diff --git a/.gitignore b/.gitignore index 1798a920eb..d155512039 100644 --- a/.gitignore +++ b/.gitignore @@ -138,3 +138,24 @@ tags *CMakeCache* *CMakeFiles* .history/ +*.txt +*.tcl +*.pc +contrib/geos +contrib/libuv +contrib/pcre2 +contrib/zlib +deps_tmp_CMakeLists.txt.in +*.a +*.ctest +pcre2-config +pcre2_test.sh +pcre2_grep_test.sh +pcre2_chartables.c +geos-config +config.h +pcre2.h +zconf.h +version.h +geos_c.h + diff --git a/cmake/cmake.define b/cmake/cmake.define index eb78b54cae..9fae397363 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -160,12 +160,21 @@ ELSE () CHECK_C_COMPILER_FLAG("-msse4.2" COMPILER_SUPPORT_SSE42) ENDIF() - CHECK_C_COMPILER_FLAG("-mfma" COMPILER_SUPPORT_FMA) - CHECK_C_COMPILER_FLAG("-mavx" COMPILER_SUPPORT_AVX) - CHECK_C_COMPILER_FLAG("-mavx2" COMPILER_SUPPORT_AVX2) - CHECK_C_COMPILER_FLAG("-mavx512f" COMPILER_SUPPORT_AVX512F) - CHECK_C_COMPILER_FLAG("-mavx512vbmi" COMPILER_SUPPORT_AVX512BMI) - CHECK_C_COMPILER_FLAG("-mavx512vl" COMPILER_SUPPORT_AVX512VL) + IF (TD_ARM_64 OR TD_ARM_32) + SET(COMPILER_SUPPORT_FMA false) + SET(COMPILER_SUPPORT_AVX false) + SET(COMPILER_SUPPORT_AVX2 false) + SET(COMPILER_SUPPORT_AVX512F false) + SET(COMPILER_SUPPORT_AVX512BMI false) + SET(COMPILER_SUPPORT_AVX512VL false) + ELSE() + CHECK_C_COMPILER_FLAG("-mfma" COMPILER_SUPPORT_FMA) + CHECK_C_COMPILER_FLAG("-mavx" COMPILER_SUPPORT_AVX) + CHECK_C_COMPILER_FLAG("-mavx2" COMPILER_SUPPORT_AVX2) + CHECK_C_COMPILER_FLAG("-mavx512f" COMPILER_SUPPORT_AVX512F) + CHECK_C_COMPILER_FLAG("-mavx512vbmi" COMPILER_SUPPORT_AVX512BMI) + CHECK_C_COMPILER_FLAG("-mavx512vl" COMPILER_SUPPORT_AVX512VL) + ENDIF() IF (COMPILER_SUPPORT_SSE42) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse4.2") @@ -177,15 +186,17 @@ ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfma") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfma") ENDIF() - IF (COMPILER_SUPPORT_AVX) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx") - ENDIF() - IF (COMPILER_SUPPORT_AVX2) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2") - ENDIF() - MESSAGE(STATUS "SIMD instructions (FMA/AVX/AVX2) is ACTIVATED") + MESSAGE(STATUS "FMA instructions is ACTIVATED") + ENDIF() + IF (COMPILER_SUPPORT_AVX) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx") + MESSAGE(STATUS "AVX instructions is ACTIVATED") + ENDIF() + IF (COMPILER_SUPPORT_AVX2) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2") + MESSAGE(STATUS "AVX2 instructions is ACTIVATED") ENDIF() IF ("${SIMD_AVX512_SUPPORT}" MATCHES "true") diff --git a/cmake/cmake.version b/cmake/cmake.version index 3bb764612e..c600c084fd 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.3.3.0.alpha") + SET(TD_VER_NUMBER "3.3.4.0.alpha") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/cmake/stub_CMakeLists.txt.in b/cmake/stub_CMakeLists.txt.in index 6c54d33be7..4c7a2fb002 100644 --- a/cmake/stub_CMakeLists.txt.in +++ b/cmake/stub_CMakeLists.txt.in @@ -2,7 +2,7 @@ # stub ExternalProject_Add(stub GIT_REPOSITORY https://github.com/coolxv/cpp-stub.git - GIT_TAG 5e903b8e + GIT_TAG 3137465194014d66a8402941e80d2bccc6346f51 GIT_SUBMODULES "src" SOURCE_DIR "${TD_CONTRIB_DIR}/cpp-stub" BINARY_DIR "${TD_CONTRIB_DIR}/cpp-stub/src" diff --git a/docs/en/14-reference/05-connectors/10-cpp.mdx b/docs/en/14-reference/05-connectors/10-cpp.mdx index 6a570b2490..ca32660ac7 100644 --- a/docs/en/14-reference/05-connectors/10-cpp.mdx +++ b/docs/en/14-reference/05-connectors/10-cpp.mdx @@ -19,7 +19,7 @@ After TDengine server or client installation, `taos.h` is located at The dynamic libraries for the TDengine client driver are located in. - Linux: `/usr/local/taos/driver/libtaos.so` -- Windows: `C:\TDengine\taos.dll` +- Windows: `C:\TDengine\driver\taos.dll` - macOS: `/usr/local/lib/libtaos.dylib` ## Supported platforms diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index a6e157cf74..486fe2c015 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -20,6 +20,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://t import Release from "/components/ReleaseV3"; +## 3.3.3.0 + + + ## 3.3.2.0 diff --git a/docs/examples/c/CMakeLists.txt b/docs/examples/c/CMakeLists.txt new file mode 100644 index 0000000000..f636084bab --- /dev/null +++ b/docs/examples/c/CMakeLists.txt @@ -0,0 +1,101 @@ +PROJECT(TDengine) + +IF (TD_LINUX) + INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc ${TD_SOURCE_DIR}/inc) + AUX_SOURCE_DIRECTORY(. SRC) + + add_executable(docs_connect_example "") + add_executable(docs_create_db_demo "") + add_executable(docs_insert_data_demo "") + add_executable(docs_query_data_demo "") + add_executable(docs_with_reqid_demo "") + add_executable(docs_sml_insert_demo "") + add_executable(docs_stmt_insert_demo "") + add_executable(docs_tmq_demo "") + + target_sources(docs_connect_example + PRIVATE + "connect_example.c" + ) + + target_sources(docs_create_db_demo + PRIVATE + "create_db_demo.c" + ) + + target_sources(docs_insert_data_demo + PRIVATE + "insert_data_demo.c" + ) + + target_sources(docs_query_data_demo + PRIVATE + "query_data_demo.c" + ) + + target_sources(docs_with_reqid_demo + PRIVATE + "with_reqid_demo.c" + ) + + target_sources(docs_sml_insert_demo + PRIVATE + "sml_insert_demo.c" + ) + + target_sources(docs_stmt_insert_demo + PRIVATE + "stmt_insert_demo.c" + ) + + target_sources(docs_tmq_demo + PRIVATE + "tmq_demo.c" + ) + + target_link_libraries(docs_connect_example + taos + ) + + target_link_libraries(docs_create_db_demo + taos + ) + + target_link_libraries(docs_insert_data_demo + taos + ) + + target_link_libraries(docs_query_data_demo + taos + ) + + target_link_libraries(docs_with_reqid_demo + taos + ) + + target_link_libraries(docs_sml_insert_demo + taos + ) + + target_link_libraries(docs_stmt_insert_demo + taos + ) + + target_link_libraries(docs_tmq_demo + taos + pthread + ) + + SET_TARGET_PROPERTIES(docs_connect_example PROPERTIES OUTPUT_NAME docs_connect_example) + SET_TARGET_PROPERTIES(docs_create_db_demo PROPERTIES OUTPUT_NAME docs_create_db_demo) + SET_TARGET_PROPERTIES(docs_insert_data_demo PROPERTIES OUTPUT_NAME docs_insert_data_demo) + SET_TARGET_PROPERTIES(docs_query_data_demo PROPERTIES OUTPUT_NAME docs_query_data_demo) + SET_TARGET_PROPERTIES(docs_with_reqid_demo PROPERTIES OUTPUT_NAME docs_with_reqid_demo) + SET_TARGET_PROPERTIES(docs_sml_insert_demo PROPERTIES OUTPUT_NAME docs_sml_insert_demo) + SET_TARGET_PROPERTIES(docs_stmt_insert_demo PROPERTIES OUTPUT_NAME docs_stmt_insert_demo) + SET_TARGET_PROPERTIES(docs_tmq_demo PROPERTIES OUTPUT_NAME docs_tmq_demo) +ENDIF () +IF (TD_DARWIN) + INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc ${TD_SOURCE_DIR}/inc) + AUX_SOURCE_DIRECTORY(. SRC) +ENDIF () diff --git a/docs/examples/c/Makefile b/docs/examples/c/Makefile new file mode 100644 index 0000000000..9fda575ec6 --- /dev/null +++ b/docs/examples/c/Makefile @@ -0,0 +1,34 @@ +# Makefile for building TDengine examples on TD Linux platform + +INCLUDE_DIRS = + +TARGETS = connect_example \ + create_db_demo \ + insert_data_demo \ + query_data_demo \ + with_reqid_demo \ + sml_insert_demo \ + stmt_insert_demo \ + tmq_demo + +SOURCES = connect_example.c \ + create_db_demo.c \ + insert_data_demo.c \ + query_data_demo.c \ + with_reqid_demo.c \ + sml_insert_demo.c \ + stmt_insert_demo.c \ + tmq_demo.c + +LIBS = -ltaos -lpthread + + +CFLAGS = -g + +all: $(TARGETS) + +$(TARGETS): + $(CC) $(CFLAGS) -o $@ $(wildcard $(@F).c) $(LIBS) + +clean: + rm -f $(TARGETS) \ No newline at end of file diff --git a/docs/zh/04-get-started/05-cloud.md b/docs/zh/04-get-started/05-cloud.md index bd76add527..1bca09ee91 100644 --- a/docs/zh/04-get-started/05-cloud.md +++ b/docs/zh/04-get-started/05-cloud.md @@ -15,7 +15,7 @@ TDengine Cloud 大幅减轻了用户在部署、运维等方面的人力负担 要在 TDengine Cloud 注册新用户，请遵循以下简易步骤完成注册流程： -1. 打开浏览器，访问 TDengine Cloud 的首页：https://cloud.taosdata.com，在右边的“注册”部分，填入自己的姓名以及企业邮箱地址，点击“获取验证码”按钮。 +1. 打开浏览器，访问 [TDengine Cloud](https://cloud.taosdata.com)，在右边的“注册”部分，填入自己的姓名以及企业邮箱地址，点击“获取验证码”按钮。 2. 检查企业邮箱，找到主题为“你的 TDengine Cloud 注册账户验证码”的邮件。从邮件内容中复制 6 位验证码，并将其粘贴到注册页面上的“验证码”输入框中。接着，点击“注册 TDengine Cloud”按钮，进入客户信息补全页面。 @@ -32,4 +32,4 @@ TDengine Cloud 大幅减轻了用户在部署、运维等方面的人力负担 3. 第 3 步，创建实例。在此步骤中，你需要填写实例的区域、名称、是否选择高可用选项以及计费方案等必填信息。确认无误后，点击“创建”按钮。大约等待 1min，新的TDengine 实例便会创建完成。随后，你可以在控制台中对该实例进行各种操作，如查询数据、创建订阅、创建流等。 -TDengine Cloud 提供多种级别的计费方案，包括入门版、基础版、标准版、专业版和旗舰版，以满足不同客户的需求。如果你觉得现有计费方案无法满足自己的特定需求，请联系 TDengine Cloud 的客户支持团队，他们将为你量身定制计费方案。注册后，你将获得一定的免费额度，以便体验服务 \ No newline at end of file +TDengine Cloud 提供多种级别的计费方案，包括入门版、基础版、标准版、专业版和旗舰版，以满足不同客户的需求。如果你觉得现有计费方案无法满足自己的特定需求，请联系 TDengine Cloud 的客户支持团队，他们将为你量身定制计费方案。注册后，你将获得一定的免费额度，以便体验服务 diff --git a/docs/zh/06-advanced/05-data-in/17-mongodb.md b/docs/zh/06-advanced/05-data-in/17-mongodb.md index 2434e2e122..5311bc43c6 100644 --- a/docs/zh/06-advanced/05-data-in/17-mongodb.md +++ b/docs/zh/06-advanced/05-data-in/17-mongodb.md @@ -13,7 +13,7 @@ MongoDB 是一个介于关系型数据库与非关系型数据库之间的产品 ### 1. 新增数据源 -在数据写入页面中点击左上角的 **+新增数据源** 按钮进入新增数据源页面，如下图所示： +在数据写入页面中点击右上角的 **+新增数据源** 按钮进入新增数据源页面，如下图所示： ![Common-zh00-EnterDataSourcePage.png](./pic/Common-zh00-EnterDataSourcePage.png "进入新增数据源页面") diff --git a/docs/zh/08-operation/03-deployment.md b/docs/zh/08-operation/03-deployment.md index 83b2c91843..2e0c2a7989 100644 --- a/docs/zh/08-operation/03-deployment.md +++ b/docs/zh/08-operation/03-deployment.md @@ -206,11 +206,11 @@ http { ### 部署 taosX -如果想使用 TDengine 的数据接入能力，需要部署 taosX 服务，关于它的详细说明和部署请参考[taosX 参考手册](../../reference/components/taosx)。 +如果想使用 TDengine 的数据接入能力，需要部署 taosX 服务，关于它的详细说明和部署请参考企业版参考手册。 ### 部署 taosX-Agent -有些数据源如 Pi, OPC 等，因为网络条件和数据源访问的限制，taosX 无法直接访问数据源，这种情况下需要部署一个代理服务 taosX-Agent，关于它的详细说明和部署请参考[taosX-Agent 参考手册](../../reference/components/taosx-agent)。 +有些数据源如 Pi, OPC 等，因为网络条件和数据源访问的限制，taosX 无法直接访问数据源，这种情况下需要部署一个代理服务 taosX-Agent，关于它的详细说明和部署请参考企业版参考手册。 ### 部署 taos-Explorer diff --git a/docs/zh/08-operation/12-multi.md b/docs/zh/08-operation/12-multi.md index 8f11ee4326..a5608ad5fa 100644 --- a/docs/zh/08-operation/12-multi.md +++ b/docs/zh/08-operation/12-multi.md @@ -70,7 +70,7 @@ dataDir /mnt/data6 2 0 |参数名称 | 参数含义 | |:-------------|:-----------------------------------------------| -|s3EndPoint | 用户所在地域的 COS 服务域名，支持 http 和 https，bucket 的区域需要与 endpoint 的保持一致，否则无法访问。例如：http://cos.ap-beijing.myqcloud.com | +|s3EndPoint | 用户所在地域的 COS 服务域名，支持 http 和 https，bucket 的区域需要与 endpoint 的保持一致，否则无法访问。 | |s3AccessKey |冒号分隔的用户 SecretId:SecretKey。例如：AKIDsQmwsfKxTo2A6nGVXZN0UlofKn6JRRSJ:lIdoy99ygEacU7iHfogaN2Xq0yumSm1E | |s3BucketName | 存储桶名称，减号后面是用户注册 COS 服务的 AppId。其中 AppId 是 COS 特有，AWS 和阿里云都没有，配置时需要作为 bucket name 的一部分，使用减号分隔。参数值均为字符串类型，但不需要引号。例如：test0711-1309024725 | |s3UploadDelaySec | data 文件持续多长时间不再变动后上传至 s3，单位：秒。最小值：1；最大值：2592000 (30天），默认值 60 秒 | diff --git a/docs/zh/08-operation/18-dual.md b/docs/zh/08-operation/18-dual.md index 354e715602..c7871a8e1e 100644 --- a/docs/zh/08-operation/18-dual.md +++ b/docs/zh/08-operation/18-dual.md @@ -83,7 +83,7 @@ taosx replica start ```shell taosx replica start -f td1:6030 -t td2:6030 ``` -该示例命令会自动创建除 information_schema、performance_schema、log、audit 库之外的同步任务。可以使用 http://td2:6041 指定该 endpoint 使用 websocket 接口（默认是原生接口）。也可以指定数据库同步：taosx replica start -f td1:6030 -t td2:6030 db1 仅创建指定的数据库同步任务。 +该示例命令会自动创建除 information_schema、performance_schema、log、audit 库之外的同步任务。可以使用 `http://td2:6041` 指定该 endpoint 使用 websocket 接口（默认是原生接口）。也可以指定数据库同步：taosx replica start -f td1:6030 -t td2:6030 db1 仅创建指定的数据库同步任务。 2. 方法二 diff --git a/docs/zh/10-third-party/01-collection/11-kafka.md b/docs/zh/10-third-party/01-collection/11-kafka.md index 75adefbc50..5113519e50 100644 --- a/docs/zh/10-third-party/01-collection/11-kafka.md +++ b/docs/zh/10-third-party/01-collection/11-kafka.md @@ -347,14 +347,18 @@ curl -X DELETE http://localhost:8083/connectors/TDengineSourceConnector 以下配置项对 TDengine Sink Connector 和 TDengine Source Connector 均适用。 1. `name`: connector 名称。 -2. `connector.class`: connector 的完整类名， 如: com.taosdata.kafka.connect.sink.TDengineSinkConnector。 -3. `tasks.max`: 最大任务数, 默认 1。 -4. `topics`: 需要同步的 topic 列表， 多个用逗号分隔, 如 `topic1,topic2`。 -5. `connection.url`: TDengine JDBC 连接字符串， 如 `jdbc:TAOS://127.0.0.1:6030`。 -6. `connection.user`： TDengine 用户名， 默认 root。 -7. `connection.password` ：TDengine 用户密码， 默认 taosdata。 -8. `connection.attempts` ：最大尝试连接次数。默认 3。 -9. `connection.backoff.ms` ： 创建连接失败重试时间隔时间，单位为 ms。 默认 5000。 +1. `connector.class`: connector 的完整类名， 如: com.taosdata.kafka.connect.sink.TDengineSinkConnector。 +1. `tasks.max`: 最大任务数, 默认 1。 +1. `topics`: 需要同步的 topic 列表， 多个用逗号分隔, 如 `topic1,topic2`。 +1. `connection.url`: TDengine JDBC 连接字符串， 如 `jdbc:TAOS://127.0.0.1:6030`。 +1. `connection.user`： TDengine 用户名， 默认 root。 +1. `connection.password` ：TDengine 用户密码， 默认 taosdata。 +1. `connection.attempts` ：最大尝试连接次数。默认 3。 +1. `connection.backoff.ms` ： 创建连接失败重试时间隔时间，单位为 ms。 默认 5000。 +1. `data.precision`: 使用 InfluxDB 行协议格式时，时间戳的精度。可选值为： + 1. ms ： 表示毫秒 + 1. us ： 表示微秒 + 1. ns ： 表示纳秒 ### TDengine Sink Connector 特有的配置 @@ -367,30 +371,22 @@ curl -X DELETE http://localhost:8083/connectors/TDengineSourceConnector 1. line ：代表 InfluxDB 行协议格式 2. json : 代表 OpenTSDB JSON 格式 3. telnet ：代表 OpenTSDB Telnet 行协议格式 -7. `data.precision`: 使用 InfluxDB 行协议格式时，时间戳的精度。可选值为： - 1. ms ： 表示毫秒 - 2. us ： 表示微秒 - 3. ns ： 表示纳秒。 ### TDengine Source Connector 特有的配置 1. `connection.database`: 源数据库名称，无缺省值。 -2. `topic.prefix`： 数据导入 kafka 时使用的 topic 名称的前缀。默认为空字符串 ""。 -3. `timestamp.initial`: 数据同步起始时间。格式为'yyyy-MM-dd HH:mm:ss'，若未指定则从指定 DB 中最早的一条记录开始。 -4. `poll.interval.ms`: 检查是否有新建或删除的表的时间间隔，单位为 ms。默认为 1000。 -5. `fetch.max.rows` : 检索数据库时最大检索条数。 默认为 100。 -6. `query.interval.ms`: 从 TDengine 一次读取数据的时间跨度，需要根据表中的数据特征合理配置，避免一次查询的数据量过大或过小；在具体的环境中建议通过测试设置一个较优值，默认值为 0，即获取到当前最新时间的所有数据。 -7. `out.format` : 结果集输出格式。`line` 表示输出格式为 InfluxDB Line 协议格式，`json` 表示输出格式是 json。默认为 line。 -8. `data.precision`: 使用 InfluxDB 行协议格式时，时间戳的精度。可选值为： - 1. ms ： 表示毫秒， - 2. us ： 表示微秒 - 3. ns ： 表示纳秒。 -9. `topic.per.stable`: 如果设置为 true，表示一个超级表对应一个 Kafka topic，topic的命名规则 ``；如果设置为 false，则指定的 DB 中的所有数据进入一个 Kafka topic，topic 的命名规则为 `` -10. `topic.ignore.db`: topic 命名规则是否包含 database 名称，true 表示规则为 ``，false 表示规则为 ``，默认 false。此配置项在 `topic.per.stable` 设置为 false 时不生效。 -11. `topic.delimiter`: topic 名称分割符，默认为 `-`。 -12. `read.method`: 从 TDengine 读取数据方式，query 或是 subscription。默认为 subscription。 -13. `subscription.group.id`: 指定 TDengine 数据订阅的组 id，当 `read.method` 为 subscription 时，此项为必填项。 -14. `subscription.from`: 指定 TDengine 数据订阅起始位置，latest 或是 earliest。默认为 latest。 +1. `topic.prefix`： 数据导入 kafka 时使用的 topic 名称的前缀。默认为空字符串 ""。 +1. `timestamp.initial`: 数据同步起始时间。格式为'yyyy-MM-dd HH:mm:ss'，若未指定则从指定 DB 中最早的一条记录开始。 +1. `poll.interval.ms`: 检查是否有新建或删除的表的时间间隔，单位为 ms。默认为 1000。 +1. `fetch.max.rows` : 检索数据库时最大检索条数。 默认为 100。 +1. `query.interval.ms`: 从 TDengine 一次读取数据的时间跨度，需要根据表中的数据特征合理配置，避免一次查询的数据量过大或过小；在具体的环境中建议通过测试设置一个较优值，默认值为 0，即获取到当前最新时间的所有数据。 +1. `out.format` : 结果集输出格式。`line` 表示输出格式为 InfluxDB Line 协议格式，`json` 表示输出格式是 json。默认为 line。 +1. `topic.per.stable`: 如果设置为 true，表示一个超级表对应一个 Kafka topic，topic的命名规则 ``；如果设置为 false，则指定的 DB 中的所有数据进入一个 Kafka topic，topic 的命名规则为 `` +1. `topic.ignore.db`: topic 命名规则是否包含 database 名称，true 表示规则为 ``，false 表示规则为 ``，默认 false。此配置项在 `topic.per.stable` 设置为 false 时不生效。 +1. `topic.delimiter`: topic 名称分割符，默认为 `-`。 +1. `read.method`: 从 TDengine 读取数据方式，query 或是 subscription。默认为 subscription。 +1. `subscription.group.id`: 指定 TDengine 数据订阅的组 id，当 `read.method` 为 subscription 时，此项为必填项。 +1. `subscription.from`: 指定 TDengine 数据订阅起始位置，latest 或是 earliest。默认为 latest。 ## 其他说明 diff --git a/docs/zh/14-reference/01-components/04-taosx.md b/docs/zh/14-reference/01-components/04-taosx.md index a5bf9df0c0..114a6b1ce5 100644 --- a/docs/zh/14-reference/01-components/04-taosx.md +++ b/docs/zh/14-reference/01-components/04-taosx.md @@ -239,7 +239,7 @@ d4,2017-07-14T10:40:00.006+08:00,-2.740636,10,-0.893545,7,California.LosAngles - `plugins_home`：外部数据源连接器所在目录。 - `data_dir`：数据文件存放目录。 -- `instanceId`：当前 explorer 服务的实例 ID，如果同一台机器上启动了多个 explorer 实例，必须保证各个实例的实例 ID 互不相同。 +- `instanceId`：当前 taosX 服务的实例 ID，如果同一台机器上启动了多个 taosX 实例，必须保证各个实例的实例 ID 互不相同。 - `logs_home`：日志文件存放目录，`taosX` 日志文件的前缀为 `taosx.log`，外部数据源有自己的日志文件名前缀。已弃用，请使用 `log.path` 代替。 - `log_level`：日志等级，可选级别包括 `error`、`warn`、`info`、`debug`、`trace`，默认值为 `info`。已弃用，请使用 `log.level` 代替。 - `log_keep_days`：日志的最大存储天数，`taosX` 日志将按天划分为不同的文件。已弃用，请使用 `log.keepDays` 代替。 diff --git a/docs/zh/14-reference/01-components/05-taosx-agent.md b/docs/zh/14-reference/01-components/05-taosx-agent.md index e521c8becb..bf2e6f7e78 100644 --- a/docs/zh/14-reference/01-components/05-taosx-agent.md +++ b/docs/zh/14-reference/01-components/05-taosx-agent.md @@ -11,6 +11,7 @@ sidebar_label: taosX-Agent - `endpoint`: 必填，`taosX` 的 GRPC 服务地址。 - `token`: 必填，在 `Explorer` 上创建 `Agent` 时，产生的 Token。 +- `instanceId`：当前 taosx-agent 服务的实例 ID，如果同一台机器上启动了多个 taosx-agent 实例，必须保证各个实例的实例 ID 互不相同。 - `compression`: 非必填，可配置为 `ture` 或 `false`, 默认为 `false`。配置为`true`, 则开启 `Agent` 和 `taosX` 通信数据压缩。 - `log_level`: 非必填，日志级别，默认为 `info`, 同 `taosX` 一样，支持 `error`，`warn`，`info`，`debug`，`trace` 五级。已弃用，请使用 `log.level` 代替。 - `log_keep_days`：非必填，日志保存天数，默认为 `30` 天。已弃用，请使用 `log.keepDays` 代替。 @@ -37,7 +38,7 @@ sidebar_label: taosX-Agent # server instance id # # The instanceId of each instance is unique on the host -# instanceId = 64 +# instanceId = 48 # enable communication data compression between Agent and taosX # diff --git a/docs/zh/14-reference/01-components/07-explorer.md b/docs/zh/14-reference/01-components/07-explorer.md index 5d17970b38..c63bc703e2 100644 --- a/docs/zh/14-reference/01-components/07-explorer.md +++ b/docs/zh/14-reference/01-components/07-explorer.md @@ -132,7 +132,7 @@ cors = true - `cluster`：TDengine 集群的 taosAdapter 地址。 - `cluster_native`：TDengine 集群的原生连接地址，默认关闭。 - `x_api`：taosX 的 gRPC 地址。 -- `grpc`：taosX 代理向 taosX 建立连接的 gRPC 地址. +- `grpc`：taosX 代理向 taosX 建立连接的 gRPC 地址。 - `cors`：CORS 配置开关，默认为 `false`。当为 `true` 时，允许跨域访问。 - `ssl.certificate`：SSL 证书（如果同时设置了 certificate 与 certificate_key 两个参数，则启用 HTTPS 服务，否则不启用）。 - `ssl.certificate_key`：SSL 证书密钥。 diff --git a/docs/zh/14-reference/03-taos-sql/03-table.md b/docs/zh/14-reference/03-taos-sql/03-table.md index 2f0ae7100a..cad9190bd9 100644 --- a/docs/zh/14-reference/03-taos-sql/03-table.md +++ b/docs/zh/14-reference/03-taos-sql/03-table.md @@ -79,6 +79,18 @@ CREATE TABLE [IF NOT EXISTS] tb_name1 USING stb_name TAGS (tag_value1, ...) [IF 批量建表方式要求数据表必须以超级表为模板。 在不超出 SQL 语句长度限制的前提下，单条语句中的建表数量建议控制在 1000 ～ 3000 之间，将会获得比较理想的建表速度。 +### 使用 CSV 批量创建子表 + +```sql +CREATE TABLE [IF NOT EXISTS] USING [db_name.]stb_name (field1_name [, field2_name] ....) FILE csv_file_path; +``` + +**参数说明** + +1. FILE 语法表示数据来自于 CSV 文件（英文逗号分隔、英文单引号括住每个值），CSV 文件无需表头。CSV 文件中应仅包含 table name 与 tag 值。如需插入数据，请参考数据写入章节。 +2. 为指定的 stb_name 创建子表，该超级表必须已经存在。 +3. field_name 列表顺序与 CSV 文件各列内容顺序一致。列表中不允许出现重复项，且必须包含 `tbname`，可包含零个或多个超级表中已定义的标签列。未包含在列表中的标签值将被设置为 NULL。 + ## 修改普通表 ```sql diff --git a/docs/zh/14-reference/03-taos-sql/10-function.md b/docs/zh/14-reference/03-taos-sql/10-function.md index ac69f64f08..8c882b3237 100644 --- a/docs/zh/14-reference/03-taos-sql/10-function.md +++ b/docs/zh/14-reference/03-taos-sql/10-function.md @@ -1458,7 +1458,7 @@ WEEKDAY(expr) **适用于**: 表和超级表。 **使用说明**： -- 返回值 0 代表周日，1 代表周一 ... 6 代表周六。 +- 返回值 0 代表周一，1 代表周二 ... 6 代表周日。 - 若 `expr` 为 NULL，返回 NULL。 - 输入时间戳的精度由所查询表的精度确定, 若未指定表, 则精度为毫秒. diff --git a/docs/zh/14-reference/03-taos-sql/14-stream.md b/docs/zh/14-reference/03-taos-sql/14-stream.md index c0d14f0455..d995c2a09b 100644 --- a/docs/zh/14-reference/03-taos-sql/14-stream.md +++ b/docs/zh/14-reference/03-taos-sql/14-stream.md @@ -99,7 +99,7 @@ PARTITION 子句中，为 tbname 定义了一个别名 tname, 在PARTITION 子 ## 流式计算读取历史数据 -正常情况下，流式计算不会处理创建前已经写入源表中的数据，若要处理已经写入的数据，可以在创建流时设置 fill_history 1 选项，这样创建的流式计算会自动处理创建前、创建中、创建后写入的数据。例如： +正常情况下，流式计算不会处理创建前已经写入源表中的数据，若要处理已经写入的数据，可以在创建流时设置 fill_history 1 选项，这样创建的流式计算会自动处理创建前、创建中、创建后写入的数据。流计算处理历史数据的最大窗口数是2000万，超过限制会报错。例如： ```sql create stream if not exists s1 fill_history 1 into st1 as select count(*) from t1 interval(10s) diff --git a/docs/zh/14-reference/03-taos-sql/24-show.md b/docs/zh/14-reference/03-taos-sql/24-show.md index 09333dd0b7..110c9cee6e 100644 --- a/docs/zh/14-reference/03-taos-sql/24-show.md +++ b/docs/zh/14-reference/03-taos-sql/24-show.md @@ -30,6 +30,16 @@ SHOW CLUSTER ALIVE; 查询当前集群的状态是否可用，返回值： 0：不可用 1：完全可用 2：部分可用（集群中部分节点下线，但其它节点仍可以正常使用） +## SHOW CLUSTER MACHINES + +```sql +SHOW CLUSTER MACHINES; // 从 TDengine 3.2.3.0 版本开始支持 +``` + +显示集群的机器码等信息。 + +注：企业版独有 + ## SHOW CONNECTIONS ```sql @@ -99,6 +109,7 @@ SHOW FUNCTIONS; ```sql SHOW LICENCES; SHOW GRANTS; +SHOW GRANTS FULL; // 从 TDengine 3.2.3.0 版本开始支持 ``` 显示企业版许可授权的信息。 diff --git a/docs/zh/14-reference/05-connector/10-cpp.mdx b/docs/zh/14-reference/05-connector/10-cpp.mdx index 0df6ed924c..c618601fb9 100644 --- a/docs/zh/14-reference/05-connector/10-cpp.mdx +++ b/docs/zh/14-reference/05-connector/10-cpp.mdx @@ -27,7 +27,7 @@ TDengine 服务端或客户端安装后，`taosws.h` 位于： TDengine 客户端驱动的动态库位于： - Linux: `/usr/local/taos/driver/libtaosws.so` -- Windows: `C:\TDengine\taosws.dll` +- Windows: `C:\TDengine\driver\taosws.dll` - macOS: `/usr/local/lib/libtaosws.dylib` ### 支持的平台 @@ -626,7 +626,7 @@ TDengine 服务端或客户端安装后，`taos.h` 位于： TDengine 客户端驱动的动态库位于： - Linux: `/usr/local/taos/driver/libtaos.so` -- Windows: `C:\TDengine\taos.dll` +- Windows: `C:\TDengine\driver\taos.dll` - macOS: `/usr/local/lib/libtaos.dylib` ### 支持的平台 diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index 5b3abcb341..0f9ceada50 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -24,6 +24,10 @@ TDengine 3.x 各版本安装包下载链接如下： import Release from "/components/ReleaseV3"; +## 3.3.3.0 + + + ## 3.3.2.0 diff --git a/docs/zh/28-releases/03-notes/3.3.3.0.md b/docs/zh/28-releases/03-notes/3.3.3.0.md new file mode 100644 index 0000000000..405ca83d71 --- /dev/null +++ b/docs/zh/28-releases/03-notes/3.3.3.0.md @@ -0,0 +1,89 @@ +--- +title: 3.3.3.0 版本说明 +sidebar_label: 3.3.3.0 +description: 3.3.3.0 版本说明 +--- +### 新特性/优化 +1. 增加函数支持：pi、truncate/tunc、exp、ln、mod、rand、sign、degress、radians、char、ascii、position、trim、replace、repeat、substring、substr、substring_index、week、weekday、weekofyear、dayofweek、stddev_pop、var_pop +2. 多级存储支持微软对象存储 Azure Blob [企业版] +3. 支持 MongoDB 数据源 [企业版] +4. TDengine支持macOS企业版客户端 [企业版] +5. taosX日志默认不写入syslog [企业版] +6. 服务端记录所有慢查询信息到log库 +7. show cluster machines 查询结果中添加服务端版本号 +8. 删除保留关键字LEVEL/ENCODE/COMPRESS, 可以作为列名/表名/数据库名等使用 +9. 禁止动态修改临时目录 +10. round 函数：支持四舍五入的精度 +11. timediff 函数：返回时间戳 expr1 - expr2 的结果，支持负数结果，并近似到时间单位 time_unit 指定的精度 +12. max/min 函数：支持字符串作为输入参数，当输入参数为字符串类型时，返回最大字符串值 +13. 提升了 taosX 数据同步的性能 [企业版] +14. 统一慢查询和普通日志的保存策略 +15. 优化订阅功能中offset文件过大且加载过慢的问题 +16. 升级 JDBC driver 至 3.2.5 +17. Kafka connector 提交改为同步方式 +18. 新增参数：表查询不存在时是否报异常 +19. 扩大 tsma 的 interval 的最大窗口到 1 年 +20. 支持从 CSV 文件批量建表 +### 修复问题 +1. 服务器重启后 mqtt 写入任务未自动重新启动 [企业版] +2. 修复windows上agent和PI connector连接错误导致任务终止的问题 [企业版] +3. 重启taosadapter后,mqtt 任务异常，无法同步数据 [企业版] +4. Explorer 上 mqtt 写入任务状态不正确 [企业版] +5. 重启 taosx 后，运行中的任务没有被重新调度 [企业版] +6. 对于 taosx 的 MongoDB DataIn 任务，优化了日志的输出信息 [企业版] +7. 修复在 local.toml 文件为空时备份失败的问题 [企业版] +8. taosx replica 任务在网络断开恢复后数据同步异常的问题 [企业版] +9. 修复3.1版本同步到3.3版本meta数据不同步问题 [企业版] +10. taosadapter 异常重启后, mqtt任务无法自动恢复的问题[企业版] +11. Kafka DataIn 任务状态正常，但任务停止消费数据 [企业版] +12. 修复 last + interval 查询导致 crash 的问题 +13. 在数据写入时热更stt_trigger导致taosd崩溃 +14. 修改root密码后taoskeeper反复重启 +15. 云服务中服务重启后流计算无法继续 +16. tsdb加载缓存导致taosd死锁 +17. 查询时fill值指定为0报错 +18. last_row 查询结果与预期不符 +19. 用户指定 information_schema 库登录时导致 taosd 崩溃 +20. group by 语句支持位置参数和别名语法 +21. 元数据克隆函数对象错误导致 crash +22. 缓存更新时数据填充的游标错误导致的 taosd 异常退出 +23. STDDEV 函数计算结果随机不正确的问题 +24. 多级存储以及加密场景下无法添加离线节点 +25. taos CLI 无法输入大于20字节长度的密码 +26. 拼接 sql 写入报错: int data overflow +27. 大量查询并发场景下元数据的一致性 +28. 尝试解决手动点击停止按钮，任务无法停止的问题 +29. 解决了列或者标签中包含逗号或引号时导出 CSV 文件错误的问题 +30. 多线程并发的对同一张子表进行 describe 导致 crash +31. 数据浏览器的查询结果未按照配置的时区展示的问题 +32. 创建 websocket 连接时 taosadapter 内存泄漏的问题 +33. 社区版无法在 redhat 操作系统正常启动 +34. 无效断言在两个 stt 分别位于数据块时间轴分布空洞场景引发的 crash +35. S3 长时间拉取数据会失败的问题 +36. 使用结果集函数判断时间线的有序性 +37. 解决超级表投影查询慢的问题 +38. 修复: 多节点环境，事务返回码与事务状态不一致导致事务堆积 +39. 修复：在stt_trigger = 1时，delete数据之后重新出现的错误 +40. 节点恢复阶段taosd内存占用过高导致OOM +41. limit过小时的判断错误 +42. AVEVA historian 数据源连通性及获取示例数据功能 +43. taosd 服务无法正常停止的问题 +44. last 函数查询全空列导致 crash +45. 查询 cid 不在当前版本的 schema 时触发 assert 导致 coredump +46. 写入消息体过大时 OOM 的问题，增加 syncLogBufferMemoryAllowed 参数控制 +47. timezone 设置为 UTC 且数据包含“1970”时,查询报错 +48. 取消mnode节点的消息同步的等待超时 +49. 集群节点异常恢复后，集群授权状态失效 +50. 查询空指针问题 +51. 在restore vnode时，将vgroup暂时设置成单副本 +52. 可以使用SQL保留字作为列名创建超级表，也可以使用SQL保留字作为标签名 +53. 高并发查询期间内存持续增长的问题 +54. 流计算在checkpoint处理过程中，由于断言失败导致的crash +55. 修复了 3.3.2.8.0820 版本中可以查询到 information_schema 中不存在的表perf_queries问题 +56. 节点恢复阶段taosd内存占用过高导致OOM +57. 修复在schema变更情况下，通过taosx导入数据时，taosd出现crash的问题 +58. 可能的内存泄漏 +59. 客户端内存泄漏 +60. 开源用户修改stt_trigger值升级后无法修改其他数据库选项 +61. NOT IN (NULL) 查询结果不正确 +62. taos shell和taosBenchmark不能成功连接云服务实例 diff --git a/docs/zh/28-releases/03-notes/index.md b/docs/zh/28-releases/03-notes/index.md index eadb434f53..132a72d9ca 100644 --- a/docs/zh/28-releases/03-notes/index.md +++ b/docs/zh/28-releases/03-notes/index.md @@ -3,5 +3,5 @@ title: 版本说明 sidebar_label: 版本说明 description: 各版本版本说明 --- - +[3.3.3.0](./3.3.3.0) [3.3.2.0](./3.3.2.0) diff --git a/include/client/taos.h b/include/client/taos.h index 8b0111ae97..80dbe27c47 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -251,6 +251,7 @@ DLL_EXPORT int64_t taos_affected_rows64(TAOS_RES *res); DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); +DLL_EXPORT int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); DLL_EXPORT void taos_stop_query(TAOS_RES *res); DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col); DLL_EXPORT int taos_is_null_by_column(TAOS_RES *res, int columnIndex, bool result[], int *rows); @@ -389,7 +390,7 @@ DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); DLL_EXPORT int64_t tmq_get_vgroup_offset(TAOS_RES *res); DLL_EXPORT const char *tmq_err2str(int32_t code); -/* ------------------------------ TAOSX -----------------------------------*/ +/* ------------------------------ TAOSX INTERFACE -----------------------------------*/ typedef struct tmq_raw_data { void *raw; uint32_t raw_len; diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 753f84d774..3910ea6745 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -89,32 +89,6 @@ typedef struct { int32_t exprIdx; } STupleKey; -typedef struct STuplePos { - union { - struct { - int32_t pageId; - int32_t offset; - }; - SWinKey streamTupleKey; - }; -} STuplePos; - -typedef struct SFirstLastRes { - bool hasResult; - // used for last_row function only, isNullRes in SResultRowEntry can not be passed to downstream.So, - // this attribute is required - bool isNull; - int32_t bytes; - int64_t ts; - char* pkData; - int32_t pkBytes; - int8_t pkType; - STuplePos pos; - STuplePos nullTuplePos; - bool nullTupleSaved; - char buf[]; -} SFirstLastRes; - static inline int STupleKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { STupleKey* pTuple1 = (STupleKey*)pKey1; STupleKey* pTuple2 = (STupleKey*)pKey2; diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index cbfd1efb84..99cdb53103 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -277,7 +277,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pData bool alreadyAddGroupId(char* ctbName, int64_t groupId); bool isAutoTableName(char* ctbName); -void buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId); +int32_t buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId, size_t cap); int32_t buildCtbNameByGroupId(const char* stbName, uint64_t groupId, char** pName); int32_t buildCtbNameByGroupIdImpl(const char* stbName, uint64_t groupId, char* pBuf); diff --git a/include/common/tglobal.h b/include/common/tglobal.h index bece14c17d..cf918c6e0d 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -214,7 +214,7 @@ extern int64_t tsMinDiskFreeSize; // udf extern bool tsStartUdfd; extern char tsUdfdResFuncs[]; -extern char tsUdfdLdLibPath[]; +extern char tsUdfdLdLibPath[512]; // schemaless extern char tsSmlChildTableName[]; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index bc61e21d16..1a10f02c96 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1684,6 +1684,7 @@ typedef struct { typedef struct { int32_t openVnodes; + int32_t dropVnodes; int32_t totalVnodes; int32_t masterNum; int64_t numOfSelectReqs; @@ -2822,8 +2823,8 @@ enum { TOPIC_SUB_TYPE__COLUMN, }; -#define DEFAULT_MAX_POLL_INTERVAL 300000 -#define DEFAULT_SESSION_TIMEOUT 12000 +#define DEFAULT_MAX_POLL_INTERVAL 300000 +#define DEFAULT_SESSION_TIMEOUT 12000 typedef struct { char name[TSDB_TOPIC_FNAME_LEN]; // accout.topic @@ -4048,37 +4049,39 @@ void tDeleteMqMetaRsp(SMqMetaRsp* pRsp); #define MQ_DATA_RSP_VERSION 100 typedef struct { - SMqRspHead head; - STqOffsetVal reqOffset; - STqOffsetVal rspOffset; - int32_t blockNum; - int8_t withTbName; - int8_t withSchema; - SArray* blockDataLen; - SArray* blockData; - SArray* blockTbName; - SArray* blockSchema; -} SMqDataRspCommon; + struct { + SMqRspHead head; + STqOffsetVal rspOffset; + STqOffsetVal reqOffset; + int32_t blockNum; + int8_t withTbName; + int8_t withSchema; + SArray* blockDataLen; + SArray* blockData; + SArray* blockTbName; + SArray* blockSchema; + }; + + union{ + struct{ + int64_t sleepTime; + }; + struct{ + int32_t createTableNum; + SArray* createTableLen; + SArray* createTableReq; + }; + }; -typedef struct { - SMqDataRspCommon common; - int64_t sleepTime; } SMqDataRsp; -int32_t tEncodeMqDataRsp(SEncoder* pEncoder, const void* pRsp); -int32_t tDecodeMqDataRsp(SDecoder* pDecoder, void* pRsp); -void tDeleteMqDataRsp(void* pRsp); +int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pObj); +int32_t tDecodeMqDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp); +void tDeleteMqDataRsp(SMqDataRsp* pRsp); -typedef struct { - SMqDataRspCommon common; - int32_t createTableNum; - SArray* createTableLen; - SArray* createTableReq; -} STaosxRsp; - -int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const void* pRsp); -int32_t tDecodeSTaosxRsp(SDecoder* pDecoder, void* pRsp); -void tDeleteSTaosxRsp(void* pRsp); +int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp); +int32_t tDecodeSTaosxRsp(SDecoder* pDecoder, SMqDataRsp* pRsp); +void tDeleteSTaosxRsp(SMqDataRsp* pRsp); typedef struct SMqBatchMetaRsp { SMqRspHead head; // not serialize @@ -4163,6 +4166,7 @@ typedef struct { typedef struct { SArray* topicPrivileges; // SArray + int32_t debugFlag; } SMqHbRsp; typedef struct { diff --git a/include/common/ttime.h b/include/common/ttime.h index d430f7bd2a..65bb763b1f 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -64,7 +64,7 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) { : 1000000000; time_t t = taosTime(NULL); struct tm tm; - (void) taosLocalTime(&t, &tm, NULL); + (void) taosLocalTime(&t, &tm, NULL, 0); tm.tm_hour = 0; tm.tm_min = 0; tm.tm_sec = 0; diff --git a/include/common/tvariant.h b/include/common/tvariant.h index 0d84b2414e..5ab42ae8b7 100644 --- a/include/common/tvariant.h +++ b/include/common/tvariant.h @@ -23,6 +23,7 @@ extern "C" { #endif // variant, each number/string/field_id has a corresponding struct during parsing sql +// **NOTE**: if you want to change this struct, please consider the backward compatibility of function top and bottom. typedef struct SVariant { uint32_t nType; int32_t nLen; // only used for string, for number, it is useless diff --git a/include/libs/command/command.h b/include/libs/command/command.h index 284f54e5ae..9fb2ca40b9 100644 --- a/include/libs/command/command.h +++ b/include/libs/command/command.h @@ -29,6 +29,6 @@ int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int64_t startTs) int32_t qExecExplainEnd(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp); int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t groupId, SRetrieveTableRsp **pRsp); void qExplainFreeCtx(SExplainCtx *pCtx); -int32_t formatDurationOrKeep(char* buffer, int32_t timeInMinutes); +int32_t formatDurationOrKeep(char* buffer, int64_t bufSize, int32_t timeInMinutes); #endif diff --git a/include/libs/function/function.h b/include/libs/function/function.h index ec01cf1f6f..7ca046762a 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -23,6 +23,7 @@ extern "C" { #include "tcommon.h" #include "tsimplehash.h" #include "tvariant.h" +#include "functionResInfo.h" struct SqlFunctionCtx; struct SResultRowEntryInfo; @@ -85,14 +86,7 @@ enum { PRE_SCAN = 0x2u, // pre-scan belongs to the main scan and occurs before main scan }; -typedef struct SPoint1 { - int64_t key; - union { - double val; - char *ptr; - }; -} SPoint1; - +struct SPoint1; struct SqlFunctionCtx; struct SResultRowEntryInfo; diff --git a/include/libs/function/functionResInfo.h b/include/libs/function/functionResInfo.h new file mode 100644 index 0000000000..d79caf3f8c --- /dev/null +++ b/include/libs/function/functionResInfo.h @@ -0,0 +1,90 @@ +/* +* Copyright (c) 2019 TAOS Data, Inc. +* +* This program is free software: you can use, redistribute, and/or modify +* it under the terms of the GNU Affero General Public License, version 3 +* or later ("AGPL"), as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, but WITHOUT +* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +* FITNESS FOR A PARTICULAR PURPOSE. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . + */ + +#ifndef TDENGINE_FUNCTIONRESINFO_H +#define TDENGINE_FUNCTIONRESINFO_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" +#include "tcommon.h" + +typedef struct STuplePos { + union { + struct { + int32_t pageId; + int32_t offset; + }; + SWinKey streamTupleKey; + }; +} STuplePos; + +typedef struct SCentroid { + double mean; + int64_t weight; +} SCentroid; + +typedef struct SPt { + double value; + int64_t weight; +} SPt; + +typedef struct TDigest { + double compression; + int32_t threshold; + int64_t size; + + int64_t total_weight; + double min; + double max; + + int32_t num_buffered_pts; + SPt *buffered_pts; + + int32_t num_centroids; + SCentroid *centroids; +} TDigest; + +typedef struct SFirstLastRes { + bool hasResult; + // used for last_row function only, isNullRes in SResultRowEntry can not be passed to downstream.So, + // this attribute is required + bool isNull; + int32_t bytes; + int64_t ts; + char* pkData; + int32_t pkBytes; + int8_t pkType; + STuplePos pos; + STuplePos nullTuplePos; + bool nullTupleSaved; + char buf[]; +} SFirstLastRes; + +typedef struct SPoint1 { + int64_t key; + union { + double val; + char *ptr; + }; +} SPoint1; + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_FUNCTIONRESINFO_H diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index a691433ee6..bfe9e9555b 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -274,7 +274,7 @@ typedef enum EWindowType { WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE, WINDOW_TYPE_EVENT, - WINDOW_TYPE_COUNT + WINDOW_TYPE_COUNT, } EWindowType; typedef enum EWindowAlgorithm { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 5f9a4a1110..f5567c735e 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -190,7 +190,6 @@ typedef struct SFunctionNode { bool hasOriginalFunc; int32_t originalFuncId; ETrimType trimType; - bool hasSMA; bool dual; // whether select stmt without from stmt, true for without. } SFunctionNode; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index bcb1aa90d1..81a3952463 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -336,7 +336,7 @@ char* jobTaskStatusStr(int32_t status); SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name); void destroyQueryExecRes(SExecResult* pRes); -int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len); +int32_t dataConverToStr(char* str, int64_t capacity, int type, void* buf, int32_t bufSize, int32_t* len); void parseTagDatatoJson(void* p, char** jsonStr); int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); void getColumnTypeFromMeta(STableMeta* pMeta, char* pName, ETableColumnType* pType); @@ -407,29 +407,29 @@ void* getTaskPoolWorkerCb(); #define IS_AUDIT_CTB_NAME(_ctbname) \ ((*(_ctbname) == 't') && (0 == strncmp(_ctbname, TSDB_AUDIT_CTB_OPERATION, TSDB_AUDIT_CTB_OPERATION_LEN))) -#define qFatal(...) \ - do { \ - if (qDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); \ - } \ +#define qFatal(...) \ + do { \ + if (qDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("QRY FATAL ", DEBUG_FATAL, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define qError(...) \ - do { \ - if (qDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); \ - } \ +#define qError(...) \ + do { \ + if (qDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("QRY ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define qWarn(...) \ - do { \ - if (qDebugFlag & DEBUG_WARN) { \ - taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); \ - } \ +#define qWarn(...) \ + do { \ + if (qDebugFlag & DEBUG_WARN) { \ + taosPrintLog("QRY WARN ", DEBUG_WARN, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define qInfo(...) \ - do { \ - if (qDebugFlag & DEBUG_INFO) { \ - taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); \ - } \ +#define qInfo(...) \ + do { \ + if (qDebugFlag & DEBUG_INFO) { \ + taosPrintLog("QRY ", DEBUG_INFO, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + } \ } while (0) #define qDebug(...) \ do { \ diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 07d56f9b07..50c096258e 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -47,7 +47,7 @@ extern "C" { #define SYNC_HEARTBEAT_SLOW_MS 1500 #define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500 #define SYNC_SNAP_RESEND_MS 1000 * 60 -#define SYNC_SNAP_TIMEOUT_MS 1000 * 300 +#define SYNC_SNAP_TIMEOUT_MS 1000 * 180 #define SYNC_VND_COMMIT_MIN_MS 3000 diff --git a/include/os/osDef.h b/include/os/osDef.h index ff30265afa..be37c50264 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -247,9 +247,12 @@ void syslog(int unused, const char *format, ...); #define TD_DIRSEP_CHAR '/' #endif +#define TD_FQDN_LEN 128 #define TD_LOCALE_LEN 64 #define TD_CHARSET_LEN 64 #define TD_TIMEZONE_LEN 96 +#define TD_TIME_STR_LEN 128 +#define TD_IP_LEN 64 #ifdef __cplusplus } diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 5ee3f30ddf..1cedfc4dcd 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -85,7 +85,6 @@ typedef int32_t SOCKET; #else #define TAOS_EPOLL_WAIT_TIME 500 typedef int32_t SOCKET; -typedef SOCKET EpollFd; #define EpollClose(pollFd) taosCloseSocket(pollFd) #endif diff --git a/include/os/osString.h b/include/os/osString.h index 5f211ad2ee..995aa4a591 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -51,18 +51,25 @@ typedef enum { M2C = 0, C2M } ConvType; #define strtod STR_TO_LD_FUNC_TAOS_FORBID #define strtold STR_TO_D_FUNC_TAOS_FORBID #define strtof STR_TO_F_FUNC_TAOS_FORBID + +#ifdef strndup +#undef strndup +#endif #define strndup STR_TO_F_FUNC_TAOS_FORBID + #endif -#define tstrncpy(dst, src, size) \ - do { \ +#define tstrncpy(dst, src, size) \ + do { \ (void)strncpy((dst), (src), (size)); \ - (dst)[(size)-1] = 0; \ + (dst)[(size) - 1] = 0; \ } while (0) +int64_t tsnprintf(char *dst, int64_t size, const char *format, ...); #define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src)) #define TAOS_STRNCPY(_dst, _src, _size) ((void)strncpy(_dst, _src, _size)) #define TAOS_STRCAT(_dst, _src) ((void)strcat(_dst, _src)) +#define TAOS_STRNCAT(_dst, _src, len) ((void)strncat(_dst, _src, len)) char *tstrdup(const char *src); int32_t taosUcs4len(TdUcs4 *ucs4); @@ -81,7 +88,7 @@ bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes); int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4); bool taosValidateEncodec(const char *encodec); -int32_t taosHexEncode(const unsigned char *src, char *dst, int32_t len); +int32_t taosHexEncode(const unsigned char *src, char *dst, int32_t len, int32_t bufSize); int32_t taosHexDecode(const char *src, char *dst, int32_t len); int32_t taosWcharWidth(TdWchar wchar); diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index fb6acf2a0e..d292b326d5 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -52,7 +52,8 @@ int32_t taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes); void taosSetDefaultCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes); void taosKillSystem(); -int32_t taosGetSystemUUID(char *uid, int32_t uidlen); +int32_t taosGetSystemUUIDLimit36(char *uid, int32_t uidlen); +int32_t taosGetSystemUUIDLen(char *uid, int32_t uidlen); char *taosGetCmdlineByPID(int32_t pid); void taosSetCoreDump(bool enable); diff --git a/include/os/osTime.h b/include/os/osTime.h index c367416175..5d74146e9c 100644 --- a/include/os/osTime.h +++ b/include/os/osTime.h @@ -91,7 +91,7 @@ static FORCE_INLINE int64_t taosGetMonoTimestampMs() { } char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm); -struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf); +struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf, int32_t bufSize); struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst); time_t taosTime(time_t *t); time_t taosMktime(struct tm *timep); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 558902075b..2b116514e6 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -474,8 +474,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_DNODE_INVALID_TTL_CHG_ON_WR TAOS_DEF_ERROR_CODE(0, 0x0427) #define TSDB_CODE_DNODE_INVALID_EN_WHITELIST TAOS_DEF_ERROR_CODE(0, 0x0428) #define TSDB_CODE_DNODE_INVALID_MONITOR_PARAS TAOS_DEF_ERROR_CODE(0, 0x0429) -#define TSDB_CODE_MNODE_STOPPED TAOS_DEF_ERROR_CODE(0, 0x042A) - +#define TSDB_CODE_MNODE_STOPPED TAOS_DEF_ERROR_CODE(0, 0x042A) // mnode-sma #define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index fef6c0713c..1f09b750cb 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -152,11 +152,15 @@ int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int // for internal usage int32_t getWordLength(char type); +#ifdef __AVX2__ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, char *const output, const char type); -void tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output); -void tsDecompressFloatImplAvx2(const char *const input, const int32_t nelements, char *const output); +int32_t tsDecompressFloatImpAvx2(const char *input, int32_t nelements, char *output); +int32_t tsDecompressDoubleImpAvx2(const char *input, int32_t nelements, char *output); +#endif +#ifdef __AVX512VL__ +void tsDecompressTimestampAvx2(const char *input, int32_t nelements, char *output, bool bigEndian); void tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output, bool bigEndian); -void tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, char *const output, bool bigEndian); +#endif /************************************************************************* * REGULAR COMPRESSION 2 @@ -213,8 +217,8 @@ typedef int32_t (*__data_compress_init)(char *lossyColumns, float fPrecision, do uint32_t intervals, int32_t ifAdtFse, const char *compressor); typedef int32_t (*__data_compress_l1_fn_t)(const char *const input, const int32_t nelements, char *const output, const char type); -typedef int32_t (*__data_decompress_l1_fn_t)(const char *const input, const int32_t nelements, char *const output, - const char type); +typedef int32_t (*__data_decompress_l1_fn_t)(const char *const input, int32_t ninput, const int32_t nelements, + char *const output, const char type); typedef int32_t (*__data_compress_l2_fn_t)(const char *const input, const int32_t nelements, char *const output, int32_t outputSize, const char type, int8_t level); @@ -289,4 +293,4 @@ int8_t tUpdateCompress(uint32_t oldCmpr, uint32_t newCmpr, uint8_t l2Disabled, u } #endif -#endif /*_TD_UTIL_COMPRESSION_H_*/ \ No newline at end of file +#endif /*_TD_UTIL_COMPRESSION_H_*/ diff --git a/include/util/tdef.h b/include/util/tdef.h index 07be21e044..e513e0460c 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -325,7 +325,7 @@ typedef enum ELogicConditionType { #define TSDB_CLUSTER_ID_LEN 40 #define TSDB_MACHINE_ID_LEN 24 -#define TSDB_FQDN_LEN 128 +#define TSDB_FQDN_LEN TD_FQDN_LEN #define TSDB_EP_LEN (TSDB_FQDN_LEN + 6) #define TSDB_IPv4ADDR_LEN 16 #define TSDB_FILENAME_LEN 128 @@ -644,6 +644,7 @@ enum { RAND_ERR_MEMORY = 1, RAND_ERR_FILE = 2, RAND_ERR_NETWORK = 4 }; #define MONITOR_TAG_NAME_LEN 100 #define MONITOR_TAG_VALUE_LEN 300 #define MONITOR_METRIC_NAME_LEN 100 + #ifdef __cplusplus } #endif diff --git a/include/util/tdigest.h b/include/util/tdigest.h index 03519b4c7b..d807a7f0cf 100644 --- a/include/util/tdigest.h +++ b/include/util/tdigest.h @@ -23,6 +23,7 @@ #define TDIGEST_H #include "os.h" +#include "libs/function/functionResInfo.h" #ifndef M_PI #define M_PI 3.14159265358979323846264338327950288 /* pi */ @@ -37,32 +38,6 @@ #define TDIGEST_SIZE(compression) \ (sizeof(TDigest) + sizeof(SCentroid) * GET_CENTROID(compression) + sizeof(SPt) * GET_THRESHOLD(compression)) -typedef struct SCentroid { - double mean; - int64_t weight; -} SCentroid; - -typedef struct SPt { - double value; - int64_t weight; -} SPt; - -typedef struct TDigest { - double compression; - int32_t threshold; - int64_t size; - - int64_t total_weight; - double min; - double max; - - int32_t num_buffered_pts; - SPt *buffered_pts; - - int32_t num_centroids; - SCentroid *centroids; -} TDigest; - TDigest *tdigestNewFrom(void *pBuf, int32_t compression); int32_t tdigestAdd(TDigest *t, double x, int64_t w); int32_t tdigestMerge(TDigest *t1, TDigest *t2); diff --git a/include/util/tlog.h b/include/util/tlog.h index 832dc7dbc1..e80e94de32 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -69,6 +69,8 @@ extern int32_t tdbDebugFlag; extern int32_t sndDebugFlag; extern int32_t simDebugFlag; +extern int32_t tqClientDebug; + int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc); void taosCloseLog(); void taosResetLog(); diff --git a/include/util/tuuid.h b/include/util/tuuid.h index a4e1059371..f7e71b910f 100644 --- a/include/util/tuuid.h +++ b/include/util/tuuid.h @@ -48,3 +48,6 @@ int64_t tGenIdPI64(void); * @return */ int64_t tGenQid64(int8_t dnodeId); + +int32_t taosGetSystemUUIDU32(uint32_t *uuid); +int32_t taosGetSystemUUIDU64(uint64_t *uuid); diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile index 99a10e0285..a67724d5a8 100644 --- a/packaging/docker/Dockerfile +++ b/packaging/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:latest +FROM ubuntu:22.04 WORKDIR /root diff --git a/packaging/docker/bin/entrypoint.sh b/packaging/docker/bin/entrypoint.sh index a60254d7ef..02895f4755 100755 --- a/packaging/docker/bin/entrypoint.sh +++ b/packaging/docker/bin/entrypoint.sh @@ -10,6 +10,12 @@ fi DISABLE_ADAPTER=${TAOS_DISABLE_ADAPTER:-0} unset TAOS_DISABLE_ADAPTER +DISABLE_KEEPER=${TAOS_DISABLE_KEEPER:-0} +unset TAOS_DISABLE_KEEPER + +DISABLE_EXPLORER=${TAOS_DISABLE_EXPLORER:-0} +unset TAOS_DISABLE_EXPLORER + # to get mnodeEpSet from data dir DATA_DIR=$(taosd -C|grep -E 'dataDir.*(\S+)' -o |head -n1|sed 's/dataDir *//') DATA_DIR=${DATA_DIR:-/var/lib/taos} @@ -33,26 +39,18 @@ ulimit -c unlimited sysctl -w kernel.core_pattern=/corefile/core-$FQDN-%e-%p >/dev/null >&1 set -e -if [ "$DISABLE_ADAPTER" = "0" ]; then - which taosadapter >/dev/null && taosadapter & - # wait for 6041 port ready - for _ in $(seq 1 20); do - nc -z localhost 6041 && break - sleep 0.5 - done -fi + # if dnode has been created or has mnode ep set or the host is first ep or not for cluster, just start. if [ -f "$DATA_DIR/dnode/dnode.json" ] || [ -f "$DATA_DIR/dnode/mnodeEpSet.json" ] || [ "$TAOS_FQDN" = "$FIRST_EP_HOST" ]; then - $@ + $@ & # others will first wait the first ep ready. else if [ "$TAOS_FIRST_EP" = "" ]; then echo "run TDengine with single node." - $@ - exit $? + $@ & fi while true; do es=$(taos -h $FIRST_EP_HOST -P $FIRST_EP_PORT --check | grep "^[0-9]*:") @@ -64,5 +62,36 @@ else fi sleep 1s done - $@ + if ps aux | grep -v grep | grep taosd > dev/null; then + echo "TDengine is running" + else + $@ & + fi fi + +if [ "$DISABLE_ADAPTER" = "0" ]; then + which taosadapter >/dev/null && taosadapter & + # wait for 6041 port ready + for _ in $(seq 1 20); do + nc -z localhost 6041 && break + sleep 0.5 + done +fi + +if [ "$DISABLE_KEEPER" = "0" ]; then + sleep 3 + which taoskeeper >/dev/null && taoskeeper & + # wait for 6043 port ready + for _ in $(seq 1 20); do + nc -z localhost 6043 && break + sleep 0.5 + done +fi + + +which taos-explorer >/dev/null && taos-explorer +# wait for 6060 port ready +for _ in $(seq 1 20); do + nc -z localhost 6060 && break + sleep 0.5 +done diff --git a/packaging/tools/TDengine.pkgproj b/packaging/tools/TDengine.pkgproj index 7aaac62735..3171d80d71 100644 --- a/packaging/tools/TDengine.pkgproj +++ b/packaging/tools/TDengine.pkgproj @@ -663,7 +663,19 @@ SUMMARY LOCALIZATIONS - + + + LANGUAGE + English + VALUE + + PATH + /opt/taos/TDengine/packaging/tools/mac_install_summary.txt + PATH_TYPE + 0 + + + TITLE diff --git a/packaging/tools/com.taosdata.taos-explorer.plist b/packaging/tools/com.taosdata.taos-explorer.plist new file mode 100644 index 0000000000..2edb5552ad --- /dev/null +++ b/packaging/tools/com.taosdata.taos-explorer.plist @@ -0,0 +1,33 @@ + + + + + Label + com.tdengine.taos-explorer + ProgramArguments + + /usr/local/bin/taos-explorer + + ProcessType + Interactive + Disabled + + RunAtLoad + + LaunchOnlyOnce + + SessionCreate + + ExitTimeOut + 600 + KeepAlive + + SuccessfulExit + + AfterInitialDemand + + + Program + /usr/local/bin/taos-explorer + + \ No newline at end of file diff --git a/packaging/tools/mac_install_summary.txt b/packaging/tools/mac_install_summary.txt new file mode 100644 index 0000000000..d13f1c280b --- /dev/null +++ b/packaging/tools/mac_install_summary.txt @@ -0,0 +1,13 @@ +TDengine is installed successfully. Please open an Mac terminal and execute the commands below: + +To configure TDengine, sudo vi /etc/taos/taos.cfg +To configure taosadapter, sudo vi /etc/taos/taoadapter.toml +To configure taos-explorer, sudo vi /etc/taos/explorer.toml +To start service, sudo launchctl start com.tdengine.taosd +To start Taos Adapter, sudo launchctl start com.tdengine.taosadapter +To start Taos Explorer, sudo launchctl start com.tdengine.taos-explorer + +To start all the components, sudo start-all.sh +To access TDengine Commnd Line Interface, taos -h YourServerName +To access TDengine Graphic User Interface, http://YourServerName:6060 +To read the user manual, http://YourServerName:6060/docs-en diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index ea19125bf5..13447bd5e4 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -616,8 +616,8 @@ function update_TDengine() { [ -f ${installDir}/bin/taosadapter ] && \ echo -e "${GREEN_DARK}To start Adapter ${NC}: taosadapter &${NC}" else - echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}" - echo -e "${GREEN_DARK}To start Adapter ${NC}: launchctl start com.tdengine.taosadapter${NC}" + echo -e "${GREEN_DARK}To start service ${NC}: sudo launchctl start com.tdengine.taosd${NC}" + echo -e "${GREEN_DARK}To start Adapter ${NC}: sudo launchctl start com.tdengine.taosadapter${NC}" fi fi @@ -668,8 +668,8 @@ function install_TDengine() { [ -f ${installDir}/bin/taosadapter ] && \ echo -e "${GREEN_DARK}To start Adapter ${NC}: taosadapter &${NC}" else - echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}" - echo -e "${GREEN_DARK}To start Adapter ${NC}: launchctl start com.tdengine.taosadapter${NC}" + echo -e "${GREEN_DARK}To start service ${NC}: sudo launchctl start com.tdengine.taosd${NC}" + echo -e "${GREEN_DARK}To start Adapter ${NC}: sudo launchctl start com.tdengine.taosadapter${NC}" fi fi diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 58a17e2a50..c3f459ca9c 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -206,10 +206,17 @@ function clean_log() { } function clean_service_on_launchctl() { - ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : - ${csudo}rm /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : - ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || : - ${csudo}rm /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || : + ${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist || : + ${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${PREFIX}adapter.plist || : + ${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${PREFIX}keeper.plist || : + ${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${PREFIX}-explorer.plist || : + + ${csudo}launchctl remove com.tdengine.taosd || : + ${csudo}launchctl remove com.tdengine.${PREFIX}adapter || : + ${csudo}launchctl remove com.tdengine.${PREFIX}keeper || : + ${csudo}launchctl remove com.tdengine.${PREFIX}-explorer || : + + ${csudo}rm /Library/LaunchDaemons/com.taosdata.* > /dev/null 2>&1 || : } function remove_data_and_config() { @@ -250,6 +257,12 @@ if [ -e ${install_main_dir}/uninstall_${PREFIX}x.sh ]; then fi fi + +if [ "$osType" = "Darwin" ]; then + clean_service_on_launchctl + ${csudo}rm -rf /Applications/TDengine.app +fi + remove_bin clean_header # Remove lib file @@ -282,10 +295,7 @@ elif echo $osinfo | grep -qwi "centos"; then # echo "this is centos system" ${csudo}rpm -e --noscripts tdengine >/dev/null 2>&1 || : fi -if [ "$osType" = "Darwin" ]; then - clean_service_on_launchctl - ${csudo}rm -rf /Applications/TDengine.app -fi + command -v systemctl >/dev/null 2>&1 && ${csudo}systemctl daemon-reload >/dev/null 2>&1 || true echo diff --git a/packaging/tools/tdengine.iss b/packaging/tools/tdengine.iss index 8085c55e3e..c3eb6f9f68 100644 --- a/packaging/tools/tdengine.iss +++ b/packaging/tools/tdengine.iss @@ -71,8 +71,8 @@ Source: {#MyAppSourceDir}\taosdump.exe; DestDir: "{app}"; DestName: "{#CusPrompt Filename: {sys}\sc.exe; Parameters: "create taosd start= DEMAND binPath= ""C:\\TDengine\\taosd.exe --win_service""" ; Flags: runhidden Filename: {sys}\sc.exe; Parameters: "create taosadapter start= DEMAND binPath= ""C:\\TDengine\\taosadapter.exe""" ; Flags: runhidden -Filename: "C:\Windows\System32\odbcconf.exe"; Parameters: "/S /F win_odbcinst.ini"; WorkingDir: "{app}\taos_odbc\x64"; Flags: runhidden; StatusMsg: "Configuring ODBC x64" -Filename: "C:\Windows\SysWOW64\odbcconf.exe"; Parameters: "/S /F win_odbcinst.ini"; WorkingDir: "{app}\taos_odbc\x86"; Flags: runhidden; StatusMsg: "Configuring ODBC x86" +Filename: "C:\Windows\System32\odbcconf.exe"; Parameters: "/S /F win_odbc_install.ini"; WorkingDir: "{app}\taos_odbc\x64"; Flags: runhidden; StatusMsg: "Configuring ODBC x64" +Filename: "C:\Windows\SysWOW64\odbcconf.exe"; Parameters: "/S /F win_odbc_install.ini"; WorkingDir: "{app}\taos_odbc\x86"; Flags: runhidden; StatusMsg: "Configuring ODBC x86" [UninstallRun] RunOnceId: "stoptaosd"; Filename: {sys}\sc.exe; Parameters: "stop taosd" ; Flags: runhidden diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index a70c3b8e64..8d45e8b4a8 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -226,31 +226,17 @@ typedef struct { SSchemaWrapper schema; int32_t resIter; SReqResultInfo resInfo; -} SMqRspObjCommon; - -typedef struct { - SMqRspObjCommon common; - SMqDataRsp rsp; + union{ + struct{ + SMqRspHead head; + STqOffsetVal rspOffset; + }; + SMqDataRsp dataRsp; + SMqMetaRsp metaRsp; + SMqBatchMetaRsp batchMetaRsp; + }; } SMqRspObj; -typedef struct { - int8_t resType; - char topic[TSDB_TOPIC_FNAME_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int32_t vgId; - SMqMetaRsp metaRsp; -} SMqMetaRspObj; - -typedef struct { - SMqRspObjCommon common; - STaosxRsp rsp; -} SMqTaosxRspObj; - -typedef struct { - SMqRspObjCommon common; - SMqBatchMetaRsp rsp; -} SMqBatchMetaRspObj; - typedef struct SReqRelInfo { uint64_t userRefId; uint64_t prevRefId; @@ -332,7 +318,7 @@ int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols); static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { SMqRspObj* msg = (SMqRspObj*)res; - return (SReqResultInfo*)&msg->common.resInfo; + return (SReqResultInfo*)&msg->resInfo; } int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pResInfo); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 3b755c2921..21f7c93036 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -114,10 +114,10 @@ static void concatStrings(SArray *list, char *buf, int size) { db = dot + 1; } if (i != 0) { - (void)strcat(buf, ","); + (void)strncat(buf, ",", size - 1 - len); len += 1; } - int ret = snprintf(buf + len, size - len, "%s", db); + int ret = tsnprintf(buf + len, size - len, "%s", db); if (ret < 0) { tscError("snprintf failed, buf:%s, ret:%d", buf, ret); break; @@ -1094,18 +1094,14 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { * @return */ uint64_t generateRequestId() { - static uint64_t hashId = 0; - static uint32_t requestSerialId = 0; + static uint32_t hashId = 0; + static int32_t requestSerialId = 0; if (hashId == 0) { - char uid[64] = {0}; - int32_t code = taosGetSystemUUID(uid, tListLen(uid)); + int32_t code = taosGetSystemUUIDU32(&hashId); if (code != TSDB_CODE_SUCCESS) { tscError("Failed to get the system uid to generated request id, reason:%s. use ip address instead", - tstrerror(TAOS_SYSTEM_ERROR(errno))); - - } else { - hashId = MurmurHash3_32(uid, strlen(uid)); + tstrerror(code)); } } @@ -1117,7 +1113,7 @@ uint64_t generateRequestId() { uint32_t val = atomic_add_fetch_32(&requestSerialId, 1); if (val >= 0xFFFF) atomic_store_32(&requestSerialId, 0); - id = ((hashId & 0x0FFF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); + id = (((uint64_t)(hashId & 0x0FFF)) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); if (id) { break; } @@ -1132,27 +1128,27 @@ static setConfRet taos_set_config_imp(const char *config){ static bool setConfFlag = false; if (setConfFlag) { ret.retCode = SET_CONF_RET_ERR_ONLY_ONCE; - strcpy(ret.retMsg, "configuration can only set once"); + tstrncpy(ret.retMsg, "configuration can only set once", RET_MSG_LENGTH); return ret; } taosInitGlobalCfg(); cJSON *root = cJSON_Parse(config); if (root == NULL){ ret.retCode = SET_CONF_RET_ERR_JSON_PARSE; - strcpy(ret.retMsg, "parse json error"); + tstrncpy(ret.retMsg, "parse json error", RET_MSG_LENGTH); return ret; } int size = cJSON_GetArraySize(root); if(!cJSON_IsObject(root) || size == 0) { ret.retCode = SET_CONF_RET_ERR_JSON_INVALID; - strcpy(ret.retMsg, "json content is invalid, must be not empty object"); + tstrncpy(ret.retMsg, "json content is invalid, must be not empty object", RET_MSG_LENGTH); return ret; } if(size >= 1000) { ret.retCode = SET_CONF_RET_ERR_TOO_LONG; - strcpy(ret.retMsg, "json object size is too long"); + tstrncpy(ret.retMsg, "json object size is too long", RET_MSG_LENGTH); return ret; } @@ -1160,7 +1156,7 @@ static setConfRet taos_set_config_imp(const char *config){ cJSON *item = cJSON_GetArrayItem(root, i); if(!item) { ret.retCode = SET_CONF_RET_ERR_INNER; - strcpy(ret.retMsg, "inner error"); + tstrncpy(ret.retMsg, "inner error", RET_MSG_LENGTH); return ret; } if(!taosReadConfigOption(item->string, item->valuestring, NULL, NULL, TAOS_CFG_CSTATUS_OPTION, TSDB_CFG_CTYPE_B_CLIENT)){ diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 6ee6d753e4..75716d0bd2 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -55,7 +55,7 @@ static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SC for (int32_t i = 0; i < numOfBatchs; ++i) { SGetUserAuthRsp *rsp = taosArrayGet(batchRsp.pArray, i); if (NULL == rsp) { - code = TSDB_CODE_OUT_OF_RANGE; + code = terrno; goto _return; } tscDebug("hb to update user auth, user:%s, version:%d", rsp->user, rsp->version); @@ -217,7 +217,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog for (int32_t i = 0; i < numOfBatchs; ++i) { SDbHbRsp *rsp = taosArrayGet(batchRsp.pArray, i); if (NULL == rsp) { - code = TSDB_CODE_OUT_OF_RANGE; + code = terrno; goto _return; } if (rsp->useDbRsp) { @@ -291,7 +291,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo for (int32_t i = 0; i < numOfMeta; ++i) { STableMetaRsp *rsp = taosArrayGet(hbRsp.pMetaRsp, i); if (NULL == rsp) { - code = TSDB_CODE_OUT_OF_RANGE; + code = terrno; goto _return; } if (rsp->numOfColumns < 0) { @@ -313,7 +313,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo for (int32_t i = 0; i < numOfIndex; ++i) { STableIndexRsp *rsp = taosArrayGet(hbRsp.pIndexRsp, i); if (NULL == rsp) { - code = TSDB_CODE_OUT_OF_RANGE; + code = terrno; goto _return; } TSC_ERR_JRET(catalogUpdateTableIndex(pCatalog, rsp)); @@ -354,7 +354,7 @@ static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatal for (int32_t i = 0; i < numOfMeta; ++i) { SViewMetaRsp *rsp = taosArrayGetP(hbRsp.pViewRsp, i); if (NULL == rsp) { - code = TSDB_CODE_OUT_OF_RANGE; + code = terrno; goto _return; } if (rsp->numOfCols < 0) { @@ -772,7 +772,7 @@ static int32_t hbGetUserAuthInfo(SClientHbKey *connKey, SHbParam *param, SClient SUserAuthVersion *qUserAuth = (SUserAuthVersion *)taosMemoryRealloc(pKv->value, (userNum + 1) * sizeof(SUserAuthVersion)); if (qUserAuth) { - (void)strncpy((qUserAuth + userNum)->user, pTscObj->user, TSDB_USER_LEN); + tstrncpy((qUserAuth + userNum)->user, pTscObj->user, TSDB_USER_LEN); (qUserAuth + userNum)->version = htonl(-1); // force get userAuthInfo pKv->value = qUserAuth; pKv->valueLen += sizeof(SUserAuthVersion); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 15bd5795e2..2c67cafdf5 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -949,7 +949,7 @@ int32_t handleQueryExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, for (int32_t i = 0; i < tbNum; ++i) { STbVerInfo* tbInfo = taosArrayGet(pTbArray, i); if (NULL == tbInfo) { - code = TSDB_CODE_OUT_OF_RANGE; + code = terrno; goto _return; } STbSVersion tbSver = {.tbFName = tbInfo->tbFName, .sver = tbInfo->sversion, .tver = tbInfo->tversion}; @@ -1921,19 +1921,19 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons return NULL; } -TAOS* taos_connect_l(const char* ip, int ipLen, const char* user, int userLen, const char* pass, int passLen, - const char* db, int dbLen, uint16_t port) { - char ipStr[TSDB_EP_LEN] = {0}; - char dbStr[TSDB_DB_NAME_LEN] = {0}; - char userStr[TSDB_USER_LEN] = {0}; - char passStr[TSDB_PASSWORD_LEN] = {0}; - - (void)strncpy(ipStr, ip, TMIN(TSDB_EP_LEN - 1, ipLen)); - (void)strncpy(userStr, user, TMIN(TSDB_USER_LEN - 1, userLen)); - (void)strncpy(passStr, pass, TMIN(TSDB_PASSWORD_LEN - 1, passLen)); - (void)strncpy(dbStr, db, TMIN(TSDB_DB_NAME_LEN - 1, dbLen)); - return taos_connect(ipStr, userStr, passStr, dbStr, port); -} +//TAOS* taos_connect_l(const char* ip, int ipLen, const char* user, int userLen, const char* pass, int passLen, +// const char* db, int dbLen, uint16_t port) { +// char ipStr[TSDB_EP_LEN] = {0}; +// char dbStr[TSDB_DB_NAME_LEN] = {0}; +// char userStr[TSDB_USER_LEN] = {0}; +// char passStr[TSDB_PASSWORD_LEN] = {0}; +// +// tstrncpy(ipStr, ip, TMIN(TSDB_EP_LEN - 1, ipLen)); +// tstrncpy(userStr, user, TMIN(TSDB_USER_LEN - 1, userLen)); +// tstrncpy(passStr, pass, TMIN(TSDB_PASSWORD_LEN - 1, passLen)); +// tstrncpy(dbStr, db, TMIN(TSDB_DB_NAME_LEN - 1, dbLen)); +// return taos_connect(ipStr, userStr, passStr, dbStr, port); +//} void doSetOneRowPtr(SReqResultInfo* pResultInfo) { for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) { @@ -2275,7 +2275,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int char* jsonInnerData = data + CHAR_BYTES; char dst[TSDB_MAX_JSON_TAG_LEN] = {0}; if (jsonInnerType == TSDB_DATA_TYPE_NULL) { - (void)sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L); + (void)snprintf(varDataVal(dst), TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE, "%s", TSDB_DATA_NULL_STR_L); varDataSetLen(dst, strlen(varDataVal(dst))); } else if (tTagIsJson(data)) { char* jsonString = NULL; @@ -2298,10 +2298,10 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int *(char*)POINTER_SHIFT(varDataVal(dst), length + CHAR_BYTES) = '\"'; } else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) { double jsonVd = *(double*)(jsonInnerData); - (void)sprintf(varDataVal(dst), "%.9lf", jsonVd); + (void)snprintf(varDataVal(dst), TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE, "%.9lf", jsonVd); varDataSetLen(dst, strlen(varDataVal(dst))); } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) { - (void)sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false"); + (void)snprintf(varDataVal(dst), TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE, "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false"); varDataSetLen(dst, strlen(varDataVal(dst))); } else { tscError("doConvertJson error: invalid type:%d", jsonInnerType); @@ -2658,8 +2658,8 @@ int32_t appendTbToReq(SHashObj* pHash, int32_t pos1, int32_t len1, int32_t pos2, return -1; } - char dbFName[TSDB_DB_FNAME_LEN]; - (void)sprintf(dbFName, "%d.%.*s", acctId, dbLen, dbName); + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + (void)snprintf(dbFName, TSDB_DB_FNAME_LEN, "%d.%.*s", acctId, dbLen, dbName); STablesReq* pDb = taosHashGet(pHash, dbFName, strlen(dbFName)); if (pDb) { @@ -2672,7 +2672,7 @@ int32_t appendTbToReq(SHashObj* pHash, int32_t pos1, int32_t len1, int32_t pos2, if (NULL == db.pTables) { return terrno; } - (void)strcpy(db.dbFName, dbFName); + tstrncpy(db.dbFName, dbFName, TSDB_DB_FNAME_LEN); if (NULL == taosArrayPush(db.pTables, &name)) { return terrno; } diff --git a/source/client/src/clientJniConnector.c b/source/client/src/clientJniConnector.c index d783c6d8e4..f87611ac00 100644 --- a/source/client/src/clientJniConnector.c +++ b/source/client/src/clientJniConnector.c @@ -1083,14 +1083,14 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_stmtErrorMsgIm TAOS *tscon = (TAOS *)con; if (tscon == NULL) { jniError("jobj:%p, connection already closed", jobj); - (void)sprintf(errMsg, "jobj:%p, connection already closed", jobj); + (void)snprintf(errMsg, sizeof(errMsg), "jobj:%p, connection already closed", jobj); return (*env)->NewStringUTF(env, errMsg); } TAOS_STMT *pStmt = (TAOS_STMT *)stmt; if (pStmt == NULL) { jniError("jobj:%p, conn:%p, invalid stmt", jobj, tscon); - (void)sprintf(errMsg, "jobj:%p, conn:%p, invalid stmt", jobj, tscon); + (void)snprintf(errMsg, sizeof(errMsg), "jobj:%p, conn:%p, invalid stmt", jobj, tscon); return (*env)->NewStringUTF(env, errMsg); } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 1b5f95ce16..a35c7c7a4c 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -373,25 +373,22 @@ void taos_free_result(TAOS_RES *res) { SRequestObj *pRequest = (SRequestObj *)res; tscDebug("0x%" PRIx64 " taos_free_result start to free query", pRequest->requestId); destroyRequest(pRequest); - } else if (TD_RES_TMQ_METADATA(res)) { - SMqTaosxRspObj *pRsp = (SMqTaosxRspObj *)res; - tDeleteSTaosxRsp(&pRsp->rsp); - doFreeReqResultInfo(&pRsp->common.resInfo); - taosMemoryFree(pRsp); - } else if (TD_RES_TMQ(res)) { - SMqRspObj *pRsp = (SMqRspObj *)res; - tDeleteMqDataRsp(&pRsp->rsp); - doFreeReqResultInfo(&pRsp->common.resInfo); - taosMemoryFree(pRsp); - } else if (TD_RES_TMQ_META(res)) { - SMqMetaRspObj *pRspObj = (SMqMetaRspObj *)res; - tDeleteMqMetaRsp(&pRspObj->metaRsp); - taosMemoryFree(pRspObj); - } else if (TD_RES_TMQ_BATCH_META(res)) { - SMqBatchMetaRspObj *pBtRspObj = (SMqBatchMetaRspObj *)res; - tDeleteMqBatchMetaRsp(&pBtRspObj->rsp); - taosMemoryFree(pBtRspObj); + return; } + SMqRspObj *pRsp = (SMqRspObj *)res; + if (TD_RES_TMQ(res)) { + tDeleteMqDataRsp(&pRsp->dataRsp); + doFreeReqResultInfo(&pRsp->resInfo); + } else if (TD_RES_TMQ_METADATA(res)) { + tDeleteSTaosxRsp(&pRsp->dataRsp); + doFreeReqResultInfo(&pRsp->resInfo); + } else if (TD_RES_TMQ_META(res)) { + tDeleteMqMetaRsp(&pRsp->metaRsp); + } else if (TD_RES_TMQ_BATCH_META(res)) { + tDeleteMqBatchMetaRsp(&pRsp->batchMetaRsp); + } + taosMemoryFree(pRsp); + } void taos_kill_query(TAOS *taos) { @@ -454,7 +451,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { } else if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { SMqRspObj *msg = ((SMqRspObj *)res); SReqResultInfo *pResultInfo = NULL; - if (msg->common.resIter == -1) { + if (msg->resIter == -1) { if (tmqGetNextResInfo(res, true, &pResultInfo) != 0) { return NULL; } @@ -485,71 +482,75 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { } int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { + return taos_print_row_with_size(str, INT32_MAX, row, fields, num_fields); +} +int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields){ int32_t len = 0; for (int i = 0; i < num_fields; ++i) { - if (i > 0) { + if (i > 0 && len < size - 1) { str[len++] = ' '; } if (row[i] == NULL) { - len += sprintf(str + len, "%s", TSDB_DATA_NULL_STR); + len += snprintf(str + len, size - len, "%s", TSDB_DATA_NULL_STR); continue; } switch (fields[i].type) { case TSDB_DATA_TYPE_TINYINT: - len += sprintf(str + len, "%d", *((int8_t *)row[i])); + len += snprintf(str + len, size - len, "%d", *((int8_t *)row[i])); break; case TSDB_DATA_TYPE_UTINYINT: - len += sprintf(str + len, "%u", *((uint8_t *)row[i])); + len += snprintf(str + len, size - len, "%u", *((uint8_t *)row[i])); break; case TSDB_DATA_TYPE_SMALLINT: - len += sprintf(str + len, "%d", *((int16_t *)row[i])); + len += snprintf(str + len, size - len, "%d", *((int16_t *)row[i])); break; case TSDB_DATA_TYPE_USMALLINT: - len += sprintf(str + len, "%u", *((uint16_t *)row[i])); + len += snprintf(str + len, size - len, "%u", *((uint16_t *)row[i])); break; case TSDB_DATA_TYPE_INT: - len += sprintf(str + len, "%d", *((int32_t *)row[i])); + len += snprintf(str + len, size - len, "%d", *((int32_t *)row[i])); break; case TSDB_DATA_TYPE_UINT: - len += sprintf(str + len, "%u", *((uint32_t *)row[i])); + len += snprintf(str + len, size - len, "%u", *((uint32_t *)row[i])); break; case TSDB_DATA_TYPE_BIGINT: - len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); + len += snprintf(str + len, size - len, "%" PRId64, *((int64_t *)row[i])); break; case TSDB_DATA_TYPE_UBIGINT: - len += sprintf(str + len, "%" PRIu64, *((uint64_t *)row[i])); + len += snprintf(str + len, size - len, "%" PRIu64, *((uint64_t *)row[i])); break; case TSDB_DATA_TYPE_FLOAT: { float fv = 0; fv = GET_FLOAT_VAL(row[i]); - len += sprintf(str + len, "%f", fv); + len += snprintf(str + len, size - len, "%f", fv); } break; case TSDB_DATA_TYPE_DOUBLE: { double dv = 0; dv = GET_DOUBLE_VAL(row[i]); - len += sprintf(str + len, "%lf", dv); + len += snprintf(str + len, size - len, "%lf", dv); } break; case TSDB_DATA_TYPE_VARBINARY: { void *data = NULL; - uint32_t size = 0; + uint32_t tmp = 0; int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); - if (taosAscii2Hex(row[i], charLen, &data, &size) < 0) { + if (taosAscii2Hex(row[i], charLen, &data, &tmp) < 0) { break; } - (void)memcpy(str + len, data, size); - len += size; + uint32_t copyLen = TMIN(size - len - 1, tmp); + (void)memcpy(str + len, data, copyLen); + len += copyLen; taosMemoryFree(data); } break; case TSDB_DATA_TYPE_BINARY: @@ -569,21 +570,28 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) } } - (void)memcpy(str + len, row[i], charLen); - len += charLen; + uint32_t copyLen = TMIN(size - len - 1, charLen); + (void)memcpy(str + len, row[i], copyLen); + len += copyLen; } break; case TSDB_DATA_TYPE_TIMESTAMP: - len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); + len += snprintf(str + len, size - len, "%" PRId64, *((int64_t *)row[i])); break; case TSDB_DATA_TYPE_BOOL: - len += sprintf(str + len, "%d", *((int8_t *)row[i])); + len += snprintf(str + len, size - len, "%d", *((int8_t *)row[i])); default: break; } + + if (len >= size - 1) { + break; + } + } + if (len < size){ + str[len] = 0; } - str[len] = 0; return len; } @@ -948,7 +956,7 @@ int taos_get_current_db(TAOS *taos, char *database, int len, int *required) { if (required) *required = strlen(pTscObj->db) + 1; TSC_ERR_JRET(TSDB_CODE_INVALID_PARA); } else { - (void)strcpy(database, pTscObj->db); + tstrncpy(database, pTscObj->db, len); code = 0; } _return: diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index 6667c4c741..595c871953 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -21,7 +21,7 @@ char tmpSlowLogPath[PATH_MAX] = {0}; TdThread monitorThread; static int32_t getSlowLogTmpDir(char* tmpPath, int32_t size) { - int ret = snprintf(tmpPath, size, "%s/tdengine_slow_log/", tsTempDir); + int ret = tsnprintf(tmpPath, size, "%s/tdengine_slow_log/", tsTempDir); if (ret < 0) { tscError("failed to get tmp path ret:%d", ret); return TSDB_CODE_TSC_INTERNAL_ERROR; @@ -183,7 +183,7 @@ FAILED: static void generateClusterReport(taos_collector_registry_t* registry, void* pTransporter, SEpSet* epSet) { char ts[50] = {0}; - (void)sprintf(ts, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); + (void)snprintf(ts, sizeof(ts), "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); char* pCont = (char*)taos_collector_registry_bridge_new(registry, ts, "%" PRId64, NULL); if (NULL == pCont) { tscError("generateClusterReport failed, get null content."); @@ -401,7 +401,7 @@ static void monitorWriteSlowLog2File(MonitorSlowLogData* slowLogData, char* tmpP return; } pClient->lastCheckTime = taosGetMonoTimestampMs(); - (void)strcpy(pClient->path, path); + tstrncpy(pClient->path, path, PATH_MAX); pClient->offset = 0; pClient->pFile = pFile; if (taosHashPut(monitorSlowLogHash, &slowLogData->clusterId, LONG_BYTES, &pClient, POINTER_BYTES) != 0) { @@ -458,7 +458,7 @@ static char* readFile(TdFilePtr pFile, int64_t* offset, int64_t size) { return NULL; } char* buf = pCont; - (void)strcat(buf++, "["); + (void)strncat(buf++, "[", totalSize - 1); int64_t readSize = taosReadFile(pFile, buf, totalSize - 4); // 4 reserved for [] if (readSize <= 0) { if (readSize < 0) { diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index d0ea7055de..acba8117c6 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -23,12 +23,12 @@ #include "tglobal.h" #include "tmsgtype.h" -#define RAW_NULL_CHECK(c) \ - do { \ - if (c == NULL) { \ - code = terrno; \ - goto end; \ - } \ +#define RAW_NULL_CHECK(c) \ + do { \ + if (c == NULL) { \ + code = terrno; \ + goto end; \ + } \ } while (0) #define RAW_FALSE_CHECK(c) \ @@ -47,7 +47,7 @@ } \ } while (0) -#define LOG_ID_TAG "connId:0x%" PRIx64 ",qid:0x%" PRIx64 +#define LOG_ID_TAG "connId:0x%" PRIx64 ",QID:0x%" PRIx64 #define LOG_ID_VALUE *(int64_t*)taos, pRequest->requestId #define TMQ_META_VERSION "1.0" @@ -458,15 +458,17 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { cJSON* tvalue = NULL; if (IS_VAR_DATA_TYPE(pTagVal->type)) { char* buf = NULL; + int64_t bufSize = 0; if (pTagVal->type == TSDB_DATA_TYPE_VARBINARY) { - buf = taosMemoryCalloc(pTagVal->nData * 2 + 2 + 3, 1); + bufSize = pTagVal->nData * 2 + 2 + 3; } else { - buf = taosMemoryCalloc(pTagVal->nData + 3, 1); + bufSize = pTagVal->nData + 3; } + buf = taosMemoryCalloc(bufSize, 1); RAW_NULL_CHECK(buf); if (!buf) goto end; - if (dataConverToStr(buf, pTagVal->type, pTagVal->pData, pTagVal->nData, NULL) != TSDB_CODE_SUCCESS) { + if (dataConverToStr(buf, bufSize, pTagVal->type, pTagVal->pData, pTagVal->nData, NULL) != TSDB_CODE_SUCCESS) { taosMemoryFree(buf); goto end; } @@ -548,7 +550,7 @@ end: tDecoderClear(&decoder); } -static void processAutoCreateTable(STaosxRsp* rsp, char** string) { +static void processAutoCreateTable(SMqDataRsp* rsp, char** string) { SDecoder* decoder = NULL; SVCreateTbReq* pCreateReq = NULL; int32_t code = 0; @@ -736,13 +738,15 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { goto end; } } else { + int64_t bufSize = 0; if (vAlterTbReq.tagType == TSDB_DATA_TYPE_VARBINARY) { - buf = taosMemoryCalloc(vAlterTbReq.nTagVal * 2 + 2 + 3, 1); + bufSize = vAlterTbReq.nTagVal * 2 + 2 + 3; } else { - buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 3, 1); + bufSize = vAlterTbReq.nTagVal + 3; } + buf = taosMemoryCalloc(bufSize, 1); RAW_NULL_CHECK(buf); - if (dataConverToStr(buf, vAlterTbReq.tagType, vAlterTbReq.pTagVal, vAlterTbReq.nTagVal, NULL) != + if (dataConverToStr(buf, bufSize, vAlterTbReq.tagType, vAlterTbReq.pTagVal, vAlterTbReq.nTagVal, NULL) != TSDB_CODE_SUCCESS) { taosMemoryFree(buf); goto end; @@ -922,7 +926,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { for (int32_t i = 0; i < req.schemaRow.nCols; i++) { SSchema* pSchema = req.schemaRow.pSchema + i; SFieldWithOptions field = {.type = pSchema->type, .flags = pSchema->flags, .bytes = pSchema->bytes}; - (void)strcpy(field.name, pSchema->name); + tstrncpy(field.name, pSchema->name, TSDB_COL_NAME_LEN); if (createDefaultCompress) { field.compress = createDefaultColCmprByType(pSchema->type); @@ -937,7 +941,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { for (int32_t i = 0; i < req.schemaTag.nCols; i++) { SSchema* pSchema = req.schemaTag.pSchema + i; SField field = {.type = pSchema->type, .flags = pSchema->flags, .bytes = pSchema->bytes}; - (void)strcpy(field.name, pSchema->name); + tstrncpy(field.name, pSchema->name, TSDB_COL_NAME_LEN); RAW_NULL_CHECK(taosArrayPush(pReq.pTags, &field)); } @@ -1188,7 +1192,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { pCreateReq->ctb.suid = pTableMeta->uid; SArray* pTagVals = NULL; - code = tTagToValArray((STag *)pCreateReq->ctb.pTag, &pTagVals); + code = tTagToValArray((STag*)pCreateReq->ctb.pTag, &pTagVals); if (code != TSDB_CODE_SUCCESS) { taosMemoryFreeClear(pTableMeta); goto end; @@ -1206,18 +1210,19 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { if (strcmp(tag->name, tName) == 0 && tag->type != TSDB_DATA_TYPE_JSON) { STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, i); if (pTagVal) { - if (pTagVal->cid != tag->colId){ + if (pTagVal->cid != tag->colId) { pTagVal->cid = tag->colId; rebuildTag = true; } } else { - uError("create tb invalid data %s, size:%d index:%d cid:%d", pCreateReq->name, (int)taosArrayGetSize(pTagVals), i, tag->colId); + uError("create tb invalid data %s, size:%d index:%d cid:%d", pCreateReq->name, + (int)taosArrayGetSize(pTagVals), i, tag->colId); } } } } taosMemoryFreeClear(pTableMeta); - if (rebuildTag){ + if (rebuildTag) { STag* ppTag = NULL; code = tTagNew(pTagVals, 1, false, &ppTag); taosArrayDestroy(pTagVals); @@ -1239,7 +1244,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { if (pTableBatch == NULL) { SVgroupCreateTableBatch tBatch = {0}; tBatch.info = pInfo; - (void)strcpy(tBatch.dbName, pRequest->pDb); + tstrncpy(tBatch.dbName, pRequest->pDb, TSDB_DB_NAME_LEN); tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq)); RAW_NULL_CHECK(tBatch.req.pArray); @@ -1720,8 +1725,8 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { uDebug(LOG_ID_TAG " write raw data, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen); pRequest->syncQuery = true; - rspObj.common.resIter = -1; - rspObj.common.resType = RES_TYPE__TMQ; + rspObj.resIter = -1; + rspObj.resType = RES_TYPE__TMQ; int8_t dataVersion = *(int8_t*)data; if (dataVersion >= MQ_DATA_RSP_VERSION) { @@ -1729,7 +1734,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { dataLen -= sizeof(int8_t) + sizeof(int32_t); } tDecoderInit(&decoder, data, dataLen); - code = tDecodeMqDataRsp(&decoder, &rspObj.rsp); + code = tDecodeMqDataRsp(&decoder, &rspObj.dataRsp); if (code != 0) { SET_ERROR_MSG("decode mq data rsp failed"); code = TSDB_CODE_INVALID_MSG; @@ -1753,19 +1758,19 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { RAW_RETURN_CHECK(smlInitHandle(&pQuery)); pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); RAW_NULL_CHECK(pVgHash); - while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) { - void* pRetrieve = taosArrayGetP(rspObj.rsp.common.blockData, rspObj.common.resIter); + while (++rspObj.resIter < rspObj.dataRsp.blockNum) { + void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter); RAW_NULL_CHECK(pRetrieve); - if (!rspObj.rsp.common.withSchema) { + if (!rspObj.dataRsp.withSchema) { goto end; } - const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.common.blockTbName, rspObj.common.resIter); + const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter); RAW_NULL_CHECK(tbName); SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; - (void)strcpy(pName.dbname, pRequest->pDb); - (void)strcpy(pName.tname, tbName); + tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN); + tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN); RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta)); @@ -1777,7 +1782,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { RAW_RETURN_CHECK(taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); } - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.common.blockSchema, rspObj.common.resIter); + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter); RAW_NULL_CHECK(pSW); TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD)); RAW_NULL_CHECK(fields); @@ -1804,7 +1809,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { end: uDebug(LOG_ID_TAG " write raw data return, msg:%s", LOG_ID_VALUE, tstrerror(code)); - tDeleteMqDataRsp(&rspObj.rsp); + tDeleteMqDataRsp(&rspObj.dataRsp); tDecoderClear(&decoder); qDestroyQuery(pQuery); destroyRequest(pRequest); @@ -1813,9 +1818,9 @@ end: return code; } -static int32_t buildCreateTbMap(STaosxRsp* rsp, SHashObj* pHashObj) { +static int32_t buildCreateTbMap(SMqDataRsp* rsp, SHashObj* pHashObj) { // find schema data info - int32_t code = 0; + int32_t code = 0; SVCreateTbReq pCreateReq = {0}; SDecoder decoderTmp = {0}; @@ -1826,15 +1831,16 @@ static int32_t buildCreateTbMap(STaosxRsp* rsp, SHashObj* pHashObj) { RAW_NULL_CHECK(lenTmp); tDecoderInit(&decoderTmp, *dataTmp, *lenTmp); - RAW_RETURN_CHECK (tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq)); + RAW_RETURN_CHECK(tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq)); if (pCreateReq.type != TSDB_CHILD_TABLE) { code = TSDB_CODE_INVALID_MSG; goto end; } - if (taosHashGet(pHashObj, pCreateReq.name, strlen(pCreateReq.name)) == NULL){ - RAW_RETURN_CHECK(taosHashPut(pHashObj, pCreateReq.name, strlen(pCreateReq.name), &pCreateReq, sizeof(SVCreateTbReq))); - } else{ + if (taosHashGet(pHashObj, pCreateReq.name, strlen(pCreateReq.name)) == NULL) { + RAW_RETURN_CHECK( + taosHashPut(pHashObj, pCreateReq.name, strlen(pCreateReq.name), &pCreateReq, sizeof(SVCreateTbReq))); + } else { tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE); pCreateReq = (SVCreateTbReq){0}; } @@ -1857,7 +1863,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) int32_t code = TSDB_CODE_SUCCESS; SHashObj* pVgHash = NULL; SQuery* pQuery = NULL; - SMqTaosxRspObj rspObj = {0}; + SMqRspObj rspObj = {0}; SDecoder decoder = {0}; STableMeta* pTableMeta = NULL; SHashObj* pCreateTbHash = NULL; @@ -1867,8 +1873,8 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) uDebug(LOG_ID_TAG " write raw metadata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen); pRequest->syncQuery = true; - rspObj.common.resIter = -1; - rspObj.common.resType = RES_TYPE__TMQ_METADATA; + rspObj.resIter = -1; + rspObj.resType = RES_TYPE__TMQ_METADATA; int8_t dataVersion = *(int8_t*)data; if (dataVersion >= MQ_DATA_RSP_VERSION) { @@ -1877,7 +1883,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) } tDecoderInit(&decoder, data, dataLen); - code = tDecodeSTaosxRsp(&decoder, &rspObj.rsp); + code = tDecodeSTaosxRsp(&decoder, &rspObj.dataRsp); if (code != 0) { SET_ERROR_MSG("decode mq taosx data rsp failed"); code = TSDB_CODE_INVALID_MSG; @@ -1903,34 +1909,34 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) RAW_NULL_CHECK(pVgHash); pCreateTbHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); RAW_NULL_CHECK(pCreateTbHash); - RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.rsp, pCreateTbHash)); + RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.dataRsp, pCreateTbHash)); - uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.common.blockNum); - while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) { - void* pRetrieve = taosArrayGetP(rspObj.rsp.common.blockData, rspObj.common.resIter); + uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.dataRsp.blockNum); + while (++rspObj.resIter < rspObj.dataRsp.blockNum) { + void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter); RAW_NULL_CHECK(pRetrieve); - if (!rspObj.rsp.common.withSchema) { + if (!rspObj.dataRsp.withSchema) { goto end; } - const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.common.blockTbName, rspObj.common.resIter); + const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter); if (!tbName) { SET_ERROR_MSG("block tbname is null"); - code = TSDB_CODE_TMQ_INVALID_MSG; + code = terrno; goto end; } uDebug(LOG_ID_TAG " write raw metadata block tbname:%s", LOG_ID_VALUE, tbName); SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; - (void)strcpy(pName.dbname, pRequest->pDb); - (void)strcpy(pName.tname, tbName); + tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN); + tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN); // find schema data info SVCreateTbReq* pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, tbName, strlen(tbName)); - SVgroupInfo vg = {0}; + SVgroupInfo vg = {0}; RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg)); if (pCreateReqDst) { // change stable name to get meta - (void)strcpy(pName.tname, pCreateReqDst->ctb.stbName); + tstrncpy(pName.tname, pCreateReqDst->ctb.stbName, TSDB_TABLE_NAME_LEN); } RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta)); @@ -1944,7 +1950,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) RAW_RETURN_CHECK(taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); } - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.common.blockSchema, rspObj.common.resIter); + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter); RAW_NULL_CHECK(pSW); TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD)); if (fields == NULL) { @@ -1957,10 +1963,10 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) fields[i].bytes = pSW->pSchema[i].bytes; tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name)); } - void* rawData = getRawDataFromRes(pRetrieve); - char err[ERR_MSG_LEN] = {0}; + void* rawData = getRawDataFromRes(pRetrieve); + char err[ERR_MSG_LEN] = {0}; SVCreateTbReq* pCreateReqTmp = NULL; - if (pCreateReqDst){ + if (pCreateReqDst) { RAW_RETURN_CHECK(cloneSVreateTbReq(pCreateReqDst, &pCreateReqTmp)); } code = rawBlockBindData(pQuery, pTableMeta, rawData, &pCreateReqTmp, fields, pSW->nCols, true, err, ERR_MSG_LEN); @@ -1989,7 +1995,7 @@ end: pIter = taosHashIterate(pCreateTbHash, pIter); } taosHashCleanup(pCreateTbHash); - tDeleteSTaosxRsp(&rspObj.rsp); + tDeleteSTaosxRsp(&rspObj.dataRsp); tDecoderClear(&decoder); qDestroyQuery(pQuery); destroyRequest(pRequest); @@ -2070,31 +2076,28 @@ char* tmq_get_json_meta(TAOS_RES* res) { return NULL; } + char* string = NULL; + SMqRspObj* rspObj = (SMqRspObj*)res; if (TD_RES_TMQ_METADATA(res)) { - SMqTaosxRspObj* pMetaDataRspObj = (SMqTaosxRspObj*)res; - char* string = NULL; - processAutoCreateTable(&pMetaDataRspObj->rsp, &string); - return string; + processAutoCreateTable(&rspObj->dataRsp, &string); } else if (TD_RES_TMQ_BATCH_META(res)) { - SMqBatchMetaRspObj* pBatchMetaRspObj = (SMqBatchMetaRspObj*)res; - char* string = NULL; - processBatchMetaToJson(&pBatchMetaRspObj->rsp, &string); - return string; + processBatchMetaToJson(&rspObj->batchMetaRsp, &string); + } else if (TD_RES_TMQ_META(res)) { + cJSON* pJson = NULL; + processSimpleMeta(&rspObj->metaRsp, &pJson); + string = cJSON_PrintUnformatted(pJson); + cJSON_Delete(pJson); + } else{ + uError("tmq_get_json_meta res:%d, invalid type", *(int8_t*)res); } - SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; - cJSON* pJson = NULL; - processSimpleMeta(&pMetaRspObj->metaRsp, &pJson); - char* string = cJSON_PrintUnformatted(pJson); - cJSON_Delete(pJson); uDebug("tmq_get_json_meta string:%s", string); return string; } void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); } -static int32_t getOffSetLen(const void* rsp) { - const SMqDataRspCommon* pRsp = rsp; +static int32_t getOffSetLen(const SMqDataRsp* pRsp) { SEncoder coder = {0}; tEncoderInit(&coder, NULL, 0); if (tEncodeSTqOffsetVal(&coder, &pRsp->reqOffset) < 0) return -1; @@ -2104,9 +2107,8 @@ static int32_t getOffSetLen(const void* rsp) { return pos; } -typedef int32_t __encode_func__(SEncoder* pEncoder, const void* pRsp); - -static int32_t encodeMqDataRsp(__encode_func__* encodeFunc, void* rspObj, tmq_raw_data* raw) { +typedef int32_t __encode_func__(SEncoder* pEncoder, const SMqDataRsp* pRsp); +static int32_t encodeMqDataRsp(__encode_func__* encodeFunc, SMqDataRsp* rspObj, tmq_raw_data* raw) { int32_t len = 0; int32_t code = 0; SEncoder encoder = {0}; @@ -2155,15 +2157,14 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { if (!raw || !res) { return TSDB_CODE_INVALID_PARA; } + SMqRspObj* rspObj = ((SMqRspObj*)res); if (TD_RES_TMQ_META(res)) { - SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; - raw->raw = pMetaRspObj->metaRsp.metaRsp; - raw->raw_len = pMetaRspObj->metaRsp.metaRspLen; - raw->raw_type = pMetaRspObj->metaRsp.resMsgType; + raw->raw = rspObj->metaRsp.metaRsp; + raw->raw_len = rspObj->metaRsp.metaRspLen; + raw->raw_type = rspObj->metaRsp.resMsgType; uDebug("tmq get raw type meta:%p", raw); } else if (TD_RES_TMQ(res)) { - SMqRspObj* rspObj = ((SMqRspObj*)res); - int32_t code = encodeMqDataRsp(tEncodeMqDataRsp, &rspObj->rsp, raw); + int32_t code = encodeMqDataRsp(tEncodeMqDataRsp, &rspObj->dataRsp, raw); if (code != 0) { uError("tmq get raw type error:%d", terrno); return code; @@ -2171,9 +2172,7 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { raw->raw_type = RES_TYPE__TMQ; uDebug("tmq get raw type data:%p", raw); } else if (TD_RES_TMQ_METADATA(res)) { - SMqTaosxRspObj* rspObj = ((SMqTaosxRspObj*)res); - - int32_t code = encodeMqDataRsp(tEncodeSTaosxRsp, &rspObj->rsp, raw); + int32_t code = encodeMqDataRsp(tEncodeSTaosxRsp, &rspObj->dataRsp, raw); if (code != 0) { uError("tmq get raw type error:%d", terrno); return code; @@ -2181,10 +2180,9 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { raw->raw_type = RES_TYPE__TMQ_METADATA; uDebug("tmq get raw type metadata:%p", raw); } else if (TD_RES_TMQ_BATCH_META(res)) { - SMqBatchMetaRspObj* pBtMetaRspObj = (SMqBatchMetaRspObj*)res; - raw->raw = pBtMetaRspObj->rsp.pMetaBuff; - raw->raw_len = pBtMetaRspObj->rsp.metaBuffLen; - raw->raw_type = RES_TYPE__TMQ_BATCH_META; + raw->raw = rspObj->batchMetaRsp.pMetaBuff; + raw->raw_len = rspObj->batchMetaRsp.metaBuffLen; + raw->raw_type = rspObj->resType; uDebug("tmq get raw batch meta:%p", raw); } else { uError("tmq get raw error type:%d", *(int8_t*)res); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index f3a22bff75..d5cca55701 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -137,7 +137,7 @@ void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2 } (void)memset(pBuf->buf, 0, pBuf->len); if (msg1) { - (void)strncat(pBuf->buf, msg1, pBuf->len); + (void)strncat(pBuf->buf, msg1, pBuf->len - 1); } int32_t left = pBuf->len - strlen(pBuf->buf); if (left > 2 && msg2) { @@ -393,7 +393,7 @@ int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements) { tinfo->tags = taosArrayDup(info->preLineTagKV, NULL); if (tinfo->tags == NULL) { smlDestroyTableInfo(&tinfo); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } for (size_t i = 0; i < taosArrayGetSize(info->preLineTagKV); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(info->preLineTagKV, i); @@ -515,9 +515,9 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam if (tag == NULL) { return TSDB_CODE_SML_INVALID_DATA; } - (void)strncat(childTableName, tag->value, tag->length); + (void)strncat(childTableName, tag->value, TMIN(tag->length, TSDB_TABLE_NAME_LEN - 1 - strlen(childTableName))); if (i != taosArrayGetSize(tags) - 1) { - (void)strcat(childTableName, tsSmlAutoChildTableNameDelimiter); + (void)strncat(childTableName, tsSmlAutoChildTableNameDelimiter, TSDB_TABLE_NAME_LEN - 1 - strlen(childTableName)); } } if (tsSmlDot2Underline) { @@ -538,8 +538,7 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam // handle child table name if (childTableNameLen == tag->keyLen && strncmp(tag->key, tbnameKey, tag->keyLen) == 0) { (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - (void)strncpy(childTableName, tag->value, - (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); + tstrncpy(childTableName, tag->value, TMIN(TSDB_TABLE_NAME_LEN, tag->length + 1)); if (tsSmlDot2Underline) { smlStrReplace(childTableName, strlen(childTableName)); } @@ -561,7 +560,7 @@ int32_t smlSetCTableName(SSmlTableInfo *oneTable, char *tbnameKey) { if (strlen(oneTable->childTableName) == 0) { SArray *dst = taosArrayDup(oneTable->tags, NULL); if (dst == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (oneTable->sTableNameLen >= TSDB_TABLE_NAME_LEN) { uError("SML:smlSetCTableName super table name is too long"); @@ -957,7 +956,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool for (; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); if (kv == NULL) { - code = TSDB_CODE_SML_INVALID_DATA; + code = terrno; goto END; } if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { @@ -1053,7 +1052,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, for (int32_t i = 0; i < pReq.numOfColumns; ++i) { SField *pField = taosArrayGet(pColumns, i); if (pField == NULL) { - code = TSDB_CODE_SML_INVALID_DATA; + code = terrno; goto end; } SFieldWithOptions fieldWithOption = {0}; diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 866d0cc272..e56d4cc4f6 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -236,7 +236,7 @@ int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, } (void)memcpy(&pStmt->bInfo.sname, tbName, sizeof(*tbName)); - (void)strncpy(pStmt->bInfo.tbFName, tbFName, sizeof(pStmt->bInfo.tbFName) - 1); + tstrncpy(pStmt->bInfo.tbFName, tbFName, TSDB_TABLE_FNAME_LEN); pStmt->bInfo.tbFName[sizeof(pStmt->bInfo.tbFName) - 1] = 0; pStmt->bInfo.tbUid = autoCreateTbl ? 0 : pTableMeta->uid; @@ -983,7 +983,7 @@ int stmtSetDbName(TAOS_STMT* stmt, const char* dbName) { taosMemoryFreeClear(pStmt->exec.pRequest->pDb); pStmt->exec.pRequest->pDb = taosStrdup(dbName); if (pStmt->exec.pRequest->pDb == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } return TSDB_CODE_SUCCESS; } @@ -1018,13 +1018,13 @@ int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) { STMT_ERR_RET(stmtGetFromCache(pStmt)); if (pStmt->bInfo.needParse) { - (void)strncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName) - 1); + tstrncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName)); pStmt->bInfo.tbName[sizeof(pStmt->bInfo.tbName) - 1] = 0; STMT_ERR_RET(stmtParseSql(pStmt)); } } else { - (void)strncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName) - 1); + tstrncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName)); pStmt->bInfo.tbName[sizeof(pStmt->bInfo.tbName) - 1] = 0; pStmt->exec.pRequest->requestId++; pStmt->bInfo.needParse = false; @@ -1172,7 +1172,7 @@ int32_t stmtAppendTablePostHandle(STscStmt* pStmt, SStmtQNode* param) { } if (0 == pStmt->sql.siInfo.firstName[0]) { - (void)strcpy(pStmt->sql.siInfo.firstName, pStmt->bInfo.tbName); + tstrncpy(pStmt->sql.siInfo.firstName, pStmt->bInfo.tbName, TSDB_TABLE_NAME_LEN); } param->tblData.getFromHash = pStmt->sql.siInfo.tbFromHash; @@ -1313,7 +1313,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { // param->tblData.aCol = taosArrayInit(20, POINTER_BYTES); param->restoreTbCols = false; - (void)strcpy(param->tblData.tbName, pStmt->bInfo.tbName); + tstrncpy(param->tblData.tbName, pStmt->bInfo.tbName, TSDB_TABLE_NAME_LEN); } int64_t startUs3 = taosGetTimestampUs(); diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index a0fd49ac86..b78e0d0f56 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -187,7 +187,7 @@ static int32_t stmtUpdateBindInfo(TAOS_STMT2* stmt, STableMeta* pTableMeta, void } (void)memcpy(&pStmt->bInfo.sname, tbName, sizeof(*tbName)); - (void)strncpy(pStmt->bInfo.tbFName, tbFName, sizeof(pStmt->bInfo.tbFName) - 1); + tstrncpy(pStmt->bInfo.tbFName, tbFName, sizeof(pStmt->bInfo.tbFName)); pStmt->bInfo.tbFName[sizeof(pStmt->bInfo.tbFName) - 1] = 0; pStmt->bInfo.tbUid = autoCreateTbl ? 0 : pTableMeta->uid; @@ -850,7 +850,7 @@ static int stmtSetDbName2(TAOS_STMT2* stmt, const char* dbName) { taosMemoryFreeClear(pStmt->exec.pRequest->pDb); pStmt->exec.pRequest->pDb = taosStrdup(dbName); if (pStmt->exec.pRequest->pDb == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } return TSDB_CODE_SUCCESS; } @@ -961,13 +961,13 @@ int stmtSetTbName2(TAOS_STMT2* stmt, const char* tbName) { STMT_ERR_RET(stmtGetFromCache(pStmt)); if (pStmt->bInfo.needParse) { - (void)strncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName) - 1); + tstrncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName)); pStmt->bInfo.tbName[sizeof(pStmt->bInfo.tbName) - 1] = 0; STMT_ERR_RET(stmtParseSql(pStmt)); } } else { - (void)strncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName) - 1); + tstrncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName)); pStmt->bInfo.tbName[sizeof(pStmt->bInfo.tbName) - 1] = 0; pStmt->exec.pRequest->requestId++; pStmt->bInfo.needParse = false; @@ -1113,7 +1113,7 @@ static int32_t stmtAppendTablePostHandle(STscStmt2* pStmt, SStmtQNode* param) { } if (0 == pStmt->sql.siInfo.firstName[0]) { - (void)strcpy(pStmt->sql.siInfo.firstName, pStmt->bInfo.tbName); + tstrncpy(pStmt->sql.siInfo.firstName, pStmt->bInfo.tbName, TSDB_TABLE_NAME_LEN); } param->tblData.getFromHash = pStmt->sql.siInfo.tbFromHash; @@ -1200,22 +1200,6 @@ static int stmtAddBatch2(TAOS_STMT2* stmt) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_ADD_BATCH)); - if (pStmt->sql.stbInterlaceMode) { - int64_t startUs2 = taosGetTimestampUs(); - pStmt->stat.addBatchUs += startUs2 - startUs; - - pStmt->sql.siInfo.tableColsReady = false; - - SStmtQNode* param = NULL; - STMT_ERR_RET(stmtAllocQNodeFromBuf(&pStmt->sql.siInfo.tbBuf, (void**)¶m)); - param->restoreTbCols = true; - param->next = NULL; - - stmtEnqueue(pStmt, param); - - return TSDB_CODE_SUCCESS; - } - STMT_ERR_RET(stmtCacheBlock(pStmt)); return TSDB_CODE_SUCCESS; @@ -1367,7 +1351,7 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) { // param->tblData.aCol = taosArrayInit(20, POINTER_BYTES); param->restoreTbCols = false; - (void)strcpy(param->tblData.tbName, pStmt->bInfo.tbName); + tstrncpy(param->tblData.tbName, pStmt->bInfo.tbName, TSDB_TABLE_NAME_LEN); } int64_t startUs3 = taosGetTimestampUs(); @@ -1627,6 +1611,22 @@ int stmtExec2(TAOS_STMT2* stmt, int* affected_rows) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE)); + if (pStmt->sql.stbInterlaceMode) { + int64_t startUs2 = taosGetTimestampUs(); + pStmt->stat.addBatchUs += startUs2 - startUs; + + pStmt->sql.siInfo.tableColsReady = false; + + SStmtQNode* param = NULL; + STMT_ERR_RET(stmtAllocQNodeFromBuf(&pStmt->sql.siInfo.tbBuf, (void**)¶m)); + param->restoreTbCols = true; + param->next = NULL; + + stmtEnqueue(pStmt, param); + + return TSDB_CODE_SUCCESS; + } + if (STMT_TYPE_QUERY != pStmt->sql.type) { if (pStmt->sql.stbInterlaceMode) { int64_t startTs = taosGetTimestampUs(); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 2c92824855..fd6ca831d1 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -24,33 +24,61 @@ #include "tref.h" #include "ttimer.h" +#define tqFatalC(...) do { if (cDebugFlag & DEBUG_FATAL || tqClientDebug) { taosPrintLog("TQ FATAL ", DEBUG_FATAL, tqDebugFlag, __VA_ARGS__); }} while(0) +#define tqErrorC(...) do { if (cDebugFlag & DEBUG_ERROR || tqClientDebug) { taosPrintLog("TQ ERROR ", DEBUG_ERROR, tqDebugFlag, __VA_ARGS__); }} while(0) +#define tqWarnC(...) do { if (cDebugFlag & DEBUG_WARN || tqClientDebug) { taosPrintLog("TQ WARN ", DEBUG_WARN, tqDebugFlag, __VA_ARGS__); }} while(0) +#define tqInfoC(...) do { if (cDebugFlag & DEBUG_INFO || tqClientDebug) { taosPrintLog("TQ ", DEBUG_INFO, tqDebugFlag, __VA_ARGS__); }} while(0) +#define tqDebugC(...) do { if (cDebugFlag & DEBUG_DEBUG || tqClientDebug) { taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); }} while(0) +#define tqTraceC(...) do { if (cDebugFlag & DEBUG_TRACE || tqClientDebug) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0) + #define EMPTY_BLOCK_POLL_IDLE_DURATION 10 #define DEFAULT_AUTO_COMMIT_INTERVAL 5000 #define DEFAULT_HEARTBEAT_INTERVAL 3000 #define DEFAULT_ASKEP_INTERVAL 1000 +#define DEFAULT_COMMIT_CNT 1 +#define SUBSCRIBE_RETRY_MAX_COUNT 240 +#define SUBSCRIBE_RETRY_INTERVAL 500 -struct SMqMgmt { - tmr_h timer; - int32_t rsetId; + +#define SET_ERROR_MSG_TMQ(MSG) \ + if (errstr != NULL && errstrLen > 0) (void)snprintf(errstr, errstrLen, MSG); + +#define PROCESS_POLL_RSP(FUNC,DATA) \ + SDecoder decoder = {0}; \ + tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); \ + if (FUNC(&decoder, DATA) < 0) { \ + tDecoderClear(&decoder); \ + code = TSDB_CODE_OUT_OF_MEMORY; \ + goto END;\ + }\ + tDecoderClear(&decoder);\ + (void)memcpy(DATA, pMsg->pData, sizeof(SMqRspHead)); + +#define DELETE_POLL_RSP(FUNC,DATA) \ + SMqPollRspWrapper* pRsp = &rspWrapper->pollRsp;\ + taosMemoryFreeClear(pRsp->pEpset);\ + FUNC(DATA); + +enum { + TMQ_VG_STATUS__IDLE = 0, + TMQ_VG_STATUS__WAIT, }; -static TdThreadOnce tmqInit = PTHREAD_ONCE_INIT; // initialize only once -volatile int32_t tmqInitRes = 0; // initialize rsp code -static struct SMqMgmt tmqMgmt = {0}; -static int8_t pollFlag = 0; +enum { + TMQ_CONSUMER_STATUS__INIT = 0, + TMQ_CONSUMER_STATUS__READY, + TMQ_CONSUMER_STATUS__CLOSED, +}; + +enum { + TMQ_DELAYED_TASK__ASK_EP = 1, + TMQ_DELAYED_TASK__COMMIT, +}; typedef struct { - int32_t code; - int8_t tmqRspType; - int32_t epoch; -} SMqRspWrapper; - -typedef struct { - int32_t code; - int8_t tmqRspType; - int32_t epoch; - SMqAskEpRsp msg; -} SMqAskEpRspWrapper; + tmr_h timer; + int32_t rsetId; +} SMqMgmt; struct tmq_list_t { SArray container; @@ -106,6 +134,7 @@ struct tmq_t { // poll info int64_t pollCnt; int64_t totalRows; + int8_t pollFlag; // timer tmr_h hbLiveTimer; @@ -119,30 +148,12 @@ struct tmq_t { tsem2_t rspSem; }; -typedef struct SAskEpInfo { +typedef struct { int32_t code; tsem2_t sem; } SAskEpInfo; -enum { - TMQ_VG_STATUS__IDLE = 0, - TMQ_VG_STATUS__WAIT, -}; - -enum { - TMQ_CONSUMER_STATUS__INIT = 0, - TMQ_CONSUMER_STATUS__READY, - TMQ_CONSUMER_STATUS__NO_TOPIC, - TMQ_CONSUMER_STATUS__RECOVER, - TMQ_CONSUMER_STATUS__CLOSED, -}; - -enum { - TMQ_DELAYED_TASK__ASK_EP = 1, - TMQ_DELAYED_TASK__COMMIT, -}; - -typedef struct SVgOffsetInfo { +typedef struct { STqOffsetVal committedOffset; STqOffsetVal endOffset; // the last version in TAOS_RES + 1 STqOffsetVal beginOffset; // the first version in TAOS_RES @@ -173,23 +184,32 @@ typedef struct { } SMqClientTopic; typedef struct { - int32_t code; - int8_t tmqRspType; - int32_t epoch; // epoch can be used to guard the vgHandle int32_t vgId; char topicName[TSDB_TOPIC_FNAME_LEN]; - SMqClientVg* vgHandle; SMqClientTopic* topicHandle; uint64_t reqId; SEpSet* pEpset; union { + struct{ + SMqRspHead head; + STqOffsetVal rspOffset; + }; SMqDataRsp dataRsp; SMqMetaRsp metaRsp; - STaosxRsp taosxRsp; SMqBatchMetaRsp batchMetaRsp; }; } SMqPollRspWrapper; +typedef struct { + int32_t code; + int8_t tmqRspType; + int32_t epoch; + union{ + SMqPollRspWrapper pollRsp; + SMqAskEpRsp epRsp; + }; +} SMqRspWrapper; + typedef struct { tsem2_t rspSem; int32_t rspErr; @@ -208,7 +228,7 @@ typedef struct { uint64_t requestId; // request id for debug purpose } SMqPollCbParam; -typedef struct SMqVgCommon { +typedef struct { tsem2_t rsp; int32_t numOfRsp; SArray* pList; @@ -218,18 +238,18 @@ typedef struct SMqVgCommon { int32_t code; } SMqVgCommon; -typedef struct SMqSeekParam { +typedef struct { tsem2_t sem; int32_t code; } SMqSeekParam; -typedef struct SMqCommittedParam { +typedef struct { tsem2_t sem; int32_t code; SMqVgOffset vgOffset; } SMqCommittedParam; -typedef struct SMqVgWalInfoParam { +typedef struct { int32_t vgId; int32_t epoch; int32_t totalReq; @@ -252,17 +272,22 @@ typedef struct { int64_t consumerId; } SMqCommitCbParam; -typedef struct SSyncCommitInfo { +typedef struct { tsem2_t sem; int32_t code; } SSyncCommitInfo; -static int32_t syncAskEp(tmq_t* tmq); -static int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet); -static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffsetVal* offset, const char* pTopicName, - SMqCommitCbParamSet* pParamSet); -static int32_t commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId); -static int32_t askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpset); +typedef struct { + STqOffsetVal currentOffset; + STqOffsetVal commitOffset; + STqOffsetVal seekOffset; + int64_t numOfRows; + int32_t vgStatus; +} SVgroupSaveInfo; + +static TdThreadOnce tmqInit = PTHREAD_ONCE_INIT; // initialize only once +volatile int32_t tmqInitRes = 0; // initialize rsp code +static SMqMgmt tmqMgmt = {0}; tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t)); @@ -300,7 +325,7 @@ void tmq_conf_destroy(tmq_conf_t* conf) { tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { int32_t code = 0; if (conf == NULL || key == NULL || value == NULL) { - tscError("tmq_conf_set null, conf:%p key:%p value:%p", conf, key, value); + tqErrorC("tmq_conf_set null, conf:%p key:%p value:%p", conf, key, value); return TMQ_CONF_INVALID; } if (strcasecmp(key, "group.id") == 0) { @@ -321,7 +346,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value conf->autoCommit = false; return TMQ_CONF_OK; } else { - tscError("invalid value for enable.auto.commit: %s", value); + tqErrorC("invalid value for enable.auto.commit: %s", value); return TMQ_CONF_INVALID; } } @@ -330,7 +355,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value int64_t tmp; code = taosStr2int64(value, &tmp); if (tmp < 0 || code != 0) { - tscError("invalid value for auto.commit.interval.ms: %s", value); + tqErrorC("invalid value for auto.commit.interval.ms: %s", value); return TMQ_CONF_INVALID; } conf->autoCommitInterval = (tmp > INT32_MAX ? INT32_MAX : tmp); @@ -341,7 +366,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value int64_t tmp; code = taosStr2int64(value, &tmp); if (tmp < 6000 || tmp > 1800000 || code != 0) { - tscError("invalid value for session.timeout.ms: %s", value); + tqErrorC("invalid value for session.timeout.ms: %s", value); return TMQ_CONF_INVALID; } conf->sessionTimeoutMs = tmp; @@ -352,7 +377,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value int64_t tmp; code = taosStr2int64(value, &tmp); if (tmp < 1000 || tmp >= conf->sessionTimeoutMs || code != 0) { - tscError("invalid value for heartbeat.interval.ms: %s", value); + tqErrorC("invalid value for heartbeat.interval.ms: %s", value); return TMQ_CONF_INVALID; } conf->heartBeatIntervalMs = tmp; @@ -363,7 +388,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value int32_t tmp; code = taosStr2int32(value, &tmp); if (tmp < 1000 || code != 0) { - tscError("invalid value for max.poll.interval.ms: %s", value); + tqErrorC("invalid value for max.poll.interval.ms: %s", value); return TMQ_CONF_INVALID; } conf->maxPollIntervalMs = tmp; @@ -381,7 +406,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value conf->resetOffset = TMQ_OFFSET__RESET_LATEST; return TMQ_CONF_OK; } else { - tscError("invalid value for auto.offset.reset: %s", value); + tqErrorC("invalid value for auto.offset.reset: %s", value); return TMQ_CONF_INVALID; } } @@ -394,7 +419,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value conf->withTbName = false; return TMQ_CONF_OK; } else { - tscError("invalid value for msg.with.table.name: %s", value); + tqErrorC("invalid value for msg.with.table.name: %s", value); return TMQ_CONF_INVALID; } } @@ -407,7 +432,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value conf->snapEnable = false; return TMQ_CONF_OK; } else { - tscError("invalid value for experimental.snapshot.enable: %s", value); + tqErrorC("invalid value for experimental.snapshot.enable: %s", value); return TMQ_CONF_INVALID; } } @@ -415,7 +440,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value if (strcasecmp(key, "td.connect.ip") == 0) { void *tmp = taosStrdup(value); if (tmp == NULL) { - tscError("tmq_conf_set out of memory:%d", terrno); + tqErrorC("tmq_conf_set out of memory:%d", terrno); return TMQ_CONF_INVALID; } conf->ip = tmp; @@ -425,7 +450,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value if (strcasecmp(key, "td.connect.user") == 0) { void *tmp = taosStrdup(value); if (tmp == NULL) { - tscError("tmq_conf_set out of memory:%d", terrno); + tqErrorC("tmq_conf_set out of memory:%d", terrno); return TMQ_CONF_INVALID; } conf->user = tmp; @@ -435,7 +460,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value if (strcasecmp(key, "td.connect.pass") == 0) { void *tmp = taosStrdup(value); if (tmp == NULL) { - tscError("tmq_conf_set out of memory:%d", terrno); + tqErrorC("tmq_conf_set out of memory:%d", terrno); return TMQ_CONF_INVALID; } conf->pass = tmp; @@ -446,7 +471,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value int64_t tmp; code = taosStr2int64(value, &tmp); if (tmp <= 0 || tmp > 65535 || code != 0) { - tscError("invalid value for td.connect.port: %s", value); + tqErrorC("invalid value for td.connect.port: %s", value); return TMQ_CONF_INVALID; } @@ -462,7 +487,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value conf->replayEnable = false; return TMQ_CONF_OK; } else { - tscError("invalid value for enable.replay: %s", value); + tqErrorC("invalid value for enable.replay: %s", value); return TMQ_CONF_INVALID; } } @@ -484,16 +509,22 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value return TMQ_CONF_OK; } - tscError("unknown key: %s", key); + tqErrorC("unknown key: %s", key); return TMQ_CONF_UNKNOWN; } -tmq_list_t* tmq_list_new() { return (tmq_list_t*)taosArrayInit(0, sizeof(void*)); } +tmq_list_t* tmq_list_new() { + return (tmq_list_t*)taosArrayInit(0, sizeof(void*)); +} int32_t tmq_list_append(tmq_list_t* list, const char* src) { - if (list == NULL) return TSDB_CODE_INVALID_PARA; + if (list == NULL) { + return TSDB_CODE_INVALID_PARA; + } SArray* container = &list->container; - if (src == NULL || src[0] == 0) return TSDB_CODE_INVALID_PARA; + if (src == NULL || src[0] == 0) { + return TSDB_CODE_INVALID_PARA; + } char* topic = taosStrdup(src); if (topic == NULL) return terrno; if (taosArrayPush(container, &topic) == NULL) { @@ -510,24 +541,66 @@ void tmq_list_destroy(tmq_list_t* list) { } int32_t tmq_list_get_size(const tmq_list_t* list) { - if (list == NULL) return -1; + if (list == NULL) { + return TSDB_CODE_INVALID_PARA; + } const SArray* container = &list->container; return taosArrayGetSize(container); } char** tmq_list_to_c_array(const tmq_list_t* list) { - if (list == NULL) return NULL; + if (list == NULL) { + return NULL; + } const SArray* container = &list->container; return container->pData; } +static int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { + int64_t refId = pParamSet->refId; + int32_t code = 0; + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); + if (tmq == NULL) { + code = TSDB_CODE_TMQ_CONSUMER_CLOSED; + } + + // if no more waiting rsp + if (pParamSet->callbackFn != NULL) { + pParamSet->callbackFn(tmq, pParamSet->code, pParamSet->userParam); + } + + taosMemoryFree(pParamSet); + if (tmq != NULL) { + code = taosReleaseRef(tmqMgmt.rsetId, refId); + } + + return code; +} + +static int32_t commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId) { + int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); + if (waitingRspNum == 0) { + tqDebugC("consumer:0x%" PRIx64 " topic:%s vgId:%d all commit-rsp received, commit completed", consumerId, pTopic, + vgId); + return tmqCommitDone(pParamSet); + } else { + tqDebugC("consumer:0x%" PRIx64 " topic:%s vgId:%d commit-rsp received, remain:%d", consumerId, pTopic, vgId, + waitingRspNum); + } + return 0; +} + static int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { + if (pBuf){ + taosMemoryFreeClear(pBuf->pData); + taosMemoryFreeClear(pBuf->pEpSet); + } + if(param == NULL){ + return TSDB_CODE_INVALID_PARA; + } SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params; - taosMemoryFree(pBuf->pData); - taosMemoryFree(pBuf->pEpSet); - return commitRspCountDown(pParamSet, pParam->consumerId, pParam->topicName, pParam->vgId); } @@ -598,7 +671,6 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, epSet, NULL, pMsgSendInfo); if (code != 0) { (void)atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); - return code; } return code; } @@ -614,7 +686,7 @@ static int32_t getTopicByName(tmq_t* tmq, const char* pTopicName, SMqClientTopic return 0; } - tscError("consumer:0x%" PRIx64 ", total:%d, failed to find topic:%s", tmq->consumerId, numOfTopics, pTopicName); + tqErrorC("consumer:0x%" PRIx64 ", total:%d, failed to find topic:%s", tmq->consumerId, numOfTopics, pTopicName); return TSDB_CODE_TMQ_INVALID_TOPIC; } @@ -638,7 +710,7 @@ static int32_t getClientVg(tmq_t* tmq, char* pTopicName, int32_t vgId, SMqClient SMqClientTopic* pTopic = NULL; int32_t code = getTopicByName(tmq, pTopicName, &pTopic); if (code != 0) { - tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); + tqErrorC("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); return code; } @@ -654,22 +726,15 @@ static int32_t getClientVg(tmq_t* tmq, char* pTopicName, int32_t vgId, SMqClient return *pVg == NULL ? TSDB_CODE_TMQ_INVALID_VGID : TSDB_CODE_SUCCESS; } -static int32_t asyncCommitOffset(tmq_t* tmq, char* pTopicName, int32_t vgId, STqOffsetVal* offsetVal, - tmq_commit_cb* pCommitFp, void* userParam) { - tscInfo("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId); - taosRLockLatch(&tmq->lock); - SMqClientVg* pVg = NULL; - int32_t code = getClientVg(tmq, pTopicName, vgId, &pVg); - if (code != 0) { - goto end; - } +static int32_t innerCommit(tmq_t* tmq, char* pTopicName, STqOffsetVal* offsetVal, SMqClientVg* pVg, SMqCommitCbParamSet* pParamSet){ + int32_t code = 0; if (offsetVal->type <= 0) { code = TSDB_CODE_TMQ_INVALID_MSG; - goto end; + return code; } if (tOffsetEqual(offsetVal, &pVg->offsetInfo.committedOffset)) { code = TSDB_CODE_TMQ_SAME_COMMITTED_VALUE; - goto end; + return code; } char offsetBuf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(offsetBuf, tListLen(offsetBuf), offsetVal); @@ -677,26 +742,39 @@ static int32_t asyncCommitOffset(tmq_t* tmq, char* pTopicName, int32_t vgId, STq char commitBuf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->offsetInfo.committedOffset); - SMqCommitCbParamSet* pParamSet = NULL; - code = prepareCommitCbParamSet(tmq, pCommitFp, userParam, 0, &pParamSet); - if (code != 0) { - goto end; - } - code = doSendCommitMsg(tmq, pVg->vgId, &pVg->epSet, offsetVal, pTopicName, pParamSet); if (code != TSDB_CODE_SUCCESS) { - tscError("consumer:0x%" PRIx64 " topic:%s on vgId:%d end commit msg failed, send offset:%s committed:%s, code:%s", + tqErrorC("consumer:0x%" PRIx64 " topic:%s on vgId:%d end commit msg failed, send offset:%s committed:%s, code:%s", tmq->consumerId, pTopicName, pVg->vgId, offsetBuf, commitBuf, tstrerror(terrno)); - taosMemoryFree(pParamSet); - goto end; + return code; } - tscInfo("consumer:0x%" PRIx64 " topic:%s on vgId:%d send commit msg success, send offset:%s committed:%s", + tqDebugC("consumer:0x%" PRIx64 " topic:%s on vgId:%d send commit msg success, send offset:%s committed:%s", tmq->consumerId, pTopicName, pVg->vgId, offsetBuf, commitBuf); tOffsetCopy(&pVg->offsetInfo.committedOffset, offsetVal); + return code; +} -end: +static int32_t asyncCommitOffset(tmq_t* tmq, char* pTopicName, int32_t vgId, STqOffsetVal* offsetVal, + tmq_commit_cb* pCommitFp, void* userParam) { + tqInfoC("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId); + SMqCommitCbParamSet* pParamSet = NULL; + int32_t code = prepareCommitCbParamSet(tmq, pCommitFp, userParam, 0, &pParamSet); + if (code != 0){ + return code; + } + + taosRLockLatch(&tmq->lock); + SMqClientVg* pVg = NULL; + code = getClientVg(tmq, pTopicName, vgId, &pVg); + if (code == 0) { + code = innerCommit(tmq, pTopicName, offsetVal, pVg, pParamSet); + } taosRUnLockLatch(&tmq->lock); + + if (code != 0){ + taosMemoryFree(pParamSet); + } return code; } @@ -711,26 +789,12 @@ static void asyncCommitFromResult(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_c goto end; } - if (TD_RES_TMQ(pRes)) { + if (TD_RES_TMQ(pRes) || TD_RES_TMQ_META(pRes) || + TD_RES_TMQ_METADATA(pRes) || TD_RES_TMQ_BATCH_META(pRes)) { SMqRspObj* pRspObj = (SMqRspObj*)pRes; - pTopicName = pRspObj->common.topic; - vgId = pRspObj->common.vgId; - offsetVal = pRspObj->rsp.common.rspOffset; - } else if (TD_RES_TMQ_META(pRes)) { - SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)pRes; - pTopicName = pMetaRspObj->topic; - vgId = pMetaRspObj->vgId; - offsetVal = pMetaRspObj->metaRsp.rspOffset; - } else if (TD_RES_TMQ_METADATA(pRes)) { - SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)pRes; - pTopicName = pRspObj->common.topic; - vgId = pRspObj->common.vgId; - offsetVal = pRspObj->rsp.common.rspOffset; - } else if (TD_RES_TMQ_BATCH_META(pRes)) { - SMqBatchMetaRspObj* pBtRspObj = (SMqBatchMetaRspObj*)pRes; - pTopicName = pBtRspObj->common.topic; - vgId = pBtRspObj->common.vgId; - offsetVal = pBtRspObj->rsp.rspOffset; + pTopicName = pRspObj->topic; + vgId = pRspObj->vgId; + offsetVal = pRspObj->rspOffset; } else { code = TSDB_CODE_TMQ_INVALID_MSG; goto end; @@ -745,83 +809,61 @@ end: } } -static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam) { +static int32_t innerCommitAll(tmq_t* tmq, SMqCommitCbParamSet* pParamSet){ int32_t code = 0; - // init as 1 to prevent concurrency issue - SMqCommitCbParamSet* pParamSet = NULL; - code = prepareCommitCbParamSet(tmq, pCommitFp, userParam, 1, &pParamSet); - if (code != 0) { - goto end; - } taosRLockLatch(&tmq->lock); int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); - tscDebug("consumer:0x%" PRIx64 " start to commit offset for %d topics", tmq->consumerId, numOfTopics); + tqDebugC("consumer:0x%" PRIx64 " start to commit offset for %d topics", tmq->consumerId, numOfTopics); for (int32_t i = 0; i < numOfTopics; i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); if (pTopic == NULL) { code = TSDB_CODE_TMQ_INVALID_TOPIC; - taosRUnLockLatch(&tmq->lock); - goto end; + goto END; } int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); - tscDebug("consumer:0x%" PRIx64 " commit offset for topics:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName, - numOfVgroups); + tqDebugC("consumer:0x%" PRIx64 " commit offset for topics:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName, numOfVgroups); for (int32_t j = 0; j < numOfVgroups; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); if (pVg == NULL) { - code = TSDB_CODE_INVALID_PARA; - taosRUnLockLatch(&tmq->lock); - goto end; + code = terrno; + goto END; } - if (pVg->offsetInfo.endOffset.type > 0 && - !tOffsetEqual(&pVg->offsetInfo.endOffset, &pVg->offsetInfo.committedOffset)) { - char offsetBuf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(offsetBuf, tListLen(offsetBuf), &pVg->offsetInfo.endOffset); - char commitBuf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->offsetInfo.committedOffset); - - code = doSendCommitMsg(tmq, pVg->vgId, &pVg->epSet, &pVg->offsetInfo.endOffset, pTopic->topicName, pParamSet); - if (code != TSDB_CODE_SUCCESS) { - tscError("consumer:0x%" PRIx64 - " topic:%s on vgId:%d end commit msg failed, send offset:%s committed:%s, code:%s ordinal:%d/%d", - tmq->consumerId, pTopic->topicName, pVg->vgId, offsetBuf, commitBuf, tstrerror(terrno), j + 1, - numOfVgroups); - continue; - } - - tscDebug("consumer:0x%" PRIx64 - " topic:%s on vgId:%d send commit msg success, send offset:%s committed:%s, ordinal:%d/%d", - tmq->consumerId, pTopic->topicName, pVg->vgId, offsetBuf, commitBuf, j + 1, numOfVgroups); - tOffsetCopy(&pVg->offsetInfo.committedOffset, &pVg->offsetInfo.endOffset); - } else { - tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, current:%" PRId64 ", ordinal:%d/%d", - tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->offsetInfo.endOffset.version, j + 1, numOfVgroups); + code = innerCommit(tmq, pTopic->topicName, &pVg->offsetInfo.endOffset, pVg, pParamSet); + if (code != 0){ + tqDebugC("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, code:%s, current offset version:%" PRId64 ", ordinal:%d/%d", + tmq->consumerId, pTopic->topicName, pVg->vgId, tstrerror(code), pVg->offsetInfo.endOffset.version, j + 1, numOfVgroups); } } } - taosRUnLockLatch(&tmq->lock); - - tscDebug("consumer:0x%" PRIx64 " total commit:%d for %d topics", tmq->consumerId, pParamSet->waitingRspNum - 1, + tqDebugC("consumer:0x%" PRIx64 " total commit:%d for %d topics", tmq->consumerId, pParamSet->waitingRspNum - DEFAULT_COMMIT_CNT, numOfTopics); +END: + taosRUnLockLatch(&tmq->lock); + return code; +} - // request is sent - if (pParamSet->waitingRspNum != 1) { - // count down since waiting rsp num init as 1 - code = commitRspCountDown(pParamSet, tmq->consumerId, "", 0); - if (code != 0) { - tscError("consumer:0x%" PRIx64 " commit rsp count down failed, code:%s", tmq->consumerId, tstrerror(code)); - pParamSet = NULL; - goto end; +static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam) { + int32_t code = 0; + SMqCommitCbParamSet* pParamSet = NULL; + // init waitingRspNum as DEFAULT_COMMIT_CNT to prevent concurrency issue + code = prepareCommitCbParamSet(tmq, pCommitFp, userParam, DEFAULT_COMMIT_CNT, &pParamSet); + if (code != 0) { + tqErrorC("consumer:0x%" PRIx64 " prepareCommitCbParamSet failed, code:%s", tmq->consumerId, tstrerror(code)); + if (pCommitFp != NULL) { + pCommitFp(tmq, code, userParam); } return; } + code = innerCommitAll(tmq, pParamSet); + if (code != 0){ + tqErrorC("consumer:0x%" PRIx64 " innerCommitAll failed, code:%s", tmq->consumerId, tstrerror(code)); + } -end: - taosMemoryFree(pParamSet); - if (pCommitFp != NULL) { - pCommitFp(tmq, code, userParam); + code = commitRspCountDown(pParamSet, tmq->consumerId, "init", -1); + if (code != 0) { + tqErrorC("consumer:0x%" PRIx64 " commit rsp count down failed, code:%s", tmq->consumerId, tstrerror(code)); } return; } @@ -839,7 +881,7 @@ static void generateTimedTask(int64_t refId, int32_t type) { *pTaskType = type; if (taosWriteQitem(tmq->delayedTask, pTaskType) == 0) { if (tsem2_post(&tmq->rspSem) != 0){ - tscError("consumer:0x%" PRIx64 " failed to post sem, type:%d", tmq->consumerId, type); + tqErrorC("consumer:0x%" PRIx64 " failed to post sem, type:%d", tmq->consumerId, type); } }else{ taosFreeQitem(pTaskType); @@ -848,7 +890,7 @@ static void generateTimedTask(int64_t refId, int32_t type) { code = taosReleaseRef(tmqMgmt.rsetId, refId); if (code != 0){ - tscError("failed to release ref:%"PRId64 ", type:%d, code:%d", refId, type, code); + tqErrorC("failed to release ref:%"PRId64 ", type:%d, code:%d", refId, type, code); } } @@ -863,11 +905,11 @@ void tmqReplayTask(void* param, void* tmrId) { if (tmq == NULL) return; if (tsem2_post(&tmq->rspSem) != 0){ - tscError("consumer:0x%" PRIx64 " failed to post sem, replay", tmq->consumerId); + tqErrorC("consumer:0x%" PRIx64 " failed to post sem, replay", tmq->consumerId); } int32_t code = taosReleaseRef(tmqMgmt.rsetId, refId); if (code != 0){ - tscError("failed to release ref:%"PRId64 ", code:%d", refId, code); + tqErrorC("failed to release ref:%"PRId64 ", code:%d", refId, code); } } @@ -877,11 +919,11 @@ void tmqAssignDelayedCommitTask(void* param, void* tmrId) { } int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { - if (code != 0) { - goto END; + if (pMsg == NULL) { + return TSDB_CODE_INVALID_PARA; } - if (pMsg == NULL || param == NULL) { - code = TSDB_CODE_INVALID_PARA; + + if (param == NULL || code != 0){ goto END; } @@ -902,7 +944,7 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { for (int32_t j = 0; j < topicNumCur; j++) { SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, j); if (pTopicCur && strcmp(pTopicCur->topicName, privilege->topic) == 0) { - tscInfo("consumer:0x%" PRIx64 ", has no privilege, topic:%s", tmq->consumerId, privilege->topic); + tqInfoC("consumer:0x%" PRIx64 ", has no privilege, topic:%s", tmq->consumerId, privilege->topic); pTopicCur->noPrivilege = 1; } } @@ -911,10 +953,11 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { taosWUnLockLatch(&tmq->lock); code = taosReleaseRef(tmqMgmt.rsetId, refId); if (code != 0){ - tscError("failed to release ref:%"PRId64 ", code:%d", refId, code); + tqErrorC("failed to release ref:%"PRId64 ", code:%d", refId, code); } } + tqClientDebug = rsp.debugFlag; tDestroySMqHbRsp(&rsp); END: @@ -934,10 +977,10 @@ void tmqSendHbReq(void* param, void* tmrId) { SMqHbReq req = {0}; req.consumerId = tmq->consumerId; req.epoch = tmq->epoch; - req.pollFlag = atomic_load_8(&pollFlag); + req.pollFlag = atomic_load_8(&tmq->pollFlag); req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows)); if (req.topics == NULL) { - return; + goto END; } taosRLockLatch(&tmq->lock); for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { @@ -950,7 +993,7 @@ void tmqSendHbReq(void* param, void* tmrId) { if (data == NULL) { continue; } - (void)strcpy(data->topicName, pTopic->topicName); + tstrncpy(data->topicName, pTopic->topicName, TSDB_TOPIC_FNAME_LEN); data->offsetRows = taosArrayInit(numOfVgroups, sizeof(OffsetRows)); if (data->offsetRows == NULL) { continue; @@ -970,7 +1013,7 @@ void tmqSendHbReq(void* param, void* tmrId) { offRows->ever = pVg->offsetInfo.walVerEnd == -1 ? 0 : pVg->offsetInfo.walVerEnd; char buf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(buf, TSDB_OFFSET_LEN, &offRows->offset); - tscDebug("consumer:0x%" PRIx64 ",report offset, group:%s vgId:%d, offset:%s/%" PRId64 ", rows:%" PRId64, + tqDebugC("consumer:0x%" PRIx64 ",report offset, group:%s vgId:%d, offset:%s/%" PRId64 ", rows:%" PRId64, tmq->consumerId, tmq->groupId, offRows->vgId, buf, offRows->ever, offRows->rows); } } @@ -978,26 +1021,26 @@ void tmqSendHbReq(void* param, void* tmrId) { int32_t tlen = tSerializeSMqHbReq(NULL, 0, &req); if (tlen < 0) { - tscError("tSerializeSMqHbReq failed, size:%d", tlen); - goto OVER; + tqErrorC("tSerializeSMqHbReq failed, size:%d", tlen); + goto END; } void* pReq = taosMemoryCalloc(1, tlen); if (pReq == NULL) { - tscError("failed to malloc MqHbReq msg, code:%d", terrno); - goto OVER; + tqErrorC("failed to malloc MqHbReq msg, code:%d", terrno); + goto END; } if (tSerializeSMqHbReq(pReq, tlen, &req) < 0) { - tscError("tSerializeSMqHbReq %d failed", tlen); + tqErrorC("tSerializeSMqHbReq %d failed", tlen); taosMemoryFree(pReq); - goto OVER; + goto END; } SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { taosMemoryFree(pReq); - goto OVER; + goto END; } sendInfo->msgInfo = (SDataBuf){.pData = pReq, .len = tlen, .handle = NULL}; @@ -1012,35 +1055,351 @@ void tmqSendHbReq(void* param, void* tmrId) { int32_t code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo); if (code != 0) { - tscError("tmqSendHbReq asyncSendMsgToServer failed"); + tqErrorC("tmqSendHbReq asyncSendMsgToServer failed"); } + (void)atomic_val_compare_exchange_8(&tmq->pollFlag, 1, 0); - (void)atomic_val_compare_exchange_8(&pollFlag, 1, 0); -OVER: +END: tDestroySMqHbReq(&req); if (tmrId != NULL) { bool ret = taosTmrReset(tmqSendHbReq, tmq->heartBeatIntervalMs, param, tmqMgmt.timer, &tmq->hbLiveTimer); - tscDebug("reset timer fo tmq hb:%d", ret); + tqDebugC("reset timer fo tmq hb:%d", ret); } int32_t ret = taosReleaseRef(tmqMgmt.rsetId, refId); if (ret != 0){ - tscError("failed to release ref:%"PRId64 ", code:%d", refId, ret); + tqErrorC("failed to release ref:%"PRId64 ", code:%d", refId, ret); } } static void defaultCommitCbFn(tmq_t* pTmq, int32_t code, void* param) { if (code != 0) { - tscError("consumer:0x%" PRIx64 ", failed to commit offset, code:%s", pTmq->consumerId, tstrerror(code)); + tqErrorC("consumer:0x%" PRIx64 ", failed to commit offset, code:%s", pTmq->consumerId, tstrerror(code)); } } +static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { + if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) { + tDeleteSMqAskEpRsp(&rspWrapper->epRsp); + } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { + DELETE_POLL_RSP(tDeleteMqDataRsp, &pRsp->dataRsp) + } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP){ + DELETE_POLL_RSP(tDeleteSTaosxRsp, &pRsp->dataRsp) + } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) { + DELETE_POLL_RSP(tDeleteMqMetaRsp,&pRsp->metaRsp) + } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { + DELETE_POLL_RSP(tDeleteMqBatchMetaRsp,&pRsp->batchMetaRsp) + } +} + +static void freeClientVg(void* param) { + SMqClientVg* pVg = param; + tOffsetDestroy(&pVg->offsetInfo.endOffset); + tOffsetDestroy(&pVg->offsetInfo.beginOffset); + tOffsetDestroy(&pVg->offsetInfo.committedOffset); +} +static void freeClientTopic(void* param) { + SMqClientTopic* pTopic = param; + taosMemoryFreeClear(pTopic->schema.pSchema); + taosArrayDestroyEx(pTopic->vgs, freeClientVg); +} + +static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopicEp, SHashObj* pVgOffsetHashMap, + tmq_t* tmq) { + pTopic->schema = pTopicEp->schema; + pTopicEp->schema.nCols = 0; + pTopicEp->schema.pSchema = NULL; + + char vgKey[TSDB_TOPIC_FNAME_LEN + 22] = {0}; + int32_t vgNumGet = taosArrayGetSize(pTopicEp->vgs); + + tstrncpy(pTopic->topicName, pTopicEp->topic, TSDB_TOPIC_FNAME_LEN); + tstrncpy(pTopic->db, pTopicEp->db, TSDB_DB_FNAME_LEN); + + tqInfoC("consumer:0x%" PRIx64 ", update topic:%s, new numOfVgs:%d", tmq->consumerId, pTopic->topicName, vgNumGet); + pTopic->vgs = taosArrayInit(vgNumGet, sizeof(SMqClientVg)); + if (pTopic->vgs == NULL) { + tqErrorC("consumer:0x%" PRIx64 ", failed to init vgs for topic:%s", tmq->consumerId, pTopic->topicName); + return; + } + for (int32_t j = 0; j < vgNumGet; j++) { + SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); + if (pVgEp == NULL) { + continue; + } + (void)snprintf(vgKey, sizeof(vgKey), "%s:%d", pTopic->topicName, pVgEp->vgId); + SVgroupSaveInfo* pInfo = taosHashGet(pVgOffsetHashMap, vgKey, strlen(vgKey)); + + STqOffsetVal offsetNew = {0}; + offsetNew.type = tmq->resetOffsetCfg; + + tqInfoC("consumer:0x%" PRIx64 ", update topic:%s, new numOfVgs:%d, num:%d, port:%d", tmq->consumerId, + pTopic->topicName, vgNumGet, pVgEp->epSet.numOfEps, pVgEp->epSet.eps[pVgEp->epSet.inUse].port); + + SMqClientVg clientVg = { + .pollCnt = 0, + .vgId = pVgEp->vgId, + .epSet = pVgEp->epSet, + .vgStatus = pInfo ? pInfo->vgStatus : TMQ_VG_STATUS__IDLE, + .vgSkipCnt = 0, + .emptyBlockReceiveTs = 0, + .blockReceiveTs = 0, + .blockSleepForReplay = 0, + .numOfRows = pInfo ? pInfo->numOfRows : 0, + }; + + clientVg.offsetInfo.walVerBegin = -1; + clientVg.offsetInfo.walVerEnd = -1; + clientVg.seekUpdated = false; + if (pInfo) { + tOffsetCopy(&clientVg.offsetInfo.endOffset, &pInfo->currentOffset); + tOffsetCopy(&clientVg.offsetInfo.committedOffset, &pInfo->commitOffset); + tOffsetCopy(&clientVg.offsetInfo.beginOffset, &pInfo->seekOffset); + } else { + clientVg.offsetInfo.endOffset = offsetNew; + clientVg.offsetInfo.committedOffset = offsetNew; + clientVg.offsetInfo.beginOffset = offsetNew; + } + if (taosArrayPush(pTopic->vgs, &clientVg) == NULL) { + tqErrorC("consumer:0x%" PRIx64 ", failed to push vg:%d into topic:%s", tmq->consumerId, pVgEp->vgId, + pTopic->topicName); + freeClientVg(&clientVg); + } + } +} + +static void buildNewTopicList(tmq_t* tmq, SArray* newTopics, const SMqAskEpRsp* pRsp){ + SHashObj* pVgOffsetHashMap = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); + if (pVgOffsetHashMap == NULL) { + tqErrorC("consumer:0x%" PRIx64 " taos hash init null, code:%d", tmq->consumerId, terrno); + return; + } + + int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); + for (int32_t i = 0; i < topicNumCur; i++) { + // find old topic + SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); + if (pTopicCur && pTopicCur->vgs) { + int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); + tqInfoC("consumer:0x%" PRIx64 ", current vg num: %d", tmq->consumerId, vgNumCur); + for (int32_t j = 0; j < vgNumCur; j++) { + SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); + if (pVgCur == NULL) { + continue; + } + char vgKey[TSDB_TOPIC_FNAME_LEN + 22] = {0}; + (void)snprintf(vgKey, sizeof(vgKey), "%s:%d", pTopicCur->topicName, pVgCur->vgId); + + char buf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(buf, TSDB_OFFSET_LEN, &pVgCur->offsetInfo.endOffset); + tqInfoC("consumer:0x%" PRIx64 ", vgId:%d vgKey:%s, offset:%s", tmq->consumerId, pVgCur->vgId, vgKey, buf); + + SVgroupSaveInfo info = {.currentOffset = pVgCur->offsetInfo.endOffset, + .seekOffset = pVgCur->offsetInfo.beginOffset, + .commitOffset = pVgCur->offsetInfo.committedOffset, + .numOfRows = pVgCur->numOfRows, + .vgStatus = pVgCur->vgStatus}; + if (taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)) != 0) { + tqErrorC("consumer:0x%" PRIx64 ", failed to put vg:%d into hashmap", tmq->consumerId, pVgCur->vgId); + } + } + } + } + + for (int32_t i = 0; i < taosArrayGetSize(pRsp->topics); i++) { + SMqClientTopic topic = {0}; + SMqSubTopicEp* pTopicEp = taosArrayGet(pRsp->topics, i); + if (pTopicEp == NULL) { + continue; + } + initClientTopicFromRsp(&topic, pTopicEp, pVgOffsetHashMap, tmq); + if (taosArrayPush(newTopics, &topic) == NULL) { + tqErrorC("consumer:0x%" PRIx64 ", failed to push topic:%s into new topics", tmq->consumerId, topic.topicName); + freeClientTopic(&topic); + } + } + + taosHashCleanup(pVgOffsetHashMap); +} + +static void doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { + int32_t topicNumGet = taosArrayGetSize(pRsp->topics); + // vnode transform (epoch == tmq->epoch && topicNumGet != 0) + // ask ep rsp (epoch == tmq->epoch && topicNumGet == 0) + if (epoch < tmq->epoch || (epoch == tmq->epoch && topicNumGet == 0)) { + tqDebugC("consumer:0x%" PRIx64 " no update ep epoch from %d to epoch %d, incoming topics:%d", tmq->consumerId, + tmq->epoch, epoch, topicNumGet); + return; + } + + SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic)); + if (newTopics == NULL) { + tqErrorC("consumer:0x%" PRIx64 " taos array init null, code:%d", tmq->consumerId, terrno); + return; + } + tqInfoC("consumer:0x%" PRIx64 " update ep epoch from %d to epoch %d, incoming topics:%d, existed topics:%d", + tmq->consumerId, tmq->epoch, epoch, topicNumGet, (int)taosArrayGetSize(tmq->clientTopics)); + + taosWLockLatch(&tmq->lock); + if (topicNumGet > 0){ + buildNewTopicList(tmq, newTopics, pRsp); + } + // destroy current buffered existed topics info + if (tmq->clientTopics) { + taosArrayDestroyEx(tmq->clientTopics, freeClientTopic); + } + tmq->clientTopics = newTopics; + taosWUnLockLatch(&tmq->lock); + + atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY); + atomic_store_32(&tmq->epoch, epoch); + + tqInfoC("consumer:0x%" PRIx64 " update topic info completed", tmq->consumerId); +} + +static int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { + SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; + if (pParam == NULL) { + goto FAIL; + } + + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId); + if (tmq == NULL) { + code = TSDB_CODE_TMQ_CONSUMER_CLOSED; + goto FAIL; + } + + if (code != TSDB_CODE_SUCCESS) { + tqErrorC("consumer:0x%" PRIx64 ", get topic endpoint error, code:%s", tmq->consumerId, tstrerror(code)); + goto END; + } + + if (pMsg == NULL) { + goto END; + } + SMqRspHead* head = pMsg->pData; + int32_t epoch = atomic_load_32(&tmq->epoch); + tqDebugC("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d", tmq->consumerId, head->epoch, epoch); + if (pParam->sync) { + SMqAskEpRsp rsp = {0}; + if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp) != NULL) { + doUpdateLocalEp(tmq, head->epoch, &rsp); + } + tDeleteSMqAskEpRsp(&rsp); + } else { + SMqRspWrapper* pWrapper = NULL; + code = taosAllocateQitem(sizeof(SMqRspWrapper), DEF_QITEM, 0, (void**)&pWrapper); + if (code) { + goto END; + } + + pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP; + pWrapper->epoch = head->epoch; + (void)memcpy(&pWrapper->epRsp, pMsg->pData, sizeof(SMqRspHead)); + if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->epRsp) == NULL) { + tmqFreeRspWrapper((SMqRspWrapper*)pWrapper); + taosFreeQitem(pWrapper); + } else { + code = taosWriteQitem(tmq->mqueue, pWrapper); + if (code != 0) { + tmqFreeRspWrapper((SMqRspWrapper*)pWrapper); + taosFreeQitem(pWrapper); + tqErrorC("consumer:0x%" PRIx64 " put ep res into mqueue failed, code:%d", tmq->consumerId, code); + } + } + } + + END: + { + int32_t ret = taosReleaseRef(tmqMgmt.rsetId, pParam->refId); + if (ret != 0){ + tqErrorC("failed to release ref:%"PRId64 ", code:%d", pParam->refId, ret); + } + } + + FAIL: + if (pParam && pParam->sync) { + SAskEpInfo* pInfo = pParam->pParam; + if (pInfo) { + pInfo->code = code; + if (tsem2_post(&pInfo->sem) != 0){ + tqErrorC("failed to post rsp sem askep cb"); + } + } + } + + if (pMsg) { + taosMemoryFree(pMsg->pEpSet); + taosMemoryFree(pMsg->pData); + } + + return code; +} + +static int32_t askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) { + SMqAskEpReq req = {0}; + req.consumerId = pTmq->consumerId; + req.epoch = updateEpSet ? -1 : pTmq->epoch; + tstrncpy(req.cgroup, pTmq->groupId, TSDB_CGROUP_LEN); + int code = 0; + SMqAskEpCbParam* pParam = NULL; + void* pReq = NULL; + + int32_t tlen = tSerializeSMqAskEpReq(NULL, 0, &req); + if (tlen < 0) { + tqErrorC("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq failed", pTmq->consumerId); + return TSDB_CODE_INVALID_PARA; + } + + pReq = taosMemoryCalloc(1, tlen); + if (pReq == NULL) { + tqErrorC("consumer:0x%" PRIx64 ", failed to malloc askEpReq msg, size:%d", pTmq->consumerId, tlen); + return terrno; + } + + if (tSerializeSMqAskEpReq(pReq, tlen, &req) < 0) { + tqErrorC("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq %d failed", pTmq->consumerId, tlen); + taosMemoryFree(pReq); + return TSDB_CODE_INVALID_PARA; + } + + pParam = taosMemoryCalloc(1, sizeof(SMqAskEpCbParam)); + if (pParam == NULL) { + tqErrorC("consumer:0x%" PRIx64 ", failed to malloc subscribe param", pTmq->consumerId); + taosMemoryFree(pReq); + return terrno; + } + + pParam->refId = pTmq->refId; + pParam->sync = sync; + pParam->pParam = param; + + SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (sendInfo == NULL) { + taosMemoryFree(pReq); + taosMemoryFree(pParam); + return terrno; + } + + sendInfo->msgInfo = (SDataBuf){.pData = pReq, .len = tlen, .handle = NULL}; + sendInfo->requestId = generateRequestId(); + sendInfo->requestObjRefId = 0; + sendInfo->param = pParam; + sendInfo->paramFreeFp = taosMemoryFree; + sendInfo->fp = askEpCb; + sendInfo->msgType = TDMT_MND_TMQ_ASK_EP; + + SEpSet epSet = getEpSet_s(&pTmq->pTscObj->pAppInfo->mgmtEp); + tqDebugC("consumer:0x%" PRIx64 " ask ep from mnode,QID:0x%" PRIx64, pTmq->consumerId, sendInfo->requestId); + return asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo); +} + void tmqHandleAllDelayedTask(tmq_t* pTmq) { STaosQall* qall = NULL; int32_t code = 0; code = taosAllocateQall(&qall); if (code) { - tscError("consumer:0x%" PRIx64 ", failed to allocate qall, code:%s", pTmq->consumerId, tstrerror(code)); + tqErrorC("consumer:0x%" PRIx64 ", failed to allocate qall, code:%s", pTmq->consumerId, tstrerror(code)); return; } @@ -1050,29 +1409,30 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) { return; } - tscDebug("consumer:0x%" PRIx64 " handle delayed %d tasks before poll data", pTmq->consumerId, numOfItems); + tqDebugC("consumer:0x%" PRIx64 " handle delayed %d tasks before poll data", pTmq->consumerId, numOfItems); int8_t* pTaskType = NULL; while (taosGetQitem(qall, (void**)&pTaskType) != 0) { if (*pTaskType == TMQ_DELAYED_TASK__ASK_EP) { + tqDebugC("consumer:0x%" PRIx64 " retrieve ask ep timer", pTmq->consumerId); code = askEp(pTmq, NULL, false, false); if (code != 0) { - tscError("consumer:0x%" PRIx64 " failed to ask ep, code:%s", pTmq->consumerId, tstrerror(code)); + tqErrorC("consumer:0x%" PRIx64 " failed to ask ep, code:%s", pTmq->consumerId, tstrerror(code)); continue; } - tscDebug("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId); + tqDebugC("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId); bool ret = taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer, - &pTmq->epTimer); - tscDebug("reset timer fo tmq ask ep:%d", ret); + &pTmq->epTimer); + tqDebugC("reset timer fo tmq ask ep:%d", ret); } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { - tmq_commit_cb* pCallbackFn = pTmq->commitCb ? pTmq->commitCb : defaultCommitCbFn; + tmq_commit_cb* pCallbackFn = (pTmq->commitCb != NULL) ? pTmq->commitCb : defaultCommitCbFn; asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam); - tscDebug("consumer:0x%" PRIx64 " next commit to vnode(s) in %.2fs", pTmq->consumerId, + tqDebugC("consumer:0x%" PRIx64 " next commit to vnode(s) in %.2fs", pTmq->consumerId, pTmq->autoCommitInterval / 1000.0); bool ret = taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer, - &pTmq->commitTimer); - tscDebug("reset timer fo commit:%d", ret); + &pTmq->commitTimer); + tqDebugC("reset timer fo commit:%d", ret); } else { - tscError("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType); + tqErrorC("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType); } taosFreeQitem(pTaskType); @@ -1081,35 +1441,11 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) { taosFreeQall(qall); } -static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { - if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) { - SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)rspWrapper; - tDeleteSMqAskEpRsp(&pEpRspWrapper->msg); - } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { - SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; - taosMemoryFreeClear(pRsp->pEpset); - - tDeleteMqDataRsp(&pRsp->dataRsp); - } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) { - SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; - taosMemoryFreeClear(pRsp->pEpset); - tDeleteMqMetaRsp(&pRsp->metaRsp); - } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { - SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; - taosMemoryFreeClear(pRsp->pEpset); - tDeleteSTaosxRsp(&pRsp->taosxRsp); - } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { - SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; - taosMemoryFreeClear(pRsp->pEpset); - tDeleteMqBatchMetaRsp(&pRsp->batchMetaRsp); - } -} - void tmqClearUnhandleMsg(tmq_t* tmq) { SMqRspWrapper* rspWrapper = NULL; while (taosGetQitem(tmq->qall, (void**)&rspWrapper) != 0) { - tmqFreeRspWrapper(rspWrapper); - taosFreeQitem(rspWrapper); + tmqFreeRspWrapper(rspWrapper); + taosFreeQitem(rspWrapper); } rspWrapper = NULL; @@ -1123,6 +1459,10 @@ void tmqClearUnhandleMsg(tmq_t* tmq) { } int32_t tmqSubscribeCb(void* param, SDataBuf* pMsg, int32_t code) { + if (pMsg) { + taosMemoryFreeClear(pMsg->pEpSet); + } + if (param == NULL) { return code; } @@ -1130,11 +1470,8 @@ int32_t tmqSubscribeCb(void* param, SDataBuf* pMsg, int32_t code) { SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param; pParam->rspErr = code; - if (pMsg) { - taosMemoryFree(pMsg->pEpSet); - } if (tsem2_post(&pParam->rspSem) != 0){ - tscError("failed to post sem, subscribe cb"); + tqErrorC("failed to post sem, subscribe cb"); } return 0; } @@ -1151,16 +1488,16 @@ int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { SMqClientTopic* topic = taosArrayGet(tmq->clientTopics, i); if (topic == NULL) { - tscError("topic is null"); + tqErrorC("topic is null"); continue; } char* tmp = strchr(topic->topicName, '.'); if (tmp == NULL) { - tscError("topic name is invalid:%s", topic->topicName); + tqErrorC("topic name is invalid:%s", topic->topicName); continue; } if (tmq_list_append(*topics, tmp + 1) != 0) { - tscError("failed to append topic:%s", tmp + 1); + tqErrorC("failed to append topic:%s", tmp + 1); continue; } } @@ -1168,18 +1505,6 @@ int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { return 0; } -static void freeClientVg(void* param) { - SMqClientVg* pVg = param; - tOffsetDestroy(&pVg->offsetInfo.endOffset); - tOffsetDestroy(&pVg->offsetInfo.beginOffset); - tOffsetDestroy(&pVg->offsetInfo.committedOffset); -} -static void freeClientTopic(void* param) { - SMqClientTopic* pTopic = param; - taosMemoryFreeClear(pTopic->schema.pSchema); - taosArrayDestroyEx(pTopic->vgs, freeClientVg); -} - void tmqFreeImpl(void* handle) { tmq_t* tmq = (tmq_t*)handle; int64_t id = tmq->consumerId; @@ -1195,7 +1520,7 @@ void tmqFreeImpl(void* handle) { taosFreeQall(tmq->qall); if(tsem2_destroy(&tmq->rspSem) != 0) { - tscError("failed to destroy sem in free tmq"); + tqErrorC("failed to destroy sem in free tmq"); } taosArrayDestroyEx(tmq->clientTopics, freeClientTopic); @@ -1203,22 +1528,22 @@ void tmqFreeImpl(void* handle) { if (tmq->commitTimer) { if (!taosTmrStopA(&tmq->commitTimer)) { - tscError("failed to stop commit timer"); + tqErrorC("failed to stop commit timer"); } } if (tmq->epTimer) { if (!taosTmrStopA(&tmq->epTimer)) { - tscError("failed to stop ep timer"); + tqErrorC("failed to stop ep timer"); } } if (tmq->hbLiveTimer) { if (!taosTmrStopA(&tmq->hbLiveTimer)) { - tscError("failed to stop hb timer"); + tqErrorC("failed to stop hb timer"); } } taosMemoryFree(tmq); - tscDebug("consumer:0x%" PRIx64 " closed", id); + tqInfoC("consumer:0x%" PRIx64 " closed", id); } static void tmqMgmtInit(void) { @@ -1247,9 +1572,6 @@ void tmqMgmtClose(void) { } } -#define SET_ERROR_MSG_TMQ(MSG) \ - if (errstr != NULL) (void)snprintf(errstr, errstrLen, MSG); - tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { int32_t code = 0; @@ -1269,7 +1591,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t)); if (pTmq == NULL) { - tscError("failed to create consumer, groupId:%s", conf->groupId); + tqErrorC("failed to create consumer, groupId:%s", conf->groupId); SET_ERROR_MSG_TMQ("malloc tmq failed") return NULL; } @@ -1279,13 +1601,13 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); if (pTmq->clientTopics == NULL) { - tscError("failed to create consumer, groupId:%s", conf->groupId); + tqErrorC("failed to create consumer, groupId:%s", conf->groupId); SET_ERROR_MSG_TMQ("malloc client topics failed") goto _failed; } code = taosOpenQueue(&pTmq->mqueue); if (code) { - tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), + tqErrorC("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId); SET_ERROR_MSG_TMQ("open queue failed") goto _failed; @@ -1293,7 +1615,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { code = taosOpenQueue(&pTmq->delayedTask); if (code) { - tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), + tqErrorC("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId); SET_ERROR_MSG_TMQ("open delayed task queue failed") goto _failed; @@ -1301,14 +1623,14 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { code = taosAllocateQall(&pTmq->qall); if (code) { - tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), + tqErrorC("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId); SET_ERROR_MSG_TMQ("allocate qall failed") goto _failed; } if (conf->groupId[0] == 0) { - tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), + tqErrorC("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId); SET_ERROR_MSG_TMQ("malloc tmq element failed or group is empty") goto _failed; @@ -1318,10 +1640,11 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->status = TMQ_CONSUMER_STATUS__INIT; pTmq->pollCnt = 0; pTmq->epoch = 0; + pTmq->pollFlag = 0; // set conf - (void)strcpy(pTmq->clientId, conf->clientId); - (void)strcpy(pTmq->groupId, conf->groupId); + tstrncpy(pTmq->clientId, conf->clientId, TSDB_CLIENT_ID_LEN); + tstrncpy(pTmq->groupId, conf->groupId, TSDB_CGROUP_LEN); pTmq->withTbName = conf->withTbName; pTmq->useSnapshot = conf->snapEnable; pTmq->autoCommit = conf->autoCommit; @@ -1337,7 +1660,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->enableBatchMeta = conf->enableBatchMeta; tstrncpy(pTmq->user, user, TSDB_USER_LEN); if (taosGetFqdn(pTmq->fqdn) != 0) { - (void)strcpy(pTmq->fqdn, "localhost"); + tstrncpy(pTmq->fqdn, "localhost", TSDB_FQDN_LEN); } if (conf->replayEnable) { pTmq->autoCommit = false; @@ -1349,7 +1672,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { // init semaphore if (tsem2_init(&pTmq->rspSem, 0, 0) != 0) { - tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, + tqErrorC("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, tstrerror(TAOS_SYSTEM_ERROR(errno)), pTmq->groupId); SET_ERROR_MSG_TMQ("init t_sem failed") goto _failed; @@ -1359,7 +1682,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { code = taos_connect_internal(conf->ip, user, pass, NULL, NULL, conf->port, CONN_TYPE__TMQ, &pTmq->pTscObj); if (code) { terrno = code; - tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); + tqErrorC("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); SET_ERROR_MSG_TMQ("init tscObj failed") goto _failed; } @@ -1378,7 +1701,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { char buf[TSDB_OFFSET_LEN] = {0}; STqOffsetVal offset = {.type = pTmq->resetOffsetCfg}; tFormatOffset(buf, tListLen(buf), &offset); - tscInfo("consumer:0x%" PRIx64 " is setup, refId:%" PRId64 + tqInfoC("consumer:0x%" PRIx64 " is setup, refId:%" PRId64 ", groupId:%s, snapshot:%d, autoCommit:%d, commitInterval:%dms, offset:%s", pTmq->consumerId, pTmq->refId, pTmq->groupId, pTmq->useSnapshot, pTmq->autoCommit, pTmq->autoCommitInterval, buf); @@ -1390,9 +1713,31 @@ _failed: return NULL; } +static int32_t syncAskEp(tmq_t* pTmq) { + SAskEpInfo* pInfo = taosMemoryMalloc(sizeof(SAskEpInfo)); + if (pInfo == NULL) return terrno; + if (tsem2_init(&pInfo->sem, 0, 0) != 0) { + taosMemoryFree(pInfo); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } + + int32_t code = askEp(pTmq, pInfo, true, false); + if (code == 0) { + if (tsem2_wait(&pInfo->sem) != 0){ + tqErrorC("consumer:0x%" PRIx64 ", failed to wait for sem", pTmq->consumerId); + } + code = pInfo->code; + } + + if(tsem2_destroy(&pInfo->sem) != 0) { + tqErrorC("failed to destroy sem sync ask ep"); + } + taosMemoryFree(pInfo); + return code; +} + int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { if (tmq == NULL || topic_list == NULL) return TSDB_CODE_INVALID_PARA; - const int32_t MAX_RETRY_COUNT = 120 * 2; // let's wait for 2 mins at most const SArray* container = &topic_list->container; int32_t sz = taosArrayGetSize(container); void* buf = NULL; @@ -1400,7 +1745,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { SCMSubscribeReq req = {0}; int32_t code = 0; - tscInfo("consumer:0x%" PRIx64 " cgroup:%s, subscribe %d topics", tmq->consumerId, tmq->groupId, sz); + tqInfoC("consumer:0x%" PRIx64 " cgroup:%s, subscribe %d topics", tmq->consumerId, tmq->groupId, sz); req.consumerId = tmq->consumerId; tstrncpy(req.clientId, tmq->clientId, TSDB_CLIENT_ID_LEN); @@ -1411,7 +1756,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { req.topicNames = taosArrayInit(sz, sizeof(void*)); if (req.topicNames == NULL) { code = terrno; - goto FAIL; + goto END; } req.withTbName = tmq->withTbName; @@ -1426,43 +1771,43 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { for (int32_t i = 0; i < sz; i++) { char* topic = taosArrayGetP(container, i); if (topic == NULL) { - code = TSDB_CODE_INVALID_PARA; - goto FAIL; + code = terrno; + goto END; } SName name = {0}; code = tNameSetDbName(&name, tmq->pTscObj->acctId, topic, strlen(topic)); if (code) { - tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to set topic name, code:%d", tmq->consumerId, tmq->groupId, + tqErrorC("consumer:0x%" PRIx64 " cgroup:%s, failed to set topic name, code:%d", tmq->consumerId, tmq->groupId, code); - goto FAIL; + goto END; } char* topicFName = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN); if (topicFName == NULL) { code = terrno; - goto FAIL; + goto END; } code = tNameExtractFullName(&name, topicFName); if (code) { - tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to extract topic name, code:%d", tmq->consumerId, tmq->groupId, + tqErrorC("consumer:0x%" PRIx64 " cgroup:%s, failed to extract topic name, code:%d", tmq->consumerId, tmq->groupId, code); taosMemoryFree(topicFName); - goto FAIL; + goto END; } if (taosArrayPush(req.topicNames, &topicFName) == NULL) { code = terrno; taosMemoryFree(topicFName); - goto FAIL; + goto END; } - tscInfo("consumer:0x%" PRIx64 " subscribe topic:%s", tmq->consumerId, topicFName); + tqInfoC("consumer:0x%" PRIx64 " subscribe topic:%s", tmq->consumerId, topicFName); } int32_t tlen = tSerializeSCMSubscribeReq(NULL, &req); buf = taosMemoryMalloc(tlen); if (buf == NULL) { code = terrno; - goto FAIL; + goto END; } void* abuf = buf; @@ -1472,7 +1817,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { if (sendInfo == NULL) { code = terrno; taosMemoryFree(buf); - goto FAIL; + goto END; } SMqSubscribeCbParam param = {.rspErr = 0}; @@ -1480,7 +1825,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { code = TSDB_CODE_TSC_INTERNAL_ERROR; taosMemoryFree(buf); taosMemoryFree(sendInfo); - goto FAIL; + goto END; } sendInfo->msgInfo = (SDataBuf){.pData = buf, .len = tlen, .handle = NULL}; @@ -1494,47 +1839,49 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo); if (code != 0) { - goto FAIL; + goto END; } if (tsem2_wait(¶m.rspSem) != 0){ - tscError("consumer:0x%" PRIx64 ", failed to wait semaphore in subscribe", tmq->consumerId); + tqErrorC("consumer:0x%" PRIx64 ", failed to wait semaphore in subscribe", tmq->consumerId); } if(tsem2_destroy(¶m.rspSem) != 0) { - tscError("consumer:0x%" PRIx64 ", failed to destroy semaphore in subscribe", tmq->consumerId); + tqErrorC("consumer:0x%" PRIx64 ", failed to destroy semaphore in subscribe", tmq->consumerId); } if (param.rspErr != 0) { code = param.rspErr; - goto FAIL; + goto END; } int32_t retryCnt = 0; while ((code = syncAskEp(tmq)) != 0) { - if (retryCnt++ > MAX_RETRY_COUNT || code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) { - tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes, code:%s", + if (retryCnt++ > SUBSCRIBE_RETRY_MAX_COUNT || code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) { + tqErrorC("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes, code:%s", tmq->consumerId, tstrerror(code)); if (code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) { code = 0; } - goto FAIL; + goto END; } - tscInfo("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry:%d in 500ms", tmq->consumerId, retryCnt); - taosMsleep(500); + tqInfoC("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry:%d in 500ms", tmq->consumerId, retryCnt); + taosMsleep(SUBSCRIBE_RETRY_INTERVAL); } - tmq->epTimer = taosTmrStart(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(tmq->refId), tmqMgmt.timer); - tmq->commitTimer = - taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, (void*)(tmq->refId), tmqMgmt.timer); + if (tmq->epTimer == NULL){ + tmq->epTimer = taosTmrStart(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(tmq->refId), tmqMgmt.timer); + } + if (tmq->commitTimer == NULL){ + tmq->commitTimer = taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, (void*)(tmq->refId), tmqMgmt.timer); + } if (tmq->epTimer == NULL || tmq->commitTimer == NULL) { code = TSDB_CODE_TSC_INTERNAL_ERROR; - goto FAIL; + goto END; } -FAIL: +END: taosArrayDestroyP(req.topicNames, taosMemoryFree); - return code; } @@ -1572,19 +1919,9 @@ static SMqClientTopic* getTopicInfo(tmq_t* tmq, char* topicName) { return NULL; } -static void setVgIdle(tmq_t* tmq, char* topicName, int32_t vgId) { - taosWLockLatch(&tmq->lock); - SMqClientVg* pVg = NULL; - getVgInfo(tmq, topicName, vgId, &pVg); - if (pVg) { - atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - } - taosWUnLockLatch(&tmq->lock); -} - int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tmq_t* tmq = NULL; - SMqPollRspWrapper* pRspWrapper = NULL; + SMqRspWrapper* pRspWrapper = NULL; int8_t rspType = 0; int32_t vgId = 0; uint64_t requestId = 0; @@ -1593,21 +1930,19 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { return TSDB_CODE_TSC_INTERNAL_ERROR; } if (pParam == NULL) { - taosMemoryFreeClear(pMsg->pData); - taosMemoryFreeClear(pMsg->pEpSet); - return TSDB_CODE_TSC_INTERNAL_ERROR; + code = TSDB_CODE_TSC_INTERNAL_ERROR; + goto EXIT; } int64_t refId = pParam->refId; vgId = pParam->vgId; requestId = pParam->requestId; tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { - taosMemoryFreeClear(pMsg->pData); - taosMemoryFreeClear(pMsg->pEpSet); - return TSDB_CODE_TMQ_CONSUMER_CLOSED; + code = TSDB_CODE_TMQ_CONSUMER_CLOSED; + goto EXIT; } - int32_t ret = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM, 0, (void**)&pRspWrapper); + int32_t ret = taosAllocateQitem(sizeof(SMqRspWrapper), DEF_QITEM, 0, (void**)&pRspWrapper); if (ret) { code = ret; tscWarn("consumer:0x%" PRIx64 " msg discard from vgId:%d, since out of memory", tmq->consumerId, vgId); @@ -1619,280 +1954,72 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { } if (pMsg->pData == NULL) { - tscError("consumer:0x%" PRIx64 " msg discard from vgId:%d, since msg is NULL", tmq->consumerId, vgId); + tqErrorC("consumer:0x%" PRIx64 " msg discard from vgId:%d, since msg is NULL", tmq->consumerId, vgId); code = TSDB_CODE_TSC_INTERNAL_ERROR; goto END; } int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch; int32_t clientEpoch = atomic_load_32(&tmq->epoch); - if (msgEpoch < clientEpoch) { - // do not write into queue since updating epoch reset - tscWarn("consumer:0x%" PRIx64 - " msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d,QID:0x%" PRIx64, - tmq->consumerId, vgId, msgEpoch, clientEpoch, requestId); - code = TSDB_CODE_TMQ_CONSUMER_MISMATCH; - goto END; - } if (msgEpoch != clientEpoch) { - tscError("consumer:0x%" PRIx64 - " msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, reqId:0x%" PRIx64, + tqErrorC("consumer:0x%" PRIx64 + " msg discard from vgId:%d since epoch not equal, rsp epoch %d, current epoch %d, reqId:0x%" PRIx64, tmq->consumerId, vgId, msgEpoch, clientEpoch, requestId); code = TSDB_CODE_TMQ_CONSUMER_MISMATCH; goto END; } - // handle meta rsp rspType = ((SMqRspHead*)pMsg->pData)->mqMsgType; - pRspWrapper->tmqRspType = rspType; - pRspWrapper->reqId = requestId; - pRspWrapper->pEpset = pMsg->pEpSet; - pMsg->pEpSet = NULL; - + tqDebugC("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, type %d,QID:0x%" PRIx64, tmq->consumerId, vgId, rspType, requestId); if (rspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - if (tDecodeMqDataRsp(&decoder, &pRspWrapper->dataRsp) < 0) { - tDecoderClear(&decoder); - code = TSDB_CODE_OUT_OF_MEMORY; - goto END; - } - tDecoderClear(&decoder); - (void)memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); - - char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &pRspWrapper->dataRsp.common.rspOffset); - tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d,QID:0x%" PRIx64, - tmq->consumerId, vgId, pRspWrapper->dataRsp.common.reqOffset.version, buf, rspType, requestId); + PROCESS_POLL_RSP(tDecodeMqDataRsp, &pRspWrapper->pollRsp.dataRsp) } else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - if (tDecodeMqMetaRsp(&decoder, &pRspWrapper->metaRsp) < 0) { - tDecoderClear(&decoder); - code = TSDB_CODE_OUT_OF_MEMORY; - goto END; - } - tDecoderClear(&decoder); - (void)memcpy(&pRspWrapper->metaRsp, pMsg->pData, sizeof(SMqRspHead)); + PROCESS_POLL_RSP(tDecodeMqMetaRsp, &pRspWrapper->pollRsp.metaRsp) } else if (rspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - if (tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp) < 0) { - tDecoderClear(&decoder); - code = TSDB_CODE_OUT_OF_MEMORY; - goto END; - } - tDecoderClear(&decoder); - (void)memcpy(&pRspWrapper->taosxRsp, pMsg->pData, sizeof(SMqRspHead)); + PROCESS_POLL_RSP(tDecodeSTaosxRsp, &pRspWrapper->pollRsp.dataRsp) } else if (rspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - if (tSemiDecodeMqBatchMetaRsp(&decoder, &pRspWrapper->batchMetaRsp) < 0) { - tDecoderClear(&decoder); - code = TSDB_CODE_OUT_OF_MEMORY; - goto END; - } - tDecoderClear(&decoder); - (void)memcpy(&pRspWrapper->batchMetaRsp, pMsg->pData, sizeof(SMqRspHead)); - tscDebug("consumer:0x%" PRIx64 " recv poll batchmeta rsp, vgId:%d,QID:0x%" PRIx64, tmq->consumerId, vgId, - requestId); + PROCESS_POLL_RSP(tSemiDecodeMqBatchMetaRsp, &pRspWrapper->pollRsp.batchMetaRsp) } else { // invalid rspType - tscError("consumer:0x%" PRIx64 " invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType); + tqErrorC("consumer:0x%" PRIx64 " invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType); + code = TSDB_CODE_TSC_INTERNAL_ERROR; + goto END; } + pRspWrapper->tmqRspType = rspType; + pRspWrapper->pollRsp.reqId = requestId; + pRspWrapper->pollRsp.pEpset = pMsg->pEpSet; + pMsg->pEpSet = NULL; END: if (pRspWrapper) { pRspWrapper->code = code; - pRspWrapper->vgId = vgId; - (void)strcpy(pRspWrapper->topicName, pParam->topicName); + pRspWrapper->pollRsp.vgId = vgId; + tstrncpy(pRspWrapper->pollRsp.topicName, pParam->topicName, TSDB_TOPIC_FNAME_LEN); code = taosWriteQitem(tmq->mqueue, pRspWrapper); if (code != 0) { - tmqFreeRspWrapper((SMqRspWrapper*)pRspWrapper); + tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pRspWrapper); - tscError("consumer:0x%" PRIx64 " put poll res into mqueue failed, code:%d", tmq->consumerId, code); + tqErrorC("consumer:0x%" PRIx64 " put poll res into mqueue failed, code:%d", tmq->consumerId, code); + } else { + tqDebugC("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d,QID:0x%" PRIx64, + tmq ? tmq->consumerId : 0, rspType, vgId, taosQueueItemSize(tmq->mqueue), requestId); } } - int32_t total = taosQueueItemSize(tmq->mqueue); - tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d,QID:0x%" PRIx64, - tmq ? tmq->consumerId : 0, rspType, vgId, total, requestId); - if (tmq) { - if (tsem2_post(&tmq->rspSem) != 0){ - tscError("failed to post rsp sem, consumer:0x%" PRIx64, tmq->consumerId); - } + + if (tsem2_post(&tmq->rspSem) != 0){ + tqErrorC("failed to post rsp sem, consumer:0x%" PRIx64, tmq->consumerId); } - if (pMsg) taosMemoryFreeClear(pMsg->pData); - if (pMsg) taosMemoryFreeClear(pMsg->pEpSet); ret = taosReleaseRef(tmqMgmt.rsetId, refId); if (ret != 0){ - tscError("failed to release ref:%"PRId64 ", code:%d", refId, ret); + tqErrorC("failed to release ref:%"PRId64 ", code:%d", refId, ret); } +EXIT: + taosMemoryFreeClear(pMsg->pData); + taosMemoryFreeClear(pMsg->pEpSet); return code; } -typedef struct SVgroupSaveInfo { - STqOffsetVal currentOffset; - STqOffsetVal commitOffset; - STqOffsetVal seekOffset; - int64_t numOfRows; - int32_t vgStatus; -} SVgroupSaveInfo; - -static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopicEp, SHashObj* pVgOffsetHashMap, - tmq_t* tmq) { - pTopic->schema = pTopicEp->schema; - pTopicEp->schema.nCols = 0; - pTopicEp->schema.pSchema = NULL; - - char vgKey[TSDB_TOPIC_FNAME_LEN + 22] = {0}; - int32_t vgNumGet = taosArrayGetSize(pTopicEp->vgs); - - tstrncpy(pTopic->topicName, pTopicEp->topic, TSDB_TOPIC_FNAME_LEN); - tstrncpy(pTopic->db, pTopicEp->db, TSDB_DB_FNAME_LEN); - - tscInfo("consumer:0x%" PRIx64 ", update topic:%s, new numOfVgs:%d", tmq->consumerId, pTopic->topicName, vgNumGet); - pTopic->vgs = taosArrayInit(vgNumGet, sizeof(SMqClientVg)); - if (pTopic->vgs == NULL) { - tscError("consumer:0x%" PRIx64 ", failed to init vgs for topic:%s", tmq->consumerId, pTopic->topicName); - return; - } - for (int32_t j = 0; j < vgNumGet; j++) { - SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); - if (pVgEp == NULL) { - continue; - } - (void)sprintf(vgKey, "%s:%d", pTopic->topicName, pVgEp->vgId); - SVgroupSaveInfo* pInfo = taosHashGet(pVgOffsetHashMap, vgKey, strlen(vgKey)); - - STqOffsetVal offsetNew = {0}; - offsetNew.type = tmq->resetOffsetCfg; - - tscInfo("consumer:0x%" PRIx64 ", update topic:%s, new numOfVgs:%d, num:%d, port:%d", tmq->consumerId, - pTopic->topicName, vgNumGet, pVgEp->epSet.numOfEps, pVgEp->epSet.eps[pVgEp->epSet.inUse].port); - - SMqClientVg clientVg = { - .pollCnt = 0, - .vgId = pVgEp->vgId, - .epSet = pVgEp->epSet, - .vgStatus = pInfo ? pInfo->vgStatus : TMQ_VG_STATUS__IDLE, - .vgSkipCnt = 0, - .emptyBlockReceiveTs = 0, - .blockReceiveTs = 0, - .blockSleepForReplay = 0, - .numOfRows = pInfo ? pInfo->numOfRows : 0, - }; - - clientVg.offsetInfo.walVerBegin = -1; - clientVg.offsetInfo.walVerEnd = -1; - clientVg.seekUpdated = false; - if (pInfo) { - tOffsetCopy(&clientVg.offsetInfo.endOffset, &pInfo->currentOffset); - tOffsetCopy(&clientVg.offsetInfo.committedOffset, &pInfo->commitOffset); - tOffsetCopy(&clientVg.offsetInfo.beginOffset, &pInfo->seekOffset); - } else { - clientVg.offsetInfo.endOffset = offsetNew; - clientVg.offsetInfo.committedOffset = offsetNew; - clientVg.offsetInfo.beginOffset = offsetNew; - } - if (taosArrayPush(pTopic->vgs, &clientVg) == NULL) { - tscError("consumer:0x%" PRIx64 ", failed to push vg:%d into topic:%s", tmq->consumerId, pVgEp->vgId, - pTopic->topicName); - freeClientVg(&clientVg); - } - } -} - -static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { - bool set = false; - - int32_t topicNumGet = taosArrayGetSize(pRsp->topics); - if (epoch < tmq->epoch || (epoch == tmq->epoch && topicNumGet == 0)) { - tscDebug("consumer:0x%" PRIx64 " no update ep epoch from %d to epoch %d, incoming topics:%d", tmq->consumerId, - tmq->epoch, epoch, topicNumGet); - if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER) { - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY); - } - return false; - } - - SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic)); - if (newTopics == NULL) { - return false; - } - - SHashObj* pVgOffsetHashMap = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); - if (pVgOffsetHashMap == NULL) { - taosArrayDestroy(newTopics); - return false; - } - - taosWLockLatch(&tmq->lock); - int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); - - char vgKey[TSDB_TOPIC_FNAME_LEN + 22] = {0}; - tscInfo("consumer:0x%" PRIx64 " update ep epoch from %d to epoch %d, incoming topics:%d, existed topics:%d", - tmq->consumerId, tmq->epoch, epoch, topicNumGet, topicNumCur); - for (int32_t i = 0; i < topicNumCur; i++) { - // find old topic - SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); - if (pTopicCur && pTopicCur->vgs) { - int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); - tscInfo("consumer:0x%" PRIx64 ", current vg num: %d", tmq->consumerId, vgNumCur); - for (int32_t j = 0; j < vgNumCur; j++) { - SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); - if (pVgCur == NULL) { - continue; - } - (void)sprintf(vgKey, "%s:%d", pTopicCur->topicName, pVgCur->vgId); - - char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &pVgCur->offsetInfo.endOffset); - tscInfo("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch, pVgCur->vgId, - vgKey, buf); - - SVgroupSaveInfo info = {.currentOffset = pVgCur->offsetInfo.endOffset, - .seekOffset = pVgCur->offsetInfo.beginOffset, - .commitOffset = pVgCur->offsetInfo.committedOffset, - .numOfRows = pVgCur->numOfRows, - .vgStatus = pVgCur->vgStatus}; - if (taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)) != 0) { - tscError("consumer:0x%" PRIx64 ", failed to put vg:%d into hashmap", tmq->consumerId, pVgCur->vgId); - } - } - } - } - - for (int32_t i = 0; i < topicNumGet; i++) { - SMqClientTopic topic = {0}; - SMqSubTopicEp* pTopicEp = taosArrayGet(pRsp->topics, i); - if (pTopicEp == NULL) { - continue; - } - initClientTopicFromRsp(&topic, pTopicEp, pVgOffsetHashMap, tmq); - if (taosArrayPush(newTopics, &topic) == NULL) { - tscError("consumer:0x%" PRIx64 ", failed to push topic:%s into new topics", tmq->consumerId, topic.topicName); - freeClientTopic(&topic); - } - } - - taosHashCleanup(pVgOffsetHashMap); - - // destroy current buffered existed topics info - if (tmq->clientTopics) { - taosArrayDestroyEx(tmq->clientTopics, freeClientTopic); - } - tmq->clientTopics = newTopics; - taosWUnLockLatch(&tmq->lock); - - int8_t flag = (topicNumGet == 0) ? TMQ_CONSUMER_STATUS__NO_TOPIC : TMQ_CONSUMER_STATUS__READY; - atomic_store_8(&tmq->status, flag); - atomic_store_32(&tmq->epoch, epoch); - - tscInfo("consumer:0x%" PRIx64 " update topic info completed", tmq->consumerId); - return set; -} - void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) { (void)snprintf(pReq->subKey, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", tmq->groupId, TMQ_SEPARATOR, pTopic->topicName); pReq->withTbName = tmq->withTbName; @@ -1908,35 +2035,6 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl pReq->enableBatchMeta = tmq->enableBatchMeta; } -void tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqMetaRspObj** ppRspObj) { - SMqMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqMetaRspObj)); - if (pRspObj == NULL) { - return; - } - pRspObj->resType = RES_TYPE__TMQ_META; - tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); - tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); - pRspObj->vgId = pWrapper->vgHandle->vgId; - - (void)memcpy(&pRspObj->metaRsp, &pWrapper->metaRsp, sizeof(SMqMetaRsp)); - *ppRspObj = pRspObj; -} - -void tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqBatchMetaRspObj** ppRspObj) { - SMqBatchMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqBatchMetaRspObj)); - if (pRspObj == NULL) { - return; - } - pRspObj->common.resType = RES_TYPE__TMQ_BATCH_META; - tstrncpy(pRspObj->common.topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); - tstrncpy(pRspObj->common.db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); - pRspObj->common.vgId = pWrapper->vgHandle->vgId; - - (void)memcpy(&pRspObj->rsp, &pWrapper->batchMetaRsp, sizeof(SMqBatchMetaRsp)); - tscDebug("build batchmeta Rsp from wrapper"); - *ppRspObj = pRspObj; -} - void changeByteEndian(char* pData) { if (pData == NULL) { return; @@ -1947,7 +2045,7 @@ void changeByteEndian(char* pData) { // length | version: int32_t blockVersion = *(int32_t*)p; if (blockVersion != BLOCK_VERSION_1) { - tscError("invalid block version:%d", blockVersion); + tqErrorC("invalid block version:%d", blockVersion); return; } *(int32_t*)p = BLOCK_VERSION_2; @@ -1988,23 +2086,18 @@ static void tmqGetRawDataRowsPrecisionFromRes(void* pRetrieve, void** rawData, i } static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, - SMqRspObjCommon* pRspObj, SMqDataRspCommon* pDataRsp) { - (*numOfRows) = 0; - tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); - tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); - - pRspObj->vgId = pWrapper->vgHandle->vgId; + SMqRspObj* pRspObj) { pRspObj->resIter = -1; - pRspObj->resInfo.totalRows = 0; pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; + SMqDataRsp* pDataRsp = &pRspObj->dataRsp; bool needTransformSchema = !pDataRsp->withSchema; if (!pDataRsp->withSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable pDataRsp->withSchema = true; pDataRsp->blockSchema = taosArrayInit(pDataRsp->blockNum, sizeof(void*)); if (pDataRsp->blockSchema == NULL) { - tscError("failed to allocate memory for blockSchema"); + tqErrorC("failed to allocate memory for blockSchema"); return; } } @@ -2023,7 +2116,7 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg SSchemaWrapper* schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema); if (schema) { if (taosArrayPush(pDataRsp->blockSchema, &schema) == NULL) { - tscError("failed to push schema into blockSchema"); + tqErrorC("failed to push schema into blockSchema"); continue; } } @@ -2031,31 +2124,6 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg } } -void tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, - SMqRspObj** ppRspObj) { - SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); - if (pRspObj == NULL) { - return; - } - pRspObj->common.resType = RES_TYPE__TMQ; - (void)memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp)); - tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, &pRspObj->common, &pRspObj->rsp.common); - *ppRspObj = pRspObj; -} - -void tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, - SMqTaosxRspObj** ppRspObj) { - SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj)); - if (pRspObj == NULL) { - return; - } - pRspObj->common.resType = RES_TYPE__TMQ_METADATA; - (void)memcpy(&pRspObj->rsp, &pWrapper->taosxRsp, sizeof(STaosxRsp)); - - tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, &pRspObj->common, &pRspObj->rsp.common); - *ppRspObj = pRspObj; -} - static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* pVg, int64_t timeout) { SMqPollReq req = {0}; char* msg = NULL; @@ -2089,7 +2157,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p } pParam->refId = pTmq->refId; - (void)strcpy(pParam->topicName, pTopic->topicName); + tstrncpy(pParam->topicName, pTopic->topicName, TSDB_TOPIC_FNAME_LEN); pParam->vgId = pVg->vgId; pParam->requestId = req.reqId; @@ -2108,11 +2176,10 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p sendInfo->fp = tmqPollCb; sendInfo->msgType = TDMT_VND_TMQ_CONSUME; - // int64_t transporterId = 0; char offsetFormatBuf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->offsetInfo.endOffset); code = asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, NULL, sendInfo); - tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, code:%d, epoch %d, req:%s,QID:0x%" PRIx64, pTmq->consumerId, + tqDebugC("consumer:0x%" PRIx64 " send poll to %s vgId:%d, code:%d, epoch %d, req:%s,QID:0x%" PRIx64, pTmq->consumerId, pTopic->topicName, pVg->vgId, code, pTmq->epoch, offsetFormatBuf, req.reqId); if (code != 0) { return code; @@ -2127,14 +2194,11 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p // broadcast the poll request to all related vnodes static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { - if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER) { - return 0; - } int32_t code = 0; taosWLockLatch(&tmq->lock); int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); - tscDebug("consumer:0x%" PRIx64 " start to poll data, numOfTopics:%d", tmq->consumerId, numOfTopics); + tqDebugC("consumer:0x%" PRIx64 " start to poll data, numOfTopics:%d", tmq->consumerId, numOfTopics); for (int i = 0; i < numOfTopics; i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); @@ -2143,7 +2207,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { } int32_t numOfVg = taosArrayGetSize(pTopic->vgs); if (pTopic->noPrivilege) { - tscDebug("consumer:0x%" PRIx64 " has no privilegr for topic:%s", tmq->consumerId, pTopic->topicName); + tqDebugC("consumer:0x%" PRIx64 " has no privilegr for topic:%s", tmq->consumerId, pTopic->topicName); continue; } for (int j = 0; j < numOfVg; j++) { @@ -2153,14 +2217,14 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { } int64_t elapsed = taosGetTimestampMs() - pVg->emptyBlockReceiveTs; if (elapsed < EMPTY_BLOCK_POLL_IDLE_DURATION && elapsed >= 0) { // less than 10ms - tscDebug("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId, + tqDebugC("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId, tmq->epoch, pVg->vgId); continue; } elapsed = taosGetTimestampMs() - pVg->blockReceiveTs; if (tmq->replayEnable && elapsed < pVg->blockSleepForReplay && elapsed >= 0) { - tscDebug("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for %" PRId64 "ms before start next poll when replay", + tqDebugC("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for %" PRId64 "ms before start next poll when replay", tmq->consumerId, tmq->epoch, pVg->vgId, pVg->blockSleepForReplay); continue; } @@ -2168,7 +2232,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); if (vgStatus == TMQ_VG_STATUS__WAIT) { int32_t vgSkipCnt = atomic_add_fetch_32(&pVg->vgSkipCnt, 1); - tscDebug("consumer:0x%" PRIx64 " epoch %d wait poll-rsp, skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch, + tqDebugC("consumer:0x%" PRIx64 " epoch %d wait poll-rsp, skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch, pVg->vgId, vgSkipCnt); continue; } @@ -2183,20 +2247,20 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { end: taosWUnLockLatch(&tmq->lock); - tscDebug("consumer:0x%" PRIx64 " end to poll data, code:%d", tmq->consumerId, code); + tqDebugC("consumer:0x%" PRIx64 " end to poll data, code:%d", tmq->consumerId, code); return code; } static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* reqOffset, STqOffsetVal* rspOffset, int64_t sver, int64_t ever, int64_t consumerId, bool hasData) { if (!pVg->seekUpdated) { - tscDebug("consumer:0x%" PRIx64 " local offset is update, since seekupdate not set", consumerId); + tqDebugC("consumer:0x%" PRIx64 " local offset is update, since seekupdate not set", consumerId); if (hasData) { tOffsetCopy(&pVg->offsetInfo.beginOffset, reqOffset); } tOffsetCopy(&pVg->offsetInfo.endOffset, rspOffset); } else { - tscDebug("consumer:0x%" PRIx64 " local offset is NOT update, since seekupdate is set", consumerId); + tqDebugC("consumer:0x%" PRIx64 " local offset is NOT update, since seekupdate is set", consumerId); } // update the status @@ -2207,9 +2271,134 @@ static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* reqOffset, STqOffsetVal pVg->offsetInfo.walVerEnd = ever + 1; } -static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { - tscDebug("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, taosQallItemSize(tmq->qall)); +static SMqRspObj* buildRsp(SMqPollRspWrapper* pollRspWrapper){ + typedef union { + SMqDataRsp dataRsp; + SMqMetaRsp metaRsp; + SMqBatchMetaRsp batchMetaRsp; + } MEMSIZE; + SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); + if (pRspObj == NULL) { + tqErrorC("buildRsp:failed to allocate memory"); + return NULL; + } + (void)memcpy(&pRspObj->dataRsp, &pollRspWrapper->dataRsp, sizeof(MEMSIZE)); + tstrncpy(pRspObj->topic, pollRspWrapper->topicName, TSDB_TOPIC_FNAME_LEN); + tstrncpy(pRspObj->db, pollRspWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); + pRspObj->vgId = pollRspWrapper->vgId; + (void)memset(&pollRspWrapper->dataRsp, 0, sizeof(MEMSIZE)); + return pRspObj; +} + +static void processMqRspError(tmq_t* tmq, SMqRspWrapper* pRspWrapper){ + SMqPollRspWrapper* pollRspWrapper = &pRspWrapper->pollRsp; + + if (pRspWrapper->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { // for vnode transform + int32_t code = askEp(tmq, NULL, false, true); + if (code != 0) { + tqErrorC("consumer:0x%" PRIx64 " failed to ask ep, code:%s", tmq->consumerId, tstrerror(code)); + } + } else if (pRspWrapper->code == TSDB_CODE_TMQ_CONSUMER_MISMATCH) { + int32_t code = askEp(tmq, NULL, false, false); + if (code != 0) { + tqErrorC("consumer:0x%" PRIx64 " failed to ask ep, code:%s", tmq->consumerId, tstrerror(code)); + } + } + tqInfoC("consumer:0x%" PRIx64 " msg from vgId:%d discarded, since %s", tmq->consumerId, pollRspWrapper->vgId, + tstrerror(pRspWrapper->code)); + taosWLockLatch(&tmq->lock); + SMqClientVg* pVg = NULL; + getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId, &pVg); + if (pVg) { + pVg->emptyBlockReceiveTs = taosGetTimestampMs(); + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + } + taosWUnLockLatch(&tmq->lock); +} +static SMqRspObj* processMqRsp(tmq_t* tmq, SMqRspWrapper* pRspWrapper){ + SMqRspObj* pRspObj = NULL; + + if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) { + tqDebugC("consumer:0x%" PRIx64 " ep msg received", tmq->consumerId); + SMqAskEpRsp* rspMsg = &pRspWrapper->epRsp; + doUpdateLocalEp(tmq, pRspWrapper->epoch, rspMsg); + return pRspObj; + } + + SMqPollRspWrapper* pollRspWrapper = &pRspWrapper->pollRsp; + taosWLockLatch(&tmq->lock); + SMqClientVg* pVg = NULL; + getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId, &pVg); + if(pVg == NULL) { + tqErrorC("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, + pollRspWrapper->topicName, pollRspWrapper->vgId); + goto END; + } + pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); + if (pollRspWrapper->pEpset != NULL) { + pVg->epSet = *pollRspWrapper->pEpset; + } + + if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP || pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { + updateVgInfo(pVg, &pollRspWrapper->dataRsp.reqOffset, &pollRspWrapper->dataRsp.rspOffset, pollRspWrapper->head.walsver, pollRspWrapper->head.walever, + tmq->consumerId, pollRspWrapper->dataRsp.blockNum != 0); + + char buf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(buf, TSDB_OFFSET_LEN, &pollRspWrapper->rspOffset); + if (pollRspWrapper->dataRsp.blockNum == 0) { + tqDebugC("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, vg total:%" PRId64 + ", total:%" PRId64 ",QID:0x%" PRIx64, + tmq->consumerId, pVg->vgId, buf, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId); + pVg->emptyBlockReceiveTs = taosGetTimestampMs(); + } else { + pRspObj = buildRsp(pollRspWrapper); + if (pRspObj == NULL) { + tqErrorC("consumer:0x%" PRIx64 " failed to allocate memory for meta rsp", tmq->consumerId); + goto END; + } + pRspObj->resType = pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP ? RES_TYPE__TMQ : RES_TYPE__TMQ_METADATA; + int64_t numOfRows = 0; + tmqBuildRspFromWrapperInner(pollRspWrapper, pVg, &numOfRows, pRspObj); + tmq->totalRows += numOfRows; + pVg->emptyBlockReceiveTs = 0; + if (tmq->replayEnable) { + pVg->blockReceiveTs = taosGetTimestampMs(); + pVg->blockSleepForReplay = pRspObj->dataRsp.sleepTime; + if (pVg->blockSleepForReplay > 0) { + if (taosTmrStart(tmqReplayTask, pVg->blockSleepForReplay, (void*)(tmq->refId), tmqMgmt.timer) == NULL) { + tqErrorC("consumer:0x%" PRIx64 " failed to start replay timer, vgId:%d, sleep:%" PRId64, + tmq->consumerId, pVg->vgId, pVg->blockSleepForReplay); + } + } + } + tqDebugC("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 + ", vg total:%" PRId64 ", total:%" PRId64 ",QID:0x%" PRIx64, + tmq->consumerId, pVg->vgId, buf, pRspObj->dataRsp.blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, + pollRspWrapper->reqId); + } + } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP || pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { + updateVgInfo(pVg, &pollRspWrapper->rspOffset, &pollRspWrapper->rspOffset, + pollRspWrapper->head.walsver, pollRspWrapper->head.walever, tmq->consumerId, true); + + + pRspObj = buildRsp(pollRspWrapper); + if (pRspObj == NULL) { + tqErrorC("consumer:0x%" PRIx64 " failed to allocate memory for meta rsp", tmq->consumerId); + goto END; + } + pRspObj->resType = pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP ? RES_TYPE__TMQ_META : RES_TYPE__TMQ_BATCH_META; + } + + END: + taosWUnLockLatch(&tmq->lock); + return pRspObj; +} + +static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { + tqDebugC("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, taosQallItemSize(tmq->qall)); + + void* returnVal = NULL; while (1) { SMqRspWrapper* pRspWrapper = NULL; if (taosGetQitem(tmq->qall, (void**)&pRspWrapper) == 0) { @@ -2221,253 +2410,20 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { } } - tscDebug("consumer:0x%" PRIx64 " handle rsp, type:%d", tmq->consumerId, pRspWrapper->tmqRspType); - + tqDebugC("consumer:0x%" PRIx64 " handle rsp, type:%d", tmq->consumerId, pRspWrapper->tmqRspType); if (pRspWrapper->code != 0) { - SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; - if (pRspWrapper->code == TSDB_CODE_TMQ_CONSUMER_MISMATCH) { - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__RECOVER); - tscDebug("consumer:0x%" PRIx64 " wait for the rebalance, set status to be RECOVER", tmq->consumerId); - } else if (pRspWrapper->code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { - tscInfo("consumer:0x%" PRIx64 " return null since no committed offset", tmq->consumerId); - } else { - if (pRspWrapper->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { // for vnode transform - int32_t code = askEp(tmq, NULL, false, true); - if (code != 0) { - tscError("consumer:0x%" PRIx64 " failed to ask ep, code:%s", tmq->consumerId, tstrerror(code)); - } - } - tscError("consumer:0x%" PRIx64 " msg from vgId:%d discarded, since %s", tmq->consumerId, pollRspWrapper->vgId, - tstrerror(pRspWrapper->code)); - taosWLockLatch(&tmq->lock); - SMqClientVg* pVg = NULL; - getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId, &pVg); - if (pVg) pVg->emptyBlockReceiveTs = taosGetTimestampMs(); - taosWUnLockLatch(&tmq->lock); - } - setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); - taosMemoryFreeClear(pollRspWrapper->pEpset); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { - SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; - - int32_t consumerEpoch = atomic_load_32(&tmq->epoch); - SMqDataRspCommon* pDataRsp = (SMqDataRspCommon*)&pollRspWrapper->dataRsp; - - if (pDataRsp->head.epoch == consumerEpoch) { - taosWLockLatch(&tmq->lock); - SMqClientVg* pVg = NULL; - getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId, &pVg); - pollRspWrapper->vgHandle = pVg; - pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); - if (pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL) { - tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, - pollRspWrapper->topicName, pollRspWrapper->vgId); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - taosWUnLockLatch(&tmq->lock); - return NULL; - } - // update the epset - if (pollRspWrapper->pEpset != NULL) { - SEp* pEp = GET_ACTIVE_EP(pollRspWrapper->pEpset); - SEp* pOld = GET_ACTIVE_EP(&(pVg->epSet)); - tscDebug("consumer:0x%" PRIx64 " update epset vgId:%d, ep:%s:%d, old ep:%s:%d", tmq->consumerId, pVg->vgId, - pEp->fqdn, pEp->port, pOld->fqdn, pOld->port); - pVg->epSet = *pollRspWrapper->pEpset; - } - - updateVgInfo(pVg, &pDataRsp->reqOffset, &pDataRsp->rspOffset, pDataRsp->head.walsver, pDataRsp->head.walever, - tmq->consumerId, pDataRsp->blockNum != 0); - - char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &pDataRsp->rspOffset); - if (pDataRsp->blockNum == 0) { - tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, vg total:%" PRId64 - ", total:%" PRId64 ",QID:0x%" PRIx64, - tmq->consumerId, pVg->vgId, buf, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId); - pVg->emptyBlockReceiveTs = taosGetTimestampMs(); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - taosWUnLockLatch(&tmq->lock); - } else { // build rsp - int64_t numOfRows = 0; - SMqRspObj* pRsp = NULL; - tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp); - tmq->totalRows += numOfRows; - pVg->emptyBlockReceiveTs = 0; - if (pRsp && tmq->replayEnable) { - pVg->blockReceiveTs = taosGetTimestampMs(); - pVg->blockSleepForReplay = pRsp->rsp.sleepTime; - if (pVg->blockSleepForReplay > 0) { - if (taosTmrStart(tmqReplayTask, pVg->blockSleepForReplay, (void*)(tmq->refId), tmqMgmt.timer) == NULL) { - tscError("consumer:0x%" PRIx64 " failed to start replay timer, vgId:%d, sleep:%" PRId64, - tmq->consumerId, pVg->vgId, pVg->blockSleepForReplay); - } - } - } - tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 - ", vg total:%" PRId64 ", total:%" PRId64 ",QID:0x%" PRIx64, - tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, - pollRspWrapper->reqId); - taosMemoryFreeClear(pollRspWrapper->pEpset); - taosFreeQitem(pRspWrapper); - taosWUnLockLatch(&tmq->lock); - return pRsp; - } - } else { - tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", - tmq->consumerId, pollRspWrapper->vgId, pDataRsp->head.epoch, consumerEpoch); - setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - } - } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) { - SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; - int32_t consumerEpoch = atomic_load_32(&tmq->epoch); - - tscDebug("consumer:0x%" PRIx64 " process meta rsp", tmq->consumerId); - - if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) { - taosWLockLatch(&tmq->lock); - SMqClientVg* pVg = NULL; - getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId, &pVg); - pollRspWrapper->vgHandle = pVg; - pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); - if (pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL) { - tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, - pollRspWrapper->topicName, pollRspWrapper->vgId); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - taosWUnLockLatch(&tmq->lock); - return NULL; - } - - updateVgInfo(pVg, &pollRspWrapper->metaRsp.rspOffset, &pollRspWrapper->metaRsp.rspOffset, - pollRspWrapper->metaRsp.head.walsver, pollRspWrapper->metaRsp.head.walever, tmq->consumerId, true); - // build rsp - SMqMetaRspObj* pRsp = NULL; - tmqBuildMetaRspFromWrapper(pollRspWrapper, &pRsp); - taosMemoryFreeClear(pollRspWrapper->pEpset); - taosFreeQitem(pRspWrapper); - taosWUnLockLatch(&tmq->lock); - return pRsp; - } else { - tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", - tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->metaRsp.head.epoch, consumerEpoch); - setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - } - } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { - SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; - int32_t consumerEpoch = atomic_load_32(&tmq->epoch); - - tscDebug("consumer:0x%" PRIx64 " process meta rsp", tmq->consumerId); - - if (pollRspWrapper->batchMetaRsp.head.epoch == consumerEpoch) { - taosWLockLatch(&tmq->lock); - SMqClientVg* pVg = NULL; - getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId, &pVg); - pollRspWrapper->vgHandle = pVg; - pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); - if (pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL) { - tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, - pollRspWrapper->topicName, pollRspWrapper->vgId); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - taosWUnLockLatch(&tmq->lock); - return NULL; - } - - // build rsp - updateVgInfo(pVg, &pollRspWrapper->batchMetaRsp.rspOffset, &pollRspWrapper->batchMetaRsp.rspOffset, - pollRspWrapper->batchMetaRsp.head.walsver, pollRspWrapper->batchMetaRsp.head.walever, - tmq->consumerId, true); - SMqBatchMetaRspObj* pRsp = NULL; - tmqBuildBatchMetaRspFromWrapper(pollRspWrapper, &pRsp); - taosMemoryFreeClear(pollRspWrapper->pEpset); - taosFreeQitem(pRspWrapper); - taosWUnLockLatch(&tmq->lock); - return pRsp; - } else { - tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", - tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->batchMetaRsp.head.epoch, consumerEpoch); - setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - } - } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { - SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; - int32_t consumerEpoch = atomic_load_32(&tmq->epoch); - SMqDataRspCommon* pDataRsp = (SMqDataRspCommon*)&pollRspWrapper->taosxRsp; - - if (pDataRsp->head.epoch == consumerEpoch) { - taosWLockLatch(&tmq->lock); - SMqClientVg* pVg = NULL; - getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId, &pVg); - pollRspWrapper->vgHandle = pVg; - pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); - if (pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL) { - tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, - pollRspWrapper->topicName, pollRspWrapper->vgId); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - taosWUnLockLatch(&tmq->lock); - return NULL; - } - - updateVgInfo(pVg, &pDataRsp->reqOffset, &pDataRsp->rspOffset, pDataRsp->head.walsver, pDataRsp->head.walever, - tmq->consumerId, pDataRsp->blockNum != 0); - - if (pDataRsp->blockNum == 0) { - tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ",QID:0x%" PRIx64, - tmq->consumerId, pVg->vgId, pVg->numOfRows, pollRspWrapper->reqId); - pVg->emptyBlockReceiveTs = taosGetTimestampMs(); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - taosWUnLockLatch(&tmq->lock); - continue; - } else { - pVg->emptyBlockReceiveTs = 0; // reset the ts - } - - // build rsp - int64_t numOfRows = 0; - SMqTaosxRspObj* pRsp = NULL; - tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp); - tmq->totalRows += numOfRows; - - char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.endOffset); - tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 - ", vg total:%" PRId64 ", total:%" PRId64 ",QID:0x%" PRIx64, - tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, - pollRspWrapper->reqId); - - taosMemoryFreeClear(pollRspWrapper->pEpset); - taosFreeQitem(pRspWrapper); - taosWUnLockLatch(&tmq->lock); - return pRsp; - } else { - tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", - tmq->consumerId, pollRspWrapper->vgId, pDataRsp->head.epoch, consumerEpoch); - setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - } - } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) { - tscDebug("consumer:0x%" PRIx64 " ep msg received", tmq->consumerId); - SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)pRspWrapper; - SMqAskEpRsp* rspMsg = &pEpRspWrapper->msg; - (void)doUpdateLocalEp(tmq, pEpRspWrapper->epoch, rspMsg); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - } else { - tscError("consumer:0x%" PRIx64 " invalid msg received:%d", tmq->consumerId, pRspWrapper->tmqRspType); + processMqRspError(tmq, pRspWrapper); + }else{ + returnVal = processMqRsp(tmq, pRspWrapper); + } + tmqFreeRspWrapper(pRspWrapper); + taosFreeQitem(pRspWrapper); + if(returnVal != NULL){ + break; } } + + return returnVal; } TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { @@ -2476,45 +2432,28 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { void* rspObj = NULL; int64_t startTime = taosGetTimestampMs(); - tscDebug("consumer:0x%" PRIx64 " start to poll at %" PRId64 ", timeout:%" PRId64, tmq->consumerId, startTime, + tqDebugC("consumer:0x%" PRIx64 " start to poll at %" PRId64 ", timeout:%" PRId64, tmq->consumerId, startTime, timeout); // in no topic status, delayed task also need to be processed if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__INIT) { - tscInfo("consumer:0x%" PRIx64 " poll return since consumer is init", tmq->consumerId); + tqInfoC("consumer:0x%" PRIx64 " poll return since consumer is init", tmq->consumerId); taosMsleep(500); // sleep for a while return NULL; } - while (1) { - if (atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__RECOVER) { - break; - } - tscInfo("consumer:0x%" PRIx64 " tmq status is recover", tmq->consumerId); - - int32_t retryCnt = 0; - while (syncAskEp(tmq) != 0) { - if (retryCnt++ > 40) { - return NULL; - } - - tscInfo("consumer:0x%" PRIx64 " not ready, retry:%d/40 in 500ms", tmq->consumerId, retryCnt); - taosMsleep(500); - } - } - - (void)atomic_val_compare_exchange_8(&pollFlag, 0, 1); + (void)atomic_val_compare_exchange_8(&tmq->pollFlag, 0, 1); while (1) { tmqHandleAllDelayedTask(tmq); if (tmqPollImpl(tmq, timeout) < 0) { - tscError("consumer:0x%" PRIx64 " return due to poll error", tmq->consumerId); + tqErrorC("consumer:0x%" PRIx64 " return due to poll error", tmq->consumerId); } rspObj = tmqHandleAllRsp(tmq, timeout); if (rspObj) { - tscDebug("consumer:0x%" PRIx64 " return rsp %p", tmq->consumerId, rspObj); + tqDebugC("consumer:0x%" PRIx64 " return rsp %p", tmq->consumerId, rspObj); return (TAOS_RES*)rspObj; } @@ -2522,7 +2461,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { int64_t currentTime = taosGetTimestampMs(); int64_t elapsedTime = currentTime - startTime; if (elapsedTime > timeout || elapsedTime < 0) { - tscDebug("consumer:0x%" PRIx64 " (epoch %d) timeout, no rsp, start time %" PRId64 ", current time %" PRId64, + tqDebugC("consumer:0x%" PRIx64 " (epoch %d) timeout, no rsp, start time %" PRId64 ", current time %" PRId64, tmq->consumerId, tmq->epoch, startTime, currentTime); return NULL; } @@ -2536,77 +2475,68 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { static void displayConsumeStatistics(tmq_t* pTmq) { taosRLockLatch(&pTmq->lock); int32_t numOfTopics = taosArrayGetSize(pTmq->clientTopics); - tscDebug("consumer:0x%" PRIx64 " closing poll:%" PRId64 " rows:%" PRId64 " topics:%d, final epoch:%d", + tqInfoC("consumer:0x%" PRIx64 " closing poll:%" PRId64 " rows:%" PRId64 " topics:%d, final epoch:%d", pTmq->consumerId, pTmq->pollCnt, pTmq->totalRows, numOfTopics, pTmq->epoch); - tscDebug("consumer:0x%" PRIx64 " rows dist begin: ", pTmq->consumerId); + tqInfoC("consumer:0x%" PRIx64 " rows dist begin: ", pTmq->consumerId); for (int32_t i = 0; i < numOfTopics; ++i) { SMqClientTopic* pTopics = taosArrayGet(pTmq->clientTopics, i); if (pTopics == NULL) continue; - tscDebug("consumer:0x%" PRIx64 " topic:%d", pTmq->consumerId, i); + tqInfoC("consumer:0x%" PRIx64 " topic:%d", pTmq->consumerId, i); int32_t numOfVgs = taosArrayGetSize(pTopics->vgs); for (int32_t j = 0; j < numOfVgs; ++j) { SMqClientVg* pVg = taosArrayGet(pTopics->vgs, j); if (pVg == NULL) continue; - tscDebug("topic:%s, %d. vgId:%d rows:%" PRId64, pTopics->topicName, j, pVg->vgId, pVg->numOfRows); + tqInfoC("topic:%s, %d. vgId:%d rows:%" PRId64, pTopics->topicName, j, pVg->vgId, pVg->numOfRows); } } taosRUnLockLatch(&pTmq->lock); - tscDebug("consumer:0x%" PRIx64 " rows dist end", pTmq->consumerId); + tqInfoC("consumer:0x%" PRIx64 " rows dist end", pTmq->consumerId); } -static int32_t innerClose(tmq_t* tmq) { - if (tmq->status != TMQ_CONSUMER_STATUS__READY) { - tscInfo("consumer:0x%" PRIx64 " not in ready state, unsubscribe it directly", tmq->consumerId); - return 0; +int32_t tmq_unsubscribe(tmq_t* tmq) { + if (tmq == NULL) return TSDB_CODE_INVALID_PARA; + int32_t code = 0; + int8_t status = atomic_load_8(&tmq->status); + tqInfoC("consumer:0x%" PRIx64 " start to unsubscribe consumer, status:%d", tmq->consumerId, status); + + displayConsumeStatistics(tmq); + if (status != TMQ_CONSUMER_STATUS__READY) { + tqInfoC("consumer:0x%" PRIx64 " status:%d, already closed or not in ready state, no need unsubscribe", tmq->consumerId, status); + goto END; } if (tmq->autoCommit) { - int32_t code = tmq_commit_sync(tmq, NULL); + code = tmq_commit_sync(tmq, NULL); if (code != 0) { - return code; + goto END; } } tmqSendHbReq((void*)(tmq->refId), NULL); tmq_list_t* lst = tmq_list_new(); if (lst == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto END; } - int32_t code = tmq_subscribe(tmq, lst); + code = tmq_subscribe(tmq, lst); tmq_list_destroy(lst); - return code; -} -int32_t tmq_unsubscribe(tmq_t* tmq) { - if (tmq == NULL) return TSDB_CODE_INVALID_PARA; - - tscInfo("consumer:0x%" PRIx64 " start to unsubscribe consumer, status:%d", tmq->consumerId, tmq->status); - int32_t code = 0; - if (atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__CLOSED) { - code = innerClose(tmq); - if (code == 0) { - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__CLOSED); - } + if(code != 0){ + goto END; } + +END: return code; } int32_t tmq_consumer_close(tmq_t* tmq) { if (tmq == NULL) return TSDB_CODE_INVALID_PARA; - - tscInfo("consumer:0x%" PRIx64 " start to close consumer, status:%d", tmq->consumerId, tmq->status); - displayConsumeStatistics(tmq); - int32_t code = 0; - if (atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__CLOSED) { - code = innerClose(tmq); - if (code == 0) { - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__CLOSED); - } - } - + tqInfoC("consumer:0x%" PRIx64 " start to close consumer, status:%d", tmq->consumerId, tmq->status); + int32_t code = tmq_unsubscribe(tmq); if (code == 0) { + atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__CLOSED); code = taosRemoveRef(tmqMgmt.rsetId, tmq->refId); if (code != 0){ - tscError("tmq close failed to remove ref:%" PRId64 ", code:%d", tmq->refId, code); + tqErrorC("tmq close failed to remove ref:%" PRId64 ", code:%d", tmq->refId, code); } } return code; @@ -2648,14 +2578,9 @@ const char* tmq_get_topic_name(TAOS_RES* res) { if (res == NULL) { return NULL; } - if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { - char* tmp = strchr(((SMqRspObjCommon*)res)->topic, '.'); - if (tmp == NULL) { - return NULL; - } - return tmp + 1; - } else if (TD_RES_TMQ_META(res)) { - char* tmp = strchr(((SMqMetaRspObj*)res)->topic, '.'); + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || + TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + char* tmp = strchr(((SMqRspObj*)res)->topic, '.'); if (tmp == NULL) { return NULL; } @@ -2670,14 +2595,9 @@ const char* tmq_get_db_name(TAOS_RES* res) { return NULL; } - if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { - char* tmp = strchr(((SMqRspObjCommon*)res)->db, '.'); - if (tmp == NULL) { - return NULL; - } - return tmp + 1; - } else if (TD_RES_TMQ_META(res)) { - char* tmp = strchr(((SMqMetaRspObj*)res)->db, '.'); + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || + TD_RES_TMQ_BATCH_META(res) || TD_RES_TMQ_META(res)) { + char* tmp = strchr(((SMqRspObj*)res)->db, '.'); if (tmp == NULL) { return NULL; } @@ -2691,10 +2611,9 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) { if (res == NULL) { return TSDB_CODE_INVALID_PARA; } - if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { - return ((SMqRspObjCommon*)res)->vgId; - } else if (TD_RES_TMQ_META(res)) { - return ((SMqMetaRspObj*)res)->vgId; + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || + TD_RES_TMQ_BATCH_META(res) || TD_RES_TMQ_META(res)) { + return ((SMqRspObj*)res)->vgId; } else { return TSDB_CODE_INVALID_PARA; } @@ -2705,25 +2624,20 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) { return TSDB_CODE_INVALID_PARA; } if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { - SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); - STqOffsetVal* pOffset = &common->reqOffset; - if (common->reqOffset.type == TMQ_OFFSET__LOG) { - return common->reqOffset.version; + SMqRspObj* pRspObj = (SMqRspObj*)res; + STqOffsetVal* pOffset = &pRspObj->dataRsp.reqOffset; + if (pOffset->type == TMQ_OFFSET__LOG) { + return pOffset->version; } else { - tscError("invalid offset type:%d", common->reqOffset.type); + tqErrorC("invalid offset type:%d", pOffset->type); } - } else if (TD_RES_TMQ_META(res)) { - SMqMetaRspObj* pRspObj = (SMqMetaRspObj*)res; - if (pRspObj->metaRsp.rspOffset.type == TMQ_OFFSET__LOG) { - return pRspObj->metaRsp.rspOffset.version; - } - } else if (TD_RES_TMQ_BATCH_META(res)) { - SMqBatchMetaRspObj* pBtRspObj = (SMqBatchMetaRspObj*)res; - if (pBtRspObj->rsp.rspOffset.type == TMQ_OFFSET__LOG) { - return pBtRspObj->rsp.rspOffset.version; + } else if (TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + SMqRspObj* pRspObj = (SMqRspObj*)res; + if (pRspObj->rspOffset.type == TMQ_OFFSET__LOG) { + return pRspObj->rspOffset.version; } } else { - tscError("invalid tmq type:%d", *(int8_t*)res); + tqErrorC("invalid tmq type:%d", *(int8_t*)res); } // data from tsdb, no valid offset info @@ -2735,21 +2649,20 @@ const char* tmq_get_table_name(TAOS_RES* res) { return NULL; } if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { - SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); - - SMqRspObjCommon* pRspObj = (SMqRspObjCommon*)res; - if (!common->withTbName || common->blockTbName == NULL || pRspObj->resIter < 0 || - pRspObj->resIter >= common->blockNum) { + SMqRspObj* pRspObj = (SMqRspObj*)res; + SMqDataRsp* data = &pRspObj->dataRsp; + if (!data->withTbName || data->blockTbName == NULL || pRspObj->resIter < 0 || + pRspObj->resIter >= data->blockNum) { return NULL; } - return (const char*)taosArrayGetP(common->blockTbName, pRspObj->resIter); + return (const char*)taosArrayGetP(data->blockTbName, pRspObj->resIter); } return NULL; } void tmq_commit_async(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* cb, void* param) { if (tmq == NULL) { - tscError("invalid tmq handle, null"); + tqErrorC("invalid tmq handle, null"); if (cb != NULL) { cb(tmq, TSDB_CODE_INVALID_PARA, param); } @@ -2766,13 +2679,13 @@ static void commitCallBackFn(tmq_t* UNUSED_PARAM(tmq), int32_t code, void* param SSyncCommitInfo* pInfo = (SSyncCommitInfo*)param; pInfo->code = code; if (tsem2_post(&pInfo->sem) != 0){ - tscError("failed to post rsp sem in commit cb"); + tqErrorC("failed to post rsp sem in commit cb"); } } int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { if (tmq == NULL) { - tscError("invalid tmq handle, null"); + tqErrorC("invalid tmq handle, null"); return TSDB_CODE_INVALID_PARA; } @@ -2780,11 +2693,11 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); if (pInfo == NULL) { - tscError("failed to allocate memory for sync commit"); + tqErrorC("failed to allocate memory for sync commit"); return terrno; } if (tsem2_init(&pInfo->sem, 0, 0) != 0) { - tscError("failed to init sem for sync commit"); + tqErrorC("failed to init sem for sync commit"); taosMemoryFree(pInfo); return TSDB_CODE_OUT_OF_MEMORY; } @@ -2797,28 +2710,28 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { } if (tsem2_wait(&pInfo->sem) != 0){ - tscError("failed to wait sem for sync commit"); + tqErrorC("failed to wait sem for sync commit"); } code = pInfo->code; if(tsem2_destroy(&pInfo->sem) != 0) { - tscError("failed to destroy sem for sync commit"); + tqErrorC("failed to destroy sem for sync commit"); } taosMemoryFree(pInfo); - tscInfo("consumer:0x%" PRIx64 " sync res commit done, code:%s", tmq->consumerId, tstrerror(code)); + tqInfoC("consumer:0x%" PRIx64 " sync res commit done, code:%s", tmq->consumerId, tstrerror(code)); return code; } // wal range will be ok after calling tmq_get_topic_assignment or poll interface static int32_t checkWalRange(SVgOffsetInfo* offset, int64_t value) { if (offset->walVerBegin == -1 || offset->walVerEnd == -1) { - tscError("Assignment or poll interface need to be called first"); + tqErrorC("Assignment or poll interface need to be called first"); return TSDB_CODE_TMQ_NEED_INITIALIZED; } if (value != -1 && (value < offset->walVerBegin || value > offset->walVerEnd)) { - tscError("invalid seek params, offset:%" PRId64 ", valid range:[%" PRId64 ", %" PRId64 "]", value, + tqErrorC("invalid seek params, offset:%" PRId64 ", valid range:[%" PRId64 ", %" PRId64 "]", value, offset->walVerBegin, offset->walVerEnd); return TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE; } @@ -2828,7 +2741,7 @@ static int32_t checkWalRange(SVgOffsetInfo* offset, int64_t value) { int32_t tmq_commit_offset_sync(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_t offset) { if (tmq == NULL || pTopicName == NULL) { - tscError("invalid tmq handle, null"); + tqErrorC("invalid tmq handle, null"); return TSDB_CODE_INVALID_PARA; } @@ -2856,7 +2769,7 @@ int32_t tmq_commit_offset_sync(tmq_t* tmq, const char* pTopicName, int32_t vgId, SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); if (pInfo == NULL) { - tscError("consumer:0x%" PRIx64 " failed to prepare seek operation", tmq->consumerId); + tqErrorC("consumer:0x%" PRIx64 " failed to prepare seek operation", tmq->consumerId); return terrno; } @@ -2869,18 +2782,18 @@ int32_t tmq_commit_offset_sync(tmq_t* tmq, const char* pTopicName, int32_t vgId, code = asyncCommitOffset(tmq, tname, vgId, &offsetVal, commitCallBackFn, pInfo); if (code == 0) { if (tsem2_wait(&pInfo->sem) != 0){ - tscError("failed to wait sem for sync commit offset"); + tqErrorC("failed to wait sem for sync commit offset"); } code = pInfo->code; } if (code == TSDB_CODE_TMQ_SAME_COMMITTED_VALUE) code = TSDB_CODE_SUCCESS; if(tsem2_destroy(&pInfo->sem) != 0) { - tscError("failed to destroy sem for sync commit offset"); + tqErrorC("failed to destroy sem for sync commit offset"); } taosMemoryFree(pInfo); - tscInfo("consumer:0x%" PRIx64 " sync send commit to vgId:%d, offset:%" PRId64 " code:%s", tmq->consumerId, vgId, + tqInfoC("consumer:0x%" PRIx64 " sync send commit to vgId:%d, offset:%" PRId64 " code:%s", tmq->consumerId, vgId, offset, tstrerror(code)); return code; @@ -2890,7 +2803,7 @@ void tmq_commit_offset_async(tmq_t* tmq, const char* pTopicName, int32_t vgId, i void* param) { int32_t code = 0; if (tmq == NULL || pTopicName == NULL) { - tscError("invalid tmq handle, null"); + tqErrorC("invalid tmq handle, null"); code = TSDB_CODE_INVALID_PARA; goto end; } @@ -2919,7 +2832,7 @@ void tmq_commit_offset_async(tmq_t* tmq, const char* pTopicName, int32_t vgId, i code = asyncCommitOffset(tmq, tname, vgId, &offsetVal, cb, param); - tscInfo("consumer:0x%" PRIx64 " async send commit to vgId:%d, offset:%" PRId64 " code:%s", tmq->consumerId, vgId, + tqInfoC("consumer:0x%" PRIx64 " async send commit to vgId:%d, offset:%" PRId64 " code:%s", tmq->consumerId, vgId, offset, tstrerror(code)); end: @@ -2929,210 +2842,21 @@ end: } } -int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { - SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; - if (pParam == NULL) { - goto FAIL; - } - - tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId); - if (tmq == NULL) { - code = TSDB_CODE_TMQ_CONSUMER_CLOSED; - goto FAIL; - } - - if (code != TSDB_CODE_SUCCESS) { - tscError("consumer:0x%" PRIx64 ", get topic endpoint error, code:%s", tmq->consumerId, tstrerror(code)); - goto END; - } - - if (pMsg == NULL) { - goto END; - } - SMqRspHead* head = pMsg->pData; - int32_t epoch = atomic_load_32(&tmq->epoch); - tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d", tmq->consumerId, head->epoch, epoch); - if (pParam->sync) { - SMqAskEpRsp rsp = {0}; - if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp) != NULL) { - (void)doUpdateLocalEp(tmq, head->epoch, &rsp); - } - tDeleteSMqAskEpRsp(&rsp); - } else { - SMqAskEpRspWrapper* pWrapper = NULL; - code = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM, 0, (void**)&pWrapper); - if (code) { - goto END; - } - - pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP; - pWrapper->epoch = head->epoch; - (void)memcpy(&pWrapper->msg, pMsg->pData, sizeof(SMqRspHead)); - if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg) == NULL) { - tmqFreeRspWrapper((SMqRspWrapper*)pWrapper); - taosFreeQitem(pWrapper); - } else { - code = taosWriteQitem(tmq->mqueue, pWrapper); - if (code != 0) { - tmqFreeRspWrapper((SMqRspWrapper*)pWrapper); - taosFreeQitem(pWrapper); - tscError("consumer:0x%" PRIx64 " put ep res into mqueue failed, code:%d", tmq->consumerId, code); - } - } - } - -END: - { - int32_t ret = taosReleaseRef(tmqMgmt.rsetId, pParam->refId); - if (ret != 0){ - tscError("failed to release ref:%"PRId64 ", code:%d", pParam->refId, ret); - } - } -FAIL: - if (pParam && pParam->sync) { - SAskEpInfo* pInfo = pParam->pParam; - if (pInfo) { - pInfo->code = code; - if (tsem2_post(&pInfo->sem) != 0){ - tscError("failed to post rsp sem askep cb"); - } - } - } - - if (pMsg) { - taosMemoryFree(pMsg->pEpSet); - taosMemoryFree(pMsg->pData); - } - - return code; -} - -int32_t syncAskEp(tmq_t* pTmq) { - SAskEpInfo* pInfo = taosMemoryMalloc(sizeof(SAskEpInfo)); - if (pInfo == NULL) return terrno; - if (tsem2_init(&pInfo->sem, 0, 0) != 0) { - taosMemoryFree(pInfo); - return TSDB_CODE_TSC_INTERNAL_ERROR; - } - - int32_t code = askEp(pTmq, pInfo, true, false); - if (code == 0) { - if (tsem2_wait(&pInfo->sem) != 0){ - tscError("consumer:0x%" PRIx64 ", failed to wait for sem", pTmq->consumerId); - } - code = pInfo->code; - } - - if(tsem2_destroy(&pInfo->sem) != 0) { - tscError("failed to destroy sem sync ask ep"); - } - taosMemoryFree(pInfo); - return code; -} - -int32_t askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) { - SMqAskEpReq req = {0}; - req.consumerId = pTmq->consumerId; - req.epoch = updateEpSet ? -1 : pTmq->epoch; - (void)strcpy(req.cgroup, pTmq->groupId); - int code = 0; - SMqAskEpCbParam* pParam = NULL; - void* pReq = NULL; - - int32_t tlen = tSerializeSMqAskEpReq(NULL, 0, &req); - if (tlen < 0) { - tscError("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq failed", pTmq->consumerId); - return TSDB_CODE_INVALID_PARA; - } - - pReq = taosMemoryCalloc(1, tlen); - if (pReq == NULL) { - tscError("consumer:0x%" PRIx64 ", failed to malloc askEpReq msg, size:%d", pTmq->consumerId, tlen); - return terrno; - } - - if (tSerializeSMqAskEpReq(pReq, tlen, &req) < 0) { - tscError("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq %d failed", pTmq->consumerId, tlen); - taosMemoryFree(pReq); - return TSDB_CODE_INVALID_PARA; - } - - pParam = taosMemoryCalloc(1, sizeof(SMqAskEpCbParam)); - if (pParam == NULL) { - tscError("consumer:0x%" PRIx64 ", failed to malloc subscribe param", pTmq->consumerId); - taosMemoryFree(pReq); - return terrno; - } - - pParam->refId = pTmq->refId; - pParam->sync = sync; - pParam->pParam = param; - - SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (sendInfo == NULL) { - taosMemoryFree(pReq); - taosMemoryFree(pParam); - return terrno; - } - - sendInfo->msgInfo = (SDataBuf){.pData = pReq, .len = tlen, .handle = NULL}; - sendInfo->requestId = generateRequestId(); - sendInfo->requestObjRefId = 0; - sendInfo->param = pParam; - sendInfo->paramFreeFp = taosMemoryFree; - sendInfo->fp = askEpCb; - sendInfo->msgType = TDMT_MND_TMQ_ASK_EP; - - SEpSet epSet = getEpSet_s(&pTmq->pTscObj->pAppInfo->mgmtEp); - tscDebug("consumer:0x%" PRIx64 " ask ep from mnode,QID:0x%" PRIx64, pTmq->consumerId, sendInfo->requestId); - return asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo); -} - -int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { - int64_t refId = pParamSet->refId; - - tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); - if (tmq == NULL) { - taosMemoryFree(pParamSet); - return TSDB_CODE_TMQ_CONSUMER_CLOSED; - } - - // if no more waiting rsp - if (pParamSet->callbackFn != NULL) { - pParamSet->callbackFn(tmq, pParamSet->code, pParamSet->userParam); - } - - taosMemoryFree(pParamSet); - return taosReleaseRef(tmqMgmt.rsetId, refId); -} - -int32_t commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId) { - int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); - if (waitingRspNum == 0) { - tscInfo("consumer:0x%" PRIx64 " topic:%s vgId:%d all commit-rsp received, commit completed", consumerId, pTopic, - vgId); - return tmqCommitDone(pParamSet); - } else { - tscInfo("consumer:0x%" PRIx64 " topic:%s vgId:%d commit-rsp received, remain:%d", consumerId, pTopic, vgId, - waitingRspNum); - } - return 0; -} - int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pResInfo) { - SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); - SMqRspObjCommon* pRspObj = (SMqRspObjCommon*)res; + SMqRspObj* pRspObj = (SMqRspObj*)res; + SMqDataRsp* data = &pRspObj->dataRsp; + pRspObj->resIter++; - if (pRspObj->resIter < common->blockNum) { - if (common->withSchema) { + if (pRspObj->resIter < data->blockNum) { + if (data->withSchema) { doFreeReqResultInfo(&pRspObj->resInfo); - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(common->blockSchema, pRspObj->resIter); + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(data->blockSchema, pRspObj->resIter); if (pSW) { TAOS_CHECK_RETURN(setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols)); } } - void* pRetrieve = taosArrayGetP(common->blockData, pRspObj->resIter); + void* pRetrieve = taosArrayGetP(data->blockData, pRspObj->resIter); void* rawData = NULL; int64_t rows = 0; int32_t precision = 0; @@ -3165,7 +2889,7 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { int32_t total = atomic_add_fetch_32(&pCommon->numOfRsp, 1); if (code != TSDB_CODE_SUCCESS) { - tscError("consumer:0x%" PRIx64 " failed to get the wal info from vgId:%d for topic:%s", pCommon->consumerId, + tqErrorC("consumer:0x%" PRIx64 " failed to get the wal info from vgId:%d for topic:%s", pCommon->consumerId, pParam->vgId, pCommon->pTopicName); } else { @@ -3181,12 +2905,12 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { SMqRspHead* pHead = pMsg->pData; tmq_topic_assignment assignment = {.begin = pHead->walsver, .end = pHead->walever + 1, - .currentOffset = rsp.common.rspOffset.version, + .currentOffset = rsp.rspOffset.version, .vgId = pParam->vgId}; (void)taosThreadMutexLock(&pCommon->mutex); if (taosArrayPush(pCommon->pList, &assignment) == NULL) { - tscError("consumer:0x%" PRIx64 " failed to push the wal info from vgId:%d for topic:%s", pCommon->consumerId, + tqErrorC("consumer:0x%" PRIx64 " failed to push the wal info from vgId:%d for topic:%s", pCommon->consumerId, pParam->vgId, pCommon->pTopicName); code = TSDB_CODE_TSC_INTERNAL_ERROR; } @@ -3197,7 +2921,7 @@ END: pCommon->code = code; if (total == pParam->totalReq) { if (tsem2_post(&pCommon->rsp) != 0) { - tscError("failed to post semaphore in get wal cb"); + tqErrorC("failed to post semaphore in get wal cb"); } } @@ -3215,7 +2939,7 @@ static void destroyCommonInfo(SMqVgCommon* pCommon) { } taosArrayDestroy(pCommon->pList); if(tsem2_destroy(&pCommon->rsp) != 0) { - tscError("failed to destroy semaphore for topic:%s", pCommon->pTopicName); + tqErrorC("failed to destroy semaphore for topic:%s", pCommon->pTopicName); } (void)taosThreadMutexDestroy(&pCommon->mutex); taosMemoryFree(pCommon->pTopicName); @@ -3253,7 +2977,7 @@ end: } pParam->code = code; if (tsem2_post(&pParam->sem) != 0){ - tscError("failed to post semaphore in tmCommittedCb"); + tqErrorC("failed to post semaphore in tmCommittedCb"); } return code; } @@ -3319,14 +3043,14 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, epSet, NULL, sendInfo); if (code != 0) { if(tsem2_destroy(&pParam->sem) != 0) { - tscError("failed to destroy semaphore in get committed from server1"); + tqErrorC("failed to destroy semaphore in get committed from server1"); } taosMemoryFree(pParam); return code; } if (tsem2_wait(&pParam->sem) != 0){ - tscError("failed to wait semaphore in get committed from server"); + tqErrorC("failed to wait semaphore in get committed from server"); } code = pParam->code; if (code == TSDB_CODE_SUCCESS) { @@ -3338,7 +3062,7 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep } } if(tsem2_destroy(&pParam->sem) != 0) { - tscError("failed to destroy semaphore in get committed from server2"); + tqErrorC("failed to destroy semaphore in get committed from server2"); } taosMemoryFree(pParam); @@ -3347,7 +3071,7 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep int64_t tmq_position(tmq_t* tmq, const char* pTopicName, int32_t vgId) { if (tmq == NULL || pTopicName == NULL) { - tscError("invalid tmq handle, null"); + tqErrorC("invalid tmq handle, null"); return TSDB_CODE_INVALID_PARA; } @@ -3367,7 +3091,7 @@ int64_t tmq_position(tmq_t* tmq, const char* pTopicName, int32_t vgId) { SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; int32_t type = pOffsetInfo->endOffset.type; if (isInSnapshotMode(type, tmq->useSnapshot)) { - tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, position error", tmq->consumerId, type); + tqErrorC("consumer:0x%" PRIx64 " offset type:%d not wal version, position error", tmq->consumerId, type); taosWUnLockLatch(&tmq->lock); return TSDB_CODE_TMQ_SNAPSHOT_ERROR; } @@ -3397,16 +3121,16 @@ int64_t tmq_position(tmq_t* tmq, const char* pTopicName, int32_t vgId) { position = code; } } else { - tscError("consumer:0x%" PRIx64 " offset type:%d can not be reach here", tmq->consumerId, type); + tqErrorC("consumer:0x%" PRIx64 " offset type:%d can not be reach here", tmq->consumerId, type); } - tscInfo("consumer:0x%" PRIx64 " tmq_position vgId:%d position:%" PRId64, tmq->consumerId, vgId, position); + tqInfoC("consumer:0x%" PRIx64 " tmq_position vgId:%d position:%" PRId64, tmq->consumerId, vgId, position); return position; } int64_t tmq_committed(tmq_t* tmq, const char* pTopicName, int32_t vgId) { if (tmq == NULL || pTopicName == NULL) { - tscError("invalid tmq handle, null"); + tqErrorC("invalid tmq handle, null"); return TSDB_CODE_INVALID_PARA; } @@ -3425,14 +3149,14 @@ int64_t tmq_committed(tmq_t* tmq, const char* pTopicName, int32_t vgId) { SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; if (isInSnapshotMode(pOffsetInfo->endOffset.type, tmq->useSnapshot)) { - tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, committed error", tmq->consumerId, + tqErrorC("consumer:0x%" PRIx64 " offset type:%d not wal version, committed error", tmq->consumerId, pOffsetInfo->endOffset.type); taosWUnLockLatch(&tmq->lock); return TSDB_CODE_TMQ_SNAPSHOT_ERROR; } if (isInSnapshotMode(pOffsetInfo->committedOffset.type, tmq->useSnapshot)) { - tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, committed error", tmq->consumerId, + tqErrorC("consumer:0x%" PRIx64 " offset type:%d not wal version, committed error", tmq->consumerId, pOffsetInfo->committedOffset.type); taosWUnLockLatch(&tmq->lock); return TSDB_CODE_TMQ_SNAPSHOT_ERROR; @@ -3450,14 +3174,14 @@ int64_t tmq_committed(tmq_t* tmq, const char* pTopicName, int32_t vgId) { committed = getCommittedFromServer(tmq, tname, vgId, &epSet); end: - tscInfo("consumer:0x%" PRIx64 " tmq_committed vgId:%d committed:%" PRId64, tmq->consumerId, vgId, committed); + tqInfoC("consumer:0x%" PRIx64 " tmq_committed vgId:%d committed:%" PRId64, tmq->consumerId, vgId, committed); return committed; } int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_assignment** assignment, int32_t* numOfAssignment) { if (tmq == NULL || pTopicName == NULL || assignment == NULL || numOfAssignment == NULL) { - tscError("invalid tmq handle, null"); + tqErrorC("invalid tmq handle, null"); return TSDB_CODE_INVALID_PARA; } *numOfAssignment = 0; @@ -3473,7 +3197,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a SMqClientTopic* pTopic = NULL; int32_t code = getTopicByName(tmq, tname, &pTopic); if (code != 0) { - tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); + tqErrorC("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); goto end; } @@ -3486,7 +3210,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a } int32_t type = pClientVg->offsetInfo.beginOffset.type; if (isInSnapshotMode(type, tmq->useSnapshot)) { - tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, assignment not allowed", tmq->consumerId, type); + tqErrorC("consumer:0x%" PRIx64 " offset type:%d not wal version, assignment not allowed", tmq->consumerId, type); code = TSDB_CODE_TMQ_SNAPSHOT_ERROR; goto end; } @@ -3494,7 +3218,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a *assignment = taosMemoryCalloc(*numOfAssignment, sizeof(tmq_topic_assignment)); if (*assignment == NULL) { - tscError("consumer:0x%" PRIx64 " failed to malloc buffer, size:%" PRIzu, tmq->consumerId, + tqErrorC("consumer:0x%" PRIx64 " failed to malloc buffer, size:%" PRIzu, tmq->consumerId, (*numOfAssignment) * sizeof(tmq_topic_assignment)); code = terrno; goto end; @@ -3517,7 +3241,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a pAssignment->begin = pClientVg->offsetInfo.walVerBegin; pAssignment->end = pClientVg->offsetInfo.walVerEnd; pAssignment->vgId = pClientVg->vgId; - tscInfo("consumer:0x%" PRIx64 " get assignment from local:%d->%" PRId64, tmq->consumerId, pAssignment->vgId, + tqInfoC("consumer:0x%" PRIx64 " get assignment from local:%d->%" PRId64, tmq->consumerId, pAssignment->vgId, pAssignment->currentOffset); } @@ -3606,7 +3330,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a char offsetFormatBuf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.beginOffset); - tscInfo("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s,QID:0x%" PRIx64, tmq->consumerId, + tqInfoC("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s,QID:0x%" PRIx64, tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId); code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pClientVg->epSet, NULL, sendInfo); if (code != 0) { @@ -3615,7 +3339,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a } if (tsem2_wait(&pCommon->rsp) != 0){ - tscError("consumer:0x%" PRIx64 " failed to wait sem in get assignment", tmq->consumerId); + tqErrorC("consumer:0x%" PRIx64 " failed to wait sem in get assignment", tmq->consumerId); } code = pCommon->code; @@ -3641,7 +3365,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a } SVgOffsetInfo* pOffsetInfo = &pClientVg->offsetInfo; - tscInfo("consumer:0x%" PRIx64 " %s vgId:%d offset is update to:%" PRId64, tmq->consumerId, pTopic->topicName, + tqInfoC("consumer:0x%" PRIx64 " %s vgId:%d offset is update to:%" PRId64, tmq->consumerId, pTopic->topicName, p->vgId, p->currentOffset); pOffsetInfo->walVerBegin = p->begin; @@ -3680,7 +3404,7 @@ static int32_t tmqSeekCb(void* param, SDataBuf* pMsg, int32_t code) { SMqSeekParam* pParam = param; pParam->code = code; if (tsem2_post(&pParam->sem) != 0){ - tscError("failed to post sem in tmqSeekCb"); + tqErrorC("failed to post sem in tmqSeekCb"); } return 0; } @@ -3689,7 +3413,7 @@ static int32_t tmqSeekCb(void* param, SDataBuf* pMsg, int32_t code) { // there is no data to poll int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_t offset) { if (tmq == NULL || pTopicName == NULL) { - tscError("invalid tmq handle, null"); + tqErrorC("invalid tmq handle, null"); return TSDB_CODE_INVALID_PARA; } @@ -3710,7 +3434,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ int32_t type = pOffsetInfo->endOffset.type; if (isInSnapshotMode(type, tmq->useSnapshot)) { - tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, seek not allowed", tmq->consumerId, type); + tqErrorC("consumer:0x%" PRIx64 " offset type:%d not wal version, seek not allowed", tmq->consumerId, type); taosWUnLockLatch(&tmq->lock); return TSDB_CODE_TMQ_SNAPSHOT_ERROR; } @@ -3721,7 +3445,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ return code; } - tscInfo("consumer:0x%" PRIx64 " seek to %" PRId64 " on vgId:%d", tmq->consumerId, offset, vgId); + tqInfoC("consumer:0x%" PRIx64 " seek to %" PRId64 " on vgId:%d", tmq->consumerId, offset, vgId); // update the offset, and then commit to vnode pOffsetInfo->endOffset.type = TMQ_OFFSET__LOG; pOffsetInfo->endOffset.version = offset; @@ -3779,22 +3503,22 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo); if (code != 0) { if(tsem2_destroy(&pParam->sem) != 0) { - tscError("consumer:0x%" PRIx64 "destroy rsp sem failed in seek offset", tmq->consumerId); + tqErrorC("consumer:0x%" PRIx64 "destroy rsp sem failed in seek offset", tmq->consumerId); } taosMemoryFree(pParam); return code; } if (tsem2_wait(&pParam->sem) != 0){ - tscError("consumer:0x%" PRIx64 "wait rsp sem failed in seek offset", tmq->consumerId); + tqErrorC("consumer:0x%" PRIx64 "wait rsp sem failed in seek offset", tmq->consumerId); } code = pParam->code; if(tsem2_destroy(&pParam->sem) != 0) { - tscError("consumer:0x%" PRIx64 "destroy rsp sem failed in seek offset", tmq->consumerId); + tqErrorC("consumer:0x%" PRIx64 "destroy rsp sem failed in seek offset", tmq->consumerId); } taosMemoryFree(pParam); - tscInfo("consumer:0x%" PRIx64 "send seek to vgId:%d, return code:%s", tmq->consumerId, vgId, tstrerror(code)); + tqInfoC("consumer:0x%" PRIx64 "send seek to vgId:%d, return code:%s", tmq->consumerId, vgId, tstrerror(code)); return code; } diff --git a/source/common/src/cos.c b/source/common/src/cos.c index c2b9fe34e1..453b924f31 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -260,19 +260,19 @@ static void responseCompleteCallback(S3Status status, const S3ErrorDetails *erro const int elen = sizeof(cbd->err_msg); if (error) { if (error->message && elen - len > 0) { - len += snprintf(&(cbd->err_msg[len]), elen - len, " Message: %s\n", error->message); + len += tsnprintf(&(cbd->err_msg[len]), elen - len, " Message: %s\n", error->message); } if (error->resource && elen - len > 0) { - len += snprintf(&(cbd->err_msg[len]), elen - len, " Resource: %s\n", error->resource); + len += tsnprintf(&(cbd->err_msg[len]), elen - len, " Resource: %s\n", error->resource); } if (error->furtherDetails && elen - len > 0) { - len += snprintf(&(cbd->err_msg[len]), elen - len, " Further Details: %s\n", error->furtherDetails); + len += tsnprintf(&(cbd->err_msg[len]), elen - len, " Further Details: %s\n", error->furtherDetails); } if (error->extraDetailsCount && elen - len > 0) { - len += snprintf(&(cbd->err_msg[len]), elen - len, "%s", " Extra Details:\n"); + len += tsnprintf(&(cbd->err_msg[len]), elen - len, "%s", " Extra Details:\n"); for (int i = 0; i < error->extraDetailsCount; i++) { if (elen - len > 0) { - len += snprintf(&(cbd->err_msg[len]), elen - len, " %s: %s\n", error->extraDetails[i].name, + len += tsnprintf(&(cbd->err_msg[len]), elen - len, " %s: %s\n", error->extraDetails[i].name, error->extraDetails[i].value); } } @@ -753,7 +753,7 @@ upload: if (!manager.etags[i]) { TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); } - n = snprintf(buf, sizeof(buf), + n = tsnprintf(buf, sizeof(buf), "%d" "%s", i + 1, manager.etags[i]); @@ -919,7 +919,7 @@ upload: char buf[256]; int n; for (int i = 0; i < cp.part_num; ++i) { - n = snprintf(buf, sizeof(buf), + n = tsnprintf(buf, sizeof(buf), "%d" "%s", // i + 1, manager.etags[i]); diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c index 47a452eab7..eef889429b 100644 --- a/source/common/src/rsync.c +++ b/source/common/src/rsync.c @@ -94,7 +94,7 @@ static int32_t generateConfigFile(char* confDir) { #endif ); uDebug("[rsync] conf:%s", confContent); - if (taosWriteFile(pFile, confContent, strlen(confContent)) != TSDB_CODE_SUCCESS) { + if (taosWriteFile(pFile, confContent, strlen(confContent)) <= 0) { uError("[rsync] write conf file error," ERRNO_ERR_FORMAT, ERRNO_ERR_DATA); (void)taosCloseFile(&pFile); code = terrno; diff --git a/source/common/src/systable.c b/source/common/src/systable.c index be841d9682..19df0f5a78 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -398,6 +398,7 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = { {.name = "finished", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, }; + static const SSysDbTableSchema tsmaSchema[] = { {.name = "tsma_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 7a67522231..5372533455 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2446,9 +2446,11 @@ _error: return NULL; } -static char* formatTimestamp(char* buf, int64_t val, int precision) { +static int32_t formatTimestamp(char* buf, size_t cap, int64_t val, int precision) { time_t tt; int32_t ms = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (precision == TSDB_TIME_PRECISION_NANO) { tt = (time_t)(val / 1000000000); ms = val % 1000000000; @@ -2460,14 +2462,6 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { ms = val % 1000; } - /* comment out as it make testcases like select_with_tags.sim fail. - but in windows, this may cause the call to localtime crash if tt < 0, - need to find a better solution. - if (tt < 0) { - tt = 0; - } - */ - if (tt <= 0 && ms < 0) { tt--; if (precision == TSDB_TIME_PRECISION_NANO) { @@ -2479,28 +2473,44 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { } } struct tm ptm = {0}; - if (taosLocalTime(&tt, &ptm, buf) == NULL) { - return buf; + if (taosLocalTime(&tt, &ptm, buf, cap) == NULL) { + code = TSDB_CODE_INTERNAL_ERROR; + TSDB_CHECK_CODE(code, lino, _end); } - size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); + size_t pos = strftime(buf, cap, "%Y-%m-%d %H:%M:%S", &ptm); + if (pos == 0) { + code = TSDB_CODE_OUT_OF_BUFFER; + TSDB_CHECK_CODE(code, lino, _end); + } + int32_t nwritten = 0; if (precision == TSDB_TIME_PRECISION_NANO) { - sprintf(buf + pos, ".%09d", ms); + nwritten = snprintf(buf + pos, cap - pos, ".%09d", ms); } else if (precision == TSDB_TIME_PRECISION_MICRO) { - sprintf(buf + pos, ".%06d", ms); + nwritten = snprintf(buf + pos, cap - pos, ".%06d", ms); } else { - sprintf(buf + pos, ".%03d", ms); + nwritten = snprintf(buf + pos, cap - pos, ".%03d", ms); } - return buf; + if (nwritten >= cap - pos) { + code = TSDB_CODE_OUT_OF_BUFFER; + TSDB_CHECK_CODE(code, lino, _end); + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } // for debug int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf, const char* taskIdStr) { + int32_t lino = 0; int32_t size = 2048 * 1024; int32_t code = 0; char* dumpBuf = NULL; - char pBuf[128] = {0}; + char pBuf[TD_TIME_STR_LEN] = {0}; int32_t rows = pDataBlock->info.rows; int32_t len = 0; @@ -2510,7 +2520,7 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf } int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); - len += snprintf(dumpBuf + len, size - len, + len += tsnprintf(dumpBuf + len, size - len, "%s===stream===%s|block type %d|child id %d|group id:%" PRIu64 "|uid:%" PRId64 "|rows:%" PRId64 "|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "|tbl:%s\n", taskIdStr, flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, @@ -2521,7 +2531,7 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf } for (int32_t j = 0; j < rows; j++) { - len += snprintf(dumpBuf + len, size - len, "%s|", flag); + len += tsnprintf(dumpBuf + len, size - len, "%s|", flag); if (len >= size - 1) { goto _exit; } @@ -2530,11 +2540,12 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); if (pColInfoData == NULL) { code = terrno; + lino = __LINE__; goto _exit; } if (colDataIsNull(pColInfoData, rows, j, NULL) || !pColInfoData->pData) { - len += snprintf(dumpBuf + len, size - len, " %15s |", "NULL"); + len += tsnprintf(dumpBuf + len, size - len, " %15s |", "NULL"); if (len >= size - 1) goto _exit; continue; } @@ -2543,52 +2554,55 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf switch (pColInfoData->info.type) { case TSDB_DATA_TYPE_TIMESTAMP: memset(pBuf, 0, sizeof(pBuf)); - (void)formatTimestamp(pBuf, *(uint64_t*)var, pColInfoData->info.precision); - len += snprintf(dumpBuf + len, size - len, " %25s |", pBuf); + code = formatTimestamp(pBuf, sizeof(pBuf), *(uint64_t*)var, pColInfoData->info.precision); + if (code != TSDB_CODE_SUCCESS) { + TAOS_UNUSED(tsnprintf(pBuf, sizeof(pBuf), "NaN")); + } + len += tsnprintf(dumpBuf + len, size - len, " %25s |", pBuf); if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_TINYINT: - len += snprintf(dumpBuf + len, size - len, " %15d |", *(int8_t*)var); + len += tsnprintf(dumpBuf + len, size - len, " %15d |", *(int8_t*)var); if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_UTINYINT: - len += snprintf(dumpBuf + len, size - len, " %15d |", *(uint8_t*)var); + len += tsnprintf(dumpBuf + len, size - len, " %15d |", *(uint8_t*)var); if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_SMALLINT: - len += snprintf(dumpBuf + len, size - len, " %15d |", *(int16_t*)var); + len += tsnprintf(dumpBuf + len, size - len, " %15d |", *(int16_t*)var); if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_USMALLINT: - len += snprintf(dumpBuf + len, size - len, " %15d |", *(uint16_t*)var); + len += tsnprintf(dumpBuf + len, size - len, " %15d |", *(uint16_t*)var); if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_INT: - len += snprintf(dumpBuf + len, size - len, " %15d |", *(int32_t*)var); + len += tsnprintf(dumpBuf + len, size - len, " %15d |", *(int32_t*)var); if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_UINT: - len += snprintf(dumpBuf + len, size - len, " %15u |", *(uint32_t*)var); + len += tsnprintf(dumpBuf + len, size - len, " %15u |", *(uint32_t*)var); if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_BIGINT: - len += snprintf(dumpBuf + len, size - len, " %15" PRId64 " |", *(int64_t*)var); + len += tsnprintf(dumpBuf + len, size - len, " %15" PRId64 " |", *(int64_t*)var); if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_UBIGINT: - len += snprintf(dumpBuf + len, size - len, " %15" PRIu64 " |", *(uint64_t*)var); + len += tsnprintf(dumpBuf + len, size - len, " %15" PRIu64 " |", *(uint64_t*)var); if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_FLOAT: - len += snprintf(dumpBuf + len, size - len, " %15f |", *(float*)var); + len += tsnprintf(dumpBuf + len, size - len, " %15f |", *(float*)var); if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_DOUBLE: - len += snprintf(dumpBuf + len, size - len, " %15f |", *(double*)var); + len += tsnprintf(dumpBuf + len, size - len, " %15f |", *(double*)var); if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_BOOL: - len += snprintf(dumpBuf + len, size - len, " %15d |", *(bool*)var); + len += tsnprintf(dumpBuf + len, size - len, " %15d |", *(bool*)var); if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_VARCHAR: @@ -2599,7 +2613,7 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf int32_t dataSize = TMIN(sizeof(pBuf), varDataLen(pData)); dataSize = TMIN(dataSize, 50); memcpy(pBuf, varDataVal(pData), dataSize); - len += snprintf(dumpBuf + len, size - len, " %15s |", pBuf); + len += tsnprintf(dumpBuf + len, size - len, " %15s |", pBuf); if (len >= size - 1) goto _exit; } break; case TSDB_DATA_TYPE_NCHAR: { @@ -2609,24 +2623,25 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf code = taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf); if (code < 0) { uError("func %s failed to convert to ucs charset since %s", __func__, tstrerror(code)); + lino = __LINE__; goto _exit; } - len += snprintf(dumpBuf + len, size - len, " %15s |", pBuf); + len += tsnprintf(dumpBuf + len, size - len, " %15s |", pBuf); if (len >= size - 1) goto _exit; } break; } } - len += snprintf(dumpBuf + len, size - len, "%d\n", j); + len += tsnprintf(dumpBuf + len, size - len, "%d\n", j); if (len >= size - 1) goto _exit; } - len += snprintf(dumpBuf + len, size - len, "%s |end\n", flag); + len += tsnprintf(dumpBuf + len, size - len, "%s |end\n", flag); _exit: if (code == TSDB_CODE_SUCCESS) { *pDataBuf = dumpBuf; dumpBuf = NULL; } else { - uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); if (dumpBuf) { taosMemoryFree(dumpBuf); } @@ -2857,27 +2872,98 @@ _end: return code; } -void buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId) { - char tmp[TSDB_TABLE_NAME_LEN] = {0}; - if (stbName == NULL){ - snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%"PRIu64, groupId); - }else{ +// Construct the child table name in the form of __ and store it in `ctbName`. +// If the name length exceeds TSDB_TABLE_NAME_LEN, first convert _ to an MD5 value and then +// concatenate. If the length is still too long, convert to an MD5 value as well. +int32_t buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId, size_t cap) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + char tmp[TSDB_TABLE_NAME_LEN] = {0}; + char* suffix = tmp; + size_t suffixCap = sizeof(tmp); + size_t suffixLen = 0; + size_t prefixLen = 0; + T_MD5_CTX context; + + if (ctbName == NULL || cap < TSDB_TABLE_NAME_LEN) { + code = TSDB_CODE_INTERNAL_ERROR; + TSDB_CHECK_CODE(code, lino, _end); + } + + prefixLen = strlen(ctbName); + + if (stbName == NULL) { + suffixLen = snprintf(suffix, suffixCap, "%" PRIu64, groupId); + if (suffixLen >= suffixCap) { + code = TSDB_CODE_INTERNAL_ERROR; + TSDB_CHECK_CODE(code, lino, _end); + } + } else { int32_t i = strlen(stbName) - 1; - for(; i >= 0; i--){ - if (stbName[i] == '.'){ + for (; i >= 0; i--) { + if (stbName[i] == '.') { break; } } - snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%s_%"PRIu64, stbName + i + 1, groupId); - } - - ctbName[TSDB_TABLE_NAME_LEN - strlen(tmp) - 1] = 0; // put stbname + groupId to the end - (void)strcat(ctbName, tmp); - for(int i = 0; i < strlen(ctbName); i++){ - if(ctbName[i] == '.'){ - ctbName[i] = '_'; + suffixLen = snprintf(suffix, suffixCap, "%s_%" PRIu64, stbName + i + 1, groupId); + if (suffixLen >= suffixCap) { + suffixCap = suffixLen + 1; + suffix = taosMemoryMalloc(suffixCap); + TSDB_CHECK_NULL(suffix, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY); + suffixLen = snprintf(suffix, suffixCap, "%s_%" PRIu64, stbName + i + 1, groupId); + if (suffixLen >= suffixCap) { + code = TSDB_CODE_INTERNAL_ERROR; + TSDB_CHECK_CODE(code, lino, _end); + } } } + + if (prefixLen + suffixLen + 1 >= TSDB_TABLE_NAME_LEN) { + // If the name length exceeeds the limit, convert the suffix to MD5 value. + tMD5Init(&context); + tMD5Update(&context, (uint8_t*)suffix, suffixLen); + tMD5Final(&context); + suffixLen = snprintf(suffix, suffixCap, "%016" PRIx64 "%016" PRIx64, *(uint64_t*)context.digest, + *(uint64_t*)(context.digest + 8)); + if (suffixLen >= suffixCap) { + code = TSDB_CODE_INTERNAL_ERROR; + TSDB_CHECK_CODE(code, lino, _end); + } + } + + if (prefixLen + suffixLen + 1 >= TSDB_TABLE_NAME_LEN) { + // If the name is still too long, convert the ctbName to MD5 value. + tMD5Init(&context); + tMD5Update(&context, (uint8_t*)ctbName, prefixLen); + tMD5Final(&context); + prefixLen = snprintf(ctbName, cap, "t_%016" PRIx64 "%016" PRIx64, *(uint64_t*)context.digest, + *(uint64_t*)(context.digest + 8)); + if (prefixLen >= cap) { + code = TSDB_CODE_INTERNAL_ERROR; + TSDB_CHECK_CODE(code, lino, _end); + } + } + + if (prefixLen + suffixLen + 1 >= TSDB_TABLE_NAME_LEN) { + code = TSDB_CODE_INTERNAL_ERROR; + TSDB_CHECK_CODE(code, lino, _end); + } + + ctbName[prefixLen] = '_'; + tstrncpy(&ctbName[prefixLen + 1], suffix, cap - prefixLen - 1); + + for (char* p = ctbName; *p; ++p) { + if (*p == '.') *p = '_'; + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + if (suffix != tmp) { + taosMemoryFree(suffix); + } + return code; } // auto stream subtable name starts with 't_', followed by the first segment of MD5 digest for group vals. diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 2aef97ed1b..3c05294264 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -362,7 +362,7 @@ static int32_t taosSplitS3Cfg(SConfig *pCfg, const char *name, char gVarible[TSD char *strDup = NULL; if ((strDup = taosStrdup(pItem->str))== NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } @@ -542,6 +542,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqClientDebug", tqClientDebug, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); @@ -644,11 +645,6 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableCoreFile", tsEnableCoreFile, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "ssd42", tsSSE42Supported, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "avx", tsAVXSupported, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "avx2", tsAVX2Supported, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "fma", tsFMASupported, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "avx512", tsAVX512Supported, CFG_SCOPE_BOTH, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "simdEnable", tsSIMDEnable, CFG_SCOPE_BOTH, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "AVX512Enable", tsAVX512Enable, CFG_SCOPE_BOTH, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, CFG_SCOPE_BOTH, CFG_DYN_NONE)); @@ -1396,6 +1392,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "simdEnable"); tsSIMDEnable = (bool)pItem->bval; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "AVX512Enable"); + tsAVX512Enable = (bool)pItem->bval; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "tagFilterCache"); tsTagFilterCache = (bool)pItem->bval; @@ -1724,6 +1723,7 @@ int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char * TAOS_CHECK_RETURN(cfgInit(&pCfg)); TAOS_CHECK_GOTO(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE), NULL, _exit); + TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "debugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER), NULL, _exit); TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) ,NULL, _exit); if ((code = taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) { @@ -1957,7 +1957,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"smaDebugFlag", &smaDebugFlag}, {"idxDebugFlag", &idxDebugFlag}, {"tdbDebugFlag", &tdbDebugFlag}, {"tmrDebugFlag", &tmrDebugFlag}, {"uDebugFlag", &uDebugFlag}, {"smaDebugFlag", &smaDebugFlag}, {"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag}, {"metaDebugFlag", &metaDebugFlag}, - {"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag}, + {"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag}, {"tqClientDebug", &tqClientDebug}, }; static OptionNameAndVar options[] = {{"audit", &tsEnableAudit}, diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c index 8de557a881..10375ba857 100644 --- a/source/common/src/tmisce.c +++ b/source/common/src/tmisce.c @@ -284,7 +284,7 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, col++); if (pColInfo == NULL) { - code = TSDB_CODE_OUT_OF_RANGE; + code = terrno; TAOS_CHECK_GOTO(code, NULL, _exit); } @@ -297,7 +297,7 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { pColInfo = taosArrayGet(pBlock->pDataBlock, col++); if (pColInfo == NULL) { - code = TSDB_CODE_OUT_OF_RANGE; + code = terrno; TAOS_CHECK_GOTO(code, NULL, _exit); } @@ -309,7 +309,7 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { pColInfo = taosArrayGet(pBlock->pDataBlock, col++); if (pColInfo == NULL) { - code = TSDB_CODE_OUT_OF_RANGE; + code = terrno; TAOS_CHECK_GOTO(code, NULL, _exit); } TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, scope, false), NULL, _exit); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 557ac07bbd..4c4b78278e 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1703,6 +1703,7 @@ int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) { if (!tDecodeIsEnd(&decoder)) { TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pRsp->ipWhiteVer)); } + tEndDecode(&decoder); _exit: tDecoderClear(&decoder); @@ -8163,6 +8164,7 @@ int32_t tSerializeSMqHbRsp(void *buf, int32_t bufLen, SMqHbRsp *pRsp) { TAOS_CHECK_EXIT(tEncodeI8(&encoder, privilege->noPrivilege)); } + if (tEncodeI32(&encoder, pRsp->debugFlag) < 0) return -1; tEndEncode(&encoder); _exit: @@ -8196,6 +8198,10 @@ int32_t tDeserializeSMqHbRsp(void *buf, int32_t bufLen, SMqHbRsp *pRsp) { TAOS_CHECK_EXIT(tDecodeI8(&decoder, &data->noPrivilege)); } } + + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI32(&decoder, &pRsp->debugFlag) < 0) return -1; + } tEndDecode(&decoder); _exit: @@ -10681,7 +10687,7 @@ int32_t tDecodeMqMetaRsp(SDecoder *pDecoder, SMqMetaRsp *pRsp) { void tDeleteMqMetaRsp(SMqMetaRsp *pRsp) { taosMemoryFree(pRsp->metaRsp); } -int32_t tEncodeMqDataRspCommon(SEncoder *pEncoder, const SMqDataRspCommon *pRsp) { +int32_t tEncodeMqDataRspCommon(SEncoder *pEncoder, const SMqDataRsp *pRsp) { int32_t code = 0; int32_t lino; @@ -10711,19 +10717,20 @@ _exit: return code; } -int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const void *pRsp) { +int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { TAOS_CHECK_RETURN(tEncodeMqDataRspCommon(pEncoder, pRsp)); - TAOS_CHECK_RETURN(tEncodeI64(pEncoder, ((SMqDataRsp *)pRsp)->sleepTime)); + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pRsp->sleepTime)); return 0; } -int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRspCommon *pRsp) { +int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRsp *pRsp) { int32_t code = 0; int32_t lino; TAOS_CHECK_EXIT(tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset)); TAOS_CHECK_EXIT(tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset)); TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->blockNum)); + if (pRsp->blockNum != 0) { if ((pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void *))) == NULL) { TAOS_CHECK_EXIT(terrno); @@ -10787,17 +10794,16 @@ _exit: return code; } -int32_t tDecodeMqDataRsp(SDecoder *pDecoder, void *pRsp) { +int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { TAOS_CHECK_RETURN(tDecodeMqDataRspCommon(pDecoder, pRsp)); if (!tDecodeIsEnd(pDecoder)) { - TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &((SMqDataRsp *)pRsp)->sleepTime)); + TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pRsp->sleepTime)); } return 0; } -static void tDeleteMqDataRspCommon(void *rsp) { - SMqDataRspCommon *pRsp = rsp; +static void tDeleteMqDataRspCommon(SMqDataRsp *pRsp) { taosArrayDestroy(pRsp->blockDataLen); pRsp->blockDataLen = NULL; taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree); @@ -10810,15 +10816,13 @@ static void tDeleteMqDataRspCommon(void *rsp) { tOffsetDestroy(&pRsp->rspOffset); } -void tDeleteMqDataRsp(void *rsp) { tDeleteMqDataRspCommon(rsp); } +void tDeleteMqDataRsp(SMqDataRsp *rsp) { tDeleteMqDataRspCommon(rsp); } -int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const void *rsp) { +int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { int32_t code = 0; int32_t lino; - TAOS_CHECK_EXIT(tEncodeMqDataRspCommon(pEncoder, rsp)); - - const STaosxRsp *pRsp = (const STaosxRsp *)rsp; + TAOS_CHECK_EXIT(tEncodeMqDataRspCommon(pEncoder, pRsp)); TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->createTableNum)); if (pRsp->createTableNum) { for (int32_t i = 0; i < pRsp->createTableNum; i++) { @@ -10831,13 +10835,11 @@ _exit: return code; } -int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, void *rsp) { +int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { int32_t code = 0; int32_t lino; - TAOS_CHECK_EXIT(tDecodeMqDataRspCommon(pDecoder, rsp)); - - STaosxRsp *pRsp = (STaosxRsp *)rsp; + TAOS_CHECK_EXIT(tDecodeMqDataRspCommon(pDecoder, pRsp)); TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->createTableNum)); if (pRsp->createTableNum) { if ((pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t))) == NULL) { @@ -10864,10 +10866,9 @@ _exit: return code; } -void tDeleteSTaosxRsp(void *rsp) { - tDeleteMqDataRspCommon(rsp); +void tDeleteSTaosxRsp(SMqDataRsp *pRsp) { + tDeleteMqDataRspCommon(pRsp); - STaosxRsp *pRsp = (STaosxRsp *)rsp; taosArrayDestroy(pRsp->createTableLen); pRsp->createTableLen = NULL; taosArrayDestroyP(pRsp->createTableReq, (FDelete)taosMemoryFree); diff --git a/source/common/src/tname.c b/source/common/src/tname.c index a9dda87591..964741e2c4 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -33,7 +33,7 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in } struct tm tm; time_t t = (time_t)start; - taosLocalTime(&t, &tm); + taosLocalTime(&t, &tm, NULL, 0); tm.tm_sec = 0; tm.tm_min = 0; tm.tm_hour = 0; @@ -103,7 +103,7 @@ int32_t tNameExtractFullName(const SName* name, char* dst) { return TSDB_CODE_INVALID_PARA; } - int32_t len = snprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname); + int32_t len = tsnprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname); size_t tnameLen = strlen(name->tname); if (tnameLen > 0) { diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 98e46ab672..75624593d9 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -693,7 +693,7 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { struct tm tm; time_t tt = (time_t)(t / TSDB_TICK_PER_SECOND(precision)); - struct tm* ptm = taosLocalTime(&tt, &tm, NULL); + struct tm* ptm = taosLocalTime(&tt, &tm, NULL, 0); int32_t mon = tm.tm_year * 12 + tm.tm_mon + (int32_t)numOfMonth; tm.tm_year = mon / 12; tm.tm_mon = mon % 12; @@ -754,11 +754,11 @@ int32_t taosTimeCountIntervalForFill(int64_t skey, int64_t ekey, int64_t interva struct tm tm; time_t t = (time_t)skey; - struct tm* ptm = taosLocalTime(&t, &tm, NULL); + struct tm* ptm = taosLocalTime(&t, &tm, NULL, 0); int32_t smon = tm.tm_year * 12 + tm.tm_mon; t = (time_t)ekey; - ptm = taosLocalTime(&t, &tm, NULL); + ptm = taosLocalTime(&t, &tm, NULL, 0); int32_t emon = tm.tm_year * 12 + tm.tm_mon; if (unit == 'y') { @@ -782,7 +782,7 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { start /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); struct tm tm; time_t tt = (time_t)start; - struct tm* ptm = taosLocalTime(&tt, &tm, NULL); + struct tm* ptm = taosLocalTime(&tt, &tm, NULL, 0); tm.tm_sec = 0; tm.tm_min = 0; tm.tm_hour = 0; @@ -911,13 +911,13 @@ int64_t taosTimeGetIntervalEnd(int64_t intervalStart, const SInterval* pInterval // 2020-07-03 17:48:42 // and the parameter can also be a variable. const char* fmtts(int64_t ts) { - static char buf[96] = {0}; + static char buf[TD_TIME_STR_LEN] = {0}; size_t pos = 0; struct tm tm; if (ts > -62135625943 && ts < 32503651200) { time_t t = (time_t)ts; - if (taosLocalTime(&t, &tm, buf) == NULL) { + if (taosLocalTime(&t, &tm, buf, sizeof(buf)) == NULL) { return buf; } pos += strftime(buf + pos, sizeof(buf), "s=%Y-%m-%d %H:%M:%S", &tm); @@ -925,7 +925,7 @@ const char* fmtts(int64_t ts) { if (ts > -62135625943000 && ts < 32503651200000) { time_t t = (time_t)(ts / 1000); - if (taosLocalTime(&t, &tm, buf) == NULL) { + if (taosLocalTime(&t, &tm, buf, sizeof(buf)) == NULL) { return buf; } if (pos > 0) { @@ -939,7 +939,7 @@ const char* fmtts(int64_t ts) { { time_t t = (time_t)(ts / 1000000); - if (taosLocalTime(&t, &tm, buf) == NULL) { + if (taosLocalTime(&t, &tm, buf, sizeof(buf)) == NULL) { return buf; } if (pos > 0) { @@ -993,11 +993,11 @@ int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precisio TAOS_RETURN(TSDB_CODE_INVALID_PARA); } - if (NULL == taosLocalTime(", &ptm, buf)) { + if (NULL == taosLocalTime(", &ptm, buf, bufLen)) { TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm); - length += snprintf(ts + length, fractionLen, format, mod); + length += tsnprintf(ts + length, fractionLen, format, mod); length += (int32_t)strftime(ts + length, 40 - length, "%z", &ptm); tstrncpy(buf, ts, bufLen); @@ -1007,7 +1007,7 @@ int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precisio int32_t taosTs2Tm(int64_t ts, int32_t precision, struct STm* tm) { tm->fsec = ts % TICK_PER_SECOND[precision] * (TICK_PER_SECOND[TSDB_TIME_PRECISION_NANO] / TICK_PER_SECOND[precision]); time_t t = ts / TICK_PER_SECOND[precision]; - if (NULL == taosLocalTime(&t, &tm->tm, NULL)) { + if (NULL == taosLocalTime(&t, &tm->tm, NULL, 0)) { TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } return TSDB_CODE_SUCCESS; diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index e8b7b132f2..ebf91025bb 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -480,7 +480,7 @@ TEST(testCase, StreamAllNormTest) { char ctbName[TSDB_TABLE_NAME_LEN] = {0}; uint64_t groupId = 12345; - buildCtbNameAddGroupId(NULL, ctbName, groupId); + buildCtbNameAddGroupId(NULL, ctbName, groupId, sizeof(ctbName)); ASSERT_STREQ("_12345", ctbName); } @@ -490,7 +490,7 @@ TEST(testCase, StreamWithStbName) { char ctbName[TSDB_TABLE_NAME_LEN] = {0}; uint64_t groupId = 12345; - buildCtbNameAddGroupId(stbName, ctbName, groupId); + buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)); ASSERT_STREQ("_stb_12345", ctbName); } @@ -500,7 +500,7 @@ TEST(testCase, StreamWithoutDotInStbName) { char ctbName[TSDB_TABLE_NAME_LEN] = {0}; uint64_t groupId = 12345; - buildCtbNameAddGroupId(stbName, ctbName, groupId); + buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)); ASSERT_STREQ("_table_12345", ctbName); } @@ -510,11 +510,59 @@ TEST(testCase, StreamWithoutDotInStbName2) { char ctbName[TSDB_TABLE_NAME_LEN] = {0}; uint64_t groupId = 12345; - buildCtbNameAddGroupId(stbName, ctbName, groupId); + buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)); ASSERT_STREQ("__12345", ctbName); } +TEST(testCase, StreamWithLongStbName) { + char ctbName[TSDB_TABLE_NAME_LEN]; + char expectName[TSDB_TABLE_NAME_LEN]; + char *stbName = "a_simle_stb_name"; + uint64_t groupId = UINT64_MAX; + + // test basic function + strcpy(ctbName, "a_simple_ctb_name"); + EXPECT_EQ(buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)), TSDB_CODE_SUCCESS); + EXPECT_STREQ(ctbName, "a_simple_ctb_name_a_simle_stb_name_18446744073709551615"); + + // test null stbName + strcpy(ctbName, "a_simple_ctb_name"); + stbName = NULL; + EXPECT_EQ(buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)), TSDB_CODE_SUCCESS); + EXPECT_STREQ(ctbName, "a_simple_ctb_name_18446744073709551615"); + + // test buffer capcity check + EXPECT_EQ(buildCtbNameAddGroupId(stbName, NULL, groupId, sizeof(ctbName)), TSDB_CODE_INTERNAL_ERROR); + EXPECT_EQ(buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName) - 1), TSDB_CODE_INTERNAL_ERROR); + + // test md5 conversion of stbName with groupid + for (int32_t i = 0; i < 159; ++i) ctbName[i] = 'A'; + ctbName[159] = '\0'; + stbName = taosStrdup(ctbName); + snprintf(expectName, TSDB_TABLE_NAME_LEN, "%s_d85f0d87946d76eeedd7b7b78b7492a2", ctbName); + EXPECT_EQ(buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)), TSDB_CODE_SUCCESS); + EXPECT_STREQ(ctbName, expectName); + + // test md5 conversion of all parts + for (int32_t i = 0; i < 190; ++i) ctbName[i] = 'A'; + ctbName[190] = '\0'; + tstrncpy(expectName, "t_d38a8b2df999bef0082ffc80a59a9cd7_d85f0d87946d76eeedd7b7b78b7492a2", TSDB_TABLE_NAME_LEN); + EXPECT_EQ(buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)), TSDB_CODE_SUCCESS); + EXPECT_STREQ(ctbName, expectName); + + // test larger stbName + taosMemoryFree(stbName); + for (int32_t i = 0; i < 190; ++i) ctbName[i] = 'A'; + ctbName[190] = '\0'; + stbName = taosStrdup(ctbName); + tstrncpy(expectName, "t_d38a8b2df999bef0082ffc80a59a9cd7_9c99cc7c52073b63fb750af402d9b84b", TSDB_TABLE_NAME_LEN); + EXPECT_EQ(buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)), TSDB_CODE_SUCCESS); + EXPECT_STREQ(ctbName, expectName); + + taosMemoryFree(stbName); +} + #if 1 TEST(testCase, NoneTest) { const static int nCols = 14; @@ -570,4 +618,4 @@ for (int r = 0; r < nRows; ++r) { taosArrayDestroy(pArray); taosMemoryFree(pTSchema); } -#endif \ No newline at end of file +#endif diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 89569d69d6..ddef0537f8 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -280,9 +280,9 @@ static void dmPrintArgs(int32_t argc, char const *argv[]) { taosGetCwd(path, sizeof(path)); char args[1024] = {0}; - int32_t arglen = snprintf(args, sizeof(args), "%s", argv[0]); + int32_t arglen = tsnprintf(args, sizeof(args), "%s", argv[0]); for (int32_t i = 1; i < argc; ++i) { - arglen = arglen + snprintf(args + arglen, sizeof(args) - arglen, " %s", argv[i]); + arglen = arglen + tsnprintf(args + arglen, sizeof(args) - arglen, " %s", argv[i]); } dInfo("startup path:%s args:%s", path, args); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 419c669103..f1f3a3bee7 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -39,6 +39,7 @@ static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) { (void)taosThreadRwlockUnlock(&pMgmt->pData->lock); } } + static void dmMayShouldUpdateIpWhiteList(SDnodeMgmt *pMgmt, int64_t ver) { int32_t code = 0; dDebug("ip-white-list on dnode ver: %" PRId64 ", status ver: %" PRId64 "", pMgmt->pData->ipWhiteVer, ver); @@ -84,6 +85,7 @@ static void dmMayShouldUpdateIpWhiteList(SDnodeMgmt *pMgmt, int64_t ver) { dError("failed to send retrieve ip white list request since:%s", tstrerror(code)); } } + static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { const STraceId *trace = &pRsp->info.traceId; dGTrace("status rsp received from mnode, statusSeq:%d code:0x%x", pMgmt->statusSeq, pRsp->code); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index 1561ab0a6b..22c2b2f5b2 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -77,7 +77,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { } if ((code = udfStartUdfd(pMgmt->pData->dnodeId)) != 0) { - dError("failed to start udfd"); + dError("failed to start udfd since %s", tstrerror(code)); } pOutput->pMgmt = pMgmt; diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index 6b01b92445..0e1a4bc98e 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -77,6 +77,7 @@ typedef struct { typedef struct { int32_t vnodeNum; int32_t opened; + int32_t dropped; int32_t failed; bool updateVnodesList; int32_t threadIndex; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index d081e70ff0..3cf0382eba 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -311,6 +311,8 @@ static void *vmOpenVnodeInThread(void *param) { snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pCfg->vgId); vnodeDestroy(pCfg->vgId, path, pMgmt->pTfs, 0); pThread->updateVnodesList = true; + pThread->dropped++; + (void)atomic_add_fetch_32(&pMgmt->state.dropVnodes, 1); continue; } @@ -352,8 +354,8 @@ static void *vmOpenVnodeInThread(void *param) { (void)atomic_add_fetch_32(&pMgmt->state.openVnodes, 1); } - dInfo("thread:%d, numOfVnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, - pThread->failed); + dInfo("thread:%d, numOfVnodes:%d, opened:%d dropped:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, + pThread->opened, pThread->dropped, pThread->failed); return NULL; } @@ -427,7 +429,7 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) { taosMemoryFree(threads); taosMemoryFree(pCfgs); - if (pMgmt->state.openVnodes != pMgmt->state.totalVnodes) { + if ((pMgmt->state.openVnodes + pMgmt->state.dropVnodes) != pMgmt->state.totalVnodes) { dError("there are total vnodes:%d, opened:%d", pMgmt->state.totalVnodes, pMgmt->state.openVnodes); terrno = TSDB_CODE_VND_INIT_FAILED; return -1; @@ -774,6 +776,7 @@ static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) { } pMgmt->state.openVnodes = 0; + pMgmt->state.dropVnodes = 0; dInfo("restore %d vnodes with %d threads", numOfVnodes, threadNum); for (int32_t t = 0; t < threadNum; ++t) { diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index f77571c665..277dd2e02a 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -65,7 +65,7 @@ int32_t dmInitDnode(SDnode *pDnode) { snprintf(path, sizeof(path), "%s%s%s", tsDataDir, TD_DIRSEP, pWrapper->name); pWrapper->path = taosStrdup(path); if (pWrapper->path == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index b9f4ab54f4..3090903805 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -105,6 +105,7 @@ static bool dmIsForbiddenIp(int8_t forbidden, char *user, uint32_t clientIp) { return false; } } + static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { SDnodeTrans *pTrans = &pDnode->trans; int32_t code = -1; @@ -150,10 +151,9 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { dmSetMnodeEpSet(&pDnode->data, pEpSet); } break; - case TDMT_MND_RETRIEVE_IP_WHITE_RSP: { + case TDMT_MND_RETRIEVE_IP_WHITE_RSP: dmUpdateRpcIpWhite(&pDnode->data, pTrans->serverRpc, pRpc); return; - } break; default: break; } diff --git a/source/dnode/mgmt/node_util/src/dmEps.c b/source/dnode/mgmt/node_util/src/dmEps.c index db401375c7..49c1bdf3df 100644 --- a/source/dnode/mgmt/node_util/src/dmEps.c +++ b/source/dnode/mgmt/node_util/src/dmEps.c @@ -461,12 +461,12 @@ void dmGetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet) { void dmEpSetToStr(char *buf, int32_t len, SEpSet *epSet) { int32_t n = 0; - n += snprintf(buf + n, len - n, "%s", "{"); + n += tsnprintf(buf + n, len - n, "%s", "{"); for (int i = 0; i < epSet->numOfEps; i++) { - n += snprintf(buf + n, len - n, "%s:%d%s", epSet->eps[i].fqdn, epSet->eps[i].port, + n += tsnprintf(buf + n, len - n, "%s:%d%s", epSet->eps[i].fqdn, epSet->eps[i].port, (i + 1 < epSet->numOfEps ? ", " : "")); } - n += snprintf(buf + n, len - n, "%s", "}"); + n += tsnprintf(buf + n, len - n, "%s", "}"); } static FORCE_INLINE void dmSwapEps(SEp *epLhs, SEp *epRhs) { diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index ceaf086dc1..cb148c1949 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -14,6 +14,7 @@ IF (TD_ENTERPRISE) ELSEIF(${BUILD_WITH_COS}) add_definitions(-DUSE_COS) ENDIF() + ENDIF () add_library(mnode STATIC ${MNODE_SRC}) diff --git a/source/dnode/mnode/impl/src/mndArbGroup.c b/source/dnode/mnode/impl/src/mndArbGroup.c index 61dd13eb45..97bf661bc3 100644 --- a/source/dnode/mnode/impl/src/mndArbGroup.c +++ b/source/dnode/mnode/impl/src/mndArbGroup.c @@ -445,7 +445,7 @@ static int32_t mndProcessArbHbTimer(SRpcMsg *pReq) { SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId); if (pDnode == NULL) { - mError("dnodeId:%d, timer failed to send arb-hb request, failed find dnode", dnodeId); + mError("dnodeId:%d, timer failed to acquire dnode", dnodeId); taosArrayDestroy(hbMembers); continue; } @@ -453,7 +453,10 @@ static int32_t mndProcessArbHbTimer(SRpcMsg *pReq) { int64_t mndTerm = mndGetTerm(pMnode); if (mndIsDnodeOnline(pDnode, nowMs)) { - TAOS_CHECK_RETURN(mndSendArbHeartBeatReq(pDnode, arbToken, mndTerm, hbMembers)); + int32_t sendCode = mndSendArbHeartBeatReq(pDnode, arbToken, mndTerm, hbMembers); + if (TSDB_CODE_SUCCESS != sendCode) { + mError("dnodeId:%d, timer failed to send arb-hb request", dnodeId); + } } mndReleaseDnode(pMnode, pDnode); diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 5e38a91ab0..3779bea564 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -241,7 +241,7 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) { clusterObj.createdTime = taosGetTimestampMs(); clusterObj.updateTime = clusterObj.createdTime; - int32_t code = taosGetSystemUUID(clusterObj.name, TSDB_CLUSTER_ID_LEN); + int32_t code = taosGetSystemUUIDLen(clusterObj.name, TSDB_CLUSTER_ID_LEN); if (code != 0) { (void)strcpy(clusterObj.name, "tdengine3.0"); mError("failed to get name from system, set to default val %s", clusterObj.name); diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c index 2e40965c18..106680da7f 100644 --- a/source/dnode/mnode/impl/src/mndCompact.c +++ b/source/dnode/mnode/impl/src/mndCompact.c @@ -640,10 +640,10 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) { rpcMsg.pCont = pHead; char detail[1024] = {0}; - int32_t len = snprintf(detail, sizeof(detail), "msgType:%s numOfEps:%d inUse:%d", + int32_t len = tsnprintf(detail, sizeof(detail), "msgType:%s numOfEps:%d inUse:%d", TMSG_INFO(TDMT_VND_QUERY_COMPACT_PROGRESS), epSet.numOfEps, epSet.inUse); for (int32_t i = 0; i < epSet.numOfEps; ++i) { - len += snprintf(detail + len, sizeof(detail) - len, " ep:%d-%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); + len += tsnprintf(detail + len, sizeof(detail) - len, " ep:%d-%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); } mDebug("compact:%d, send update progress msg to %s", pDetail->compactId, detail); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 4365b83f86..9f7c163eec 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -167,7 +167,7 @@ static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbRs } STopicPrivilege *data = taosArrayReserve(rsp->topicPrivileges, 1); MND_TMQ_NULL_CHECK(data); - (void)strcpy(data->topic, topic); + tstrncpy(data->topic, topic, TSDB_TOPIC_FNAME_LEN); if (mndCheckTopicPrivilege(pMnode, user, MND_OPER_SUBSCRIBE, pTopic) != 0 || grantCheckExpire(TSDB_GRANT_SUBSCRIPTION) < 0) { data->noPrivilege = 1; @@ -244,6 +244,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { } storeOffsetRows(pMnode, &req, pConsumer); + rsp.debugFlag = tqClientDebug; code = buildMqHbRsp(pMsg, &rsp); END: @@ -277,7 +278,7 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e taosRLockLatch(&pSub->lock); SMqSubTopicEp topicEp = {0}; - (void)strcpy(topicEp.topic, topic); + tstrncpy(topicEp.topic, topic, TSDB_TOPIC_FNAME_LEN); // 2.1 fetch topic schema SMqTopicObj *pTopic = NULL; @@ -587,8 +588,8 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { SCMSubscribeReq subscribe = {0}; MND_TMQ_RETURN_CHECK(tDeserializeSCMSubscribeReq(msgStr, &subscribe, pMsg->contLen)); - bool ubSubscribe = (taosArrayGetSize(subscribe.topicNames) == 0); - if(ubSubscribe){ + bool unSubscribe = (taosArrayGetSize(subscribe.topicNames) == 0); + if(unSubscribe){ SMqConsumerObj *pConsumerTmp = NULL; MND_TMQ_RETURN_CHECK(mndAcquireConsumer(pMnode, subscribe.consumerId, &pConsumerTmp)); if (taosArrayGetSize(pConsumerTmp->assignedTopics) == 0){ @@ -599,7 +600,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { } MND_TMQ_RETURN_CHECK(checkAndSortTopic(pMnode, subscribe.topicNames)); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, - (ubSubscribe ? TRN_CONFLICT_NOTHING :TRN_CONFLICT_DB_INSIDE), + (unSubscribe ? TRN_CONFLICT_NOTHING :TRN_CONFLICT_DB_INSIDE), pMsg, "subscribe"); MND_TMQ_NULL_CHECK(pTrans); @@ -909,7 +910,7 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * // consumer id char consumerIdHex[TSDB_CONSUMER_ID_LEN + VARSTR_HEADER_SIZE] = {0}; - (void)sprintf(varDataVal(consumerIdHex), "0x%" PRIx64, pConsumer->consumerId); + (void)snprintf(varDataVal(consumerIdHex), TSDB_CONSUMER_ID_LEN, "0x%" PRIx64, pConsumer->consumerId); varDataSetLen(consumerIdHex, strlen(varDataVal(consumerIdHex))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -992,7 +993,7 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * parasStr = taosMemoryCalloc(1, pShow->pMeta->pSchemas[cols].bytes); MND_TMQ_NULL_CHECK(parasStr); - (void)sprintf(varDataVal(parasStr), "tbname:%d,commit:%d,interval:%dms,reset:%s", pConsumer->withTbName, + (void)snprintf(varDataVal(parasStr), pShow->pMeta->pSchemas[cols].bytes - VARSTR_HEADER_SIZE, "tbname:%d,commit:%d,interval:%dms,reset:%s", pConsumer->withTbName, pConsumer->autoCommit, pConsumer->autoCommitInterval, buf); varDataSetLen(parasStr, strlen(varDataVal(parasStr))); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 0403029f74..7c42564f4c 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -2366,7 +2366,7 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)strictVstr, false), &lino, _OVER); char durationVstr[128] = {0}; - int32_t len = formatDurationOrKeep(&durationVstr[VARSTR_HEADER_SIZE], pDb->cfg.daysPerFile); + int32_t len = formatDurationOrKeep(&durationVstr[VARSTR_HEADER_SIZE], sizeof(durationVstr) - VARSTR_HEADER_SIZE, pDb->cfg.daysPerFile); varDataSetLen(durationVstr, len); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -2377,9 +2377,9 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, char keep1Str[128] = {0}; char keep2Str[128] = {0}; - int32_t lenKeep0 = formatDurationOrKeep(keep0Str, pDb->cfg.daysToKeep0); - int32_t lenKeep1 = formatDurationOrKeep(keep1Str, pDb->cfg.daysToKeep1); - int32_t lenKeep2 = formatDurationOrKeep(keep2Str, pDb->cfg.daysToKeep2); + int32_t lenKeep0 = formatDurationOrKeep(keep0Str, sizeof(keep0Str), pDb->cfg.daysToKeep0); + int32_t lenKeep1 = formatDurationOrKeep(keep1Str, sizeof(keep1Str), pDb->cfg.daysToKeep1); + int32_t lenKeep2 = formatDurationOrKeep(keep2Str, sizeof(keep2Str), pDb->cfg.daysToKeep2); if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) { len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%s,%s,%s", keep1Str, keep2Str, keep0Str); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index a94a471e4b..04041646eb 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -693,7 +693,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { int64_t clusterid = mndGetClusterId(pMnode); if (statusReq.clusterId != 0 && statusReq.clusterId != clusterid) { code = TSDB_CODE_MND_DNODE_DIFF_CLUSTER; - mWarn("dnode:%d, %s, its clusterid:%" PRId64 " differ from current cluster:%" PRId64 ", code:0x%x", + mWarn("dnode:%d, %s, its clusterid:%" PRId64 " differ from current clusterid:%" PRId64 ", code:0x%x", statusReq.dnodeId, statusReq.dnodeEp, statusReq.clusterId, clusterid, code); goto _OVER; } @@ -740,7 +740,6 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { bool enableWhiteListChanged = statusReq.clusterCfg.enableWhiteList != (tsEnableWhiteList ? 1 : 0); bool needCheck = !online || dnodeChanged || reboot || supportVnodesChanged || pMnode->ipWhiteVer != statusReq.ipWhiteVer || encryptKeyChanged || enableWhiteListChanged; - const STraceId *trace = &pReq->info.traceId; mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id, pDnode->accessTimes, needCheck, online, reboot, dnodeChanged, statusReq.statusSeq); diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c index 0b3a0998f0..718c34e85a 100644 --- a/source/dnode/mnode/impl/src/mndIndex.c +++ b/source/dnode/mnode/impl/src/mndIndex.c @@ -157,7 +157,7 @@ static void *mndBuildDropIdxReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStbOb pHead->contLen = htonl(len); pHead->vgId = htonl(pVgroup->vgId); - void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); + void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); int32_t ret = 0; if ((ret = tSerializeSDropIdxReq(pBuf, len - sizeof(SMsgHead), &req)) < 0) { terrno = ret; @@ -662,6 +662,8 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb pNew->pTags = NULL; pNew->pColumns = NULL; + pNew->pCmpr = NULL; + pNew->pTags = NULL; pNew->updateTime = taosGetTimestampMs(); pNew->lock = 0; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index eb855d28a8..bee971b966 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -495,7 +495,7 @@ static int32_t mndCreateDir(SMnode *pMnode, const char *path) { int32_t code = 0; pMnode->path = taosStrdup(path); if (pMnode->path == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; TAOS_RETURN(code); } @@ -574,6 +574,8 @@ static int32_t mndOpenSdb(SMnode *pMnode) { code = sdbReadFile(pMnode->pSdb); } + mInfo("vgId:1, mnode sdb is opened, with applied index:%" PRId64, pMnode->pSdb->commitIndex); + atomic_store_64(&pMnode->applied, pMnode->pSdb->commitIndex); return code; } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 4dc2f093e8..a1ffee9b06 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -140,7 +140,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType SProfileMgmt *pMgmt = &pMnode->profileMgmt; char connStr[255] = {0}; - int32_t len = snprintf(connStr, sizeof(connStr), "%s%d%d%d%s", user, ip, port, pid, app); + int32_t len = tsnprintf(connStr, sizeof(connStr), "%s%d%d%d%s", user, ip, port, pid, app); uint32_t connId = mndGenerateUid(connStr, len); if (startTime == 0) startTime = taosGetTimestampMs(); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index a258155223..fa2538f245 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -1231,7 +1231,7 @@ static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp SNode *node = NULL; FOREACH(node, pList) { SFunctionNode *pFunc = (SFunctionNode *)node; - extOffset += snprintf(rsp->indexExts + extOffset, sizeof(rsp->indexExts) - extOffset - 1, "%s%s", + extOffset += tsnprintf(rsp->indexExts + extOffset, sizeof(rsp->indexExts) - extOffset - 1, "%s%s", (extOffset ? "," : ""), pFunc->functionName); } @@ -2221,10 +2221,10 @@ static int32_t mndRetrieveTSMA(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo int32_t len = 0; if (TSDB_CODE_SUCCESS == code) { if (!IS_CALENDAR_TIME_DURATION(pSma->intervalUnit)) { - len = snprintf(interval + VARSTR_HEADER_SIZE, 64, "%" PRId64 "%c", pSma->interval, + len = tsnprintf(interval + VARSTR_HEADER_SIZE, 64, "%" PRId64 "%c", pSma->interval, getPrecisionUnit(pSrcDb->cfg.precision)); } else { - len = snprintf(interval + VARSTR_HEADER_SIZE, 64, "%" PRId64 "%c", pSma->interval, pSma->intervalUnit); + len = tsnprintf(interval + VARSTR_HEADER_SIZE, 64, "%" PRId64 "%c", pSma->interval, pSma->intervalUnit); } varDataSetLen(interval, len); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -2235,7 +2235,7 @@ static int32_t mndRetrieveTSMA(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo if (TSDB_CODE_SUCCESS == code) { // create sql pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - len = snprintf(buf + VARSTR_HEADER_SIZE, TSDB_MAX_SAVED_SQL_LEN, "%s", pSma->sql); + len = tsnprintf(buf + VARSTR_HEADER_SIZE, TSDB_MAX_SAVED_SQL_LEN, "%s", pSma->sql); varDataSetLen(buf, TMIN(len, TSDB_MAX_SAVED_SQL_LEN)); code = colDataSetVal(pColInfo, numOfRows, buf, false); } @@ -2350,7 +2350,7 @@ int32_t dumpTSMAInfoFromSmaObj(const SSmaObj* pSma, const SStbObj* pDestStb, STa nodesDestroyNode(pNode); } pInfo->ast = taosStrdup(pSma->ast); - if (!pInfo->ast) code = TSDB_CODE_OUT_OF_MEMORY; + if (!pInfo->ast) code = terrno; if (code == TSDB_CODE_SUCCESS && pDestStb->numOfTags > 0) { pInfo->pTags = taosArrayInit(pDestStb->numOfTags, sizeof(SSchema)); diff --git a/source/dnode/mnode/impl/src/mndStreamTransAct.c b/source/dnode/mnode/impl/src/mndStreamTransAct.c index 3ecd192222..4e0bf97587 100644 --- a/source/dnode/mnode/impl/src/mndStreamTransAct.c +++ b/source/dnode/mnode/impl/src/mndStreamTransAct.c @@ -234,6 +234,7 @@ static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); if (code != TSDB_CODE_SUCCESS || !hasEpset) { mError("failed to extract epset during create update epset, code:%s", tstrerror(code)); + taosMemoryFree(pBuf); return code; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 93d8b6dcde..bcca01a230 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -187,12 +187,12 @@ static void mndSplitSubscribeKey(const char *key, char *topic, char *cgroup, boo (void)memcpy(cgroup, key, i); cgroup[i] = 0; if (fullName) { - (void)strcpy(topic, &key[i + 1]); + tstrncpy(topic, &key[i + 1], TSDB_TOPIC_FNAME_LEN); } else { while (key[i] != '.') { i++; } - (void)strcpy(topic, &key[i + 1]); + tstrncpy(topic, &key[i + 1], TSDB_CGROUP_LEN); } } @@ -1361,7 +1361,7 @@ static int32_t buildResult(SSDataBlock *pBlock, int32_t *numOfRows, int64_t cons // consumer id char consumerIdHex[TSDB_CONSUMER_ID_LEN] = {0}; - (void)sprintf(varDataVal(consumerIdHex), "0x%" PRIx64, consumerId); + (void)snprintf(varDataVal(consumerIdHex), TSDB_CONSUMER_ID_LEN - VARSTR_HEADER_SIZE, "0x%" PRIx64, consumerId); varDataSetLen(consumerIdHex, strlen(varDataVal(consumerIdHex))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -1398,7 +1398,8 @@ static int32_t buildResult(SSDataBlock *pBlock, int32_t *numOfRows, int64_t cons // vg id char buf[TSDB_OFFSET_LEN * 2 + VARSTR_HEADER_SIZE] = {0}; (void)tFormatOffset(varDataVal(buf), TSDB_OFFSET_LEN, &data->offset); - (void)sprintf(varDataVal(buf) + strlen(varDataVal(buf)), "/%" PRId64, data->ever); + (void)snprintf(varDataVal(buf) + strlen(varDataVal(buf)), + sizeof(buf) - VARSTR_HEADER_SIZE - strlen(varDataVal(buf)), "/%" PRId64, data->ever); varDataSetLen(buf, strlen(varDataVal(buf))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); MND_TMQ_NULL_CHECK(pColInfo); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 50b8b3e275..b5a74e865f 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -637,7 +637,7 @@ void mndSyncStop(SMnode *pMnode) { (void)taosThreadMutexLock(&pMgmt->lock); if (pMgmt->transId != 0) { - mInfo("vgId:1, is stopped and post sem, trans:%d", pMgmt->transId); + mInfo("vgId:1, trans:%d, is stopped and post sem", pMgmt->transId); pMgmt->transId = 0; pMgmt->transSec = 0; pMgmt->errCode = TSDB_CODE_APP_IS_STOPPING; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 6f7b24eab2..fcf7905cd1 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -589,6 +589,7 @@ STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) { SSdb *pSdb = pMnode->pSdb; + if (pTrans != NULL) mInfo("vgId:1, trans:%d, release transaction", pTrans->id); sdbRelease(pSdb, pTrans); } @@ -1131,10 +1132,11 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { if (!sendRsp) { return; } else { - mInfo("trans:%d, send rsp, stage:%s failedTimes:%d code:0x%x", pTrans->id, mndTransStr(pTrans->stage), - pTrans->failedTimes, code); + mInfo("vgId:1, trans:%d, start to send rsp, stage:%s failedTimes:%d code:0x%x", pTrans->id, + mndTransStr(pTrans->stage), pTrans->failedTimes, code); } + mInfo("vgId:1, trans:%d, start to lock rpc array", pTrans->id); taosWLockLatch(&pTrans->lockRpcArray); int32_t size = taosArrayGetSize(pTrans->pRpcArray); if (size <= 0) { @@ -1155,8 +1157,8 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { if (i != 0 && code == 0) { code = TSDB_CODE_MNODE_NOT_FOUND; } - mInfo("trans:%d, client:%d send rsp, code:0x%x stage:%s app:%p", pTrans->id, i, code, mndTransStr(pTrans->stage), - pInfo->ahandle); + mInfo("vgId:1, trans:%d, client:%d start to send rsp, code:0x%x stage:%s app:%p", pTrans->id, i, code, + mndTransStr(pTrans->stage), pInfo->ahandle); SRpcMsg rspMsg = {.code = code, .info = *pInfo}; @@ -1199,6 +1201,9 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { } tmsgSendRsp(&rspMsg); + + mInfo("vgId:1, trans:%d, client:%d send rsp finished, code:0x%x stage:%s app:%p", pTrans->id, i, code, + mndTransStr(pTrans->stage), pInfo->ahandle); } } taosArrayClear(pTrans->pRpcArray); @@ -1338,10 +1343,10 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen); char detail[1024] = {0}; - int32_t len = snprintf(detail, sizeof(detail), "msgType:%s numOfEps:%d inUse:%d", TMSG_INFO(pAction->msgType), + int32_t len = tsnprintf(detail, sizeof(detail), "msgType:%s numOfEps:%d inUse:%d", TMSG_INFO(pAction->msgType), pAction->epSet.numOfEps, pAction->epSet.inUse); for (int32_t i = 0; i < pAction->epSet.numOfEps; ++i) { - len += snprintf(detail + len, sizeof(detail) - len, " ep:%d-%s:%u", i, pAction->epSet.eps[i].fqdn, + len += tsnprintf(detail + len, sizeof(detail) - len, " ep:%d-%s:%u", i, pAction->epSet.eps[i].fqdn, pAction->epSet.eps[i].port); } @@ -1495,7 +1500,7 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr return code; } - mInfo("trans:%d, execute %d actions serial, current redoAction:%d", pTrans->id, numOfActions, pTrans->actionPos); + mInfo("trans:%d, execute %d actions serial, current action:%d", pTrans->id, numOfActions, pTrans->actionPos); for (int32_t action = pTrans->actionPos; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pActions, action); @@ -1637,6 +1642,7 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool pTrans->code = code; bool continueExec = true; if (code != 0 && code != TSDB_CODE_MND_TRANS_CTX_SWITCH) { + taosMsleep(100); continueExec = true; } else { continueExec = false; @@ -1767,7 +1773,8 @@ static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans, bool to if (code == 0) { pTrans->stage = TRN_STAGE_UNDO_ACTION; - mInfo("trans:%d, stage from rollback to undoAction", pTrans->id); + pTrans->actionPos = 0; + mInfo("trans:%d, stage from rollback to undoAction, actionPos:%d", pTrans->id, pTrans->actionPos); continueExec = true; } else { pTrans->failedTimes++; @@ -2017,14 +2024,14 @@ static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl char lastInfo[TSDB_TRANS_ERROR_LEN + VARSTR_HEADER_SIZE] = {0}; char detail[TSDB_TRANS_ERROR_LEN + 1] = {0}; - int32_t len = snprintf(detail, sizeof(detail), "action:%d code:0x%x(%s) ", pTrans->lastAction, + int32_t len = tsnprintf(detail, sizeof(detail), "action:%d code:0x%x(%s) ", pTrans->lastAction, pTrans->lastErrorNo & 0xFFFF, tstrerror(pTrans->lastErrorNo)); SEpSet epset = pTrans->lastEpset; if (epset.numOfEps > 0) { - len += snprintf(detail + len, sizeof(detail) - len, "msgType:%s numOfEps:%d inUse:%d ", + len += tsnprintf(detail + len, sizeof(detail) - len, "msgType:%s numOfEps:%d inUse:%d ", TMSG_INFO(pTrans->lastMsgType), epset.numOfEps, epset.inUse); for (int32_t i = 0; i < pTrans->lastEpset.numOfEps; ++i) { - len += snprintf(detail + len, sizeof(detail) - len, "ep:%d-%s:%u ", i, epset.eps[i].fqdn, epset.eps[i].port); + len += tsnprintf(detail + len, sizeof(detail) - len, "ep:%d-%s:%u ", i, epset.eps[i].fqdn, epset.eps[i].port); } } STR_WITH_MAXSIZE_TO_VARSTR(lastInfo, detail, pShow->pMeta->pSchemas[cols].bytes); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 99472ca457..63390d4772 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -594,7 +594,7 @@ int32_t mndFetchAllIpWhite(SMnode *pMnode, SHashObj **ppIpWhiteTab) { if (name == NULL) { sdbRelease(pSdb, pUser); sdbCancelFetch(pSdb, pIter); - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } if (taosArrayPush(pUserNames, &name) == NULL) { taosMemoryFree(name); @@ -617,7 +617,7 @@ int32_t mndFetchAllIpWhite(SMnode *pMnode, SHashObj **ppIpWhiteTab) { if (found == false) { char *name = taosStrdup(TSDB_DEFAULT_USER); if (name == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } if (taosArrayPush(pUserNames, &name) == NULL) { taosMemoryFree(name); diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index 03098d93e0..8d0898e8ac 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -168,11 +168,10 @@ static int32_t sdbCreateDir(SSdb *pSdb) { } void sdbSetApplyInfo(SSdb *pSdb, int64_t index, int64_t term, int64_t config) { -#if 1 - mTrace("mnode apply info changed from index:%" PRId64 " term:%" PRId64 " config:%" PRId64 " to index:%" PRId64 - " term:%" PRId64 " config:%" PRId64, - pSdb->applyIndex, pSdb->applyTerm, pSdb->applyConfig, index, term, config); -#endif + mInfo("vgId:1, mnode apply info changed from index:%" PRId64 " term:%" PRId64 " config:%" PRId64 " to index:%" PRId64 + " term:%" PRId64 " config:%" PRId64, + pSdb->applyIndex, pSdb->applyTerm, pSdb->applyConfig, index, term, config); + pSdb->applyIndex = index; pSdb->applyTerm = term; pSdb->applyConfig = config; diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 34a017a907..0a6dc45d51 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -173,6 +173,8 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) { return terrno; } + mInfo("vgId:1, write sdb file with sdb applyIndex:%" PRId64 " term:%" PRId64 " config:%" PRId64, pSdb->applyIndex, + pSdb->applyTerm, pSdb->applyConfig); if (taosWriteFile(pFile, &pSdb->applyIndex, sizeof(int64_t)) != sizeof(int64_t)) { return terrno; } @@ -554,6 +556,9 @@ int32_t sdbWriteFile(SSdb *pSdb, int32_t delta) { } if (code != 0) { mError("failed to write sdb file since %s", tstrerror(code)); + } else { + mInfo("write sdb file success, apply index:%" PRId64 " term:%" PRId64 " config:%" PRId64, pSdb->applyIndex, + pSdb->applyTerm, pSdb->applyConfig); } (void)taosThreadMutexUnlock(&pSdb->filelock); return code; diff --git a/source/dnode/mnode/sdb/src/sdbRow.c b/source/dnode/mnode/sdb/src/sdbRow.c index da5a232851..f76c9923e3 100644 --- a/source/dnode/mnode/sdb/src/sdbRow.c +++ b/source/dnode/mnode/sdb/src/sdbRow.c @@ -42,6 +42,7 @@ void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc) { // remove attached object such as trans SdbDeleteFp deleteFp = pSdb->deleteFps[pRow->type]; if (deleteFp != NULL) { + mInfo("vgId:1, deleteFp:%p, type:%s", deleteFp, sdbTableName(pRow->type)); (void)(*deleteFp)(pSdb, pRow->pObj, callFunc); } diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 141aff0337..653b47ff14 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -112,14 +112,14 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle); void tqDestroyTqHandle(void* data); // tqRead -int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqBatchMetaRsp* pBatchMetaRsp, STqOffsetVal* offset); +int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBatchMetaRsp* pBatchMetaRsp, STqOffsetVal* offset); int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset, const SMqPollReq* pRequest); int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t reqId); // tqExec -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded); -int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, void* pRsp, int32_t numOfCols, int8_t precision); -int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const void* pRsp, +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded); +int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision); +int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t type, int32_t vgId); void tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId); @@ -148,9 +148,9 @@ int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name); // tq util int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type); int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg); -int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const void* pRsp, int32_t epoch, int64_t consumerId, +int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, int32_t type, int64_t sver, int64_t ever); -int32_t tqInitDataRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset); +int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset); void tqUpdateNodeStage(STQ* pTq, bool isLeader); int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock, SSubmitTbData* pTableData, int64_t earlyTs, const char* id); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 30efee42e5..ed8f99ec75 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -342,6 +342,7 @@ typedef struct { rocksdb_writeoptions_t *writeoptions; rocksdb_readoptions_t *readoptions; rocksdb_writebatch_t *writebatch; + TdThreadMutex writeBatchMutex; STSchema *pTSchema; } SRocksCache; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 14207e7fb3..484c5c0a16 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1523,6 +1523,7 @@ int32_t metaGetTableTags(void *pVnode, uint64_t suid, SArray *pUidTagInfo) { } memcpy(info.pTagVal, pCur->pVal, pCur->vLen); if (taosArrayPush(pUidTagInfo, &info) == NULL) { + taosMemoryFreeClear(info.pTagVal); metaCloseCtbCursor(pCur); taosHashCleanup(pSepecifiedUidMap); return terrno; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 8814e87140..08ee422126 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -3133,7 +3133,12 @@ static void colCompressDebug(SHashObj *pColCmprObj) { int32_t metaGetColCmpr(SMeta *pMeta, tb_uid_t uid, SHashObj **ppColCmprObj) { int rc = 0; - SHashObj *pColCmprObj = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); + SHashObj *pColCmprObj = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); + if (pColCmprObj == NULL) { + pColCmprObj = NULL; + return TSDB_CODE_OUT_OF_MEMORY; + } + void *pData = NULL; int nData = 0; SMetaEntry e = {0}; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index d0047b8e3f..a37a9787c9 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -166,14 +166,14 @@ void tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) { } SMqDataRsp dataRsp = {0}; - code = tqInitDataRsp(&dataRsp.common, req.reqOffset); + code = tqInitDataRsp(&dataRsp, req.reqOffset); if (code != 0) { tqError("tqInitDataRsp failed, code:%d", code); return; } - dataRsp.common.blockNum = 0; + dataRsp.blockNum = 0; char buf[TSDB_OFFSET_LEN] = {0}; - (void)tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.reqOffset); + (void)tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.reqOffset); tqInfo("tqPushEmptyDataRsp to consumer:0x%" PRIx64 " vgId:%d, offset:%s,QID:0x%" PRIx64, req.consumerId, vgId, buf, req.reqId); @@ -184,18 +184,18 @@ void tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) { tDeleteMqDataRsp(&dataRsp); } -int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const void* pRsp, int32_t type, +int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t type, int32_t vgId) { int64_t sver = 0, ever = 0; walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); char buf1[TSDB_OFFSET_LEN] = {0}; char buf2[TSDB_OFFSET_LEN] = {0}; - (void)tFormatOffset(buf1, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->reqOffset); - (void)tFormatOffset(buf2, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->rspOffset); + (void)tFormatOffset(buf1, TSDB_OFFSET_LEN, &(pRsp->reqOffset)); + (void)tFormatOffset(buf2, TSDB_OFFSET_LEN, &(pRsp->rspOffset)); tqDebug("tmq poll vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s,QID:0x%" PRIx64, - vgId, pReq->consumerId, pReq->epoch, ((SMqDataRspCommon*)pRsp)->blockNum, buf1, buf2, pReq->reqId); + vgId, pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2, pReq->reqId); return tqDoSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type, sver, ever); } @@ -518,7 +518,7 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { taosRUnLockLatch(&pTq->lock); SMqDataRsp dataRsp = {0}; - code = tqInitDataRsp(&dataRsp.common, req.reqOffset); + code = tqInitDataRsp(&dataRsp, req.reqOffset); if (code != 0) { return code; } @@ -529,10 +529,10 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { goto END; } - dataRsp.common.rspOffset.type = TMQ_OFFSET__LOG; + dataRsp.rspOffset.type = TMQ_OFFSET__LOG; if (reqOffset.type == TMQ_OFFSET__LOG) { - dataRsp.common.rspOffset.version = reqOffset.version; + dataRsp.rspOffset.version = reqOffset.version; } else if (reqOffset.type < 0) { STqOffset* pOffset = NULL; code = tqMetaGetOffset(pTq, req.subKey, &pOffset); @@ -543,17 +543,17 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { goto END; } - dataRsp.common.rspOffset.version = pOffset->val.version; + dataRsp.rspOffset.version = pOffset->val.version; tqInfo("consumer:0x%" PRIx64 " vgId:%d subkey:%s get assignment from store:%" PRId64, consumerId, vgId, - req.subKey, dataRsp.common.rspOffset.version); + req.subKey, dataRsp.rspOffset.version); } else { if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEST) { - dataRsp.common.rspOffset.version = sver; // not consume yet, set the earliest position + dataRsp.rspOffset.version = sver; // not consume yet, set the earliest position } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { - dataRsp.common.rspOffset.version = ever; + dataRsp.rspOffset.version = ever; } tqInfo("consumer:0x%" PRIx64 " vgId:%d subkey:%s get assignment from init:%" PRId64, consumerId, vgId, req.subKey, - dataRsp.common.rspOffset.version); + dataRsp.rspOffset.version); } } else { tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s invalid offset type:%d", consumerId, vgId, req.subKey, diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 2c892998d2..f392269b9f 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -22,7 +22,7 @@ int32_t tqBuildFName(char** data, const char* path, char* name) { if(fname == NULL) { return terrno; } - int32_t code = snprintf(fname, len, "%s%s%s", path, TD_DIRSEP, name); + int32_t code = tsnprintf(fname, len, "%s%s%s", path, TD_DIRSEP, name); if (code < 0){ code = TAOS_SYSTEM_ERROR(errno); taosMemoryFree(fname); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 6dc5453d50..f2f85773b5 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -562,9 +562,18 @@ int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrap return 0; } -static int32_t buildResSDataBlock(SSDataBlock* pBlock, SSchemaWrapper* pSchema, const SArray* pColIdList) { +static int32_t buildResSDataBlock(STqReader* pReader, SSchemaWrapper* pSchema, const SArray* pColIdList) { + SSDataBlock* pBlock = pReader->pResBlock; if (blockDataGetNumOfCols(pBlock) > 0) { - return TSDB_CODE_SUCCESS; + blockDataDestroy(pBlock); + int32_t code = createDataBlock(&pReader->pResBlock); + if (code) { + return code; + } + pBlock = pReader->pResBlock; + + pBlock->info.id.uid = pReader->cachedSchemaUid; + pBlock->info.version = pReader->msg.ver; } int32_t numOfCols = taosArrayGetSize(pColIdList); @@ -678,10 +687,10 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* vgId, suid, uid, sversion, pReader->pSchemaWrapper->version); return TSDB_CODE_TQ_INTERNAL_ERROR; } - if (blockDataGetNumOfCols(pBlock) == 0) { - code = buildResSDataBlock(pReader->pResBlock, pReader->pSchemaWrapper, pReader->pColIdList); - TSDB_CHECK_CODE(code, line, END); - } + code = buildResSDataBlock(pReader, pReader->pSchemaWrapper, pReader->pColIdList); + TSDB_CHECK_CODE(code, line, END); + pBlock = pReader->pResBlock; + *pRes = pBlock; } int32_t numOfRows = 0; diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index f71669f158..dbc1b16cf5 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -15,7 +15,7 @@ #include "tq.h" -int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, void* pRsp, int32_t numOfCols, int8_t precision) { +int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision) { int32_t dataStrLen = sizeof(SRetrieveTableRspForTmq) + blockGetEncodeSize(pBlock); void* buf = taosMemoryCalloc(1, dataStrLen); if (buf == NULL) { @@ -34,11 +34,11 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, void* pRsp, int32_t numOf return terrno; } actualLen += sizeof(SRetrieveTableRspForTmq); - if (taosArrayPush(((SMqDataRspCommon*)pRsp)->blockDataLen, &actualLen) == NULL){ + if (taosArrayPush(pRsp->blockDataLen, &actualLen) == NULL){ taosMemoryFree(buf); return terrno; } - if (taosArrayPush(((SMqDataRspCommon*)pRsp)->blockData, &buf) == NULL) { + if (taosArrayPush(pRsp->blockData, &buf) == NULL) { taosMemoryFree(buf); return terrno; } @@ -46,18 +46,18 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, void* pRsp, int32_t numOf return TSDB_CODE_SUCCESS; } -static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, void* pRsp) { +static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, SMqDataRsp* pRsp) { SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pTqReader->pSchemaWrapper); if (pSW == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } - if (taosArrayPush(((SMqDataRspCommon*)pRsp)->blockSchema, &pSW) == NULL) { + if (taosArrayPush(pRsp->blockSchema, &pSW) == NULL) { return terrno; } return 0; } -static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, void* pRsp, int32_t n) { +static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, int32_t n) { SMetaReader mr = {0}; metaReaderDoInit(&mr, pTq->pVnode->pMeta, META_READER_LOCK); @@ -73,7 +73,8 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, void* pRsp, int32_t metaReaderClear(&mr); return terrno; } - if(taosArrayPush(((SMqDataRspCommon*)pRsp)->blockTbName, &tbName) == NULL){ + if(taosArrayPush(pRsp->blockTbName, &tbName) == NULL){ + tqError("failed to push tbName to blockTbName:%s", tbName); continue; } } @@ -143,7 +144,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* code = tqAddBlockDataToRsp(pHandle->block, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); TSDB_CHECK_CODE(code, line, END); - pRsp->common.blockNum++; + pRsp->blockNum++; if (pDataBlock == NULL) { blockDataDestroy(pHandle->block); pHandle->block = NULL; @@ -167,7 +168,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* code = tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); TSDB_CHECK_CODE(code, line, END); - pRsp->common.blockNum++; + pRsp->blockNum++; totalRows += pDataBlock->info.rows; if (totalRows >= tmqRowSize || (taosGetTimestampMs() - st > 1000)) { break; @@ -176,8 +177,8 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* } tqDebug("consumer:0x%" PRIx64 " vgId:%d tmq task executed finished, total blocks:%d, totalRows:%d", - pHandle->consumerId, vgId, pRsp->common.blockNum, totalRows); - code = qStreamExtractOffset(task, &pRsp->common.rspOffset); + pHandle->consumerId, vgId, pRsp->blockNum, totalRows); + code = qStreamExtractOffset(task, &pRsp->rspOffset); END: if (code != 0) { tqError("consumer:0x%" PRIx64 " vgId:%d tmq task executed error, line:%d code:%d", pHandle->consumerId, vgId, line, @@ -186,7 +187,7 @@ END: return code; } -int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqBatchMetaRsp* pBatchMetaRsp, STqOffsetVal* pOffset) { +int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBatchMetaRsp* pBatchMetaRsp, STqOffsetVal* pOffset) { const STqExecHandle* pExec = &pHandle->execHandle; qTaskInfo_t task = pExec->task; int code = qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType); @@ -208,7 +209,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqBatc tqDebug("tmqsnap task execute end, get %p", pDataBlock); if (pDataBlock != NULL && pDataBlock->info.rows > 0) { - if (pRsp->common.withTbName) { + if (pRsp->withTbName) { if (pOffset->type == TMQ_OFFSET__LOG) { int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, 1) < 0) { @@ -221,13 +222,13 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqBatc tqError("vgId:%d, failed to add tbname to rsp msg, null", pTq->pVnode->config.vgId); return terrno; } - if (taosArrayPush(pRsp->common.blockTbName, &tbName) == NULL){ + if (taosArrayPush(pRsp->blockTbName, &tbName) == NULL){ tqError("vgId:%d, failed to add tbname to rsp msg", pTq->pVnode->config.vgId); continue; } } } - if (pRsp->common.withSchema) { + if (pRsp->withSchema) { if (pOffset->type == TMQ_OFFSET__LOG) { if (tqAddBlockSchemaToRsp(pExec, pRsp) != 0){ tqError("vgId:%d, failed to add schema to rsp msg", pTq->pVnode->config.vgId); @@ -235,19 +236,19 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqBatc } } else { SSchemaWrapper* pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task)); - if(taosArrayPush(pRsp->common.blockSchema, &pSW) == NULL){ + if(taosArrayPush(pRsp->blockSchema, &pSW) == NULL){ tqError("vgId:%d, failed to add schema to rsp msg", pTq->pVnode->config.vgId); continue; } } } - if (tqAddBlockDataToRsp(pDataBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pDataBlock->pDataBlock), + if (tqAddBlockDataToRsp(pDataBlock, pRsp, taosArrayGetSize(pDataBlock->pDataBlock), pTq->pVnode->config.tsdbCfg.precision) != 0) { tqError("vgId:%d, failed to add block to rsp msg", pTq->pVnode->config.vgId); continue; } - pRsp->common.blockNum++; + pRsp->blockNum++; if (pOffset->type == TMQ_OFFSET__LOG) { continue; } else { @@ -281,13 +282,13 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqBatc tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode), pHandle->snapshotVer + 1); - code = qStreamExtractOffset(task, &pRsp->common.rspOffset); + code = qStreamExtractOffset(task, &pRsp->rspOffset); break; } - if (pRsp->common.blockNum > 0) { + if (pRsp->blockNum > 0) { tqDebug("tmqsnap task exec exited, get data"); - code = qStreamExtractOffset(task, &pRsp->common.rspOffset); + code = qStreamExtractOffset(task, &pRsp->rspOffset); break; } } @@ -296,7 +297,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqBatc } -static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, STaosxRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded){ +static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded){ int32_t code = 0; STqExecHandle* pExec = &pHandle->execHandle; STqReader* pReader = pExec->pTqReader; @@ -323,7 +324,7 @@ static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, STaosxRsp* pRsp, int3 if ((pSubmitTbDataRet->flags & sourceExcluded) != 0) { goto END; } - if (pRsp->common.withTbName) { + if (pRsp->withTbName) { int64_t uid = pExec->pTqReader->lastBlkUid; code = tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)); if (code != 0) { @@ -381,7 +382,7 @@ static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, STaosxRsp* pRsp, int3 if (pBlock == NULL) { continue; } - if (tqAddBlockDataToRsp(pBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlock->pDataBlock), + if (tqAddBlockDataToRsp(pBlock, pRsp, taosArrayGetSize(pBlock->pDataBlock), pTq->pVnode->config.tsdbCfg.precision) != 0){ tqError("vgId:%d, failed to add block to rsp msg", pTq->pVnode->config.vgId); continue; @@ -389,11 +390,11 @@ static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, STaosxRsp* pRsp, int3 *totalRows += pBlock->info.rows; blockDataFreeRes(pBlock); SSchemaWrapper* pSW = taosArrayGetP(pSchemas, i); - if (taosArrayPush(pRsp->common.blockSchema, &pSW) == NULL){ + if (taosArrayPush(pRsp->blockSchema, &pSW) == NULL){ tqError("vgId:%d, failed to add schema to rsp msg", pTq->pVnode->config.vgId); continue; } - pRsp->common.blockNum++; + pRsp->blockNum++; } taosArrayDestroy(pBlocks); @@ -405,7 +406,7 @@ END: taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper); } -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows, +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded) { STqExecHandle* pExec = &pHandle->execHandle; int32_t code = 0; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index b18fa42cae..6daa9213aa 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -73,14 +73,19 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p } if (varTbName != NULL && varTbName != (void*)-1) { - name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN); + size_t cap = TMAX(TSDB_TABLE_NAME_LEN, varDataLen(varTbName) + 1); + name = taosMemoryMalloc(cap); if (name == NULL) { return terrno; } memcpy(name, varDataVal(varTbName), varDataLen(varTbName)); + name[varDataLen(varTbName)] = '\0'; if (newSubTableRule && !isAutoTableName(name) && !alreadyAddGroupId(name, groupId) && groupId != 0 && stbFullName) { - buildCtbNameAddGroupId(stbFullName, name, groupId); + int32_t code = buildCtbNameAddGroupId(stbFullName, name, groupId, cap); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } else if (stbFullName) { int32_t code = buildCtbNameByGroupId(stbFullName, groupId, &name); @@ -107,7 +112,7 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p groupId, name, skey, ekey); SSingleDeleteReq req = {.startTs = skey, .endTs = ekey}; - strncpy(req.tbname, name, TSDB_TABLE_NAME_LEN - 1); + tstrncpy(req.tbname, name, TSDB_TABLE_NAME_LEN); void* p = taosArrayPush(deleteReq->deleteReqs, &req); if (p == NULL) { return terrno; @@ -235,8 +240,11 @@ int32_t setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* return terrno; } - strcpy(pCreateTableReq->name, pDataBlock->info.parTbName); - buildCtbNameAddGroupId(stbFullName, pCreateTableReq->name, gid); + tstrncpy(pCreateTableReq->name, pDataBlock->info.parTbName, TSDB_TABLE_NAME_LEN); + int32_t code = buildCtbNameAddGroupId(stbFullName, pCreateTableReq->name, gid, TSDB_TABLE_NAME_LEN); + if (code != TSDB_CODE_SUCCESS) { + return code; + } // tqDebug("gen name from:%s", pDataBlock->info.parTbName); } else { pCreateTableReq->name = taosStrdup(pDataBlock->info.parTbName); @@ -852,9 +860,12 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat !alreadyAddGroupId(dstTableName, groupId) && groupId != 0) { tqDebug("s-task:%s append groupId:%" PRId64 " for generated dstTable:%s", id, groupId, dstTableName); if (pTask->ver == SSTREAM_TASK_SUBTABLE_CHANGED_VER) { - buildCtbNameAddGroupId(NULL, dstTableName, groupId); + code = buildCtbNameAddGroupId(NULL, dstTableName, groupId, sizeof(pDataBlock->info.parTbName)); } else if (pTask->ver > SSTREAM_TASK_SUBTABLE_CHANGED_VER && stbFullName) { - buildCtbNameAddGroupId(stbFullName, dstTableName, groupId); + code = buildCtbNameAddGroupId(stbFullName, dstTableName, groupId, sizeof(pDataBlock->info.parTbName)); + } + if (code != TSDB_CODE_SUCCESS) { + return code; } } } @@ -1250,4 +1261,4 @@ int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* } return TSDB_CODE_SUCCESS; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 5b83df3217..b4866b8c65 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -20,7 +20,7 @@ static int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const static int32_t tqSendBatchMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqBatchMetaRsp* pRsp, int32_t vgId); -int32_t tqInitDataRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset) { +int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) { pRsp->blockData = taosArrayInit(0, sizeof(void*)); pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); @@ -40,7 +40,7 @@ void tqUpdateNodeStage(STQ* pTq, bool isLeader) { streamMetaUpdateStageRole(pTq->pStreamMeta, state.term, isLeader); } -static int32_t tqInitTaosxRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset) { +static int32_t tqInitTaosxRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) { tOffsetCopy(&pRsp->reqOffset, &pOffset); tOffsetCopy(&pRsp->rspOffset, &pOffset); @@ -116,12 +116,12 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand SMqDataRsp dataRsp = {0}; tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer + 1); - code = tqInitDataRsp(&dataRsp.common, *pOffsetVal); + code = tqInitDataRsp(&dataRsp, *pOffsetVal); if (code != 0) { return code; } tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, - pHandle->subKey, vgId, dataRsp.common.rspOffset.version); + pHandle->subKey, vgId, dataRsp.rspOffset.version); code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); tDeleteMqDataRsp(&dataRsp); @@ -145,24 +145,27 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, terrno = 0; SMqDataRsp dataRsp = {0}; - - int code = tqInitDataRsp(&dataRsp.common, *pOffset); + int code = tqInitDataRsp(&dataRsp, *pOffset); if (code != 0) { goto end; } code = qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); + if (code != 0) { + goto end; + } + code = tqScanData(pTq, pHandle, &dataRsp, pOffset, pRequest); if (code != 0 && terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) { goto end; } // till now, all data has been transferred to consumer, new data needs to push client once arrived. - if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST && dataRsp.common.blockNum == 0) { + if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST && dataRsp.blockNum == 0) { // lock taosWLockLatch(&pTq->lock); int64_t ver = walGetCommittedVer(pTq->pVnode->pWal); - if (dataRsp.common.rspOffset.version > ver) { // check if there are data again to avoid lost data + if (dataRsp.rspOffset.version > ver) { // check if there are data again to avoid lost data code = tqRegisterPushHandle(pTq, pHandle, pMsg); taosWUnLockLatch(&pTq->lock); goto end; @@ -170,16 +173,16 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, taosWUnLockLatch(&pTq->lock); } - tOffsetCopy(&dataRsp.common.reqOffset, - pOffset); // reqOffset represents the current date offset, may be changed if wal not exists + // reqOffset represents the current date offset, may be changed if wal not exists + tOffsetCopy(&dataRsp.reqOffset, pOffset); code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); end : { char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.rspOffset); + tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.rspOffset); tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s,QID:0x%" PRIx64 " code:%d", - consumerId, pHandle->subKey, vgId, dataRsp.common.blockNum, buf, pRequest->reqId, code); + consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId, code); tDeleteMqDataRsp(&dataRsp); return code; } @@ -208,11 +211,11 @@ static void tDeleteCommon(void* parm) {} static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal* offset) { int32_t vgId = TD_VID(pTq->pVnode); - STaosxRsp taosxRsp = {0}; + SMqDataRsp taosxRsp = {0}; SMqBatchMetaRsp btMetaRsp = {0}; int32_t code = 0; - TQ_ERR_GO_TO_END(tqInitTaosxRsp(&taosxRsp.common, *offset)); + TQ_ERR_GO_TO_END(tqInitTaosxRsp(&taosxRsp, *offset)); if (offset->type != TMQ_OFFSET__LOG) { TQ_ERR_GO_TO_END(tqScanTaosx(pTq, pHandle, &taosxRsp, &btMetaRsp, offset)); @@ -227,13 +230,13 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send data blockNum:%d, offset type:%d,uid:%" PRId64 ",ts:%" PRId64, - pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.common.blockNum, taosxRsp.common.rspOffset.type, - taosxRsp.common.rspOffset.uid, taosxRsp.common.rspOffset.ts); - if (taosxRsp.common.blockNum > 0) { + pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, + taosxRsp.rspOffset.uid, taosxRsp.rspOffset.ts); + if (taosxRsp.blockNum > 0) { code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto END; } else { - tOffsetCopy(offset, &taosxRsp.common.rspOffset); + tOffsetCopy(offset, &taosxRsp.rspOffset); } } @@ -264,7 +267,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, } goto END; } - tqOffsetResetToLog(&taosxRsp.common.rspOffset, fetchVer); + tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); code = tqSendDataRsp( pHandle, pMsg, pRequest, &taosxRsp, taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); @@ -278,7 +281,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, // process meta if (pHead->msgType != TDMT_VND_SUBMIT) { if (totalRows > 0) { - tqOffsetResetToLog(&taosxRsp.common.rspOffset, fetchVer); + tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); code = tqSendDataRsp( pHandle, pMsg, pRequest, &taosxRsp, taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); @@ -387,7 +390,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, } if (totalRows >= tmqRowSize || (taosGetTimestampMs() - st > 1000)) { - tqOffsetResetToLog(&taosxRsp.common.rspOffset, fetchVer + 1); + tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer + 1); code = tqSendDataRsp( pHandle, pMsg, pRequest, &taosxRsp, taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); @@ -522,7 +525,7 @@ int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPoll return 0; } -int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const void* pRsp, int32_t epoch, int64_t consumerId, +int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, int32_t type, int64_t sver, int64_t ever) { int32_t len = 0; int32_t code = 0; diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 30be253b65..6b7e857120 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -84,7 +84,6 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) { code = qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId); if (code) { - return code; } } @@ -363,7 +362,7 @@ int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { } pRspHead->vgId = htonl(req.upstreamNodeId); - if(pRspHead->vgId == 0) { + if (pRspHead->vgId == 0) { tqError("vgId:%d invalid dispatch msg from upstream to task:0x%x", pMeta->vgId, req.taskId); return TSDB_CODE_INVALID_MSG; } @@ -460,7 +459,7 @@ int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { if (code != TSDB_CODE_SUCCESS) { // return error not send rsp manually tqError("s-task:0x%x vgId:%d failed to process retrieve request from 0x%x, code:%s", req.dstTaskId, req.dstNodeId, req.srcTaskId, tstrerror(code)); - } else { // send rsp manually only on success. + } else { // send rsp manually only on success. SRpcMsg rsp = {.info = pMsg->info, .code = 0}; streamTaskSendRetrieveRsp(&req, &rsp); } @@ -515,7 +514,7 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe } tDecoderClear(&decoder); - tqDebug("tq task:0x%x (vgId:%d) recv check rsp(qid:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", rsp.upstreamTaskId, + tqDebug("tq task:0x%x (vgId:%d) recv check rsp(QID:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); if (!isLeader) { @@ -1272,7 +1271,7 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { streamMutexLock(&pTask->lock); if (pTask->chkInfo.checkpointId < req.checkpointId) { - tqFatal("s-task:%s vgId:%d invalid consensus-checkpointId:%" PRId64 ", greater than existed checkpointId:%"PRId64, + tqFatal("s-task:%s vgId:%d invalid consensus-checkpointId:%" PRId64 ", greater than existed checkpointId:%" PRId64, pTask->id.idStr, vgId, req.checkpointId, pTask->chkInfo.checkpointId); streamMutexUnlock(&pTask->lock); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 37ab0a53f9..89a51eb0f5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -22,6 +22,12 @@ #define ROCKS_BATCH_SIZE (4096) +void tsdbLRUCacheRelease(SLRUCache *cache, LRUHandle *handle, bool eraseIfLastRef) { + if (!taosLRUCacheRelease(cache, handle, eraseIfLastRef)) { + tsdbTrace(" release lru cache failed"); + } +} + static int32_t tsdbOpenBCache(STsdb *pTsdb) { int32_t code = 0, lino = 0; int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; @@ -215,6 +221,8 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { rocksdb_writebatch_t *writebatch = rocksdb_writebatch_create(); + TAOS_CHECK_GOTO(taosThreadMutexInit(&pTsdb->rCache.writeBatchMutex, NULL), &lino, _err6) ; + pTsdb->rCache.writebatch = writebatch; pTsdb->rCache.my_comparator = cmp; pTsdb->rCache.options = options; @@ -226,6 +234,8 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { TAOS_RETURN(code); +_err6: + rocksdb_writebatch_destroy(writebatch); _err5: rocksdb_close(pTsdb->rCache.db); _err4: @@ -244,6 +254,7 @@ _err: static void tsdbCloseRocksCache(STsdb *pTsdb) { rocksdb_close(pTsdb->rCache.db); + (void)taosThreadMutexDestroy(&pTsdb->rCache.writeBatchMutex); rocksdb_flushoptions_destroy(pTsdb->rCache.flushoptions); rocksdb_writebatch_destroy(pTsdb->rCache.writebatch); rocksdb_readoptions_destroy(pTsdb->rCache.readoptions); @@ -579,7 +590,7 @@ static void tsdbCacheDeleter(const void *key, size_t klen, void *value, void *ud if (pLastCol->dirty) { if (tsdbCacheFlushDirty(key, klen, pLastCol, ud) != 0) { STsdb *pTsdb = (STsdb *)ud; - tsdbError("tsdb/cache: vgId:%d, flush cache %s failed at line %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__); + tsdbTrace("tsdb/cache: vgId:%d, flush cache %s failed at line %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__); } } @@ -713,9 +724,13 @@ static int32_t tsdbCacheDropTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, { SLastCol *pLastCol = NULL; code = tsdbCacheDeserialize(values_list[0], values_list_sizes[0], &pLastCol); - if (code != TSDB_CODE_SUCCESS) { - tsdbWarn("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, - tstrerror(code)); + if (code == TSDB_CODE_INVALID_PARA) { + tsdbTrace("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tstrerror(code)); + } else if (code != TSDB_CODE_SUCCESS) { + tsdbError("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tstrerror(code)); + goto _exit; } if (NULL != pLastCol) { rocksdb_writebatch_delete(wb, keys_list[0], klen); @@ -724,9 +739,13 @@ static int32_t tsdbCacheDropTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, pLastCol = NULL; code = tsdbCacheDeserialize(values_list[1], values_list_sizes[1], &pLastCol); - if (code != TSDB_CODE_SUCCESS) { - tsdbWarn("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, - tstrerror(code)); + if (code == TSDB_CODE_INVALID_PARA) { + tsdbTrace("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tstrerror(code)); + } else if (code != TSDB_CODE_SUCCESS) { + tsdbError("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tstrerror(code)); + goto _exit; } if (NULL != pLastCol) { rocksdb_writebatch_delete(wb, keys_list[1], klen); @@ -739,9 +758,7 @@ static int32_t tsdbCacheDropTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, for (int i = 0; i < 2; i++) { LRUHandle *h = taosLRUCacheLookup(pTsdb->lruCache, keys_list[i], klen); if (h) { - if (taosLRUCacheRelease(pTsdb->lruCache, h, true)) { - tsdbInfo("vgId:%d, %s release lru cache failed at line %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__); - } + tsdbLRUCacheRelease(pTsdb->lruCache, h, true); taosLRUCacheErase(pTsdb->lruCache, keys_list[i], klen); } } @@ -770,17 +787,13 @@ int32_t tsdbCacheNewTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWrap code = tsdbCacheNewTableColumn(pTsdb, uid, cid, col_type, LFLAG_LAST_ROW); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } code = tsdbCacheNewTableColumn(pTsdb, uid, cid, col_type, LFLAG_LAST); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } } } else { @@ -798,17 +811,13 @@ int32_t tsdbCacheNewTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWrap code = tsdbCacheNewTableColumn(pTsdb, uid, cid, col_type, LFLAG_LAST_ROW); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } code = tsdbCacheNewTableColumn(pTsdb, uid, cid, col_type, LFLAG_LAST); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } } @@ -827,10 +836,8 @@ int32_t tsdbCacheDropTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWra code = tsdbCacheCommitNoLock(pTsdb); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s commit with no lock failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s commit with no lock failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } if (pSchemaRow != NULL) { @@ -845,10 +852,8 @@ int32_t tsdbCacheDropTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWra code = tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s drop table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s drop table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } } } else { @@ -871,10 +876,8 @@ int32_t tsdbCacheDropTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWra code = tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s drop table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s drop table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } } @@ -895,10 +898,8 @@ int32_t tsdbCacheDropSubTables(STsdb *pTsdb, SArray *uids, tb_uid_t suid) { code = tsdbCacheCommitNoLock(pTsdb); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s commit with no lock failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s commit with no lock failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } STSchema *pTSchema = NULL; @@ -924,11 +925,8 @@ int32_t tsdbCacheDropSubTables(STsdb *pTsdb, SArray *uids, tb_uid_t suid) { code = tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s drop table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s drop table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - taosMemoryFree(pTSchema); - TAOS_RETURN(code); } } } @@ -949,17 +947,13 @@ int32_t tsdbCacheNewNTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, int8_t code = tsdbCacheNewTableColumn(pTsdb, uid, cid, col_type, 0); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } code = tsdbCacheNewTableColumn(pTsdb, uid, cid, col_type, 1); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } // rocksMayWrite(pTsdb, true, false, false); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -974,18 +968,14 @@ int32_t tsdbCacheDropNTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, bool h code = tsdbCacheCommitNoLock(pTsdb); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s commit with no lock failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s commit with no lock failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } code = tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s drop table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s drop table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } rocksMayWrite(pTsdb, false); @@ -1005,17 +995,13 @@ int32_t tsdbCacheNewSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, int8_t code = tsdbCacheNewTableColumn(pTsdb, uid, cid, col_type, 0); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } code = tsdbCacheNewTableColumn(pTsdb, uid, cid, col_type, 1); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } } @@ -1031,10 +1017,8 @@ int32_t tsdbCacheDropSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, bool code = tsdbCacheCommitNoLock(pTsdb); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s commit with no lock failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s commit with no lock failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } for (int i = 0; i < TARRAY_SIZE(uids); ++i) { @@ -1042,10 +1026,8 @@ int32_t tsdbCacheDropSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, bool code = tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s drop table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s drop table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } } @@ -1100,7 +1082,9 @@ static int32_t tsdbCachePutToRocksdb(STsdb *pTsdb, SLastKey *pLastKey, SLastCol } rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; + (void)taosThreadMutexLock(&pTsdb->rCache.writeBatchMutex); rocksdb_writebatch_put(wb, (char *)pLastKey, ROCKS_KEY_LEN, rocks_value, vlen); + (void)taosThreadMutexUnlock(&pTsdb->rCache.writeBatchMutex); taosMemoryFree(rocks_value); @@ -1174,9 +1158,7 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray } } - if (!taosLRUCacheRelease(pCache, h, false)) { - tsdbInfo("vgId:%d, %s release lru cache failed at line %d", TD_VID(pTsdb->pVnode), __func__, __LINE__); - } + tsdbLRUCacheRelease(pCache, h, false); TAOS_CHECK_EXIT(code); } else { if (!remainCols) { @@ -1237,9 +1219,13 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray SLastCol *pLastCol = NULL; code = tsdbCacheDeserialize(values_list[i], values_list_sizes[i], &pLastCol); - if (code != TSDB_CODE_SUCCESS) { - tsdbWarn("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, - tstrerror(code)); + if (code == TSDB_CODE_INVALID_PARA) { + tsdbTrace("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tstrerror(code)); + } else if (code != TSDB_CODE_SUCCESS) { + tsdbError("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tstrerror(code)); + goto _exit; } /* if (code) { @@ -1391,9 +1377,8 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6 } code = tSimpleHashIterateRemove(iColHash, &iCol, sizeof(iCol), &pIte, &iter); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s tSimpleHashIterateRemove failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, + tsdbTrace("vgId:%d, %s tSimpleHashIterateRemove failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - TAOS_CHECK_GOTO(code, &lino, _exit); } } } @@ -1402,9 +1387,8 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6 // 3. do update code = tsdbCacheUpdate(pTsdb, suid, uid, ctxArray); if (code < TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s tsdbCacheUpdate failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s tsdbCacheUpdate failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - TAOS_CHECK_GOTO(code, &lino, _exit); } _exit: @@ -1491,9 +1475,8 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo // 3. do update code = tsdbCacheUpdate(pTsdb, suid, uid, ctxArray); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s tsdbCacheUpdate failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s tsdbCacheUpdate failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - TAOS_CHECK_GOTO(code, &lino, _exit); } _exit: @@ -1709,9 +1692,13 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA } code = tsdbCacheDeserialize(values_list[i], values_list_sizes[i], &pLastCol); - if (code != TSDB_CODE_SUCCESS) { - tsdbWarn("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, - tstrerror(code)); + if (code == TSDB_CODE_INVALID_PARA) { + tsdbTrace("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tstrerror(code)); + } else if (code != TSDB_CODE_SUCCESS) { + tsdbError("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tstrerror(code)); + goto _exit; } SLastCol *pToFree = pLastCol; SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[j]; @@ -1787,10 +1774,14 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache SLastCol *pLastCol = h ? (SLastCol *)taosLRUCacheValue(pCache, h) : NULL; if (h && pLastCol->cacheStatus != TSDB_LAST_CACHE_NO_CACHE) { SLastCol lastCol = *pLastCol; - TAOS_CHECK_GOTO(tsdbCacheReallocSLastCol(&lastCol, NULL), NULL, _exit); + if (TSDB_CODE_SUCCESS != (code = tsdbCacheReallocSLastCol(&lastCol, NULL))) { + tsdbLRUCacheRelease(pCache, h, false); + TAOS_CHECK_GOTO(code, NULL, _exit); + } if (taosArrayPush(pLastArray, &lastCol) == NULL) { code = terrno; + tsdbLRUCacheRelease(pCache, h, false); goto _exit; } } else { @@ -1800,38 +1791,39 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache if (taosArrayPush(pLastArray, &noneCol) == NULL) { code = terrno; + tsdbLRUCacheRelease(pCache, h, false); goto _exit; } if (!remainCols) { if ((remainCols = taosArrayInit(numKeys, sizeof(SIdxKey))) == NULL) { code = terrno; + tsdbLRUCacheRelease(pCache, h, false); goto _exit; } } if (!ignoreFromRocks) { if ((ignoreFromRocks = taosArrayInit(numKeys, sizeof(bool))) == NULL) { code = terrno; + tsdbLRUCacheRelease(pCache, h, false); goto _exit; } } if (taosArrayPush(remainCols, &(SIdxKey){i, key}) == NULL) { code = terrno; + tsdbLRUCacheRelease(pCache, h, false); goto _exit; } bool ignoreRocks = pLastCol ? (pLastCol->cacheStatus == TSDB_LAST_CACHE_NO_CACHE) : false; if (taosArrayPush(ignoreFromRocks, &ignoreRocks) == NULL) { code = terrno; + tsdbLRUCacheRelease(pCache, h, false); goto _exit; } } if (h) { - code = taosLRUCacheRelease(pCache, h, false); - if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s release lru cache failed at line %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__); - goto _exit; - } + tsdbLRUCacheRelease(pCache, h, false); } } @@ -1846,6 +1838,7 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache SLastCol lastCol = *pLastCol; code = tsdbCacheReallocSLastCol(&lastCol, NULL); if (code) { + tsdbLRUCacheRelease(pCache, h, false); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); TAOS_RETURN(code); } @@ -1858,13 +1851,8 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache // no cache or cache is invalid ++i; } - if (h) { - code = taosLRUCacheRelease(pCache, h, false); - if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s release lru cache failed at line %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__); - goto _exit; - } + tsdbLRUCacheRelease(pCache, h, false); } } @@ -1900,10 +1888,8 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE code = tsdbCacheCommit(pTsdb); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s commit failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s commit failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } (void)taosThreadMutexLock(&pTsdb->lruMutex); @@ -1922,9 +1908,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE .cacheStatus = TSDB_LAST_CACHE_NO_CACHE}; code = tsdbCachePutToLRU(pTsdb, &lastKey, &noneCol, 1); } - if (taosLRUCacheRelease(pTsdb->lruCache, h, false) != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s release lru cache failed at line %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__); - } + tsdbLRUCacheRelease(pTsdb->lruCache, h, false); TAOS_CHECK_EXIT(code); } else { if (!remainCols) { @@ -1976,9 +1960,13 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE for (int i = 0; i < numKeys; ++i) { SLastCol *pLastCol = NULL; code = tsdbCacheDeserialize(values_list[i], values_list_sizes[i], &pLastCol); - if (code != TSDB_CODE_SUCCESS) { - tsdbWarn("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, - tstrerror(code)); + if (code == TSDB_CODE_INVALID_PARA) { + tsdbTrace("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tstrerror(code)); + } else if (code != TSDB_CODE_SUCCESS) { + tsdbError("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tstrerror(code)); + goto _exit; } SIdxKey *idxKey = taosArrayGet(remainCols, i); SLastKey *pLastKey = &idxKey->key; @@ -3483,11 +3471,7 @@ _err: TAOS_RETURN(code); } -void tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h) { - if (taosLRUCacheRelease(pCache, h, false)) { - tsdbError("%s release lru cache failed at line %d.", __func__, __LINE__); - } -} +void tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h) { tsdbLRUCacheRelease(pCache, h, false); } void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity) { taosLRUCacheSetCapacity(pVnode->pTsdb->lruCache, capacity); diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 9e37e785f8..d508d75922 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -14,6 +14,7 @@ */ #include "functionMgt.h" +#include "functionResInfo.h" #include "taoserror.h" #include "tarray.h" #include "tcommon.h" diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index d3b783847c..0c9d9e56cf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -433,6 +433,21 @@ static int32_t tsdbFSCreateFileObjHash(STFileSystem *fs, STFileHash *hash) { if (fset->farr[i] != NULL) { code = tsdbFSAddEntryToFileObjHash(hash, fset->farr[i]->fname); TSDB_CHECK_CODE(code, lino, _exit); + + if (TSDB_FTYPE_DATA == i && fset->farr[i]->f->lcn > 0) { + STFileObj *fobj = fset->farr[i]; + int32_t lcn = fobj->f->lcn; + char lcn_name[TSDB_FILENAME_LEN]; + + snprintf(lcn_name, TSDB_FQDN_LEN, "%s", fobj->fname); + char *dot = strrchr(lcn_name, '.'); + if (dot) { + snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lcn_name), "%d.data", lcn); + + code = tsdbFSAddEntryToFileObjHash(hash, lcn_name); + TSDB_CHECK_CODE(code, lino, _exit); + } + } } } @@ -535,9 +550,7 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) { for (const STfsFile *file = NULL; (file = tfsReaddir(dir)) != NULL;) { if (taosIsDir(file->aname)) continue; - if (tsdbFSGetFileObjHashEntry(&fobjHash, file->aname) == NULL && - strncmp(file->aname + strlen(file->aname) - 3, ".cp", 3) && - strncmp(file->aname + strlen(file->aname) - 5, ".data", 5)) { + if (tsdbFSGetFileObjHashEntry(&fobjHash, file->aname) == NULL) { tsdbRemoveFile(file->aname); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c index ea404142fe..60e42bd2b8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c @@ -46,8 +46,11 @@ static int32_t tsdbFSetWriteTableDataBegin(SFSetWriter *writer, const TABLEID *t code = tsdbUpdateSkmTb(writer->config->tsdb, writer->ctx->tbid, writer->skmTb); TSDB_CHECK_CODE(code, lino, _exit); + if (writer->pColCmprObj != NULL) { + taosHashCleanup(writer->pColCmprObj); + writer->pColCmprObj = NULL; + } code = metaGetColCmpr(writer->config->tsdb->pVnode->pMeta, tbid->suid ? tbid->suid : tbid->uid, &writer->pColCmprObj); - // TODO: TSDB_CHECK_CODE(code, lino, _exit); writer->blockDataIdx = 0; for (int32_t i = 0; i < ARRAY_SIZE(writer->blockData); i++) { @@ -127,6 +130,8 @@ _exit: TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); } taosHashCleanup(writer->pColCmprObj); + writer->pColCmprObj = NULL; + return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 5b6511a38e..d4b906fe2a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -596,7 +596,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void pReader->status.pPrimaryTsCol = taosArrayGet(pReader->resBlockInfo.pResBlock->pDataBlock, pSup->slotId[0]); if (pReader->status.pPrimaryTsCol == NULL) { - code = TSDB_CODE_INVALID_PARA; + code = terrno; goto _end; } @@ -704,7 +704,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileRead pReader->cost.headFileLoadTime += (et1 - st) / 1000.0; -_end: +//_end: // tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle); return code; } @@ -1872,9 +1872,9 @@ static void doPinSttBlock(SSttBlockReader* pSttBlockReader) { tMergeTreePinSttBl static void doUnpinSttBlock(SSttBlockReader* pSttBlockReader) { tMergeTreeUnpinSttBlock(&pSttBlockReader->mergeTree); } -static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SSttBlockReader* pSttBlockReader, - STableBlockScanInfo* pScanInfo, SRowKey* pSttKey, STsdbReader* pReader, - bool* copied) { +static int32_t tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SSttBlockReader* pSttBlockReader, + STableBlockScanInfo* pScanInfo, SRowKey* pSttKey, STsdbReader* pReader, + bool* copied) { int32_t code = TSDB_CODE_SUCCESS; *copied = false; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index d0ea58c28a..e8740a0650 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -623,6 +623,7 @@ static int32_t tsdbSnapWriteFileSetOpenReader(STsdbSnapWriter* writer) { int32_t lino = 0; if (writer->ctx->fset) { +#if 0 // open data reader SDataFileReaderConfig dataFileReaderConfig = { .tsdb = writer->tsdb, @@ -650,6 +651,7 @@ static int32_t tsdbSnapWriteFileSetOpenReader(STsdbSnapWriter* writer) { code = tsdbDataFileReaderOpen(NULL, &dataFileReaderConfig, &writer->ctx->dataReader); TSDB_CHECK_CODE(code, lino, _exit); +#endif // open stt reader array SSttLvl* lvl; @@ -791,6 +793,15 @@ static int32_t tsdbSnapWriteFileSetOpenWriter(STsdbSnapWriter* writer) { .did = writer->ctx->did, .level = 0, }; + // merge stt files to either data or a new stt file + if (writer->ctx->fset) { + for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (writer->ctx->fset->farr[ftype] != NULL) { + config.files[ftype].exist = true; + config.files[ftype].file = writer->ctx->fset->farr[ftype]->f[0]; + } + } + } code = tsdbFSetWriterOpen(&config, &writer->ctx->fsetWriter); TSDB_CHECK_CODE(code, lino, _exit); @@ -842,6 +853,8 @@ static int32_t tsdbSnapWriteFileSetBegin(STsdbSnapWriter* writer, int32_t fid) { _exit: if (code) { TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } else { + tsdbInfo("vgId:%d %s succeeded, fid:%d", TD_VID(writer->tsdb->pVnode), __func__, fid); } return code; } @@ -922,6 +935,8 @@ static int32_t tsdbSnapWriteFileSetEnd(STsdbSnapWriter* writer) { _exit: if (code) { TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } else { + tsdbInfo("vgId:%d %s succeeded, fid:%d", TD_VID(writer->tsdb->pVnode), __func__, writer->ctx->fid); } return code; } @@ -1175,7 +1190,7 @@ _exit: if (code) { TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); } else { - tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); + tsdbInfo("vgId:%d %s done, rollback:%d", TD_VID(tsdb->pVnode), __func__, rollback); } return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 438083f9b9..4a4d305f25 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -102,9 +102,8 @@ static int32_t vnodeGetBufPoolToUse(SVnode *pVnode) { ts.tv_sec = tv.tv_sec; } - int32_t rc = taosThreadCondTimedWait(&pVnode->poolNotEmpty, &pVnode->mutex, &ts); - if (rc && rc != ETIMEDOUT) { - code = TAOS_SYSTEM_ERROR(rc); + code = taosThreadCondTimedWait(&pVnode->poolNotEmpty, &pVnode->mutex, &ts); + if (code && code != TSDB_CODE_TIMEOUT_ERROR) { TSDB_CHECK_CODE(code, lino, _exit); } } diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index d616bfd4ce..7c6a2e7313 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -254,7 +254,7 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { if (mer1.me.ctbEntry.commentLen > 0) { cfgRsp.pComment = taosStrdup(mer1.me.ctbEntry.comment); if (NULL == cfgRsp.pComment) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } } @@ -273,7 +273,7 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { if (mer1.me.ntbEntry.commentLen > 0) { cfgRsp.pComment = taosStrdup(mer1.me.ntbEntry.comment); if (NULL == cfgRsp.pComment) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } } @@ -399,7 +399,7 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { for (int32_t i = 0; i < msgNum; ++i) { req = taosArrayGet(batchReq.pMsgs, i); if (req == NULL) { - code = TSDB_CODE_OUT_OF_RANGE; + code = terrno; goto _exit; } diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 46f4f86484..c1dcdf2741 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -72,7 +72,7 @@ int32_t ctgInitGetTbMetaTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; @@ -94,7 +94,7 @@ int32_t ctgInitGetTbMetasTask(SCtgJob* pJob, int32_t taskIdx, void* param) { ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->tbMetaNum, sizeof(SMetaRes)); if (NULL == ctx->pResList) { - qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbMetaNum, + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbMetaNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(terrno); @@ -105,7 +105,7 @@ int32_t ctgInitGetTbMetasTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbMetaNum); return TSDB_CODE_SUCCESS; @@ -133,7 +133,7 @@ int32_t ctgInitGetDbVgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); return TSDB_CODE_SUCCESS; @@ -161,7 +161,7 @@ int32_t ctgInitGetDbCfgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); return TSDB_CODE_SUCCESS; @@ -189,7 +189,7 @@ int32_t ctgInitGetDbInfoTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); return TSDB_CODE_SUCCESS; @@ -223,7 +223,7 @@ int32_t ctgInitGetTbHashTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, tableName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tableName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; @@ -245,7 +245,7 @@ int32_t ctgInitGetTbHashsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->tbHashNum, sizeof(SMetaRes)); if (NULL == ctx->pResList) { - qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbHashNum, + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbHashNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(terrno); @@ -256,7 +256,7 @@ int32_t ctgInitGetTbHashsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbHashNum); return TSDB_CODE_SUCCESS; @@ -275,7 +275,7 @@ int32_t ctgInitGetQnodeTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); return TSDB_CODE_SUCCESS; } @@ -293,7 +293,7 @@ int32_t ctgInitGetDnodeTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); return TSDB_CODE_SUCCESS; } @@ -320,7 +320,7 @@ int32_t ctgInitGetIndexTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, indexFName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, indexFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name); return TSDB_CODE_SUCCESS; @@ -348,7 +348,7 @@ int32_t ctgInitGetUdfTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, udfName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, udfName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name); return TSDB_CODE_SUCCESS; @@ -376,7 +376,7 @@ int32_t ctgInitGetUserTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, user:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, user:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), user->user); return TSDB_CODE_SUCCESS; @@ -394,7 +394,7 @@ int32_t ctgInitGetSvrVerTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); return TSDB_CODE_SUCCESS; } @@ -426,7 +426,7 @@ int32_t ctgInitGetTbIndexTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; @@ -459,7 +459,7 @@ int32_t ctgInitGetTbCfgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; @@ -492,7 +492,7 @@ int32_t ctgInitGetTbTagTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; @@ -514,7 +514,7 @@ int32_t ctgInitGetViewsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->viewNum, sizeof(SMetaRes)); if (NULL == ctx->pResList) { - qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->viewNum, + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->viewNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(terrno); @@ -525,7 +525,7 @@ int32_t ctgInitGetViewsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, viewNum:%d", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, viewNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->viewNum); return TSDB_CODE_SUCCESS; @@ -546,7 +546,7 @@ int32_t ctgInitGetTbTSMATask(SCtgJob* pJob, int32_t taskId, void* param) { pTaskCtx->pNames = param; pTaskCtx->pResList = taosArrayInit(pJob->tbTsmaNum, sizeof(SMetaRes)); if (NULL == pTaskCtx->pResList) { - qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbTsmaNum, + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbTsmaNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(terrno); @@ -574,7 +574,7 @@ int32_t ctgInitGetTSMATask(SCtgJob* pJob, int32_t taskId, void* param) { pTaskCtx->pNames = param; pTaskCtx->pResList = taosArrayInit(pJob->tsmaNum, sizeof(SMetaRes)); if (NULL == pTaskCtx->pResList) { - qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tsmaNum, + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tsmaNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(terrno); @@ -603,7 +603,7 @@ static int32_t ctgInitGetTbNamesTask(SCtgJob* pJob, int32_t taskId, void* param) pTaskCtx->pNames = param; pTaskCtx->pResList = taosArrayInit(pJob->tbNameNum, sizeof(SMetaRes)); if (NULL == pTaskCtx->pResList) { - qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbNameNum, + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbNameNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(terrno); @@ -1048,7 +1048,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const } double el = (taosGetTimestampUs() - st) / 1000.0; - qDebug("qid:0x%" PRIx64 ", jobId: 0x%" PRIx64 " initialized, task num %d, forceUpdate %d, elapsed time:%.2f ms", + qDebug("QID:0x%" PRIx64 ", jobId: 0x%" PRIx64 " initialized, task num %d, forceUpdate %d, elapsed time:%.2f ms", pJob->queryId, pJob->refId, taskNum, pReq->forceUpdate, el); return TSDB_CODE_SUCCESS; @@ -1450,16 +1450,17 @@ _return: int32_t ctgCallUserCb(void* param) { SCtgJob* pJob = (SCtgJob*)param; - qDebug("qid:0x%" PRIx64 " ctg start to call user cb with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode)); + qDebug("QID:0x%" PRIx64 " ctg start to call user cb with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode)); (*pJob->userFp)(&pJob->jobRes, pJob->userParam, pJob->jobResCode); - qDebug("qid:0x%" PRIx64 " ctg end to call user cb", pJob->queryId); + qDebug("QID:0x%" PRIx64 " ctg end to call user cb", pJob->queryId); int64_t refId = pJob->refId; int32_t code = taosRemoveRef(gCtgMgmt.jobPool, refId); if (code) { - qError("qid:0x%" PRIx64 " remove ctg job %" PRId64 " from jobPool failed, error:%s", pJob->queryId, refId, tstrerror(code)); + qError("QID:0x%" PRIx64 " remove ctg job %" PRId64 " from jobPool failed, error:%s", pJob->queryId, refId, + tstrerror(code)); } return TSDB_CODE_SUCCESS; @@ -1469,7 +1470,7 @@ void ctgUpdateJobErrCode(SCtgJob* pJob, int32_t errCode) { if (!NEED_CLIENT_REFRESH_VG_ERROR(errCode) || errCode == TSDB_CODE_SUCCESS) return; atomic_store_32(&pJob->jobResCode, errCode); - qDebug("qid:0x%" PRIx64 " ctg job errCode updated to %s", pJob->queryId, tstrerror(errCode)); + qDebug("QID:0x%" PRIx64 " ctg job errCode updated to %s", pJob->queryId, tstrerror(errCode)); return; } @@ -1481,7 +1482,7 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { return TSDB_CODE_SUCCESS; } - qDebug("qid:0x%" PRIx64 " task %d end with res %s", pJob->queryId, pTask->taskId, tstrerror(rspCode)); + qDebug("QID:0x%" PRIx64 " task %d end with res %s", pJob->queryId, pTask->taskId, tstrerror(rspCode)); pTask->code = rspCode; pTask->status = CTG_TASK_DONE; @@ -1490,7 +1491,7 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { int32_t taskDone = atomic_add_fetch_32(&pJob->taskDone, 1); if (taskDone < taosArrayGetSize(pJob->pTasks)) { - qDebug("qid:0x%" PRIx64 " task done: %d, total: %d", pJob->queryId, taskDone, + qDebug("QID:0x%" PRIx64 " task done: %d, total: %d", pJob->queryId, taskDone, (int32_t)taosArrayGetSize(pJob->pTasks)); ctgUpdateJobErrCode(pJob, rspCode); @@ -2915,7 +2916,7 @@ int32_t ctgHandleGetTbTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf if (META_TYPE_BOTH_TABLE == pOut->metaType) { // rewrite tsma fetch table with it's super table name - (void)sprintf(pFetch->tsmaSourceTbName.tname, "%s", pOut->tbName); + (void)snprintf(pFetch->tsmaSourceTbName.tname, sizeof(pFetch->tsmaSourceTbName.tname), "%s", pOut->tbName); } CTG_ERR_JRET(ctgGetTbTSMAFromMnode(pCtg, pConn, &pFetch->tsmaSourceTbName, NULL, tReq, TDMT_MND_GET_TABLE_TSMA)); @@ -4347,7 +4348,7 @@ int32_t ctgLaunchJob(SCtgJob* pJob) { CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - qDebug("qid:0x%" PRIx64 " ctg launch [%dth] task", pJob->queryId, pTask->taskId); + qDebug("QID:0x%" PRIx64 " ctg launch [%dth] task", pJob->queryId, pTask->taskId); CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); pTask = taosArrayGet(pJob->pTasks, i); @@ -4360,7 +4361,7 @@ int32_t ctgLaunchJob(SCtgJob* pJob) { } if (taskNum <= 0) { - qDebug("qid:0x%" PRIx64 " ctg call user callback with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode)); + qDebug("QID:0x%" PRIx64 " ctg call user callback with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode)); CTG_ERR_RET(taosAsyncExec(ctgCallUserCb, pJob, NULL)); #if CTG_BATCH_FETCH diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 8cbb5c226a..9db3913375 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -906,7 +906,7 @@ int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation) { if (gCtgMgmt.queue.stopQueue) { ctgFreeQNode(node); CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); - CTG_RET(TSDB_CODE_CTG_EXIT); + CTG_ERR_JRET(TSDB_CODE_CTG_EXIT); } gCtgMgmt.queue.tail->next = node; @@ -924,7 +924,7 @@ int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation) { code = tsem_post(&gCtgMgmt.queue.reqSem); if (TSDB_CODE_SUCCESS != code) { qError("tsem_post failed, code:%x", code); - CTG_RET(code); + CTG_ERR_JRET(code); } if (syncOp) { @@ -935,9 +935,15 @@ int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation) { if (!operation->unLocked) { CTG_LOCK(CTG_READ, &gCtgMgmt.lock); } - taosMemoryFree(operation); + TAOS_UNUSED(tsem_destroy(&operation->rspSem)); + taosMemoryFreeClear(operation); } + return code; +_return: + if (syncOp && operation) { + TAOS_UNUSED(tsem_destroy(&operation->rspSem)); + } return code; } diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index 0ac606d4d6..f3a0b04457 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -176,22 +176,22 @@ int32_t ctgdLaunchAsyncCall(SCatalog *pCtg, SRequestConnInfo *pConn, uint64_t re taosArrayPush(req.pTableMeta, &name); taosArrayPush(req.pTableHash, &name); - strcpy(dbFName, "1.db1"); + tstrncpy(dbFName, "1.db1", sizeof(dbFName)); taosArrayPush(req.pDbVgroup, dbFName); taosArrayPush(req.pDbCfg, dbFName); taosArrayPush(req.pDbInfo, dbFName); - strcpy(dbFName, "1.db2"); + tstrncpy(dbFName, "1.db2", sizeof(dbFName)); taosArrayPush(req.pDbVgroup, dbFName); taosArrayPush(req.pDbCfg, dbFName); taosArrayPush(req.pDbInfo, dbFName); - strcpy(funcName, "udf1"); + tstrncpy(funcName, "udf1", sizeof(funcName)); taosArrayPush(req.pUdf, funcName); - strcpy(funcName, "udf2"); + tstrncpy(funcName, "udf2", sizeof(funcName)); taosArrayPush(req.pUdf, funcName); - strcpy(user.user, "root"); - strcpy(user.dbFName, "1.db1"); + tstrncpy(user.user, "root", sizeof(user.user)); + tstrncpy(user.dbFName, "1.db1", sizeof(user.dbFName)); user.type = AUTH_TYPE_READ; taosArrayPush(req.pUser, &user); user.type = AUTH_TYPE_WRITE; @@ -199,8 +199,8 @@ int32_t ctgdLaunchAsyncCall(SCatalog *pCtg, SRequestConnInfo *pConn, uint64_t re user.type = AUTH_TYPE_OTHER; taosArrayPush(req.pUser, &user); - strcpy(user.user, "user1"); - strcpy(user.dbFName, "1.db2"); + tstrncpy(user.user, "user1", sizeof(user.user)); + tstrncpy(user.dbFName, "1.db2", sizeof(user.dbFName)); user.type = AUTH_TYPE_READ; taosArrayPush(req.pUser, &user); user.type = AUTH_TYPE_WRITE; @@ -335,7 +335,7 @@ int32_t ctgdHandleDbgCommand(char *command) { CTG_RET(TSDB_CODE_INVALID_PARA); } - bool enable = atoi(param); + bool enable = taosStr2Int32(param, NULL, 10); int32_t code = ctgdEnableDebug(option, enable); diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index d6e941c819..46a615aeed 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -47,7 +47,7 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu msgNum = 0; } - ctgDebug("qid:0x%" PRIx64 " ctg got batch %d rsp %s", pJob->queryId, cbParam->batchId, + ctgDebug("QID:0x%" PRIx64 " ctg got batch %d rsp %s", pJob->queryId, cbParam->batchId, TMSG_INFO(cbParam->reqType + 1)); SHashObj* pBatchs = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -114,7 +114,7 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu pMsgCtx->pBatchs = pBatchs; - ctgDebug("qid:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s, pBatchs: %p", pJob->queryId, pTask->taskId, + ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s, pBatchs: %p", pJob->queryId, pTask->taskId, pRsp->msgIdx, TMSG_INFO(taskMsg.msgType + 1), pBatchs); (void)(*gCtgAsyncFps[pTask->type].handleRspFp)( @@ -454,7 +454,7 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) { CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - qDebug("qid:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, + qDebug("QID:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(cbParam->reqType + 1)); #if CTG_BATCH_FETCH @@ -702,7 +702,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT if (TDMT_VND_TABLE_CFG == msgType) { SCtgTbCfgCtx* ctx = (SCtgTbCfgCtx*)pTask->taskCtx; pName = ctx->pName; - } else if (TDMT_VND_TABLE_META == msgType || TDMT_VND_TABLE_NAME == msgType) { + } else if (TDMT_VND_TABLE_META == msgType || TDMT_VND_TABLE_NAME == msgType) { if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); @@ -808,7 +808,7 @@ int32_t ctgLaunchBatchs(SCatalog* pCtg, SCtgJob* pJob, SHashObj* pBatchs) { SCtgBatch* pBatch = (SCtgBatch*)p; int32_t msgSize = 0; - ctgDebug("qid:0x%" PRIx64 " ctg start to launch batch %d", pJob->queryId, pBatch->batchId); + ctgDebug("QID:0x%" PRIx64 " ctg start to launch batch %d", pJob->queryId, pBatch->batchId); CTG_ERR_JRET(ctgBuildBatchReqMsg(pBatch, *vgId, &msg, &msgSize)); code = ctgAsyncSendMsg(pCtg, &pBatch->conn, pJob, pBatch->pTaskIds, pBatch->batchId, pBatch->pMsgIdxs, @@ -1124,10 +1124,11 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n int32_t code = tNameExtractFullName(name, tbFName); if (code) { - ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), name->type, name->dbname, name->tname); + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), name->type, + name->dbname, name->tname); CTG_ERR_RET(code); } - + code = queryBuildMsg[TMSG_INDEX(reqType)]((void*)tbFName, &msg, 0, &msgLen, mallocFp); if (code) { ctgError("Build get index msg failed, code:%s, tbFName:%s", tstrerror(code), tbFName); @@ -1302,7 +1303,7 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, const int32_t msgLen = 0; int32_t reqType = TDMT_MND_TABLE_META; char tbFName[TSDB_TABLE_FNAME_LEN]; - (void)sprintf(tbFName, "%s.%s", dbFName, tbName); + (void)snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName); void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; ctgDebug("try to get table meta from mnode, tbFName:%s", tbFName); @@ -1368,7 +1369,7 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa (void)tNameGetFullDbName(pTableName, dbFName); int32_t reqType = (pTask && pTask->type == CTG_TASK_GET_TB_NAME ? TDMT_VND_TABLE_NAME : TDMT_VND_TABLE_META); char tbFName[TSDB_TABLE_FNAME_LEN]; - (void)sprintf(tbFName, "%s.%s", dbFName, pTableName->tname); + (void)snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, pTableName->tname); void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; SEp* pEp = &vgroupInfo->epSet.eps[vgroupInfo->epSet.inUse]; @@ -1450,7 +1451,8 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S int32_t code = tNameExtractFullName(pTableName, tbFName); if (code) { - ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTableName->type, pTableName->dbname, pTableName->tname); + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTableName->type, + pTableName->dbname, pTableName->tname); CTG_ERR_RET(code); } @@ -1523,7 +1525,8 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S int32_t code = tNameExtractFullName(pTableName, tbFName); if (code) { - ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTableName->type, pTableName->dbname, pTableName->tname); + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTableName->type, + pTableName->dbname, pTableName->tname); CTG_ERR_RET(code); } @@ -1632,10 +1635,11 @@ int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* int32_t reqType = TDMT_MND_VIEW_META; SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; - char fullName[TSDB_TABLE_FNAME_LEN]; + char fullName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pName, fullName); if (code) { - ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pName->type, pName->dbname, pName->tname); + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pName->type, + pName->dbname, pName->tname); CTG_ERR_RET(code); } @@ -1693,10 +1697,11 @@ int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa int32_t msgLen = 0; SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; - char tbFName[TSDB_TABLE_FNAME_LEN]; + char tbFName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(name, tbFName); if (code) { - ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), name->type, name->dbname, name->tname); + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), name->type, + name->dbname, name->tname); CTG_ERR_RET(code); } @@ -1757,10 +1762,11 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c char tbFName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pTbName, tbFName); if (code) { - ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTbName->type, pTbName->dbname, pTbName->tname); + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTbName->type, + pTbName->dbname, pTbName->tname); CTG_ERR_RET(code); } - + SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 96cd783d2f..f8591b2121 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -452,7 +452,8 @@ void ctgClearHandleMeta(SCatalog* pCtg, int64_t* pClearedSize, int64_t* pCleardN code = taosHashRemove(dbCache->tbCache, key, len); if (code) { - qError("taosHashRemove table cache failed, key:%s, len:%d, error:%s", (char*)key, (int32_t)len, tstrerror(code)); + qError("taosHashRemove table cache failed, key:%s, len:%d, error:%s", (char*)key, (int32_t)len, + tstrerror(code)); } cacheSize = @@ -1096,7 +1097,7 @@ void ctgFreeJob(void* job) { taosMemoryFree(job); - qDebug("qid:0x%" PRIx64 ", ctg job 0x%" PRIx64 " freed", qid, rid); + qDebug("QID:0x%" PRIx64 ", ctg job 0x%" PRIx64 " freed", qid, rid); } int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target) { @@ -1107,7 +1108,7 @@ int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* targ if (target) { pCtx->target = taosStrdup(target); if (NULL == pCtx->target) { - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_RET(terrno); } } else { pCtx->target = NULL; @@ -1124,7 +1125,7 @@ int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target) { if (target) { ctx.target = taosStrdup(target); if (NULL == ctx.target) { - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_RET(terrno); } } @@ -1241,10 +1242,11 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SEpSet* pMgmtEps, SDBVgInfo* d char tbFullName[TSDB_TABLE_FNAME_LEN]; code = tNameExtractFullName(pTableName, tbFullName); if (code) { - ctgError("tNameExtractFullName failed, error:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTableName->type, pTableName->dbname, pTableName->tname); + ctgError("tNameExtractFullName failed, error:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTableName->type, + pTableName->dbname, pTableName->tname); CTG_ERR_RET(code); } - + uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix); @@ -1384,7 +1386,7 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskR } char tbFullName[TSDB_TABLE_FNAME_LEN]; - (void)sprintf(tbFullName, "%s.", dbFName); + (void)snprintf(tbFullName, sizeof(tbFullName), "%s.", dbFName); int32_t offset = strlen(tbFullName); SName* pName = NULL; int32_t tbNameLen = 0; @@ -1629,7 +1631,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst) { if (NULL == (*dst)->vgArray) { taosHashCleanup((*dst)->vgHash); taosMemoryFreeClear(*dst); - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_RET(terrno); } } @@ -1696,7 +1698,7 @@ int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes) { } pInfo->expr = taosStrdup(pInfo->expr); if (NULL == pInfo->expr) { - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_RET(terrno); } } @@ -1704,12 +1706,13 @@ int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes) { } int32_t ctgUpdateSendTargetInfo(SMsgSendInfo* pMsgSendInfo, int32_t msgType, char* dbFName, int32_t vgId) { - if (msgType == TDMT_VND_TABLE_META || msgType == TDMT_VND_TABLE_CFG || msgType == TDMT_VND_BATCH_META || msgType == TDMT_VND_TABLE_NAME) { + if (msgType == TDMT_VND_TABLE_META || msgType == TDMT_VND_TABLE_CFG || msgType == TDMT_VND_BATCH_META || + msgType == TDMT_VND_TABLE_NAME) { pMsgSendInfo->target.type = TARGET_TYPE_VNODE; pMsgSendInfo->target.vgId = vgId; pMsgSendInfo->target.dbFName = taosStrdup(dbFName); if (NULL == pMsgSendInfo->target.dbFName) { - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_RET(terrno); } } else { pMsgSendInfo->target.type = TARGET_TYPE_MNODE; @@ -2010,7 +2013,8 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { char dbFName[TSDB_DB_FNAME_LEN]; code = tNameExtractFullName(&req->pRawReq->tbName, tbFName); if (code) { - ctgError("tNameExtractFullName failed, error:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), req->pRawReq->tbName.type, req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); + ctgError("tNameExtractFullName failed, error:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), + req->pRawReq->tbName.type, req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); CTG_ERR_RET(code); } @@ -2066,7 +2070,7 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { continue; } - (void)sprintf(tbFName, "%s.%s", dbFName, stbName); + (void)snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, stbName); continue; } @@ -2201,7 +2205,8 @@ int32_t ctgChkSetViewAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) } else { code = tNameExtractFullName(&req->pRawReq->tbName, viewFName); if (code) { - ctgError("tNameExtractFullName failed, error:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), req->pRawReq->tbName.type, req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); + ctgError("tNameExtractFullName failed, error:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), + req->pRawReq->tbName.type, req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); CTG_ERR_RET(code); } } diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index d24f830ea7..27a43f7523 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -291,7 +291,8 @@ static int32_t buildRetension(SArray* pRetension, char** ppRetentions) { return TSDB_CODE_SUCCESS; } - char* p1 = taosMemoryCalloc(1, 100); + const int lMaxLen = 128; + char* p1 = taosMemoryCalloc(1, lMaxLen); if (NULL == p1) { return terrno; } @@ -302,13 +303,13 @@ static int32_t buildRetension(SArray* pRetension, char** ppRetentions) { int64_t v1 = getValOfDiffPrecision(p->freqUnit, p->freq); int64_t v2 = getValOfDiffPrecision(p->keepUnit, p->keep); if (i == 0) { - len += sprintf(p1 + len, "-:%" PRId64 "%c", v2, p->keepUnit); + len += tsnprintf(p1 + len, lMaxLen - len, "-:%" PRId64 "%c", v2, p->keepUnit); } else { - len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit); + len += tsnprintf(p1 + len, lMaxLen - len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit); } if (i < size - 1) { - len += sprintf(p1 + len, ","); + len += tsnprintf(p1 + len, lMaxLen - len, ","); } } @@ -344,16 +345,19 @@ static const char* encryptAlgorithmStr(int8_t encryptAlgorithm) { return TSDB_CACHE_MODEL_NONE_STR; } -int32_t formatDurationOrKeep(char* buffer, int32_t timeInMinutes) { +int32_t formatDurationOrKeep(char* buffer, int64_t bufSize, int32_t timeInMinutes) { + if (buffer == NULL || bufSize <= 0) { + return 0; + } int32_t len = 0; if (timeInMinutes % 1440 == 0) { - int32_t days = timeInMinutes / 1440; - len = sprintf(buffer, "%dd", days); + int32_t days = timeInMinutes / 1440; + len = tsnprintf(buffer, bufSize, "%dd", days); } else if (timeInMinutes % 60 == 0) { - int32_t hours = timeInMinutes / 60; - len = sprintf(buffer, "%dh", hours); + int32_t hours = timeInMinutes / 60; + len = tsnprintf(buffer, bufSize, "%dh", hours); } else { - len = sprintf(buffer, "%dm", timeInMinutes); + len = tsnprintf(buffer, bufSize, "%dm", timeInMinutes); } return len; } @@ -400,15 +404,15 @@ static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, char keep1Str[128] = {0}; char keep2Str[128] = {0}; - int32_t lenDuration = formatDurationOrKeep(durationStr, pCfg->daysPerFile); - int32_t lenKeep0 = formatDurationOrKeep(keep0Str, pCfg->daysToKeep0); - int32_t lenKeep1 = formatDurationOrKeep(keep1Str, pCfg->daysToKeep1); - int32_t lenKeep2 = formatDurationOrKeep(keep2Str, pCfg->daysToKeep2); + int32_t lenDuration = formatDurationOrKeep(durationStr, sizeof(durationStr), pCfg->daysPerFile); + int32_t lenKeep0 = formatDurationOrKeep(keep0Str, sizeof(keep0Str), pCfg->daysToKeep0); + int32_t lenKeep1 = formatDurationOrKeep(keep1Str, sizeof(keep1Str), pCfg->daysToKeep1); + int32_t lenKeep2 = formatDurationOrKeep(keep2Str, sizeof(keep2Str), pCfg->daysToKeep2); if (IS_SYS_DBNAME(dbName)) { - len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE DATABASE `%s`", dbName); + len += tsnprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, "CREATE DATABASE `%s`", dbName); } else { - len += sprintf(buf2 + VARSTR_HEADER_SIZE, + len += tsnprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, "CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %s " "WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %s,%s,%s PAGES %d PAGESIZE %d " "PRECISION '%s' REPLICA %d " @@ -426,7 +430,7 @@ static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, pCfg->s3KeepLocal, pCfg->s3Compact); if (pRetentions) { - len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", pRetentions); + len += tsnprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, " RETENTIONS %s", pRetentions); } } @@ -503,28 +507,33 @@ static int32_t buildCreateViewResultDataBlock(SSDataBlock** pOutput) { void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { for (int32_t i = 0; i < pCfg->numOfColumns; ++i) { SSchema* pSchema = pCfg->pSchemas + i; - char type[32 + 60]; // 60 byte for compress info - sprintf(type, "%s", tDataTypes[pSchema->type].name); +#define LTYPE_LEN (32 + 60) // 60 byte for compress info + char type[LTYPE_LEN]; + snprintf(type, LTYPE_LEN, "%s", tDataTypes[pSchema->type].name); + int typeLen = strlen(type); if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_VARBINARY == pSchema->type || TSDB_DATA_TYPE_GEOMETRY == pSchema->type) { - sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE)); + typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE)); } else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) { - sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); + typeLen += snprintf(type + typeLen, LTYPE_LEN - typeLen, "(%d)", + (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } if (useCompress(pCfg->tableType) && pCfg->pSchemaExt) { - sprintf(type + strlen(type), " ENCODE \'%s\'", - columnEncodeStr(COMPRESS_L1_TYPE_U32(pCfg->pSchemaExt[i].compress))); - sprintf(type + strlen(type), " COMPRESS \'%s\'", - columnCompressStr(COMPRESS_L2_TYPE_U32(pCfg->pSchemaExt[i].compress))); - sprintf(type + strlen(type), " LEVEL \'%s\'", - columnLevelStr(COMPRESS_L2_TYPE_LEVEL_U32(pCfg->pSchemaExt[i].compress))); + typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, " ENCODE \'%s\'", + columnEncodeStr(COMPRESS_L1_TYPE_U32(pCfg->pSchemaExt[i].compress))); + typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, " COMPRESS \'%s\'", + columnCompressStr(COMPRESS_L2_TYPE_U32(pCfg->pSchemaExt[i].compress))); + typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, " LEVEL \'%s\'", + columnLevelStr(COMPRESS_L2_TYPE_LEVEL_U32(pCfg->pSchemaExt[i].compress))); } if (!(pSchema->flags & COL_IS_KEY)) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type); + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), "%s`%s` %s", + ((i > 0) ? ", " : ""), pSchema->name, type); } else { char* pk = "PRIMARY KEY"; - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s %s", ((i > 0) ? ", " : ""), pSchema->name, type, pk); + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), "%s`%s` %s %s", + ((i > 0) ? ", " : ""), pSchema->name, type, pk); } } } @@ -533,22 +542,25 @@ void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) { for (int32_t i = 0; i < pCfg->numOfTags; ++i) { SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; char type[32]; - sprintf(type, "%s", tDataTypes[pSchema->type].name); + snprintf(type, sizeof(type), "%s", tDataTypes[pSchema->type].name); if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_VARBINARY == pSchema->type || TSDB_DATA_TYPE_GEOMETRY == pSchema->type) { - sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE)); + snprintf(type + strlen(type), sizeof(type) - strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE)); } else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) { - sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); + snprintf(type + strlen(type), sizeof(type) - strlen(type), "(%d)", + (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type); + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, sizeof(type) - (VARSTR_HEADER_SIZE + *len), "%s`%s` %s", + ((i > 0) ? ", " : ""), pSchema->name, type); } } void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) { for (int32_t i = 0; i < pCfg->numOfTags; ++i) { SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s`", ((i > 0) ? ", " : ""), pSchema->name); + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), + "%s`%s`", ((i > 0) ? ", " : ""), pSchema->name); } } @@ -565,11 +577,12 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { if (tTagIsJson(pTag)) { char* pJson = NULL; parseTagDatatoJson(pTag, &pJson); - if(NULL == pJson) { + if (NULL == pJson) { qError("failed to parse tag to json, pJson is NULL"); return terrno; } - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s", pJson); + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), + "%s", pJson); taosMemoryFree(pJson); return TSDB_CODE_SUCCESS; @@ -582,11 +595,13 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { for (int32_t i = 0; i < pCfg->numOfTags; ++i) { SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; if (i > 0) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", "); + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), + ", "); } if (j >= valueNum) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "NULL"); + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), + "NULL"); continue; } @@ -599,17 +614,24 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { char type = pTagVal->type; int32_t tlen = 0; + int64_t leftSize = SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len); + if (leftSize <= 0) { + qError("no enough space to store tag value, leftSize:%" PRId64, leftSize); + code = TSDB_CODE_APP_ERROR; + TAOS_CHECK_ERRNO(code); + } if (IS_VAR_DATA_TYPE(type)) { - code = dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, type, pTagVal->pData, pTagVal->nData, &tlen); + code = dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, leftSize, type, pTagVal->pData, pTagVal->nData, &tlen); TAOS_CHECK_ERRNO(code); } else { - code = dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, type, &pTagVal->i64, tDataTypes[type].bytes, &tlen); + code = dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, leftSize, type, &pTagVal->i64, tDataTypes[type].bytes, &tlen); TAOS_CHECK_ERRNO(code); } *len += tlen; j++; } else { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "NULL"); + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), + "NULL"); } } _exit: @@ -620,37 +642,47 @@ _exit: void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg* pCfg) { if (pCfg->commentLen > 0) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " COMMENT '%s'", pCfg->pComment); + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), + " COMMENT '%s'", pCfg->pComment); } else if (0 == pCfg->commentLen) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " COMMENT ''"); + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), + " COMMENT ''"); } if (NULL != pDbCfg->pRetensions && pCfg->watermark1 > 0) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " WATERMARK %" PRId64 "a", pCfg->watermark1); + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), + " WATERMARK %" PRId64 "a", pCfg->watermark1); if (pCfg->watermark2 > 0) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", %" PRId64 "a", pCfg->watermark2); + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), + ", %" PRId64 "a", pCfg->watermark2); } } if (NULL != pDbCfg->pRetensions && pCfg->delay1 > 0) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " MAX_DELAY %" PRId64 "a", pCfg->delay1); + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), + " MAX_DELAY %" PRId64 "a", pCfg->delay1); if (pCfg->delay2 > 0) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", %" PRId64 "a", pCfg->delay2); + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), + ", %" PRId64 "a", pCfg->delay2); } } int32_t funcNum = taosArrayGetSize(pCfg->pFuncs); if (NULL != pDbCfg->pRetensions && funcNum > 0) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " ROLLUP("); + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), + " ROLLUP("); for (int32_t i = 0; i < funcNum; ++i) { char* pFunc = taosArrayGet(pCfg->pFuncs, i); - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s%s", ((i > 0) ? ", " : ""), pFunc); + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), + "%s%s", ((i > 0) ? ", " : ""), pFunc); } - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ")"); + *len += + snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), ")"); } if (pCfg->ttl > 0) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " TTL %d", pCfg->ttl); + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), + " TTL %d", pCfg->ttl); } if (TSDB_SUPER_TABLE == pCfg->tableType || TSDB_NORMAL_TABLE == pCfg->tableType) { @@ -663,18 +695,23 @@ void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg* if (nSma < pCfg->numOfColumns && nSma > 0) { bool smaOn = false; - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " SMA("); + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), + " SMA("); for (int32_t i = 0; i < pCfg->numOfColumns; ++i) { if (IS_BSMA_ON(pCfg->pSchemas + i)) { if (smaOn) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ",`%s`", (pCfg->pSchemas + i)->name); + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, + SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), ",`%s`", + (pCfg->pSchemas + i)->name); } else { smaOn = true; - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "`%s`", (pCfg->pSchemas + i)->name); + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, + SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), "`%s`", + (pCfg->pSchemas + i)->name); } } } - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ")"); + *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, ")"); } } } @@ -698,24 +735,32 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p int32_t len = 0; if (TSDB_SUPER_TABLE == pCfg->tableType) { - len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE STABLE `%s` (", tbName); + len += tsnprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_TB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, + "CREATE STABLE `%s` (", tbName); appendColumnFields(buf2, &len, pCfg); - len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ") TAGS ("); + len += tsnprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + len), + ") TAGS ("); appendTagFields(buf2, &len, pCfg); - len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")"); + len += + snprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + len), ")"); appendTableOptions(buf2, &len, pDbCfg, pCfg); } else if (TSDB_CHILD_TABLE == pCfg->tableType) { - len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE TABLE `%s` USING `%s` (", tbName, pCfg->stbName); + len += tsnprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_TB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, + "CREATE TABLE `%s` USING `%s` (", tbName, pCfg->stbName); appendTagNameFields(buf2, &len, pCfg); - len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ") TAGS ("); + len += tsnprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + len), + ") TAGS ("); code = appendTagValues(buf2, &len, pCfg); TAOS_CHECK_ERRNO(code); - len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")"); + len += + snprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + len), ")"); appendTableOptions(buf2, &len, pDbCfg, pCfg); } else { - len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE TABLE `%s` (", tbName); + len += tsnprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_TB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, + "CREATE TABLE `%s` (", tbName); appendColumnFields(buf2, &len, pCfg); - len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")"); + len += + snprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + len), ")"); appendTableOptions(buf2, &len, pDbCfg, pCfg); } @@ -792,9 +837,21 @@ static int32_t execAlterCmd(char* cmd, char* value, bool* processed) { taosResetLog(); cfgDumpCfg(tsCfg, 0, false); } else if (0 == strcasecmp(cmd, COMMAND_SCHEDULE_POLICY)) { - code = schedulerUpdatePolicy(atoi(value)); + int32_t tmp = 0; + code = taosStr2int32(value, &tmp); + if (code) { + qError("invalid value:%s, error:%s", value, tstrerror(code)); + return code; + } + code = schedulerUpdatePolicy(tmp); } else if (0 == strcasecmp(cmd, COMMAND_ENABLE_RESCHEDULE)) { - code = schedulerEnableReSchedule(atoi(value)); + int32_t tmp = 0; + code = taosStr2int32(value, &tmp); + if (code) { + qError("invalid value:%s, error:%s", value, tstrerror(code)); + return code; + } + code = schedulerEnableReSchedule(tmp != 0); } else if (0 == strcasecmp(cmd, COMMAND_CATALOG_DEBUG)) { code = ctgdHandleDbgCommand(value); } else if (0 == strcasecmp(cmd, COMMAND_ENABLE_MEM_DEBUG)) { diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index e391d274e3..572ff88be9 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -833,6 +833,9 @@ void cleanupResultInfoInStream(SExecTaskInfo* pTaskInfo, void* pState, SExpr SGroupResInfo* pGroupResInfo); void cleanupResultInfo(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap); +void cleanupResultInfoWithoutHash(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SDiskbasedBuf* pBuf, + SGroupResInfo* pGroupResInfo); + int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, const char* pkey, void* pState, SFunctionStateStore* pStore); void cleanupAggSup(SAggSupporter* pAggSup); diff --git a/source/libs/executor/inc/operator.h b/source/libs/executor/inc/operator.h index 0df676c6e2..fce0b05fde 100644 --- a/source/libs/executor/inc/operator.h +++ b/source/libs/executor/inc/operator.h @@ -190,6 +190,9 @@ int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdSt int32_t getOperatorExplainExecInfo(struct SOperatorInfo* operatorInfo, SArray* pExecInfoList); void * getOperatorParam(int32_t opType, SOperatorParam* param, int32_t idx); +void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts, uint64_t groupId); +void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex, uint64_t groupId); + #ifdef __cplusplus } #endif diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index 9e5ad132f7..91b435fbec 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -159,8 +159,8 @@ void destroyAggOperatorInfo(void* param) { cleanupBasicInfo(&pInfo->binfo); if (pInfo->pOperator) { - cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf, - &pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable); + cleanupResultInfoWithoutHash(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf, + &pInfo->groupResInfo); pInfo->pOperator = NULL; } cleanupAggSup(&pInfo->aggSup); @@ -627,6 +627,42 @@ void cleanupResultInfoInStream(SExecTaskInfo* pTaskInfo, void* pState, SExprSupp } } +void cleanupResultInfoWithoutHash(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SDiskbasedBuf* pBuf, + SGroupResInfo* pGroupResInfo) { + int32_t numOfExprs = pSup->numOfExprs; + int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; + SqlFunctionCtx* pCtx = pSup->pCtx; + int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); + bool needCleanup = false; + + for (int32_t j = 0; j < numOfExprs; ++j) { + needCleanup |= pCtx[j].needCleanup; + } + if (!needCleanup) { + return; + } + + for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) { + SResultRow* pRow = NULL; + SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i); + SFilePage* page = getBufPage(pBuf, pPos->pos.pageId); + if (page == NULL) { + qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo)); + continue; + } + pRow = (SResultRow*)((char*)page + pPos->pos.offset); + + + for (int32_t j = 0; j < numOfExprs; ++j) { + pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset); + if (pCtx[j].fpSet.cleanup) { + pCtx[j].fpSet.cleanup(&pCtx[j]); + } + } + releaseBufPage(pBuf, page); + } +} + void cleanupResultInfo(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap) { int32_t numOfExprs = pSup->numOfExprs; diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index 57f4289ebf..c284e9a8a9 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -273,10 +273,18 @@ static int32_t getCacheSize(struct SDataSinkHandle* pHandle, uint64_t* size) { int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, void* pParam) { int32_t code = TSDB_CODE_SUCCESS; + if (pParam == NULL) { + code = TSDB_CODE_QRY_INVALID_INPUT; + qError("invalid input param in creating data deleter, code%s", tstrerror(code)); + goto _end; + } + + SDeleterParam* pDeleterParam = (SDeleterParam*)pParam; SDataDeleterHandle* deleter = taosMemoryCalloc(1, sizeof(SDataDeleterHandle)); if (NULL == deleter) { code = terrno; + taosArrayDestroy(pDeleterParam->pUidList); taosMemoryFree(pParam); goto _end; } @@ -292,12 +300,6 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData deleter->pDeleter = pDeleterNode; deleter->pSchema = pDataSink->pInputDataBlockDesc; - if (pParam == NULL) { - code = TSDB_CODE_QRY_INVALID_INPUT; - qError("invalid input param in creating data deleter, code%s", tstrerror(code)); - goto _end; - } - deleter->pParam = pParam; deleter->status = DS_BUF_EMPTY; deleter->queryEnd = false; diff --git a/source/libs/executor/src/eventwindowoperator.c b/source/libs/executor/src/eventwindowoperator.c index f473626953..e68a91d97d 100644 --- a/source/libs/executor/src/eventwindowoperator.c +++ b/source/libs/executor/src/eventwindowoperator.c @@ -44,22 +44,6 @@ static int32_t eventWindowAggregateNext(SOperatorInfo* pOperator, SSDataBlock** static void destroyEWindowOperatorInfo(void* param); static int32_t eventWindowAggImpl(SOperatorInfo* pOperator, SEventWindowOperatorInfo* pInfo, SSDataBlock* pBlock); -// todo : move to util -static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex, - uint64_t groupId) { - pRowSup->startRowIndex = rowIndex; - pRowSup->numOfRows = 0; - pRowSup->win.skey = tsList[rowIndex]; - pRowSup->groupId = groupId; -} - -static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts, uint64_t groupId) { - pRowSup->win.ekey = ts; - pRowSup->prevTs = ts; - pRowSup->numOfRows += 1; - pRowSup->groupId = groupId; -} - int32_t createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) { QRY_PARAM_CHECK(pOptrInfo); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 04b3a83264..0cadffbfdf 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1143,11 +1143,11 @@ SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, S varDataSetLen(tmp, tagVal.nData); memcpy(tmp + VARSTR_HEADER_SIZE, tagVal.pData, tagVal.nData); code = colDataSetVal(pColInfo, i, tmp, false); - QUERY_CHECK_CODE(code, lino, _end); #if TAG_FILTER_DEBUG qDebug("tagfilter varch:%s", tmp + 2); #endif taosMemoryFree(tmp); + QUERY_CHECK_CODE(code, lino, _end); } else { code = colDataSetVal(pColInfo, i, (const char*)&tagVal.i64, false); QUERY_CHECK_CODE(code, lino, _end); @@ -1797,7 +1797,6 @@ int32_t createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) { SDataType* pType = &pFuncNode->node.resType; pExp->base.resSchema = createResSchema(pType->type, pType->bytes, slotId, pType->scale, pType->precision, pFuncNode->node.aliasName); - tExprNode* pExprNode = pExp->pExpr; pExprNode->_function.functionId = pFuncNode->funcId; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index cbf392f67e..27dd687f40 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1635,13 +1635,17 @@ int32_t getTableListInfo(const SExecTaskInfo* pTaskInfo, SArray** pList) { int32_t qStreamOperatorReleaseState(qTaskInfo_t tInfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tInfo; - pTaskInfo->pRoot->fpSet.releaseStreamStateFn(pTaskInfo->pRoot); + if (pTaskInfo->pRoot->fpSet.releaseStreamStateFn != NULL) { + pTaskInfo->pRoot->fpSet.releaseStreamStateFn(pTaskInfo->pRoot); + } return 0; } int32_t qStreamOperatorReloadState(qTaskInfo_t tInfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tInfo; - pTaskInfo->pRoot->fpSet.reloadStreamStateFn(pTaskInfo->pRoot); + if (pTaskInfo->pRoot->fpSet.reloadStreamStateFn != NULL) { + pTaskInfo->pRoot->fpSet.reloadStreamStateFn(pTaskInfo->pRoot); + } return 0; } diff --git a/source/libs/executor/src/groupcacheoperator.c b/source/libs/executor/src/groupcacheoperator.c index d785a1e619..648a2ea6d2 100644 --- a/source/libs/executor/src/groupcacheoperator.c +++ b/source/libs/executor/src/groupcacheoperator.c @@ -78,14 +78,15 @@ static void logGroupCacheExecInfo(SGroupCacheOperatorInfo* pGrpCacheOperator) { if (pGrpCacheOperator->downstreamNum <= 0 || NULL == pGrpCacheOperator->execInfo.pDownstreamBlkNum) { return; } - - char* buf = taosMemoryMalloc(pGrpCacheOperator->downstreamNum * 32 + 100); + + int32_t bufSize = pGrpCacheOperator->downstreamNum * 32 + 100; + char* buf = taosMemoryMalloc(bufSize); if (NULL == buf) { return; } - int32_t offset = sprintf(buf, "groupCache exec info, downstreamBlkNum:"); + int32_t offset = snprintf(buf, bufSize, "groupCache exec info, downstreamBlkNum:"); for (int32_t i = 0; i < pGrpCacheOperator->downstreamNum; ++i) { - offset += sprintf(buf + offset, " %" PRId64 , pGrpCacheOperator->execInfo.pDownstreamBlkNum[i]); + offset += snprintf(buf + offset, bufSize, " %" PRId64 , pGrpCacheOperator->execInfo.pDownstreamBlkNum[i]); } qDebug("%s", buf); taosMemoryFree(buf); @@ -234,7 +235,7 @@ static int32_t acquireFdFromFileCtx(SGcFileCacheCtx* pFileCtx, int32_t fileId, S SGroupCacheFileInfo* pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId)); if (NULL == pTmp) { - (void)sprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], "_%d", fileId); + (void)snprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], sizeof(pFileCtx->baseFilename) - pFileCtx->baseNameLen, "_%d", fileId); SGroupCacheFileInfo newFile = {0}; if (taosHashPut(pFileCtx->pCacheFile, &fileId, sizeof(fileId), &newFile, sizeof(newFile))) { @@ -439,7 +440,7 @@ static FORCE_INLINE void chkRemoveVgroupCurrFile(SGcFileCacheCtx* pFileCtx, int3 #if 0 /* debug only */ - sprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], "_%d", pFileCtx->fileId); + snprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], sizeof(pFileCtx->baseFilename) - pFileCtx->baseNameLen, "_%d", pFileCtx->fileId); taosRemoveFile(pFileCtx->baseFilename); /* debug only */ #endif @@ -522,7 +523,7 @@ static int32_t buildGroupCacheBaseBlock(SSDataBlock** ppDst, SSDataBlock* pSrc) (*ppDst)->pDataBlock = taosArrayDup(pSrc->pDataBlock, NULL); if (NULL == (*ppDst)->pDataBlock) { taosMemoryFree(*ppDst); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } TAOS_MEMCPY(&(*ppDst)->info, &pSrc->info, sizeof(pSrc->info)); blockDataDeepClear(*ppDst); @@ -813,7 +814,7 @@ static int32_t addFileRefTableNum(SGcFileCacheCtx* pFileCtx, int32_t fileId, int SGroupCacheFileInfo* pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId)); if (NULL == pTmp) { - (void)sprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], "_%u", fileId); + (void)snprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], sizeof(pFileCtx->baseFilename) - pFileCtx->baseNameLen, "_%u", fileId); SGroupCacheFileInfo newFile = {0}; newFile.groupNum = 1; @@ -1377,7 +1378,7 @@ static void freeRemoveGroupCacheData(void* p) { #if 0 /* debug only */ - sprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], "_%d", pGroup->fileId); + snprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], sizeof(pFileCtx->baseFilename) - pFileCtx->baseNameLen, "_%d", pGroup->fileId); taosRemoveFile(pFileCtx->baseFilename); /* debug only */ #endif diff --git a/source/libs/executor/src/streamfilloperator.c b/source/libs/executor/src/streamfilloperator.c index 291cc3b67b..826220581a 100644 --- a/source/libs/executor/src/streamfilloperator.c +++ b/source/libs/executor/src/streamfilloperator.c @@ -1165,12 +1165,12 @@ _end: return code; } -static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { - pFillSup->rowSize = sizeof(SResultCellData) * pFillSup->numOfAllCols; - for (int i = 0; i < pFillSup->numOfAllCols; i++) { - SFillColInfo* pCol = &pFillSup->pAllColInfo[i]; - SResSchema* pSchema = &pCol->pExpr->base.resSchema; - pFillSup->rowSize += pSchema->bytes; +static int32_t initResultBuf(SSDataBlock* pInputRes, SStreamFillSupporter* pFillSup) { + int32_t numOfCols = taosArrayGetSize(pInputRes->pDataBlock); + pFillSup->rowSize = sizeof(SResultCellData) * numOfCols; + for (int i = 0; i < numOfCols; i++) { + SColumnInfoData* pCol = taosArrayGet(pInputRes->pDataBlock, i); + pFillSup->rowSize += pCol->info.bytes; } pFillSup->next.key = INT64_MIN; pFillSup->nextNext.key = INT64_MIN; @@ -1185,7 +1185,7 @@ static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { } static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNode, SInterval* pInterval, - SExprInfo* pFillExprInfo, int32_t numOfFillCols, SStorageAPI* pAPI) { + SExprInfo* pFillExprInfo, int32_t numOfFillCols, SStorageAPI* pAPI, SSDataBlock* pInputRes) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter)); @@ -1214,7 +1214,7 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod pFillSup->interval = *pInterval; pFillSup->pAPI = pAPI; - code = initResultBuf(pFillSup); + code = initResultBuf(pInputRes, pFillSup); QUERY_CHECK_CODE(code, lino, _end); SExprInfo* noFillExpr = NULL; @@ -1371,7 +1371,11 @@ int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysi code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI.functionStore); QUERY_CHECK_CODE(code, lino, _error); - pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI); + pInfo->pSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); + QUERY_CHECK_NULL(pInfo->pSrcBlock, code, lino, _error, terrno); + + pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI, + pInfo->pSrcBlock); if (!pInfo->pFillSup) { code = TSDB_CODE_FAILED; QUERY_CHECK_CODE(code, lino, _error); @@ -1380,8 +1384,7 @@ int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysi initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); QUERY_CHECK_NULL(pInfo->pRes, code, lino, _error, terrno); - pInfo->pSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); - QUERY_CHECK_NULL(pInfo->pSrcBlock, code, lino, _error, terrno); + code = blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); QUERY_CHECK_CODE(code, lino, _error); diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index e8322d6911..be27f277c0 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -49,7 +49,7 @@ #define STREAM_SESSION_OP_CHECKPOINT_NAME "StreamSessionOperator_Checkpoint" #define STREAM_STATE_OP_CHECKPOINT_NAME "StreamStateOperator_Checkpoint" -#define MAX_STREAM_HISTORY_RESULT 100000000 +#define MAX_STREAM_HISTORY_RESULT 20000000 typedef struct SStateWindowInfo { SResultWindowInfo winInfo; @@ -449,7 +449,7 @@ void destroyFlusedPos(void* pRes) { } void destroyFlusedppPos(void* ppRes) { - void *pRes = *(void **)ppRes; + void* pRes = *(void**)ppRes; destroyFlusedPos(pRes); } @@ -507,7 +507,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { } taosArrayDestroy(pInfo->pMidPullDatas); - if (pInfo->pState !=NULL && pInfo->pState->dump == 1) { + if (pInfo->pState != NULL && pInfo->pState->dump == 1) { taosMemoryFreeClear(pInfo->pState->pTdbState->pOwner); taosMemoryFreeClear(pInfo->pState->pTdbState); } @@ -548,7 +548,8 @@ void reloadFromDownStream(SOperatorInfo* downstream, SStreamIntervalOperatorInfo bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { return pInfo->primaryPkIndex != -1; } -int32_t initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamIntervalOperatorInfo* pInfo, struct SSteamOpBasicInfo* pBasic) { +int32_t initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamIntervalOperatorInfo* pInfo, + struct SSteamOpBasicInfo* pBasic) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SStateStore* pAPI = &downstream->pTaskInfo->storageAPI.stateStore; @@ -1028,7 +1029,7 @@ static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pN } static int32_t doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, uint64_t groupId, - SSHashObj* pUpdatedMap, SSHashObj* pDeletedMap) { + SSHashObj* pUpdatedMap, SSHashObj* pDeletedMap) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info; @@ -1661,7 +1662,7 @@ static int32_t doStreamFinalIntervalAggNext(SOperatorInfo* pOperator, SSDataBloc pInfo->binfo.pRes->info.type = pBlock->info.type; } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { - SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); + SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); QUERY_CHECK_NULL(delWins, code, lino, _end, terrno); SHashObj* finalMap = IS_FINAL_INTERVAL_OP(pOperator) ? pInfo->pFinalPullDataMap : NULL; code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, finalMap); @@ -1897,9 +1898,8 @@ _end: } } -int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, - SExecTaskInfo* pTaskInfo, int32_t numOfChild, - SReadHandle* pHandle, SOperatorInfo** pOptrInfo) { +int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, + int32_t numOfChild, SReadHandle* pHandle, SOperatorInfo** pOptrInfo) { QRY_PARAM_CHECK(pOptrInfo); int32_t code = TSDB_CODE_SUCCESS; @@ -1959,8 +1959,8 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex); - int32_t numOfCols = 0; - SExprInfo* pExprInfo = NULL; + int32_t numOfCols = 0; + SExprInfo* pExprInfo = NULL; code = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &pExprInfo, &numOfCols); QUERY_CHECK_CODE(code, lino, _error); @@ -2042,11 +2042,13 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN pOperator->info = pInfo; if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) { - pOperator->fpSet = createOperatorFpSet(NULL, doStreamMidIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo, - optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = + createOperatorFpSet(NULL, doStreamMidIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); } else { - pOperator->fpSet = createOperatorFpSet(NULL, doStreamFinalIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo, - optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = + createOperatorFpSet(NULL, doStreamFinalIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); } setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState); if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || @@ -2220,10 +2222,10 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, in *(pSup->pState) = *pState; pSup->stateStore.streamStateSetNumber(pSup->pState, -1, tsIndex); int32_t funResSize = getMaxFunResSize(pExpSup, numOfOutput); - pSup->pState->pFileState = NULL; - code = pSup->stateStore.streamFileStateInit( - tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize, funResSize, sesionTs, pSup->pState, - pTwAggSup->deleteMark, taskIdStr, pHandle->checkpointId, STREAM_STATE_BUFF_SORT, &pSup->pState->pFileState); + pSup->pState->pFileState = NULL; + code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize, funResSize, + sesionTs, pSup->pState, pTwAggSup->deleteMark, taskIdStr, + pHandle->checkpointId, STREAM_STATE_BUFF_SORT, &pSup->pState->pFileState); QUERY_CHECK_CODE(code, lino, _end); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); @@ -3309,8 +3311,8 @@ int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpe QUERY_CHECK_CONDITION((winCode == TSDB_CODE_SUCCESS), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); - code = - tSimpleHashPut(pInfo->streamAggSup.pResultRows, &winfo.sessionWin, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + code = tSimpleHashPut(pInfo->streamAggSup.pResultRows, &winfo.sessionWin, sizeof(SSessionKey), &winfo, + sizeof(SResultWindowInfo)); QUERY_CHECK_CODE(code, lino, _end); } @@ -3772,8 +3774,8 @@ _end: } } -int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, - SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pOptrInfo) { +int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, + SReadHandle* pHandle, SOperatorInfo** pOptrInfo) { QRY_PARAM_CHECK(pOptrInfo); SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; @@ -3802,12 +3804,12 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode } } SExprSupp* pExpSup = &pOperator->exprSupp; - + SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno); pInfo->binfo.pRes = pResBlock; - SExprInfo* pExprInfo = NULL; + SExprInfo* pExprInfo = NULL; code = createExprInfo(pSessionNode->window.pFuncs, NULL, &pExprInfo, &numOfCols); QUERY_CHECK_CODE(code, lino, _error); @@ -3886,8 +3888,9 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode QUERY_CHECK_CODE(code, lino, _error); } } - pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionAggNext, NULL, destroyStreamSessionAggOperatorInfo, - optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = + createOperatorFpSet(optrDummyOpenFn, doStreamSessionAggNext, NULL, destroyStreamSessionAggOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState); if (downstream) { @@ -4102,8 +4105,8 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { } int32_t createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, - SExecTaskInfo* pTaskInfo, int32_t numOfChild, - SReadHandle* pHandle, SOperatorInfo** pOptrInfo) { + SExecTaskInfo* pTaskInfo, int32_t numOfChild, SReadHandle* pHandle, + SOperatorInfo** pOptrInfo) { QRY_PARAM_CHECK(pOptrInfo); int32_t code = TSDB_CODE_SUCCESS; @@ -4111,7 +4114,7 @@ int32_t createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhys SOperatorInfo* pOperator = NULL; code = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle, &pOperator); if (pOperator == NULL || code != 0) { - downstream = NULL; + downstream = NULL; QUERY_CHECK_CODE(code, lino, _error); } @@ -4618,8 +4621,8 @@ int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera QUERY_CHECK_CODE(code, lino, _end); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); - code = - tSimpleHashPut(pInfo->streamAggSup.pResultRows, &winfo.sessionWin, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + code = tSimpleHashPut(pInfo->streamAggSup.pResultRows, &winfo.sessionWin, sizeof(SSessionKey), &winfo, + sizeof(SResultWindowInfo)); QUERY_CHECK_CODE(code, lino, _end); } @@ -4846,7 +4849,7 @@ void streamStateReleaseState(SOperatorInfo* pOperator) { int32_t resSize = winSize + sizeof(TSKEY); char* pBuff = taosMemoryCalloc(1, resSize); if (!pBuff) { - return ; + return; } memcpy(pBuff, pInfo->historyWins->pData, winSize); memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY)); @@ -4944,7 +4947,7 @@ void streamStateReloadState(SOperatorInfo* pOperator) { QUERY_CHECK_CODE(code, lino, _end); } } - taosMemoryFree(pBuf); + taosMemoryFreeClear(pBuf); SOperatorInfo* downstream = pOperator->pDownstream[0]; if (downstream->fpSet.reloadStreamStateFn) { @@ -4953,6 +4956,7 @@ void streamStateReloadState(SOperatorInfo* pOperator) { reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); _end: + taosMemoryFreeClear(pBuf); if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo)); } @@ -5001,9 +5005,9 @@ int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno); pInfo->binfo.pRes = pResBlock; - SExprSupp* pExpSup = &pOperator->exprSupp; - int32_t numOfCols = 0; - SExprInfo* pExprInfo = NULL; + SExprSupp* pExpSup = &pOperator->exprSupp; + int32_t numOfCols = 0; + SExprInfo* pExprInfo = NULL; code = createExprInfo(pStateNode->window.pFuncs, NULL, &pExprInfo, &numOfCols); QUERY_CHECK_CODE(code, lino, _error); @@ -5335,7 +5339,7 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* *(pInfo->pState) = *(pTaskInfo->streamInfo.pState); pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex); - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; SExprInfo* pExprInfo = NULL; code = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &pExprInfo, &numOfCols); QUERY_CHECK_CODE(code, lino, _error); @@ -5383,7 +5387,8 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pInfo->pState->pFileState = NULL; code = pTaskInfo->storageAPI.stateStore.streamFileStateInit( tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, compareTs, pInfo->pState, - pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo), pHandle->checkpointId, STREAM_STATE_BUFF_HASH, &pInfo->pState->pFileState); + pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo), pHandle->checkpointId, STREAM_STATE_BUFF_HASH, + &pInfo->pState->pFileState); QUERY_CHECK_CODE(code, lino, _error); pInfo->pOperator = pOperator; @@ -5397,7 +5402,7 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pInfo->recvGetAll = false; code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes); - QUERY_CHECK_CODE(code, lino, _error); + QUERY_CHECK_CODE(code, lino, _error); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pDeletedMap = tSimpleHashInit(4096, hashFn); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 8164281871..34ecda6ce7 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -89,14 +89,14 @@ static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindo return setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset); } -static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts, uint64_t groupId) { +void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts, uint64_t groupId) { pRowSup->win.ekey = ts; pRowSup->prevTs = ts; pRowSup->numOfRows += 1; pRowSup->groupId = groupId; } -static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex, +void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex, uint64_t groupId) { pRowSup->startRowIndex = rowIndex; pRowSup->numOfRows = 0; @@ -1229,10 +1229,9 @@ static void destroyStateWindowOperatorInfo(void* param) { SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); taosMemoryFreeClear(pInfo->stateKey.pData); - - if (pInfo->pOperator != NULL) { - cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf, - &pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable); + if (pInfo->pOperator) { + cleanupResultInfoWithoutHash(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf, + &pInfo->groupResInfo); pInfo->pOperator = NULL; } @@ -1257,10 +1256,9 @@ void destroyIntervalOperatorInfo(void* param) { SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); - - if (pInfo->pOperator != NULL) { - cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf, - &pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable); + if (pInfo->pOperator) { + cleanupResultInfoWithoutHash(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf, + &pInfo->groupResInfo); pInfo->pOperator = NULL; } @@ -1764,10 +1762,9 @@ void destroySWindowOperatorInfo(void* param) { cleanupBasicInfo(&pInfo->binfo); colDataDestroy(&pInfo->twAggSup.timeWindowData); - - if (pInfo->pOperator != NULL) { - cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf, - &pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable); + if (pInfo->pOperator) { + cleanupResultInfoWithoutHash(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf, + &pInfo->groupResInfo); pInfo->pOperator = NULL; } diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 0b2fb70eba..970bdc6305 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -22,29 +22,7 @@ extern "C" { #include "function.h" #include "functionMgt.h" - -typedef struct SSumRes { - union { - int64_t isum; - uint64_t usum; - double dsum; - }; - int16_t type; - int64_t prevTs; - bool isPrevTsSet; - bool overflow; // if overflow is true, dsum to be used for any type; -} SSumRes; - -typedef struct SMinmaxResInfo { - bool assign; // assign the first value or not - int64_t v; - char *str; - STuplePos tuplePos; - - STuplePos nullTuplePos; - bool nullTupleSaved; - int16_t type; -} SMinmaxResInfo; +#include "functionResInfoInt.h" int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems); diff --git a/source/libs/function/inc/functionResInfoInt.h b/source/libs/function/inc/functionResInfoInt.h new file mode 100644 index 0000000000..9ee1e884b3 --- /dev/null +++ b/source/libs/function/inc/functionResInfoInt.h @@ -0,0 +1,366 @@ +/* +* Copyright (c) 2019 TAOS Data, Inc. +* +* This program is free software: you can use, redistribute, and/or modify +* it under the terms of the GNU Affero General Public License, version 3 +* or later ("AGPL"), as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, but WITHOUT +* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +* FITNESS FOR A PARTICULAR PURPOSE. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +*/ + +#ifndef TDENGINE_FUNCTIONRESINFOINT_H +#define TDENGINE_FUNCTIONRESINFOINT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" +#include "thistogram.h" +#include "tdigest.h" +#include "functionResInfo.h" +#include "tpercentile.h" + +#define USE_ARRAYLIST + +#define HLL_BUCKET_BITS 14 // The bits of the bucket +#define HLL_DATA_BITS (64 - HLL_BUCKET_BITS) +#define HLL_BUCKETS (1 << HLL_BUCKET_BITS) +#define HLL_BUCKET_MASK (HLL_BUCKETS - 1) +#define HLL_ALPHA_INF 0.721347520444481703680 // constant for 0.5/ln(2) + +typedef struct SSumRes { + union { + int64_t isum; + uint64_t usum; + double dsum; + }; + int16_t type; + int64_t prevTs; + bool isPrevTsSet; + bool overflow; // if overflow is true, dsum to be used for any type; +} SSumRes; + +typedef struct SMinmaxResInfo { + bool assign; // assign the first value or not + int64_t v; + char *str; + STuplePos tuplePos; + + STuplePos nullTuplePos; + bool nullTupleSaved; + int16_t type; +} SMinmaxResInfo; + +typedef struct SStdRes { + double result; + int64_t count; + union { + double quadraticDSum; + int64_t quadraticISum; + uint64_t quadraticUSum; + }; + union { + double dsum; + int64_t isum; + uint64_t usum; + }; + int16_t type; +} SStdRes; + +typedef struct SHistBin { + double val; + int64_t num; + +#if !defined(USE_ARRAYLIST) + double delta; + int32_t index; // index in min-heap list +#endif +} SHistBin; + +typedef struct SHistogramInfo { + int64_t numOfElems; + int32_t numOfEntries; + int32_t maxEntries; + double min; + double max; +#if defined(USE_ARRAYLIST) + SHistBin* elems; +#else + tSkipList* pList; + SMultiwayMergeTreeInfo* pLoserTree; + int32_t maxIndex; + bool ordered; +#endif +} SHistogramInfo; + +typedef struct SAPercentileInfo { + double result; + double percent; + int8_t algo; + SHistogramInfo* pHisto; + TDigest* pTDigest; +} SAPercentileInfo; + +typedef struct SSpreadInfo { + double result; + bool hasResult; + double min; + double max; +} SSpreadInfo; + +typedef struct SHLLFuncInfo { + uint64_t result; + uint64_t totalCount; + uint8_t buckets[HLL_BUCKETS]; +} SHLLInfo; + +typedef struct SGroupKeyInfo { + bool hasResult; + bool isNull; + char data[]; +} SGroupKeyInfo; + +typedef struct SAvgRes { + double result; + SSumRes sum; + int64_t count; + int16_t type; // store the original input type, used in merge function +} SAvgRes; + + +// structs above are used in stream + +#define HISTOGRAM_MAX_BINS_NUM 1000 +#define MAVG_MAX_POINTS_NUM 1000 +#define TAIL_MAX_POINTS_NUM 100 +#define TAIL_MAX_OFFSET 100 + +typedef struct STopBotResItem { + SVariant v; + uint64_t uid; // it is a table uid, used to extract tag data during building of the final result for the tag data + STuplePos tuplePos; // tuple data of this chosen row +} STopBotResItem; + +typedef struct STopBotRes { + int32_t maxSize; + int16_t type; + + STuplePos nullTuplePos; + bool nullTupleSaved; + + STopBotResItem* pItems; +} STopBotRes; + +typedef struct SLeastSQRInfo { + double matrix[2][3]; + double startVal; + double stepVal; + int64_t num; +} SLeastSQRInfo; + +typedef struct MinMaxEntry { + union { + double dMinVal; + // double i64MinVal; + uint64_t u64MinVal; + }; + union { + double dMaxVal; + // double i64MaxVal; + int64_t u64MaxVal; + }; +} MinMaxEntry; + +typedef struct { + int32_t size; + int32_t pageId; + SFilePage *data; +} SSlotInfo; + +typedef struct tMemBucketSlot { + SSlotInfo info; + MinMaxEntry range; +} tMemBucketSlot; + +struct tMemBucket; +typedef int32_t (*__perc_hash_func_t)(struct tMemBucket *pBucket, const void *value, int32_t *index); + +typedef struct tMemBucket { + int16_t numOfSlots; + int16_t type; + int32_t bytes; + int32_t total; + int32_t elemPerPage; // number of elements for each object + int32_t maxCapacity; // maximum allowed number of elements that can be sort directly to get the result + int32_t bufPageSize; // disk page size + MinMaxEntry range; // value range + int32_t times; // count that has been checked for deciding the correct data value buckets. + __compar_fn_t comparFn; + tMemBucketSlot *pSlots; + SDiskbasedBuf *pBuffer; + __perc_hash_func_t hashFunc; + SHashObj *groupPagesMap; // disk page map for different groups; +} tMemBucket; + +typedef struct SPercentileInfo { + double result; + tMemBucket* pMemBucket; + int32_t stage; + double minval; + double maxval; + int64_t numOfElems; +} SPercentileInfo; + +typedef struct SDiffInfo { + bool hasPrev; + bool isFirstRow; + int8_t ignoreOption; // replace the ignore with case when + union { + int64_t i64; + double d64; + } prev; + + int64_t prevTs; +} SDiffInfo; + +typedef struct SElapsedInfo { + double result; + TSKEY min; + TSKEY max; + int64_t timeUnit; +} SElapsedInfo; + +typedef struct STwaInfo { + double dOutput; + int64_t numOfElems; + SPoint1 p; + STimeWindow win; +} STwaInfo; + +typedef struct SHistoFuncBin { + double lower; + double upper; + int64_t count; + double percentage; +} SHistoFuncBin; + +typedef struct SHistoFuncInfo { + int32_t numOfBins; + int32_t totalCount; + bool normalized; + SHistoFuncBin bins[]; +} SHistoFuncInfo; + +typedef struct SStateInfo { + union { + int64_t count; + int64_t durationStart; + }; + int64_t prevTs; + bool isPrevTsSet; +} SStateInfo; + +typedef struct SMavgInfo { + int32_t pos; + double sum; + int64_t prevTs; + bool isPrevTsSet; + int32_t numOfPoints; + bool pointsMeet; + double points[]; +} SMavgInfo; + +typedef struct SSampleInfo { + int32_t samples; + int32_t totalPoints; + int32_t numSampled; + uint8_t colType; + uint16_t colBytes; + + STuplePos nullTuplePos; + bool nullTupleSaved; + + char* data; + STuplePos* tuplePos; +} SSampleInfo; + +typedef struct STailItem { + int64_t timestamp; + bool isNull; + char data[]; +} STailItem; + +typedef struct STailInfo { + int32_t numOfPoints; + int32_t numAdded; + int32_t offset; + uint8_t colType; + uint16_t colBytes; + STailItem** pItems; +} STailInfo; + +typedef struct SUniqueItem { + int64_t timestamp; + bool isNull; + char data[]; +} SUniqueItem; + +typedef struct SUniqueInfo { + int32_t numOfPoints; + uint8_t colType; + uint16_t colBytes; + bool hasNull; // null is not hashable, handle separately + SHashObj* pHash; + char pItems[]; +} SUniqueInfo; + +typedef struct SModeItem { + int64_t count; + STuplePos dataPos; + STuplePos tuplePos; +} SModeItem; + +typedef struct SModeInfo { + uint8_t colType; + uint16_t colBytes; + SHashObj* pHash; + + STuplePos nullTuplePos; + bool nullTupleSaved; + + char* buf; // serialize data buffer +} SModeInfo; + +typedef struct SDerivInfo { + double prevValue; // previous value + TSKEY prevTs; // previous timestamp + bool ignoreNegative; // ignore the negative value + int64_t tsWindow; // time window for derivative + bool valueSet; // the value has been set already +} SDerivInfo; + +typedef struct SRateInfo { + double firstValue; + TSKEY firstKey; + double lastValue; + TSKEY lastKey; + int8_t hasResult; // flag to denote has value + + char* firstPk; + char* lastPk; + int8_t pkType; + int32_t pkBytes; + char pkData[]; +} SRateInfo; + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_FUNCTIONRESINFOINT_H diff --git a/source/libs/function/inc/thistogram.h b/source/libs/function/inc/thistogram.h index 08bff7117e..26681d0426 100644 --- a/source/libs/function/inc/thistogram.h +++ b/source/libs/function/inc/thistogram.h @@ -16,6 +16,8 @@ #ifndef TDENGINE_HISTOGRAM_H #define TDENGINE_HISTOGRAM_H +#include "functionResInfoInt.h" + #ifdef __cplusplus extern "C" { #endif @@ -24,51 +26,28 @@ extern "C" { #define MAX_HISTOGRAM_BIN 500 -typedef struct SHistBin { - double val; - int64_t num; - -#if !defined(USE_ARRAYLIST) - double delta; - int32_t index; // index in min-heap list -#endif -} SHistBin; - typedef struct SHeapEntry { void* pData; double val; } SHeapEntry; -typedef struct SHistogramInfo { - int64_t numOfElems; - int32_t numOfEntries; - int32_t maxEntries; - double min; - double max; -#if defined(USE_ARRAYLIST) - SHistBin* elems; -#else - tSkipList* pList; - SMultiwayMergeTreeInfo* pLoserTree; - int32_t maxIndex; - bool ordered; -#endif -} SHistogramInfo; +struct SHistogramInfo; +struct SHistBin; -int32_t tHistogramCreate(int32_t numOfEntries, SHistogramInfo** pHisto); -SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins); +int32_t tHistogramCreate(int32_t numOfEntries, struct SHistogramInfo** pHisto); +struct SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins); -int32_t tHistogramAdd(SHistogramInfo** pHisto, double val); -int32_t tHistogramSum(SHistogramInfo* pHisto, double v, int64_t *res); +int32_t tHistogramAdd(struct SHistogramInfo** pHisto, double val); +int32_t tHistogramSum(struct SHistogramInfo* pHisto, double v, int64_t *res); -int32_t tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num, double** pVal); -int32_t tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries, - SHistogramInfo** pResHistogram); -void tHistogramDestroy(SHistogramInfo** pHisto); +int32_t tHistogramUniform(struct SHistogramInfo* pHisto, double* ratio, int32_t num, double** pVal); +int32_t tHistogramMerge(struct SHistogramInfo* pHisto1, struct SHistogramInfo* pHisto2, int32_t numOfEntries, + struct SHistogramInfo** pResHistogram); +void tHistogramDestroy(struct SHistogramInfo** pHisto); -void tHistogramPrint(SHistogramInfo* pHisto); +void tHistogramPrint(struct SHistogramInfo* pHisto); -int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val); +int32_t histoBinarySearch(struct SHistBin* pEntry, int32_t len, double val); SHeapEntry* tHeapCreate(int32_t numOfEntries); void tHeapSort(SHeapEntry* pEntry, int32_t len); diff --git a/source/libs/function/inc/tpercentile.h b/source/libs/function/inc/tpercentile.h index 09df42d3a3..35067fa3ea 100644 --- a/source/libs/function/inc/tpercentile.h +++ b/source/libs/function/inc/tpercentile.h @@ -21,59 +21,18 @@ extern "C" { #endif #include "tpagedbuf.h" - -typedef struct MinMaxEntry { - union { - double dMinVal; - // double i64MinVal; - uint64_t u64MinVal; - }; - union { - double dMaxVal; - // double i64MaxVal; - int64_t u64MaxVal; - }; -} MinMaxEntry; - -typedef struct { - int32_t size; - int32_t pageId; - SFilePage *data; -} SSlotInfo; - -typedef struct tMemBucketSlot { - SSlotInfo info; - MinMaxEntry range; -} tMemBucketSlot; +#include "functionResInfoInt.h" struct tMemBucket; -typedef int32_t (*__perc_hash_func_t)(struct tMemBucket *pBucket, const void *value, int32_t *index); - -typedef struct tMemBucket { - int16_t numOfSlots; - int16_t type; - int32_t bytes; - int32_t total; - int32_t elemPerPage; // number of elements for each object - int32_t maxCapacity; // maximum allowed number of elements that can be sort directly to get the result - int32_t bufPageSize; // disk page size - MinMaxEntry range; // value range - int32_t times; // count that has been checked for deciding the correct data value buckets. - __compar_fn_t comparFn; - tMemBucketSlot *pSlots; - SDiskbasedBuf *pBuffer; - __perc_hash_func_t hashFunc; - SHashObj *groupPagesMap; // disk page map for different groups; -} tMemBucket; int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval, bool hasWindowOrGroup, - tMemBucket **pBucket); + struct tMemBucket **pBucket); -void tMemBucketDestroy(tMemBucket **pBucket); +void tMemBucketDestroy(struct tMemBucket **pBucket); -int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size); +int32_t tMemBucketPut(struct tMemBucket *pBucket, const void *data, size_t size); -int32_t getPercentile(tMemBucket *pMemBucket, double percent, double *result); +int32_t getPercentile(struct tMemBucket *pMemBucket, double percent, double *result); #endif // TDENGINE_TPERCENTILE_H diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 111daa7c8f..8b06a883fd 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -16,9 +16,9 @@ #include "builtins.h" #include "builtinsimpl.h" #include "cJSON.h" +#include "geomFunc.h" #include "querynodes.h" #include "scalar.h" -#include "geomFunc.h" #include "taoserror.h" #include "ttime.h" @@ -208,10 +208,10 @@ static int32_t countTrailingSpaces(const SValueNode* pVal, bool isLtrim) { } static int32_t addTimezoneParam(SNodeList* pList) { - char buf[6] = {0}; + char buf[TD_TIME_STR_LEN] = {0}; time_t t = taosTime(NULL); struct tm tmInfo; - if (taosLocalTime(&t, &tmInfo, buf) != NULL) { + if (taosLocalTime(&t, &tmInfo, buf, sizeof(buf)) != NULL) { (void)strftime(buf, sizeof(buf), "%z", &tmInfo); } int32_t len = (int32_t)strlen(buf); @@ -237,7 +237,7 @@ static int32_t addTimezoneParam(SNodeList* pList) { return terrno; } varDataSetLen(pVal->datum.p, len); - (void)strncpy(varDataVal(pVal->datum.p), pVal->literal, len); + tstrncpy(varDataVal(pVal->datum.p), pVal->literal, len + 1); code = nodesListAppend(pList, (SNode*)pVal); if (TSDB_CODE_SUCCESS != code) { @@ -327,7 +327,6 @@ static int32_t translateMinMax(SFunctionNode* pFunc, char* pErrBuf, int32_t len) } else if (IS_NULL_TYPE(paraType)) { paraType = TSDB_DATA_TYPE_BIGINT; } - pFunc->hasSMA = !IS_VAR_DATA_TYPE(paraType); int32_t bytes = IS_STR_DATA_TYPE(paraType) ? dataType->bytes : tDataTypes[paraType].bytes; pFunc->node.resType = (SDataType){.bytes = bytes, .type = paraType}; return TSDB_CODE_SUCCESS; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 157d44b3de..9f50e705ca 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -16,6 +16,7 @@ #include "builtinsimpl.h" #include "cJSON.h" #include "function.h" +#include "functionResInfoInt.h" #include "query.h" #include "querynodes.h" #include "tcompare.h" @@ -26,82 +27,12 @@ #include "thistogram.h" #include "tpercentile.h" -#define HISTOGRAM_MAX_BINS_NUM 1000 -#define MAVG_MAX_POINTS_NUM 1000 -#define TAIL_MAX_POINTS_NUM 100 -#define TAIL_MAX_OFFSET 100 - -#define HLL_BUCKET_BITS 14 // The bits of the bucket -#define HLL_DATA_BITS (64 - HLL_BUCKET_BITS) -#define HLL_BUCKETS (1 << HLL_BUCKET_BITS) -#define HLL_BUCKET_MASK (HLL_BUCKETS - 1) -#define HLL_ALPHA_INF 0.721347520444481703680 // constant for 0.5/ln(2) - -// typedef struct SMinmaxResInfo { -// bool assign; // assign the first value or not -// int64_t v; -// STuplePos tuplePos; -// -// STuplePos nullTuplePos; -// bool nullTupleSaved; -// int16_t type; -// } SMinmaxResInfo; - -typedef struct STopBotResItem { - SVariant v; - uint64_t uid; // it is a table uid, used to extract tag data during building of the final result for the tag data - STuplePos tuplePos; // tuple data of this chosen row -} STopBotResItem; - -typedef struct STopBotRes { - int32_t maxSize; - int16_t type; - - STuplePos nullTuplePos; - bool nullTupleSaved; - - STopBotResItem* pItems; -} STopBotRes; - -typedef struct SStdRes { - double result; - int64_t count; - union { - double quadraticDSum; - int64_t quadraticISum; - uint64_t quadraticUSum; - }; - union { - double dsum; - int64_t isum; - uint64_t usum; - }; - int16_t type; -} SStdRes; - -typedef struct SLeastSQRInfo { - double matrix[2][3]; - double startVal; - double stepVal; - int64_t num; -} SLeastSQRInfo; - -typedef struct SPercentileInfo { - double result; - tMemBucket* pMemBucket; - int32_t stage; - double minval; - double maxval; - int64_t numOfElems; -} SPercentileInfo; - -typedef struct SAPercentileInfo { - double result; - double percent; - int8_t algo; - SHistogramInfo* pHisto; - TDigest* pTDigest; -} SAPercentileInfo; +bool ignoreNegative(int8_t ignoreOption){ + return (ignoreOption & 0x1) == 0x1; +} +bool ignoreNull(int8_t ignoreOption){ + return (ignoreOption & 0x2) == 0x2; +} typedef enum { APERCT_ALGO_UNKNOWN = 0, @@ -109,76 +40,8 @@ typedef enum { APERCT_ALGO_TDIGEST, } EAPerctAlgoType; -typedef struct SDiffInfo { - bool hasPrev; - bool isFirstRow; - int8_t ignoreOption; // replace the ignore with case when - union { - int64_t i64; - double d64; - } prev; - - int64_t prevTs; -} SDiffInfo; - -bool ignoreNegative(int8_t ignoreOption){ - return (ignoreOption & 0x1) == 0x1; -} -bool ignoreNull(int8_t ignoreOption){ - return (ignoreOption & 0x2) == 0x2; -} -typedef struct SSpreadInfo { - double result; - bool hasResult; - double min; - double max; -} SSpreadInfo; - -typedef struct SElapsedInfo { - double result; - TSKEY min; - TSKEY max; - int64_t timeUnit; -} SElapsedInfo; - -typedef struct STwaInfo { - double dOutput; - int64_t numOfElems; - SPoint1 p; - STimeWindow win; -} STwaInfo; - -typedef struct SHistoFuncBin { - double lower; - double upper; - int64_t count; - double percentage; -} SHistoFuncBin; - -typedef struct SHistoFuncInfo { - int32_t numOfBins; - int32_t totalCount; - bool normalized; - SHistoFuncBin bins[]; -} SHistoFuncInfo; - typedef enum { UNKNOWN_BIN = 0, USER_INPUT_BIN, LINEAR_BIN, LOG_BIN } EHistoBinType; -typedef struct SHLLFuncInfo { - uint64_t result; - uint64_t totalCount; - uint8_t buckets[HLL_BUCKETS]; -} SHLLInfo; - -typedef struct SStateInfo { - union { - int64_t count; - int64_t durationStart; - }; - int64_t prevTs; - bool isPrevTsSet; -} SStateInfo; - typedef enum { STATE_OPER_INVALID = 0, STATE_OPER_LT, @@ -189,105 +52,6 @@ typedef enum { STATE_OPER_EQ, } EStateOperType; -typedef struct SMavgInfo { - int32_t pos; - double sum; - int64_t prevTs; - bool isPrevTsSet; - int32_t numOfPoints; - bool pointsMeet; - double points[]; -} SMavgInfo; - -typedef struct SSampleInfo { - int32_t samples; - int32_t totalPoints; - int32_t numSampled; - uint8_t colType; - uint16_t colBytes; - - STuplePos nullTuplePos; - bool nullTupleSaved; - - char* data; - STuplePos* tuplePos; -} SSampleInfo; - -typedef struct STailItem { - int64_t timestamp; - bool isNull; - char data[]; -} STailItem; - -typedef struct STailInfo { - int32_t numOfPoints; - int32_t numAdded; - int32_t offset; - uint8_t colType; - uint16_t colBytes; - STailItem** pItems; -} STailInfo; - -typedef struct SUniqueItem { - int64_t timestamp; - bool isNull; - char data[]; -} SUniqueItem; - -typedef struct SUniqueInfo { - int32_t numOfPoints; - uint8_t colType; - uint16_t colBytes; - bool hasNull; // null is not hashable, handle separately - SHashObj* pHash; - char pItems[]; -} SUniqueInfo; - -typedef struct SModeItem { - int64_t count; - STuplePos dataPos; - STuplePos tuplePos; -} SModeItem; - -typedef struct SModeInfo { - uint8_t colType; - uint16_t colBytes; - SHashObj* pHash; - - STuplePos nullTuplePos; - bool nullTupleSaved; - - char* buf; // serialize data buffer -} SModeInfo; - -typedef struct SDerivInfo { - double prevValue; // previous value - TSKEY prevTs; // previous timestamp - bool ignoreNegative; // ignore the negative value - int64_t tsWindow; // time window for derivative - bool valueSet; // the value has been set already -} SDerivInfo; - -typedef struct SRateInfo { - double firstValue; - TSKEY firstKey; - double lastValue; - TSKEY lastKey; - int8_t hasResult; // flag to denote has value - - char* firstPk; - char* lastPk; - int8_t pkType; - int32_t pkBytes; - char pkData[]; -} SRateInfo; - -typedef struct SGroupKeyInfo { - bool hasResult; - bool isNull; - char data[]; -} SGroupKeyInfo; - #define SET_VAL(_info, numOfElem, res) \ do { \ if ((numOfElem) <= 0) { \ @@ -1948,11 +1712,11 @@ int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { char buf[LEASTSQUARES_BUFF_LENGTH] = {0}; char slopBuf[64] = {0}; char interceptBuf[64] = {0}; - int n = snprintf(slopBuf, 64, "%.6lf", param02); + int n = tsnprintf(slopBuf, 64, "%.6lf", param02); if (n > LEASTSQUARES_DOUBLE_ITEM_LENGTH) { (void)snprintf(slopBuf, 64, "%." DOUBLE_PRECISION_DIGITS, param02); } - n = snprintf(interceptBuf, 64, "%.6lf", param12); + n = tsnprintf(interceptBuf, 64, "%.6lf", param12); if (n > LEASTSQUARES_DOUBLE_ITEM_LENGTH) { (void)snprintf(interceptBuf, 64, "%." DOUBLE_PRECISION_DIGITS, param12); } @@ -2144,16 +1908,16 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } if (i == pCtx->numOfParams - 1) { - len += snprintf(varDataVal(buf) + len, sizeof(buf) - VARSTR_HEADER_SIZE - len, "%.6lf]", ppInfo->result); + len += tsnprintf(varDataVal(buf) + len, sizeof(buf) - VARSTR_HEADER_SIZE - len, "%.6lf]", ppInfo->result); } else { - len += snprintf(varDataVal(buf) + len, sizeof(buf) - VARSTR_HEADER_SIZE - len, "%.6lf, ", ppInfo->result); + len += tsnprintf(varDataVal(buf) + len, sizeof(buf) - VARSTR_HEADER_SIZE - len, "%.6lf, ", ppInfo->result); } } int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); if (NULL == pCol) { - code = TSDB_CODE_OUT_OF_RANGE; + code = terrno; goto _fin_error; } @@ -3185,7 +2949,8 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } else { code = colDataSetVal(pCol, pBlock->info.rows, res, false); if (TSDB_CODE_SUCCESS != code) { - return TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(res); + return code; } code = setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); } @@ -3675,7 +3440,7 @@ int32_t diffFunctionByRow(SArray* pCtxArray) { for (int i = 0; i < diffColNum; ++i) { SqlFunctionCtx* pCtx = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, i); if (NULL == pCtx) { - code = TSDB_CODE_OUT_OF_RANGE; + code = terrno; goto _exit; } funcInputUpdate(pCtx); @@ -3689,7 +3454,7 @@ int32_t diffFunctionByRow(SArray* pCtxArray) { SqlFunctionCtx* pCtx0 = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, 0); SFuncInputRow* pRow0 = (SFuncInputRow*)taosArrayGet(pRows, 0); if (NULL == pCtx0 || NULL == pRow0) { - code = TSDB_CODE_OUT_OF_RANGE; + code = terrno; goto _exit; } int32_t startOffset = pCtx0->offset; @@ -3707,7 +3472,7 @@ int32_t diffFunctionByRow(SArray* pCtxArray) { SqlFunctionCtx* pCtx = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, i); SFuncInputRow* pRow = (SFuncInputRow*)taosArrayGet(pRows, i); if (NULL == pCtx || NULL == pRow) { - code = TSDB_CODE_OUT_OF_RANGE; + code = terrno; goto _exit; } code = funcInputGetNextRow(pCtx, pRow, &result); @@ -3730,7 +3495,7 @@ int32_t diffFunctionByRow(SArray* pCtxArray) { SqlFunctionCtx* pCtx = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, i); SFuncInputRow* pRow = (SFuncInputRow*)taosArrayGet(pRows, i); if (NULL == pCtx || NULL == pRow) { - code = TSDB_CODE_OUT_OF_RANGE; + code = terrno; goto _exit; } if ((keepNull || hasNotNullValue) && !isFirstRow(pCtx, pRow)){ @@ -3752,7 +3517,7 @@ int32_t diffFunctionByRow(SArray* pCtxArray) { for (int i = 0; i < diffColNum; ++i) { SqlFunctionCtx* pCtx = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, i); if (NULL == pCtx) { - code = TSDB_CODE_OUT_OF_RANGE; + code = terrno; goto _exit; } SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); @@ -4429,7 +4194,7 @@ int32_t spreadPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t code = TSDB_CODE_SUCCESS; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); if (NULL == pCol) { - code = TSDB_CODE_OUT_OF_RANGE; + code = terrno; goto _exit; } @@ -4619,7 +4384,7 @@ int32_t elapsedPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t code = TSDB_CODE_SUCCESS; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); if (NULL == pCol) { - code = TSDB_CODE_OUT_OF_RANGE; + code = terrno; goto _exit; } @@ -4969,10 +4734,10 @@ int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t len; char buf[512] = {0}; if (!pInfo->normalized) { - len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%" PRId64 "}", + len = snprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%" PRId64 "}", pInfo->bins[i].lower, pInfo->bins[i].upper, pInfo->bins[i].count); } else { - len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}", pInfo->bins[i].lower, + len = snprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}", pInfo->bins[i].lower, pInfo->bins[i].upper, pInfo->bins[i].percentage); } varDataSetLen(buf, len); @@ -5002,7 +4767,7 @@ int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t code = TSDB_CODE_SUCCESS; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); if (NULL == pCol) { - code = TSDB_CODE_OUT_OF_RANGE; + code = terrno; goto _exit; } code = colDataSetVal(pCol, pBlock->info.rows, res, false); @@ -5235,7 +5000,7 @@ int32_t hllPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t code = TSDB_CODE_SUCCESS; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); if (NULL == pCol) { - code = TSDB_CODE_OUT_OF_RANGE; + code = terrno; goto _exit; } @@ -6600,7 +6365,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { compRatio = pData->totalSize * 100 / (double)totalRawSize; } - int32_t len = sprintf(st + VARSTR_HEADER_SIZE, + int32_t len = snprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Total_Blocks=[%d] Total_Size=[%.2f KiB] Average_size=[%.2f KiB] Compression_Ratio=[%.2f %c]", pData->numOfBlocks, pData->totalSize / 1024.0, averageSize / 1024.0, compRatio, '%'); @@ -6615,7 +6380,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { avgRows = pData->totalRows / pData->numOfBlocks; } - len = sprintf(st + VARSTR_HEADER_SIZE, "Block_Rows=[%" PRId64 "] MinRows=[%d] MaxRows=[%d] AvgRows=[%" PRId64 "]", + len = snprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Block_Rows=[%" PRId64 "] MinRows=[%d] MaxRows=[%d] AvgRows=[%" PRId64 "]", pData->totalRows, pData->minRows, pData->maxRows, avgRows); varDataSetLen(st, len); code = colDataSetVal(pColInfo, row++, st, false); @@ -6623,14 +6388,14 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return code; } - len = sprintf(st + VARSTR_HEADER_SIZE, "Inmem_Rows=[%d] Stt_Rows=[%d] ", pData->numOfInmemRows, pData->numOfSttRows); + len = snprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Inmem_Rows=[%d] Stt_Rows=[%d] ", pData->numOfInmemRows, pData->numOfSttRows); varDataSetLen(st, len); code = colDataSetVal(pColInfo, row++, st, false); if (TSDB_CODE_SUCCESS != code) { return code; } - len = sprintf(st + VARSTR_HEADER_SIZE, "Total_Tables=[%d] Total_Filesets=[%d] Total_Vgroups=[%d]", pData->numOfTables, + len = snprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Total_Tables=[%d] Total_Filesets=[%d] Total_Vgroups=[%d]", pData->numOfTables, pData->numOfFiles, pData->numOfVgroups); varDataSetLen(st, len); @@ -6639,7 +6404,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return code; } - len = sprintf(st + VARSTR_HEADER_SIZE, + len = snprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "--------------------------------------------------------------------------------"); varDataSetLen(st, len); code = colDataSetVal(pColInfo, row++, st, false); @@ -6666,7 +6431,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t bucketRange = ceil(((double) (pData->defMaxRows - pData->defMinRows)) / numOfBuckets); for (int32_t i = 0; i < tListLen(pData->blockRowsHisto); ++i) { - len = sprintf(st + VARSTR_HEADER_SIZE, "%04d |", pData->defMinRows + bucketRange * (i + 1)); + len = snprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "%04d |", pData->defMinRows + bucketRange * (i + 1)); int32_t num = 0; if (pData->blockRowsHisto[i] > 0) { @@ -6674,13 +6439,13 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } for (int32_t j = 0; j < num; ++j) { - int32_t x = sprintf(st + VARSTR_HEADER_SIZE + len, "%c", '|'); + int32_t x = snprintf(varDataVal(st) + len, sizeof(st) - VARSTR_HEADER_SIZE - len, "%c", '|'); len += x; } if (pData->blockRowsHisto[i] > 0) { double v = pData->blockRowsHisto[i] * 100.0 / pData->numOfBlocks; - len += sprintf(st + VARSTR_HEADER_SIZE + len, " %d (%.2f%c)", pData->blockRowsHisto[i], v, '%'); + len += snprintf(varDataVal(st) + len, sizeof(st) - VARSTR_HEADER_SIZE - len, " %d (%.2f%c)", pData->blockRowsHisto[i], v, '%'); } varDataSetLen(st, len); diff --git a/source/libs/function/src/detail/tavgfunction.c b/source/libs/function/src/detail/tavgfunction.c index 854260c354..7313fc82f7 100644 --- a/source/libs/function/src/detail/tavgfunction.c +++ b/source/libs/function/src/detail/tavgfunction.c @@ -92,261 +92,6 @@ out->sum.usum += val; \ } -typedef struct SAvgRes { - double result; - SSumRes sum; - int64_t count; - int16_t type; // store the original input type, used in merge function -} SAvgRes; - -static void floatVectorSumAVX(const float* plist, int32_t numOfRows, SAvgRes* pRes) { - const int32_t bitWidth = 256; - -#if __AVX__ - // find the start position that are aligned to 32bytes address in memory - int32_t width = (bitWidth>>3u) / sizeof(float); - - int32_t remainder = numOfRows % width; - int32_t rounds = numOfRows / width; - - const float* p = plist; - - __m256 val; - __m256 sum = _mm256_setzero_ps(); - - for (int32_t i = 0; i < rounds; ++i) { - val = _mm256_loadu_ps(p); - sum = _mm256_add_ps(sum, val); - p += width; - } - - // let sum up the final results - const float* q = (const float*)∑ - pRes->sum.dsum += q[0] + q[1] + q[2] + q[3] + q[4] + q[5] + q[6] + q[7]; - - int32_t startIndex = rounds * width; - for (int32_t j = 0; j < remainder; ++j) { - pRes->sum.dsum += plist[j + startIndex]; - } -#endif -} - -static void doubleVectorSumAVX(const double* plist, int32_t numOfRows, SAvgRes* pRes) { - const int32_t bitWidth = 256; - -#if __AVX__ - // find the start position that are aligned to 32bytes address in memory - int32_t width = (bitWidth>>3u) / sizeof(int64_t); - - int32_t remainder = numOfRows % width; - int32_t rounds = numOfRows / width; - - const double* p = plist; - - __m256d val; - __m256d sum = _mm256_setzero_pd(); - - for (int32_t i = 0; i < rounds; ++i) { - val = _mm256_loadu_pd(p); - sum = _mm256_add_pd(sum, val); - p += width; - } - - // let sum up the final results - const double* q = (const double*)∑ - pRes->sum.dsum += q[0] + q[1] + q[2] + q[3]; - - int32_t startIndex = rounds * width; - for (int32_t j = 0; j < remainder; ++j) { - pRes->sum.dsum += plist[j + startIndex]; - } -#endif -} - -static void i8VectorSumAVX2(const int8_t* plist, int32_t numOfRows, int32_t type, SAvgRes* pRes) { - const int32_t bitWidth = 256; - -#if __AVX2__ - // find the start position that are aligned to 32bytes address in memory - int32_t width = (bitWidth>>3u) / sizeof(int64_t); - - int32_t remainder = numOfRows % width; - int32_t rounds = numOfRows / width; - - __m256i sum = _mm256_setzero_si256(); - - if (type == TSDB_DATA_TYPE_TINYINT) { - const int8_t* p = plist; - - for (int32_t i = 0; i < rounds; ++i) { - __m128i val = _mm_lddqu_si128((__m128i*)p); - __m256i extVal = _mm256_cvtepi8_epi64(val); // only four items will be converted into __m256i - sum = _mm256_add_epi64(sum, extVal); - p += width; - } - - // let sum up the final results - const int64_t* q = (const int64_t*)∑ - pRes->sum.isum += q[0] + q[1] + q[2] + q[3]; - - for (int32_t j = 0; j < remainder; ++j) { - pRes->sum.isum += plist[j + rounds * width]; - } - } else { - const uint8_t* p = (const uint8_t*)plist; - - for(int32_t i = 0; i < rounds; ++i) { - __m128i val = _mm_lddqu_si128((__m128i*)p); - __m256i extVal = _mm256_cvtepu8_epi64(val); // only four items will be converted into __m256i - sum = _mm256_add_epi64(sum, extVal); - p += width; - } - - // let sum up the final results - const uint64_t* q = (const uint64_t*)∑ - pRes->sum.usum += q[0] + q[1] + q[2] + q[3]; - - for (int32_t j = 0; j < remainder; ++j) { - pRes->sum.usum += (uint8_t)plist[j + rounds * width]; - } - } - -#endif -} - -static void i16VectorSumAVX2(const int16_t* plist, int32_t numOfRows, int32_t type, SAvgRes* pRes) { - const int32_t bitWidth = 256; - -#if __AVX2__ - // find the start position that are aligned to 32bytes address in memory - int32_t width = (bitWidth>>3u) / sizeof(int64_t); - - int32_t remainder = numOfRows % width; - int32_t rounds = numOfRows / width; - - __m256i sum = _mm256_setzero_si256(); - - if (type == TSDB_DATA_TYPE_SMALLINT) { - const int16_t* p = plist; - - for (int32_t i = 0; i < rounds; ++i) { - __m128i val = _mm_lddqu_si128((__m128i*)p); - __m256i extVal = _mm256_cvtepi16_epi64(val); // only four items will be converted into __m256i - sum = _mm256_add_epi64(sum, extVal); - p += width; - } - - // let sum up the final results - const int64_t* q = (const int64_t*)∑ - pRes->sum.isum += q[0] + q[1] + q[2] + q[3]; - - for (int32_t j = 0; j < remainder; ++j) { - pRes->sum.isum += plist[j + rounds * width]; - } - } else { - const uint16_t* p = (const uint16_t*)plist; - - for(int32_t i = 0; i < rounds; ++i) { - __m128i val = _mm_lddqu_si128((__m128i*)p); - __m256i extVal = _mm256_cvtepu16_epi64(val); // only four items will be converted into __m256i - sum = _mm256_add_epi64(sum, extVal); - p += width; - } - - // let sum up the final results - const uint64_t* q = (const uint64_t*)∑ - pRes->sum.usum += q[0] + q[1] + q[2] + q[3]; - - for (int32_t j = 0; j < remainder; ++j) { - pRes->sum.usum += (uint16_t)plist[j + rounds * width]; - } - } - -#endif -} - -static void i32VectorSumAVX2(const int32_t* plist, int32_t numOfRows, int32_t type, SAvgRes* pRes) { - const int32_t bitWidth = 256; - -#if __AVX2__ - // find the start position that are aligned to 32bytes address in memory - int32_t width = (bitWidth>>3u) / sizeof(int64_t); - - int32_t remainder = numOfRows % width; - int32_t rounds = numOfRows / width; - - __m256i sum = _mm256_setzero_si256(); - - if (type == TSDB_DATA_TYPE_INT) { - const int32_t* p = plist; - - for (int32_t i = 0; i < rounds; ++i) { - __m128i val = _mm_lddqu_si128((__m128i*)p); - __m256i extVal = _mm256_cvtepi32_epi64(val); // only four items will be converted into __m256i - sum = _mm256_add_epi64(sum, extVal); - p += width; - } - - // let sum up the final results - const int64_t* q = (const int64_t*)∑ - pRes->sum.isum += q[0] + q[1] + q[2] + q[3]; - - for (int32_t j = 0; j < remainder; ++j) { - pRes->sum.isum += plist[j + rounds * width]; - } - } else { - const uint32_t* p = (const uint32_t*)plist; - - for(int32_t i = 0; i < rounds; ++i) { - __m128i val = _mm_lddqu_si128((__m128i*)p); - __m256i extVal = _mm256_cvtepu32_epi64(val); // only four items will be converted into __m256i - sum = _mm256_add_epi64(sum, extVal); - p += width; - } - - // let sum up the final results - const uint64_t* q = (const uint64_t*)∑ - pRes->sum.usum += q[0] + q[1] + q[2] + q[3]; - - for (int32_t j = 0; j < remainder; ++j) { - pRes->sum.usum += (uint32_t)plist[j + rounds * width]; - } - } - -#endif -} - -static void i64VectorSumAVX2(const int64_t* plist, int32_t numOfRows, SAvgRes* pRes) { - const int32_t bitWidth = 256; - -#if __AVX2__ - // find the start position that are aligned to 32bytes address in memory - int32_t width = (bitWidth >> 3u) / sizeof(int64_t); - - int32_t remainder = numOfRows % width; - int32_t rounds = numOfRows / width; - - __m256i sum = _mm256_setzero_si256(); - - const int64_t* p = plist; - - for (int32_t i = 0; i < rounds; ++i) { - __m256i val = _mm256_lddqu_si256((__m256i*)p); - sum = _mm256_add_epi64(sum, val); - p += width; - } - - // let sum up the final results - const int64_t* q = (const int64_t*)∑ - pRes->sum.isum += q[0] + q[1] + q[2] + q[3]; - - for (int32_t j = 0; j < remainder; ++j) { - pRes->sum.isum += plist[j + rounds * width]; - } - -#endif -} - int32_t getAvgInfoSize() { return (int32_t)sizeof(SAvgRes); } bool getAvgFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { @@ -568,23 +313,16 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { numOfElem = pInput->numOfRows; pAvgRes->count += pInput->numOfRows; - bool simdAvailable = tsAVXSupported && tsSIMDEnable && (numOfRows > THRESHOLD_SIZE); - switch(type) { case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_TINYINT: { const int8_t* plist = (const int8_t*) pCol->pData; - // 1. If the CPU supports AVX, let's employ AVX instructions to speedup this loop - if (simdAvailable) { - i8VectorSumAVX2(plist, numOfRows, type, pAvgRes); - } else { - for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - if (type == TSDB_DATA_TYPE_TINYINT) { - CHECK_OVERFLOW_SUM_SIGNED(pAvgRes, plist[i]) - } else { - CHECK_OVERFLOW_SUM_UNSIGNED(pAvgRes, (uint8_t)plist[i]) - } + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + if (type == TSDB_DATA_TYPE_TINYINT) { + CHECK_OVERFLOW_SUM_SIGNED(pAvgRes, plist[i]) + } else { + CHECK_OVERFLOW_SUM_UNSIGNED(pAvgRes, (uint8_t)plist[i]) } } break; @@ -594,16 +332,11 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { case TSDB_DATA_TYPE_SMALLINT: { const int16_t* plist = (const int16_t*)pCol->pData; - // 1. If the CPU supports AVX, let's employ AVX instructions to speedup this loop - if (simdAvailable) { - i16VectorSumAVX2(plist, numOfRows, type, pAvgRes); - } else { - for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - if (type == TSDB_DATA_TYPE_SMALLINT) { - CHECK_OVERFLOW_SUM_SIGNED(pAvgRes, plist[i]) - } else { - CHECK_OVERFLOW_SUM_UNSIGNED(pAvgRes, (uint16_t)plist[i]) - } + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + if (type == TSDB_DATA_TYPE_SMALLINT) { + CHECK_OVERFLOW_SUM_SIGNED(pAvgRes, plist[i]) + } else { + CHECK_OVERFLOW_SUM_UNSIGNED(pAvgRes, (uint16_t)plist[i]) } } break; @@ -613,16 +346,11 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { case TSDB_DATA_TYPE_INT: { const int32_t* plist = (const int32_t*) pCol->pData; - // 1. If the CPU supports AVX, let's employ AVX instructions to speedup this loop - if (simdAvailable) { - i32VectorSumAVX2(plist, numOfRows, type, pAvgRes); - } else { - for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - if (type == TSDB_DATA_TYPE_INT) { - CHECK_OVERFLOW_SUM_SIGNED(pAvgRes, plist[i]) - } else { - CHECK_OVERFLOW_SUM_UNSIGNED(pAvgRes, (uint32_t)plist[i]) - } + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + if (type == TSDB_DATA_TYPE_INT) { + CHECK_OVERFLOW_SUM_SIGNED(pAvgRes, plist[i]) + } else { + CHECK_OVERFLOW_SUM_UNSIGNED(pAvgRes, (uint32_t)plist[i]) } } break; @@ -632,16 +360,11 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { case TSDB_DATA_TYPE_BIGINT: { const int64_t* plist = (const int64_t*) pCol->pData; - // 1. If the CPU supports AVX, let's employ AVX instructions to speedup this loop - if (simdAvailable && type == TSDB_DATA_TYPE_BIGINT) { - i64VectorSumAVX2(plist, numOfRows, pAvgRes); - } else { - for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - if (type == TSDB_DATA_TYPE_BIGINT) { - CHECK_OVERFLOW_SUM_SIGNED(pAvgRes, plist[i]) - } else { - CHECK_OVERFLOW_SUM_UNSIGNED(pAvgRes, (uint64_t)plist[i]) - } + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + if (type == TSDB_DATA_TYPE_BIGINT) { + CHECK_OVERFLOW_SUM_SIGNED(pAvgRes, plist[i]) + } else { + CHECK_OVERFLOW_SUM_UNSIGNED(pAvgRes, (uint64_t)plist[i]) } } break; @@ -650,26 +373,16 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { case TSDB_DATA_TYPE_FLOAT: { const float* plist = (const float*) pCol->pData; - // 1. If the CPU supports AVX, let's employ AVX instructions to speedup this loop - if (simdAvailable) { - floatVectorSumAVX(plist, numOfRows, pAvgRes); - } else { - for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - pAvgRes->sum.dsum += plist[i]; - } + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + pAvgRes->sum.dsum += plist[i]; } break; } case TSDB_DATA_TYPE_DOUBLE: { const double* plist = (const double*)pCol->pData; - // 1. If the CPU supports AVX, let's employ AVX instructions to speedup this loop - if (simdAvailable) { - doubleVectorSumAVX(plist, numOfRows, pAvgRes); - } else { - for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - pAvgRes->sum.dsum += plist[i]; - } + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + pAvgRes->sum.dsum += plist[i]; } break; } diff --git a/source/libs/function/src/detail/tminmax.c b/source/libs/function/src/detail/tminmax.c index e3c12e9a57..69c1a8a6dd 100644 --- a/source/libs/function/src/detail/tminmax.c +++ b/source/libs/function/src/detail/tminmax.c @@ -72,6 +72,7 @@ #define GET_INVOKE_INTRINSIC_THRESHOLD(_bits, _bytes) ((_bits) / ((_bytes) << 3u)) +#ifdef __AVX2__ static void calculateRounds(int32_t numOfRows, int32_t bytes, int32_t* remainder, int32_t* rounds, int32_t* width) { const int32_t bitWidth = 256; @@ -81,224 +82,104 @@ static void calculateRounds(int32_t numOfRows, int32_t bytes, int32_t* remainder } #define EXTRACT_MAX_VAL(_first, _sec, _width, _remain, _v) \ - (_v) = TMAX((_first)[0], (_first)[1]); \ - for (int32_t k = 1; k < (_width); ++k) { \ - (_v) = TMAX((_v), (_first)[k]); \ - } \ - \ - for (int32_t j = 0; j < (_remain); ++j) { \ - if ((_v) < (_sec)[j]) { \ - (_v) = (_sec)[j]; \ - } \ - } + __COMPARE_EXTRACT_MAX(0, (_width), (_v), (_first)) \ + __COMPARE_EXTRACT_MAX(0, (_remain), (_v), (_sec)) #define EXTRACT_MIN_VAL(_first, _sec, _width, _remain, _v) \ - (_v) = TMIN((_first)[0], (_first)[1]); \ - for (int32_t k = 1; k < (_width); ++k) { \ - (_v) = TMIN((_v), (_first)[k]); \ - } \ - \ - for (int32_t j = 0; j < (_remain); ++j) { \ - if ((_v) > (_sec)[j]) { \ - (_v) = (_sec)[j]; \ - } \ - } + __COMPARE_EXTRACT_MIN(0, (_width), (_v), (_first)) \ + __COMPARE_EXTRACT_MIN(0, (_remain), (_v), (_sec)) -static int8_t i8VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFunc, bool signVal) { - int8_t v = 0; +#define CMP_TYPE_MIN_MAX(type, cmp) \ + const type* p = pData; \ + __m256i initVal = _mm256_lddqu_si256((__m256i*)p); \ + p += width; \ + for (int32_t i = 1; i < (rounds); ++i) { \ + __m256i next = _mm256_lddqu_si256((__m256i*)p); \ + initVal = CMP_FUNC_##cmp##_##type(initVal, next); \ + p += width; \ + } \ + const type* q = (const type*)&initVal; \ + type* v = (type*)res; \ + EXTRACT_##cmp##_VAL(q, p, width, remain, *v) + +static void i8VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFunc, bool signVal, int64_t* res) { const int8_t* p = pData; int32_t width, remain, rounds; calculateRounds(numOfRows, sizeof(int8_t), &remain, &rounds, &width); -#if __AVX2__ - __m256i next; - __m256i initVal = _mm256_lddqu_si256((__m256i*)p); - p += width; +#define CMP_FUNC_MIN_int8_t _mm256_min_epi8 +#define CMP_FUNC_MAX_int8_t _mm256_max_epi8 +#define CMP_FUNC_MIN_uint8_t _mm256_min_epu8 +#define CMP_FUNC_MAX_uint8_t _mm256_max_epu8 if (!isMinFunc) { // max function if (signVal) { - for (int32_t i = 0; i < rounds; ++i) { - next = _mm256_lddqu_si256((__m256i*)p); - initVal = _mm256_max_epi8(initVal, next); - p += width; - } - - const int8_t* q = (const int8_t*)&initVal; - EXTRACT_MAX_VAL(q, p, width, remain, v) - } else { // unsigned value - for (int32_t i = 0; i < rounds; ++i) { - next = _mm256_lddqu_si256((__m256i*)p); - initVal = _mm256_max_epu8(initVal, next); - p += width; - } - - const uint8_t* q = (const uint8_t*)&initVal; - EXTRACT_MAX_VAL(q, p, width, remain, v) + CMP_TYPE_MIN_MAX(int8_t, MAX); + } else { + CMP_TYPE_MIN_MAX(uint8_t, MAX); } - } else { // min function if (signVal) { - for (int32_t i = 0; i < rounds; ++i) { - next = _mm256_lddqu_si256((__m256i*)p); - initVal = _mm256_min_epi8(initVal, next); - p += width; - } - - // let sum up the final results - const int8_t* q = (const int8_t*)&initVal; - EXTRACT_MIN_VAL(q, p, width, remain, v) + CMP_TYPE_MIN_MAX(int8_t, MIN); } else { - for (int32_t i = 0; i < rounds; ++i) { - next = _mm256_lddqu_si256((__m256i*)p); - initVal = _mm256_min_epu8(initVal, next); - p += width; - } - - // let sum up the final results - const uint8_t* q = (const uint8_t*)&initVal; - EXTRACT_MIN_VAL(q, p, width, remain, v) + CMP_TYPE_MIN_MAX(uint8_t, MIN); } } -#endif - - return v; } -static int16_t i16VectorCmpAVX2(const int16_t* pData, int32_t numOfRows, bool isMinFunc, bool signVal) { - int16_t v = 0; - const int16_t* p = pData; - +static void i16VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFunc, bool signVal, int64_t* res) { int32_t width, remain, rounds; calculateRounds(numOfRows, sizeof(int16_t), &remain, &rounds, &width); -#if __AVX2__ - __m256i next; - __m256i initVal = _mm256_lddqu_si256((__m256i*)p); - p += width; - +#define CMP_FUNC_MIN_int16_t _mm256_min_epi16 +#define CMP_FUNC_MAX_int16_t _mm256_max_epi16 +#define CMP_FUNC_MIN_uint16_t _mm256_min_epu16 +#define CMP_FUNC_MAX_uint16_t _mm256_max_epu16 if (!isMinFunc) { // max function if (signVal) { - for (int32_t i = 0; i < rounds; ++i) { - next = _mm256_lddqu_si256((__m256i*)p); - initVal = _mm256_max_epi16(initVal, next); - p += width; - } - - // let sum up the final results - const int16_t* q = (const int16_t*)&initVal; - EXTRACT_MAX_VAL(q, p, width, remain, v) + CMP_TYPE_MIN_MAX(int16_t, MAX); } else { - for (int32_t i = 0; i < rounds; ++i) { - next = _mm256_lddqu_si256((__m256i*)p); - initVal = _mm256_max_epu16(initVal, next); - p += width; - } - - // let sum up the final results - const uint16_t* q = (const uint16_t*)&initVal; - EXTRACT_MAX_VAL(q, p, width, remain, v) + CMP_TYPE_MIN_MAX(uint16_t, MAX); } - } else { // min function if (signVal) { - for (int32_t i = 0; i < rounds; ++i) { - next = _mm256_lddqu_si256((__m256i*)p); - initVal = _mm256_min_epi16(initVal, next); - p += width; - } - - // let sum up the final results - const int16_t* q = (const int16_t*)&initVal; - EXTRACT_MIN_VAL(q, p, width, remain, v) + CMP_TYPE_MIN_MAX(int16_t, MIN); } else { - for (int32_t i = 0; i < rounds; ++i) { - next = _mm256_lddqu_si256((__m256i*)p); - initVal = _mm256_min_epi16(initVal, next); - p += width; - } - - // let sum up the final results - const uint16_t* q = (const uint16_t*)&initVal; - EXTRACT_MIN_VAL(q, p, width, remain, v) + CMP_TYPE_MIN_MAX(uint16_t, MIN); } } -#endif - - return v; } -static int32_t i32VectorCmpAVX2(const int32_t* pData, int32_t numOfRows, bool isMinFunc, bool signVal) { - int32_t v = 0; - const int32_t* p = pData; - +static void i32VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFunc, bool signVal, int64_t* res) { int32_t width, remain, rounds; calculateRounds(numOfRows, sizeof(int32_t), &remain, &rounds, &width); -#if __AVX2__ - __m256i next; - __m256i initVal = _mm256_lddqu_si256((__m256i*)p); - p += width; - +#define CMP_FUNC_MIN_int32_t _mm256_min_epi32 +#define CMP_FUNC_MAX_int32_t _mm256_max_epi32 +#define CMP_FUNC_MIN_uint32_t _mm256_min_epu32 +#define CMP_FUNC_MAX_uint32_t _mm256_max_epu32 if (!isMinFunc) { // max function if (signVal) { - for (int32_t i = 0; i < rounds; ++i) { - next = _mm256_lddqu_si256((__m256i*)p); - initVal = _mm256_max_epi32(initVal, next); - p += width; - } - - // let compare the final results - const int32_t* q = (const int32_t*)&initVal; - EXTRACT_MAX_VAL(q, p, width, remain, v) - } else { // unsigned value - for (int32_t i = 0; i < rounds; ++i) { - next = _mm256_lddqu_si256((__m256i*)p); - initVal = _mm256_max_epi32(initVal, next); - p += width; - } - - // let compare the final results - const uint32_t* q = (const uint32_t*)&initVal; - EXTRACT_MAX_VAL(q, p, width, remain, v) + CMP_TYPE_MIN_MAX(int32_t, MAX); + } else { + CMP_TYPE_MIN_MAX(uint32_t, MAX); } } else { // min function if (signVal) { - for (int32_t i = 0; i < rounds; ++i) { - next = _mm256_lddqu_si256((__m256i*)p); - initVal = _mm256_min_epi32(initVal, next); - p += width; - } - - // let sum up the final results - const int32_t* q = (const int32_t*)&initVal; - EXTRACT_MIN_VAL(q, p, width, remain, v) + CMP_TYPE_MIN_MAX(int32_t, MIN); } else { - for (int32_t i = 0; i < rounds; ++i) { - next = _mm256_lddqu_si256((__m256i*)p); - initVal = _mm256_min_epu32(initVal, next); - p += width; - } - - // let sum up the final results - const uint32_t* q = (const uint32_t*)&initVal; - EXTRACT_MIN_VAL(q, p, width, remain, v) + CMP_TYPE_MIN_MAX(uint32_t, MIN); } } -#endif - - return v; } -static float floatVectorCmpAVX(const float* pData, int32_t numOfRows, bool isMinFunc) { - float v = 0; +static void floatVectorCmpAVX2(const float* pData, int32_t numOfRows, bool isMinFunc, float* res) { const float* p = pData; int32_t width, remain, rounds; calculateRounds(numOfRows, sizeof(float), &remain, &rounds, &width); -#if __AVX__ - __m256 next; __m256 initVal = _mm256_loadu_ps(p); p += width; @@ -311,7 +192,7 @@ static float floatVectorCmpAVX(const float* pData, int32_t numOfRows, bool isMin } const float* q = (const float*)&initVal; - EXTRACT_MAX_VAL(q, p, width, remain, v) + EXTRACT_MAX_VAL(q, p, width, remain, *res) } else { // min function for (int32_t i = 1; i < rounds; ++i) { next = _mm256_loadu_ps(p); @@ -320,22 +201,16 @@ static float floatVectorCmpAVX(const float* pData, int32_t numOfRows, bool isMin } const float* q = (const float*)&initVal; - EXTRACT_MIN_VAL(q, p, width, remain, v) + EXTRACT_MIN_VAL(q, p, width, remain, *res) } -#endif - - return v; } -static double doubleVectorCmpAVX(const double* pData, int32_t numOfRows, bool isMinFunc) { - double v = 0; +static void doubleVectorCmpAVX2(const double* pData, int32_t numOfRows, bool isMinFunc, double* res) { const double* p = pData; int32_t width, remain, rounds; calculateRounds(numOfRows, sizeof(double), &remain, &rounds, &width); -#if __AVX__ - __m256d next; __m256d initVal = _mm256_loadu_pd(p); p += width; @@ -349,7 +224,7 @@ static double doubleVectorCmpAVX(const double* pData, int32_t numOfRows, bool is // let sum up the final results const double* q = (const double*)&initVal; - EXTRACT_MAX_VAL(q, p, width, remain, v) + EXTRACT_MAX_VAL(q, p, width, remain, *res) } else { // min function for (int32_t i = 1; i < rounds; ++i) { next = _mm256_loadu_pd(p); @@ -359,12 +234,10 @@ static double doubleVectorCmpAVX(const double* pData, int32_t numOfRows, bool is // let sum up the final results const double* q = (const double*)&initVal; - EXTRACT_MIN_VAL(q, p, width, remain, v) + EXTRACT_MIN_VAL(q, p, width, remain, *res) } -#endif - - return v; } +#endif static int32_t findFirstValPosition(const SColumnInfoData* pCol, int32_t start, int32_t numOfRows, bool isStr) { int32_t i = start; @@ -378,14 +251,17 @@ static int32_t findFirstValPosition(const SColumnInfoData* pCol, int32_t start, static void handleInt8Col(const void* data, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc, bool signVal) { - // AVX2 version to speedup the loop - if (tsAVX2Supported && tsSIMDEnable) { - pBuf->v = i8VectorCmpAVX2(data, numOfRows, isMinFunc, signVal); - } else { - if (!pBuf->assign) { - pBuf->v = ((int8_t*)data)[start]; - } + if (!pBuf->assign) { + pBuf->v = ((const int8_t*)data)[start]; + } +#ifdef __AVX2__ + if (tsAVX2Supported && tsSIMDEnable && numOfRows * sizeof(int8_t) >= sizeof(__m256i)) { + i8VectorCmpAVX2(data + start * sizeof(int8_t), numOfRows, isMinFunc, signVal, &pBuf->v); + } else { +#else + if (true) { +#endif if (signVal) { const int8_t* p = (const int8_t*)data; int8_t* v = (int8_t*)&pBuf->v; @@ -412,14 +288,17 @@ static void handleInt8Col(const void* data, int32_t start, int32_t numOfRows, SM static void handleInt16Col(const void* data, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc, bool signVal) { - // AVX2 version to speedup the loop - if (tsAVX2Supported && tsSIMDEnable) { - pBuf->v = i16VectorCmpAVX2(data, numOfRows, isMinFunc, signVal); - } else { - if (!pBuf->assign) { - pBuf->v = ((int16_t*)data)[start]; - } + if (!pBuf->assign) { + pBuf->v = ((const int16_t*)data)[start]; + } +#ifdef __AVX2__ + if (tsAVX2Supported && tsSIMDEnable && numOfRows * sizeof(int16_t) >= sizeof(__m256i)) { + i16VectorCmpAVX2(data + start * sizeof(int16_t), numOfRows, isMinFunc, signVal, &pBuf->v); + } else { +#else + if (true) { +#endif if (signVal) { const int16_t* p = (const int16_t*)data; int16_t* v = (int16_t*)&pBuf->v; @@ -446,14 +325,17 @@ static void handleInt16Col(const void* data, int32_t start, int32_t numOfRows, S static void handleInt32Col(const void* data, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc, bool signVal) { - // AVX2 version to speedup the loop - if (tsAVX2Supported && tsSIMDEnable) { - pBuf->v = i32VectorCmpAVX2(data, numOfRows, isMinFunc, signVal); - } else { - if (!pBuf->assign) { - pBuf->v = ((int32_t*)data)[start]; - } + if (!pBuf->assign) { + pBuf->v = ((const int32_t*)data)[start]; + } +#ifdef __AVX2__ + if (tsAVX2Supported && tsSIMDEnable && numOfRows * sizeof(int32_t) >= sizeof(__m256i)) { + i32VectorCmpAVX2(data + start * sizeof(int32_t), numOfRows, isMinFunc, signVal, &pBuf->v); + } else { +#else + if (true) { +#endif if (signVal) { const int32_t* p = (const int32_t*)data; int32_t* v = (int32_t*)&pBuf->v; @@ -481,7 +363,7 @@ static void handleInt32Col(const void* data, int32_t start, int32_t numOfRows, S static void handleInt64Col(const void* data, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc, bool signVal) { if (!pBuf->assign) { - pBuf->v = ((int64_t*)data)[start]; + pBuf->v = ((const int64_t*)data)[start]; } if (signVal) { @@ -503,33 +385,29 @@ static void handleInt64Col(const void* data, int32_t start, int32_t numOfRows, S __COMPARE_EXTRACT_MAX(start, start + numOfRows, *v, p); } } + + pBuf->assign = true; } static void handleFloatCol(SColumnInfoData* pCol, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc) { float* pData = (float*)pCol->pData; float* val = (float*)&pBuf->v; + if (!pBuf->assign) { + *val = pData[start]; + } - // AVX version to speedup the loop - if (tsAVXSupported && tsSIMDEnable) { - *val = floatVectorCmpAVX(pData, numOfRows, isMinFunc); +#ifdef __AVX2__ + if (tsAVXSupported && tsSIMDEnable && numOfRows * sizeof(float) >= sizeof(__m256i)) { + floatVectorCmpAVX2(pData + start, numOfRows, isMinFunc, val); } else { - if (!pBuf->assign) { - *val = pData[start]; - } - +#else + if (true) { +#endif if (isMinFunc) { // min - for (int32_t i = start; i < start + numOfRows; ++i) { - if (*val > pData[i]) { - *val = pData[i]; - } - } + __COMPARE_EXTRACT_MIN(start, start + numOfRows, *val, pData); } else { // max - for (int32_t i = start; i < start + numOfRows; ++i) { - if (*val < pData[i]) { - *val = pData[i]; - } - } + __COMPARE_EXTRACT_MAX(start, start + numOfRows, *val, pData); } } @@ -540,27 +418,21 @@ static void handleDoubleCol(SColumnInfoData* pCol, int32_t start, int32_t numOfR bool isMinFunc) { double* pData = (double*)pCol->pData; double* val = (double*)&pBuf->v; + if (!pBuf->assign) { + *val = pData[start]; + } - // AVX version to speedup the loop - if (tsAVXSupported && tsSIMDEnable) { - *val = (double)doubleVectorCmpAVX(pData, numOfRows, isMinFunc); +#ifdef __AVX2__ + if (tsAVXSupported && tsSIMDEnable && numOfRows * sizeof(double) >= sizeof(__m256i)) { + doubleVectorCmpAVX2(pData + start, numOfRows, isMinFunc, val); } else { - if (!pBuf->assign) { - *val = pData[start]; - } - +#else + if (true) { +#endif if (isMinFunc) { // min - for (int32_t i = start; i < start + numOfRows; ++i) { - if (*val > pData[i]) { - *val = pData[i]; - } - } + __COMPARE_EXTRACT_MIN(start, start + numOfRows, *val, pData); } else { // max - for (int32_t i = start; i < start + numOfRows; ++i) { - if (*val < pData[i]) { - *val = pData[i]; - } - } + __COMPARE_EXTRACT_MAX(start, start + numOfRows, *val, pData); } } @@ -581,7 +453,7 @@ static int32_t findRowIndex(int32_t start, int32_t num, SColumnInfoData* pCol, c } static int32_t doExtractVal(SColumnInfoData* pCol, int32_t i, int32_t end, SqlFunctionCtx* pCtx, SMinmaxResInfo* pBuf, - bool isMinFunc) { + bool isMinFunc) { if (isMinFunc) { switch (pCol->info.type) { case TSDB_DATA_TYPE_BOOL: @@ -652,8 +524,8 @@ static int32_t doExtractVal(SColumnInfoData* pCol, int32_t i, int32_t end, SqlFu if (colDataIsNull_var(pCol, i)) { continue; } - char *pLeft = (char *)colDataGetData(pCol, i); - char *pRight = (char *)pBuf->str; + char* pLeft = (char*)colDataGetData(pCol, i); + char* pRight = (char*)pBuf->str; int32_t ret = compareLenBinaryVal(pLeft, pRight); if (ret < 0) { @@ -674,8 +546,8 @@ static int32_t doExtractVal(SColumnInfoData* pCol, int32_t i, int32_t end, SqlFu if (colDataIsNull_var(pCol, i)) { continue; } - char *pLeft = (char *)colDataGetData(pCol, i); - char *pRight = (char *)pBuf->str; + char* pLeft = (char*)colDataGetData(pCol, i); + char* pRight = (char*)pBuf->str; int32_t ret = compareLenPrefixedWStr(pLeft, pRight); if (ret < 0) { @@ -761,8 +633,8 @@ static int32_t doExtractVal(SColumnInfoData* pCol, int32_t i, int32_t end, SqlFu if (colDataIsNull_var(pCol, i)) { continue; } - char *pLeft = (char *)colDataGetData(pCol, i); - char *pRight = (char *)pBuf->str; + char* pLeft = (char*)colDataGetData(pCol, i); + char* pRight = (char*)pBuf->str; int32_t ret = compareLenBinaryVal(pLeft, pRight); if (ret > 0) { @@ -784,8 +656,8 @@ static int32_t doExtractVal(SColumnInfoData* pCol, int32_t i, int32_t end, SqlFu if (colDataIsNull_var(pCol, i)) { continue; } - char *pLeft = (char *)colDataGetData(pCol, i); - char *pRight = (char *)pBuf->str; + char* pLeft = (char*)colDataGetData(pCol, i); + char* pRight = (char*)pBuf->str; int32_t ret = compareLenPrefixedWStr(pLeft, pRight); if (ret > 0) { @@ -838,7 +710,6 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems) // data in current data block are qualified to the query if (pInput->colDataSMAIsSet && !IS_STR_DATA_TYPE(type)) { - numOfElems = pInput->numOfRows - pAgg->numOfNull; if (numOfElems == 0) { goto _over; diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 886772b36c..6c67a888a2 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -408,7 +408,7 @@ static int32_t createColumnByFunc(const SFunctionNode* pFunc, SColumnNode** ppCo if (NULL == *ppCol) { return code; } - (void)strcpy((*ppCol)->colName, pFunc->node.aliasName); + tstrncpy((*ppCol)->colName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); (*ppCol)->node.resType = pFunc->node.resType; return TSDB_CODE_SUCCESS; } @@ -437,11 +437,12 @@ static int32_t createPartialFunction(const SFunctionNode* pSrcFunc, SFunctionNod (*pPartialFunc)->hasOriginalFunc = true; (*pPartialFunc)->originalFuncId = pSrcFunc->hasOriginalFunc ? pSrcFunc->originalFuncId : pSrcFunc->funcId; char name[TSDB_FUNC_NAME_LEN + TSDB_NAME_DELIMITER_LEN + TSDB_POINTER_PRINT_BYTES + 1] = {0}; - int32_t len = snprintf(name, sizeof(name) - 1, "%s.%p", (*pPartialFunc)->functionName, pSrcFunc); + + int32_t len = tsnprintf(name, sizeof(name), "%s.%p", (*pPartialFunc)->functionName, pSrcFunc); if (taosHashBinary(name, len) < 0) { return TSDB_CODE_FAILED; } - (void)strncpy((*pPartialFunc)->node.aliasName, name, TSDB_COL_NAME_LEN - 1); + tstrncpy((*pPartialFunc)->node.aliasName, name, TSDB_COL_NAME_LEN); (*pPartialFunc)->hasPk = pSrcFunc->hasPk; (*pPartialFunc)->pkBytes = pSrcFunc->pkBytes; return TSDB_CODE_SUCCESS; @@ -475,7 +476,7 @@ static int32_t createMidFunction(const SFunctionNode* pSrcFunc, const SFunctionN } } if (TSDB_CODE_SUCCESS == code) { - (void)strcpy(pFunc->node.aliasName, pPartialFunc->node.aliasName); + tstrncpy(pFunc->node.aliasName, pPartialFunc->node.aliasName, TSDB_COL_NAME_LEN); } if (TSDB_CODE_SUCCESS == code) { @@ -504,7 +505,7 @@ static int32_t createMergeFunction(const SFunctionNode* pSrcFunc, const SFunctio if (fmIsSameInOutType(pSrcFunc->funcId)) { pFunc->node.resType = pSrcFunc->node.resType; } - (void)strcpy(pFunc->node.aliasName, pSrcFunc->node.aliasName); + tstrncpy(pFunc->node.aliasName, pSrcFunc->node.aliasName, TSDB_COL_NAME_LEN); } if (TSDB_CODE_SUCCESS == code) { @@ -558,8 +559,8 @@ static int32_t fmCreateStateFunc(const SFunctionNode* pFunc, SFunctionNode** pSt nodesDestroyList(pParams); return code; } - (void)strcpy((*pStateFunc)->node.aliasName, pFunc->node.aliasName); - (void)strcpy((*pStateFunc)->node.userAlias, pFunc->node.userAlias); + tstrncpy((*pStateFunc)->node.aliasName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); + tstrncpy((*pStateFunc)->node.userAlias, pFunc->node.userAlias, TSDB_COL_NAME_LEN); } return TSDB_CODE_SUCCESS; } @@ -605,8 +606,8 @@ static int32_t fmCreateStateMergeFunc(SFunctionNode* pFunc, SFunctionNode** pSta nodesDestroyList(pParams); return code; } - (void)strcpy((*pStateMergeFunc)->node.aliasName, pFunc->node.aliasName); - (void)strcpy((*pStateMergeFunc)->node.userAlias, pFunc->node.userAlias); + tstrncpy((*pStateMergeFunc)->node.aliasName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); + tstrncpy((*pStateMergeFunc)->node.userAlias, pFunc->node.userAlias, TSDB_COL_NAME_LEN); } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/function/src/tscript.c b/source/libs/function/src/tscript.c index 768581285b..eecc66d6d6 100644 --- a/source/libs/function/src/tscript.c +++ b/source/libs/function/src/tscript.c @@ -92,7 +92,7 @@ void taosValueToLuaType(lua_State *lua, int32_t type, char *val) { int taosLoadScriptInit(void* pInit) { ScriptCtx *pCtx = pInit; char funcName[MAX_FUNC_NAME] = {0}; - sprintf(funcName, "%s_init", pCtx->funcName); + snprintf(funcName, MAX_FUNC_NAME, "%s_init", pCtx->funcName); lua_State* lua = pCtx->pEnv->lua_state; lua_getglobal(lua, funcName); @@ -106,7 +106,7 @@ void taosLoadScriptNormal(void *pInit, char *pInput, int16_t iType, int16_t iByt int64_t *ptsList, int64_t key, char* pOutput, char *ptsOutput, int32_t *numOfOutput, int16_t oType, int16_t oBytes) { ScriptCtx* pCtx = pInit; char funcName[MAX_FUNC_NAME] = {0}; - sprintf(funcName, "%s_add", pCtx->funcName); + snprintf(funcName, MAX_FUNC_NAME, "%s_add", pCtx->funcName); lua_State* lua = pCtx->pEnv->lua_state; lua_getglobal(lua, funcName); @@ -143,7 +143,7 @@ void taosLoadScriptNormal(void *pInit, char *pInput, int16_t iType, int16_t iByt void taosLoadScriptMerge(void *pInit, char* data, int32_t numOfRows, char* pOutput, int32_t* numOfOutput) { ScriptCtx *pCtx = pInit; char funcName[MAX_FUNC_NAME] = {0}; - sprintf(funcName, "%s_merge", pCtx->funcName); + snprintf(funcName, MAX_FUNC_NAME, "%s_merge", pCtx->funcName); lua_State* lua = pCtx->pEnv->lua_state; lua_getglobal(lua, funcName); @@ -167,7 +167,7 @@ void taosLoadScriptMerge(void *pInit, char* data, int32_t numOfRows, char* pOutp void taosLoadScriptFinalize(void *pInit,int64_t key, char *pOutput, int32_t* numOfOutput) { ScriptCtx *pCtx = pInit; char funcName[MAX_FUNC_NAME] = {0}; - sprintf(funcName, "%s_finalize", pCtx->funcName); + snprintf(funcName, MAX_FUNC_NAME, "%s_finalize", pCtx->funcName); lua_State* lua = pCtx->pEnv->lua_state; lua_getglobal(lua, funcName); diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index ffdd4ea500..6bc9f84d6d 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -143,10 +143,10 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) { char udfdPathLdLib[1024] = {0}; size_t udfdLdLibPathLen = strlen(tsUdfdLdLibPath); - strncpy(udfdPathLdLib, tsUdfdLdLibPath, tListLen(udfdPathLdLib)); + tstrncpy(udfdPathLdLib, tsUdfdLdLibPath, sizeof(udfdPathLdLib)); udfdPathLdLib[udfdLdLibPathLen] = ':'; - strncpy(udfdPathLdLib + udfdLdLibPathLen + 1, pathTaosdLdLib, sizeof(udfdPathLdLib) - udfdLdLibPathLen - 1); + tstrncpy(udfdPathLdLib + udfdLdLibPathLen + 1, pathTaosdLdLib, sizeof(udfdPathLdLib) - udfdLdLibPathLen - 1); if (udfdLdLibPathLen + taosdLdLibPathLen < 1024) { fnInfo("[UDFD]udfd LD_LIBRARY_PATH: %s", udfdPathLdLib); } else { @@ -158,10 +158,12 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) { char *taosFqdnEnvItem = NULL; char *taosFqdn = getenv("TAOS_FQDN"); if (taosFqdn != NULL) { - taosFqdnEnvItem = taosMemoryMalloc(strlen("TAOS_FQDN=") + strlen(taosFqdn) + 1); + int subLen = strlen(taosFqdn); + int len = strlen("TAOS_FQDN=") + subLen + 1; + taosFqdnEnvItem = taosMemoryMalloc(len); if (taosFqdnEnvItem != NULL) { - strcpy(taosFqdnEnvItem, "TAOS_FQDN="); - TAOS_STRCAT(taosFqdnEnvItem, taosFqdn); + tstrncpy(taosFqdnEnvItem, "TAOS_FQDN=", len); + TAOS_STRNCAT(taosFqdnEnvItem, taosFqdn, subLen); fnInfo("[UDFD]Succsess to set TAOS_FQDN:%s", taosFqdn); } else { fnError("[UDFD]Failed to allocate memory for TAOS_FQDN"); @@ -1072,7 +1074,7 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) { int32_t code = 0, line = 0; uv_mutex_lock(&gUdfcProxy.udfStubsMutex); SUdfcFuncStub key = {0}; - strncpy(key.udfName, udfName, TSDB_FUNC_NAME_LEN); + tstrncpy(key.udfName, udfName, TSDB_FUNC_NAME_LEN); int32_t stubIndex = taosArraySearchIdx(gUdfcProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); if (stubIndex != -1) { SUdfcFuncStub *foundStub = taosArrayGet(gUdfcProxy.udfStubs, stubIndex); @@ -1105,7 +1107,7 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) { code = doSetupUdf(udfName, pHandle); if (code == TSDB_CODE_SUCCESS) { SUdfcFuncStub stub = {0}; - strncpy(stub.udfName, udfName, TSDB_FUNC_NAME_LEN); + tstrncpy(stub.udfName, udfName, TSDB_FUNC_NAME_LEN); stub.handle = *pHandle; ++stub.refCount; stub.createTime = taosGetTimestampUs(); @@ -1129,7 +1131,7 @@ _exit: void releaseUdfFuncHandle(char *udfName, UdfcFuncHandle handle) { uv_mutex_lock(&gUdfcProxy.udfStubsMutex); SUdfcFuncStub key = {0}; - strncpy(key.udfName, udfName, TSDB_FUNC_NAME_LEN); + tstrncpy(key.udfName, udfName, TSDB_FUNC_NAME_LEN); SUdfcFuncStub *foundStub = taosArraySearch(gUdfcProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); SUdfcFuncStub *expiredStub = taosArraySearch(gUdfcProxy.expiredUdfStubs, &key, compareUdfcFuncSub, TD_EQ); if (!foundStub && !expiredStub) { @@ -2020,7 +2022,7 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) { task->type = UDF_TASK_SETUP; SUdfSetupRequest *req = &task->_setup.req; - strncpy(req->udfName, udfName, TSDB_FUNC_NAME_LEN); + tstrncpy(req->udfName, udfName, TSDB_FUNC_NAME_LEN); code = udfcRunUdfUvTask(task, UV_TASK_CONNECT); TAOS_CHECK_GOTO(code, &lino, _exit); @@ -2033,7 +2035,7 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) { task->session->outputType = rsp->outputType; task->session->bytes = rsp->bytes; task->session->bufSize = rsp->bufSize; - strncpy(task->session->udfName, udfName, TSDB_FUNC_NAME_LEN); + tstrncpy(task->session->udfName, udfName, TSDB_FUNC_NAME_LEN); fnInfo("successfully setup udf func handle. udfName: %s, handle: %p", udfName, task->session); *funcHandle = task->session; taosMemoryFree(task); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index e1dfd686d4..c360cf6894 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -396,7 +396,7 @@ int32_t udfdLoadSharedLib(char *libPath, uv_lib_t *pLib, const char *funcName[], int32_t udfdInitializePythonPlugin(SUdfScriptPlugin *plugin) { plugin->scriptType = TSDB_FUNC_SCRIPT_PYTHON; // todo: windows support - sprintf(plugin->libPath, "%s", "libtaospyudf.so"); + snprintf(plugin->libPath, PATH_MAX, "%s", "libtaospyudf.so"); plugin->libLoaded = false; const char *funcName[UDFD_MAX_PLUGIN_FUNCS] = {"pyOpen", "pyClose", "pyUdfInit", "pyUdfDestroy", "pyUdfScalarProc", "pyUdfAggStart", @@ -617,7 +617,7 @@ int32_t udfdNewUdf(SUdf **pUdf, const char *udfName) { } udfNew->refCount = 1; udfNew->lastFetchTime = taosGetTimestampMs(); - strncpy(udfNew->name, udfName, TSDB_FUNC_NAME_LEN); + tstrncpy(udfNew->name, udfName, TSDB_FUNC_NAME_LEN); udfNew->state = UDF_STATE_INIT; if (uv_mutex_init(&udfNew->lock) != 0) return TSDB_CODE_UDF_UV_EXEC_FAILURE; @@ -997,7 +997,7 @@ int32_t udfdSaveFuncBodyToFile(SFuncInfo *pFuncInfo, SUdf *udf) { udfdGetFuncBodyPath(udf, path); bool fileExist = !(taosStatFile(path, NULL, NULL, NULL) < 0); if (fileExist) { - strncpy(udf->path, path, PATH_MAX); + tstrncpy(udf->path, path, PATH_MAX); fnInfo("udfd func body file. reuse existing file %s", path); return TSDB_CODE_SUCCESS; } @@ -1017,7 +1017,7 @@ int32_t udfdSaveFuncBodyToFile(SFuncInfo *pFuncInfo, SUdf *udf) { return TSDB_CODE_FILE_CORRUPTED; } - strncpy(udf->path, path, PATH_MAX); + tstrncpy(udf->path, path, PATH_MAX); return TSDB_CODE_SUCCESS; } @@ -1612,7 +1612,7 @@ int32_t udfdInitResidentFuncs() { char *token; while ((token = strtok_r(pSave, ",", &pSave)) != NULL) { char func[TSDB_FUNC_NAME_LEN + 1] = {0}; - strncpy(func, token, TSDB_FUNC_NAME_LEN); + tstrncpy(func, token, TSDB_FUNC_NAME_LEN); fnInfo("udfd add resident function %s", func); if(taosArrayPush(global.residentFuncs, func) == NULL) { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index f70540bfe1..4b3ca97bbf 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -3953,9 +3953,10 @@ static int32_t datumToJson(const void* pObj, SJson* pJson) { break; case TSDB_DATA_TYPE_NCHAR: { // cJSON only support utf-8 encoding. Convert memory content to hex string. - char* buf = taosMemoryCalloc(varDataLen(pNode->datum.p) * 2 + 1, sizeof(char)); + int32_t bufSize = varDataLen(pNode->datum.p) * 2 + 1; + char* buf = taosMemoryCalloc(bufSize, sizeof(char)); if (!buf) return terrno; - code = taosHexEncode(varDataVal(pNode->datum.p), buf, varDataLen(pNode->datum.p)); + code = taosHexEncode(varDataVal(pNode->datum.p), buf, varDataLen(pNode->datum.p), bufSize); if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(buf); return TSDB_CODE_TSC_INVALID_VALUE; @@ -3971,9 +3972,10 @@ static int32_t datumToJson(const void* pObj, SJson* pJson) { break; case TSDB_DATA_TYPE_JSON: { int32_t len = getJsonValueLen(pNode->datum.p); - char* buf = taosMemoryCalloc(len * 2 + 1, sizeof(char)); + int32_t bufSize = len * 2 + 1; + char* buf = taosMemoryCalloc(bufSize, sizeof(char)); if (!buf) return terrno; - code = taosHexEncode(pNode->datum.p, buf, len); + code = taosHexEncode(pNode->datum.p, buf, len, bufSize); if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(buf); return TSDB_CODE_TSC_INVALID_VALUE; diff --git a/source/libs/nodes/src/nodesToSQLFuncs.c b/source/libs/nodes/src/nodesToSQLFuncs.c index b57bba0cc9..de0cd54f16 100644 --- a/source/libs/nodes/src/nodesToSQLFuncs.c +++ b/source/libs/nodes/src/nodesToSQLFuncs.c @@ -110,19 +110,19 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { case QUERY_NODE_COLUMN: { SColumnNode *colNode = (SColumnNode *)pNode; if (colNode->dbName[0]) { - *len += snprintf(buf + *len, bufSize - *len, "`%s`.", colNode->dbName); + *len += tsnprintf(buf + *len, bufSize - *len, "`%s`.", colNode->dbName); } if (colNode->tableAlias[0]) { - *len += snprintf(buf + *len, bufSize - *len, "`%s`.", colNode->tableAlias); + *len += tsnprintf(buf + *len, bufSize - *len, "`%s`.", colNode->tableAlias); } else if (colNode->tableName[0]) { - *len += snprintf(buf + *len, bufSize - *len, "`%s`.", colNode->tableName); + *len += tsnprintf(buf + *len, bufSize - *len, "`%s`.", colNode->tableName); } if (colNode->tableAlias[0]) { - *len += snprintf(buf + *len, bufSize - *len, "`%s`", colNode->node.userAlias); + *len += tsnprintf(buf + *len, bufSize - *len, "`%s`", colNode->node.userAlias); } else { - *len += snprintf(buf + *len, bufSize - *len, "%s", colNode->node.userAlias); + *len += tsnprintf(buf + *len, bufSize - *len, "%s", colNode->node.userAlias); } return TSDB_CODE_SUCCESS; @@ -137,9 +137,9 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { int32_t tlen = strlen(t); if (tlen > 32) { - *len += snprintf(buf + *len, bufSize - *len, "%.*s...%s", 32, t, t + tlen - 1); + *len += tsnprintf(buf + *len, bufSize - *len, "%.*s...%s", 32, t, t + tlen - 1); } else { - *len += snprintf(buf + *len, bufSize - *len, "%s", t); + *len += tsnprintf(buf + *len, bufSize - *len, "%s", t); } taosMemoryFree(t); @@ -147,18 +147,18 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { } case QUERY_NODE_OPERATOR: { SOperatorNode *pOpNode = (SOperatorNode *)pNode; - *len += snprintf(buf + *len, bufSize - *len, "("); + *len += tsnprintf(buf + *len, bufSize - *len, "("); if (pOpNode->pLeft) { NODES_ERR_RET(nodesNodeToSQL(pOpNode->pLeft, buf, bufSize, len)); } - *len += snprintf(buf + *len, bufSize - *len, " %s ", operatorTypeStr(pOpNode->opType)); + *len += tsnprintf(buf + *len, bufSize - *len, " %s ", operatorTypeStr(pOpNode->opType)); if (pOpNode->pRight) { NODES_ERR_RET(nodesNodeToSQL(pOpNode->pRight, buf, bufSize, len)); } - *len += snprintf(buf + *len, bufSize - *len, ")"); + *len += tsnprintf(buf + *len, bufSize - *len, ")"); return TSDB_CODE_SUCCESS; } @@ -167,17 +167,17 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { SNode *node = NULL; bool first = true; - *len += snprintf(buf + *len, bufSize - *len, "("); + *len += tsnprintf(buf + *len, bufSize - *len, "("); FOREACH(node, pLogicNode->pParameterList) { if (!first) { - *len += snprintf(buf + *len, bufSize - *len, " %s ", logicConditionTypeStr(pLogicNode->condType)); + *len += tsnprintf(buf + *len, bufSize - *len, " %s ", logicConditionTypeStr(pLogicNode->condType)); } NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); first = false; } - *len += snprintf(buf + *len, bufSize - *len, ")"); + *len += tsnprintf(buf + *len, bufSize - *len, ")"); return TSDB_CODE_SUCCESS; } @@ -186,17 +186,17 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { SNode *node = NULL; bool first = true; - *len += snprintf(buf + *len, bufSize - *len, "%s(", pFuncNode->functionName); + *len += tsnprintf(buf + *len, bufSize - *len, "%s(", pFuncNode->functionName); FOREACH(node, pFuncNode->pParameterList) { if (!first) { - *len += snprintf(buf + *len, bufSize - *len, ", "); + *len += tsnprintf(buf + *len, bufSize - *len, ", "); } NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); first = false; } - *len += snprintf(buf + *len, bufSize - *len, ")"); + *len += tsnprintf(buf + *len, bufSize - *len, ")"); return TSDB_CODE_SUCCESS; } @@ -206,13 +206,13 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { bool first = true; int32_t num = 0; - *len += snprintf(buf + *len, bufSize - *len, "("); + *len += tsnprintf(buf + *len, bufSize - *len, "("); FOREACH(node, pListNode->pNodeList) { if (!first) { - *len += snprintf(buf + *len, bufSize - *len, ", "); + *len += tsnprintf(buf + *len, bufSize - *len, ", "); if (++num >= 10) { - *len += snprintf(buf + *len, bufSize - *len, "..."); + *len += tsnprintf(buf + *len, bufSize - *len, "..."); break; } } @@ -220,7 +220,7 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { first = false; } - *len += snprintf(buf + *len, bufSize - *len, ")"); + *len += tsnprintf(buf + *len, bufSize - *len, ")"); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 382584bb76..77b39b8f0d 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -2273,9 +2273,9 @@ static EDealRes doCollect(SCollectColumnsCxt* pCxt, SColumnNode* pCol, SNode* pN char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; int32_t len = 0; if ('\0' == pCol->tableAlias[0]) { - len = snprintf(name, sizeof(name), "%s", pCol->colName); + len = tsnprintf(name, sizeof(name), "%s", pCol->colName); } else { - len = snprintf(name, sizeof(name), "%s.%s", pCol->tableAlias, pCol->colName); + len = tsnprintf(name, sizeof(name), "%s.%s", pCol->tableAlias, pCol->colName); } if (pCol->projRefIdx > 0) { len = taosHashBinary(name, strlen(name)); diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index cd3095ede8..5db7e18fc5 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -3651,6 +3651,7 @@ SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, STokenPair* CHECK_PARSER_STATUS(pCxt); CHECK_NAME(checkDbName(pCxt, &pPrivLevel->first, false)); CHECK_NAME(checkUserName(pCxt, pUserName)); + CHECK_NAME(checkTableName(pCxt, &pPrivLevel->second)); SRevokeStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_REVOKE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index da9c9d5b8d..cca35d9c9a 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -113,7 +113,7 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem SSchema* pTagSchema = &pSchema[tags->pColIndex[i]]; SSmlKv* kv = taosArrayGet(cols, i); if (kv == NULL){ - code = TSDB_CODE_SML_INVALID_DATA; + code = terrno; uError("SML smlBuildTagRow error kv is null"); goto end; } @@ -381,7 +381,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc for (int32_t r = 0; r < rowNum; ++r) { void* rowData = taosArrayGetP(cols, r); if (rowData == NULL) { - ret = TSDB_CODE_SML_INVALID_DATA; + ret = terrno; goto end; } // 1. set the parsed value from sql string @@ -389,7 +389,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]]; SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]); if (pVal == NULL) { - ret = TSDB_CODE_SML_INVALID_DATA; + ret = terrno; goto end; } void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 500d0e213e..ee61611bf2 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -218,6 +218,10 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch } else { memcpy(&val.i64, bind[c].buffer, colLen); } + if (IS_VAR_DATA_TYPE(pTagSchema->type) && val.nData > pTagSchema->bytes) { + code = TSDB_CODE_PAR_VALUE_TOO_LONG; + goto end; + } if (NULL == taosArrayPush(pTagArray, &val)) { code = terrno; goto end; @@ -566,6 +570,10 @@ int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const c } else { memcpy(&val.i64, bind[c].buffer, colLen); } + if (IS_VAR_DATA_TYPE(pTagSchema->type) && val.nData > pTagSchema->bytes) { + code = TSDB_CODE_PAR_VALUE_TOO_LONG; + goto end; + } if (NULL == taosArrayPush(pTagArray, &val)) { code = terrno; goto end; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3ae4583013..5c6f619397 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2738,7 +2738,7 @@ static void setFuncClassification(STranslateContext* pCxt, SFunctionNode* pFunc) pSelect->returnRows = fmGetFuncReturnRows(pFunc); } else if (fmIsInterpFunc(pFunc->funcId)) { pSelect->returnRows = fmGetFuncReturnRows(pFunc); - } + } if (fmIsProcessByRowFunc(pFunc->funcId)) { pSelect->lastProcessByRowFuncId = pFunc->funcId; } @@ -2840,7 +2840,7 @@ static int32_t rewriteServerStatusFunc(STranslateContext* pCxt, SNode** pNode) { static int32_t rewriteUserFunc(STranslateContext* pCxt, SNode** pNode) { char userConn[TSDB_USER_LEN + 1 + TSDB_FQDN_LEN] = {0}; // format 'user@host' - int32_t len = snprintf(userConn, sizeof(userConn), "%s@", pCxt->pParseCxt->pUser); + int32_t len = tsnprintf(userConn, sizeof(userConn), "%s@", pCxt->pParseCxt->pUser); if (TSDB_CODE_SUCCESS != taosGetFqdn(userConn + len)) { return terrno; } @@ -3781,8 +3781,8 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) } static int32_t checkWinJoinAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) { - if (!isWindowJoinStmt(pSelect) || - (!pSelect->hasAggFuncs && !pSelect->hasIndefiniteRowsFunc && !pSelect->hasInterpFunc)) { + if (!isWindowJoinStmt(pSelect) || (!pSelect->hasAggFuncs && !pSelect->hasIndefiniteRowsFunc && + !pSelect->hasInterpFunc)) { return TSDB_CODE_SUCCESS; } if (!pSelect->onlyHasKeepOrderFunc) { @@ -4126,7 +4126,7 @@ static int32_t setTableTsmas(STranslateContext* pCxt, SName* pName, SRealTableNo SVgroupInfo vgInfo = {0}; bool exists = false; toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetTbName); - int32_t len = snprintf(buf, TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN, "%s.%s_%s", pTsma->dbFName, pTsma->name, + int32_t len = tsnprintf(buf, TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN, "%s.%s_%s", pTsma->dbFName, pTsma->name, pRealTable->table.tableName); len = taosCreateMD5Hash(buf, len); strncpy(tsmaTargetTbName.tname, buf, MD5_OUTPUT_LEN); @@ -4935,18 +4935,18 @@ static int32_t createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr, SNo strcpy(pFunc->node.userAlias, pCol->colName); strcpy(pFunc->node.aliasName, pCol->colName); } else { - len = snprintf(buf, sizeof(buf) - 1, "%s(%s.%s)", pSrcFunc->functionName, pCol->tableAlias, pCol->colName); + len = tsnprintf(buf, sizeof(buf) - 1, "%s(%s.%s)", pSrcFunc->functionName, pCol->tableAlias, pCol->colName); (void)taosHashBinary(buf, len); strncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN - 1); - len = snprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pCol->colName); + len = tsnprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pCol->colName); // note: userAlias could be truncated here strncpy(pFunc->node.userAlias, buf, TSDB_COL_NAME_LEN - 1); } } else { - len = snprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pExpr->aliasName); + len = tsnprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pExpr->aliasName); (void)taosHashBinary(buf, len); strncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN - 1); - len = snprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pExpr->userAlias); + len = tsnprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pExpr->userAlias); // note: userAlias could be truncated here strncpy(pFunc->node.userAlias, buf, TSDB_COL_NAME_LEN - 1); } @@ -6402,7 +6402,7 @@ static int32_t replaceToChildTableQuery(STranslateContext* pCxt, SEqCondTbNameTa STableTSMAInfo* pTsma = taosArrayGetP(pRealTable->pTsmas, i); SName tsmaTargetTbName = {0}; toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetTbName); - int32_t len = snprintf(buf, TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN, "%s.%s_%s", pTsma->dbFName, pTsma->name, + int32_t len = tsnprintf(buf, TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN, "%s.%s_%s", pTsma->dbFName, pTsma->name, pRealTable->table.tableName); len = taosCreateMD5Hash(buf, len); strncpy(tsmaTargetTbName.tname, buf, MD5_OUTPUT_LEN); @@ -8724,7 +8724,7 @@ static int32_t makeIntervalVal(SRetention* pRetension, int8_t precision, SNode** return code; } char buf[20] = {0}; - int32_t len = snprintf(buf, sizeof(buf), "%" PRId64 "%c", timeVal, pRetension->freqUnit); + int32_t len = tsnprintf(buf, sizeof(buf), "%" PRId64 "%c", timeVal, pRetension->freqUnit); pVal->literal = taosStrndup(buf, len); if (NULL == pVal->literal) { nodesDestroyNode((SNode*)pVal); @@ -10186,7 +10186,8 @@ static int32_t translateKillTransaction(STranslateContext* pCxt, SKillStmt* pStm static bool crossTableWithoutAggOper(SSelectStmt* pSelect) { return NULL == pSelect->pWindow && !pSelect->hasAggFuncs && !pSelect->hasIndefiniteRowsFunc && - !pSelect->hasInterpFunc && TSDB_SUPER_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType && + !pSelect->hasInterpFunc && + TSDB_SUPER_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType && !hasTbnameFunction(pSelect->pPartitionByList); } @@ -10603,6 +10604,11 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported Group by"); } + if (NULL == pSelect->pWindow && pSelect->hasAggFuncs) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "Non window query only support scalar function, aggregate function is not allowed"); + } + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 1ce8b04324..98676160cb 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -987,13 +987,13 @@ static int32_t reserveTableReqInCacheImpl(const char* pTbFName, int32_t len, SHa static int32_t reserveTableReqInCache(int32_t acctId, const char* pDb, const char* pTable, SHashObj** pTables) { char fullName[TSDB_TABLE_FNAME_LEN]; - int32_t len = snprintf(fullName, sizeof(fullName), "%d.%s.%s", acctId, pDb, pTable); + int32_t len = tsnprintf(fullName, sizeof(fullName), "%d.%s.%s", acctId, pDb, pTable); return reserveTableReqInCacheImpl(fullName, len, pTables); } static int32_t reserveTableReqInDbCacheImpl(int32_t acctId, const char* pDb, const char* pTable, SHashObj* pDbs) { SParseTablesMetaReq req = {0}; - int32_t len = snprintf(req.dbFName, sizeof(req.dbFName), "%d.%s", acctId, pDb); + int32_t len = tsnprintf(req.dbFName, sizeof(req.dbFName), "%d.%s", acctId, pDb); int32_t code = reserveTableReqInCache(acctId, pDb, pTable, &req.pTables); if (TSDB_CODE_SUCCESS == code) { code = taosHashPut(pDbs, req.dbFName, len, &req, sizeof(SParseTablesMetaReq)); @@ -1009,7 +1009,7 @@ static int32_t reserveTableReqInDbCache(int32_t acctId, const char* pDb, const c } } char fullName[TSDB_DB_FNAME_LEN]; - int32_t len = snprintf(fullName, sizeof(fullName), "%d.%s", acctId, pDb); + int32_t len = tsnprintf(fullName, sizeof(fullName), "%d.%s", acctId, pDb); SParseTablesMetaReq* pReq = taosHashGet(*pDbs, fullName, len); if (NULL == pReq) { return reserveTableReqInDbCacheImpl(acctId, pDb, pTable, *pDbs); @@ -1109,7 +1109,7 @@ static int32_t reserveDbReqInCache(int32_t acctId, const char* pDb, SHashObj** p } } char fullName[TSDB_TABLE_FNAME_LEN]; - int32_t len = snprintf(fullName, sizeof(fullName), "%d.%s", acctId, pDb); + int32_t len = tsnprintf(fullName, sizeof(fullName), "%d.%s", acctId, pDb); return taosHashPut(*pDbs, fullName, len, &nullPointer, POINTER_BYTES); } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 401b4f93d1..97c4fa9dde 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -294,9 +294,6 @@ static bool scanPathOptIsSpecifiedFuncType(const SFunctionNode* pFunc, bool (*ty return true; } -static bool isMinMaxFunction(int32_t funcType) { - return funcType == FUNCTION_TYPE_MIN || funcType == FUNCTION_TYPE_MAX; -} static int32_t scanPathOptGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs, SNodeList** pDsoFuncs) { SNodeList* pAllFuncs = scanPathOptGetAllFuncs(pScan->node.pParent); SNodeList* pTmpSdrFuncs = NULL; @@ -306,8 +303,7 @@ static int32_t scanPathOptGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSd FOREACH(pNode, pAllFuncs) { SFunctionNode* pFunc = (SFunctionNode*)pNode; int32_t code = TSDB_CODE_SUCCESS; - if ((!isMinMaxFunction(pFunc->funcType) && scanPathOptIsSpecifiedFuncType(pFunc, fmIsSpecialDataRequiredFunc)) || - (isMinMaxFunction(pFunc->funcType) && pFunc->hasSMA)) { + if (scanPathOptIsSpecifiedFuncType(pFunc, fmIsSpecialDataRequiredFunc)) { SNode* pNew = NULL; code = nodesCloneNode(pNode, &pNew); if (TSDB_CODE_SUCCESS == code) { @@ -1170,9 +1166,9 @@ static EDealRes pdcJoinCollectCondCol(SNode* pNode, void* pContext) { char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; int32_t len = 0; if ('\0' == pCol->tableAlias[0]) { - len = snprintf(name, sizeof(name), "%s", pCol->colName); + len = tsnprintf(name, sizeof(name), "%s", pCol->colName); } else { - len = snprintf(name, sizeof(name), "%s.%s", pCol->tableAlias, pCol->colName); + len = tsnprintf(name, sizeof(name), "%s.%s", pCol->tableAlias, pCol->colName); } if (NULL == taosHashGet(pCxt->pColHash, name, len)) { pCxt->errCode = taosHashPut(pCxt->pColHash, name, len, NULL, 0); @@ -3166,7 +3162,7 @@ static int32_t partTagsOptRebuildTbanme(SNodeList* pPartKeys) { // todo refact: just to mask compilation warnings static void partTagsSetAlias(char* pAlias, const char* pTableAlias, const char* pColName) { char name[TSDB_COL_FNAME_LEN + 1] = {0}; - int32_t len = snprintf(name, TSDB_COL_FNAME_LEN, "%s.%s", pTableAlias, pColName); + int32_t len = tsnprintf(name, TSDB_COL_FNAME_LEN, "%s.%s", pTableAlias, pColName); (void)taosHashBinary(name, len); strncpy(pAlias, name, TSDB_COL_NAME_LEN - 1); @@ -3845,7 +3841,7 @@ static int32_t rewriteUniqueOptCreateFirstFunc(SFunctionNode* pSelectValue, SNod } else { int64_t pointer = (int64_t)pFunc; char name[TSDB_FUNC_NAME_LEN + TSDB_POINTER_PRINT_BYTES + TSDB_NAME_DELIMITER_LEN + 1] = {0}; - int32_t len = snprintf(name, sizeof(name) - 1, "%s.%" PRId64 "", pFunc->functionName, pointer); + int32_t len = tsnprintf(name, sizeof(name) - 1, "%s.%" PRId64 "", pFunc->functionName, pointer); (void)taosHashBinary(name, len); strncpy(pFunc->node.aliasName, name, TSDB_COL_NAME_LEN - 1); } @@ -4355,7 +4351,7 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic } FOREACH(pParamNode, pFunc->pParameterList) { if (FUNCTION_TYPE_LAST_ROW == funcType || FUNCTION_TYPE_LAST == funcType) { - int32_t len = snprintf(pFunc->functionName, sizeof(pFunc->functionName), + int32_t len = tsnprintf(pFunc->functionName, sizeof(pFunc->functionName), FUNCTION_TYPE_LAST_ROW == funcType ? "_cache_last_row" : "_cache_last"); pFunc->functionName[len] = '\0'; code = fmGetFuncInfo(pFunc, NULL, 0); @@ -7238,7 +7234,7 @@ static int32_t tsmaOptCreateWStart(int8_t precision, SFunctionNode** pWStartOut) strcpy(pWStart->functionName, "_wstart"); int64_t pointer = (int64_t)pWStart; char name[TSDB_COL_NAME_LEN + TSDB_POINTER_PRINT_BYTES + TSDB_NAME_DELIMITER_LEN + 1] = {0}; - int32_t len = snprintf(name, sizeof(name) - 1, "%s.%" PRId64 "", pWStart->functionName, pointer); + int32_t len = tsnprintf(name, sizeof(name) - 1, "%s.%" PRId64 "", pWStart->functionName, pointer); (void)taosHashBinary(name, len); strncpy(pWStart->node.aliasName, name, TSDB_COL_NAME_LEN - 1); pWStart->node.resType.precision = precision; diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 706394507a..bd2462243d 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -431,7 +431,7 @@ static int32_t stbSplAppendWStart(SNodeList* pFuncs, int32_t* pIndex, uint8_t pr strcpy(pWStart->functionName, "_wstart"); int64_t pointer = (int64_t)pWStart; char name[TSDB_COL_NAME_LEN + TSDB_POINTER_PRINT_BYTES + TSDB_NAME_DELIMITER_LEN + 1] = {0}; - int32_t len = snprintf(name, sizeof(name) - 1, "%s.%" PRId64 "", pWStart->functionName, pointer); + int32_t len = tsnprintf(name, sizeof(name) - 1, "%s.%" PRId64 "", pWStart->functionName, pointer); (void)taosHashBinary(name, len); strncpy(pWStart->node.aliasName, name, TSDB_COL_NAME_LEN - 1); pWStart->node.resType.precision = precision; @@ -463,7 +463,7 @@ static int32_t stbSplAppendWEnd(SWindowLogicNode* pWin, int32_t* pIndex) { strcpy(pWEnd->functionName, "_wend"); int64_t pointer = (int64_t)pWEnd; char name[TSDB_COL_NAME_LEN + TSDB_POINTER_PRINT_BYTES + TSDB_NAME_DELIMITER_LEN + 1] = {0}; - int32_t len = snprintf(name, sizeof(name) - 1, "%s.%" PRId64 "", pWEnd->functionName, pointer); + int32_t len = tsnprintf(name, sizeof(name) - 1, "%s.%" PRId64 "", pWEnd->functionName, pointer); (void)taosHashBinary(name, len); strncpy(pWEnd->node.aliasName, name, TSDB_COL_NAME_LEN - 1); diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index b5f0bc50e8..ae958de6e8 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -630,7 +630,7 @@ SFunctionNode* createGroupKeyAggFunc(SColumnNode* pGroupCol) { } if (TSDB_CODE_SUCCESS == code) { char name[TSDB_FUNC_NAME_LEN + TSDB_NAME_DELIMITER_LEN + TSDB_POINTER_PRINT_BYTES + 1] = {0}; - int32_t len = snprintf(name, sizeof(name) - 1, "%s.%p", pFunc->functionName, pFunc); + int32_t len = tsnprintf(name, sizeof(name) - 1, "%s.%p", pFunc->functionName, pFunc); (void)taosHashBinary(name, len); strncpy(pFunc->node.aliasName, name, TSDB_COL_NAME_LEN - 1); } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 43159bce20..c3aa95f5b7 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -37,7 +37,7 @@ static int32_t dumpQueryPlan(SQueryPlan* pPlan) { char* pStr = NULL; code = nodesNodeToString((SNode*)pPlan, false, &pStr, NULL); if (TSDB_CODE_SUCCESS == code) { - planDebugL("qid:0x%" PRIx64 " Query Plan, JsonPlan: %s", pPlan->queryId, pStr); + planDebugL("QID:0x%" PRIx64 " Query Plan, JsonPlan: %s", pPlan->queryId, pStr); taosMemoryFree(pStr); } return code; @@ -123,7 +123,7 @@ int32_t qContinuePlanPostQuery(void* pPostPlan) { } int32_t qSetSubplanExecutionNode(SSubplan* subplan, int32_t groupId, SDownstreamSourceNode* pSource) { - planDebug("qid:0x%" PRIx64 " set subplan execution node, groupId:%d", subplan->id.queryId, groupId); + planDebug("QID:0x%" PRIx64 " set subplan execution node, groupId:%d", subplan->id.queryId, groupId); return setSubplanExecutionNode(subplan->pNode, groupId, pSource); } @@ -143,7 +143,7 @@ static void clearSubplanExecutionNode(SPhysiNode* pNode) { } void qClearSubplanExecutionNode(SSubplan* pSubplan) { - planDebug("qid:0x%" PRIx64 " clear subplan execution node, groupId:%d", pSubplan->id.queryId, pSubplan->id.groupId); + planDebug("QID:0x%" PRIx64 " clear subplan execution node, groupId:%d", pSubplan->id.queryId, pSubplan->id.groupId); clearSubplanExecutionNode(pSubplan->pNode); } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index ff20211af2..847fc9b8e9 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -313,42 +313,41 @@ void destroyQueryExecRes(SExecResult* pRes) { } } // clang-format on - -int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len) { +int32_t dataConverToStr(char* str, int64_t capacity, int type, void* buf, int32_t bufSize, int32_t* len) { int32_t n = 0; switch (type) { case TSDB_DATA_TYPE_NULL: - n = sprintf(str, "null"); + n = tsnprintf(str, capacity, "null"); break; case TSDB_DATA_TYPE_BOOL: - n = sprintf(str, (*(int8_t*)buf) ? "true" : "false"); + n = tsnprintf(str, capacity, (*(int8_t*)buf) ? "true" : "false"); break; case TSDB_DATA_TYPE_TINYINT: - n = sprintf(str, "%d", *(int8_t*)buf); + n = tsnprintf(str, capacity, "%d", *(int8_t*)buf); break; case TSDB_DATA_TYPE_SMALLINT: - n = sprintf(str, "%d", *(int16_t*)buf); + n = tsnprintf(str, capacity, "%d", *(int16_t*)buf); break; case TSDB_DATA_TYPE_INT: - n = sprintf(str, "%d", *(int32_t*)buf); + n = tsnprintf(str, capacity, "%d", *(int32_t*)buf); break; case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: - n = sprintf(str, "%" PRId64, *(int64_t*)buf); + n = tsnprintf(str, capacity, "%" PRId64, *(int64_t*)buf); break; case TSDB_DATA_TYPE_FLOAT: - n = sprintf(str, "%e", GET_FLOAT_VAL(buf)); + n = tsnprintf(str, capacity, "%e", GET_FLOAT_VAL(buf)); break; case TSDB_DATA_TYPE_DOUBLE: - n = sprintf(str, "%e", GET_DOUBLE_VAL(buf)); + n = tsnprintf(str, capacity, "%e", GET_DOUBLE_VAL(buf)); break; case TSDB_DATA_TYPE_VARBINARY: { @@ -395,19 +394,19 @@ int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t n = length + 2; break; case TSDB_DATA_TYPE_UTINYINT: - n = sprintf(str, "%d", *(uint8_t*)buf); + n = tsnprintf(str, capacity, "%d", *(uint8_t*)buf); break; case TSDB_DATA_TYPE_USMALLINT: - n = sprintf(str, "%d", *(uint16_t*)buf); + n = tsnprintf(str, capacity, "%d", *(uint16_t*)buf); break; case TSDB_DATA_TYPE_UINT: - n = sprintf(str, "%u", *(uint32_t*)buf); + n = tsnprintf(str, capacity, "%u", *(uint32_t*)buf); break; case TSDB_DATA_TYPE_UBIGINT: - n = sprintf(str, "%" PRIu64, *(uint64_t*)buf); + n = tsnprintf(str, capacity, "%" PRIu64, *(uint64_t*)buf); break; default: diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 542e549d40..60c760a60e 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -107,7 +107,7 @@ int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *ms } SUseDbReq usedbReq = {0}; - strncpy(usedbReq.db, pInput->db, sizeof(usedbReq.db)); + tstrncpy(usedbReq.db, pInput->db, TSDB_DB_FNAME_LEN); usedbReq.db[sizeof(usedbReq.db) - 1] = 0; usedbReq.vgVersion = pInput->vgVersion; usedbReq.dbId = pInput->dbId; @@ -207,7 +207,7 @@ int32_t queryBuildGetDBCfgMsg(void *input, char **msg, int32_t msgSize, int32_t } SDbCfgReq dbCfgReq = {0}; - strncpy(dbCfgReq.db, input, sizeof(dbCfgReq.db) - 1); + tstrncpy(dbCfgReq.db, input, TSDB_DB_FNAME_LEN); int32_t bufLen = tSerializeSDbCfgReq(NULL, 0, &dbCfgReq); void *pBuf = (*mallcFp)(bufLen); @@ -231,7 +231,7 @@ int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t } SUserIndexReq indexReq = {0}; - strncpy(indexReq.indexFName, input, sizeof(indexReq.indexFName) - 1); + tstrncpy(indexReq.indexFName, input, TSDB_INDEX_FNAME_LEN); int32_t bufLen = tSerializeSUserIndexReq(NULL, 0, &indexReq); void *pBuf = (*mallcFp)(bufLen); @@ -293,7 +293,7 @@ int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32 } SGetUserAuthReq req = {0}; - strncpy(req.user, input, sizeof(req.user) - 1); + tstrncpy(req.user, input, TSDB_USER_LEN); int32_t bufLen = tSerializeSGetUserAuthReq(NULL, 0, &req); void *pBuf = (*mallcFp)(bufLen); @@ -316,7 +316,7 @@ int32_t queryBuildGetTbIndexMsg(void *input, char **msg, int32_t msgSize, int32_ } STableIndexReq indexReq = {0}; - strncpy(indexReq.tbFName, input, sizeof(indexReq.tbFName) - 1); + tstrncpy(indexReq.tbFName, input, TSDB_TABLE_FNAME_LEN); int32_t bufLen = tSerializeSTableIndexReq(NULL, 0, &indexReq); void *pBuf = (*mallcFp)(bufLen); @@ -342,8 +342,8 @@ int32_t queryBuildGetTbCfgMsg(void *input, char **msg, int32_t msgSize, int32_t SBuildTableInput *pInput = input; STableCfgReq cfgReq = {0}; cfgReq.header.vgId = pInput->vgId; - strncpy(cfgReq.dbFName, pInput->dbFName, sizeof(cfgReq.dbFName) - 1); - strncpy(cfgReq.tbName, pInput->tbName, sizeof(cfgReq.tbName) - 1); + tstrncpy(cfgReq.dbFName, pInput->dbFName, TSDB_DB_FNAME_LEN); + tstrncpy(cfgReq.tbName, pInput->tbName, TSDB_TABLE_NAME_LEN); int32_t bufLen = tSerializeSTableCfgReq(NULL, 0, &cfgReq); void *pBuf = (*mallcFp)(bufLen); @@ -367,7 +367,7 @@ int32_t queryBuildGetViewMetaMsg(void *input, char **msg, int32_t msgSize, int32 } SViewMetaReq req = {0}; - strncpy(req.fullname, input, sizeof(req.fullname) - 1); + tstrncpy(req.fullname, input, TSDB_VIEW_FNAME_LEN); int32_t bufLen = tSerializeSViewMetaReq(NULL, 0, &req); void *pBuf = (*mallcFp)(bufLen); @@ -392,7 +392,7 @@ int32_t queryBuildGetTableTSMAMsg(void *input, char **msg, int32_t msgSize, int3 } STableTSMAInfoReq req = {0}; - strncpy(req.name, input, sizeof(req.name) - 1); + tstrncpy(req.name, input, TSDB_TABLE_FNAME_LEN); int32_t bufLen = tSerializeTableTSMAInfoReq(NULL, 0, &req); void * pBuf = (*mallcFp)(bufLen); @@ -417,7 +417,7 @@ int32_t queryBuildGetTSMAMsg(void *input, char **msg, int32_t msgSize, int32_t * STableTSMAInfoReq req = {0}; req.fetchingWithTsmaName = true; - strncpy(req.name, input, sizeof(req.name) - 1); + tstrncpy(req.name, input, TSDB_TABLE_FNAME_LEN); int32_t bufLen = tSerializeTableTSMAInfoReq(NULL, 0, &req); void * pBuf = (*mallcFp)(bufLen); @@ -688,14 +688,14 @@ int32_t queryProcessTableMetaRsp(void *output, char *msg, int32_t msgSize) { } STableMetaOutput *pOut = output; - strcpy(pOut->dbFName, metaRsp.dbFName); + tstrncpy(pOut->dbFName, metaRsp.dbFName, TSDB_DB_FNAME_LEN); pOut->dbId = metaRsp.dbId; if (metaRsp.tableType == TSDB_CHILD_TABLE) { SET_META_TYPE_BOTH_TABLE(pOut->metaType); - strcpy(pOut->ctbName, metaRsp.tbName); - strcpy(pOut->tbName, metaRsp.stbName); + tstrncpy(pOut->ctbName, metaRsp.tbName, TSDB_TABLE_NAME_LEN); + tstrncpy(pOut->tbName, metaRsp.stbName, TSDB_TABLE_NAME_LEN); pOut->ctbMeta.vgId = metaRsp.vgId; pOut->ctbMeta.tableType = metaRsp.tableType; @@ -705,7 +705,7 @@ int32_t queryProcessTableMetaRsp(void *output, char *msg, int32_t msgSize) { code = queryCreateTableMetaFromMsg(&metaRsp, true, &pOut->tbMeta); } else { SET_META_TYPE_TABLE(pOut->metaType); - strcpy(pOut->tbName, metaRsp.tbName); + tstrncpy(pOut->tbName, metaRsp.tbName, TSDB_TABLE_NAME_LEN); code = queryCreateTableMetaFromMsg(&metaRsp, (metaRsp.tableType == TSDB_SUPER_TABLE), &pOut->tbMeta); } @@ -744,14 +744,14 @@ static int32_t queryProcessTableNameRsp(void *output, char *msg, int32_t msgSize } STableMetaOutput *pOut = output; - strcpy(pOut->dbFName, metaRsp.dbFName); + tstrncpy(pOut->dbFName, metaRsp.dbFName, TSDB_DB_FNAME_LEN); pOut->dbId = metaRsp.dbId; if (metaRsp.tableType == TSDB_CHILD_TABLE) { SET_META_TYPE_BOTH_TABLE(pOut->metaType); - strcpy(pOut->ctbName, metaRsp.tbName); - strcpy(pOut->tbName, metaRsp.stbName); + tstrncpy(pOut->ctbName, metaRsp.tbName, TSDB_TABLE_NAME_LEN); + tstrncpy(pOut->tbName, metaRsp.stbName, TSDB_TABLE_NAME_LEN); pOut->ctbMeta.vgId = metaRsp.vgId; pOut->ctbMeta.tableType = metaRsp.tableType; @@ -761,7 +761,7 @@ static int32_t queryProcessTableNameRsp(void *output, char *msg, int32_t msgSize code = queryCreateTableMetaExFromMsg(&metaRsp, true, &pOut->tbMeta); } else { SET_META_TYPE_TABLE(pOut->metaType); - strcpy(pOut->tbName, metaRsp.tbName); + tstrncpy(pOut->tbName, metaRsp.tbName, TSDB_TABLE_NAME_LEN); code = queryCreateTableMetaExFromMsg(&metaRsp, (metaRsp.tableType == TSDB_SUPER_TABLE), &pOut->tbMeta); } diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index d6266afa02..bebb9b288a 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -539,7 +539,7 @@ int32_t qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx) { } if (dbFName[0] && tbName[0]) { - (void)sprintf(tbInfo.tbFName, "%s.%s", dbFName, tbName); + (void)snprintf(tbInfo.tbFName, sizeof(tbInfo.tbFName), "%s.%s", dbFName, tbName); } else { tbInfo.tbFName[0] = 0; } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index a9a765c0fa..e07ef69990 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1764,41 +1764,41 @@ _return: return DEAL_RES_ERROR; } -int32_t fltConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) { +int32_t fltConverToStr(char *str, int32_t strMaxLen, int type, void *buf, int32_t bufSize, int32_t *len) { int32_t n = 0; switch (type) { case TSDB_DATA_TYPE_NULL: - n = sprintf(str, "null"); + n = snprintf(str, strMaxLen, "null"); break; case TSDB_DATA_TYPE_BOOL: - n = sprintf(str, (*(int8_t *)buf) ? "true" : "false"); + n = snprintf(str, strMaxLen, (*(int8_t *)buf) ? "true" : "false"); break; case TSDB_DATA_TYPE_TINYINT: - n = sprintf(str, "%d", *(int8_t *)buf); + n = snprintf(str, strMaxLen, "%d", *(int8_t *)buf); break; case TSDB_DATA_TYPE_SMALLINT: - n = sprintf(str, "%d", *(int16_t *)buf); + n = snprintf(str, strMaxLen, "%d", *(int16_t *)buf); break; case TSDB_DATA_TYPE_INT: - n = sprintf(str, "%d", *(int32_t *)buf); + n = snprintf(str, strMaxLen, "%d", *(int32_t *)buf); break; case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: - n = sprintf(str, "%" PRId64, *(int64_t *)buf); + n = snprintf(str, strMaxLen, "%" PRId64, *(int64_t *)buf); break; case TSDB_DATA_TYPE_FLOAT: - n = sprintf(str, "%e", GET_FLOAT_VAL(buf)); + n = snprintf(str, strMaxLen, "%e", GET_FLOAT_VAL(buf)); break; case TSDB_DATA_TYPE_DOUBLE: - n = sprintf(str, "%e", GET_DOUBLE_VAL(buf)); + n = snprintf(str, strMaxLen, "%e", GET_DOUBLE_VAL(buf)); break; case TSDB_DATA_TYPE_BINARY: @@ -1817,19 +1817,19 @@ int32_t fltConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t break; case TSDB_DATA_TYPE_UTINYINT: - n = sprintf(str, "%d", *(uint8_t *)buf); + n = snprintf(str, strMaxLen, "%d", *(uint8_t *)buf); break; case TSDB_DATA_TYPE_USMALLINT: - n = sprintf(str, "%d", *(uint16_t *)buf); + n = snprintf(str, strMaxLen, "%d", *(uint16_t *)buf); break; case TSDB_DATA_TYPE_UINT: - n = sprintf(str, "%u", *(uint32_t *)buf); + n = snprintf(str, strMaxLen, "%u", *(uint32_t *)buf); break; case TSDB_DATA_TYPE_UBIGINT: - n = sprintf(str, "%" PRIu64, *(uint64_t *)buf); + n = snprintf(str, strMaxLen, "%" PRIu64, *(uint64_t *)buf); break; default: @@ -1886,8 +1886,8 @@ int32_t filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t optio SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); SColumnNode *refNode = (SColumnNode *)left->desc; if (unit->compare.optr <= OP_TYPE_JSON_CONTAINS) { - len = sprintf(str, "UNIT[%d] => [%d][%d] %s [", i, refNode->dataBlockId, refNode->slotId, - operatorTypeStr(unit->compare.optr)); + len += snprintf(str, sizeof(str), "UNIT[%d] => [%d][%d] %s [", i, refNode->dataBlockId, refNode->slotId, + operatorTypeStr(unit->compare.optr)); } if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != OP_TYPE_IN) { @@ -1898,18 +1898,22 @@ int32_t filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t optio data += VARSTR_HEADER_SIZE; } if (data) { - FLT_ERR_RET(fltConverToStr(str + len, type, data, tlen > 32 ? 32 : tlen, &tlen)); + FLT_ERR_RET(fltConverToStr(str + len, sizeof(str) - len, type, data, tlen > 32 ? 32 : tlen, &tlen)); + len += tlen; } } else { - (void)strcat(str, "NULL"); + (void)strncat(str, "NULL", sizeof(str) - len - 1); + len += 4; } - (void)strcat(str, "]"); + (void)strncat(str, "]", sizeof(str) - len - 1); + len += 1; if (unit->compare.optr2) { - (void)strcat(str, " && "); + (void)strncat(str, " && ", sizeof(str) - len - 1); + len += 4; if (unit->compare.optr2 <= OP_TYPE_JSON_CONTAINS) { - (void)sprintf(str + strlen(str), "[%d][%d] %s [", refNode->dataBlockId, refNode->slotId, - operatorTypeStr(unit->compare.optr2)); + len += snprintf(str + len, sizeof(str) - len, "[%d][%d] %s [", refNode->dataBlockId, + refNode->slotId, operatorTypeStr(unit->compare.optr2)); } if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != OP_TYPE_IN) { @@ -1919,11 +1923,14 @@ int32_t filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t optio tlen = varDataLen(data); data += VARSTR_HEADER_SIZE; } - FLT_ERR_RET(fltConverToStr(str + strlen(str), type, data, tlen > 32 ? 32 : tlen, &tlen)); + FLT_ERR_RET(fltConverToStr(str + len, sizeof(str) - len, type, data, tlen > 32 ? 32 : tlen, &tlen)); + len += tlen; } else { - (void)strcat(str, "NULL"); + (void)strncat(str, "NULL", sizeof(str) - len - 1); + len += 4; } - (void)strcat(str, "]"); + (void)strncat(str, "]", sizeof(str) - len - 1); + len += 1; } qDebug("%s", str); // TODO @@ -1955,21 +1962,39 @@ int32_t filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t optio SFilterRangeNode *r = ctx->rs; int32_t tlen = 0; while (r) { - char str[256] = {0}; + char str[256] = {0}; + int32_t len = 0; if (FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) { - (void)strcat(str, "(NULL)"); + (void)strncat(str, "(NULL)", sizeof(str) - len - 1); + len += 6; } else { - FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str, "(") : strcat(str, "["); - FLT_ERR_RET(fltConverToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen)); - FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str, ")") : strcat(str, "]"); + FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? + (void)strncat(str, "(", sizeof(str) - len - 1) : + (void)strncat(str, "[", sizeof(str) - len - 1); + len += 1; + FLT_ERR_RET(fltConverToStr(str + len, sizeof(str) - len, ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen)); + len += tlen; + FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? + (void)strncat(str, ")", sizeof(str) - len - 1) : + (void)strncat(str, "]", sizeof(str) - len - 1); + len += 1; } - (void)strcat(str, " - "); + (void)strncat(str, " - ", sizeof(str) - len - 1); + len += 3; if (FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL)) { - (void)strcat(str, "(NULL)"); + (void)strncat(str, "(NULL)", sizeof(str) - len - 1); + len += 6; } else { - FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str, "(") : strcat(str, "["); - FLT_ERR_RET(fltConverToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen)); - FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str, ")") : strcat(str, "]"); + FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? + (void)strncat(str, "(", sizeof(str) - len - 1) : + (void)strncat(str, "[", sizeof(str) - len - 1); + len += 1; + FLT_ERR_RET(fltConverToStr(str + len, sizeof(str) - len, ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen)); + len += tlen; + FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? + (void)strncat(str, ")", sizeof(str) - len - 1) : + (void)strncat(str, "]", sizeof(str) - len - 1); + len += 1; } qDebug("range: %s", str); @@ -3809,13 +3834,13 @@ int32_t fltInitFromNode(SNode *tree, SFilterInfo *info, uint32_t options) { SFltBuildGroupCtx tctx = {.info = info, .group = group}; nodesWalkExpr(tree, fltTreeToGroup, (void *)&tctx); if (TSDB_CODE_SUCCESS != tctx.code) { - taosArrayDestroy(group); + taosArrayDestroyEx(group, filterFreeGroup); code = tctx.code; goto _return; } code = filterConvertGroupFromArray(info, group); if (TSDB_CODE_SUCCESS != code) { - taosArrayDestroy(group); + taosArrayDestroyEx(group, filterFreeGroup); goto _return; } taosArrayDestroy(group); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 4f08c93c1e..209110b014 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -628,8 +628,8 @@ int32_t sclWalkCaseWhenList(SScalarCtx *ctx, SNodeList *pList, struct SListCell cell = cell->pNext) { pWhenThen = (SWhenThenNode *)node; - SCL_ERR_RET(sclGetNodeRes(pWhenThen->pWhen, ctx, &pWhen)); - SCL_ERR_RET(sclGetNodeRes(pWhenThen->pThen, ctx, &pThen)); + SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pWhen, ctx, &pWhen)); + SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pThen, ctx, &pThen)); SCL_ERR_JRET(vectorCompareImpl(pCase, pWhen, pComp, rowIdx, 1, TSDB_ORDER_ASC, OP_TYPE_EQUAL)); @@ -646,6 +646,10 @@ int32_t sclWalkCaseWhenList(SScalarCtx *ctx, SNodeList *pList, struct SListCell goto _return; } + sclFreeParam(pWhen); + sclFreeParam(pThen); + taosMemoryFreeClear(pWhen); + taosMemoryFreeClear(pThen); } if (pElse) { @@ -672,8 +676,8 @@ _return: sclFreeParam(pWhen); sclFreeParam(pThen); - taosMemoryFree(pWhen); - taosMemoryFree(pThen); + taosMemoryFreeClear(pWhen); + taosMemoryFreeClear(pThen); SCL_RET(code); } @@ -1207,7 +1211,7 @@ EDealRes sclRewriteFunction(SNode **pNode, SScalarCtx *ctx) { res->translate = true; - (void)strcpy(res->node.aliasName, node->node.aliasName); + tstrncpy(res->node.aliasName, node->node.aliasName, TSDB_COL_NAME_LEN); res->node.resType.type = output.columnData->info.type; res->node.resType.bytes = output.columnData->info.bytes; res->node.resType.scale = output.columnData->info.scale; @@ -1282,7 +1286,7 @@ EDealRes sclRewriteLogic(SNode **pNode, SScalarCtx *ctx) { res->node.resType = node->node.resType; res->translate = true; - (void)strcpy(res->node.aliasName, node->node.aliasName); + tstrncpy(res->node.aliasName, node->node.aliasName, TSDB_COL_NAME_LEN); int32_t type = output.columnData->info.type; if (IS_VAR_DATA_TYPE(type)) { res->datum.p = output.columnData->pData; @@ -1352,7 +1356,7 @@ EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) { res->translate = true; - (void)strcpy(res->node.aliasName, node->node.aliasName); + tstrncpy(res->node.aliasName, node->node.aliasName, TSDB_COL_NAME_LEN); res->node.resType = node->node.resType; if (colDataIsNull_s(output.columnData, 0)) { res->isNull = true; @@ -1415,7 +1419,7 @@ EDealRes sclRewriteCaseWhen(SNode **pNode, SScalarCtx *ctx) { res->translate = true; - (void)strcpy(res->node.aliasName, node->node.aliasName); + tstrncpy(res->node.aliasName, node->node.aliasName, TSDB_COL_NAME_LEN); res->node.resType = node->node.resType; if (colDataIsNull_s(output.columnData, 0)) { res->isNull = true; diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 377009a07f..341ce760f5 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2067,9 +2067,9 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_GEOMETRY: { if (inputType == TSDB_DATA_TYPE_BOOL) { - // NOTE: sprintf will append '\0' at the end of string - int32_t len = sprintf(varDataVal(output), "%.*s", (int32_t)(outputLen - VARSTR_HEADER_SIZE), - *(int8_t *)input ? "true" : "false"); + // NOTE: snprintf will append '\0' at the end of string + int32_t len = snprintf(varDataVal(output), outputLen + TSDB_NCHAR_SIZE - VARSTR_HEADER_SIZE, "%.*s", + (int32_t)(outputLen - VARSTR_HEADER_SIZE), *(int8_t *)input ? "true" : "false"); varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_BINARY) { int32_t len = TMIN(varDataLen(input), outputLen - VARSTR_HEADER_SIZE); @@ -2109,7 +2109,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int32_t len; if (inputType == TSDB_DATA_TYPE_BOOL) { char tmp[8] = {0}; - len = sprintf(tmp, "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false"); + len = snprintf(tmp, sizeof(tmp), "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false"); bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); if (!ret) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; @@ -2193,7 +2193,7 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * NUM_TO_STRING(type, input, sizeof(fraction), fraction); int32_t fractionLen; - char buf[64] = {0}; + char buf[TD_TIME_STR_LEN] = {0}; int64_t timeVal; char* format = NULL; int64_t quot = 0; @@ -2244,14 +2244,14 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * struct tm tmInfo; int32_t len = 0; - if (taosLocalTime((const time_t *)", &tmInfo, buf) == NULL) { + if (taosLocalTime((const time_t *)", &tmInfo, buf, sizeof(buf)) == NULL) { len = (int32_t)strlen(buf); goto _end; } len = (int32_t)strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &tmInfo); - len += snprintf(buf + len, fractionLen, format, mod); + len += tsnprintf(buf + len, fractionLen, format, mod); // add timezone string if (tzLen > 0) { @@ -3006,20 +3006,56 @@ static int32_t doScalarFunction2(SScalarParam *pInput, int32_t inputNum, SScalar } break; } - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT:{ + case TSDB_DATA_TYPE_TINYINT: { + int8_t *in = (int8_t *)pInputData[0]->pData; + int8_t *out = (int8_t *)pOutputData->pData; + int8_t result = (int8_t)d1((double)in[i], in2); + out[i] = result; + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + int16_t *in = (int16_t *)pInputData[0]->pData; + int16_t *out = (int16_t *)pOutputData->pData; + int16_t result = (int16_t)d1((double)in[i], in2); + out[i] = result; + break; + } + case TSDB_DATA_TYPE_INT: { + int32_t *in = (int32_t *)pInputData[0]->pData; + int32_t *out = (int32_t *)pOutputData->pData; + int32_t result = (int32_t)d1((double)in[i], in2); + out[i] = result; + break; + } + case TSDB_DATA_TYPE_BIGINT: { int64_t *in = (int64_t *)pInputData[0]->pData; int64_t *out = (int64_t *)pOutputData->pData; int64_t result = (int64_t)d1((double)in[i], in2); out[i] = result; break; } - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_UBIGINT:{ + case TSDB_DATA_TYPE_UTINYINT: { + uint8_t *in = (uint8_t *)pInputData[0]->pData; + uint8_t *out = (uint8_t *)pOutputData->pData; + uint8_t result = (uint8_t)d1((double)in[i], in2); + out[i] = result; + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + uint16_t *in = (uint16_t *)pInputData[0]->pData; + uint16_t *out = (uint16_t *)pOutputData->pData; + uint16_t result = (uint16_t)d1((double)in[i], in2); + out[i] = result; + break; + } + case TSDB_DATA_TYPE_UINT: { + uint32_t *in = (uint32_t *)pInputData[0]->pData; + uint32_t *out = (uint32_t *)pOutputData->pData; + uint32_t result = (uint32_t)d1((double)in[i], in2); + out[i] = result; + break; + } + case TSDB_DATA_TYPE_UBIGINT: { uint64_t *in = (uint64_t *)pInputData[0]->pData; uint64_t *out = (uint64_t *)pOutputData->pData; uint64_t result = (uint64_t)d1((double)in[i], in2); @@ -3062,20 +3098,56 @@ static int32_t doScalarFunction2(SScalarParam *pInput, int32_t inputNum, SScalar } break; } - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT:{ + case TSDB_DATA_TYPE_TINYINT: { + int8_t *in = (int8_t *)pInputData[0]->pData; + int8_t *out = (int8_t *)pOutputData->pData; + int8_t result = (int8_t)d1((double)in[0], in2); + out[i] = result; + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + int16_t *in = (int16_t *)pInputData[0]->pData; + int16_t *out = (int16_t *)pOutputData->pData; + int16_t result = (int16_t)d1((double)in[0], in2); + out[i] = result; + break; + } + case TSDB_DATA_TYPE_INT: { + int32_t *in = (int32_t *)pInputData[0]->pData; + int32_t *out = (int32_t *)pOutputData->pData; + int32_t result = (int32_t)d1((double)in[0], in2); + out[i] = result; + break; + } + case TSDB_DATA_TYPE_BIGINT: { int64_t *in = (int64_t *)pInputData[0]->pData; int64_t *out = (int64_t *)pOutputData->pData; int64_t result = (int64_t)d1((double)in[0], in2); out[i] = result; break; } - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_UBIGINT:{ + case TSDB_DATA_TYPE_UTINYINT: { + uint8_t *in = (uint8_t *)pInputData[0]->pData; + uint8_t *out = (uint8_t *)pOutputData->pData; + uint8_t result = (uint8_t)d1((double)in[0], in2); + out[i] = result; + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + uint16_t *in = (uint16_t *)pInputData[0]->pData; + uint16_t *out = (uint16_t *)pOutputData->pData; + uint16_t result = (uint16_t)d1((double)in[0], in2); + out[i] = result; + break; + } + case TSDB_DATA_TYPE_UINT: { + uint32_t *in = (uint32_t *)pInputData[0]->pData; + uint32_t *out = (uint32_t *)pOutputData->pData; + uint32_t result = (uint32_t)d1((double)in[0], in2); + out[i] = result; + break; + } + case TSDB_DATA_TYPE_UBIGINT: { uint64_t *in = (uint64_t *)pInputData[0]->pData; uint64_t *out = (uint64_t *)pOutputData->pData; uint64_t result = (uint64_t)d1((double)in[0], in2); @@ -3119,20 +3191,56 @@ static int32_t doScalarFunction2(SScalarParam *pInput, int32_t inputNum, SScalar } break; } - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT:{ + case TSDB_DATA_TYPE_TINYINT: { + int8_t *in = (int8_t *)pInputData[0]->pData; + int8_t *out = (int8_t *)pOutputData->pData; + int8_t result = (int8_t)d1((double)in[i], in2); + out[i] = result; + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + int16_t *in = (int16_t *)pInputData[0]->pData; + int16_t *out = (int16_t *)pOutputData->pData; + int16_t result = (int16_t)d1((double)in[i], in2); + out[i] = result; + break; + } + case TSDB_DATA_TYPE_INT: { + int32_t *in = (int32_t *)pInputData[0]->pData; + int32_t *out = (int32_t *)pOutputData->pData; + int32_t result = (int32_t)d1((double)in[i], in2); + out[i] = result; + break; + } + case TSDB_DATA_TYPE_BIGINT: { int64_t *in = (int64_t *)pInputData[0]->pData; int64_t *out = (int64_t *)pOutputData->pData; int64_t result = (int64_t)d1((double)in[i], in2); out[i] = result; break; } - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_UBIGINT:{ + case TSDB_DATA_TYPE_UTINYINT: { + uint8_t *in = (uint8_t *)pInputData[0]->pData; + uint8_t *out = (uint8_t *)pOutputData->pData; + uint8_t result = (uint8_t)d1((double)in[i], in2); + out[i] = result; + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + uint16_t *in = (uint16_t *)pInputData[0]->pData; + uint16_t *out = (uint16_t *)pOutputData->pData; + uint16_t result = (uint16_t)d1((double)in[i], in2); + out[i] = result; + break; + } + case TSDB_DATA_TYPE_UINT: { + uint32_t *in = (uint32_t *)pInputData[0]->pData; + uint32_t *out = (uint32_t *)pOutputData->pData; + uint32_t result = (uint32_t)d1((double)in[i], in2); + out[i] = result; + break; + } + case TSDB_DATA_TYPE_UBIGINT: { uint64_t *in = (uint64_t *)pInputData[0]->pData; uint64_t *out = (uint64_t *)pOutputData->pData; uint64_t result = (uint64_t)d1((double)in[i], in2); @@ -3844,11 +3952,11 @@ int32_t leastSQRScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPa char buf[LEASTSQUARES_BUFF_LENGTH] = {0}; char slopBuf[64] = {0}; char interceptBuf[64] = {0}; - int n = snprintf(slopBuf, 64, "%.6lf", matrix02); + int n = tsnprintf(slopBuf, 64, "%.6lf", matrix02); if (n > LEASTSQUARES_DOUBLE_ITEM_LENGTH) { (void)snprintf(slopBuf, 64, "%." DOUBLE_PRECISION_DIGITS, matrix02); } - n = snprintf(interceptBuf, 64, "%.6lf", matrix12); + n = tsnprintf(interceptBuf, 64, "%.6lf", matrix12); if (n > LEASTSQUARES_DOUBLE_ITEM_LENGTH) { (void) snprintf(interceptBuf, 64, "%." DOUBLE_PRECISION_DIGITS, matrix12); } @@ -4407,11 +4515,11 @@ int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP int32_t len; char buf[512] = {0}; if (!normalized) { - len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%" PRId64 "}", bins[k].lower, - bins[k].upper, bins[k].count); + len = snprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%" PRId64 "}", + bins[k].lower, bins[k].upper, bins[k].count); } else { - len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}", bins[k].lower, - bins[k].upper, bins[k].percentage); + len = snprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}", + bins[k].lower, bins[k].upper, bins[k].percentage); } varDataSetLen(buf, len); SCL_ERR_JRET(colDataSetVal(pOutputData, k, buf, false)); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 230454483d..a7c842172a 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -734,7 +734,7 @@ int32_t vectorConvertToVarData(SSclVectorConvCtx *pCtx) { int64_t value = 0; GET_TYPED_DATA(value, int64_t, pCtx->inType, colDataGetData(pInputCol, i)); - int32_t len = sprintf(varDataVal(tmp), "%" PRId64, value); + int32_t len = snprintf(varDataVal(tmp), sizeof(tmp) - VARSTR_HEADER_SIZE, "%" PRId64, value); varDataLen(tmp) = len; if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { SCL_ERR_RET(varToNchar(tmp, pCtx->pOut, i, NULL)); @@ -751,7 +751,7 @@ int32_t vectorConvertToVarData(SSclVectorConvCtx *pCtx) { uint64_t value = 0; GET_TYPED_DATA(value, uint64_t, pCtx->inType, colDataGetData(pInputCol, i)); - int32_t len = sprintf(varDataVal(tmp), "%" PRIu64, value); + int32_t len = snprintf(varDataVal(tmp), sizeof(tmp) - VARSTR_HEADER_SIZE, "%" PRIu64, value); varDataLen(tmp) = len; if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { SCL_ERR_RET(varToNchar(tmp, pCtx->pOut, i, NULL)); @@ -768,7 +768,7 @@ int32_t vectorConvertToVarData(SSclVectorConvCtx *pCtx) { double value = 0; GET_TYPED_DATA(value, double, pCtx->inType, colDataGetData(pInputCol, i)); - int32_t len = sprintf(varDataVal(tmp), "%lf", value); + int32_t len = snprintf(varDataVal(tmp), sizeof(tmp) - VARSTR_HEADER_SIZE, "%lf", value); varDataLen(tmp) = len; if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { SCL_ERR_RET(varToNchar(tmp, pCtx->pOut, i, NULL)); diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 70d6f7d0ae..8bbadd0e22 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -55,7 +55,7 @@ void flttInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - (void)strcpy(tsLogDir, TD_LOG_DIR_PATH); + tstrncpy(tsLogDir, TD_LOG_DIR_PATH, PATH_MAX); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); @@ -101,7 +101,7 @@ int32_t flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, rnode->node.resType.bytes = dataBytes; rnode->dataBlockId = 0; - sprintf(rnode->dbName, "%" PRIu64, dbidx++); + snprintf(rnode->dbName, TSDB_DB_NAME_LEN, "%" PRIu64, dbidx++); if (NULL == block) { rnode->slotId = 2; @@ -666,7 +666,7 @@ TEST(columnTest, binary_column_like_binary) { int32_t rowNum = sizeof(leftv) / sizeof(leftv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); - sprintf(&rightv[2], "%s", "__0"); + snprintf(&rightv[2], sizeof(rightv) - 2, "%s", "__0"); varDataSetLen(rightv, strlen(&rightv[2])); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); flttMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index e14b772ea8..4cab644582 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -81,7 +81,7 @@ void scltInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - (void)strcpy(tsLogDir, TD_LOG_DIR_PATH); + tstrncpy(tsLogDir, TD_LOG_DIR_PATH, PATH_MAX); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { (void)printf("failed to open log file in directory:%s\n", tsLogDir); diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index f475c974cc..03145da939 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -728,7 +728,7 @@ void schFreeJobImpl(void *job) { uint64_t queryId = pJob->queryId; int64_t refId = pJob->refId; - qDebug("qid:0x%" PRIx64 " begin to free sch job, refId:0x%" PRIx64 ", pointer:%p", queryId, refId, pJob); + qDebug("QID:0x%" PRIx64 " begin to free sch job, refId:0x%" PRIx64 ", pointer:%p", queryId, refId, pJob); schDropJobAllTasks(pJob); @@ -775,7 +775,7 @@ void schFreeJobImpl(void *job) { taosMemoryFreeClear(pJob->userRes.execRes); taosMemoryFreeClear(pJob->fetchRes); taosMemoryFreeClear(pJob->sql); - int32_t code = tsem_destroy(&pJob->rspSem); + int32_t code = tsem_destroy(&pJob->rspSem); if (code) { qError("tsem_destroy failed, error:%s", tstrerror(code)); } @@ -786,7 +786,7 @@ void schFreeJobImpl(void *job) { schCloseJobRef(); } - qDebug("qid:0x%" PRIx64 " sch job freed, refId:0x%" PRIx64 ", pointer:%p", queryId, refId, pJob); + qDebug("QID:0x%" PRIx64 " sch job freed, refId:0x%" PRIx64 ", pointer:%p", queryId, refId, pJob); } int32_t schJobFetchRows(SSchJob *pJob) { @@ -797,7 +797,7 @@ int32_t schJobFetchRows(SSchJob *pJob) { if (schChkCurrentOp(pJob, SCH_OP_FETCH, true)) { SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); - code = tsem_wait(&pJob->rspSem); + code = tsem_wait(&pJob->rspSem); if (code) { qError("tsem_wait for fetch rspSem failed, error:%s", tstrerror(code)); SCH_RET(code); @@ -821,7 +821,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { int64_t refId = -1; SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); if (NULL == pJob) { - qError("qid:0x%" PRIx64 " calloc %d failed", pReq->pDag->queryId, (int32_t)sizeof(SSchJob)); + qError("QID:0x%" PRIx64 " calloc %d failed", pReq->pDag->queryId, (int32_t)sizeof(SSchJob)); SCH_ERR_JRET(terrno); } @@ -831,7 +831,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { if (pReq->sql) { pJob->sql = taosStrdup(pReq->sql); if (NULL == pJob->sql) { - qError("qid:0x%" PRIx64 " strdup sql %s failed", pReq->pDag->queryId, pReq->sql); + qError("QID:0x%" PRIx64 " strdup sql %s failed", pReq->pDag->queryId, pReq->sql); SCH_ERR_JRET(terrno); } } @@ -839,7 +839,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { if (pReq->allocatorRefId > 0) { pJob->allocatorRefId = nodesMakeAllocatorWeakRef(pReq->allocatorRefId); if (pJob->allocatorRefId <= 0) { - qError("qid:0x%" PRIx64 " nodesMakeAllocatorWeakRef failed", pReq->pDag->queryId); + qError("QID:0x%" PRIx64 " nodesMakeAllocatorWeakRef failed", pReq->pDag->queryId); SCH_ERR_JRET(terrno); } } @@ -851,11 +851,11 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { pJob->pWorkerCb = pReq->pWorkerCb; if (pReq->pNodeList == NULL || taosArrayGetSize(pReq->pNodeList) <= 0) { - qDebug("qid:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId); + qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId); } else { pJob->nodeList = taosArrayDup(pReq->pNodeList, NULL); if (NULL == pJob->nodeList) { - qError("qid:0x%" PRIx64 " taosArrayDup failed, origNum:%d", pReq->pDag->queryId, + qError("QID:0x%" PRIx64 " taosArrayDup failed, origNum:%d", pReq->pDag->queryId, (int32_t)taosArrayGetSize(pReq->pNodeList)); SCH_ERR_JRET(terrno); } @@ -918,7 +918,7 @@ _return: int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq) { int32_t code = 0; - qDebug("qid:0x%" PRIx64 " sch job refId 0x%" PRIx64 " started", pReq->pDag->queryId, pJob->refId); + qDebug("QID:0x%" PRIx64 " sch job refId 0x%" PRIx64 " started", pReq->pDag->queryId, pJob->refId); SCH_ERR_RET(schLaunchJob(pJob)); @@ -926,7 +926,7 @@ int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq) { SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); code = tsem_wait(&pJob->rspSem); if (code) { - qError("qid:0x%" PRIx64 " tsem_wait sync rspSem failed, error:%s", pReq->pDag->queryId, tstrerror(code)); + qError("QID:0x%" PRIx64 " tsem_wait sync rspSem failed, error:%s", pReq->pDag->queryId, tstrerror(code)); SCH_ERR_RET(code); } } @@ -1191,7 +1191,7 @@ int32_t schProcessOnCbBegin(SSchJob **job, SSchTask **task, uint64_t qId, int64_ (void)schAcquireJob(rId, &pJob); if (NULL == pJob) { - qWarn("qid:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, qId, tId, rId); + qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, qId, tId, rId); SCH_ERR_RET(TSDB_CODE_QRY_JOB_NOT_EXIST); } diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index fe601c6b86..a8e747ccd2 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -500,7 +500,7 @@ _return: int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, code); // called if drop task rsp received code (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error @@ -513,7 +513,7 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { int32_t schHandleNotifyCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 " task notify rsp received, code:0x%x", pParam->queryId, pParam->taskId, + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " task notify rsp received, code:0x%x", pParam->queryId, pParam->taskId, code); if (pMsg) { taosMemoryFree(pMsg->pData); @@ -729,7 +729,7 @@ int32_t schMakeHbCallbackParam(SSchJob *pJob, SSchTask *pTask, void **pParam) { param->nodeEpId.nodeId = addr->nodeId; SEp* pEp = SCH_GET_CUR_EP(addr); - strcpy(param->nodeEpId.ep.fqdn, pEp->fqdn); + tstrncpy(param->nodeEpId.ep.fqdn, pEp->fqdn, sizeof(param->nodeEpId.ep.fqdn)); param->nodeEpId.ep.port = pEp->port; param->pTrans = pJob->pTrans; diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 375ad5fa37..e6b68051f9 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -288,7 +288,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { /* if (SCH_IS_DATA_SRC_TASK(task) && job->dataSrcEps.numOfEps < SCH_MAX_CANDIDATE_EP_NUM) { - strncpy(job->dataSrcEps.fqdn[job->dataSrcEps.numOfEps], task->execAddr.fqdn, sizeof(task->execAddr.fqdn)); + tstrncpy(job->dataSrcEps.fqdn[job->dataSrcEps.numOfEps], task->execAddr.fqdn, sizeof(task->execAddr.fqdn)); job->dataSrcEps.port[job->dataSrcEps.numOfEps] = task->execAddr.port; ++job->dataSrcEps.numOfEps; @@ -840,7 +840,7 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { /* for (int32_t i = 0; i < job->dataSrcEps.numOfEps && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { - strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i])); + tstrncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i])); epSet->port[epSet->numOfEps] = job->dataSrcEps.port[i]; ++epSet->numOfEps; @@ -996,7 +996,7 @@ int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList) { int32_t code = 0; - qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s", pStatus->queryId, pStatus->taskId, + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s", pStatus->queryId, pStatus->taskId, pStatus->execId, jobTaskStatusStr(pStatus->status)); if (schProcessOnCbBegin(&pJob, &pTask, pStatus->queryId, pStatus->refId, pStatus->taskId)) { @@ -1043,12 +1043,12 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { continue; } - qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg", localRsp->qId, localRsp->tId); + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg", localRsp->qId, localRsp->tId); pJob = NULL; (void)schAcquireJob(localRsp->rId, &pJob); if (NULL == pJob) { - qWarn("qid:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, localRsp->qId, + qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, localRsp->qId, localRsp->tId, localRsp->rId); SCH_ERR_JRET(TSDB_CODE_QRY_JOB_NOT_EXIST); } @@ -1068,7 +1068,7 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { (void)schReleaseJob(pJob->refId); - qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", localRsp->qId, + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", localRsp->qId, localRsp->tId, code); SCH_ERR_JRET(code); diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 14447ba856..b68f665200 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -63,10 +63,10 @@ int32_t schDumpEpSet(SEpSet *pEpSet, char** ppRes) { } int32_t n = 0; - n += snprintf(str + n, maxSize - n, "numOfEps:%d, inUse:%d eps:", pEpSet->numOfEps, pEpSet->inUse); + n += tsnprintf(str + n, maxSize - n, "numOfEps:%d, inUse:%d eps:", pEpSet->numOfEps, pEpSet->inUse); for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { SEp *pEp = &pEpSet->eps[i]; - n += snprintf(str + n, maxSize - n, "[%s:%d]", pEp->fqdn, pEp->port); + n += tsnprintf(str + n, maxSize - n, "[%s:%d]", pEp->fqdn, pEp->port); } *ppRes = str; @@ -297,16 +297,13 @@ uint64_t schGenTaskId(void) { return atomic_add_fetch_64(&schMgmt.taskId, 1); } #ifdef BUILD_NO_CALL uint64_t schGenUUID(void) { - static uint64_t hashId = 0; + static uint32_t hashId = 0; static int32_t requestSerialId = 0; if (hashId == 0) { - char uid[64] = {0}; - int32_t code = taosGetSystemUUID(uid, tListLen(uid) - 1); + int32_t code = taosGetSystemUUID32(&hashId); if (code != TSDB_CODE_SUCCESS) { qError("Failed to get the system uid, reason:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); - } else { - hashId = MurmurHash3_32(uid, strlen(uid)); } } @@ -314,7 +311,7 @@ uint64_t schGenUUID(void) { uint64_t pid = taosGetPId(); int32_t val = atomic_add_fetch_32(&requestSerialId, 1); - uint64_t id = ((hashId & 0x0FFF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); + uint64_t id = ((uint64_t)((hashId & 0x0FFF)) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); return id; } #endif diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 0c74e99a29..091de5c048 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -56,7 +56,7 @@ int32_t schedulerInit() { SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - if (taosGetSystemUUID((char *)&schMgmt.sId, sizeof(schMgmt.sId))) { + if (taosGetSystemUUIDU64(&schMgmt.sId)) { qError("generate schedulerId failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_SYS_ERROR); } diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index a2c9012df5..c88971ab75 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -4928,7 +4928,7 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) { } char content[256] = {0}; - nBytes = snprintf(content, sizeof(content), META_ON_S3_FORMATE, p->pCurrent, p->curChkpId, p->pManifest, p->curChkpId, + nBytes = tsnprintf(content, sizeof(content), META_ON_S3_FORMATE, p->pCurrent, p->curChkpId, p->pManifest, p->curChkpId, "processVer", processId); if (nBytes <= 0 || nBytes >= sizeof(content)) { code = TSDB_CODE_OUT_OF_RANGE; diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 2688617823..75bcc326b3 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -108,7 +108,7 @@ void streamTaskSendCheckMsg(SStreamTask* pTask) { pRange->range.maxVer, pWindow->skey, pWindow->ekey, req.reqId); code = streamSendCheckMsg(pTask, &req, pTask->outputInfo.fixedDispatcher.nodeId, - &pTask->outputInfo.fixedDispatcher.epSet); + &pTask->outputInfo.fixedDispatcher.epSet); } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { streamTaskStartMonitorCheckRsp(pTask); @@ -171,14 +171,14 @@ void streamTaskProcessCheckMsg(SStreamMeta* pMeta, SStreamTaskCheckReq* pReq, SS streamTaskCheckStatus(pTask, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->stage, &pRsp->oldStage); SStreamTaskState pState = streamTaskGetStatus(pTask); - stDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(qid:0x%" PRIx64 + stDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(QID:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d", pTask->id.idStr, pState.name, pRsp->oldStage, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->status); streamMetaReleaseTask(pMeta, pTask); } else { pRsp->status = TASK_DOWNSTREAM_NOT_READY; - stDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(qid:0x%" PRIx64 + stDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(QID:0x%" PRIx64 ") from task:0x%x (vgId:%d), rsp check_status %d", pReq->streamId, taskId, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->status); } @@ -259,7 +259,8 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa void* buf = rpcMallocCont(sizeof(SMsgHead) + len); if (buf == NULL) { - stError("s-task:0x%x vgId:%d failed prepare msg, %s at line:%d code:%s", taskId, pMeta->vgId, __func__, __LINE__, tstrerror(code)); + stError("s-task:0x%x vgId:%d failed prepare msg, %s at line:%d code:%s", taskId, pMeta->vgId, __func__, __LINE__, + tstrerror(code)); return terrno; } @@ -332,7 +333,7 @@ void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) { /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void processDownstreamReadyRsp(SStreamTask* pTask) { EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST; - int32_t code = streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL); + int32_t code = streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL); if (code) { stError("s-task:%s failed to set event succ, code:%s", pTask->id.idStr, tstrerror(code)); } @@ -354,7 +355,7 @@ void processDownstreamReadyRsp(SStreamTask* pTask) { stDebug("s-task:%s level:%d initial status is %s from mnode, set it to be halt", pTask->id.idStr, pTask->info.taskLevel, streamTaskGetStatusStr(pTask->status.taskStatus)); code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_HALT); - if (code != 0) { // todo: handle error + if (code != 0) { // todo: handle error stError("s-task:%s failed to handle halt event, code:%s", pTask->id.idStr, tstrerror(code)); } } @@ -373,8 +374,9 @@ void processDownstreamReadyRsp(SStreamTask* pTask) { int32_t addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { int32_t vgId = pTask->pMeta->vgId; - int32_t code = 0;; - bool existed = false; + int32_t code = 0; + ; + bool existed = false; streamMutexLock(&pTask->lock); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 699774ed52..e44bca123b 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -120,38 +120,39 @@ int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSo } int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTriggerRsp* pRsp) { + SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; + bool unQualified = false; + const char* id = pTask->id.idStr; + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { - stError("s-task:%s invalid msg recv, checkpoint-trigger rsp not handled", pTask->id.idStr); + stError("s-task:%s invalid msg recv, checkpoint-trigger rsp not handled", id); return TSDB_CODE_INVALID_MSG; } if (pRsp->rspCode != TSDB_CODE_SUCCESS) { - stDebug("s-task:%s retrieve checkpoint-trgger rsp from upstream:0x%x invalid, code:%s", pTask->id.idStr, - pRsp->upstreamTaskId, tstrerror(pRsp->rspCode)); + stDebug("s-task:%s retrieve checkpoint-trgger rsp from upstream:0x%x invalid, code:%s", id, pRsp->upstreamTaskId, + tstrerror(pRsp->rspCode)); return TSDB_CODE_SUCCESS; } streamMutexLock(&pTask->lock); SStreamTaskState status = streamTaskGetStatus(pTask); - if (status.state != TASK_STATUS__CK) { - stError("s-task:%s status:%s not in checkpoint status, discard the checkpoint-trigger msg", pTask->id.idStr, status.name); - streamMutexUnlock(&pTask->lock); - return TSDB_CODE_STREAM_TASK_IVLD_STATUS; - } - streamMutexUnlock(&pTask->lock); - SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; - streamMutexLock(&pInfo->lock); - if (pInfo->activeId != pRsp->checkpointId || pInfo->transId != pRsp->transId) { - stError("s-task:%s status:%s not in checkpoint status, discard the checkpoint-trigger msg", pTask->id.idStr, status.name); - - streamMutexUnlock(&pInfo->lock); + if (status.state != TASK_STATUS__CK) { + stError("s-task:%s status:%s not in checkpoint status, discard the checkpoint-trigger msg", id, status.name); return TSDB_CODE_STREAM_TASK_IVLD_STATUS; } + streamMutexLock(&pInfo->lock); + unQualified = (pInfo->activeId != pRsp->checkpointId || pInfo->transId != pRsp->transId); streamMutexUnlock(&pInfo->lock); + if (unQualified) { + stError("s-task:%s status:%s not in checkpoint status, discard the checkpoint-trigger msg", id, status.name); + return TSDB_CODE_STREAM_TASK_IVLD_STATUS; + } + // NOTE: here we do not do the duplicated checkpoint-trigger msg check, since it will be done by following functions. int32_t code = appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER, pRsp->checkpointId, pRsp->transId, pRsp->upstreamTaskId); @@ -903,7 +904,7 @@ static int32_t doChkptStatusCheck(SStreamTask* pTask) { return -1; } - if ((pTmrInfo->launchChkptId != pActiveInfo->activeId) || (pActiveInfo->activeId == 0)) { + if (pTmrInfo->launchChkptId != pActiveInfo->activeId) { int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); stWarn("s-task:%s vgId:%d checkpoint-trigger retrieve by previous checkpoint procedure, checkpointId:%" PRId64 ", quit, ref:%d", @@ -963,11 +964,44 @@ static int32_t doFindNotSendUpstream(SStreamTask* pTask, SArray* pList, SArray** return 0; } +static int32_t chkptTriggerRecvMonitorHelper(SStreamTask* pTask, SArray* pNotSendList) { + const char* id = pTask->id.idStr; + SArray* pList = pTask->upstreamInfo.pList; // send msg to retrieve checkpoint trigger msg + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptTriggerMsgTmr; + int32_t vgId = pTask->pMeta->vgId; + + int32_t code = doChkptStatusCheck(pTask); + if (code) { + return code; + } + + code = doFindNotSendUpstream(pTask, pList, &pNotSendList); + if (code) { + int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); + stDebug("s-task:%s failed to find not send upstream, code:%s, out of tmr, ref:%d", id, tstrerror(code), ref); + return code; + } + + // do send retrieve checkpoint trigger msg to upstream + code = doSendRetrieveTriggerMsg(pTask, pNotSendList); + if (code) { + stError("s-task:%s vgId:%d failed to retrieve trigger msg, code:%s", pTask->id.idStr, vgId, tstrerror(code)); + code = 0; + } + + return code; +} + void checkpointTriggerMonitorFn(void* param, void* tmrId) { SStreamTask* pTask = param; int32_t vgId = pTask->pMeta->vgId; int64_t now = taosGetTimestampMs(); const char* id = pTask->id.idStr; + SArray* pNotSendList = NULL; + SArray* pList = pTask->upstreamInfo.pList; // send msg to retrieve checkpoint trigger msg + int32_t code = 0; + int32_t numOfNotSend = 0; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptTriggerMsgTmr; @@ -1008,42 +1042,21 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { } streamMutexLock(&pActiveInfo->lock); + code = chkptTriggerRecvMonitorHelper(pTask, pNotSendList); + streamMutexUnlock(&pActiveInfo->lock); - int32_t code = doChkptStatusCheck(pTask); - if (code) { - streamMutexUnlock(&pActiveInfo->lock); + if (code != TSDB_CODE_SUCCESS) { streamMetaReleaseTask(pTask->pMeta, pTask); - return; - } - - // send msg to retrieve checkpoint trigger msg - SArray* pList = pTask->upstreamInfo.pList; - SArray* pNotSendList = NULL; - - code = doFindNotSendUpstream(pTask, pList, &pNotSendList); - if (code) { - int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); - stDebug("s-task:%s failed to find not send upstream, code:%s, out of tmr, ref:%d", id, tstrerror(code), ref); - streamMutexUnlock(&pActiveInfo->lock); - streamMetaReleaseTask(pTask->pMeta, pTask); - taosArrayDestroy(pNotSendList); return; } - // do send retrieve checkpoint trigger msg to upstream - int32_t size = taosArrayGetSize(pNotSendList); - code = doSendRetrieveTriggerMsg(pTask, pNotSendList); - if (code) { - stError("s-task:%s vgId:%d failed to retrieve trigger msg, code:%s", pTask->id.idStr, vgId, tstrerror(code)); - } - - streamMutexUnlock(&pActiveInfo->lock); - // check every 100ms - if (size > 0) { + numOfNotSend = taosArrayGetSize(pNotSendList); + if (numOfNotSend > 0) { stDebug("s-task:%s start to monitor checkpoint-trigger in 10s", id); - streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "trigger-recv-monitor"); + streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, + "trigger-recv-monitor"); } else { int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); stDebug("s-task:%s all checkpoint-trigger recved, quit from monitor checkpoint-trigger tmr, ref:%d", id, ref); @@ -1106,19 +1119,13 @@ int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList) { return code; } -bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) { +static int32_t isAlreadySendTriggerNoLock(SStreamTask* pTask, int32_t downstreamNodeId) { int64_t now = taosGetTimestampMs(); const char* id = pTask->id.idStr; SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; SStreamTaskState pStatus = streamTaskGetStatus(pTask); - if (pStatus.state != TASK_STATUS__CK) { - return false; - } - - streamMutexLock(&pInfo->lock); if (!pInfo->dispatchTrigger) { - streamMutexUnlock(&pInfo->lock); return false; } @@ -1146,14 +1153,29 @@ bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) id, pSendInfo->sendTs, before, pInfo->activeId, pInfo->transId); } - streamMutexUnlock(&pInfo->lock); return true; } - streamMutexUnlock(&pInfo->lock); return false; } +bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) { + int64_t now = taosGetTimestampMs(); + const char* id = pTask->id.idStr; + SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; + SStreamTaskState pStatus = streamTaskGetStatus(pTask); + + if (pStatus.state != TASK_STATUS__CK) { + return false; + } + + streamMutexLock(&pInfo->lock); + bool send = isAlreadySendTriggerNoLock(pTask, downstreamNodeId); + streamMutexUnlock(&pInfo->lock); + + return send; +} + void streamTaskGetTriggerRecvStatus(SStreamTask* pTask, int32_t* pRecved, int32_t* pTotal) { *pRecved = taosArrayGetSize(pTask->chkInfo.pActiveInfo->pReadyMsgList); @@ -1169,8 +1191,10 @@ void streamTaskGetTriggerRecvStatus(SStreamTask* pTask, int32_t* pRecved, int32_ int32_t streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) { SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; int64_t now = taosGetTimestampMs(); + int32_t code = 0; streamMutexLock(&pInfo->lock); + pInfo->dispatchTrigger = true; if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { STaskDispatcherFixed* pDispatch = &pTask->outputInfo.fixedDispatcher; @@ -1178,8 +1202,7 @@ int32_t streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) { STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pDispatch->nodeId, .taskId = pDispatch->taskId}; void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p); if (px == NULL) { // pause the stream task, if memory not enough - streamMutexUnlock(&pInfo->lock); - return terrno; + code = terrno; } } else { for (int32_t i = 0; i < streamTaskGetNumOfDownstream(pTask); ++i) { @@ -1191,14 +1214,15 @@ int32_t streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) { STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pVgInfo->vgId, .taskId = pVgInfo->taskId}; void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p); if (px == NULL) { // pause the stream task, if memory not enough - streamMutexUnlock(&pInfo->lock); - return terrno; + code = terrno; + break; } } } streamMutexUnlock(&pInfo->lock); - return 0; + + return code; } int32_t streamTaskGetNumOfConfirmed(SActiveCheckpointInfo* pInfo) { diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index a3146ae9d4..133663ac28 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -526,6 +526,7 @@ static void doMonitorDispatchData(void* param, void* tmrId) { int32_t msgId = pMsgInfo->msgId; int32_t code = 0; int64_t now = taosGetTimestampMs(); + bool inDispatch = true; stDebug("s-task:%s start monitor dispatch data", id); @@ -550,12 +551,15 @@ static void doMonitorDispatchData(void* param, void* tmrId) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s not in dispatch procedure, abort from timer, ref:%d", pTask->id.idStr, ref); - pTask->msgInfo.inMonitor = 0; - streamMutexUnlock(&pMsgInfo->lock); - return; + pMsgInfo->inMonitor = 0; + inDispatch = false; } streamMutexUnlock(&pMsgInfo->lock); + if (!inDispatch) { + return; + } + int32_t numOfFailed = getFailedDispatchInfo(pMsgInfo, now); if (numOfFailed == 0) { stDebug("s-task:%s no error occurs, check again in %dms", id, DISPATCH_RETRY_INTERVAL_MS); @@ -638,15 +642,54 @@ void streamStartMonitorDispatchData(SStreamTask* pTask, int64_t waitDuration) { "dispatch-monitor"); } +static int32_t doAddDispatchBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, SSDataBlock* pDataBlock, + SArray* vgInfo, uint32_t hashValue, int64_t now, bool* pFound) { + size_t numOfVgroups = taosArrayGetSize(vgInfo); + int32_t code = 0; + + *pFound = false; + + for (int32_t j = 0; j < numOfVgroups; j++) { + SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j); + if (pVgInfo == NULL) { + continue; + } + + if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) { + if ((code = streamAddBlockIntoDispatchMsg(pDataBlock, &pReqs[j])) < 0) { + stError("s-task:%s failed to add dispatch block, code:%s", pTask->id.idStr, tstrerror(terrno)); + return code; + } + + if (pReqs[j].blockNum == 0) { + SVgroupInfo* pDstVgroupInfo = taosArrayGet(vgInfo, j); + if (pDstVgroupInfo != NULL) { + addDispatchEntry(&pTask->msgInfo, pDstVgroupInfo->vgId, now, false); + } + } + + pReqs[j].blockNum++; + *pFound = true; + break; + } + } + + return code; +} + int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, SSDataBlock* pDataBlock, int64_t groupId, int64_t now) { bool found = false; uint32_t hashValue = 0; - int32_t numOfVgroups = 0; + int32_t code = 0; + SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; - SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; if (pTask->pNameMap == NULL) { pTask->pNameMap = tSimpleHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); + if (pTask->pNameMap == NULL) { + stError("s-task:%s failed to init the name map, code:%s", pTask->id.idStr, tstrerror(terrno)); + return terrno; + } } void* pVal = tSimpleHashGet(pTask->pNameMap, &groupId, sizeof(int64_t)); @@ -663,17 +706,21 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S if (pTask->subtableWithoutMd5 != 1 && !isAutoTableName(pDataBlock->info.parTbName) && !alreadyAddGroupId(pDataBlock->info.parTbName, groupId) && groupId != 0) { if (pTask->ver == SSTREAM_TASK_SUBTABLE_CHANGED_VER) { - buildCtbNameAddGroupId(NULL, pDataBlock->info.parTbName, groupId); + code = buildCtbNameAddGroupId(NULL, pDataBlock->info.parTbName, groupId, sizeof(pDataBlock->info.parTbName)); } else if (pTask->ver > SSTREAM_TASK_SUBTABLE_CHANGED_VER) { - buildCtbNameAddGroupId(pTask->outputInfo.shuffleDispatcher.stbFullName, pDataBlock->info.parTbName, groupId); + code = buildCtbNameAddGroupId(pTask->outputInfo.shuffleDispatcher.stbFullName, pDataBlock->info.parTbName, + groupId, sizeof(pDataBlock->info.parTbName)); + } + if (code != TSDB_CODE_SUCCESS) { + return code; } } } else { - int32_t code = buildCtbNameByGroupIdImpl(pTask->outputInfo.shuffleDispatcher.stbFullName, groupId, - pDataBlock->info.parTbName); + code = buildCtbNameByGroupIdImpl(pTask->outputInfo.shuffleDispatcher.stbFullName, groupId, + pDataBlock->info.parTbName); if (code) { - stError("s-task:%s failed to build child table name for group:%" PRId64 ", code:%s", pTask->id.idStr, - groupId, tstrerror(code)); + stError("s-task:%s failed to build child table name for group:%" PRId64 ", code:%s", pTask->id.idStr, groupId, + tstrerror(code)); } } @@ -688,44 +735,21 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S memcpy(bln.parTbName, pDataBlock->info.parTbName, strlen(pDataBlock->info.parTbName)); // failed to put into name buffer, no need to do anything - if (tSimpleHashGetSize(pTask->pNameMap) < MAX_BLOCK_NAME_NUM) { // allow error, and do nothing - int32_t code = tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName)); + if (tSimpleHashGetSize(pTask->pNameMap) < MAX_BLOCK_NAME_NUM) { // allow error, and do nothing + code = tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName)); } } - numOfVgroups = taosArrayGetSize(vgInfo); - - // TODO: optimize search streamMutexLock(&pTask->msgInfo.lock); + code = doAddDispatchBlock(pTask, pReqs, pDataBlock, vgInfo, hashValue, now, &found); + streamMutexUnlock(&pTask->msgInfo.lock); - for (int32_t j = 0; j < numOfVgroups; j++) { - SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j); - if (pVgInfo == NULL) { - continue; - } - - if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) { - if (streamAddBlockIntoDispatchMsg(pDataBlock, &pReqs[j]) < 0) { - streamMutexUnlock(&pTask->msgInfo.lock); - return -1; - } - - if (pReqs[j].blockNum == 0) { - SVgroupInfo* pDstVgroupInfo = taosArrayGet(vgInfo, j); - if (pDstVgroupInfo != NULL) { - addDispatchEntry(&pTask->msgInfo, pDstVgroupInfo->vgId, now, false); - } - } - - pReqs[j].blockNum++; - found = true; - break; - } + if (code) { + return code; } - streamMutexUnlock(&pTask->msgInfo.lock); if (!found) { - stError("s-task:%s not found req hash value:%u", pTask->id.idStr, hashValue); + stError("s-task:%s not found req hash value:%u, failed to add dispatch block", pTask->id.idStr, hashValue); return TSDB_CODE_STREAM_INTERNAL_ERROR; } else { return 0; @@ -919,7 +943,7 @@ static int32_t doTaskChkptStatusCheck(SStreamTask* pTask, int32_t num) { } static int32_t doFindNotConfirmUpstream(SArray** ppNotRspList, SArray* pList, int32_t num, int32_t vgId, int32_t level, - const char* id) { + const char* id) { SArray* pTmp = taosArrayInit(4, sizeof(int32_t)); if (pTmp == NULL) { return terrno; @@ -940,8 +964,8 @@ static int32_t doFindNotConfirmUpstream(SArray** ppNotRspList, SArray* pList, in stError("s-task:%s vgId:%d failed to record not rsp task, code: out of memory", id, vgId); return terrno; } else { - stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId, - level, pInfo->upstreamTaskId); + stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId, level, + pInfo->upstreamTaskId); } } @@ -987,13 +1011,48 @@ static void doSendChkptReadyMsg(SStreamTask* pTask, SArray* pNotRspList, int64_t } } -static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { +static int32_t chkptReadyMsgSendHelper(SStreamTask* pTask, SArray* pNotRspList) { + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptReadyMsgTmr; + SArray* pList = pActiveInfo->pReadyMsgList; + int32_t num = taosArrayGetSize(pList); + int32_t vgId = pTask->pMeta->vgId; + int32_t checkpointId = pActiveInfo->activeId; + const char* id = pTask->id.idStr; + int32_t notRsp = 0; + + int32_t code = doTaskChkptStatusCheck(pTask, num); + if (code) { + return code; + } + + code = doFindNotConfirmUpstream(&pNotRspList, pList, num, vgId, pTask->info.taskLevel, id); + if (code) { + int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); + stError("s-task:%s failed to find not rsp checkpoint-ready downstream, code:%s, out of tmr, ref:%d", id, + tstrerror(code), ref); + return code; + } + + notRsp = taosArrayGetSize(pNotRspList); + if (notRsp == 0) { + streamClearChkptReadyMsg(pActiveInfo); + } else { + doSendChkptReadyMsg(pTask, pNotRspList, checkpointId, pList); + } + + return code; +} + +static void chkptReadyMsgSendMonitorFn(void* param, void* tmrId) { SStreamTask* pTask = param; int32_t vgId = pTask->pMeta->vgId; const char* id = pTask->id.idStr; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptReadyMsgTmr; SArray* pNotRspList = NULL; + int32_t code = 0; + int32_t notRsp = 0; // check the status every 100ms if (streamTaskShouldStop(pTask)) { @@ -1004,7 +1063,7 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { } if (++pTmrInfo->activeCounter < 50) { - streamTmrStart(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, + streamTmrStart(chkptReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "chkpt-ready-monitor"); return; } @@ -1027,45 +1086,26 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { } streamMutexLock(&pActiveInfo->lock); + code = chkptReadyMsgSendHelper(pTask, pNotRspList); + streamMutexUnlock(&pActiveInfo->lock); - SArray* pList = pActiveInfo->pReadyMsgList; - int32_t num = taosArrayGetSize(pList); - int32_t code = doTaskChkptStatusCheck(pTask, num); - if (code) { - streamMutexUnlock(&pActiveInfo->lock); + if (code != TSDB_CODE_SUCCESS) { streamMetaReleaseTask(pTask->pMeta, pTask); - return; - } - - code = doFindNotConfirmUpstream(&pNotRspList, pList, num, vgId, pTask->info.taskLevel, id); - if (code) { - int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); - stError("s-task:%s failed to find not rsp checkpoint-ready downstream, code:%s, out of tmr, ref:%d", id, - tstrerror(code), ref); - streamMutexUnlock(&pActiveInfo->lock); - streamMetaReleaseTask(pTask->pMeta, pTask); - taosArrayDestroy(pNotRspList); return; } - int32_t checkpointId = pActiveInfo->activeId; - int32_t notRsp = taosArrayGetSize(pNotRspList); - doSendChkptReadyMsg(pTask, pNotRspList, checkpointId, pList); - + notRsp = taosArrayGetSize(pNotRspList); if (notRsp > 0) { // send checkpoint-ready msg again - streamTmrStart(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, + stDebug("s-task:%s start to monitor checkpoint-ready msg recv status in 10s", id); + streamTmrStart(chkptReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "chkpt-ready-monitor"); - streamMutexUnlock(&pActiveInfo->lock); } else { int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); stDebug( "s-task:%s vgId:%d checkpoint-ready msg confirmed by all upstream task(s), clear checkpoint-ready msg and quit " "from timer, ref:%d", id, vgId, ref); - - streamClearChkptReadyMsg(pActiveInfo); - streamMutexUnlock(&pActiveInfo->lock); // release should be the last execution, since pTask may be destroy after it immidiately. streamMetaReleaseTask(pTask->pMeta, pTask); } @@ -1124,7 +1164,7 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) { stDebug("s-task:%s start checkpoint-ready monitor in 10s, ref:%d ", pTask->id.idStr, ref); streamMetaAcquireOneTask(pTask); - streamTmrStart(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, + streamTmrStart(chkptReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "chkpt-ready-monitor"); // mark the timer monitor checkpointId @@ -1190,6 +1230,7 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch taosMemoryFree(buf); return terrno; } + SET_PAYLOAD_LEN(pRetrieve->data, actualLen, actualLen); int32_t payloadLen = actualLen + PAYLOAD_PREFIX_LEN; @@ -1359,29 +1400,11 @@ void initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstr pReadyInfo->childId = childId; } -int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, int32_t index, int64_t checkpointId) { - if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { - return TSDB_CODE_SUCCESS; - } - - SStreamUpstreamEpInfo* pInfo = NULL; - streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId, &pInfo); - if (pInfo == NULL) { - return TSDB_CODE_STREAM_TASK_NOT_EXIST; - } - - STaskCheckpointReadyInfo info = {0}; - initCheckpointReadyInfo(&info, pInfo->nodeId, pInfo->taskId, pInfo->childId, &pInfo->epSet, checkpointId); - - stDebug("s-task:%s (level:%d) prepare checkpoint-ready msg to upstream s-task:0x%" PRIx64 "-0x%x (vgId:%d) idx:%d", - pTask->id.idStr, pTask->info.taskLevel, pTask->id.streamId, pInfo->taskId, pInfo->nodeId, index); - +static int32_t doAddChkptReadyMsg(SStreamTask* pTask, STaskCheckpointReadyInfo* pInfo) { SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; - streamMutexLock(&pActiveInfo->lock); - void* px = taosArrayPush(pActiveInfo->pReadyMsgList, &info); + void* px = taosArrayPush(pActiveInfo->pReadyMsgList, pInfo); if (px == NULL) { - streamMutexUnlock(&pActiveInfo->lock); stError("s-task:%s failed to add readyMsg info, code: out of memory", pTask->id.idStr); return terrno; } @@ -1395,10 +1418,36 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, stDebug("s-task:%s %d/%d checkpoint-trigger recv", pTask->id.idStr, numOfRecv, total); } - streamMutexUnlock(&pActiveInfo->lock); return 0; } +int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, int32_t index, int64_t checkpointId) { + int32_t code = 0; + STaskCheckpointReadyInfo info = {0}; + + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + return TSDB_CODE_SUCCESS; + } + + SStreamUpstreamEpInfo* pInfo = NULL; + streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId, &pInfo); + if (pInfo == NULL) { + return TSDB_CODE_STREAM_TASK_NOT_EXIST; + } + + initCheckpointReadyInfo(&info, pInfo->nodeId, pInfo->taskId, pInfo->childId, &pInfo->epSet, checkpointId); + + stDebug("s-task:%s (level:%d) prepare checkpoint-ready msg to upstream s-task:0x%" PRIx64 "-0x%x (vgId:%d) idx:%d", + pTask->id.idStr, pTask->info.taskLevel, pTask->id.streamId, pInfo->taskId, pInfo->nodeId, index); + + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + + streamMutexLock(&pActiveInfo->lock); + code = doAddChkptReadyMsg(pTask, &info); + streamMutexUnlock(&pActiveInfo->lock); + return code; +} + void streamClearChkptReadyMsg(SActiveCheckpointInfo* pActiveInfo) { if (pActiveInfo == NULL) { return; diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 0eb87df9b0..2e06813071 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -794,7 +794,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { // dispatch checkpoint msg to all downstream tasks int32_t type = pInput->type; if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) { - int32_t code = streamProcessCheckpointTriggerBlock(pTask, (SStreamDataBlock*)pInput); + code = streamProcessCheckpointTriggerBlock(pTask, (SStreamDataBlock*)pInput); if (code != 0) { stError("s-task:%s failed to process checkpoint-trigger block, code:%s", pTask->id.idStr, tstrerror(code)); } diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c index dde3595eb3..19391bf7a0 100644 --- a/source/libs/stream/src/streamHb.c +++ b/source/libs/stream/src/streamHb.c @@ -42,7 +42,7 @@ static bool waitForEnoughDuration(SMetaHbInfo* pInfo) { static bool existInHbMsg(SStreamHbMsg* pMsg, SDownstreamTaskEpset* pTaskEpset) { int32_t numOfExisted = taosArrayGetSize(pMsg->pUpdateNodes); - for (int k = 0; k < numOfExisted; ++k) { + for (int32_t k = 0; k < numOfExisted; ++k) { if (pTaskEpset->nodeId == *(int32_t*)taosArrayGet(pMsg->pUpdateNodes, k)) { return true; } @@ -56,7 +56,7 @@ static void addUpdateNodeIntoHbMsg(SStreamTask* pTask, SStreamHbMsg* pMsg) { streamMutexLock(&pTask->lock); int32_t num = taosArrayGetSize(pTask->outputInfo.pNodeEpsetUpdateList); - for (int j = 0; j < num; ++j) { + for (int32_t j = 0; j < num; ++j) { SDownstreamTaskEpset* pTaskEpset = taosArrayGet(pTask->outputInfo.pNodeEpsetUpdateList, j); bool exist = existInHbMsg(pMsg, pTaskEpset); diff --git a/source/libs/stream/src/streamSched.c b/source/libs/stream/src/streamSched.c index 63e24b0975..98920e6f70 100644 --- a/source/libs/stream/src/streamSched.c +++ b/source/libs/stream/src/streamSched.c @@ -83,12 +83,14 @@ void streamTaskResumeInFuture(SStreamTask* pTask) { ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void streamTaskResumeHelper(void* param, void* tmrId) { - SStreamTask* pTask = (SStreamTask*)param; - SStreamTaskId* pId = &pTask->id; - SStreamTaskState p = streamTaskGetStatus(pTask); + SStreamTask* pTask = (SStreamTask*)param; + SStreamTaskId* pId = &pTask->id; + SStreamTaskState p = streamTaskGetStatus(pTask); + int32_t code = 0; if (p.state == TASK_STATUS__DROPPING || p.state == TASK_STATUS__STOP) { int8_t status = streamTaskSetSchedStatusInactive(pTask); + TAOS_UNUSED(status); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s status:%s not resume task, ref:%d", pId->idStr, p.name, ref); @@ -97,13 +99,12 @@ void streamTaskResumeHelper(void* param, void* tmrId) { return; } - int32_t code = streamTaskSchedTask(pTask->pMsgCb, pTask->info.nodeId, pId->streamId, pId->taskId, STREAM_EXEC_T_RESUME_TASK); + code = streamTaskSchedTask(pTask->pMsgCb, pTask->info.nodeId, pId->streamId, pId->taskId, STREAM_EXEC_T_RESUME_TASK); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); if (code) { stError("s-task:%s sched task failed, code:%s, ref:%d", pId->idStr, tstrerror(code), ref); } else { - stDebug("trigger to resume s-task:%s after being idled for %dms, ref:%d", pId->idStr, pTask->status.schedIdleTime, - ref); + stDebug("trigger to resume s-task:%s after idled for %dms, ref:%d", pId->idStr, pTask->status.schedIdleTime, ref); // release the task ref count streamTaskClearSchedIdleInfo(pTask); diff --git a/source/libs/stream/src/streamStartHistory.c b/source/libs/stream/src/streamStartHistory.c index b376dbd16b..4d7bf2ba87 100644 --- a/source/libs/stream/src/streamStartHistory.c +++ b/source/libs/stream/src/streamStartHistory.c @@ -64,7 +64,6 @@ static int32_t streamTaskSetReady(SStreamTask* pTask) { int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated) { SStreamScanHistoryReq req; - int32_t code = 0; initScanHistoryReq(pTask, &req, igUntreated); int32_t len = sizeof(SStreamScanHistoryReq); @@ -173,7 +172,7 @@ int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask) { code = streamTaskStartScanHistory(pTask); } - // NOTE: there will be an deadlock if launch fill history here. + // NOTE: there will be a deadlock if launch fill history here. // start the related fill-history task, when current task is ready // if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { // streamLaunchFillHistoryTask(pTask); @@ -219,7 +218,7 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) { stDebug("s-task:%s start to launch related fill-history task:0x%" PRIx64 "-0x%x", idStr, hStreamId, hTaskId); - // Set the execute conditions, including the query time window and the version range + // Set the execution conditions, including the query time window and the version range streamMetaRLock(pMeta); SStreamTask** pHTask = taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id)); streamMetaRUnLock(pMeta); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 1a90eb8ab0..451e82c7d4 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2897,12 +2897,12 @@ void syncNodeLogConfigInfo(SSyncNode* ths, SSyncCfg* cfg, char* str) { char buf[256]; int32_t len = 256; int32_t n = 0; - n += snprintf(buf + n, len - n, "%s", "{"); + n += tsnprintf(buf + n, len - n, "%s", "{"); for (int i = 0; i < ths->peersEpset->numOfEps; i++) { - n += snprintf(buf + n, len - n, "%s:%d%s", ths->peersEpset->eps[i].fqdn, ths->peersEpset->eps[i].port, + n += tsnprintf(buf + n, len - n, "%s:%d%s", ths->peersEpset->eps[i].fqdn, ths->peersEpset->eps[i].port, (i + 1 < ths->peersEpset->numOfEps ? ", " : "")); } - n += snprintf(buf + n, len - n, "%s", "}"); + n += tsnprintf(buf + n, len - n, "%s", "}"); sInfo("vgId:%d, %s, peersEpset%d, %s, inUse:%d", ths->vgId, str, i, buf, ths->peersEpset->inUse); } diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 6f7ea9375c..212a75c2ae 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -24,14 +24,14 @@ #include "tglobal.h" static void syncCfg2SimpleStr(const SSyncCfg* pCfg, char* buf, int32_t bufLen) { - int32_t len = snprintf(buf, bufLen, "{num:%d, as:%d, [", pCfg->replicaNum, pCfg->myIndex); + int32_t len = tsnprintf(buf, bufLen, "{num:%d, as:%d, [", pCfg->replicaNum, pCfg->myIndex); for (int32_t i = 0; i < pCfg->replicaNum; ++i) { - len += snprintf(buf + len, bufLen - len, "%s:%d", pCfg->nodeInfo[i].nodeFqdn, pCfg->nodeInfo[i].nodePort); + len += tsnprintf(buf + len, bufLen - len, "%s:%d", pCfg->nodeInfo[i].nodeFqdn, pCfg->nodeInfo[i].nodePort); if (i < pCfg->replicaNum - 1) { - len += snprintf(buf + len, bufLen - len, "%s", ", "); + len += tsnprintf(buf + len, bufLen - len, "%s", ", "); } } - len += snprintf(buf + len, bufLen - len, "%s", "]}"); + len += tsnprintf(buf + len, bufLen - len, "%s", "]}"); } void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet) { @@ -60,7 +60,7 @@ bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* return false; } - char ipbuf[128] = {0}; + char ipbuf[TD_IP_LEN] = {0}; tinet_ntoa(ipbuf, ipv4); raftId->addr = SYNC_ADDR(pInfo); raftId->vgId = vgId; @@ -111,29 +111,29 @@ void syncUtilGenerateArbToken(int32_t nodeId, int32_t groupId, char* buf) { // for leader static void syncHearbeatReplyTime2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { int32_t len = 0; - len += snprintf(buf + len, bufLen - len, "%s", "{"); + len += tsnprintf(buf + len, bufLen - len, "%s", "{"); for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { int64_t tsMs = syncIndexMgrGetRecvTime(pSyncNode->pMatchIndex, &(pSyncNode->replicasId[i])); - len += snprintf(buf + len, bufLen - len, "%d:%" PRId64, i, tsMs); + len += tsnprintf(buf + len, bufLen - len, "%d:%" PRId64, i, tsMs); if (i < pSyncNode->replicaNum - 1) { - len += snprintf(buf + len, bufLen - len, "%s", ","); + len += tsnprintf(buf + len, bufLen - len, "%s", ","); } } - len += snprintf(buf + len, bufLen - len, "%s", "}"); + len += tsnprintf(buf + len, bufLen - len, "%s", "}"); } // for follower static void syncHearbeatTime2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { int32_t len = 0; - len += snprintf(buf + len, bufLen - len, "%s", "{"); + len += tsnprintf(buf + len, bufLen - len, "%s", "{"); for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { int64_t tsMs = syncIndexMgrGetRecvTime(pSyncNode->pNextIndex, &(pSyncNode->replicasId[i])); - len += snprintf(buf + len, bufLen - len, "%d:%" PRId64, i, tsMs); + len += tsnprintf(buf + len, bufLen - len, "%d:%" PRId64, i, tsMs); if (i < pSyncNode->replicaNum - 1) { - len += snprintf(buf + len, bufLen - len, "%s", ","); + len += tsnprintf(buf + len, bufLen - len, "%s", ","); } } - len += snprintf(buf + len, bufLen - len, "%s", "}"); + len += tsnprintf(buf + len, bufLen - len, "%s", "}"); } static void syncLogBufferStates2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { @@ -142,35 +142,35 @@ static void syncLogBufferStates2Str(SSyncNode* pSyncNode, char* buf, int32_t buf return; } int32_t len = 0; - len += snprintf(buf + len, bufLen - len, "[%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pBuf->startIndex, + len += tsnprintf(buf + len, bufLen - len, "[%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); } static void syncLogReplStates2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { int32_t len = 0; - len += snprintf(buf + len, bufLen - len, "%s", "{"); + len += tsnprintf(buf + len, bufLen - len, "%s", "{"); for (int32_t i = 0; i < pSyncNode->replicaNum; i++) { SSyncLogReplMgr* pMgr = pSyncNode->logReplMgrs[i]; if (pMgr == NULL) break; - len += snprintf(buf + len, bufLen - len, "%d:%d [%" PRId64 " %" PRId64 ", %" PRId64 "]", i, pMgr->restored, + len += tsnprintf(buf + len, bufLen - len, "%d:%d [%" PRId64 " %" PRId64 ", %" PRId64 "]", i, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex); if (i + 1 < pSyncNode->replicaNum) { - len += snprintf(buf + len, bufLen - len, "%s", ", "); + len += tsnprintf(buf + len, bufLen - len, "%s", ", "); } } - len += snprintf(buf + len, bufLen - len, "%s", "}"); + len += tsnprintf(buf + len, bufLen - len, "%s", "}"); } static void syncPeerState2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { int32_t len = 0; - len += snprintf(buf + len, bufLen - len, "%s", "{"); + len += tsnprintf(buf + len, bufLen - len, "%s", "{"); for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { SPeerState* pState = syncNodeGetPeerState(pSyncNode, &(pSyncNode->replicasId[i])); if (pState == NULL) break; - len += snprintf(buf + len, bufLen - len, "%d:%" PRId64 " %" PRId64 "%s", i, pState->lastSendIndex, + len += tsnprintf(buf + len, bufLen - len, "%d:%" PRId64 " %" PRId64 "%s", i, pState->lastSendIndex, pState->lastSendTime, (i < pSyncNode->replicaNum - 1) ? ", " : ""); } - len += snprintf(buf + len, bufLen - len, "%s", "}"); + len += tsnprintf(buf + len, bufLen - len, "%s", "}"); } void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNode* pNode, const char* format, ...) { diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index be391a75f1..49a15070a6 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -102,6 +102,10 @@ void tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg tdbOsFree(pPage->apOvfl[iOvfl]); } + if (TDB_DESTROY_PAGE_LOCK(pPage) != 0) { + tdbError("tdb/page-destroy: destroy page lock failed."); + } + ptr = pPage->pData; xFree(arg, ptr); diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 7d7868f3cd..bdcbfeb1cd 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -223,7 +223,7 @@ static FORCE_INLINE int32_t taosBuildDstAddr(const char* server, uint16_t port, tError("http-report failed to resolving domain names %s, reason: %s", server, tstrerror(code)); return TSDB_CODE_RPC_FQDN_ERROR; } - char buf[256] = {0}; + char buf[TD_IP_LEN] = {0}; tinet_ntoa(buf, ip); int ret = uv_ip4_addr(buf, port, dest); if (ret != 0) { diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index c0453c7759..c3f9819119 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1790,7 +1790,7 @@ static FORCE_INLINE int32_t cliUpdateFqdnCache(SHashObj* cache, char* fqdn) { size_t len = strlen(fqdn); uint32_t* v = taosHashGet(cache, fqdn, len); if (addr != *v) { - char old[64] = {0}, new[64] = {0}; + char old[TD_IP_LEN] = {0}, new[TD_IP_LEN] = {0}; tinet_ntoa(old, *v); tinet_ntoa(new, addr); tWarn("update ip of fqdn:%s, old: %s, new: %s", fqdn, old, new); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 98b5d907a0..d3a79f330c 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -679,15 +679,15 @@ void transPrintEpSet(SEpSet* pEpSet) { return; } char buf[512] = {0}; - int len = snprintf(buf, sizeof(buf), "epset:{"); + int len = tsnprintf(buf, sizeof(buf), "epset:{"); for (int i = 0; i < pEpSet->numOfEps; i++) { if (i == pEpSet->numOfEps - 1) { - len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + len += tsnprintf(buf + len, sizeof(buf) - len, "%d. %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); } else { - len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d, ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + len += tsnprintf(buf + len, sizeof(buf) - len, "%d. %s:%d, ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); } } - len += snprintf(buf + len, sizeof(buf) - len, "}"); + len += tsnprintf(buf + len, sizeof(buf) - len, "}"); tTrace("%s, inUse:%d", buf, pEpSet->inUse); } bool transEpSetIsEqual(SEpSet* a, SEpSet* b) { diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 9943fd1701..9ade5e5638 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -796,7 +796,7 @@ int32_t walMetaSerialize(SWal* pWal, char** serialized) { TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } - if (cJSON_AddItemToObject(pRoot, "meta", pMeta) != 0) { + if (!cJSON_AddItemToObject(pRoot, "meta", pMeta)) { wInfo("vgId:%d, failed to add meta to root", pWal->cfg.vgId); } (void)sprintf(buf, "%" PRId64, pWal->vers.firstVer); @@ -816,13 +816,13 @@ int32_t walMetaSerialize(SWal* pWal, char** serialized) { wInfo("vgId:%d, failed to add lastVer to meta", pWal->cfg.vgId); } - if (cJSON_AddItemToObject(pRoot, "files", pFiles) != 0) { + if (!cJSON_AddItemToObject(pRoot, "files", pFiles)) { wInfo("vgId:%d, failed to add files to root", pWal->cfg.vgId); } SWalFileInfo* pData = pWal->fileInfoSet->pData; for (int i = 0; i < sz; i++) { SWalFileInfo* pInfo = &pData[i]; - if (cJSON_AddItemToArray(pFiles, pField = cJSON_CreateObject()) != 0) { + if (!cJSON_AddItemToArray(pFiles, pField = cJSON_CreateObject())) { wInfo("vgId:%d, failed to add field to files", pWal->cfg.vgId); } if (pField == NULL) { diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 04747cacd4..497769a71c 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -151,7 +151,7 @@ int32_t taosMulMkDir(const char *dirname) { } if (temp[1] == ':') pos += 3; #else - (void)strcpy(temp, dirname); + tstrncpy(temp, dirname, sizeof(temp)); #endif if (taosDirExist(temp)) return code; @@ -216,7 +216,7 @@ int32_t taosMulModeMkDir(const char *dirname, int mode, bool checkAccess) { } if (temp[1] == ':') pos += 3; #else - (void)strcpy(temp, dirname); + tstrncpy(temp, dirname, sizeof(temp)); #endif if (taosDirExist(temp)) { @@ -341,7 +341,7 @@ int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { } if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) { - (void)strncpy(outname, full_path.we_wordv[0], maxlen); + tstrncpy(outname, full_path.we_wordv[0], maxlen); } wordfree(&full_path); @@ -358,9 +358,9 @@ int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen) { #endif if (strlen(tmp) < maxlen) { if (realPath == NULL) { - (void)strncpy(dirname, tmp, maxlen); + tstrncpy(dirname, tmp, maxlen); } else { - (void)strncpy(realPath, tmp, maxlen); + tstrncpy(realPath, tmp, maxlen); } return 0; } @@ -440,8 +440,7 @@ TdDirPtr taosOpenDir(const char *dirname) { return NULL; } - strcpy(szFind, dirname); - strcat(szFind, "\\*.*"); //利用通配符找这个目录下的所以文件，包括目录 + snprintf(szFind, sizeof(szFind), "%s%s", dirname, "\\*.*"); //利用通配符找这个目录下的所以文件，包括目录 pDir->hFind = FindFirstFile(szFind, &(pDir->dirEntry.findFileData)); if (INVALID_HANDLE_VALUE == pDir->hFind) { @@ -560,6 +559,6 @@ void taosGetCwd(char *buf, int32_t len) { char *unused __attribute__((unused)); unused = getcwd(buf, len - 1); #else - strncpy(buf, "not implemented on windows", len - 1); + tstrncpy(buf, "not implemented on windows", len); #endif } diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index f2c90e778d..a3791eb026 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -37,7 +37,7 @@ float tsNumOfCores = 0; int64_t tsTotalMemoryKB = 0; char *tsProcPath = NULL; -char tsSIMDEnable = 0; +char tsSIMDEnable = 1; char tsAVX512Enable = 0; char tsSSE42Supported = 0; char tsAVXSupported = 0; @@ -77,21 +77,21 @@ int32_t osDefaultInit() { tmpDir = getenv("temp"); } if (tmpDir != NULL) { - (void)strcpy(tsTempDir, tmpDir); + tstrncpy(tsTempDir, tmpDir, sizeof(tsTempDir)); } - (void)strcpy(tsOsName, "Windows"); + tstrncpy(tsOsName, "Windows", sizeof(tsOsName)); #elif defined(_TD_DARWIN_64) - (void)strcpy(tsOsName, "Darwin"); + tstrncpy(tsOsName, "Darwin", sizeof(tsOsName)); #else - (void)strcpy(tsOsName, "Linux"); + tstrncpy(tsOsName, "Linux", sizeof(tsOsName)); #endif if (configDir[0] == 0) { - (void)strcpy(configDir, TD_CFG_DIR_PATH); + tstrncpy(configDir, TD_CFG_DIR_PATH, sizeof(configDir)); } - (void)strcpy(tsDataDir, TD_DATA_DIR_PATH); - (void)strcpy(tsLogDir, TD_LOG_DIR_PATH); - if(strlen(tsTempDir) == 0){ - (void)strcpy(tsTempDir, TD_TMP_DIR_PATH); + tstrncpy(tsDataDir, TD_DATA_DIR_PATH, sizeof(tsDataDir)); + tstrncpy(tsLogDir, TD_LOG_DIR_PATH, sizeof(tsLogDir)); + if (strlen(tsTempDir) == 0){ + tstrncpy(tsTempDir, TD_TMP_DIR_PATH, sizeof(tsTempDir)); } return code; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index ef8c1eb860..46cca99e5f 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -91,11 +91,7 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, cha tmpPath[len++] = '\\'; } - strcpy(tmpPath + len, TD_TMP_FILE_PREFIX); - if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) { - strcat(tmpPath, fileNamePrefix); - strcat(tmpPath, "-%d-%s"); - } + snprintf(tmpPath + len, sizeof(tmpPath) - len, "%s%s%s", TD_TMP_FILE_PREFIX, fileNamePrefix, "-%d-%s"); char rand[8] = {0}; taosRandStr(rand, tListLen(rand) - 1); @@ -112,15 +108,11 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, cha tmpPath[len++] = '/'; } - (void)strcpy(tmpPath + len, TD_TMP_FILE_PREFIX); - if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) { - (void)strcat(tmpPath, fileNamePrefix); - (void)strcat(tmpPath, "-%d-%s"); - } + snprintf(tmpPath + len, sizeof(tmpPath) - len, "%s%s%s", TD_TMP_FILE_PREFIX, fileNamePrefix, "-%d-%s"); char rand[32] = {0}; - (void)sprintf(rand, "%" PRIu64, atomic_add_fetch_64(&seqId, 1)); + (void)snprintf(rand, sizeof(rand), "%" PRIu64, atomic_add_fetch_64(&seqId, 1)); (void)snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c index d7fbb05a5d..becf0d5a70 100644 --- a/source/os/src/osLocale.c +++ b/source/os/src/osLocale.c @@ -95,7 +95,7 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) { if (locale != NULL) { tstrncpy(outLocale, locale, TD_LOCALE_LEN); } - strcpy(outCharset, "UTF-8"); + tstrncpy(outCharset, "UTF-8", TD_CHARSET_LEN); #elif defined(_TD_DARWIN_64) /* @@ -123,7 +123,7 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) { locale = setlocale(LC_CTYPE, ""); if (locale == NULL) { // printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); - strcpy(outLocale, "en_US.UTF-8"); + tstrncpy(outLocale, "en_US.UTF-8", TD_LOCALE_LEN); } else { tstrncpy(outLocale, locale, TD_LOCALE_LEN); // printf("locale not configured, set to system default:%s", outLocale); @@ -137,7 +137,7 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) { char *revisedCharset = taosCharsetReplace(str); if (NULL == revisedCharset) { - (void)strcpy(outCharset, "UTF-8"); + tstrncpy(outCharset, "UTF-8", TD_CHARSET_LEN); } else { tstrncpy(outCharset, revisedCharset, TD_CHARSET_LEN); @@ -145,7 +145,7 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) { } // printf("charset not configured, set to system default:%s", outCharset); } else { - strcpy(outCharset, "UTF-8"); + tstrncpy(outCharset, "UTF-8", TD_CHARSET_LEN); // printf("can't get locale and charset from system, set it to UTF-8"); } @@ -173,7 +173,7 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) { locale = setlocale(LC_CTYPE, ""); if (locale == NULL) { // printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); - (void)strcpy(outLocale, "en_US.UTF-8"); + tstrncpy(outLocale, "en_US.UTF-8", TD_LOCALE_LEN); } else { tstrncpy(outLocale, locale, TD_LOCALE_LEN); //printf("locale not configured, set to system default:%s\n", outLocale); @@ -186,15 +186,15 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) { char *revisedCharset = taosCharsetReplace(str); if (NULL == revisedCharset) { - (void)strcpy(outCharset, "UTF-8"); + tstrncpy(outCharset, "UTF-8", TD_CHARSET_LEN); } else { - tstrncpy(outCharset, revisedCharset, TD_LOCALE_LEN); + tstrncpy(outCharset, revisedCharset, TD_CHARSET_LEN); taosMemoryFree(revisedCharset); } // printf("charset not configured, set to system default:%s", outCharset); } else { - (void)strcpy(outCharset, "UTF-8"); + tstrncpy(outCharset, "UTF-8", TD_CHARSET_LEN); // printf("can't get locale and charset from system, set it to UTF-8"); } diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 49a5c2a2a2..c084b76485 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -168,7 +168,7 @@ void startTrace() { Dwarf_Ptr errarg = 0; FILE *fp = fopen("/proc/self/maps", "r"); - fscanf(fp, "%lx-", &addr); + ret = fscanf(fp, "%lx-", &addr); fclose(fp); ret = dwarf_init_path("/proc/self/exe", NULL, 0, DW_GROUPNUMBER_ANY, NULL, errarg, &tDbg, NULL); diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index ea9e824947..cc3a13a818 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -377,20 +377,20 @@ int32_t tsem2_wait(tsem2_t* sem) { } int32_t tsem2_timewait(tsem2_t* sem, int64_t ms) { - int ret = 0; + int32_t code = 0; - ret = taosThreadMutexLock(&sem->mutex); - if (ret) { - return ret; + code = taosThreadMutexLock(&sem->mutex); + if (code) { + return code; } if (sem->count <= 0) { struct timespec ts = {0}; if (clock_gettime(CLOCK_MONOTONIC, &ts) == -1) { - ret = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(errno); (void)taosThreadMutexUnlock(&sem->mutex); - terrno = ret; - return ret; + terrno = code; + return code; } ts.tv_sec += ms / 1000; @@ -399,22 +399,18 @@ int32_t tsem2_timewait(tsem2_t* sem, int64_t ms) { ts.tv_nsec %= 1000000000; while (sem->count <= 0) { - ret = taosThreadCondTimedWait(&sem->cond, &sem->mutex, &ts); - if (ret != 0) { + code = taosThreadCondTimedWait(&sem->cond, &sem->mutex, &ts); + if (code != 0) { (void)taosThreadMutexUnlock(&sem->mutex); - if (errno == ETIMEDOUT) { - return TSDB_CODE_TIMEOUT_ERROR; - } else { - return TAOS_SYSTEM_ERROR(errno); - } + return code; } } } sem->count--; - ret = taosThreadMutexUnlock(&sem->mutex); - return ret; + code = taosThreadMutexUnlock(&sem->mutex); + return code; } #endif diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 09a5579e13..d17cc9b239 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -332,8 +332,8 @@ int32_t taosGetFqdn(char *fqdn) { // thus, we choose AF_INET (ipv4 for the moment) to make getaddrinfo return // immediately // hints.ai_family = AF_INET; - strcpy(fqdn, hostname); - strcpy(fqdn + strlen(hostname), ".local"); + tstrncpy(fqdn, hostname, TD_FQDN_LEN); + tstrncpy(fqdn + strlen(hostname), ".local", TD_FQDN_LEN - strlen(hostname)); #else // linux #endif // linux @@ -361,7 +361,7 @@ int32_t taosGetFqdn(char *fqdn) { break; } - (void)strcpy(fqdn, result->ai_canonname); + tstrncpy(fqdn, result->ai_canonname, TD_FQDN_LEN); freeaddrinfo(result); @@ -375,7 +375,7 @@ int32_t taosGetFqdn(char *fqdn) { // fprintf(stderr, "failed to get fqdn, code:%d, hostname:%s, reason:%s\n", ret, hostname, gai_strerror(ret)); return TAOS_SYSTEM_WINSOCKET_ERROR(WSAGetLastError()); } - strcpy(fqdn, result->ai_canonname); + tstrncpy(fqdn, result->ai_canonname, TD_FQDN_LEN); freeaddrinfo(result); #endif @@ -384,7 +384,7 @@ int32_t taosGetFqdn(char *fqdn) { } void tinet_ntoa(char *ipstr, uint32_t ip) { - (void)sprintf(ipstr, "%d.%d.%d.%d", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, ip >> 24); + (void)snprintf(ipstr, TD_IP_LEN, "%d.%d.%d.%d", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, ip >> 24); } int32_t taosIgnSIGPIPE() { diff --git a/source/os/src/osString.c b/source/os/src/osString.c index d265ed510a..18da778227 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -71,7 +71,7 @@ char *taosStrndup(const char *s, int size) { if (l > size) l = size; s2 = malloc(l + 1); if (s2) { - strncpy(s2, s, l); + tstrncpy(s2, s, l + 1); s2[l] = '\0'; } else { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -442,14 +442,14 @@ int32_t taosUcs4len(TdUcs4 *ucs4) { } // dst buffer size should be at least 2*len + 1 -int32_t taosHexEncode(const unsigned char *src, char *dst, int32_t len) { +int32_t taosHexEncode(const unsigned char *src, char *dst, int32_t len, int32_t bufSize) { if (!dst) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } for (int32_t i = 0; i < len; ++i) { - (void)sprintf(dst + i * 2, "%02x", src[i]); + (void)snprintf(dst + i * 2, bufSize - i * 2, "%02x", src[i]); } return 0; @@ -710,3 +710,25 @@ int32_t taosAscii2Hex(const char *z, uint32_t n, void **data, uint32_t *size) { return 0; } + +int64_t tsnprintf(char *dst, int64_t size, const char *format, ...) { + if (size <= 0) return 0; + if (size == 1) { + dst[0] = '\0'; + return 0; + } + if (size > SIZE_MAX) { + size = SIZE_MAX; + } + + int64_t ret; + va_list args; + va_start(args, format); + ret = vsnprintf(dst, size, format, args); + va_end(args); + if (ret >= size) { + return size - 1; + } else { + return ret; + } +} diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index cb6d3a7736..bd1a058291 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -389,10 +389,10 @@ int32_t taosGetOsReleaseName(char *releaseName, char* sName, char* ver, int32_t } if (major >= 20) { major -= 9; // macOS 11 and newer - sprintf(releaseName, "macOS %u.%u", major, minor); + snprintf(releaseName, maxLen, "macOS %u.%u", major, minor); } else { major -= 4; // macOS 10.1.1 and newer - sprintf(releaseName, "macOS 10.%d.%d", major, minor); + snprintf(releaseName, maxLen, "macOS 10.%d.%d", major, minor); } return 0; @@ -474,7 +474,7 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { if (taosGetsCmd(pCmd, sizeof(buf) - 1, buf) > 0) { code = 0; done |= 2; - *numOfCores = atof(buf); + *numOfCores = taosStr2Float(buf, NULL); } taosCloseCmd(&pCmd); @@ -498,7 +498,7 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { done |= 1; } else if (((done & 2) == 0) && strncmp(line, "cpu cores", 9) == 0) { const char *v = strchr(line, ':') + 2; - *numOfCores = atof(v); + *numOfCores = taosStr2Float(v, NULL); done |= 2; } if (strncmp(line, "processor", 9) == 0) coreCount += 1; @@ -1035,14 +1035,15 @@ void taosKillSystem() { #endif } -int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { +#define UUIDLEN (36) +int32_t taosGetSystemUUIDLimit36(char *uid, int32_t uidlen) { #ifdef WINDOWS GUID guid; HRESULT h = CoCreateGuid(&guid); if (h != S_OK) { return TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); } - snprintf(uid, uidlen, "%08X-%04X-%04X-%02X%02X-%02X%02X%02X%02X%02X%02X", guid.Data1, guid.Data2, guid.Data3, + (void)snprintf(uid, uidlen, "%08X-%04X-%04X-%02X%02X-%02X%02X%02X%02X%02X%02X", guid.Data1, guid.Data2, guid.Data3, guid.Data4[0], guid.Data4[1], guid.Data4[2], guid.Data4[3], guid.Data4[4], guid.Data4[5], guid.Data4[6], guid.Data4[7]); @@ -1054,7 +1055,7 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { uuid_generate(uuid); // it's caller's responsibility to make enough space for `uid`, that's 36-char + 1-null uuid_unparse_lower(uuid, buf); - int n = snprintf(uid, uidlen, "%.*s", (int)sizeof(buf), buf); // though less performance, much safer + (void)snprintf(uid, uidlen, "%.*s", (int)sizeof(buf), buf); return 0; #else int64_t len = 0; @@ -1070,16 +1071,32 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { return terrno; } } - - if (len >= 36) { - uid[36] = 0; - return 0; + if (len >= UUIDLEN + 1) { + uid[len - 1] = 0; + } else { + uid[uidlen - 1] = 0; } return 0; #endif } +int32_t taosGetSystemUUIDLen(char *uid, int32_t uidlen) { + if (uid == NULL || uidlen <= 0) { + return TSDB_CODE_APP_ERROR; + } + int num = (uidlen % UUIDLEN == 0) ? (uidlen / UUIDLEN) : (uidlen / UUIDLEN + 1); + int left = uidlen; + for (int i = 0; i < num; ++i) { + int32_t code = taosGetSystemUUIDLimit36(uid + i * UUIDLEN, left); + if (code != 0) { + return code; + } + left -= UUIDLEN; + } + return TSDB_CODE_SUCCESS; +} + char *taosGetCmdlineByPID(int pid) { #ifdef WINDOWS return ""; @@ -1095,7 +1112,7 @@ char *taosGetCmdlineByPID(int pid) { return cmdline; #else static char cmdline[1024]; - (void)sprintf(cmdline, "/proc/%d/cmdline", pid); + (void)snprintf(cmdline, sizeof(cmdline), "/proc/%d/cmdline", pid); // int fd = open(cmdline, O_RDONLY); TdFilePtr pFile = taosOpenFile(cmdline, TD_FILE_READ); diff --git a/source/os/src/osThread.c b/source/os/src/osThread.c index d102a2a332..6a8c705cde 100644 --- a/source/os/src/osThread.c +++ b/source/os/src/osThread.c @@ -235,19 +235,22 @@ int32_t taosThreadCondWait(TdThreadCond *cond, TdThreadMutex *mutex) { int32_t taosThreadCondTimedWait(TdThreadCond *cond, TdThreadMutex *mutex, const struct timespec *abstime) { #ifdef __USE_WIN_THREAD - if (!abstime) return EINVAL; + if (!abstime) return 0; if (SleepConditionVariableCS(cond, mutex, (DWORD)(abstime->tv_sec * 1e3 + abstime->tv_nsec / 1e6))) return 0; - if (GetLastError() == ERROR_TIMEOUT) { - return ETIMEDOUT; + DWORD error = GetLastError(); + if (error == ERROR_TIMEOUT) { + return TSDB_CODE_TIMEOUT_ERROR; } - return EINVAL; + return TAOS_SYSTEM_WINAPI_ERROR(error); #else int32_t code = pthread_cond_timedwait(cond, mutex, abstime); - if (code && code != ETIMEDOUT) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + if(code == ETIMEDOUT) { + return TSDB_CODE_TIMEOUT_ERROR; + } else if (code) { + return TAOS_SYSTEM_ERROR(code); + } else { + return 0; } - return code; #endif } diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index c650ac989d..d4d9936154 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -474,22 +474,15 @@ time_t taosMktime(struct tm *timep) { #endif } -struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf) { +struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf, int32_t bufSize) { struct tm *res = NULL; - if (timep == NULL) { + if (timep == NULL || result == NULL) { return NULL; } - if (result == NULL) { - res = localtime(timep); - if (res == NULL && buf != NULL) { - (void)sprintf(buf, "NaN"); - } - return res; - } #ifdef WINDOWS if (*timep < -2208988800LL) { if (buf != NULL) { - sprintf(buf, "NaN"); + snprintf(buf, bufSize, "NaN"); } return NULL; } else if (*timep < 0) { @@ -501,7 +494,7 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf) { time_t tt = 0; if (localtime_s(&tm1, &tt) != 0) { if (buf != NULL) { - sprintf(buf, "NaN"); + snprintf(buf, bufSize, "NaN"); } return NULL; } @@ -532,7 +525,7 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf) { } else { if (localtime_s(result, timep) != 0) { if (buf != NULL) { - sprintf(buf, "NaN"); + snprintf(buf, bufSize, "NaN"); } return NULL; } @@ -540,7 +533,7 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf) { #else res = localtime_r(timep, result); if (res == NULL && buf != NULL) { - (void)sprintf(buf, "NaN"); + (void)snprintf(buf, bufSize, "NaN"); } #endif return result; @@ -559,7 +552,7 @@ static int isLeapYear(time_t year) { struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst) { if (result == NULL) { - return localtime(timep); + return NULL; } #ifdef WINDOWS if (*timep < 0) { diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index 89ced69f97..5eded97cde 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -742,6 +742,20 @@ char *tz_win[554][2] = {{"Asia/Shanghai", "China Standard Time"}, static int isdst_now = 0; +void parseTimeStr(char *p, char to[5]) { + for (int i = 0; i < 5; ++i) { + if (strlen(p) > i) { + to[i] = p[i]; + } else { + to[i] = '0'; + } + } + if (strlen(p) == 2) { + to[1] = '0'; + to[2] = p[1]; + } +} + int32_t taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8_t *outDaylight, enum TdTimezone *tsTimezone) { if (inTimezoneStr == NULL || inTimezoneStr[0] == 0) { @@ -769,15 +783,15 @@ int32_t taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, i memset(winStr, 0, sizeof(winStr)); for (size_t i = 0; i < 554; i++) { if (strcmp(tz_win[i][0], buf) == 0) { - char keyPath[100]; + char keyPath[256]; char keyValue[100]; DWORD keyValueSize = sizeof(keyValue); - sprintf(keyPath, "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Time Zones\\%s", tz_win[i][1]); + snprintf(keyPath, sizeof(keyPath), "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Time Zones\\%s", tz_win[i][1]); RegGetValue(HKEY_LOCAL_MACHINE, keyPath, "Display", RRF_RT_ANY, NULL, (PVOID)&keyValue, &keyValueSize); if (keyValueSize > 0) { keyValue[4] = (keyValue[4] == '+' ? '-' : '+'); keyValue[10] = 0; - sprintf(winStr, "TZ=%s:00", &(keyValue[1])); + snprintf(winStr, sizeof(winStr), "TZ=%s:00", &(keyValue[1])); *tsTimezone = -taosStr2Int32(&keyValue[4], NULL, 10); } break; @@ -791,14 +805,16 @@ int32_t taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, i char *ppp = strchr(inTimezoneStr, ','); int indexStr; if (pp == NULL || ppp == NULL) { - indexStr = sprintf(winStr, "TZ=UTC"); + indexStr = snprintf(winStr, sizeof(winStr), "TZ=UTC"); } else { memcpy(winStr, "TZ=", 3); pp++; memcpy(&winStr[3], pp, ppp - pp); indexStr = ppp - pp + 3; } - sprintf(&winStr[indexStr], "%c%c%c:%c%c:00", (p[0] == '+' ? '+' : '-'), p[1], p[2], p[3], p[4]); + char to[5]; + parseTimeStr(p, to); + snprintf(&winStr[indexStr], sizeof(winStr) - indexStr, "%c%c%c:%c%c:00", (to[0] == '+' ? '+' : '-'), to[1], to[2], to[3], to[4]); *tsTimezone = -taosStr2Int32(p, NULL, 10); } else { *tsTimezone = 0; @@ -806,7 +822,9 @@ int32_t taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, i } _putenv(winStr); _tzset(); - strcpy(outTimezoneStr, inTimezoneStr); + if (outTimezoneStr != inTimezoneStr) { + tstrncpy(outTimezoneStr, inTimezoneStr, TD_TIMEZONE_LEN); + } *outDaylight = 0; #elif defined(_TD_DARWIN_64) @@ -821,7 +839,7 @@ int32_t taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, i *tsTimezone = tz; tz += isdst_now; - sprintf(outTimezoneStr, "%s (%s, %s%02d00)", buf, tzname[isdst_now], tz >= 0 ? "+" : "-", abs(tz)); + snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[isdst_now], tz >= 0 ? "+" : "-", abs(tz)); *outDaylight = isdst_now; #else @@ -835,7 +853,7 @@ int32_t taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, i int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); *tsTimezone = tz; tz += isdst_now; - (void)sprintf(outTimezoneStr, "%s (%s, %s%02d00)", buf, tzname[isdst_now], tz >= 0 ? "+" : "-", abs(tz)); + (void)snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[isdst_now], tz >= 0 ? "+" : "-", abs(tz)); *outDaylight = isdst_now; #endif @@ -854,21 +872,21 @@ int32_t taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) if (result != ERROR_SUCCESS) { return TAOS_SYSTEM_WINAPI_ERROR(result); } - strcpy(outTimezoneStr, "not configured"); + tstrncpy(outTimezoneStr, "not configured", TD_TIMEZONE_LEN); *tsTimezone = 0; if (bufferSize > 0) { for (size_t i = 0; i < 139; i++) { if (strcmp(win_tz[i][0], value) == 0) { - strcpy(outTimezoneStr, win_tz[i][1]); + tstrncpy(outTimezoneStr, win_tz[i][1], TD_TIMEZONE_LEN); bufferSize = sizeof(value); - sprintf(keyPath, "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Time Zones\\%s", value); + snprintf(keyPath, sizeof(keyPath), "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Time Zones\\%s", value); result = RegGetValue(HKEY_LOCAL_MACHINE, keyPath, "Display", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); if (result != ERROR_SUCCESS) { return TAOS_SYSTEM_WINAPI_ERROR(result); } if (bufferSize > 0) { // value[4] = (value[4] == '+' ? '-' : '+'); - sprintf(outTimezoneStr, "%s (UTC, %c%c%c%c%c)", outTimezoneStr, value[4], value[5], value[6], value[8], + snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (UTC, %c%c%c%c%c)", outTimezoneStr, value[4], value[5], value[6], value[8], value[9]); *tsTimezone = taosStr2Int32(&value[4], NULL, 10); } @@ -908,7 +926,7 @@ int32_t taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) */ time_t tx1 = taosGetTimestampSec(); struct tm tm1; - if (taosLocalTime(&tx1, &tm1, NULL) == NULL) { + if (taosLocalTime(&tx1, &tm1, NULL, 0) == NULL) { return TSDB_CODE_TIME_ERROR; } daylight = tm1.tm_isdst; @@ -938,7 +956,7 @@ int32_t taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) */ time_t tx1 = taosGetTimestampSec(); struct tm tm1; - if(taosLocalTime(&tx1, &tm1, NULL) == NULL) { + if(taosLocalTime(&tx1, &tm1, NULL, 0) == NULL) { return TSDB_CODE_TIME_ERROR; } /* load time zone string from /etc/timezone */ @@ -1019,7 +1037,7 @@ int32_t taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) */ time_t tx1 = taosGetTimestampSec(); struct tm tm1; - if(taosLocalTime(&tx1, &tm1, NULL) == NULL) { + if(taosLocalTime(&tx1, &tm1, NULL, 0) == NULL) { return TSDB_CODE_TIME_ERROR; } isdst_now = tm1.tm_isdst; diff --git a/source/os/test/osStringTests.cpp b/source/os/test/osStringTests.cpp index 9565e568fb..de07d21959 100644 --- a/source/os/test/osStringTests.cpp +++ b/source/os/test/osStringTests.cpp @@ -14,6 +14,7 @@ */ #include +#include #include #pragma GCC diagnostic push @@ -111,3 +112,45 @@ TEST(osStringTests, osUcs4lenTests2) { TdUcs4 ucs4_3[] = {'C', 'h', 'i', 'n', 'a', 0x4E2D, 0x6587, '\0'}; EXPECT_EQ(taosUcs4len(ucs4_3), 7); } + +TEST(osStringTests, ostsnprintfTests) { + char buffer[50] = {0}; + int64_t ret; + + ret = tsnprintf(buffer, sizeof(buffer), "Hello, %s!", "World"); + EXPECT_EQ(ret, 13); + EXPECT_STREQ(buffer, "Hello, World!"); + + memset(buffer, 0, sizeof(buffer)); + ret = tsnprintf(buffer, 10, "Hello, %s!", "World"); + EXPECT_EQ(ret, 9); + EXPECT_EQ(strncmp(buffer, "Hello, Wo", 9), 0); + + memset(buffer, 0, sizeof(buffer)); + ret = tsnprintf(buffer, 10, "Hello%s", "World"); + EXPECT_EQ(ret, 9); + EXPECT_EQ(strncmp(buffer, "HelloWorl", 9), 0); + + memset(buffer, 0, sizeof(buffer)); + ret = tsnprintf(buffer, 0, "Hello, %s!", "World"); + EXPECT_EQ(ret, 0); + + memset(buffer, 0, sizeof(buffer)); + ret = tsnprintf(buffer, SIZE_MAX + 1, "Hello, %s!", "World"); + EXPECT_EQ(ret, 0); + + memset(buffer, 0, sizeof(buffer)); + ret = tsnprintf(buffer, sizeof(buffer), ""); + EXPECT_EQ(ret, 0); + EXPECT_STREQ(buffer, ""); + + memset(buffer, 0, sizeof(buffer)); + ret = tsnprintf(buffer, sizeof(buffer), "Number: %d", 42); + EXPECT_EQ(ret, 10); + EXPECT_STREQ(buffer, "Number: 42"); + + memset(buffer, 0, sizeof(buffer)); + ret = tsnprintf(buffer, sizeof(buffer), "Float: %.2f", 3.14); + EXPECT_EQ(ret, 11); + EXPECT_STREQ(buffer, "Float: 3.14"); +} diff --git a/source/os/test/osSystemTests.cpp b/source/os/test/osSystemTests.cpp index 3712e45451..bc122aafa0 100644 --- a/source/os/test/osSystemTests.cpp +++ b/source/os/test/osSystemTests.cpp @@ -60,3 +60,88 @@ TEST(osSystemTest, osSystem1) { (void)printf("cpu info: %s\n", tmp); ASSERT_EQ(res, 0); } + + +TEST(osSystemTest, systemUUIDTest) { + char uuid1[38]; + memset(uuid1, 0, sizeof(uuid1)); + taosGetSystemUUIDLimit36(uuid1, sizeof(uuid1)); + ASSERT_EQ(strlen(uuid1), 36); + + char uuid2[34]; + memset(uuid2, 0, sizeof(uuid2)); + taosGetSystemUUIDLimit36(uuid2, sizeof(uuid2)); + ASSERT_EQ(strlen(uuid2), 33); + + char uuid3[36]; + memset(uuid3, 0, sizeof(uuid3)); + taosGetSystemUUIDLimit36(uuid3, sizeof(uuid3)); + ASSERT_EQ(strlen(uuid3), 35); + + char uuid4[2]; + memset(uuid4, 0, sizeof(uuid4)); + taosGetSystemUUIDLimit36(uuid4, sizeof(uuid4)); + ASSERT_EQ(strlen(uuid4), 1); + + char uuid5[36]; + memset( uuid5, 0, sizeof(uuid5)); + taosGetSystemUUIDLimit36(uuid5, sizeof(uuid5)); + ASSERT_EQ(strlen(uuid5), 35); + + char uuid6[37]; + memset( uuid6, 0, sizeof(uuid6)); + taosGetSystemUUIDLimit36(uuid6, sizeof(uuid6)); + ASSERT_EQ(strlen(uuid6), 36); + + char uuid7[1]; + memset(uuid7, 0, sizeof(uuid7)); + taosGetSystemUUIDLimit36(uuid7, sizeof(uuid7)); + ASSERT_EQ(strlen(uuid7), 0); +} + +TEST(osSystemTest, systemUUIDTest2) { + char uuid1[38]; + memset(uuid1, 0, sizeof(uuid1)); + taosGetSystemUUIDLen(uuid1, sizeof(uuid1)); + ASSERT_EQ(strlen(uuid1), sizeof(uuid1) - 1); + + char uuid2[34]; + memset(uuid2, 0, sizeof(uuid2)); + taosGetSystemUUIDLen(uuid2, sizeof(uuid2)); + ASSERT_EQ(strlen(uuid2), sizeof(uuid2) - 1); + + char uuid3[36]; + memset(uuid3, 0, sizeof(uuid3)); + taosGetSystemUUIDLen(uuid3, sizeof(uuid3)); + ASSERT_EQ(strlen(uuid3), sizeof(uuid3) - 1); + + char uuid4[2]; + memset(uuid4, 0, sizeof(uuid4)); + taosGetSystemUUIDLen(uuid4, sizeof(uuid4)); + ASSERT_EQ(strlen(uuid4), sizeof(uuid4) - 1); + + char uuid5[36]; + memset( uuid5, 0, sizeof(uuid5)); + taosGetSystemUUIDLen(uuid5, sizeof(uuid5)); + ASSERT_EQ(strlen(uuid5), sizeof(uuid5) - 1); + + char uuid6[37]; + memset( uuid6, 0, sizeof(uuid6)); + taosGetSystemUUIDLen(uuid6, sizeof(uuid6)); + ASSERT_EQ(strlen(uuid6), sizeof(uuid6) - 1); + + char uuid7[1]; + memset(uuid7, 0, sizeof(uuid7)); + taosGetSystemUUIDLen(uuid7, sizeof(uuid7)); + ASSERT_EQ(strlen(uuid7), sizeof(uuid7) - 1); + + char uuid8[40]; + memset(uuid8, 0, sizeof(uuid8)); + taosGetSystemUUIDLen(uuid8, sizeof(uuid8)); + ASSERT_EQ(strlen(uuid8), sizeof(uuid8) - 1); + + char uuid9[73]; + memset(uuid9, 0, sizeof(uuid9)); + taosGetSystemUUIDLen(uuid9, sizeof(uuid9)); + ASSERT_EQ(strlen(uuid9), sizeof(uuid9) - 1); +} diff --git a/source/os/test/osTests.cpp b/source/os/test/osTests.cpp index 9786886c56..2c20348608 100644 --- a/source/os/test/osTests.cpp +++ b/source/os/test/osTests.cpp @@ -37,7 +37,7 @@ #include TEST(osTest, osFQDNSuccess) { - char fqdn[1024]; + char fqdn[TD_FQDN_LEN]; char ipString[INET_ADDRSTRLEN]; int code = taosGetFqdn(fqdn); uint32_t ipv4 = 0; diff --git a/source/os/test/osTimeTests.cpp b/source/os/test/osTimeTests.cpp index 90c089e310..42837cc537 100644 --- a/source/os/test/osTimeTests.cpp +++ b/source/os/test/osTimeTests.cpp @@ -31,20 +31,20 @@ TEST(osTimeTests, taosLocalTimeNolock) { time_t currentTime; - // Test when result is NULL - struct tm* result = taosLocalTimeNolock(NULL, ¤tTime, 0); // Test when result is not NULL struct tm expectedTime; - result = taosLocalTimeNolock(&expectedTime, ¤tTime, 1); - EXPECT_EQ(expectedTime.tm_year, result->tm_year); - EXPECT_EQ(expectedTime.tm_mon, result->tm_mon); - EXPECT_EQ(expectedTime.tm_mday, result->tm_mday); - EXPECT_EQ(expectedTime.tm_hour, result->tm_hour); - EXPECT_EQ(expectedTime.tm_min, result->tm_min); - EXPECT_EQ(expectedTime.tm_sec, result->tm_sec); - EXPECT_EQ(expectedTime.tm_wday, result->tm_wday); - EXPECT_EQ(expectedTime.tm_yday, result->tm_yday); - EXPECT_EQ(expectedTime.tm_isdst, result->tm_isdst); + struct tm* result = taosLocalTimeNolock(&expectedTime, ¤tTime, 1); + if (result) { + EXPECT_EQ(expectedTime.tm_year, result->tm_year); + EXPECT_EQ(expectedTime.tm_mon, result->tm_mon); + EXPECT_EQ(expectedTime.tm_mday, result->tm_mday); + EXPECT_EQ(expectedTime.tm_hour, result->tm_hour); + EXPECT_EQ(expectedTime.tm_min, result->tm_min); + EXPECT_EQ(expectedTime.tm_sec, result->tm_sec); + EXPECT_EQ(expectedTime.tm_wday, result->tm_wday); + EXPECT_EQ(expectedTime.tm_yday, result->tm_yday); + EXPECT_EQ(expectedTime.tm_isdst, result->tm_isdst); + } } @@ -52,7 +52,7 @@ TEST(osTimeTests, taosLocalTime) { // Test 1: Test when both timep and result are not NULL time_t timep = 1617531000; // 2021-04-04 18:10:00 struct tm result; - struct tm* local_time = taosLocalTime(&timep, &result, NULL); + struct tm* local_time = taosLocalTime(&timep, &result, NULL, 0); ASSERT_NE(local_time, nullptr); ASSERT_EQ(local_time->tm_year, 121); ASSERT_EQ(local_time->tm_mon, 3); @@ -62,20 +62,13 @@ TEST(osTimeTests, taosLocalTime) { ASSERT_EQ(local_time->tm_sec, 00); // Test 2: Test when timep is NULL - local_time = taosLocalTime(NULL, &result, NULL); + local_time = taosLocalTime(NULL, &result, NULL, 0); ASSERT_EQ(local_time, nullptr); - // Test 3: Test when result is NULL - local_time = taosLocalTime(&timep, NULL, NULL); - ASSERT_NE(local_time, nullptr); - ASSERT_EQ(local_time->tm_year, 121); - ASSERT_EQ(local_time->tm_mon, 3); - ASSERT_EQ(local_time->tm_mday, 4); - // Test 4: Test when timep is negative on Windows #ifdef WINDOWS time_t pos_timep = 1609459200; // 2021-01-01 08:00:00 - local_time = taosLocalTime(&pos_timep, &result, NULL); + local_time = taosLocalTime(&pos_timep, &result, NULL, 0); ASSERT_NE(local_time, nullptr); ASSERT_EQ(local_time->tm_year, 121); ASSERT_EQ(local_time->tm_mon, 0); @@ -85,7 +78,7 @@ TEST(osTimeTests, taosLocalTime) { ASSERT_EQ(local_time->tm_sec, 0); time_t neg_timep = -1617531000; // 1918-09-29 21:50:00 - local_time = taosLocalTime(&neg_timep, &result, NULL); + local_time = taosLocalTime(&neg_timep, &result, NULL, 0); ASSERT_NE(local_time, nullptr); ASSERT_EQ(local_time->tm_year, 18); ASSERT_EQ(local_time->tm_mon, 8); @@ -95,7 +88,7 @@ TEST(osTimeTests, taosLocalTime) { ASSERT_EQ(local_time->tm_sec, 0); time_t neg_timep2 = -315619200; // 1960-01-01 08:00:00 - local_time = taosLocalTime(&neg_timep2, &result, NULL); + local_time = taosLocalTime(&neg_timep2, &result, NULL, 0); ASSERT_NE(local_time, nullptr); ASSERT_EQ(local_time->tm_year, 60); ASSERT_EQ(local_time->tm_mon, 0); @@ -105,7 +98,7 @@ TEST(osTimeTests, taosLocalTime) { ASSERT_EQ(local_time->tm_sec, 0); time_t zero_timep = 0; // 1970-01-01 08:00:00 - local_time = taosLocalTime(&zero_timep, &result, NULL); + local_time = taosLocalTime(&zero_timep, &result, NULL, 0); ASSERT_NE(local_time, nullptr); ASSERT_EQ(local_time->tm_year, 70); ASSERT_EQ(local_time->tm_mon, 0); @@ -115,7 +108,7 @@ TEST(osTimeTests, taosLocalTime) { ASSERT_EQ(local_time->tm_sec, 0); time_t neg_timep3 = -78115158887; - local_time = taosLocalTime(&neg_timep3, &result, NULL); + local_time = taosLocalTime(&neg_timep3, &result, NULL, 0); ASSERT_EQ(local_time, nullptr); #endif } \ No newline at end of file diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 288d440d86..9c9ded693e 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -64,29 +64,33 @@ #include "td_sz.h" int32_t tsCompressPlain2(const char *const input, const int32_t nelements, char *const output, const char type); -int32_t tsDecompressPlain2(const char *const input, const int32_t nelements, char *const output, const char type); +int32_t tsDecompressPlain2(const char *const input, int32_t ninput, const int32_t nelements, char *const output, + const char type); // delta int32_t tsCompressTimestampImp2(const char *const input, const int32_t nelements, char *const output, const char type); -int32_t tsDecompressTimestampImp2(const char *const input, const int32_t nelements, char *const output, +int32_t tsDecompressTimestampImp2(const char *const input, int32_t ninput, const int32_t nelements, char *const output, const char type); // simple8b int32_t tsCompressINTImp2(const char *const input, const int32_t nelements, char *const output, const char type); -int32_t tsDecompressINTImp2(const char *const input, const int32_t nelements, char *const output, const char type); +int32_t tsDecompressINTImp2(const char *const input, int32_t ninput, const int32_t nelements, char *const output, + const char type); // bit int32_t tsCompressBoolImp2(const char *const input, const int32_t nelements, char *const output, char const type); -int32_t tsDecompressBoolImp2(const char *const input, const int32_t nelements, char *const output, char const type); +int32_t tsDecompressBoolImp2(const char *const input, int32_t ninput, const int32_t nelements, char *const output, + char const type); // double specail int32_t tsCompressDoubleImp2(const char *const input, const int32_t nelements, char *const output, char const type); -int32_t tsDecompressDoubleImp2(const char *const input, const int32_t nelements, char *const output, char const type); +int32_t tsDecompressDoubleImp2(const char *const input, int32_t ninput, const int32_t nelements, char *const output, + char const type); int32_t tsCompressDoubleImp(const char *const input, const int32_t nelements, char *const output); -int32_t tsDecompressDoubleImp(const char *const input, const int32_t nelements, char *const output); +int32_t tsDecompressDoubleImp(const char *const input, int32_t ninput, const int32_t nelements, char *const output); int32_t tsCompressFloatImp(const char *const input, const int32_t nelements, char *const output); -int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, char *const output); +int32_t tsDecompressFloatImp(const char *const input, int32_t ninput, const int32_t nelements, char *const output); int32_t l2ComressInitImpl_disabled(char *lossyColumns, float fPrecision, double dPrecision, uint32_t maxIntervals, uint32_t intervals, int32_t ifAdtFse, const char *compressor) { @@ -457,8 +461,8 @@ int32_t tsCompressINTImp(const char *const input, const int32_t nelements, char int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, char *const output, const char type) { int32_t word_length = getWordLength(type); - if (word_length == -1) { - return word_length; + if (word_length < 0) { + return -1; } // If not compressed. @@ -467,70 +471,106 @@ int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, cha return nelements * word_length; } -#if __AVX2__ - tsDecompressIntImpl_Hw(input, nelements, output, type); - return nelements * word_length; -#else +#ifdef __AVX512F__ + if (tsSIMDEnable && tsAVX512Enable && tsAVX512Supported) { + tsDecompressIntImpl_Hw(input, nelements, output, type); + return nelements * word_length; + } +#endif + // Selector value: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 char bit_per_integer[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60}; int32_t selector_to_elems[] = {240, 120, 60, 30, 20, 15, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1}; const char *ip = input + 1; + char *op = output; int32_t count = 0; - int32_t _pos = 0; int64_t prev_value = 0; - while (1) { - if (count == nelements) break; - - uint64_t w = 0; - memcpy(&w, ip, LONG_BYTES); + while (count < nelements) { + uint64_t w = *(uint64_t *)ip; char selector = (char)(w & INT64MASK(4)); // selector = 4 char bit = bit_per_integer[(int32_t)selector]; // bit = 3 int32_t elems = selector_to_elems[(int32_t)selector]; - for (int32_t i = 0; i < elems; i++) { - uint64_t zigzag_value; - - if (selector == 0 || selector == 1) { - zigzag_value = 0; - } else { - zigzag_value = ((w >> (4 + bit * i)) & INT64MASK(bit)); + switch (type) { + case TSDB_DATA_TYPE_BIGINT: { + int64_t *out = (int64_t *)op; + if (selector == 0 || selector == 1) { + for (int32_t i = 0; i < elems && count < nelements; ++i, ++count, ++out) { + *out = prev_value; + } + } else { + uint64_t zigzag_value = 0; + for (int32_t i = 0; i < elems && count < nelements; ++i, ++count, ++out) { + zigzag_value = ((w >> (4 + bit * i)) & INT64MASK(bit)); + prev_value += ZIGZAG_DECODE(int64_t, zigzag_value); + *out = prev_value; + } + } + op = (char *)out; + break; } - int64_t diff = ZIGZAG_DECODE(int64_t, zigzag_value); - int64_t curr_value = diff + prev_value; - prev_value = curr_value; - - switch (type) { - case TSDB_DATA_TYPE_BIGINT: - *((int64_t *)output + _pos) = (int64_t)curr_value; - _pos++; - break; - case TSDB_DATA_TYPE_INT: - *((int32_t *)output + _pos) = (int32_t)curr_value; - _pos++; - break; - case TSDB_DATA_TYPE_SMALLINT: - *((int16_t *)output + _pos) = (int16_t)curr_value; - _pos++; - break; - case TSDB_DATA_TYPE_TINYINT: - *((int8_t *)output + _pos) = (int8_t)curr_value; - _pos++; - break; - default: - perror("Wrong integer types.\n"); - return -1; + case TSDB_DATA_TYPE_INT: { + int32_t *out = (int32_t *)op; + if (selector == 0 || selector == 1) { + for (int32_t i = 0; i < elems && count < nelements; ++i, ++count, ++out) { + *out = (int32_t)prev_value; + } + } else { + uint64_t zigzag_value = 0; + for (int32_t i = 0; i < elems && count < nelements; ++i, ++count, ++out) { + zigzag_value = ((w >> (4 + bit * i)) & INT64MASK(bit)); + prev_value += ZIGZAG_DECODE(int64_t, zigzag_value); + *out = (int32_t)prev_value; + } + } + op = (char *)out; + break; } - count++; - if (count == nelements) break; + case TSDB_DATA_TYPE_SMALLINT: { + int16_t *out = (int16_t *)op; + if (selector == 0 || selector == 1) { + for (int32_t i = 0; i < elems && count < nelements; ++i, ++count, ++out) { + *out = (int16_t)prev_value; + } + } else { + uint64_t zigzag_value = 0; + for (int32_t i = 0; i < elems && count < nelements; ++i, ++count, ++out) { + zigzag_value = ((w >> (4 + bit * i)) & INT64MASK(bit)); + prev_value += ZIGZAG_DECODE(int64_t, zigzag_value); + *out = (int16_t)prev_value; + } + } + op = (char *)out; + break; + } + case TSDB_DATA_TYPE_TINYINT: { + int8_t *out = (int8_t *)op; + if (selector == 0 || selector == 1) { + for (int32_t i = 0; i < elems && count < nelements; ++i, ++count, ++out) { + *out = (int8_t)prev_value; + } + } else { + uint64_t zigzag_value = 0; + for (int32_t i = 0; i < elems && count < nelements; ++i, ++count, ++out) { + zigzag_value = ((w >> (4 + bit * i)) & INT64MASK(bit)); + prev_value += ZIGZAG_DECODE(int64_t, zigzag_value); + *out = (int8_t)prev_value; + } + } + op = (char *)out; + break; + } + default: + perror("Wrong integer types.\n"); + return -1; } ip += LONG_BYTES; } return nelements * word_length; -#endif } /* ----------------------------------------------Bool Compression ---------------------------------------------- */ @@ -590,7 +630,8 @@ int32_t tsDecompressBoolImp(const char *const input, const int32_t nelements, ch int32_t tsCompressBoolImp2(const char *const input, const int32_t nelements, char *const output, char const type) { return tsCompressBoolImp(input, nelements, output); } -int32_t tsDecompressBoolImp2(const char *const input, const int32_t nelements, char *const output, char const type) { +int32_t tsDecompressBoolImp2(const char *const input, int32_t ninput, const int32_t nelements, char *const output, + char const type) { return tsDecompressBoolImp(input, nelements, output); } @@ -602,18 +643,20 @@ int32_t tsCompressDoubleImp2(const char *const input, const int32_t nelements, c } return TSDB_CODE_THIRDPARTY_ERROR; } -int32_t tsDecompressDoubleImp2(const char *const input, const int32_t nelements, char *const output, char const type) { +int32_t tsDecompressDoubleImp2(const char *const input, int32_t ninput, const int32_t nelements, char *const output, + char const type) { if (type == TSDB_DATA_TYPE_FLOAT) { - return tsDecompressFloatImp(input, nelements, output); + return tsDecompressFloatImp(input, ninput, nelements, output); } else if (type == TSDB_DATA_TYPE_DOUBLE) { - return tsDecompressDoubleImp(input, nelements, output); + return tsDecompressDoubleImp(input, ninput, nelements, output); } return TSDB_CODE_THIRDPARTY_ERROR; } int32_t tsCompressINTImp2(const char *const input, const int32_t nelements, char *const output, const char type) { return tsCompressINTImp(input, nelements, output, type); } -int32_t tsDecompressINTImp2(const char *const input, const int32_t nelements, char *const output, const char type) { +int32_t tsDecompressINTImp2(const char *const input, int32_t ninput, const int32_t nelements, char *const output, + const char type) { return tsDecompressINTImp(input, nelements, output, type); } @@ -824,67 +867,68 @@ int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelement memcpy(output, input + 1, nelements * longBytes); return nelements * longBytes; } else if (input[0] == 1) { // Decompress - if (tsSIMDEnable && tsAVX512Supported && tsAVX512Enable) { - tsDecompressTimestampAvx512(input, nelements, output, false); - } else if (tsSIMDEnable && tsAVX2Supported) { - tsDecompressTimestampAvx2(input, nelements, output, false); - } else { - int64_t *ostream = (int64_t *)output; +#ifdef __AVX512VL__ + if (tsSIMDEnable && tsAVX512Enable && tsAVX512Supported) { + tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output, bool bigEndian); + return nelements * longBytes; + } +#endif - int32_t ipos = 1, opos = 0; - int8_t nbytes = 0; - int64_t prev_value = 0; - int64_t prev_delta = 0; - int64_t delta_of_delta = 0; + int64_t *ostream = (int64_t *)output; - while (1) { - uint8_t flags = input[ipos++]; - // Decode dd1 - uint64_t dd1 = 0; - nbytes = flags & INT8MASK(4); - if (nbytes == 0) { - delta_of_delta = 0; + int32_t ipos = 1, opos = 0; + int8_t nbytes = 0; + int64_t prev_value = 0; + int64_t prev_delta = 0; + int64_t delta_of_delta = 0; + + while (1) { + uint8_t flags = input[ipos++]; + // Decode dd1 + uint64_t dd1 = 0; + nbytes = flags & INT8MASK(4); + if (nbytes == 0) { + delta_of_delta = 0; + } else { + if (is_bigendian()) { + memcpy(((char *)(&dd1)) + longBytes - nbytes, input + ipos, nbytes); } else { - if (is_bigendian()) { - memcpy(((char *)(&dd1)) + longBytes - nbytes, input + ipos, nbytes); - } else { - memcpy(&dd1, input + ipos, nbytes); - } - delta_of_delta = ZIGZAG_DECODE(int64_t, dd1); + memcpy(&dd1, input + ipos, nbytes); } + delta_of_delta = ZIGZAG_DECODE(int64_t, dd1); + } - ipos += nbytes; - if (opos == 0) { - prev_value = delta_of_delta; - prev_delta = 0; - ostream[opos++] = delta_of_delta; - } else { - prev_delta = delta_of_delta + prev_delta; - prev_value = prev_value + prev_delta; - ostream[opos++] = prev_value; - } - if (opos == nelements) return nelements * longBytes; - - // Decode dd2 - uint64_t dd2 = 0; - nbytes = (flags >> 4) & INT8MASK(4); - if (nbytes == 0) { - delta_of_delta = 0; - } else { - if (is_bigendian()) { - memcpy(((char *)(&dd2)) + longBytes - nbytes, input + ipos, nbytes); - } else { - memcpy(&dd2, input + ipos, nbytes); - } - // zigzag_decoding - delta_of_delta = ZIGZAG_DECODE(int64_t, dd2); - } - ipos += nbytes; + ipos += nbytes; + if (opos == 0) { + prev_value = delta_of_delta; + prev_delta = 0; + ostream[opos++] = delta_of_delta; + } else { prev_delta = delta_of_delta + prev_delta; prev_value = prev_value + prev_delta; ostream[opos++] = prev_value; - if (opos == nelements) return nelements * longBytes; } + if (opos == nelements) return nelements * longBytes; + + // Decode dd2 + uint64_t dd2 = 0; + nbytes = (flags >> 4) & INT8MASK(4); + if (nbytes == 0) { + delta_of_delta = 0; + } else { + if (is_bigendian()) { + memcpy(((char *)(&dd2)) + longBytes - nbytes, input + ipos, nbytes); + } else { + memcpy(&dd2, input + ipos, nbytes); + } + // zigzag_decoding + delta_of_delta = ZIGZAG_DECODE(int64_t, dd2); + } + ipos += nbytes; + prev_delta = delta_of_delta + prev_delta; + prev_value = prev_value + prev_delta; + ostream[opos++] = prev_value; + if (opos == nelements) return nelements * longBytes; } } @@ -897,7 +941,8 @@ int32_t tsCompressPlain2(const char *const input, const int32_t nelements, char memcpy(output + 1, input, bytes); return bytes + 1; } -int32_t tsDecompressPlain2(const char *const input, const int32_t nelements, char *const output, const char type) { +int32_t tsDecompressPlain2(const char *const input, int32_t ninput, const int32_t nelements, char *const output, + const char type) { int32_t bytes = tDataTypes[type].bytes * nelements; memcpy(output, input + 1, bytes); return bytes; @@ -905,7 +950,7 @@ int32_t tsDecompressPlain2(const char *const input, const int32_t nelements, cha int32_t tsCompressTimestampImp2(const char *const input, const int32_t nelements, char *const output, const char type) { return tsCompressTimestampImp(input, nelements, output); } -int32_t tsDecompressTimestampImp2(const char *const input, const int32_t nelements, char *const output, +int32_t tsDecompressTimestampImp2(const char *const input, int32_t ninput, const int32_t nelements, char *const output, const char type) { return tsDecompressTimestampImp(input, nelements, output); } @@ -1023,17 +1068,10 @@ FORCE_INLINE uint64_t decodeDoubleValue(const char *const input, int32_t *const return diff; } -int32_t tsDecompressDoubleImp(const char *const input, const int32_t nelements, char *const output) { - // output stream - double *ostream = (double *)output; - - if (input[0] == 1) { - memcpy(output, input + 1, nelements * DOUBLE_BYTES); - return nelements * DOUBLE_BYTES; - } - +static int32_t tsDecompressDoubleImpHelper(const char *input, int32_t nelements, char *output) { + double *ostream = (double *)output; uint8_t flags = 0; - int32_t ipos = 1; + int32_t ipos = 0; int32_t opos = 0; uint64_t diff = 0; union { @@ -1048,7 +1086,7 @@ int32_t tsDecompressDoubleImp(const char *const input, const int32_t nelements, flags = input[ipos++]; } - diff = decodeDoubleValue(input, &ipos, flags & 0x0f); + diff = decodeDoubleValue(input, &ipos, flags & INT8MASK(4)); flags >>= 4; curr.bits ^= diff; @@ -1058,6 +1096,25 @@ int32_t tsDecompressDoubleImp(const char *const input, const int32_t nelements, return nelements * DOUBLE_BYTES; } +int32_t tsDecompressDoubleImp(const char *const input, int32_t ninput, const int32_t nelements, char *const output) { + // return the result directly if there is no compression + if (input[0] == 1) { + memcpy(output, input + 1, nelements * DOUBLE_BYTES); + return nelements * DOUBLE_BYTES; + } + +#ifdef __AVX2__ + // use AVX2 implementation when allowed and the compression ratio is not high + double compressRatio = 1.0 * nelements * DOUBLE_BYTES / ninput; + if (tsSIMDEnable && tsAVX2Supported && compressRatio < 2) { + return tsDecompressDoubleImpAvx2(input + 1, nelements, output); + } +#endif + + // use implementation without SIMD instructions by default + return tsDecompressDoubleImpHelper(input + 1, nelements, output); +} + /* --------------------------------------------Float Compression ---------------------------------------------- */ void encodeFloatValue(uint32_t diff, uint8_t flag, char *const output, int32_t *const pos) { uint8_t nbytes = (flag & INT8MASK(3)) + 1; @@ -1166,49 +1223,50 @@ uint32_t decodeFloatValue(const char *const input, int32_t *const ipos, uint8_t return diff; } -static void tsDecompressFloatHelper(const char *const input, const int32_t nelements, float *ostream) { +static int32_t tsDecompressFloatImpHelper(const char *input, int32_t nelements, char *output) { + float *ostream = (float *)output; uint8_t flags = 0; - int32_t ipos = 1; + int32_t ipos = 0; int32_t opos = 0; - uint32_t prev_value = 0; + uint32_t diff = 0; + union { + uint32_t bits; + float real; + } curr; + + curr.bits = 0; for (int32_t i = 0; i < nelements; i++) { if (i % 2 == 0) { flags = input[ipos++]; } - uint8_t flag = flags & INT8MASK(4); + diff = decodeFloatValue(input, &ipos, flags & INT8MASK(4)); flags >>= 4; - - uint32_t diff = decodeFloatValue(input, &ipos, flag); - union { - uint32_t bits; - float real; - } curr; - - uint32_t predicted = prev_value; - curr.bits = predicted ^ diff; - prev_value = curr.bits; + curr.bits ^= diff; ostream[opos++] = curr.real; } + + return nelements * FLOAT_BYTES; } -int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, char *const output) { +int32_t tsDecompressFloatImp(const char *const input, int32_t ninput, const int32_t nelements, char *const output) { if (input[0] == 1) { memcpy(output, input + 1, nelements * FLOAT_BYTES); return nelements * FLOAT_BYTES; } - if (tsSIMDEnable && tsAVX2Supported) { - tsDecompressFloatImplAvx2(input, nelements, output); - } else if (tsSIMDEnable && tsAVX512Supported && tsAVX512Enable) { - tsDecompressFloatImplAvx512(input, nelements, output); - } else { // alternative implementation without SIMD instructions. - tsDecompressFloatHelper(input, nelements, (float *)output); +#ifdef __AVX2__ + // use AVX2 implementation when allowed and the compression ratio is not high + double compressRatio = 1.0 * nelements * FLOAT_BYTES / ninput; + if (tsSIMDEnable && tsAVX2Supported && compressRatio < 2) { + return tsDecompressFloatImpAvx2(input + 1, nelements, output); } +#endif - return nelements * FLOAT_BYTES; + // use implementation without SIMD instructions by default + return tsDecompressFloatImpHelper(input + 1, nelements, output); } // @@ -1336,10 +1394,11 @@ int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int3 } else { // decompress lossless if (cmprAlg == ONE_STAGE_COMP) { - return tsDecompressFloatImp(pIn, nEle, pOut); + return tsDecompressFloatImp(pIn, nIn, nEle, pOut); } else if (cmprAlg == TWO_STAGE_COMP) { - if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; - return tsDecompressFloatImp(pBuf, nEle, pOut); + int32_t bufLen = tsDecompressStringImp(pIn, nIn, pBuf, nBuf); + if (bufLen < 0) return -1; + return tsDecompressFloatImp(pBuf, bufLen, nEle, pOut); } else { return TSDB_CODE_INVALID_PARA; } @@ -1373,10 +1432,11 @@ int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int } else { // decompress lossless if (cmprAlg == ONE_STAGE_COMP) { - return tsDecompressDoubleImp(pIn, nEle, pOut); + return tsDecompressDoubleImp(pIn, nIn, nEle, pOut); } else if (cmprAlg == TWO_STAGE_COMP) { - if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; - return tsDecompressDoubleImp(pBuf, nEle, pOut); + int32_t bufLen = tsDecompressStringImp(pIn, nIn, pBuf, nBuf); + if (bufLen < 0) return -1; + return tsDecompressDoubleImp(pBuf, bufLen, nEle, pOut); } else { return TSDB_CODE_INVALID_PARA; } @@ -1550,7 +1610,7 @@ int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int } else { \ uTrace("dencode:%s, compress:%s, level:%s, type:%s", compressL1Dict[l1].name, "disabled", "disabled", \ tDataTypes[type].name); \ - return compressL1Dict[l1].decomprFn(pIn, nEle, pOut, type); \ + return compressL1Dict[l1].decomprFn(pIn, nIn, nEle, pOut, type); \ } \ } else if (l1 != L1_DISABLED && l2 != L2_DISABLED) { \ if (compress) { \ @@ -1562,8 +1622,9 @@ int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int } else { \ uTrace("dencode:%s, decompress:%s, level:%d, type:%s", compressL1Dict[l1].name, compressL2Dict[l2].name, lvl, \ tDataTypes[type].name); \ - if (compressL2Dict[l2].decomprFn(pIn, nIn, pBuf, nBuf, type) < 0) return -1; \ - return compressL1Dict[l1].decomprFn(pBuf, nEle, pOut, type); \ + int32_t bufLen = compressL2Dict[l2].decomprFn(pIn, nIn, pBuf, nBuf, type); \ + if (bufLen < 0) return -1; \ + return compressL1Dict[l1].decomprFn(pBuf, bufLen, nEle, pOut, type); \ } \ } else if (l1 == L1_DISABLED && l2 != L2_DISABLED) { \ if (compress) { \ diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index d84a426e98..d6852b0566 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -672,17 +672,17 @@ int32_t cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t int32_t len = 0; switch (pItem->dtype) { case CFG_DTYPE_BOOL: - len = snprintf(buf, bufSize, "%u", pItem->bval); + len = tsnprintf(buf, bufSize, "%u", pItem->bval); break; case CFG_DTYPE_INT32: - len = snprintf(buf, bufSize, "%d", pItem->i32); + len = tsnprintf(buf, bufSize, "%d", pItem->i32); break; case CFG_DTYPE_INT64: - len = snprintf(buf, bufSize, "%" PRId64, pItem->i64); + len = tsnprintf(buf, bufSize, "%" PRId64, pItem->i64); break; case CFG_DTYPE_FLOAT: case CFG_DTYPE_DOUBLE: - len = snprintf(buf, bufSize, "%f", pItem->fval); + len = tsnprintf(buf, bufSize, "%f", pItem->fval); break; case CFG_DTYPE_STRING: case CFG_DTYPE_DIR: @@ -690,7 +690,7 @@ int32_t cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: case CFG_DTYPE_NONE: - len = snprintf(buf, bufSize, "%s", pItem->str); + len = tsnprintf(buf, bufSize, "%s", pItem->str); break; } @@ -710,13 +710,13 @@ int32_t cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t int32_t len = 0; switch (pItem->scope) { case CFG_SCOPE_SERVER: - len = snprintf(buf, bufSize, "server"); + len = tsnprintf(buf, bufSize, "server"); break; case CFG_SCOPE_CLIENT: - len = snprintf(buf, bufSize, "client"); + len = tsnprintf(buf, bufSize, "client"); break; case CFG_SCOPE_BOTH: - len = snprintf(buf, bufSize, "both"); + len = tsnprintf(buf, bufSize, "both"); break; } diff --git a/source/util/src/tdecompress.c b/source/util/src/tdecompress.c index 598aca730f..81223d7311 100644 --- a/source/util/src/tdecompress.c +++ b/source/util/src/tdecompress.c @@ -40,6 +40,7 @@ int32_t getWordLength(char type) { return wordLength; } +#ifdef __AVX2__ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, char *const output, const char type) { int32_t word_length = getWordLength(type); @@ -52,7 +53,6 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, int32_t _pos = 0; int64_t prevValue = 0; -#if __AVX2__ || __AVX512F__ while (_pos < nelements) { uint64_t w = *(uint64_t *)ip; @@ -309,30 +309,154 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, ip += LONG_BYTES; } -#endif return nelements * word_length; } -void tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output) { -#if __AVX512F__ - // todo add it -#endif - return; +#define M256_BYTES sizeof(__m256i) + +FORCE_INLINE __m256i decodeFloatAvx2(const char *data, const char *flag) { + __m256i dataVec = _mm256_load_si256((__m256i *)data); + __m256i flagVec = _mm256_load_si256((__m256i *)flag); + __m256i k7 = _mm256_set1_epi32(7); + __m256i lopart = _mm256_set_epi32(0, -1, 0, -1, 0, -1, 0, -1); + __m256i hipart = _mm256_set_epi32(-1, 0, -1, 0, -1, 0, -1, 0); + __m256i trTail = _mm256_cmpgt_epi32(flagVec, k7); + __m256i trHead = _mm256_andnot_si256(trTail, _mm256_set1_epi32(-1)); + __m256i shiftVec = _mm256_slli_epi32(_mm256_sub_epi32(_mm256_set1_epi32(3), _mm256_and_si256(flagVec, k7)), 3); + __m256i maskVec = hipart; + __m256i diffVec = _mm256_sllv_epi32(dataVec, _mm256_and_si256(shiftVec, maskVec)); + maskVec = _mm256_or_si256(trHead, lopart); + diffVec = _mm256_srlv_epi32(diffVec, _mm256_and_si256(shiftVec, maskVec)); + maskVec = _mm256_and_si256(trTail, lopart); + diffVec = _mm256_sllv_epi32(diffVec, _mm256_and_si256(shiftVec, maskVec)); + return diffVec; } -// todo add later -void tsDecompressFloatImplAvx2(const char *const input, const int32_t nelements, char *const output) { -#if __AVX2__ -#endif - return; +int32_t tsDecompressFloatImpAvx2(const char *input, int32_t nelements, char *output) { + // Allocate memory-aligned buffer + char buf[M256_BYTES * 3]; + memset(buf, 0, sizeof(buf)); + char *data = (char *)ALIGN_NUM((uint64_t)buf, M256_BYTES); + char *flag = data + M256_BYTES; + const char *in = input; + char *out = output; + + // Load data into the buffer for batch processing + int32_t batchSize = M256_BYTES / FLOAT_BYTES; + int32_t idx = 0; + uint32_t cur = 0; + for (int32_t i = 0; i < nelements; i += 2) { + if (idx == batchSize) { + // Start processing when the buffer is full + __m256i resVec = decodeFloatAvx2(data, flag); + _mm256_storeu_si256((__m256i *)out, resVec); + uint32_t *p = (uint32_t *)out; + for (int32_t j = 0; j < batchSize; ++j) { + p[j] = cur = (p[j] ^ cur); + } + out += M256_BYTES; + idx = 0; + } + uint8_t flag1 = (*in) & 0xF; + uint8_t flag2 = ((*in) >> 4) & 0xF; + int32_t nbytes1 = (flag1 & 0x7) + 1; + int32_t nbytes2 = (flag2 & 0x7) + 1; + in++; + flag[idx * FLOAT_BYTES] = flag1; + flag[(idx + 1) * FLOAT_BYTES] = flag2; + memcpy(data + (idx + 1) * FLOAT_BYTES - nbytes1, in, nbytes1 + nbytes2); + in += nbytes1 + nbytes2; + idx += 2; + } + if (idx) { + idx -= (nelements & 0x1); + // Process the remaining few bytes + __m256i resVec = decodeFloatAvx2(data, flag); + memcpy(out, &resVec, idx * FLOAT_BYTES); + uint32_t *p = (uint32_t *)out; + for (int32_t j = 0; j < idx; ++j) { + p[j] = cur = (p[j] ^ cur); + } + out += idx * FLOAT_BYTES; + } + return (int32_t)(out - output); } +FORCE_INLINE __m256i decodeDoubleAvx2(const char *data, const char *flag) { + __m256i dataVec = _mm256_load_si256((__m256i *)data); + __m256i flagVec = _mm256_load_si256((__m256i *)flag); + __m256i k7 = _mm256_set1_epi64x(7); + __m256i lopart = _mm256_set_epi64x(0, -1, 0, -1); + __m256i hipart = _mm256_set_epi64x(-1, 0, -1, 0); + __m256i trTail = _mm256_cmpgt_epi64(flagVec, k7); + __m256i trHead = _mm256_andnot_si256(trTail, _mm256_set1_epi64x(-1)); + __m256i shiftVec = _mm256_slli_epi64(_mm256_sub_epi64(k7, _mm256_and_si256(flagVec, k7)), 3); + __m256i maskVec = hipart; + __m256i diffVec = _mm256_sllv_epi64(dataVec, _mm256_and_si256(shiftVec, maskVec)); + maskVec = _mm256_or_si256(trHead, lopart); + diffVec = _mm256_srlv_epi64(diffVec, _mm256_and_si256(shiftVec, maskVec)); + maskVec = _mm256_and_si256(trTail, lopart); + diffVec = _mm256_sllv_epi64(diffVec, _mm256_and_si256(shiftVec, maskVec)); + return diffVec; +} + +int32_t tsDecompressDoubleImpAvx2(const char *input, const int32_t nelements, char *const output) { + // Allocate memory-aligned buffer + char buf[M256_BYTES * 3]; + memset(buf, 0, sizeof(buf)); + char *data = (char *)ALIGN_NUM((uint64_t)buf, M256_BYTES); + char *flag = data + M256_BYTES; + const char *in = input; + char *out = output; + + // Load data into the buffer for batch processing + int32_t batchSize = M256_BYTES / DOUBLE_BYTES; + int32_t idx = 0; + uint64_t cur = 0; + for (int32_t i = 0; i < nelements; i += 2) { + if (idx == batchSize) { + // Start processing when the buffer is full + __m256i resVec = decodeDoubleAvx2(data, flag); + _mm256_storeu_si256((__m256i *)out, resVec); + uint64_t *p = (uint64_t *)out; + for (int32_t j = 0; j < batchSize; ++j) { + p[j] = cur = (p[j] ^ cur); + } + out += M256_BYTES; + idx = 0; + } + uint8_t flag1 = (*in) & 0xF; + uint8_t flag2 = ((*in) >> 4) & 0xF; + int32_t nbytes1 = (flag1 & 0x7) + 1; + int32_t nbytes2 = (flag2 & 0x7) + 1; + in++; + flag[idx * DOUBLE_BYTES] = flag1; + flag[(idx + 1) * DOUBLE_BYTES] = flag2; + memcpy(data + (idx + 1) * DOUBLE_BYTES - nbytes1, in, nbytes1 + nbytes2); + in += nbytes1 + nbytes2; + idx += 2; + } + if (idx) { + idx -= (nelements & 0x1); + // Process the remaining few bytes + __m256i resVec = decodeDoubleAvx2(data, flag); + memcpy(out, &resVec, idx * DOUBLE_BYTES); + uint64_t *p = (uint64_t *)out; + for (int32_t j = 0; j < idx; ++j) { + p[j] = cur = (p[j] ^ cur); + } + out += idx * DOUBLE_BYTES; + } + return (int32_t)(out - output); +} +#endif + +#if __AVX512VL__ // decode two timestamps in one loop. void tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, char *const output, bool bigEndian) { int64_t *ostream = (int64_t *)output; int32_t ipos = 1, opos = 0; -#if __AVX2__ __m128i prevVal = _mm_setzero_si128(); __m128i prevDelta = _mm_setzero_si128(); @@ -464,7 +588,6 @@ void tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, ostream[opos++] = prevVal[1] + prevDeltaX; } } -#endif return; } @@ -473,8 +596,6 @@ void tsDecompressTimestampAvx512(const char *const input, const int32_t nelement int64_t *ostream = (int64_t *)output; int32_t ipos = 1, opos = 0; -#if __AVX512VL__ - __m128i prevVal = _mm_setzero_si128(); __m128i prevDelta = _mm_setzero_si128(); @@ -579,6 +700,6 @@ void tsDecompressTimestampAvx512(const char *const input, const int32_t nelement } } -#endif return; } +#endif diff --git a/source/util/src/terror.c b/source/util/src/terror.c index df104508da..1c94a5f2e4 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -708,7 +708,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TBNAME_ERROR, "Pseudo tag tbname n TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TBNAME_DUPLICATED, "Table name duplicated") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TAG_NAME_DUPLICATED, "Tag name duplicated") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_NOT_ALLOWED_DIFFERENT_BY_ROW_FUNC, "Some functions cannot appear in the select list at the same time") -TAOS_DEFINE_ERROR(TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR, "Syntax error in regular expression") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR, "Syntax error in regular expression") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTERNAL_ERROR, "Parser internal error") //planner diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 6a25c30704..45c8a2f6c2 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -125,6 +125,8 @@ int32_t idxDebugFlag = 131; int32_t sndDebugFlag = 131; int32_t simDebugFlag = 131; +int32_t tqClientDebug = 0; + int64_t dbgEmptyW = 0; int64_t dbgWN = 0; int64_t dbgSmallWN = 0; @@ -149,18 +151,18 @@ static int32_t taosStartLog() { return 0; } -static void getDay(char* buf){ +static void getDay(char* buf, int32_t bufSize){ time_t t = taosTime(NULL); struct tm tmInfo; - if (taosLocalTime(&t, &tmInfo, buf) != NULL) { - TAOS_UNUSED(strftime(buf, LOG_FILE_DAY_LEN, "%Y-%m-%d", &tmInfo)); + if (taosLocalTime(&t, &tmInfo, buf, bufSize) != NULL) { + TAOS_UNUSED(strftime(buf, bufSize, "%Y-%m-%d", &tmInfo)); } } static int64_t getTimestampToday() { time_t t = taosTime(NULL); struct tm tm; - if (taosLocalTime(&t, &tm, NULL) == NULL) { + if (taosLocalTime(&t, &tm, NULL, 0) == NULL) { return 0; } tm.tm_hour = 0; @@ -196,10 +198,10 @@ int32_t taosInitSlowLog() { getFullPathName(tsLogObj.slowLogName, logFileName); - char name[PATH_MAX + LOG_FILE_DAY_LEN] = {0}; - char day[LOG_FILE_DAY_LEN] = {0}; - getDay(day); - (void)snprintf(name, PATH_MAX + LOG_FILE_DAY_LEN, "%s.%s", tsLogObj.slowLogName, day); + char name[PATH_MAX + TD_TIME_STR_LEN] = {0}; + char day[TD_TIME_STR_LEN] = {0}; + getDay(day, sizeof(day)); + (void)snprintf(name, PATH_MAX + TD_TIME_STR_LEN, "%s.%s", tsLogObj.slowLogName, day); tsLogObj.timestampToday = getTimestampToday(); tsLogObj.slowHandle = taosLogBuffNew(LOG_SLOW_BUF_SIZE); @@ -428,11 +430,11 @@ static void taosOpenNewSlowLogFile() { taosWriteLog(tsLogObj.slowHandle); atomic_store_32(&tsLogObj.slowHandle->lock, 0); - char day[LOG_FILE_DAY_LEN] = {0}; - getDay(day); + char day[TD_TIME_STR_LEN] = {0}; + getDay(day, sizeof(day)); TdFilePtr pFile = NULL; - char name[PATH_MAX + LOG_FILE_DAY_LEN] = {0}; - (void)snprintf(name, PATH_MAX + LOG_FILE_DAY_LEN, "%s.%s", tsLogObj.slowLogName, day); + char name[PATH_MAX + TD_TIME_STR_LEN] = {0}; + (void)snprintf(name, PATH_MAX + TD_TIME_STR_LEN, "%s.%s", tsLogObj.slowLogName, day); pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); if (pFile == NULL) { uError("open new log file fail! reason:%s, reuse lastlog", strerror(errno)); @@ -629,7 +631,7 @@ static inline int32_t taosBuildLogHead(char *buffer, const char *flags) { TAOS_UNUSED(taosGetTimeOfDay(&timeSecs)); time_t curTime = timeSecs.tv_sec; - ptm = taosLocalTime(&curTime, &Tm, NULL); + ptm = taosLocalTime(&curTime, &Tm, NULL, 0); return sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " %s %s", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId(), diff --git a/source/util/src/tstrbuild.c b/source/util/src/tstrbuild.c index 54e815ee13..8184f57041 100644 --- a/source/util/src/tstrbuild.c +++ b/source/util/src/tstrbuild.c @@ -71,12 +71,12 @@ void taosStringBuilderAppendNull(SStringBuilder* sb) { taosStringBuilderAppendSt void taosStringBuilderAppendInteger(SStringBuilder* sb, int64_t v) { char buf[64] = {0}; - size_t len = snprintf(buf, sizeof(buf), "%" PRId64, v); + size_t len = tsnprintf(buf, sizeof(buf), "%" PRId64, v); taosStringBuilderAppendStringLen(sb, buf, TMIN(len, sizeof(buf))); } void taosStringBuilderAppendDouble(SStringBuilder* sb, double v) { char buf[512] = {0}; - size_t len = snprintf(buf, sizeof(buf), "%.9lf", v); + size_t len = tsnprintf(buf, sizeof(buf), "%.9lf", v); taosStringBuilderAppendStringLen(sb, buf, TMIN(len, sizeof(buf))); } diff --git a/source/util/src/tuuid.c b/source/util/src/tuuid.c index 4472189d37..4d50d93e74 100644 --- a/source/util/src/tuuid.c +++ b/source/util/src/tuuid.c @@ -15,19 +15,42 @@ #include "tuuid.h" -static int64_t tUUIDHashId = 0; +static uint32_t tUUIDHashId = 0; static int32_t tUUIDSerialNo = 0; +int32_t taosGetSystemUUIDU32(uint32_t *uuid) { + if (uuid == NULL) return TSDB_CODE_APP_ERROR; + char uid[37] = {0}; + int32_t code = taosGetSystemUUIDLimit36(uid, sizeof(uid)); + uid[36] = 0; + + if (code != TSDB_CODE_SUCCESS) { + return code; + } else { + *uuid = MurmurHash3_32(uid, strlen(uid)); + } + return TSDB_CODE_SUCCESS; +} + +int32_t taosGetSystemUUIDU64(uint64_t *uuid) { + if (uuid == NULL) return TSDB_CODE_APP_ERROR; + char uid[37] = {0}; + int32_t code = taosGetSystemUUIDLimit36(uid, sizeof(uid)); + uid[36] = 0; + + if (code != TSDB_CODE_SUCCESS) { + return code; + } else { + *uuid = MurmurHash3_64(uid, strlen(uid)); + } + return TSDB_CODE_SUCCESS; +} + int32_t tGenIdPI32(void) { if (tUUIDHashId == 0) { - char uid[65] = {0}; - int32_t code = taosGetSystemUUID(uid, sizeof(uid)); - uid[64] = 0; - + int32_t code = taosGetSystemUUIDU32(&tUUIDHashId); if (code != TSDB_CODE_SUCCESS) { - terrno = TAOS_SYSTEM_ERROR(errno); - } else { - tUUIDHashId = MurmurHash3_32(uid, strlen(uid)); + terrno = code; } } @@ -41,12 +64,9 @@ int32_t tGenIdPI32(void) { int64_t tGenIdPI64(void) { if (tUUIDHashId == 0) { - char uid[65] = {0}; - int32_t code = taosGetSystemUUID(uid, 64); + int32_t code = taosGetSystemUUIDU32(&tUUIDHashId); if (code != TSDB_CODE_SUCCESS) { - terrno = TAOS_SYSTEM_ERROR(errno); - } else { - tUUIDHashId = MurmurHash3_32(uid, strlen(uid)); + terrno = code; } } @@ -57,7 +77,7 @@ int64_t tGenIdPI64(void) { uint64_t pid = taosGetPId(); int32_t val = atomic_add_fetch_32(&tUUIDSerialNo, 1); - id = ((tUUIDHashId & 0x07FF) << 52) | ((pid & 0x0F) << 48) | ((ts & 0x3FFFFFF) << 20) | (val & 0xFFFFF); + id = (((uint64_t)(tUUIDHashId & 0x07FF)) << 52) | ((pid & 0x0F) << 48) | ((ts & 0x3FFFFFF) << 20) | (val & 0xFFFFF); if (id) { break; } diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index 0d8774ba41..4966a629d8 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -126,12 +126,12 @@ add_test( COMMAND regexTest ) -#add_executable(decompressTest "decompressTest.cpp") -#target_link_libraries(decompressTest os util common gtest_main) -#add_test( -# NAME decompressTest -# COMMAND decompressTest -#) +add_executable(decompressTest "decompressTest.cpp") +target_link_libraries(decompressTest os util common gtest_main) +add_test( + NAME decompressTest + COMMAND decompressTest +) if (${TD_LINUX}) # terrorTest @@ -147,4 +147,4 @@ if (${TD_LINUX}) add_custom_command(TARGET terrorTest POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different ${ERR_TBL_FILE} $ ) -endif () \ No newline at end of file +endif () diff --git a/source/util/test/decompressTest.cpp b/source/util/test/decompressTest.cpp index dfcd682255..e508c489df 100644 --- a/source/util/test/decompressTest.cpp +++ b/source/util/test/decompressTest.cpp @@ -1,14 +1,12 @@ +#define ALLOW_FORBID_FUNC #include #include #include +#include #include #include "ttypes.h" -namespace { - -} // namespace - -TEST(utilTest, decompress_ts_test) { +TEST(utilTest, DISABLED_decompress_ts_test) { { tsSIMDEnable = 1; tsAVX2Supported = 1; @@ -29,6 +27,7 @@ TEST(utilTest, decompress_ts_test) { std::cout << ((int64_t*)decompOutput)[i] << std::endl; } +#ifdef __AVX512VL__ memset(decompOutput, 0, 10 * 8); tsDecompressTimestampAvx512(reinterpret_cast(pOutput), 10, reinterpret_cast(decompOutput), false); @@ -36,13 +35,16 @@ TEST(utilTest, decompress_ts_test) { for (int32_t i = 0; i < 10; ++i) { std::cout << ((int64_t*)decompOutput)[i] << std::endl; } +#endif //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - tsList[0] = 1286; tsList[1] = 1124; tsList[2]=2681; tsList[3] = 2823; + tsList[0] = 1286; + tsList[1] = 1124; + tsList[2] = 2681; + tsList[3] = 2823; -// char* pOutput[4 * sizeof(int64_t)] = {0}; - len = tsCompressTimestamp(tsList, sizeof(tsList), sizeof(tsList) / sizeof(tsList[0]), pOutput, 4, - ONE_STAGE_COMP, NULL, 0); + len = tsCompressTimestamp(tsList, sizeof(tsList), sizeof(tsList) / sizeof(tsList[0]), pOutput, 4, ONE_STAGE_COMP, + NULL, 0); decompOutput[4 * 8] = {0}; tsDecompressTimestamp(pOutput, len, 4, decompOutput, sizeof(int64_t) * 4, ONE_STAGE_COMP, NULL, 0); @@ -56,6 +58,7 @@ TEST(utilTest, decompress_ts_test) { int32_t len1 = tsCompressTimestamp(tsList1, sizeof(tsList1), sizeof(tsList1) / sizeof(tsList1[0]), pOutput, 7, ONE_STAGE_COMP, NULL, 0); +#ifdef __AVX512VL__ memset(decompOutput, 0, 10 * 8); tsDecompressTimestampAvx512(reinterpret_cast(pOutput), 7, reinterpret_cast(decompOutput), false); @@ -63,12 +66,14 @@ TEST(utilTest, decompress_ts_test) { for (int32_t i = 0; i < 7; ++i) { std::cout << ((int64_t*)decompOutput)[i] << std::endl; } +#endif //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// int64_t tsList2[1] = {1700000000}; int32_t len2 = tsCompressTimestamp(tsList2, sizeof(tsList2), sizeof(tsList2) / sizeof(tsList2[0]), pOutput, 1, ONE_STAGE_COMP, NULL, 0); +#ifdef __AVX512VL__ memset(decompOutput, 0, 10 * 8); tsDecompressTimestampAvx512(reinterpret_cast(pOutput), 1, reinterpret_cast(decompOutput), false); @@ -76,52 +81,10 @@ TEST(utilTest, decompress_ts_test) { for (int32_t i = 0; i < 1; ++i) { std::cout << ((int64_t*)decompOutput)[i] << std::endl; } +#endif } -TEST(utilTest, decompress_bigint_avx2_test) { - { - tsSIMDEnable = 1; - tsAVX2Supported = 1; - } - - int64_t tsList[10] = {1700000000, 1700000100, 1700000200, 1700000300, 1700000400, - 1700000500, 1700000600, 1700000700, 1700000800, 1700000900}; - - char* pOutput[10 * sizeof(int64_t)] = {0}; - int32_t len = tsCompressBigint(tsList, sizeof(tsList), sizeof(tsList) / sizeof(tsList[0]), pOutput, 10, - ONE_STAGE_COMP, NULL, 0); - - char* decompOutput[10 * 8] = {0}; - - tsDecompressBigint(pOutput, len, 10, decompOutput, sizeof(int64_t) * 10, ONE_STAGE_COMP, NULL, 0); - - for (int32_t i = 0; i < 10; ++i) { - std::cout << ((int64_t*)decompOutput)[i] << std::endl; - } -} - -TEST(utilTest, decompress_int_avx2_test) { - { - tsSIMDEnable = 1; - tsAVX2Supported = 1; - } - - int32_t tsList[10] = {17000000, 17000001, 17000002, 17000003, 17000004, - 17000005, 17000006, 17000007, 17000008, 17000009}; - - char* pOutput[10 * sizeof(int32_t)] = {0}; - int32_t len = - tsCompressInt(tsList, sizeof(tsList), sizeof(tsList) / sizeof(tsList[0]), pOutput, 10, ONE_STAGE_COMP, NULL, 0); - - char* decompOutput[10 * 8] = {0}; - tsDecompressInt(pOutput, len, 10, decompOutput, sizeof(int32_t) * 10, ONE_STAGE_COMP, NULL, 0); - - for (int32_t i = 0; i < 10; ++i) { - std::cout << ((int32_t*)decompOutput)[i] << std::endl; - } -} - -TEST(utilTest, decompress_perf_test) { +TEST(utilTest, DISABLED_decompress_perf_test) { int32_t num = 10000; int64_t* pList = static_cast(taosMemoryCalloc(num, sizeof(int64_t))); @@ -149,9 +112,11 @@ TEST(utilTest, decompress_perf_test) { memset(pOutput, 0, num * sizeof(int64_t)); st = taosGetTimestampUs(); +#ifdef __AVX512VL__ for (int32_t k = 0; k < 10000; ++k) { tsDecompressTimestampAvx512(px, num, pOutput, false); } +#endif int64_t el2 = taosGetTimestampUs() - st; std::cout << "SIMD decompress elapsed time:" << el2 << " us" << std::endl; @@ -303,7 +268,7 @@ void* genCompressData_float(int32_t type, int32_t num, bool order) { } return pBuf; } -TEST(utilTest, compressAlg) { +TEST(utilTest, DISABLED_compressAlg) { int32_t num = 4096; int64_t* pList = static_cast(taosMemoryCalloc(num, sizeof(int64_t))); int64_t iniVal = 17000; @@ -479,4 +444,214 @@ TEST(utilTest, compressAlg) { } taosMemoryFree(p); } -} \ No newline at end of file +} + +static uint32_t decompressRandomSeed; + +static void refreshSeed() { + decompressRandomSeed = std::random_device()(); + std::cout << "Refresh random seed to " << decompressRandomSeed << "\n"; +} + +#ifdef __GNUC__ +template +static std::vector::value, T>::type> utilTestRandomData(int32_t n, T min, + T max) { + std::mt19937 gen(decompressRandomSeed); + std::vector data(n); + + std::uniform_int_distribution dist(min, max); + for (auto& v : data) v = dist(gen); + return data; +} + +template +static std::vector::value, T>::type> utilTestRandomData(int32_t n, + T min, + T max) { + std::mt19937 gen(decompressRandomSeed); + std::vector data(n); + + std::uniform_real_distribution dist(min, max); + for (auto& v : data) v = dist(gen); + return data; +} +#else +template +static std::vector utilTestRandomData(int32_t n, T min, T max) { + std::vector data(n); + + for (int32_t i = 0; i < n; ++i) { + data[i] = (i & 0x1) ? min : max; + } + return data; +} +#endif + +template +static double measureRunTime(const F& func, int32_t nround = 1) { + auto start = std::chrono::high_resolution_clock::now(); + for (int32_t i = 0; i < nround; ++i) { + func(); + } + auto end = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(end - start).count(); + return duration / 1000.0; +} + +template +struct DataTypeSupportAvx { + static const bool value = false; +}; + +template <> +struct DataTypeSupportAvx { + static const bool value = true; +}; + +template <> +struct DataTypeSupportAvx { + static const bool value = true; +}; + +template +static void decompressBasicTest(size_t dataSize, const CompF& compress, const DecompF& decompress, T min, T max) { + auto origData = utilTestRandomData(dataSize, min, max); + std::vector compData(origData.size() * sizeof(origData[0]) + 1); + int32_t cnt = compress(origData.data(), origData.size(), origData.size(), compData.data(), compData.size(), + ONE_STAGE_COMP, nullptr, 0); + ASSERT_LE(cnt, compData.size()); + decltype(origData) decompData(origData.size()); + + // test simple implementation without SIMD instructions + tsSIMDEnable = 0; + cnt = decompress(compData.data(), compData.size(), decompData.size(), decompData.data(), decompData.size(), + ONE_STAGE_COMP, nullptr, 0); + ASSERT_EQ(cnt, compData.size() - 1); + EXPECT_EQ(origData, decompData); + +#ifdef __AVX2__ + if (DataTypeSupportAvx::value) { + // test AVX2 implementation + tsSIMDEnable = 1; + tsAVX2Supported = 1; + cnt = decompress(compData.data(), compData.size(), decompData.size(), decompData.data(), decompData.size(), + ONE_STAGE_COMP, nullptr, 0); + ASSERT_EQ(cnt, compData.size() - 1); + EXPECT_EQ(origData, decompData); + } +#endif +} + +template +static void decompressPerfTest(const char* typname, const CompF& compress, const DecompF& decompress, T min, T max) { + constexpr size_t DATA_SIZE = 1 * 1024 * 1024; + constexpr int32_t NROUND = 1000; + auto origData = utilTestRandomData(DATA_SIZE, min, max); + std::vector compData(origData.size() * sizeof(origData[0]) + 1); + int32_t cnt = compress(origData.data(), origData.size(), origData.size(), compData.data(), compData.size(), + ONE_STAGE_COMP, nullptr, 0); + ASSERT_LE(cnt, compData.size()); + if (compData[0] == 1) std::cout << "NOT COMPRESSED!\n"; + std::cout << "Original size: " << compData.size() - 1 << "; Compressed size: " << cnt + << "; Compression ratio: " << 1.0 * (compData.size() - 1) / cnt << "\n"; + decltype(origData) decompData(origData.size()); + + tsSIMDEnable = 0; + auto ms = measureRunTime( + [&]() { + decompress(compData.data(), compData.size(), decompData.size(), decompData.data(), decompData.size(), + ONE_STAGE_COMP, nullptr, 0); + }, + NROUND); + std::cout << "Decompression of " << NROUND * DATA_SIZE << " " << typname << " without SIMD costs " << ms + << " ms, avg speed: " << NROUND * DATA_SIZE * 1000 / ms << " tuples/s\n"; + +#ifdef __AVX2__ + if (DataTypeSupportAvx::value) { + tsSIMDEnable = 1; + tsAVX2Supported = 1; + ms = measureRunTime( + [&]() { + decompress(compData.data(), compData.size(), decompData.size(), decompData.data(), decompData.size(), + ONE_STAGE_COMP, nullptr, 0); + }, + NROUND); + std::cout << "Decompression of " << NROUND * DATA_SIZE << " " << typname << " using AVX2 costs " << ms + << " ms, avg speed: " << NROUND * DATA_SIZE * 1000 / ms << " tuples/s\n"; + } +#endif +} + +#define RUN_PERF_TEST(typname, comp, decomp, min, max) \ + do { \ + refreshSeed(); \ + decompressPerfTest(#typname, comp, decomp, min, max); \ + } while (0) + +TEST(utilTest, decompressTinyintBasic) { + refreshSeed(); + for (int32_t r = 1; r <= 4096; ++r) { + decompressBasicTest(r, tsCompressTinyint, tsDecompressTinyint, 0, 100); + } +} + +TEST(utilTest, decompressTinyintPerf) { RUN_PERF_TEST(int8_t, tsCompressTinyint, tsDecompressTinyint, 0, 100); } + +TEST(utilTest, decompressSmallintBasic) { + refreshSeed(); + for (int32_t r = 1; r <= 4096; ++r) { + decompressBasicTest(r, tsCompressSmallint, tsDecompressSmallint, 0, 10000); + } +} + +TEST(utilTest, decompressSmallintPerf) { RUN_PERF_TEST(int16_t, tsCompressSmallint, tsDecompressSmallint, 0, 10000); } + +TEST(utilTest, decompressIntBasic) { + refreshSeed(); + for (int32_t r = 1; r <= 4096; ++r) { + decompressBasicTest(r, tsCompressInt, tsDecompressInt, 0, 1000000); + } +} + +TEST(utilTest, decompressIntPerf) { RUN_PERF_TEST(int32_t, tsCompressInt, tsDecompressInt, 0, 1000000); } + +TEST(utilTest, decompressBigintBasic) { + refreshSeed(); + for (int32_t r = 1; r <= 4096; ++r) { + decompressBasicTest(r, tsCompressBigint, tsDecompressBigint, 0, 1000000000L); + } +} + +TEST(utilTest, decompressBigintPerf) { RUN_PERF_TEST(int64_t, tsCompressBigint, tsDecompressBigint, 0, 1000000000L); } + +TEST(utilTest, decompressFloatBasic) { + refreshSeed(); + for (int32_t r = 1; r <= 4096; ++r) { + decompressBasicTest(r, tsCompressFloat, tsDecompressFloat, 0, 99999); + } +} + +TEST(utilTest, decompressFloatPerf) { RUN_PERF_TEST(float, tsCompressFloat, tsDecompressFloat, 0, 99999); } + +TEST(utilTest, decompressDoubleBasic) { + refreshSeed(); + for (int32_t r = 1; r <= 4096; ++r) { + decompressBasicTest(r, tsCompressDouble, tsDecompressDouble, 0, 9999999999); + } +} + +TEST(utilTest, decompressDoublePerf) { RUN_PERF_TEST(double, tsCompressDouble, tsDecompressDouble, 0, 9999999999); } + + +TEST(utilTest, decompressTimestampBasic) { + refreshSeed(); + for (int32_t r = 1; r <= 4096; ++r) { + decompressBasicTest(r, tsCompressTimestamp, tsDecompressTimestamp, 0, 1000000000L); + } +} + +TEST(utilTest, decompressTimestampPerf) { + refreshSeed(); + decompressPerfTest("timestamp", tsCompressTimestamp, tsDecompressTimestamp, 0, 1000000000L); +} diff --git a/tests/army/query/function/test_resinfo.py b/tests/army/query/function/test_resinfo.py new file mode 100644 index 0000000000..51d51f3ce1 --- /dev/null +++ b/tests/army/query/function/test_resinfo.py @@ -0,0 +1,69 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import random +import hashlib + +import taos +import frame +import frame.etool + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * + +initial_hash_resinfoInt = "e739cde34b98f13dd9ad696d18f060cc" +initial_hash_resinfo = "172d04aa7af0d8cd2e4d9df284079958" + +class TDTestCase(TBase): + def get_file_hash(self, file_path): + hasher = hashlib.md5() + with open(file_path, 'rb') as f: + buf = f.read() + hasher.update(buf) + return hasher.hexdigest() + + def testFileChanged(self): + tdLog.info(f"insert data.") + # taosBenchmark run + resinfoIntFile = etool.curFile(__file__, "../../../../source/libs/function/inc/functionResInfoInt.h") + resinfoFile = etool.curFile(__file__, "../../../../include/libs/function/functionResInfo.h") + current_hash = self.get_file_hash(resinfoIntFile) + if current_hash != initial_hash_resinfoInt: + tdLog.exit(f"{resinfoIntFile} has been modified.") + else: + tdLog.success(f"{resinfoIntFile} is not modified.") + current_hash = self.get_file_hash(resinfoFile) + if current_hash != initial_hash_resinfo: + tdLog.exit(f"{resinfoFile} has been modified.") + else: + tdLog.success(f"{resinfoFile} is not modified.") + + + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # insert data + self.testFileChanged() + + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/query/test_having.py b/tests/army/query/test_having.py new file mode 100644 index 0000000000..ff8f6a1c1d --- /dev/null +++ b/tests/army/query/test_having.py @@ -0,0 +1,378 @@ +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.eos import * +import random +import string + +""" + TD-32198: https://jira.taosdata.com:18080/browse/TD-32198 + Having关键字的专项测试,主要覆盖以下 4 种场景： + 1、普通聚合查询 + 2、关联查询 + 3、窗口切分查询 + 4、流计算中的窗口切分查询 +""" + + +class TDTestCase(TBase): + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def prepare_global_data(self): + tdSql.execute("DROP DATABASE IF EXISTS db_td32198;") + tdSql.execute("create database db_td32198;") + tdSql.execute("use db_td32198;") + + def prepare_agg_data(self): + # database for case TD-32198 + # super table + tdSql.execute("DROP STABLE IF EXISTS meters") + tdSql.execute("CREATE STABLE `meters` (`ts` TIMESTAMP , `current` FLOAT , `voltage` INT , `phase` FLOAT ) \ + TAGS (`groupid` TINYINT, `location` VARCHAR(16));") + + # child table + tdSql.execute("CREATE TABLE `ct_1` USING `meters` (`groupid`, `location`) TAGS (1, 'beijing');") + # tdSql.execute("CREATE TABLE `ct_2` USING `meters` (`groupid`, `location`) TAGS (2, 'shanghai');") + + data = [ + ('2020-06-01 00:00:00.000', 0.1000000, 12, 0.0200000), + ('2020-06-01 00:15:00.000', 0.3614670, 18, 0.1071560), + ('2020-06-01 00:30:00.000', 0.5209450, 18, 0.1736480), + ('2020-06-01 00:45:00.000', 0.8764570, 18, 0.2588190), + ('2020-06-01 01:00:00.000', 1.0260600, 14, 0.3620200), + ('2020-06-01 01:15:00.000', 1.3678550, 0, 0.4226180), + ('2020-06-01 01:30:00.000', 1.6000000, 12, 0.5200000), + ('2020-06-01 01:45:00.000', 1.8207290, 2, 0.5835760), + ('2020-06-01 02:00:00.000', 1.9283630, 18, 0.6527880), + ('2020-06-01 02:15:00.000', 2.1213200, 18, 0.7271070), + ('2020-06-01 02:30:00.000', 2.3981330, 12, 0.7760440), + ('2020-06-01 02:45:00.000', 2.4574561, 14, 0.8291520), + ('2020-06-01 03:00:00.000', 2.6980760, 14, 0.8760250), + ('2020-06-01 03:15:00.000', 2.8189230, 10, 0.9063080), + ('2020-06-01 03:30:00.000', 2.8190780, 6, 0.9396930), + ('2020-06-01 03:45:00.000', 2.8977780, 10, 0.9859260), + ('2020-06-01 04:00:00.000', 2.9544230, 4, 1.0048079), + ('2020-06-01 04:15:00.000', 2.9885840, 14, 1.0061949), + ('2020-06-01 04:30:00.000', 3.0999999, 6, 1.0200000), + ('2020-06-01 04:45:00.000', 3.0885839, 10, 1.0161951), + ('2020-06-01 05:00:00.000', 2.9544230, 18, 0.9848080), + ('2020-06-01 05:15:00.000', 2.9977770, 2, 0.9859260), + ('2020-06-01 05:30:00.000', 2.8190780, 0, 0.9496930), + ('2020-06-01 05:45:00.000', 2.7189231, 18, 0.9163080), + ('2020-06-01 06:00:00.000', 2.5980761, 10, 0.8860250) + ] + + sql = "insert into ct_1 values"; + for t in data: + sql += "('{}', {}, {}, {}),".format(t[0], t[1], t[2], t[3]) + sql += ";" + tdSql.execute(sql) + tdLog.debug("sql: %s" % sql) + + def test_agg_having(self): + tdSql.query("select voltage, sum(voltage), count(*) from ct_1 group by voltage;") + tdSql.checkRows(8) + tdSql.checkData(7, 2, 7) + tdSql.checkData(7, 1, 126) + + tdSql.query("select voltage, sum(voltage), count(*) from ct_1 group by voltage having count(voltage)>=4;"); + tdSql.checkRows(3) + tdSql.checkData(2, 2, 7) + tdSql.checkData(2, 1, 126) + + tdSql.query("select voltage, sum(voltage), count(*) from ct_1 group by voltage having count(current)>=4;"); + tdSql.checkRows(3) + tdSql.checkData(2, 2, 7) + tdSql.checkData(2, 1, 126) + + tdSql.query("select voltage, sum(voltage), count(*) from ct_1 group by voltage having voltage >=14;"); + tdSql.checkRows(2) + tdSql.checkData(0, 2, 4) + tdSql.checkData(1, 1, 126) + + tdSql.error("select voltage, count(*) from ct_1 group by voltage having current >1.0260600;"); + + def prepare_join_data(self): + # super table + tdSql.execute("DROP STABLE IF EXISTS meters") + tdSql.execute("CREATE STABLE `meters` (`ts` TIMESTAMP , `current` FLOAT , `voltage` INT , `phase` FLOAT ) \ + TAGS (`groupid` TINYINT, `location` VARCHAR(16));") + + # child table + tdSql.execute("CREATE TABLE `ct_join_1` USING `meters` (`groupid`, `location`) TAGS (1, 'beijing');") + tdSql.execute("CREATE TABLE `ct_join_2` USING `meters` (`groupid`, `location`) TAGS (2, 'shanghai');") + + # insert data for ts4806 + data_join_1 = [ + ('2020-06-01 00:00:00.000', 0.1000000, 12, 0.0200000), + ('2020-06-01 00:10:00.000', 1.2278550, 9, 0.4226180), + ('2020-06-01 00:15:00.000', 0.3614670, 18, 0.1071560), + ('2020-06-01 00:30:00.000', 0.5209450, 18, 0.1736480), + ('2020-06-01 00:40:00.000', 1.5230000, 10, 0.5200000), + ('2020-06-01 00:45:00.000', 0.8764570, 18, 0.2588190), + ('2020-06-01 00:50:00.000', 1.6507290, 11, 0.5835760), + ('2020-06-01 01:00:00.000', 1.0260600, 14, 0.3620200), + ('2020-06-01 01:15:00.000', 1.3678550, 0, 0.4226180), + ('2020-06-01 01:20:00.000', 1.1213200, 13, 0.7271070), + ('2020-06-01 01:30:00.000', 1.6000000, 12, 0.5200000), + ('2020-06-01 01:45:00.000', 1.8207290, 2, 0.5835760), + ('2020-06-01 02:00:00.000', 1.9283630, 18, 0.6527880), + ('2020-06-01 02:05:00.000', 0.9283630, 6, 0.6527880), + ('2020-06-01 02:15:00.000', 2.1213200, 18, 0.7271070) + ] + + data_join_2 = [ + ('2020-06-01 00:00:00.000', 0.3614670, 9, 0.0200000), + ('2020-06-01 00:15:00.000', 0.1000000, 12, 0.1071560), + ('2020-06-01 00:30:00.000', 0.5209450, 15, 0.1736480), + ('2020-06-01 00:45:00.000', 0.8764570, 18, 0.2588190), + ('2020-06-01 01:00:00.000', 1.0260600, 15, 0.3620200), + ('2020-06-01 01:15:00.000', 1.3678550, 7, 0.4226180), + ('2020-06-01 01:30:00.000', 1.6000000, 12, 0.5200000), + ('2020-06-01 01:45:00.000', 1.8207290, 7, 0.5835760), + ('2020-06-01 02:00:00.000', 1.0260600, 13, 0.6527880), + ('2020-06-01 02:15:00.000', 0.5209450, 18, 0.7271070) + ] + + sql = "insert into ct_join_1 values"; + for t in data_join_1: + sql += "('{}', {}, {}, {}),".format(t[0], t[1], t[2], t[3]) + sql += ";" + tdSql.execute(sql) + tdLog.debug("ct_join_1 sql: %s" % sql) + + sql = "insert into ct_join_2 values"; + for t in data_join_2: + sql += "('{}', {}, {}, {}),".format(t[0], t[1], t[2], t[3]) + sql += ";" + tdSql.execute(sql) + tdLog.debug("ct_join_2 sql: %s" % sql) + + def test_join_having(self): + tdSql.query("SELECT a.voltage, count(*) FROM ct_join_1 a JOIN ct_join_2 b ON a.ts = b.ts \ + group by a.voltage having count(*) > 4;") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 5) + tdSql.checkData(0, 0, 18) + + tdSql.error("SELECT a.voltage, count(*) FROM ct_join_1 a JOIN ct_join_2 b ON a.ts = b.ts \ + group by a.voltage having b.voltage > 14;") + + tdSql.query("SELECT a.voltage, count(*) FROM ct_join_1 a left JOIN ct_join_2 b ON a.ts = b.ts \ + group by a.voltage having count(*) > 4;"); + tdSql.checkRows(1) + tdSql.checkData(0, 1, 5) + tdSql.checkData(0, 0, 18) + + tdSql.error("SELECT a.voltage, count(*) FROM ct_join_1 a left JOIN ct_join_2 b ON a.ts = b.ts \ + group by a.voltage having b.voltage > 14;"); + + tdSql.query("SELECT a.ts, a.voltage, avg(b.voltage) FROM ct_join_2 a LEFT WINDOW JOIN ct_join_1 b \ + WINDOW_OFFSET(-15m, 15m) where a.voltage >=18 and b.voltage > 11 having avg(b.voltage) > 17;"); + tdSql.checkRows(1) + + tdSql.error("SELECT a.ts, a.voltage, avg(b.voltage) FROM ct_join_2 a LEFT WINDOW JOIN ct_join_1 b \ + WINDOW_OFFSET(-15m, 15m) where a.voltage >=18 and b.voltage > 11 having b.voltage > 17;"); + + def prepare_window_data(self): + # super table + tdSql.execute("DROP STABLE IF EXISTS meters") + tdSql.execute("CREATE STABLE `meters` (`ts` TIMESTAMP , `current` FLOAT , `voltage` INT , `phase` FLOAT ) \ + TAGS (`groupid` TINYINT, `location` VARCHAR(16));") + + # child table + tdSql.execute("CREATE TABLE `ct_win` USING `meters` (`groupid`, `location`) TAGS (1, 'beijing');") + + # insert data for ts4806 + data_win = [ + ('2020-06-01 00:00:00.000', 0.1000000, 12, 0.0200000), + ('2020-06-01 00:10:00.000', 1.2278550, 9, 0.4226180), + ('2020-06-01 00:15:00.000', 0.3614670, 18, 0.1071560), + ('2020-06-01 00:30:00.000', 0.5209450, 18, 0.1736480), + ('2020-06-01 00:40:00.000', 1.5230000, 18, 0.5200000), + ('2020-06-01 00:45:00.000', 0.8764570, 18, 0.2588190), + ('2020-06-01 00:50:00.000', 1.6507290, 11, 0.5835760), + ('2020-06-01 01:00:00.000', 1.0260600, 14, 0.3620200), + ('2020-06-01 01:15:00.000', 1.3678550, 14, 0.4226180), + ('2020-06-01 01:20:00.000', 1.1213200, 13, 0.7271070), + ('2020-06-01 01:30:00.000', 1.6000000, 12, 0.5200000), + ('2020-06-01 01:45:00.000', 1.8207290, 12, 0.5835760), + ('2020-06-01 02:00:00.000', 1.9283630, 18, 0.6527880), + ('2020-06-01 02:05:00.000', 0.9283630, 18, 0.6527880), + ('2020-06-01 02:15:00.000', 2.1213200, 18, 0.7271070) + ] + + sql = "insert into ct_win values"; + for t in data_win: + sql += "('{}', {}, {}, {}),".format(t[0], t[1], t[2], t[3]) + sql += ";" + tdSql.execute(sql) + tdLog.debug("data_win sql: %s" % sql) + + def test_window_having(self): + tdSql.query("SELECT _WSTART, _WEND, COUNT(*) FROM ct_win INTERVAL(15m) having count(*) > 1;") + tdSql.checkRows(5) + tdSql.checkData(0, 2, 2) + + tdSql.error("SELECT _WSTART, _WEND, COUNT(*) FROM ct_win INTERVAL(15m) having voltage > 12;"); + + tdSql.query("SELECT _wstart, _wend, COUNT(*) AS cnt, FIRST(ts) AS fst, voltage FROM ct_win \ + STATE_WINDOW(voltage) having count(*) > 3;"); + tdSql.checkRows(1) + tdSql.checkData(0, 2, 4) + + tdSql.error("SELECT _wstart, _wend, COUNT(*) AS cnt, FIRST(ts) AS fst, voltage FROM ct_win \ + STATE_WINDOW(voltage) having phase > 0.26;"); + + tdSql.query("SELECT _wstart, _wend, COUNT(*), FIRST(ts) FROM ct_win SESSION(ts, 10m) having count(*) > 3;"); + tdSql.checkRows(1) + tdSql.checkData(0, 2, 5) + + tdSql.error("SELECT _wstart, _wend, COUNT(*), FIRST(ts) FROM ct_win SESSION(ts, 10m) having voltage > 12;"); + + tdSql.query("select _wstart, _wend, count(*), first(voltage), last(voltage) from ct_win \ + event_window start with voltage <= 12 end with voltage >= 17 having count(*) > 3;"); + tdSql.checkRows(1) + tdSql.checkData(0, 2, 7) + tdSql.checkData(0, 3, 11) + tdSql.checkData(0, 4, 18) + + tdSql.error("select _wstart, _wend, count(*) from ct_win \ + event_window start with voltage <=12 end with voltage >= 17 having phase > 0.2;"); + + tdSql.query( + "select _wstart, _wend, count(*), sum(voltage) from ct_win count_window(4) having sum(voltage) > 57;"); + tdSql.checkRows(1) + tdSql.checkData(0, 2, 4) + tdSql.checkData(0, 3, 61) + + tdSql.error("select _wstart, _wend, count(*), sum(voltage) from ct_win count_window(4) having voltage > 12;"); + + + def prepare_stream_window_data(self): + # super table + tdSql.execute("DROP STABLE IF EXISTS meters") + tdSql.execute("CREATE STABLE `meters` (`ts` TIMESTAMP , `current` FLOAT , `voltage` INT , `phase` FLOAT ) \ + TAGS (`groupid` TINYINT, `location` VARCHAR(16));") + + # child table + tdSql.execute("CREATE TABLE `ct_steam_win` USING `meters` (`groupid`, `location`) TAGS (1, 'beijing');") + + # insert data for ts4806 + data_win = [ + ('2020-06-01 00:00:00.000', 0.1000000, 12, 0.0200000), + ('2020-06-01 00:10:00.000', 1.2278550, 9, 0.4226180), + ('2020-06-01 00:15:00.000', 0.3614670, 18, 0.1071560), + ('2020-06-01 00:30:00.000', 0.5209450, 18, 0.1736480), + ('2020-06-01 00:40:00.000', 1.5230000, 18, 0.5200000), + ('2020-06-01 00:45:00.000', 0.8764570, 18, 0.2588190), + ('2020-06-01 00:50:00.000', 1.6507290, 11, 0.5835760), + ('2020-06-01 01:00:00.000', 1.0260600, 14, 0.3620200), + ('2020-06-01 01:15:00.000', 1.3678550, 14, 0.4226180), + ('2020-06-01 01:20:00.000', 1.1213200, 13, 0.7271070), + ('2020-06-01 01:30:00.000', 1.6000000, 12, 0.5200000), + ('2020-06-01 01:45:00.000', 1.8207290, 12, 0.5835760), + ('2020-06-01 02:00:00.000', 1.9283630, 18, 0.6527880), + ('2020-06-01 02:05:00.000', 0.9283630, 18, 0.6527880), + ('2020-06-01 02:15:00.000', 2.1213200, 18, 0.7271070) + ] + + sql = "insert into ct_win values"; + for t in data_win: + sql += "('{}', {}, {}, {}),".format(t[0], t[1], t[2], t[3]) + sql += ";" + tdSql.execute(sql) + tdLog.debug("data_win sql: %s" % sql) + + # 支持会话窗口、状态窗口、滑动窗口、事件窗口和计数窗口， + # 其中，状态窗口、事件窗口 和 计数窗口 搭配超级表时必须与 partition by tbname 一起使用 + def test_stream_window_having(self): + tdSql.execute("CREATE STREAM streams0 fill_history 1 INTO streamt0 AS \ + SELECT _WSTART, _WEND, COUNT(*) FROM meters PARTITION BY tbname INTERVAL(15m) having count(*) > 1;") + tdSql.query("select * from streamt0;"); + tdSql.checkRows(5) + tdSql.checkData(0, 2, 2) + + tdSql.error("CREATE STREAM streams10 fill_history 1 INTO streamt10 AS SELECT _WSTART, _WEND, COUNT(*) \ + FROM meters PARTITION BY tbname INTERVAL(15m) having voltage > 12;"); + + + tdSql.execute("CREATE STREAM streams1 fill_history 1 INTO streamt1 AS \ + SELECT _wstart, _wend, COUNT(*) AS cnt, FIRST(ts) AS fst, voltage FROM meters PARTITION BY tbname \ + STATE_WINDOW(voltage) having count(*) > 3;"); + tdSql.query("select * from streamt1;"); + tdSql.checkRows(1) + tdSql.checkData(0, 2, 4) + + tdSql.error("CREATE STREAM streams11 fill_history 1 INTO streamt11 AS \ + SELECT _wstart, _wend, COUNT(*) AS cnt, FIRST(ts) AS fst, voltage FROM meters PARTITION BY tbname \ + STATE_WINDOW(voltage) having phase > 0.26;"); + + + tdSql.execute("CREATE STREAM streams2 fill_history 1 INTO streamt2 AS \ + SELECT _wstart, _wend, COUNT(*), FIRST(ts) FROM meters SESSION(ts, 10m) having count(*) > 3;"); + tdSql.query("select * from streamt2;"); + tdSql.checkRows(1) + tdSql.checkData(0, 2, 5) + + tdSql.error("CREATE STREAM streams12 fill_history 1 INTO streamt12 AS \ + SELECT _wstart, _wend, COUNT(*), FIRST(ts) FROM meters SESSION(ts, 10m) having voltage > 12;"); + + tdSql.execute("CREATE STREAM streams3 fill_history 1 INTO streamt3 AS \ + select _wstart, _wend, count(*), first(voltage), last(voltage) from meters PARTITION BY tbname \ + event_window start with voltage <= 12 end with voltage >= 17 having count(*) > 3;"); + tdSql.query("select * from streamt3;"); + tdSql.checkRows(1) + tdSql.checkData(0, 2, 7) + tdSql.checkData(0, 3, 11) + tdSql.checkData(0, 4, 18) + + tdSql.error("CREATE STREAM streams13 fill_history 1 INTO streamt13 AS \ + select _wstart, _wend, count(*), first(voltage), last(voltage) from meters PARTITION BY tbname \ + event_window start with voltage <= 12 end with voltage >= 17 having phase > 0.2;"); + + tdSql.execute("CREATE STREAM streams4 fill_history 1 INTO streamt4 AS \ + select _wstart, _wend, count(*), sum(voltage) from meters PARTITION BY tbname \ + count_window(4) having sum(voltage) > 57;"); + tdSql.query("select * from streamt4;"); + tdSql.checkRows(1) + tdSql.checkData(0, 2, 4) + tdSql.checkData(0, 3, 61) + + tdSql.error("CREATE STREAM streams14 fill_history 1 INTO streamt14 AS \ + select _wstart, _wend, count(*), sum(voltage) from meters PARTITION BY tbname \ + count_window(4) having voltage > 12;"); + + + + def run(self): + self.prepare_global_data() + + self.prepare_agg_data() + self.test_agg_having() + + self.prepare_join_data() + self.test_join_having() + + self.prepare_window_data() + self.test_window_having() + + ''' + self.prepare_stream_window_data() + self.test_stream_window_having() + ''' + + def stop(self): + tdSql.execute("drop database db_td32198;") + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json index 0054d985ee..5ce0d4d87d 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json @@ -63,7 +63,7 @@ "childtable_offset": 0, "insert_rows": 20, "insert_interval": 0, - "interlace_rows": 5, + "interlace_rows": 0, "disorder_ratio": 0, "disorder_range": 1000, "timestamp_step": 1, diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json index bed3598bfe..1d0ec1765d 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json @@ -64,7 +64,7 @@ "childtable_offset": 0, "insert_rows": 20, "insert_interval": 0, - "interlace_rows": 5, + "interlace_rows": 0, "disorder_ratio": 0, "disorder_range": 1000, "timestamp_step": 1, diff --git a/tests/docs-examples-test/c.sh b/tests/docs-examples-test/c.sh new file mode 100644 index 0000000000..54e334b22e --- /dev/null +++ b/tests/docs-examples-test/c.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +pgrep taosd || taosd >> /dev/null 2>&1 & +pgrep taosadapter || taosadapter >> /dev/null 2>&1 & + +GREEN='\033[0;32m' +RED='\033[0;31m' +NC='\033[0m' + +TEST_PATH="../../docs/examples/c" +echo "setting TEST_PATH: $TEST_PATH" + +cd "${TEST_PATH}" || { echo -e "${RED}Failed to change directory to ${TEST_PATH}${NC}"; exit 1; } + +LOG_FILE="docs-c-test-out.log" + +> $LOG_FILE + +make > "$LOG_FILE" 2>&1 + +if [ $? -eq 0 ]; then + echo -e "${GREEN}Make completed successfully.${NC}" +else + echo -e "${RED}Make failed. Check log file: $LOG_FILE${NC}" + cat "$LOG_FILE" + exit 1 +fi + + +declare -a TEST_EXES=( + "connect_example" + "create_db_demo" + "insert_data_demo" + "query_data_demo" + "with_reqid_demo" + "stmt_insert_demo" + "tmq_demo" + "sml_insert_demo" +) + +declare -a NEED_CLEAN=( + "true" + "false" + "false" + "false" + "false" + "false" + "false" + "true" +) + +totalCases=0 +totalFailed=0 +totalSuccess=0 + +for i in "${!TEST_EXES[@]}"; do + TEST_EXE="${TEST_EXES[$i]}" + NEED_CLEAN_FLAG="${NEED_CLEAN[$i]}" + + if [ "$NEED_CLEAN_FLAG" = "true" ]; then + echo "Cleaning database before executing $TEST_EXE..." + taos -s "drop database if exists power" >> $LOG_FILE 2>&1 + fi + + echo "Executing $TEST_EXE..." + ./$TEST_EXE >> $LOG_FILE 2>&1 + RESULT=$? + + if [ "$RESULT" -eq 0 ]; then + totalSuccess=$((totalSuccess + 1)) + echo "[$GREEN OK $NC] $TEST_EXE executed successfully." + else + totalFailed=$((totalFailed + 1)) + echo "[$RED FAILED $NC] $TEST_EXE exited with code $RESULT." + fi + + totalCases=$((totalCases + 1)) +done + +tail -n 40 $LOG_FILE + +echo -e "\nTotal number of cases executed: $totalCases" +if [ "$totalSuccess" -gt "0" ]; then + echo -e "\n${GREEN} ### Total $totalSuccess C case(s) succeed! ### ${NC}" +fi + +if [ "$totalFailed" -ne "0" ]; then + echo -e "\n${RED} ### Total $totalFailed C case(s) failed! ### ${NC}" + exit 1 +fi + +echo "All tests completed." \ No newline at end of file diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index c70989b5a8..81ffaf0d4e 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -6,7 +6,6 @@ ,,n,unit-test,bash test.sh - # # army-test # @@ -16,6 +15,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f cluster/snapshot.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f query/function/test_func_elapsed.py ,,y,army,./pytest.sh python3 ./test.py -f query/function/test_function.py +,,y,army,./pytest.sh python3 ./test.py -f query/function/test_resinfo.py ,,y,army,./pytest.sh python3 ./test.py -f query/function/concat.py ,,y,army,./pytest.sh python3 ./test.py -f query/function/cast.py ,,y,army,./pytest.sh python3 ./test.py -f query/test_join.py @@ -46,6 +46,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f query/last/test_last.py ,,y,army,./pytest.sh python3 ./test.py -f query/window/base.py ,,y,army,./pytest.sh python3 ./test.py -f query/sys/tb_perf_queries_exist_test.py -N 3 +,,y,army,./pytest.sh python3 ./test.py -f query/test_having.py # # system test @@ -161,7 +162,6 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxTopic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py -R -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqClientConsLog.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxGroupIds.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumeDiscontinuousData.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqOffset.py @@ -234,6 +234,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/basic5.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/ts-4674.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-30270.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb2.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb3.py @@ -283,7 +284,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts5466.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts-5473.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-32187.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts4563.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_td32526.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_replay.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSeekAndCommit.py ,,n,system-test,python3 ./test.py -f 7-tmq/tmq_offset.py @@ -1555,6 +1558,7 @@ ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R #docs-examples test +,,n,docs-examples-test,bash c.sh ,,n,docs-examples-test,bash python.sh ,,n,docs-examples-test,bash node.sh ,,n,docs-examples-test,bash csharp.sh diff --git a/tests/system-test/7-tmq/subscribeStb2.py b/tests/system-test/7-tmq/subscribeStb2.py index cdbc41a593..02d1630be7 100644 --- a/tests/system-test/7-tmq/subscribeStb2.py +++ b/tests/system-test/7-tmq/subscribeStb2.py @@ -266,7 +266,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt: + if totalConsumeRows < expectrowcnt: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) tdLog.exit("tmq consume rows error!") @@ -287,7 +287,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt*2: + if totalConsumeRows < expectrowcnt*2: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt*2)) tdLog.exit("tmq consume rows error!") diff --git a/tests/system-test/7-tmq/subscribeStb3.py b/tests/system-test/7-tmq/subscribeStb3.py index ed44ab1fb1..5df58d8122 100644 --- a/tests/system-test/7-tmq/subscribeStb3.py +++ b/tests/system-test/7-tmq/subscribeStb3.py @@ -20,12 +20,9 @@ class actionType(Enum): class TDTestCase: hostname = socket.gethostname() - #rpcDebugFlagVal = '143' - #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} - #clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal - #updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} - #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal - #print ("===================: ", updatecfgDict) + clientCfgDict = {'debugFlag': 135} + updatecfgDict = {'debugFlag': 135, 'asynclog': 0} + updatecfgDict["clientCfg"] = clientCfgDict def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) diff --git a/tests/system-test/7-tmq/td-30270.py b/tests/system-test/7-tmq/td-30270.py new file mode 100644 index 0000000000..73c7a88f65 --- /dev/null +++ b/tests/system-test/7-tmq/td-30270.py @@ -0,0 +1,80 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from taos.tmq import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + clientCfgDict = {'debugFlag': 135} + updatecfgDict = {'debugFlag': 135, 'clientCfg':clientCfgDict} + # updatecfgDict = {'debugFlag': 135, 'clientCfg':clientCfgDict, 'tmqRowSize':1} + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def consume_test(self): + + tdSql.execute(f'create database if not exists d1') + tdSql.execute(f'use d1') + tdSql.execute(f'create table st(ts timestamp, i int) tags(t int)') + tdSql.execute(f'insert into t1 using st tags(1) values(now, 1) (now+1s, 2)') + tdSql.execute(f'insert into t2 using st tags(2) values(now, 1) (now+1s, 2)') + tdSql.execute(f'insert into t3 using st tags(3) values(now, 1) (now+1s, 2)') + + + tdSql.execute(f'create topic topic_all as select * from st') + consumer_dict = { + "group.id": "g1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.offset.reset": "earliest", + } + consumer = Consumer(consumer_dict) + + try: + consumer.unsubscribe() + consumer.unsubscribe() + consumer.subscribe(["topic_all"]) + consumer.subscribe(["topic_all"]) + except TmqError: + tdLog.exit(f"subscribe error") + + cnt = 0 + try: + while True: + res = consumer.poll(2) + if not res: + break + val = res.value() + if val is None: + print(f"null val") + continue + for block in val: + cnt += len(block.fetchall()) + + print(f"block {cnt} rows") + + finally: + consumer.unsubscribe(); + consumer.close() + def run(self): + self.consume_test() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/td-32187.py b/tests/system-test/7-tmq/td-32187.py new file mode 100644 index 0000000000..7f971b23da --- /dev/null +++ b/tests/system-test/7-tmq/td-32187.py @@ -0,0 +1,45 @@ +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from taos.tmq import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 135, 'asynclog': 0} + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def run(self): + tdSql.execute(f'create database if not exists db_32187') + tdSql.execute(f'use db_32187') + tdSql.execute(f'create stable if not exists s5466 (ts timestamp, c1 int, c2 int) tags (t binary(32))') + tdSql.execute(f'insert into t1 using s5466 tags("__devicid__") values(1669092069068, 0, 1)') + tdSql.execute(f'insert into t1(ts, c1, c2) values(1669092069067, 0, 1)') + + tdSql.execute("create topic topic_test with meta as database db_32187") + buildPath = tdCom.getBuildPath() + cmdStr = '%s/build/bin/tmq_td32187'%(buildPath) + tdLog.info(cmdStr) + os.system(cmdStr) + + return + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/7-tmq/tmqClientConsLog.py b/tests/system-test/7-tmq/tmqClientConsLog.py deleted file mode 100644 index 83d6f93be1..0000000000 --- a/tests/system-test/7-tmq/tmqClientConsLog.py +++ /dev/null @@ -1,231 +0,0 @@ - -import taos -import sys -import time -import socket -import os -import threading -import math - -from util.log import * -from util.sql import * -from util.cases import * -from util.dnodes import * -from util.common import * -sys.path.append("./7-tmq") -from tmqCommon import * - -class TDTestCase: - - clientCfgDict = {'debugFlag': 135} - updatecfgDict = {'debugFlag': 131, 'clientCfg':clientCfgDict} - def __init__(self): - self.vgroups = 3 - self.ctbNum = 10 - self.rowsPerTbl = 1000 - - def init(self, conn, logSql, replicaVar=1): - self.replicaVar = int(replicaVar) - tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), True) - - def prepareTestEnv(self): - tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") - paraDict = {'dbName': 'dbt', - 'dropFlag': 1, - 'event': '', - 'vgroups': 2, - 'stbName': 'stb', - 'colPrefix': 'c', - 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ctb', - 'ctbStartIdx': 0, - 'ctbNum': 10, - 'rowsPerTbl': 1000, - 'batchNum': 100, - 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 10, - 'showMsg': 1, - 'showRow': 1, - 'snapshot': 0} - - paraDict['vgroups'] = self.vgroups - paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - - tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar) - tdLog.info("create stb") - tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) - tdLog.info("create ctb") - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], - ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") - tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], - ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], - startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) - - # tdLog.info("restart taosd to ensure that the data falls into the disk") - # tdDnodes.stop(1) - # tdDnodes.start(1) - # tdSql.query("flush database %s"%(paraDict['dbName'])) - return - - def updateRowsOfConsumer(self, consumerDict, consumerId, totalRowsOfConsumer): - for key in consumerDict: - if key == consumerId: - consumerDict[key] = totalRowsOfConsumer - return - - consumerDict[consumerId] = totalRowsOfConsumer - return - - def checkClientLog(self, actConsumeTotalRows, numOfConsumer): - # 01931245 TSC consumer:0x5ee20f124420000c process poll rsp, vgId:5, offset:log:3399, blocks:2, rows:6000 vg total:330000 total:654000, reqId:0xa77d2245ae20112 - # 01931245 TSC consumer:0x5ee20f124420000c process poll rsp, vgId:7, offset:log:3384, blocks:1, rows:2000 vg total:326000 total:656000, reqId:0xa77d2245b050113 - # 01931246 TSC consumer:0x5ee20f124420000d process poll rsp, vgId:6, offset:log:3400, blocks:2, rows:6000 vg total:330000 total:330000, reqId:0xa77d2245b380116 - # 01931246 TSC consumer:0x5ee20f124420000d process poll rsp, vgId:6, offset:log:3460, blocks:2, rows:6000 vg total:336000 total:336000, reqId:0xa77d2245b8f011a - # 01931246 TSC consumer:0x5ee20f124420000d process poll rsp, vgId:6, offset:log:3520, blocks:2, rows:6000 vg total:342000 total:342000, reqId:0xa77d2245beb011f - # 01931246 TSC consumer:0x5ee20f124420000d process poll rsp, vgId:6, offset:log:3567, blocks:1, rows:2000 vg total:344000 total:344000, reqId:0xa77d2245c430121 - # filter key: process poll rsp, vgId - - tdLog.printNoPrefix("======== start filter key info from client log file") - - cfgPath = tdCom.getClientCfgPath() - taosLogFile = '%s/../log/taoslog*'%(cfgPath) - filterResultFile = '%s/../log/filter'%(cfgPath) - cmdStr = 'grep -h "process poll rsp, vgId:" %s >> %s'%(taosLogFile, filterResultFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumerDict = {} - for index, line in enumerate(open(filterResultFile,'r')): - - # tdLog.info("row[%d]: %s"%(index, line)) - valueList = line.split(',') - # for i in range(len(valueList)): - # tdLog.info("index[%d]: %s"%(i, valueList[i])) - # get consumer id - list2 = valueList[0].split(':') - list3 = list2[3].split() - consumerId = list3[0] - print("consumerId: %s"%(consumerId)) - - # # get vg id - # list2 = valueList[1].split(':') - # vgId = list2[1] - # print("vgId: %s"%(vgId)) - - # get total rows of a certain consuer - list2 = valueList[6].split(':') - totalRowsOfConsumer = list2[1] - print("totalRowsOfConsumer: %s"%(totalRowsOfConsumer)) - - # update a certain info - self.updateRowsOfConsumer(consumerDict, consumerId, totalRowsOfConsumer) - - # print(consumerDict) - if numOfConsumer != len(consumerDict): - tdLog.info("expect consumer num: %d, act consumer num: %d"%(numOfConsumer, len(consumerDict))) - tdLog.exit("act consumer error!") - - # total rows of all consumers - totalRows = 0 - for key in consumerDict: - totalRows += int(consumerDict[key]) - - if totalRows < actConsumeTotalRows: - tdLog.info("expect consume total rows: %d, act consume total rows: %d"%(actConsumeTotalRows, totalRows)) - tdLog.exit("act consume rows error!") - return - - def tmqCase1(self): - tdLog.printNoPrefix("======== test case 1: ") - paraDict = {'dbName': 'dbt', - 'dropFlag': 1, - 'event': '', - 'vgroups': 2, - 'stbName': 'stb', - 'colPrefix': 'c', - 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ctb', - 'ctbStartIdx': 0, - 'ctbNum': 10, - 'rowsPerTbl': 1000, - 'batchNum': 100, - 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 10, - 'showMsg': 1, - 'showRow': 1, - 'snapshot': 0} - paraDict['vgroups'] = self.vgroups - paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - - topicNameList = ['topic1'] - expectRowsList = [] - tmqCom.initConsumerTable() - - tdLog.info("create topics from stb with filter") - queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) - # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicNameList[0], queryString) - tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - tdSql.query(queryString) - expectRowsList.append(tdSql.getRows()) - totalRowsInserted = expectRowsList[0] - - # init consume info, and start tmq_sim, then check consume result - tdLog.info("insert consume info to consume processor") - consumerId = 0 - expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] - topicList = topicNameList[0] - ifcheckdata = 0 - ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:500, auto.offset.reset:earliest' - tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - consumerId = 1 - tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - tdLog.info("start consume processor 0") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tdLog.info("wait the consume result") - - expectRows = 2 - resultList = tmqCom.selectConsumeResult(expectRows) - actConsumeTotalRows = resultList[0] + resultList[1] - - tdLog.info("two consumers poll rows: %d， %d"%(resultList[0], resultList[1])) - - tdLog.info("the consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted)) - if not (totalRowsInserted <= actConsumeTotalRows): - tdLog.exit("%d tmq consume rows error!"%consumerId) - - self.checkClientLog(actConsumeTotalRows, 2) - - time.sleep(10) - for i in range(len(topicNameList)): - tdSql.query("drop topic %s"%topicNameList[i]) - - tdLog.printNoPrefix("======== test case 1 end ...... ") - - def run(self): - tdSql.prepare() - self.prepareTestEnv() - self.tmqCase1() - - - def stop(self): - tdSql.close() - tdLog.success(f"{__file__} successfully executed") - -event = threading.Event() - -tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py index 7b31019572..96352fbe52 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py @@ -198,7 +198,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if not (expectrowcnt <= resultList[0] and totalRowsInserted >= resultList[0]): + if expectrowcnt > resultList[0]: tdLog.info("act consume rows: %d, expect consume rows between %d and %d"%(resultList[0], expectrowcnt, totalRowsInserted)) tdLog.exit("%d tmq consume rows error!"%consumerId) @@ -219,7 +219,7 @@ class TDTestCase: actConsumeTotalRows = firstConsumeRows + resultList[0] - if not (expectrowcnt >= resultList[0] and totalRowsInserted == actConsumeTotalRows): + if totalRowsInserted > actConsumeTotalRows: tdLog.info("act consume rows, first: %d, second: %d "%(firstConsumeRows, resultList[0])) tdLog.info("and sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted)) tdLog.exit("%d tmq consume rows error!"%consumerId) diff --git a/tests/system-test/7-tmq/tmq_td32526.py b/tests/system-test/7-tmq/tmq_td32526.py new file mode 100644 index 0000000000..0dd6068cdd --- /dev/null +++ b/tests/system-test/7-tmq/tmq_td32526.py @@ -0,0 +1,54 @@ +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from taos.tmq import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 135, 'asynclog': 0} + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def run(self): + tdSql.execute(f'create database if not exists db_5466') + tdSql.execute(f'use db_5466') + tdSql.execute(f'CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)') + tdSql.execute("INSERT INTO d1001 USING meters TAGS('California.SanFrancisco', 2) VALUES('2018-10-05 14:38:05.000',10.30000,219,0.31000)") + + buildPath = tdCom.getBuildPath() + cmdStr = '%s/build/bin/tmq_td32526'%(buildPath) + # tdLog.info(cmdStr) + # os.system(cmdStr) + # + # tdSql.execute("drop topic db_5466_topic") + tdSql.execute(f'alter stable meters add column item_tags nchar(500)') + tdSql.execute(f'alter stable meters add column new_col nchar(100)') + tdSql.execute("create topic db_5466_topic as select * from db_5466.meters") + + tdSql.execute("INSERT INTO d1001 USING meters TAGS('California.SanFrancisco', 2) VALUES('2018-10-06 14:38:05.000',10.30000,219,0.31000, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', '1')") + + tdLog.info(cmdStr) + if os.system(cmdStr) != 0: + tdLog.exit(cmdStr) + + return + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/win-test-file b/tests/system-test/win-test-file index 1f2b3f476c..e86047bca8 100644 --- a/tests/system-test/win-test-file +++ b/tests/system-test/win-test-file @@ -87,7 +87,6 @@ python3 ./test.py -f 7-tmq/ins_topics_test.py python3 ./test.py -f 7-tmq/tmqMaxTopic.py python3 ./test.py -f 7-tmq/tmqParamsTest.py python3 ./test.py -f 7-tmq/tmqParamsTest.py -R -python3 ./test.py -f 7-tmq/tmqClientConsLog.py python3 ./test.py -f 7-tmq/tmqMaxGroupIds.py python3 ./test.py -f 7-tmq/tmqConsumeDiscontinuousData.py python3 ./test.py -f 7-tmq/tmqOffset.py diff --git a/tools/auto/stmt2Performance/json/query.json b/tools/auto/stmt2Performance/json/query.json new file mode 100644 index 0000000000..70f1d90edc --- /dev/null +++ b/tools/auto/stmt2Performance/json/query.json @@ -0,0 +1,23 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "continue_if_fail": "yes", + "databases": "dbrate", + "query_times": 20, + "query_mode": "taosc", + "specified_table_query": { + "query_interval": 0, + "concurrent": 10, + "sqls": [ + { + "sql": "select count(*) from meters", + "result": "./query_result.txt" + } + ] + } +} diff --git a/tools/auto/stmt2Performance/json/template.json b/tools/auto/stmt2Performance/json/template.json new file mode 100644 index 0000000000..659c5966a4 --- /dev/null +++ b/tools/auto/stmt2Performance/json/template.json @@ -0,0 +1,56 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 5, + "create_table_thread_count": 1, + "thread_bind_vgroup": "yes", + "confirm_parameter_prompt": "no", + "num_of_records_per_req": 2000, + "prepared_rand": 100000, + "escape_character": "yes", + "databases": [ + { + "dbinfo": { + "name": "dbrate", + "drop": "yes", + "vgroups": 2 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 10, + "childtable_prefix": "d", + "insert_mode": "@STMT_MODE", + "interlace_rows": @INTERLACE_MODE, + "insert_rows": 100000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "auto_create_table": "no", + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc"}, + { "type": "double", "name": "dc"}, + { "type": "tinyint", "name": "ti"}, + { "type": "smallint", "name": "si"}, + { "type": "int", "name": "ic"}, + { "type": "bigint", "name": "bi"}, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi"}, + { "type": "uint", "name": "ui"}, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 32}, + { "type": "nchar", "name": "nch", "len": 64} + ], + "tags": [ + {"type": "TINYINT", "name": "groupid", "max": 10, "min": 1} + ] + } + ] + } + ] +} diff --git a/tools/auto/stmt2Performance/stmt2Perf.py b/tools/auto/stmt2Performance/stmt2Perf.py new file mode 100644 index 0000000000..e7a4d5ecbe --- /dev/null +++ b/tools/auto/stmt2Performance/stmt2Perf.py @@ -0,0 +1,356 @@ +import taos +import sys +import os +import subprocess +import time +import random +import json +from datetime import datetime + + +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +dataDir = "/var/lib/taos/" +templateFile = "json/template.json" +Number = 0 +resultContext = "" + + +def showLog(str): + print(str) + +def exec(command, show=True): + if(show): + print(f"exec {command}\n") + return os.system(command) + +# run return output and error +def run(command, timeout = 60, show=True): + if(show): + print(f"run {command} timeout={timeout}s\n") + + process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process.wait(timeout) + + output = process.stdout.read().decode(encoding="gbk") + error = process.stderr.read().decode(encoding="gbk") + + return output, error + +# return list after run +def runRetList(command, timeout=10, first=True): + output,error = run(command, timeout) + if first: + return output.splitlines() + else: + return error.splitlines() + + +def readFileContext(filename): + file = open(filename) + context = file.read() + file.close() + return context + + +def writeFileContext(filename, context): + file = open(filename, "w") + file.write(context) + file.close() + +def appendFileContext(filename, context): + global resultContext + resultContext += context + try: + file = open(filename, "a") + wsize = file.write(context) + file.close() + except: + print(f"appand file error context={context} .") + +def getFolderSize(folder): + total_size = 0 + for dirpath, dirnames, filenames in os.walk(folder): + for filename in filenames: + filepath = os.path.join(dirpath, filename) + total_size += os.path.getsize(filepath) + return total_size + +def waitClusterAlive(loop): + for i in range(loop): + command = 'taos -s "show cluster alive\G;" ' + out,err = run(command) + print(out) + if out.find("status: 1") >= 0: + showLog(f" i={i} wait cluster alive ok.\n") + return True + + showLog(f" i={i} wait cluster alive ...\n") + time.sleep(1) + + showLog(f" i={i} wait cluster alive failed.\n") + return False + +def waitCompactFinish(loop): + for i in range(loop): + command = 'taos -s "show compacts;" ' + out,err = run(command) + if out.find("Query OK, 0 row(s) in set") >= 0: + showLog(f" i={i} wait compact finish ok\n") + return True + + showLog(f" i={i} wait compact ...\n") + time.sleep(1) + + showLog(f" i={i} wait compact failed.\n") + return False + + +def getTypeName(datatype): + str1 = datatype.split(",")[0] + str2 = str1.split(":")[1] + str3 = str2.replace('"','').replace(' ','') + return str3 + + +def getMatch(datatype, algo): + if algo == "tsz": + if datatype == "float" or datatype == "double": + return True + else: + return False + else: + return True + + +def generateJsonFile(stmt, interlace): + print(f"doTest stmt: {stmt} interlace_rows={interlace}\n") + + # replace datatype + context = readFileContext(templateFile) + # replace compress + context = context.replace("@STMT_MODE", stmt) + context = context.replace("@INTERLACE_MODE", interlace) + + # write to file + fileName = f"json/test_{stmt}_{interlace}.json" + if os.path.exists(fileName): + os.remove(fileName) + writeFileContext(fileName, context) + + return fileName + +def taosdStart(): + cmd = "nohup /usr/bin/taosd 2>&1 & " + ret = exec(cmd) + print(f"exec taosd ret = {ret}\n") + time.sleep(3) + waitClusterAlive(10) + +def taosdStop(): + i = 1 + toBeKilled = "taosd" + killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" % toBeKilled + psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled + processID = subprocess.check_output(psCmd, shell=True) + while(processID): + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output(psCmd, shell=True) + print(f"i={i} kill taosd pid={processID}") + i += 1 + +def cleanAndStartTaosd(): + + # stop + taosdStop() + # clean + exec(f"rm -rf {dataDir}") + # start + taosdStart() + +def findContextValue(context, label): + start = context.find(label) + if start == -1 : + return "" + start += len(label) + 2 + # skip blank + while context[start] == ' ': + start += 1 + + # find end ',' + end = start + ends = [',','}',']', 0] + while context[end] not in ends: + end += 1 + return context[start:end] + + +def writeTemplateInfo(resultFile): + # create info + context = readFileContext(templateFile) + vgroups = findContextValue(context, "vgroups") + childCount = findContextValue(context, "childtable_count") + insertRows = findContextValue(context, "insert_rows") + line = f"vgroups = {vgroups}\nchildtable_count = {childCount}\ninsert_rows = {insertRows}\n\n" + print(line) + appendFileContext(resultFile, line) + + +def totalCompressRate(stmt, interlace, resultFile, writeSpeed, querySpeed): + global Number + # flush + command = 'taos -s "flush database dbrate;"' + rets = exec(command) + + command = 'taos -s "compact database dbrate;"' + rets = exec(command) + waitCompactFinish(60) + + # read compress rate + command = 'taos -s "show table distributed dbrate.meters\G;"' + rets = runRetList(command) + print(rets) + + str1 = rets[5] + arr = str1.split(" ") + + # Total_Size KB + str2 = arr[2] + pos = str2.find("=[") + totalSize = int(float(str2[pos+2:])/1024) + + # Compression_Ratio + str2 = arr[6] + pos = str2.find("=[") + rate = str2[pos+2:] + print("rate =" + rate) + + # total data file size + #dataSize = getFolderSize(f"{dataDir}/vnode/") + #dataSizeMB = int(dataSize/1024/1024) + + # appand to file + + Number += 1 + context = "%10s %10s %15s %10s %10s %30s %15s\n"%( Number, stmt, interlace, str(totalSize)+" MB", rate+"%", writeSpeed + " Records/second", querySpeed) + showLog(context) + appendFileContext(resultFile, context) + +def testWrite(jsonFile): + command = f"taosBenchmark -f {jsonFile}" + output, context = run(command, 60000) + # SUCC: Spent 0.960248 (real 0.947154) seconds to insert rows: 100000 with 1 thread(s) into dbrate 104139.76 (real 105579.45) records/second + + # find second real + pos = context.find("(real ") + if pos == -1: + print(f"error, run command={command} output not found first \"(real\" keyword. error={context}") + exit(1) + pos = context.find("(real ", pos + 5) + if pos == -1: + print(f"error, run command={command} output not found second \"(real\" keyword. error={context}") + exit(1) + + pos += 5 + length = len(context) + while pos < length and context[pos] == ' ': + pos += 1 + end = context.find(".", pos) + if end == -1: + print(f"error, run command={command} output not found second \".\" keyword. error={context}") + exit(1) + + speed = context[pos: end] + #print(f"write pos ={pos} end={end} speed={speed}\n output={context} \n") + return speed + +def testQuery(): + command = f"taosBenchmark -f json/query.json" + lines = runRetList(command, 60000) + # INFO: Spend 6.7350 second completed total queries: 10, the QPS of all threads: 1.485 + speed = None + + for i in range(0, len(lines)): + # find second real + context = lines[i] + pos = context.find("the QPS of all threads:") + if pos == -1 : + continue + pos += 24 + speed = context[pos:] + break + #print(f"query pos ={pos} speed={speed}\n output={context} \n") + + if speed is None: + print(f"error, run command={command} output not found second \"the QPS of all threads:\" keyword. error={lines}") + exit(1) + else: + return speed + +def doTest(stmt, interlace, resultFile): + print(f"doTest stmtMode: {stmt} interlaceRows={interlace}\n") + #cleanAndStartTaosd() + + + # json + jsonFile = generateJsonFile(stmt, interlace) + + # run taosBenchmark + t1 = time.time() + writeSpeed = testWrite(jsonFile) + t2 = time.time() + # total write speed + querySpeed = testQuery() + + # total compress rate + totalCompressRate(stmt, interlace, resultFile, writeSpeed, querySpeed) + + +def main(): + + # test compress method + stmtModes = ["stmt", "stmt2", "taosc"] + interlaceModes = ["0", "1"] + + # record result + resultFile = "./result.txt" + timestamp = time.time() + now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timestamp)) + context = f"\n---------------------- test rate ({now}) ---------------------------------\n" + + appendFileContext(resultFile, context) + # json info + writeTemplateInfo(resultFile) + # head + context = "\n%10s %10s %15s %10s %10s %30s %15s\n"%("No", "stmtMode", "interlaceRows", "dataSize", "rate", "writeSpeed", "query-QPS") + appendFileContext(resultFile, context) + + + # loop for all compression + for stmt in stmtModes: + # do test + for interlace in interlaceModes: + doTest(stmt, interlace, resultFile) + appendFileContext(resultFile, " \n") + + timestamp = time.time() + now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timestamp)) + appendFileContext(resultFile, f"\n{now} finished test!\n") + + +if __name__ == "__main__": + print("welcome use TDengine compress rate test tools.\n") + main() + # show result + print(resultContext) \ No newline at end of file diff --git a/tools/auto/testCompression/json/query.json b/tools/auto/testCompression/json/query.json new file mode 100644 index 0000000000..e810c1009f --- /dev/null +++ b/tools/auto/testCompression/json/query.json @@ -0,0 +1,23 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "continue_if_fail": "yes", + "databases": "dbrate", + "query_times": 20, + "query_mode": "taosc", + "specified_table_query": { + "query_interval": 0, + "concurrent": 10, + "sqls": [ + { + "sql": "select * from meters", + "result": "./query_res0.txt" + } + ] + } +} diff --git a/tools/auto/testCompression/testCompression.py b/tools/auto/testCompression/testCompression.py index 768fa87f71..281a097f8a 100644 --- a/tools/auto/testCompression/testCompression.py +++ b/tools/auto/testCompression/testCompression.py @@ -48,9 +48,13 @@ def run(command, timeout = 60, show=True): return output, error # return list after run -def runRetList(command, timeout=10): +def runRetList(command, timeout=10, first=True): output,error = run(command, timeout) - return output.splitlines() + if first: + return output.splitlines() + else: + return error.splitlines() + def readFileContext(filename): file = open(filename) @@ -204,7 +208,7 @@ def writeTemplateInfo(resultFile): appendFileContext(resultFile, line) -def totalCompressRate(algo, resultFile, writeSecond): +def totalCompressRate(algo, resultFile, writeSpeed, querySpeed): global Number # flush command = 'taos -s "flush database dbrate;"' @@ -239,10 +243,60 @@ def totalCompressRate(algo, resultFile, writeSecond): # appand to file Number += 1 - context = "%10s %10s %10s %10s %10s\n"%( Number, algo, str(totalSize)+" MB", rate+"%", writeSecond + " s") + context = "%10s %10s %10s %10s %30s %15s\n"%( Number, algo, str(totalSize)+" MB", rate+"%", writeSpeed + " Records/second", querySpeed) showLog(context) appendFileContext(resultFile, context) +def testWrite(jsonFile): + command = f"taosBenchmark -f {jsonFile}" + output, context = run(command, 60000) + # SUCC: Spent 0.960248 (real 0.947154) seconds to insert rows: 100000 with 1 thread(s) into dbrate 104139.76 (real 105579.45) records/second + + # find second real + pos = context.find("(real ") + if pos == -1: + print(f"error, run command={command} output not found first \"(real\" keyword. error={context}") + exit(1) + pos = context.find("(real ", pos + 5) + if pos == -1: + print(f"error, run command={command} output not found second \"(real\" keyword. error={context}") + exit(1) + + pos += 5 + length = len(context) + while pos < length and context[pos] == ' ': + pos += 1 + end = context.find(".", pos) + if end == -1: + print(f"error, run command={command} output not found second \".\" keyword. error={context}") + exit(1) + + speed = context[pos: end] + #print(f"write pos ={pos} end={end} speed={speed}\n output={context} \n") + return speed + +def testQuery(): + command = f"taosBenchmark -f json/query.json" + lines = runRetList(command, 60000) + # INFO: Spend 6.7350 second completed total queries: 10, the QPS of all threads: 1.485 + speed = None + + for i in range(20, len(lines)): + # find second real + pos = context.find("the QPS of all threads:") + context = lines[26] + if pos == -1 : + continue + pos += 24 + speed = context[pos:] + break + #print(f"query pos ={pos} speed={speed}\n output={context} \n") + + if speed is None: + print(f"error, run command={command} output not found second \"the QPS of all threads:\" keyword. error={lines}") + exit(1) + else: + return speed def doTest(algo, resultFile): print(f"doTest algo: {algo} \n") @@ -254,16 +308,20 @@ def doTest(algo, resultFile): # run taosBenchmark t1 = time.time() - exec(f"taosBenchmark -f {jsonFile}") + writeSpeed = testWrite(jsonFile) t2 = time.time() + # total write speed + querySpeed = testQuery() # total compress rate - totalCompressRate(algo, resultFile, str(int(t2-t1))) + totalCompressRate(algo, resultFile, writeSpeed, querySpeed) + def main(): # test compress method algos = ["lz4", "zlib", "zstd", "xz", "disabled"] + #algos = ["lz4"] # record result resultFile = "./result.txt" @@ -275,7 +333,7 @@ def main(): # json info writeTemplateInfo(resultFile) # head - context = "\n%10s %10s %10s %10s %10s\n"%("No", "compress", "dataSize", "rate", "insertSeconds") + context = "\n%10s %10s %10s %10s %30s %15s\n"%("No", "compress", "dataSize", "rate", "writeSpeed", "query-QPS") appendFileContext(resultFile, context) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 84da746afd..2a583f948e 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -44,7 +44,7 @@ static int32_t shellRunSingleCommand(char *command); static void shellRecordCommandToHistory(char *command); static int32_t shellRunCommand(char *command, bool recordHistory); static void shellRunSingleCommandImp(char *command); -static char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision); +static char *shellFormatTimestamp(char *buf, int32_t bufSize, int64_t val, int32_t precision); static int64_t shellDumpResultToFile(const char *fname, TAOS_RES *tres); static void shellPrintNChar(const char *str, int32_t length, int32_t width); static void shellPrintGeometry(const unsigned char *str, int32_t length, int32_t width); @@ -304,7 +304,7 @@ void shellRunSingleCommandImp(char *command) { printf("\r\n"); } -char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) { +char *shellFormatTimestamp(char *buf, int32_t bufSize, int64_t val, int32_t precision) { if (shell.args.is_raw_time) { sprintf(buf, "%" PRId64, val); return buf; @@ -335,7 +335,7 @@ char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) { } struct tm ptm = {0}; - if (taosLocalTime(&tt, &ptm, buf) == NULL) { + if (taosLocalTime(&tt, &ptm, buf, bufSize) == NULL) { return buf; } size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); @@ -405,7 +405,7 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i if (tsEnableScience) { taosFprintfFile(pFile, "%*.7e", width, GET_FLOAT_VAL(val)); } else { - n = snprintf(buf, LENGTH, "%*.7f", width, GET_FLOAT_VAL(val)); + n = tsnprintf(buf, LENGTH, "%*.7f", width, GET_FLOAT_VAL(val)); if (n > SHELL_FLOAT_WIDTH) { taosFprintfFile(pFile, "%*.7e", width, GET_FLOAT_VAL(val)); } else { @@ -419,7 +419,7 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i snprintf(buf, LENGTH, "%*.15e", width, GET_DOUBLE_VAL(val)); taosFprintfFile(pFile, "%s", buf); } else { - n = snprintf(buf, LENGTH, "%*.15f", width, GET_DOUBLE_VAL(val)); + n = tsnprintf(buf, LENGTH, "%*.15f", width, GET_DOUBLE_VAL(val)); if (n > SHELL_DOUBLE_WIDTH) { taosFprintfFile(pFile, "%*.15e", width, GET_DOUBLE_VAL(val)); } else { @@ -465,7 +465,7 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i break; } case TSDB_DATA_TYPE_TIMESTAMP: - shellFormatTimestamp(buf, *(int64_t *)val, precision); + shellFormatTimestamp(buf, sizeof(buf), *(int64_t *)val, precision); taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr); break; default: @@ -670,7 +670,7 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t if (tsEnableScience) { printf("%*.7e", width, GET_FLOAT_VAL(val)); } else { - n = snprintf(buf, LENGTH, "%*.7f", width, GET_FLOAT_VAL(val)); + n = tsnprintf(buf, LENGTH, "%*.7f", width, GET_FLOAT_VAL(val)); if (n > SHELL_FLOAT_WIDTH) { printf("%*.7e", width, GET_FLOAT_VAL(val)); } else { @@ -683,7 +683,7 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t snprintf(buf, LENGTH, "%*.15e", width, GET_DOUBLE_VAL(val)); printf("%s", buf); } else { - n = snprintf(buf, LENGTH, "%*.15f", width, GET_DOUBLE_VAL(val)); + n = tsnprintf(buf, LENGTH, "%*.15f", width, GET_DOUBLE_VAL(val)); if (n > SHELL_DOUBLE_WIDTH) { printf("%*.15e", width, GET_DOUBLE_VAL(val)); } else { @@ -710,7 +710,7 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t shellPrintGeometry(val, length, width); break; case TSDB_DATA_TYPE_TIMESTAMP: - shellFormatTimestamp(buf, *(int64_t *)val, precision); + shellFormatTimestamp(buf, sizeof(buf), *(int64_t *)val, precision); printf("%s", buf); break; default: diff --git a/utils/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt index 991a004a74..8701f208bb 100644 --- a/utils/test/c/CMakeLists.txt +++ b/utils/test/c/CMakeLists.txt @@ -4,6 +4,8 @@ add_executable(tmq_sim tmqSim.c) add_executable(create_table createTable.c) add_executable(tmq_taosx_ci tmq_taosx_ci.c) add_executable(tmq_ts5466 tmq_ts5466.c) +add_executable(tmq_td32526 tmq_td32526.c) +add_executable(tmq_td32187 tmq_td32187.c) add_executable(tmq_write_raw_test tmq_write_raw_test.c) add_executable(write_raw_block_test write_raw_block_test.c) add_executable(sml_test sml_test.c) @@ -62,6 +64,20 @@ target_link_libraries( PUBLIC common PUBLIC os ) +target_link_libraries( + tmq_td32187 + PUBLIC taos + PUBLIC util + PUBLIC common + PUBLIC os +) +target_link_libraries( + tmq_td32526 + PUBLIC taos + PUBLIC util + PUBLIC common + PUBLIC os +) target_link_libraries( tmq_taosx_ci PUBLIC taos diff --git a/utils/test/c/tmqDemo.c b/utils/test/c/tmqDemo.c index 64f536433e..40ed98132c 100644 --- a/utils/test/c/tmqDemo.c +++ b/utils/test/c/tmqDemo.c @@ -597,7 +597,7 @@ void printParaIntoFile() { time_t tTime = taosGetTimestampSec(); struct tm tm; - taosLocalTime(&tTime, &tm, NULL); + taosLocalTime(&tTime, &tm, NULL, 0); taosFprintfFile(pFile, "###################################################################\n"); taosFprintfFile(pFile, "# configDir: %s\n", configDir); diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index 006bd516d0..e2a09c4259 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -166,7 +166,7 @@ static void printHelp() { char* getCurrentTimeString(char* timeString) { time_t tTime = taosGetTimestampSec(); struct tm tm; - taosLocalTime(&tTime, &tm, NULL); + taosLocalTime(&tTime, &tm, NULL, 0); sprintf(timeString, "%d-%02d-%02d %02d:%02d:%02d", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); @@ -441,7 +441,7 @@ int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) { return 0; } -static char* shellFormatTimestamp(char* buf, int64_t val, int32_t precision) { +static char* shellFormatTimestamp(char* buf, int32_t bufSize, int64_t val, int32_t precision) { // if (shell.args.is_raw_time) { // sprintf(buf, "%" PRId64, val); // return buf; @@ -472,7 +472,7 @@ static char* shellFormatTimestamp(char* buf, int64_t val, int32_t precision) { } struct tm ptm; - if (taosLocalTime(&tt, &ptm, buf) == NULL) { + if (taosLocalTime(&tt, &ptm, buf, bufSize) == NULL) { return buf; } size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); @@ -559,7 +559,7 @@ static void shellDumpFieldToFile(TdFilePtr pFile, const char* val, TAOS_FIELD* f taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr); } break; case TSDB_DATA_TYPE_TIMESTAMP: - shellFormatTimestamp(buf, *(int64_t*)val, precision); + shellFormatTimestamp(buf, sizeof(buf), *(int64_t*)val, precision); taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr); break; default: diff --git a/utils/test/c/tmq_td32187.c b/utils/test/c/tmq_td32187.c new file mode 100644 index 0000000000..fb26e248c9 --- /dev/null +++ b/utils/test/c/tmq_td32187.c @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include "cJSON.h" +#include "taos.h" +#include "tmsg.h" +#include "types.h" + + +static TAOS_RES* tmqmessage = NULL; +static char* topic = "topic_test"; +static int32_t vgroupId = 0; +static int64_t offset = 0; + +void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { + printf("commit %d tmq %p param %p\n", code, tmq, param); +} + +tmq_t* build_consumer() { + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", "tg2"); + tmq_conf_set(conf, "client.id", "my app 1"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "msg.with.table.name", "true"); + tmq_conf_set(conf, "enable.auto.commit", "true"); + tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "msg.consume.excluded", "1"); +// tmq_conf_set(conf, "experimental.snapshot.enable", "true"); + + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + assert(tmq); + tmq_conf_destroy(conf); + return tmq; +} + +tmq_list_t* build_topic_list() { + tmq_list_t* topic_list = tmq_list_new(); + tmq_list_append(topic_list, topic); + return topic_list; +} + +static void callFunc(int i, tmq_t* tmq, tmq_list_t* topics) { + printf("call %d\n", i); + switch (i) { + case 0: + tmq_subscribe(tmq, topics); + break; + case 1: + tmq_unsubscribe(tmq); + break; + case 2:{ + tmq_list_t* t = NULL; + tmq_subscription(tmq, &t); + tmq_list_destroy(t); + break; + } + case 3: + taos_free_result(tmqmessage); + tmqmessage = tmq_consumer_poll(tmq, 5000); + break; + case 4: +// tmq_consumer_close(tmq); + break; + case 5: + tmq_commit_sync(tmq, NULL); + break; + case 6: + tmq_commit_async(tmq, NULL, NULL, NULL); + break; + case 7: + tmq_commit_offset_sync(tmq, topic, vgroupId, offset); + break; + case 8: + tmq_commit_offset_async(tmq, topic, vgroupId, offset, NULL, NULL); + break; + case 9: + tmq_get_topic_assignment(tmq, topic, NULL, NULL); + break; + case 10: + tmq_free_assignment(NULL); + break; + case 11: + tmq_offset_seek(tmq, topic, vgroupId, offset); + break; + case 12: + tmq_position(tmq, topic, vgroupId); + break; + case 13: + tmq_committed(tmq, topic, vgroupId); + break; + case 14: + tmq_get_connect(tmq); + break; + case 15: + tmq_get_table_name(tmqmessage); + break; + case 16: + vgroupId = tmq_get_vgroup_id(tmqmessage); + break; + case 17: + offset = tmq_get_vgroup_offset(tmqmessage); + break; + case 18: + tmq_get_res_type(tmqmessage); + break; + case 19: + tmq_get_topic_name(tmqmessage); + break; + case 20: + tmq_get_db_name(tmqmessage); + break; + default: + break; + } +} +void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { + int32_t code; + + if ((code = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(code)); + printf("subscribe err\n"); + return; + } + int32_t cnt = 0; + while (1) { + tmqmessage = tmq_consumer_poll(tmq, 5000); + if (tmqmessage) { + printf("poll message\n"); + while(cnt < 100){ + uint32_t i = taosRand()%21; + callFunc(i, tmq, topics); + callFunc(i, tmq, topics); + cnt++; + } + while(cnt < 300){ + uint32_t i = taosRand()%21; + callFunc(i, tmq, topics); + cnt++; + } + taos_free_result(tmqmessage); + } + break; + } + + code = tmq_consumer_close(tmq); + if (code) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); + else + fprintf(stderr, "%% Consumer closed\n"); +} + +int main(int argc, char* argv[]) { + tmq_t* tmq = build_consumer(); + tmq_list_t* topic_list = build_topic_list(); + basic_consume_loop(tmq, topic_list); + tmq_list_destroy(topic_list); +} \ No newline at end of file diff --git a/utils/test/c/tmq_td32526.c b/utils/test/c/tmq_td32526.c new file mode 100644 index 0000000000..42d38ec56c --- /dev/null +++ b/utils/test/c/tmq_td32526.c @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include "cJSON.h" +#include "taos.h" +#include "tmsg.h" +#include "types.h" + +void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { + printf("commit %d tmq %p param %p\n", code, tmq, param); +} + +tmq_t* build_consumer() { + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", "g1"); + tmq_conf_set(conf, "client.id", "my app 1"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "msg.with.table.name", "true"); + tmq_conf_set(conf, "enable.auto.commit", "true"); + tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "msg.consume.excluded", "1"); + + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + assert(tmq); + tmq_conf_destroy(conf); + return tmq; +} + +tmq_list_t* build_topic_list() { + tmq_list_t* topic_list = tmq_list_new(); + tmq_list_append(topic_list, "db_5466_topic"); + return topic_list; +} + +int printRow(char *str, TAOS_ROW row, TAOS_FIELD *fields, int numFields) { + int len = 0; + char split = ' '; + + for (int i = 0; i < numFields; ++i) { + if (i > 0) { + str[len++] = split; + } + + if (row[i] == NULL) { + len += sprintf(str + len, "%s", "NULL"); + continue; + } + + switch (fields[i].type) { + case TSDB_DATA_TYPE_TINYINT: + len += sprintf(str + len, "%d", *((int8_t *)row[i])); + break; + + case TSDB_DATA_TYPE_UTINYINT: + len += sprintf(str + len, "%u", *((uint8_t *)row[i])); + break; + + case TSDB_DATA_TYPE_SMALLINT: + len += sprintf(str + len, "%d", *((int16_t *)row[i])); + break; + + case TSDB_DATA_TYPE_USMALLINT: + len += sprintf(str + len, "%u", *((uint16_t *)row[i])); + break; + + case TSDB_DATA_TYPE_INT: + len += sprintf(str + len, "%d", *((int32_t *)row[i])); + break; + + case TSDB_DATA_TYPE_UINT: + len += sprintf(str + len, "%u", *((uint32_t *)row[i])); + break; + + case TSDB_DATA_TYPE_BIGINT: + len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); + break; + + case TSDB_DATA_TYPE_UBIGINT: + len += sprintf(str + len, "%" PRIu64, *((uint64_t *)row[i])); + break; + + case TSDB_DATA_TYPE_FLOAT: { + float fv = 0; + fv = GET_FLOAT_VAL(row[i]); + len += sprintf(str + len, "%f", fv); + } break; + + case TSDB_DATA_TYPE_DOUBLE: { + double dv = 0; + dv = GET_DOUBLE_VAL(row[i]); + len += sprintf(str + len, "%lf", dv); + } break; + + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_GEOMETRY: { + int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); + memcpy(str + len, row[i], charLen); + len += charLen; + } break; + + case TSDB_DATA_TYPE_TIMESTAMP: + len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); + break; + + case TSDB_DATA_TYPE_BOOL: + len += sprintf(str + len, "%d", *((int8_t *)row[i])); + default: + break; + } + } + + return len; +} + +/** + * @brief print column name and values of each row + * + * @param res + * @return int + */ +static int printResult(TAOS_RES *res) { + TAOS_ROW row = NULL; + int32_t cnt = 0; + while ((row = taos_fetch_row(res))) { + + int numFields = taos_num_fields(res); + TAOS_FIELD *fields = taos_fetch_fields(res); + char header[256] = {0}; + int len = 0; + for (int i = 0; i < numFields; ++i) { + len += sprintf(header + len, "%s ", fields[i].name); + } + puts(header); + + char temp[1024] = {0}; + printRow(temp, row, fields, numFields); + if (cnt == 0){ + ASSERT(strcmp(temp, "1538721485000 10.300000 219 0.310000 NULL NULL California.SanFrancisco 2") == 0); + }else if(cnt == 1){ + ASSERT(strcmp(temp, "1538807885000 10.300000 219 0.310000 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1 California.SanFrancisco 2") == 0); + } + cnt++; + } + return 0; +} + +void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { + int32_t code; + + if ((code = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(code)); + printf("subscribe err\n"); + return; + } + int32_t cnt = 0; + while (1) { + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 5000); + if (tmqmessage) { + cnt++; + printResult(tmqmessage); + taos_free_result(tmqmessage); + } else { + break; + } + } + + code = tmq_consumer_close(tmq); + if (code) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); + else + fprintf(stderr, "%% Consumer closed\n"); +} + +int main(int argc, char* argv[]) { + tmq_t* tmq = build_consumer(); + tmq_list_t* topic_list = build_topic_list(); + basic_consume_loop(tmq, topic_list); + tmq_list_destroy(topic_list); +} \ No newline at end of file diff --git a/utils/tsim/src/simExe.c b/utils/tsim/src/simExe.c index bea839057e..c7cec64dec 100644 --- a/utils/tsim/src/simExe.c +++ b/utils/tsim/src/simExe.c @@ -797,7 +797,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { tt = (*(int64_t *)row[i]) / 1000000000; } - if (taosLocalTime(&tt, &tp, timeStr) == NULL) { + if (taosLocalTime(&tt, &tp, timeStr, sizeof(timeStr)) == NULL) { break; } strftime(timeStr, 64, "%y-%m-%d %H:%M:%S", &tp);