diff --git a/cmake/addr2line_CMakeLists.txt.in b/cmake/addr2line_CMakeLists.txt.in index e514764af8..93fb9bb96c 100644 --- a/cmake/addr2line_CMakeLists.txt.in +++ b/cmake/addr2line_CMakeLists.txt.in @@ -5,8 +5,9 @@ ExternalProject_Add(addr2line GIT_TAG master SOURCE_DIR "${TD_CONTRIB_DIR}/addr2line" BINARY_DIR "${TD_CONTRIB_DIR}/addr2line" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/apr-util_CMakeLists.txt.in b/cmake/apr-util_CMakeLists.txt.in index a39100897b..8e14295ad6 100644 --- a/cmake/apr-util_CMakeLists.txt.in +++ b/cmake/apr-util_CMakeLists.txt.in @@ -11,9 +11,10 @@ ExternalProject_Add(aprutil-1 BUILD_IN_SOURCE TRUE BUILD_ALWAYS 1 #UPDATE_COMMAND "" - CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1/ --with-apr=$ENV{HOME}/.cos-local.1 + CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1/ --with-apr=$ENV{HOME}/.cos-local.1 #CONFIGURE_COMMAND ./configure --with-apr=/usr/local/apr BUILD_COMMAND make INSTALL_COMMAND make install TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/apr_CMakeLists.txt.in b/cmake/apr_CMakeLists.txt.in index 18c4eb62a1..af755c729f 100644 --- a/cmake/apr_CMakeLists.txt.in +++ b/cmake/apr_CMakeLists.txt.in @@ -16,4 +16,5 @@ ExternalProject_Add(apr-1 BUILD_COMMAND make INSTALL_COMMAND make install TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/cjson_CMakeLists.txt.in b/cmake/cjson_CMakeLists.txt.in index 32cea7baa1..941c5e786b 100644 --- a/cmake/cjson_CMakeLists.txt.in +++ b/cmake/cjson_CMakeLists.txt.in @@ -9,4 +9,5 @@ ExternalProject_Add(cjson BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" - ) \ No newline at end of file + GIT_SHALLOW true + ) diff --git a/cmake/cos_CMakeLists.txt.in b/cmake/cos_CMakeLists.txt.in index ee1e58b50f..0a0a5d4d62 100644 --- a/cmake/cos_CMakeLists.txt.in +++ b/cmake/cos_CMakeLists.txt.in @@ -9,4 +9,5 @@ ExternalProject_Add(cos BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/crashdump_CMakeLists.txt.in b/cmake/crashdump_CMakeLists.txt.in index af4b551159..3bc35a0f58 100644 --- a/cmake/crashdump_CMakeLists.txt.in +++ b/cmake/crashdump_CMakeLists.txt.in @@ -5,8 +5,9 @@ ExternalProject_Add(crashdump GIT_TAG master SOURCE_DIR "${TD_CONTRIB_DIR}/crashdump" BINARY_DIR "${TD_CONTRIB_DIR}/crashdump" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index 8e06569f4d..6494177faf 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -16,4 +16,5 @@ ExternalProject_Add(curl2 BUILD_COMMAND make -j INSTALL_COMMAND make install TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/geos_CMakeLists.txt.in b/cmake/geos_CMakeLists.txt.in index f939ccead0..e2b6ec5d4f 100644 --- a/cmake/geos_CMakeLists.txt.in +++ b/cmake/geos_CMakeLists.txt.in @@ -9,4 +9,5 @@ ExternalProject_Add(geos BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/gnuregex_CMakeLists.txt.in b/cmake/gnuregex_CMakeLists.txt.in index 817c22093a..cf748a3a54 100644 --- a/cmake/gnuregex_CMakeLists.txt.in +++ b/cmake/gnuregex_CMakeLists.txt.in @@ -10,4 +10,5 @@ ExternalProject_Add(gnuregex BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/gtest_CMakeLists.txt.in b/cmake/gtest_CMakeLists.txt.in index fe07f2318b..22498ecf8b 100644 --- a/cmake/gtest_CMakeLists.txt.in +++ b/cmake/gtest_CMakeLists.txt.in @@ -9,4 +9,5 @@ ExternalProject_Add(googletest BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" - ) \ No newline at end of file + GIT_SHALLOW true + ) diff --git a/cmake/iconv_CMakeLists.txt.in b/cmake/iconv_CMakeLists.txt.in index 5eb277ce0d..b9b3b084ca 100644 --- a/cmake/iconv_CMakeLists.txt.in +++ b/cmake/iconv_CMakeLists.txt.in @@ -9,4 +9,5 @@ ExternalProject_Add(iconv BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" - ) \ No newline at end of file + GIT_SHALLOW true + ) diff --git a/cmake/libdwarf_CMakeLists.txt.in b/cmake/libdwarf_CMakeLists.txt.in index 7de34cfbaa..30383afd89 100644 --- a/cmake/libdwarf_CMakeLists.txt.in +++ b/cmake/libdwarf_CMakeLists.txt.in @@ -5,8 +5,9 @@ ExternalProject_Add(libdwarf GIT_TAG libdwarf-0.3.1 SOURCE_DIR "${TD_CONTRIB_DIR}/libdwarf" BINARY_DIR "${TD_CONTRIB_DIR}/libdwarf" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index f2b6cac953..4884089a2b 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -13,4 +13,5 @@ ExternalProject_Add(libs3 BUILD_COMMAND make build/lib/libs3.a INSTALL_COMMAND make install_static TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/libuv_CMakeLists.txt.in b/cmake/libuv_CMakeLists.txt.in index 673c771fb0..3bfb52fe9b 100644 --- a/cmake/libuv_CMakeLists.txt.in +++ b/cmake/libuv_CMakeLists.txt.in @@ -5,8 +5,9 @@ ExternalProject_Add(libuv GIT_TAG v1.48.0 SOURCE_DIR "${TD_CONTRIB_DIR}/libuv" BINARY_DIR "${TD_CONTRIB_DIR}/libuv" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/lz4_CMakeLists.txt.in b/cmake/lz4_CMakeLists.txt.in index 381f8f5c5b..57420495ee 100644 --- a/cmake/lz4_CMakeLists.txt.in +++ b/cmake/lz4_CMakeLists.txt.in @@ -9,4 +9,5 @@ ExternalProject_Add(lz4 BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" - ) \ No newline at end of file + GIT_SHALLOW true + ) diff --git a/cmake/lzma_CMakeLists.txt.in b/cmake/lzma_CMakeLists.txt.in index fb16f89034..b6e96fb89f 100644 --- a/cmake/lzma_CMakeLists.txt.in +++ b/cmake/lzma_CMakeLists.txt.in @@ -1,9 +1,9 @@ -# xz +# xz if (${TD_LINUX}) ExternalProject_Add(lzma2 - GIT_REPOSITORY https://github.com/conor42/fast-lzma2.git + GIT_REPOSITORY https://github.com/conor42/fast-lzma2.git SOURCE_DIR "${TD_CONTRIB_DIR}/lzma2" #BINARY_DIR "" BUILD_IN_SOURCE TRUE @@ -11,5 +11,6 @@ ExternalProject_Add(lzma2 BUILD_COMMAND make INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) -endif() \ No newline at end of file +endif() diff --git a/cmake/msvcregex_CMakeLists.txt.in b/cmake/msvcregex_CMakeLists.txt.in index 686ac4cb5d..5c79c7982b 100644 --- a/cmake/msvcregex_CMakeLists.txt.in +++ b/cmake/msvcregex_CMakeLists.txt.in @@ -10,4 +10,5 @@ ExternalProject_Add(msvcregex BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" -) \ No newline at end of file + GIT_SHALLOW true +) diff --git a/cmake/mxml_CMakeLists.txt.in b/cmake/mxml_CMakeLists.txt.in index 762df40e10..464ec8c89b 100644 --- a/cmake/mxml_CMakeLists.txt.in +++ b/cmake/mxml_CMakeLists.txt.in @@ -11,4 +11,5 @@ ExternalProject_Add(mxml BUILD_COMMAND make INSTALL_COMMAND make install TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/pcre2_CMakeLists.txt.in b/cmake/pcre2_CMakeLists.txt.in index c3115d8201..b46dedef09 100644 --- a/cmake/pcre2_CMakeLists.txt.in +++ b/cmake/pcre2_CMakeLists.txt.in @@ -10,4 +10,5 @@ ExternalProject_Add(pcre2 BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/pthread_CMakeLists.txt.in b/cmake/pthread_CMakeLists.txt.in index 774f5dd972..2ec3362d3f 100644 --- a/cmake/pthread_CMakeLists.txt.in +++ b/cmake/pthread_CMakeLists.txt.in @@ -10,4 +10,5 @@ ExternalProject_Add(pthread BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/rocksdb_CMakeLists.txt.in b/cmake/rocksdb_CMakeLists.txt.in index 45599d82e3..71e59cefd8 100644 --- a/cmake/rocksdb_CMakeLists.txt.in +++ b/cmake/rocksdb_CMakeLists.txt.in @@ -11,6 +11,7 @@ if (${BUILD_CONTRIB}) BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) else() if (NOT ${TD_LINUX}) @@ -24,6 +25,7 @@ else() BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) endif() endif() diff --git a/cmake/sqlite_CMakeLists.txt.in b/cmake/sqlite_CMakeLists.txt.in index 445c49487a..6b77a82af9 100644 --- a/cmake/sqlite_CMakeLists.txt.in +++ b/cmake/sqlite_CMakeLists.txt.in @@ -10,4 +10,5 @@ ExternalProject_Add(sqlite BUILD_COMMAND "$(MAKE)" INSTALL_COMMAND "" TEST_COMMAND "" -) \ No newline at end of file + GIT_SHALLOW true +) diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in index 4778ea76e0..1098593943 100644 --- a/cmake/ssl_CMakeLists.txt.in +++ b/cmake/ssl_CMakeLists.txt.in @@ -12,4 +12,5 @@ ExternalProject_Add(openssl BUILD_COMMAND make -j INSTALL_COMMAND make install_sw -j TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/stub_CMakeLists.txt.in b/cmake/stub_CMakeLists.txt.in index 96cc924d81..6c54d33be7 100644 --- a/cmake/stub_CMakeLists.txt.in +++ b/cmake/stub_CMakeLists.txt.in @@ -10,4 +10,5 @@ ExternalProject_Add(stub BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/wcwidth_CMakeLists.txt.in b/cmake/wcwidth_CMakeLists.txt.in index 234ccbf915..3fd4e75243 100644 --- a/cmake/wcwidth_CMakeLists.txt.in +++ b/cmake/wcwidth_CMakeLists.txt.in @@ -10,4 +10,5 @@ ExternalProject_Add(wcwidth BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" -) \ No newline at end of file + GIT_SHALLOW true +) diff --git a/cmake/wingetopt_CMakeLists.txt.in b/cmake/wingetopt_CMakeLists.txt.in index b93925c170..115e91888b 100644 --- a/cmake/wingetopt_CMakeLists.txt.in +++ b/cmake/wingetopt_CMakeLists.txt.in @@ -10,4 +10,5 @@ ExternalProject_Add(wingetopt BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" -) \ No newline at end of file + GIT_SHALLOW true +) diff --git a/cmake/xml2_CMakeLists.txt.in b/cmake/xml2_CMakeLists.txt.in index be14a11c36..0e7492aea7 100644 --- a/cmake/xml2_CMakeLists.txt.in +++ b/cmake/xml2_CMakeLists.txt.in @@ -15,4 +15,5 @@ ExternalProject_Add(xml2 BUILD_COMMAND make -j INSTALL_COMMAND make install && ln -sf $ENV{HOME}/.cos-local.2/include/libxml2/libxml $ENV{HOME}/.cos-local.2/include/libxml TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/xz_CMakeLists.txt.in b/cmake/xz_CMakeLists.txt.in index e704fcbbbf..f3582e38df 100644 --- a/cmake/xz_CMakeLists.txt.in +++ b/cmake/xz_CMakeLists.txt.in @@ -1,14 +1,15 @@ # xz ExternalProject_Add(xz - GIT_REPOSITORY https://github.com/xz-mirror/xz.git - GIT_TAG v5.4.4 + GIT_REPOSITORY https://github.com/xz-mirror/xz.git + GIT_TAG v5.4.4 SOURCE_DIR "${TD_CONTRIB_DIR}/xz" BINARY_DIR "" #BUILD_IN_SOURCE TRUE - CMAKE_ARGS + CMAKE_ARGS CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" -) \ No newline at end of file + GIT_SHALLOW true +) diff --git a/cmake/zlib_CMakeLists.txt.in b/cmake/zlib_CMakeLists.txt.in index 87a17c7377..00e6d7dfce 100644 --- a/cmake/zlib_CMakeLists.txt.in +++ b/cmake/zlib_CMakeLists.txt.in @@ -12,4 +12,5 @@ ExternalProject_Add(zlib BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + GIT_SHALLOW true ) diff --git a/cmake/zstd_CMakeLists.txt.in b/cmake/zstd_CMakeLists.txt.in index f8604f9db6..d7850a5c7e 100644 --- a/cmake/zstd_CMakeLists.txt.in +++ b/cmake/zstd_CMakeLists.txt.in @@ -1,15 +1,16 @@ -# zstb +# zstb #ExternalProject_Add(zstd - #GIT_REPOSITORY https://github.com/facebook/zstd.git + #GIT_REPOSITORY https://github.com/facebook/zstd.git #GIT_TAG v1.5.5 #SOURCE_DIR "${TD_CONTRIB_DIR}/zstd" #DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" #BINARY_DIR "" - #CMAKE_COMMAND + #CMAKE_COMMAND #CMAKE_ARGS ${TD_CONTRIB_DIR}/zstd/build/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_ARCHIVE_OUTPUT_DIRECTORY=${CMAKE_BINARY_DIR}/build/lib - #BUILD_COMMAND make -j4 + #BUILD_COMMAND make -j4 #INSTALL_COMMAND "" #TEST_COMMAND "" - #) \ No newline at end of file + #GIT_SHALLOW true + #) diff --git a/docs/zh/06-advanced/05-data-in/05-opcua.md b/docs/zh/06-advanced/05-data-in/05-opcua.md index 36dfd08664..7ae7f153e9 100644 --- a/docs/zh/06-advanced/05-data-in/05-opcua.md +++ b/docs/zh/06-advanced/05-data-in/05-opcua.md @@ -169,8 +169,8 @@ CSV 文件中的每个 Row 配置一个 OPC 数据点位。Row 的规则如下 - **采集模式**：可使用 `subscribe` 或 `observe` 模式。 - `subscribe`：订阅模式，变更时上报数据并写入 TDengine。 - `observe`：按照`采集间隔`时长，轮询读取点位最新值并写入 TDengine。 -- 采集间隔：默认为 10 秒，数据点位采集间隔，从上次采集数据结束后开始计时，轮询读取点位最新值并写入 TDengine。 -- **采集超时**：向 OPC 服务器读取点位数据时如果超过设定时间未返回数据，则读取失败，默认为 10 秒。 +- **采集间隔**：默认为 10 秒，数据点位采集间隔，从上次采集数据结束后开始计时，轮询读取点位最新值并写入 TDengine。仅在 **采集模式** 为 `observe` 时可配置。 +- **采集超时**：向 OPC 服务器读取点位数据时如果超过设定时间未返回数据，则读取失败，默认为 10 秒。仅在 **采集模式** 为 `observe` 时可配置。 当 **点位集** 中使用 **选择数据点位** 方式时，采集配置中可以配置 **点位更新模式** 和 **点位更新间隔** 来启用动态点位更新。**动态点位更新** 是指，在任务运行期间，OPC Server增加或删除了点位后，符合条件的点位会自动添加到当前任务中，不需要重启 OPC 任务。 diff --git a/docs/zh/06-advanced/05-data-in/08-kafka.md b/docs/zh/06-advanced/05-data-in/08-kafka.md index 5c6d9190a8..dd5512d615 100644 --- a/docs/zh/06-advanced/05-data-in/08-kafka.md +++ b/docs/zh/06-advanced/05-data-in/08-kafka.md @@ -32,7 +32,11 @@ TDengine 可以高效地从 Kafka 读取数据并将其写入 TDengine，以实 ### 3. 配置连接信息 -在 **连接配置** 区域填写 **bootstrap-servers**，例如：`192.168.1.92:9092`。 +**bootstrap-server**，例如：`192.168.1.92`。 + +**服务端口**，例如：`9092`。 + +多个 broker 地址时，在连接配置右下增加 **新增 Broker** 按钮，成对增加 bootstrap-server 和服务端口。 ![kafka-03.png](./kafka-03.png) diff --git a/docs/zh/06-advanced/05-data-in/kafka-03.png b/docs/zh/06-advanced/05-data-in/kafka-03.png index 31a3ead3ed..a3257cc588 100644 Binary files a/docs/zh/06-advanced/05-data-in/kafka-03.png and b/docs/zh/06-advanced/05-data-in/kafka-03.png differ diff --git a/docs/zh/08-develop/01-connect/index.md b/docs/zh/08-develop/01-connect/index.md index 5cecd245e5..b16e96922f 100644 --- a/docs/zh/08-develop/01-connect/index.md +++ b/docs/zh/08-develop/01-connect/index.md @@ -132,7 +132,7 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 pip3 install taospy[ws] ``` - - **安装验证** +- **安装验证** 对于原生连接，需要验证客户端驱动和 Python 连接器本身是否都正确安装。如果能成功导入 `taos` 模块，则说明已经正确安装了客户端驱动和 Python 连接器。可在 Python 交互式 Shell 中输入： @@ -198,18 +198,18 @@ taos = { version = "*", default-features = false, features = ["ws"] } - **安装** - 使用 npm 安装 Node.js 连接器 - ``` - npm install @tdengine/websocket - ``` + ``` + npm install @tdengine/websocket + ``` :::note Node.js 目前只支持 Websocket 连接 - **安装验证** - 新建安装验证目录，例如：`~/tdengine-test`，下载 GitHub 上 [nodejsChecker.js 源代码](https://github.com/taosdata/TDengine/tree/main/docs/examples/node/websocketexample/nodejsChecker.js)到本地。 - 在命令行中执行以下命令。 - ```bash - npm init -y - npm install @tdengine/websocket - node nodejsChecker.js - ``` + ```bash + npm init -y + npm install @tdengine/websocket + node nodejsChecker.js + ``` - 执行以上步骤后，在命令行会输出 nodeChecker.js 连接 TDengine 实例，并执行简单插入和查询的结果。 diff --git a/docs/zh/08-develop/04-schemaless.md b/docs/zh/08-develop/04-schemaless.md index 130d012e8b..17a377950c 100644 --- a/docs/zh/08-develop/04-schemaless.md +++ b/docs/zh/08-develop/04-schemaless.md @@ -13,7 +13,7 @@ import TabItem from "@theme/TabItem"; 值得注意的是，通过无模式写入方式创建的超级表及其对应的子表与通过 SQL 直接创建的超级表和子表在功能上没有区别，用户仍然可以使用 SQL 直接向其中写入数据。然而，由于无模式写入方式生成的表名是基于标签值按照固定的映射规则生成的，因此这些表名可能缺乏可读性，不易于理解。 -**采用无模式写入方式时会自动创建表，无须手动创建表。** +**采用无模式写入方式时会自动创建表，无须手动创建表。手动建表的话可能会出现未知的错误。** ## 无模式写入行协议 @@ -30,6 +30,7 @@ measurement,tag_set field_set timestamp - tag_set 格式形如 `=, =`，表示标签列数据，使用英文逗号分隔，与 field_set 之间使用一个半角空格分隔。 - field_set 格式形如 `=, =`，表示普通列，同样使用英文逗号来分隔，与 timestamp 之间使用一个半角空格分隔。 - timestamp 为本行数据对应的主键时间戳。 +- 无模式写入不支持含第二主键列的表的数据写入。 tag_set 中的所有的数据自动转化为 nchar 数据类型，并不需要使用双引号。 在无模式写入数据行协议中，field_set 中的每个数据项都需要对自身的数据类型进行描述，具体要求如下。 @@ -81,7 +82,38 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 需要注意的是，如果描述数据类型后缀时出现大小写错误，或者为数据指定的数据类型有误，均可能引发报错提示而导致数据写入失败。 -TDengine提供数据写入的幂等性保证，即用户可以反复调用API进行出错数据的写入操作。无模式写入TDengine的主要处理逻辑请参考TDengine的官方网站，此处不赘述。 +TDengine 提供数据写入的幂等性保证，即您可以反复调用 API 进行出错数据的写入操作。但是不提供多行数据写入的原子性保证。即在多行数据一批次写入过程中，会出现部分数据写入成功，部分数据写入失败的情况。 + +## 无模式写入处理规则 + +无模式写入按照如下原则来处理行数据： + +1. 将使用如下规则来生成子表名：首先将 measurement 的名称和标签的 key 和 value 组合成为如下的字符串 + + ```json + "measurement,tag_key1=tag_value1,tag_key2=tag_value2" + ``` + + - 需要注意的是，这里的 tag_key1, tag_key2 并不是用户输入的标签的原始顺序，而是使用了标签名称按照字符串升序排列后的结果。所以，tag_key1 并不是在行协议中输入的第一个标签。 + 排列完成以后计算该字符串的 MD5 散列值 "md5_val"。然后将计算的结果与字符串组合生成表名：“t_md5_val”。其中的 “t_” 是固定的前缀，每个通过该映射关系自动生成的表都具有该前缀。 + + - 如果不想用自动生成的表名，有两种指定子表名的方式(第一种优先级更高)。 + 1. 通过在taos.cfg里配置 smlAutoChildTableNameDelimiter 参数来指定（`@ # 空格 回车 换行 制表符`除外)。 + 1. 举例如下：配置 smlAutoChildTableNameDelimiter=- 插入数据为 st,t0=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1-4。 + 2. 通过在taos.cfg里配置 smlChildTableName 参数来指定。 + 1. 举例如下：配置 smlChildTableName=tname 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略。 + +2. 如果解析行协议获得的超级表不存在，则会创建这个超级表（不建议手动创建超级表，不然插入数据可能异常）。 +3. 如果解析行协议获得子表不存在，则 Schemaless 会按照步骤 1 或 2 确定的子表名来创建子表。 +4. 如果数据行中指定的标签列或普通列不存在，则在超级表中增加对应的标签列或普通列（只增不减）。 +5. 如果超级表中存在一些标签列或普通列未在一个数据行中被指定取值，那么这些列的值在这一行中会被置为 NULL。 +6. 对 BINARY 或 NCHAR 列，如果数据行中所提供值的长度超出了列类型的限制，自动增加该列允许存储的字符长度上限（只增不减），以保证数据的完整保存。 +7. 整个处理过程中遇到的错误会中断写入过程，并返回错误代码。 +8. 为了提高写入的效率，默认假设同一个超级表中 field_set 的顺序是一样的（第一条数据包含所有的 field，后面的数据按照这个顺序），如果顺序不一样，需要配置参数 smlDataFormat 为 false，否则，数据写入按照相同顺序写入，库中数据会异常，从3.0.3.0开始，自动检测顺序是否一致，该配置废弃。 +9. 由于sql建表表名不支持点号（.），所以schemaless也对点号（.）做了处理，如果schemaless自动建表的表名如果有点号（.），会自动替换为下划线（\_）。如果手动指定子表名的话，子表名里有点号（.），同样转化为下划线（\_）。 +10. taos.cfg 增加 smlTsDefaultName 配置（值为字符串），只在client端起作用，配置后，schemaless自动建表的时间列名字可以通过该配置设置。不配置的话，默认为 _ts。 +11. 无模式写入的数据超级表或子表名区分大小写。 +12. 无模式写入仍然遵循 TDengine 对数据结构的底层限制，例如每行数据的总长度不能超过 48KB（从 3.0.5.0 版本开始为 64KB），标签值的总长度不超过16KB。 ## 时间分辨率识别 diff --git a/docs/zh/14-reference/01-components/01-taosd.md b/docs/zh/14-reference/01-components/01-taosd.md index 004116b478..ff6b27092d 100644 --- a/docs/zh/14-reference/01-components/01-taosd.md +++ b/docs/zh/14-reference/01-components/01-taosd.md @@ -154,6 +154,12 @@ charset 的有效值是 UTF-8。 | :-----------: | :-------------------------------------------------------------------------: | | supportVnodes | dnode 支持的最大 vnode 数目，取值范围：0-4096，缺省值： CPU 核数的 2 倍 + 5 | +### 内存相关 +| 参数名称 | 参数说明 | +| :----------------: | :---------------------------------------------: | +| rpcQueueMemoryAllowed | 一个 dnode 允许的 rpc 消息占用的内存最大值，单位 bytes，取值范围：10485760-INT64_MAX，缺省值：服务器内存的 1/10 | +| syncLogBufferMemoryAllowed | 一个 dnode 允许的 sync 日志缓存消息占用的内存最大值，单位 bytes，取值范围：10485760-INT64_MAX，缺省值：服务器内存的 1/10，3.1.3.2/3.3.2.13 版本开始生效 | + ### 性能调优 | 参数名称 | 参数说明 | diff --git a/docs/zh/14-reference/01-components/02-taosc.md b/docs/zh/14-reference/01-components/02-taosc.md old mode 100644 new mode 100755 index dfcdf22aac..32baac9a3b --- a/docs/zh/14-reference/01-components/02-taosc.md +++ b/docs/zh/14-reference/01-components/02-taosc.md @@ -33,12 +33,13 @@ TDengine 客户端驱动提供了应用编程所需要的全部 API，并且在 |logKeepDays | 日志文件的最长保存时间; 缺省值: 0，表示无限保存; 大于 0 时，日志文件会被重命名为 taosdlog.xxx，其中 xxx 为日志文件最后修改的时间戳| |smlChildTableName | schemaless 自定义的子表名的 key, 无缺省值 | |smlAutoChildTableNameDelimiter | schemaless tag之间的连接符，连起来作为子表名，无缺省值 | -|smlTagName | schemaless tag 为空时默认的 tag 名字, 无缺省值 | +|smlTagName | schemaless tag 为空时默认的 tag 名字, 缺省值 "_tag_null" | |smlTsDefaultName | schemaless自动建表的时间列名字通过该配置设置, 缺省值 "_ts" | |enableCoreFile | crash 时是否生成 core 文件，0: 不生成， 1： 生成；缺省值：1 | |enableScience | 是否开启科学计数法显示浮点数; 0: 不开始, 1: 开启；缺省值：1 | |compressMsgSize | 是否对 RPC 消息进行压缩; -1: 所有消息都不压缩; 0: 所有消息都压缩; N (N>0): 只有大于 N 个字节的消息才压缩; 缺省值 -1| +|queryTableNotExistAsEmpty | 查询表不存在时是否返回空结果集; false: 返回错误; true: 返回空结果集; 缺省值 false| ## API -请参考[连接器](../../connector) \ No newline at end of file +请参考[连接器](../../connector) diff --git a/docs/zh/14-reference/03-taos-sql/10-function.md b/docs/zh/14-reference/03-taos-sql/10-function.md index ee71abbdec..007e1dd64a 100644 --- a/docs/zh/14-reference/03-taos-sql/10-function.md +++ b/docs/zh/14-reference/03-taos-sql/10-function.md @@ -169,14 +169,43 @@ POW(expr1, expr2) #### ROUND - ```sql -ROUND(expr) +ROUND(expr[, digits]) ``` **功能说明**：获得指定字段的四舍五入的结果。 - 其他使用说明参见 CEIL 函数描述。 +**返回结果类型**：与指定字段的原始数据类型一致。 + +**适用数据类型**： +- `expr`：数值类型。 +- `digits`：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 若 `expr` 或 `digits` 为 NULL，返回 NULL。 +- 若指定了`digits`，则会保留 `digits` 位小数，默认为 0。 +- 若输入值是 INTEGER 类型, 无论 `digits` 值为多少，都只会返回 INTEGER 类型，不会保留小数。 +- `digits` 大于零表示对小数位进行操作，四舍五入到 `digits` 位小数。若小数位数小于 `digits` 位，不进行四舍五入操作，直接返回。 +- `digits` 小于零表示丢掉小数位，并将数字四舍五入到小数点左侧 `digits` 位。若小数点左侧的位数小于 `digits`位，返回 0。 +- 由于暂未支持 DECIMAL 类型，所以该函数会用 DOUBLE 和 FLOAT 来表示包含小数的结果，但是 DOUBLE 和 FLOAT 是有精度上限的，当位数太多时使用该函数可能没有意义。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select round(8888.88); + round(8888.88) | +============================ + 8889.000000000000000 | + +taos> select round(8888.88,-1); + round(8888.88,-1) | +============================ + 8890.000000000000000 | +``` #### SIN @@ -232,6 +261,297 @@ TAN(expr) **使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 +#### PI +```sql +PI() +``` + +**功能说明**：返回圆周率 π 的值。 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：无。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- π ≈ 3.141592653589793。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select pi(); + pi() | +============================ + 3.141592653589793 | +``` + +##### TRUNCATE +```sql +TRUNCATE(expr, digits) +``` + +**功能说明**：获得指定字段按照指定位数截断的值。 + +**返回结果类型**：与 `expr` 字段的原始数据类型一致。 + +**适用数据类型**： +- `expr`：数值类型。 +- `digits`：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 若 `expr` 或 `digits` 为 NULL，返回 NULL。 +- 截取指按保留位数直接进行截取，没有四舍五入。 +- `digits` 大于零表示对小数位进行操作，截取到 `digits` 位小数，若小数位数小于 `digits` 位，不进行截取操作，直接返回。 +- `digits` 等于零表示丢掉小数位。 +- `digits` 小于零表示丢掉小数位，并将小数点左侧 `digits` 位置 `0`。若小数点左侧的位数小于 `digits`位，返回 0。 +- 由于暂未支持 DECIMAL 类型，所以该函数会用 DOUBLE 和 FLOAT 来表示包含小数的结果，但是 DOUBLE 和 FLOAT 是有精度上限的，当位数太多时使用该函数可能没有意义。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select truncate(8888.88, 0); + truncate(8888.88, 0) | +============================ + 8888.000000000000000 | + +taos> select truncate(8888.88, -1); + truncate(8888.88, -1) | +============================ + 8880.000000000000000 | +``` + +#### EXP +```sql +EXP(expr) +``` +**功能说明**：返回 e（自然对数的底）的指定乘方后的值。 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 如果 `expr` 为 NULL，返回 NULL。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select exp(2); + exp(2) | +============================ + 7.389056098930650 | +``` + +#### LN +```sql +LN(expr) +``` + +**功能说明**：返回指定参数的自然对数。 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 如果 `expr` 为 NULL，返回 NULL。 +- 如果 `epxr` 小于等于 0，返回 NULL。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select ln(10); + ln(10) | +============================ + 2.302585092994046 | +``` + +#### MOD +```sql +MOD(expr1, expr2) +``` + +**功能说明**：计算 expr1 % expr2 的结果。 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 如果 `expr2` 为 0 则返回 NULL。 +- 如果 `expr1` 或 `expr2` 为 NULL，返回 NULL。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +``` sql +taos> select mod(10,3); + mod(10,3) | +============================ + 1.000000000000000 | + +taos> select mod(1,0); + mod(1,0) | +============================ + NULL | +``` + +#### RAND +```sql +RAND([seed]) +``` + +**功能说明**：返回一个从0到1均匀分布的随机数。 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**： +- `seed`：INTEGER。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 如果指定了 `seed` 值，那么将会用指定的 `seed` 作为随机种子，确保生成的随机数序列具有确定性。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +``` sql +taos> select rand(); + rand() | +============================ + 0.202092426923147 | + +taos> select rand(); + rand() | +============================ + 0.131537788143166 | + +taos> select rand(1); + rand(1) | +============================ + 0.000007826369259 | + +taos> select rand(1); + rand(1) | +============================ + 0.000007826369259 | +``` + +#### SIGN +```sql +SIGN(expr) +``` + +**功能说明**：返回指定参数的符号。 + +**返回结果类型**：与指定字段的原始数据类型一致。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 如果 `expr` 为负，返回 -1 ， +- 如果 `expr` 为正，返回 1 ， +- 如果 `expr` 为 0 ，返回 0 。 +- 如果 `expr` 为 NULL ，返回 NULL 。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select sign(-1); + sign(-1) | +======================== + -1 | + +taos> select sign(1); + sign(1) | +======================== + 1 | + +taos> select sign(0); + sign(0) | +======================== + 0 | +``` + +#### DEGREES +```sql +DEGREES(expr) +``` + +**功能说明**：计算指定参数由弧度值转为角度后的值。 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 如果 `expr` 为 NULL，则返回 NULL。 +- degree = radian * 180 / π。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select degrees(PI()); + degrees(pi()) | +============================ + 180.000000000000000 | +``` + +#### RADIANS +```sql +RADIANS(expr) +``` + +**功能说明**：计算指定参数由角度值转为弧度后的值。 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 如果 `expr` 为 NULL，则返回 NULL。 +- radian = degree * π / 180。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select radians(180); + radians(180) | +============================ + 3.141592653589793 | +``` ### 字符串函数 字符串函数的输入参数为字符串类型，返回结果为数值类型或字符串类型。 @@ -252,6 +572,27 @@ CHAR_LENGTH(expr) **适用于**: 表和超级表。 +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 与 `LENGTH()` 函数不同在于，对于多字节字符，比如中文字符， `CHAR_LENGTH()` 函数会将其算做一个字符，长度为 1，而 `LENGTH()` 会计算其字节数，长度为 3。比如 `CHAR_LENGTH('你好') = 2`， `LENGTH('你好') = 6`。 +- 如果 `expr` 为 NULL，返回 NULL。 + +**举例**： +```sql +taos> select char_length('Hello world'); + char_length('Hello world') | +============================= + 11 | + +taos> select char_length('你好 世界'); + char_length('你好 世界') | +=============================== + 5 | +``` + #### CONCAT ```sql @@ -353,23 +694,150 @@ RTRIM(expr) **适用于**: 表和超级表。 - -#### SUBSTR - +#### TRIM ```sql -SUBSTR(expr, pos [,len]) +TRIM([{LEADING | TRAILING | BOTH} [remstr] FROM] expr) +TRIM([remstr FROM] expr) ``` -**功能说明**：从源字符串 str 中的指定位置 pos 开始取一个长度为 len 的子串并返回。如果输入参数 len 被忽略，返回的子串包含从 pos 开始的整个字串。 +**功能说明**：返回去掉了所有 remstr 前缀或后缀的字符串 epxr 。 -**返回结果类型**：与输入字段的原始类型相同。 +**返回结果类型**：与输入字段 epxr 的原始类型相同。 -**适用数据类型**：VARCHAR, NCHAR。输入参数 pos 可以为正数，也可以为负数。如果 pos 是正数，表示开始位置从字符串开头正数计算。如果 pos 为负数，表示开始位置从字符串结尾倒数计算。 +**适用数据类型**： +- remstr：VARCHAR,NCHAR。 +- epxr：VARCHAR,NCHAR。 **嵌套子查询支持**：适用于内层查询和外层查询。 **适用于**: 表和超级表。 +**使用说明**： +- 第一个可选变量[LEADING | BOTH | TRAILING]指定要剪裁字符串的哪一侧： + - LEADING 将移除字符串开头的指定字符。 + - TRAILING 将移除字符串末尾的指定字符。 + - BOTH（默认值）将移除字符串开头和末尾的指定字符。 +- 第二个可选变量[remstr]指定要裁剪掉的字符串： + - 如果不指定 remstr ，默认裁剪空格。 + - remstr 可以指定多个字符，如trim('ab' from 'abacd') ，此时会将 'ab' 看做一个整体来裁剪，得到裁剪结果 'acd'。 +- 若 expr 为 NULL, 返回 NULL。 +- 该函数是多字节安全的。 + +**举例**： +```sql +taos> select trim(' a '); + trim(' a ') | +============================= + a | + +taos> select trim(leading from ' a '); + trim(leading from ' a ') | +========================================== + a | + + +taos> select trim(leading 'b' from 'bbbbbbbba '); + trim(leading 'b' from 'bbbbbbbba ') | +============================================== + a | + +taos> select trim(both 'b' from 'bbbbbabbbbbb'); + trim(both 'b' from 'bbbbbabbbbbb') | +===================================== + a | +``` + +#### SUBSTRING/SUBSTR +```sql +SUBSTRING/SUBSTR(expr, pos [, len]) +SUBSTRING/SUBSTR(expr FROM pos [FOR len]) +``` +**功能说明**：返回字符串 `expr` 在 `pos` 位置开始的子串，若指定了 `len` ，则返回在 `pos` 位置开始，长度为 `len` 的子串。 + +**返回结果类型**：与输入字段 `expr` 的原始类型相同。 + +**适用数据类型**： +- `expr`：VARCHAR,NCHAR。 +- `pos`：整数类型。 +- `len`：整数类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 若 `pos` 为正数，则返回的结果为 `expr` 从左到右开始数 `pos` 位置开始的右侧的子串。 +- 若 `pos` 为负数，则返回的结果为 `expr` 从右到左开始数 `pos` 位置开始的右侧的子串。 +- 任意参数为 NULL，返回 NULL。 +- 该函数是多字节安全的。 +- 若 `len` 小于 1，返回空串。 +- `pos` 是 1-base 的，若 `pos` 为 0，返回空串。 +- 若 `pos` + `len` 大于 `len(expr)`，返回从 `pos` 开始到字符串结尾的子串，等同于执行 `substring(expr, pos)`。 + +**举例**： +```sql +taos> select substring('tdengine', 0); + substring('tdengine', 0) | +=========================== + | + +taos> select substring('tdengine', 3); + substring('tdengine', 3) | +=========================== + engine | + +taos> select substring('tdengine', 3,3); + substring('tdengine', 3,3) | +============================= + eng | + +taos> select substring('tdengine', -3,3); + substring('tdengine', -3,3) | +============================== + ine | + +taos> select substring('tdengine', -3,-3); + substring('tdengine', -3,-3) | +=============================== + | +``` + +#### SUBSTRING_INDEX +```sql +SUBSTRING_INDEX(expr, delim, count) +``` + +**功能说明**：返回字符串 `expr` 在出现指定次数分隔符的位置截取的子串。 + +**返回结果类型**：与输入字段 `expr` 的原始类型相同。 + +**适用数据类型**： +- `expr`：VARCHAR,NCHAR。 +- `delim`：VARCHAR, NCHAR。 +- `count`：INTEGER。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 若 `count` 为正数，则返回的结果为 `expr` 从左到右开始数第 `count` 次 出现 `delim` 的位置左侧的字符串。 +- 若 `count` 为负数，则返回的结果为 `expr` 从右到左开始数第 `count` 的绝对值次 出现 `delim` 的位置右侧的字符串。 +- 任意参数为 NULL，返回 NULL。 +- 该函数是多字节安全的。 + +**举例**： +```sql +taos> select substring_index('www.taosdata.com','.',2); + substring_index('www.taosdata.com','.',2) | +============================================ + www.taosdata | + +taos> select substring_index('www.taosdata.com','.',-2); + substring_index('www.taosdata.com','.',-2) | +============================================= + taosdata.com | +``` #### UPPER @@ -387,7 +855,182 @@ UPPER(expr) **适用于**: 表和超级表。 +#### CHAR +```sql +CHAR(expr1 [, expr2] [, epxr3] ...) +``` +**功能说明**：将输入参数当作整数，并返回这些整数在 ASCII 编码中对应的字符。 + +**返回结果类型**：VARCHAR。 + +**适用数据类型**：整数类型,VARCHAR,NCHAR。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 输入的值超过 255 会被转化成多字节的结果，如 `CHAR(256)` 等同于 `CHAR(1,0)`, `CHAR(256 * 256)` 等同于 `CHAR(1,0,0)`。 +- 输入参数的 NULL 值会被跳过。 +- 输入参数若为字符串类型，会将其转换为数值类型处理。 +- 若输入的参数对应的字符为不可打印字符，返回值中仍有该参数对应的字符，但是可能无法显示出来。 + +**举例**： +```sql +taos> select char(77); + char(77) | +=========== + M | + +taos> select char(77,77); + char(77,77) | +============== + MM | + +taos> select char(77 * 256 + 77); + char(77 * 256 + 77) | +====================== + MM | + +taos> select char(77,NULL,77); + char(77,null,77) | +=================== + MM | +``` + +#### ASCII +```sql +ASCII(expr) +``` + +**功能说明**：返回字符串第一个字符的 ASCII 码。 + +**返回结果数据类型**：BIGINT。 + +**适用数据类型**：VARCHAR, NCHAR。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 如果 `expr` 为 NULL，返回 NULL。 +- 如果 `expr` 的第一个字符为多字节字符，只会返回该字符第一个字节的值对应的 ASCII 码。 + +**举例**： +```sql +taos> select ascii('testascii'); + ascii('testascii') | +===================== + 116 | +``` + +#### POSITION +```sql +POSITION(expr1 IN expr2) +``` + +**功能说明**：计算字符串 `expr1` 在字符串 `expr2` 中的位置。 + +**返回结果类型**：BIGINT。 + +**适用数据类型**： +- `expr1`：VARCHAR, NCHAR。 +- `expr2`：VARCHAR, NCHAR。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 若 `expr1` 或 `expr2` 为 NULL，返回 NULL。 +- 若 `expr1` 在 `expr2` 中不存在，返回 0。 +- 若 `expr1` 为空串，认为 `expr1` 在 `expr2` 中总能匹配成功，返回 1。 +- 返回的位置是 1-base 的。 +- 该函数是多字节安全的。 + +**举例**： +```sql +taos> select position('a' in 'cba'); + position('a' in 'cba') | +========================= + 3 | + + +taos> select position('' in 'cba'); + position('' in 'cba') | +======================== + 1 | + +taos> select position('d' in 'cba'); + position('d' in 'cba') | +========================= + 0 | +``` + +#### REPLACE +```sql +REPLACE(expr, from_str, to_str) +``` +**功能说明**：将字符串中的 `from_str` 全部替换为 `to_str`。 + +**返回结果类型**：与输入字段 `expr` 的原始类型相同。 + +**适用数据类型**： +- `expr`：VARCHAR, NCHAR。 +- `from_str`：VARCHAR, NCHAR。 +- `to_str`：VARCHAR, NCHAR。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 该函数是大小写敏感的。 +- 任意参数为 NULL，返回 NULL。 +- 该函数是多字节安全的。 + +**举例**： +```sql +taos> select replace('aabbccAABBCC', 'AA', 'DD'); + replace('aabbccAABBCC', 'AA', 'DD') | +====================================== + aabbccDDBBCC | +``` + +#### REPEAT +```sql +REPEAT(expr, count) +``` +**功能说明**：返回将字符串重复指定次数得到的字符串。 + +**返回结果类型**：与输入字段 `expr` 的原始类型相同。 + +**适用数据类型**： +- `expr`： VARCHAR,NCHAR。 +- `count`：INTEGER。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 若 `count < 1`，返回空串。 +- 若 `expr` 或 `count` 为 NULL，返回 NULL。 + +**举例**： +```sql +taos> select repeat('abc',5); + repeat('abc',5) | +============================ + abcabcabcabcabc | + +taos> select repeat('abc',-1); + repeat('abc',-1) | +=================== + | +``` ### 转换函数 转换函数将值从一种数据类型转换为另一种数据类型。 @@ -609,22 +1252,39 @@ NOW() TIMEDIFF(expr1, expr2 [, time_unit]) ``` -**功能说明**：计算两个时间戳之间的差值，并近似到时间单位 time_unit 指定的精度。 +**功能说明**：返回时间戳 `expr1` - `expr2` 的结果，结果可能为负，并近似到时间单位 `time_unit` 指定的精度。 -**返回结果数据类型**：BIGINT。 +**返回结果类型**：BIGINT。 -**应用字段**：表示 UNIX 时间戳的 BIGINT, TIMESTAMP 类型，或符合日期时间格式的 VARCHAR, NCHAR 类型。 - -**适用于**：表和超级表。 +**适用数据类型**： +- `expr1`：表示 UNIX 时间戳的 BIGINT, TIMESTAMP 类型，或符合日期时间格式的 VARCHAR, NCHAR 类型。 +- `expr2`：表示 UNIX 时间戳的 BIGINT, TIMESTAMP 类型，或符合日期时间格式的 VARCHAR, NCHAR 类型。 +- `time_unit`：见使用说明。 **嵌套子查询支持**：适用于内层查询和外层查询。 -**使用说明**： -- 支持的时间单位 time_unit 如下： - 1b(纳秒), 1u(微秒)，1a(毫秒)，1s(秒)，1m(分)，1h(小时)，1d(天), 1w(周)。 -- 如果时间单位 time_unit 未指定， 返回的时间差值精度与当前 DATABASE 设置的时间精度一致。 -- 输入包含不符合时间日期格式的字符串则返回 NULL。 +**适用于**: 表和超级表。 +**使用说明**： +- 支持的时间单位 `time_unit` 如下： 1b(纳秒), 1u(微秒)，1a(毫秒)，1s(秒)，1m(分)，1h(小时)，1d(天), 1w(周)。 +- 如果时间单位 `time_unit` 未指定， 返回的时间差值精度与当前 DATABASE 设置的时间精度一致。 +- 输入包含不符合时间日期格式的字符串则返回 NULL。 +- `expr1` 或 `expr2` 为 NULL，返回 NULL。 +- `time_unit` 为 NULL，等同于未指定时间单位。 +- 输入时间戳的精度由所查询表的精度确定, 若未指定表, 则精度为毫秒. + +**举例**： +```sql +taos> select timediff('2022-01-01 08:00:00', '2022-01-01 08:00:01',1s); + timediff('2022-01-01 08:00:00', '2022-01-01 08:00:01',1s) | +============================================================ + -1 | + +taos> select timediff('2022-01-01 08:00:01', '2022-01-01 08:00:00',1s); + timediff('2022-01-01 08:00:01', '2022-01-01 08:00:00',1s) | +============================================================ + 1 | +``` #### TIMETRUNCATE @@ -695,6 +1355,148 @@ TODAY() b(纳秒)，u(微秒)，a(毫秒)，s(秒)，m(分)，h(小时)，d(天)，w(周)。 - 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 +#### WEEK +```sql +WEEK(expr [, mode]) +``` +**功能说明**：返回输入日期的周数。 + +**返回结果类型**：BIGINT。 + +**适用数据类型**： +- `expr`：表示 UNIX 时间戳的 BIGINT, TIMESTAMP 类型，或符合日期时间格式的 VARCHAR, NCHAR 类型。 +- `mode`：0 - 7 之间的整数。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 若 `expr` 为 NULL，返回 NULL。 +- 输入时间戳的精度由所查询表的精度确定, 若未指定表, 则精度为毫秒. +- `mode` 用来指定一周是从周日开始还是周一开始，以及指定返回值范围是 1 - 53 还是 0 - 53。下表详细描述不同的 mode 对应的计算方法： + +| Mode | 每周的第一天 | 返回值范围 | 第 1 周的计算方法 | +| ---- | ------ | ------ | ------------------ | +| 0 | 周日 | 0 - 53 | 第一个包含周日的周为第 1 周 | +| 1 | 周一 | 0 - 53 | 第一个包含四天及以上的周为第 1 周 | +| 2 | 周日 | 1 - 53 | 第一个包含周日的周为第 1 周 | +| 3 | 周一 | 1 - 53 | 第一个包含四天及以上的周为第 1 周 | +| 4 | 周日 | 0 - 53 | 第一个包含四天及以上的周为第 1 周 | +| 5 | 周一 | 0 - 53 | 第一个包含周一的周为第 1 周 | +| 6 | 周日 | 1 - 53 | 第一个包含四天及以上的周为第 1 周 | +| 7 | 周一 | 1 - 53 | 第一个包含周一的周为第 1 周 | +- 当返回值范围为0 - 53时，第 1 周之前的日期为第 0 周。 +- 当返回值范围为1 - 53时，第 1 周之前的日期为上一年的最后一周。 +- 以`2000-01-01`为例， + - 在 `mode=0`时返回值为`0`，因为该年第一个周日为`2000-01-02`，从`2000-01-02`起才是第 1 周，所以 `2000-01-01`为第 0 周，返回 0。 + - 在 `mode=1`时返回值为`0`，因为`2000-01-01`所在的周只有两天，分别是 `2000-01-01(周六)`和`2000-01-02(周日)`，所以`2000-01-03`起才是第一周，`2000-01-01`为第 0 周，返回 0。 + - 在 `mode=2`时返回值为`52`，因为从`2000-01-02`起才是第 1 周，并且返回值范围为 1-53，所以`2000-01-01`算做上一年的最后一周，即 1999 年的第 52 周，返回 52。 + +**举例**： +```sql +taos> select week('2000-01-01',0); + week('2000-01-01',0) | +======================== + 0 | + +taos> select week('2000-01-01',1); + week('2000-01-01',1) | +======================== + 0 | + +taos> select week('2000-01-01',2); + week('2000-01-01',2) | +======================== + 52 | + +taos> select week('2000-01-01',3); + week('2000-01-01',3) | +======================== + 52 | +``` + +#### WEEKOFYEAR +```sql +WEEKOFYEAR(expr) +``` +**功能说明**：返回输入日期的周数。 + +**返回结果类型**：BIGINT。 + +**适用数据类型**：表示 UNIX 时间戳的 BIGINT, TIMESTAMP 类型，或符合日期时间格式的 VARCHAR, NCHAR 类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 等同于`WEEK(expr, 3)`，即在每周第一天是周一，返回值范围为 1 - 53，第一个包含四天及以上的周为第 1 周的条件下判断输入日期的周数。 +- 若 `expr` 为 NULL，返回 NULL。 +- 输入时间戳的精度由所查询表的精度确定, 若未指定表, 则精度为毫秒. + +**举例**： +```sql +taos> select weekofyear('2000-01-01'); + weekofyear('2000-01-01') | +=========================== + 52 | +``` + +#### WEEKDAY +```sql +WEEKDAY(expr) +``` +**功能说明**：返回输入日期是周几。 + +**返回结果类型**：BIGINT。 + +**适用数据类型**：表示 UNIX 时间戳的 BIGINT, TIMESTAMP 类型，或符合日期时间格式的 VARCHAR, NCHAR 类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 返回值 0 代表周日，1 代表周一 ... 6 代表周六。 +- 若 `expr` 为 NULL，返回 NULL。 +- 输入时间戳的精度由所查询表的精度确定, 若未指定表, 则精度为毫秒. + +**举例**： +```sql +taos> select weekday('2000-01-01'); + weekday('2000-01-01') | +======================== + 5 | +``` + +#### DAYOFWEEK +```sql +DAYOFWEEK(expr) +``` +**功能说明**：返回输入日期是周几。 + +**返回结果类型**：BIGINT。 + +**适用数据类型**：表示 UNIX 时间戳的 BIGINT, TIMESTAMP 类型，或符合日期时间格式的 VARCHAR, NCHAR 类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**: 表和超级表。 + +**使用说明**： +- 返回值 1 代表周日，2 代表周一 ... 7 代表周六 +- 若 `expr` 为 NULL，返回 NULL。 +- 输入时间戳的精度由所查询表的精度确定, 若未指定表, 则精度为毫秒. + +**举例**： +```sql +taos> select dayofweek('2000-01-01'); + dayofweek('2000-01-01') | +========================== + 7 | +``` + ## 聚合函数 @@ -817,13 +1619,13 @@ SPREAD(expr) **适用于**：表和超级表。 -### STDDEV +### STDDEV/STDDEV_POP ```sql -STDDEV(expr) +STDDEV/STDDEV_POP(expr) ``` -**功能说明**：统计表中某列的均方差。 +**功能说明**：统计表中某列的总体标准差。 **返回数据类型**：DOUBLE。 @@ -831,7 +1633,51 @@ STDDEV(expr) **适用于**：表和超级表。 +**举例**： +```sql +taos> select id from test_stddev; + id | +============== + 1 | + 2 | + 3 | + 4 | + 5 | +taos> select stddev_pop(id) from test_stddev; + stddev_pop(id) | +============================ + 1.414213562373095 | +``` +### VAR_POP +```sql +VAR_POP(expr) +``` + +**功能说明**：统计表中某列的总体方差。 + +**返回数据类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**适用于**：表和超级表。 + +**举例**: +```sql +taos> select id from test_var; + id | +============== + 3 | + 1 | + 2 | + 4 | + 5 | + +taos> select var_pop(id) from test_var; + var_pop(id) | +============================ + 2.000000000000000 | +``` ### SUM ```sql @@ -1045,10 +1891,11 @@ MAX(expr) **返回数据类型**：同应用的字段。 -**适用数据类型**：数值类型。 +**适用数据类型**：数值类型, VARCHAR，NCHAR。 **适用于**：表和超级表。 +**使用说明**：max 函数可以接受字符串作为输入参数，当输入参数为字符串类型时，返回最大的字符串值。 ### MIN @@ -1060,10 +1907,11 @@ MIN(expr) **返回数据类型**：同应用的字段。 -**适用数据类型**：数值类型。 +**适用数据类型**：数值类型, VARCHAR，NCHAR。 **适用于**：表和超级表。 +**使用说明**：min 函数可以接受字符串作为输入参数，当输入参数为字符串类型时，返回最大的字符串值。 ### MODE diff --git a/docs/zh/14-reference/03-taos-sql/22-meta.md b/docs/zh/14-reference/03-taos-sql/22-meta.md index 35b71feb2c..397b3d3362 100644 --- a/docs/zh/14-reference/03-taos-sql/22-meta.md +++ b/docs/zh/14-reference/03-taos-sql/22-meta.md @@ -93,8 +93,8 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数 | 4 | vgroups | INT | 数据库中有多少个 vgroup。需要注意，`vgroups` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | | 6 | replica | INT | 副本数。需要注意，`replica` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | | 7 | strict | VARCHAR(4) | 废弃参数 | -| 8 | duration | VARCHAR(10) | 单文件存储数据的时间跨度。需要注意，`duration` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 9 | keep | VARCHAR(32) | 数据保留时长。需要注意，`keep` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 8 | duration | VARCHAR(10) | 单文件存储数据的时间跨度。需要注意，`duration` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。内部存储单位为分钟，查询时有可能转换为天或小时展示 | +| 9 | keep | VARCHAR(32) | 数据保留时长。需要注意，`keep` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 内部存储单位为分钟，查询时有可能转换为天或小时展示 | | 10 | buffer | INT | 每个 vnode 写缓存的内存块大小，单位 MB。需要注意，`buffer` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | | 11 | pagesize | INT | 每个 VNODE 中元数据存储引擎的页大小，单位为 KB。需要注意，`pagesize` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | | 12 | pages | INT | 每个 vnode 元数据存储引擎的缓存页个数。需要注意，`pages` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | diff --git a/docs/zh/14-reference/05-connector/60-rest-api.mdx b/docs/zh/14-reference/05-connector/60-rest-api.mdx index a804229d27..de0e7118d4 100644 --- a/docs/zh/14-reference/05-connector/60-rest-api.mdx +++ b/docs/zh/14-reference/05-connector/60-rest-api.mdx @@ -118,13 +118,14 @@ curl -L -u username:password -d "" :/rest/sql/[db_name][?tz=timez ### HTTP 响应码 -从 `TDengine 3.0.3.0` 开始 `taosAdapter` 提供配置参数 `httpCodeServerError` 用来设置当 C 接口返回错误时是否返回非 200 的http状态码 +从 `TDengine 3.0.3.0` 开始 `taosAdapter` 提供配置参数 `httpCodeServerError` 用来设置当 C 接口返回错误时是否返回非 200 的http状态码。 +无论是否设置此参数，响应 body 里都有详细的错误码和错误信息，具体请参考 [错误](../rest-api/#错误) 。 | **说明** | **httpCodeServerError false** | **httpCodeServerError true** | |--------------------|-------------------------------|---------------------------------------| | taos_errno() 返回 0 | 200 | 200 | -| taos_errno() 返回 非0 | 200（除鉴权错误） | 500 （除鉴权错误和 400/502 错误） | -| 参数错误 | 400 (仅处理 HTTP 请求 URL 参数错误) | 400 （处理 HTTP 请求 URL 参数错误和 taosd 返回错误） | +| taos_errno() 返回 非0 | 200（除鉴权错误） | 500 （除鉴权错误和 400/502/503 错误） | +| 参数错误 | 400（仅处理 HTTP 请求 URL 参数错误） | 400 （处理 HTTP 请求 URL 参数错误和 taosd 返回错误） | | 鉴权错误 | 401 | 401 | | 接口不存在 | 404 | 404 | | 集群不可用错误 | 502 | 502 | @@ -132,27 +133,29 @@ curl -L -u username:password -d "" :/rest/sql/[db_name][?tz=timez 返回 400 的 C 错误码为： -- TSDB_CODE_TSC_SQL_SYNTAX_ERROR ( 0x0216) -- TSDB_CODE_TSC_LINE_SYNTAX_ERROR (0x021B) +- TSDB_CODE_TSC_SQL_SYNTAX_ERROR (0x0216) +- TSDB_CODE_TSC_LINE_SYNTAX_ERROR (0x021B) - TSDB_CODE_PAR_SYNTAX_ERROR (0x2600) - TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE (0x060B) -- TSDB_CODE_TSC_VALUE_OUT_OF_RANGE (0x0224) +- TSDB_CODE_TSC_VALUE_OUT_OF_RANGE (0x0224) - TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE (0x263B) 返回 401 的错误码为： -- TSDB_CODE_MND_USER_ALREADY_EXIST (0x0350) -- TSDB_CODE_MND_USER_NOT_EXIST ( 0x0351) -- TSDB_CODE_MND_INVALID_USER_FORMAT (0x0352) -- TSDB_CODE_MND_INVALID_PASS_FORMAT (0x0353) +- TSDB_CODE_MND_USER_ALREADY_EXIST (0x0350) +- TSDB_CODE_MND_USER_NOT_EXIST (0x0351) +- TSDB_CODE_MND_INVALID_USER_FORMAT (0x0352) +- TSDB_CODE_MND_INVALID_PASS_FORMAT (0x0353) - TSDB_CODE_MND_NO_USER_FROM_CONN (0x0354) - TSDB_CODE_MND_TOO_MANY_USERS (0x0355) - TSDB_CODE_MND_INVALID_ALTER_OPER (0x0356) - TSDB_CODE_MND_AUTH_FAILURE (0x0357) -返回 403 的错误码为： +返回 502 的错误码为： -- TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED (0x0020) +- TSDB_CODE_RPC_NETWORK_UNAVAIL (0x000B) + +错误码和错误描述请参考[错误码](../../../reference/error-code) ### HTTP body 结构 @@ -333,6 +336,8 @@ curl --location 'http://:/rest/sql' \ - code：（`int`）错误码。 - desc：（`string`）错误描述。 +错误码和错误描述请参考[错误码](../../../reference/error-code) + #### 返回key-value形式数据 当指定 url 参数 `row_with_meta=true` 时，返回的 data 中的数据会从数组的形式变成对象的形式，对象的 key 为列名，value 为数据，如下所示： diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 2d4d437649..bece14c17d 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -94,6 +94,7 @@ extern int32_t tsElectInterval; extern int32_t tsHeartbeatInterval; extern int32_t tsHeartbeatTimeout; extern int32_t tsSnapReplMaxWaitN; +extern int64_t tsLogBufferMemoryAllowed; // maximum allowed log buffer size in bytes for each dnode // arbitrator extern int32_t tsArbHeartBeatIntervalSec; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 64cc7bca95..cd78e97e39 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -74,7 +74,7 @@ typedef enum { * @param vgId * @return */ -qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t vgId, int32_t taskId); +int32_t qCreateStreamExecTaskInfo(qTaskInfo_t* pInfo, void* msg, SReadHandle* readers, int32_t vgId, int32_t taskId); /** * Create the exec task for queue mode @@ -93,7 +93,7 @@ int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray **tableList * @param taskId * @param queryId */ -void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId); +int32_t qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId); int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 0cf52faa4f..07cf69171b 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -174,7 +174,7 @@ typedef struct TsdReader { int32_t (*tsdReaderOpen)(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, SSDataBlock* pResBlock, void** ppReader, const char* idstr, SHashObj** pIgnoreTables); void (*tsdReaderClose)(); - void (*tsdSetReaderTaskId)(void *pReader, const char *pId); + int32_t (*tsdSetReaderTaskId)(void *pReader, const char *pId); int32_t (*tsdSetQueryTableList)(); int32_t (*tsdNextDataBlock)(); @@ -419,9 +419,9 @@ typedef struct SStateStore { SStreamStateCur* (*streamStateSessionSeekKeyCurrentPrev)(SStreamState* pState, const SSessionKey* key); SStreamStateCur* (*streamStateSessionSeekKeyCurrentNext)(SStreamState* pState, const SSessionKey* key); - struct SStreamFileState* (*streamFileStateInit)(int64_t memSize, uint32_t keySize, uint32_t rowSize, - uint32_t selectRowSize, GetTsFun fp, void* pFile, TSKEY delMark, - const char* id, int64_t ckId, int8_t type); + int32_t (*streamFileStateInit)(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, + GetTsFun fp, void* pFile, TSKEY delMark, const char* id, int64_t ckId, int8_t type, + struct SStreamFileState** ppFileState); int32_t (*streamStateGroupPut)(SStreamState* pState, int64_t groupId, void* value, int32_t vLen); SStreamStateCur* (*streamStateGroupGetCur)(SStreamState* pState); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index ee8f0450a4..a493478736 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -496,6 +496,14 @@ typedef struct SScanWalInfo { tmr_h scanTimer; } SScanWalInfo; +typedef struct SFatalErrInfo { + int32_t code; + int64_t ts; + int32_t threadId; + int32_t line; + char func[128]; +} SFatalErrInfo; + // meta typedef struct SStreamMeta { char* path; @@ -525,14 +533,13 @@ typedef struct SStreamMeta { int32_t numOfStreamTasks; // this value should be increased when a new task is added into the meta int32_t numOfPausedTasks; int64_t rid; - - int64_t chkpId; - int32_t chkpCap; - SArray* chkpSaved; - SArray* chkpInUse; - SRWLatch chkpDirLock; - void* qHandle; // todo remove it - void* bkdChkptMgt; + SFatalErrInfo fatalInfo; // fatal error occurs, stream stop to execute + int64_t chkpId; + int32_t chkpCap; + SArray* chkpSaved; + SArray* chkpInUse; + SRWLatch chkpDirLock; + void* bkdChkptMgt; } SStreamMeta; typedef struct STaskUpdateEntry { @@ -676,7 +683,7 @@ int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask); int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt); int32_t streamExecTask(SStreamTask* pTask); -void streamResumeTask(SStreamTask* pTask); +int32_t streamResumeTask(SStreamTask* pTask); int32_t streamTrySchedExec(SStreamTask* pTask); int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId, int32_t execType); void streamTaskResumeInFuture(SStreamTask* pTask); @@ -779,6 +786,9 @@ void streamMetaRLock(SStreamMeta* pMeta); void streamMetaRUnLock(SStreamMeta* pMeta); void streamMetaWLock(SStreamMeta* pMeta); void streamMetaWUnLock(SStreamMeta* pMeta); +void streamSetFatalError(SStreamMeta* pMeta, int32_t code, const char* funcName, int32_t lino); +int32_t streamGetFatalError(const SStreamMeta* pMeta); + void streamMetaResetStartInfo(STaskStartInfo* pMeta, int32_t vgId); int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pTaskList); void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader); @@ -794,7 +804,7 @@ void streamTaskSetReqConsenChkptId(SStreamTask* pTask, int64_t ts); // timer int32_t streamTimerGetInstance(tmr_h* pTmr); -void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId, +void streamTmrStart(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* pParam, void* pHandle, tmr_h* pTmrId, int32_t vgId, const char* pMsg); void streamTmrStop(tmr_h tmrId); diff --git a/include/libs/stream/tstreamFileState.h b/include/libs/stream/tstreamFileState.h index 17c01bad94..1497fb4afa 100644 --- a/include/libs/stream/tstreamFileState.h +++ b/include/libs/stream/tstreamFileState.h @@ -47,9 +47,11 @@ typedef int32_t (*range_cmpr_fn)(const SSessionKey* pWin1, const SSessionKey* pW typedef int (*__session_compare_fn_t)(const void* pWin, const void* pDatas, int pos); -SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, - GetTsFun fp, void* pFile, TSKEY delMark, const char* taskId, int64_t checkpointId, - int8_t type); +typedef int (*__session_compare_fn_t)(const void* pWin, const void* pDatas, int pos); + +int32_t streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, GetTsFun fp, + void* pFile, TSKEY delMark, const char* taskId, int64_t checkpointId, int8_t type, + struct SStreamFileState** ppFileState); void streamFileStateDestroy(SStreamFileState* pFileState); void streamFileStateClear(SStreamFileState* pFileState); bool needClearDiskBuff(SStreamFileState* pFileState); @@ -67,7 +69,7 @@ int32_t putFreeBuff(SStreamFileState* pFileState, SRowBuffPos* pPos); SStreamSnapshot* getSnapshot(SStreamFileState* pFileState); void flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState); -void recoverSnapshot(SStreamFileState* pFileState, int64_t ckId); +int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId); int32_t getSnapshotIdList(SStreamFileState* pFileState, SArray* list); int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark); @@ -95,7 +97,7 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen); SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen); -void recoverSesssion(SStreamFileState* pFileState, int64_t ckId); +int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId); void sessionWinStateClear(SStreamFileState* pFileState); void sessionWinStateCleanup(void* pBuff); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index a5d5316d23..5a2cf3a3a0 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -112,8 +112,8 @@ typedef struct SWal { int64_t totSize; int64_t lastRollSeq; // ctl - int64_t refId; - TdThreadMutex mutex; + int64_t refId; + TdThreadRwlock mutex; // ref SHashObj *pRefHash; // refId -> SWalRef // path diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 48478c8f49..5ee3f30ddf 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -165,7 +165,6 @@ int32_t taosGetFqdn(char *); void tinet_ntoa(char *ipstr, uint32_t ip); uint32_t ip2uint(const char *const ip_addr); int32_t taosIgnSIGPIPE(); -uint32_t taosInetAddr(const char *ipAddr); const char *taosInetNtoa(struct in_addr ipInt, char *dstStr, int32_t len); uint64_t taosHton64(uint64_t val); diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index a6d8a6502f..fb6acf2a0e 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -80,7 +80,7 @@ typedef struct { SysNameInfo taosGetSysNameInfo(); bool taosCheckCurrentInDll(); -int taosGetlocalhostname(char *hostname, size_t maxLen); +int32_t taosGetlocalhostname(char *hostname, size_t maxLen); #ifdef __cplusplus } diff --git a/include/os/osSystem.h b/include/os/osSystem.h index 01fad7ad97..44910ba94d 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -81,8 +81,11 @@ int32_t taosResetTerminalMode(); unw_get_reg(&cursor, UNW_REG_IP, &pc); \ fname[0] = '\0'; \ (void)unw_get_proc_name(&cursor, fname, sizeof(fname), &offset); \ - size += 1; \ array[size] = (char *)taosMemoryMalloc(sizeof(char) * STACKSIZE + 1); \ + if(NULL == array[size]) { \ + break; \ + } \ + size += 1; \ snprintf(array[size], STACKSIZE, "0x%lx : (%s+0x%lx) [0x%lx]\n", (long)pc, fname, (long)offset, (long)pc); \ } \ if (ignoreNum < size && size > 0) { \ diff --git a/include/os/osTimezone.h b/include/os/osTimezone.h index fab42e8e11..2a8d5a442d 100644 --- a/include/os/osTimezone.h +++ b/include/os/osTimezone.h @@ -54,7 +54,7 @@ enum TdTimezone { TdEastZone12 }; -void taosGetSystemTimezone(char *outTimezone, enum TdTimezone *tsTimezone); +int32_t taosGetSystemTimezone(char *outTimezone, enum TdTimezone *tsTimezone); int32_t taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight, enum TdTimezone *tsTimezone); #ifdef __cplusplus diff --git a/include/util/taoserror.h b/include/util/taoserror.h index eb329192a2..558902075b 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -157,6 +157,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x0138) #define TSDB_CODE_SOCKET_ERROR TAOS_DEF_ERROR_CODE(0, 0x0139) #define TSDB_CODE_UNSUPPORT_OS TAOS_DEF_ERROR_CODE(0, 0x013A) +#define TSDB_CODE_TIME_ERROR TAOS_DEF_ERROR_CODE(0, 0x013B) //client #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) @@ -366,6 +367,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_MND_ARBGROUP_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AA) #define TSDB_CODE_MND_ARBGROUP_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AB) #define TSDB_CODE_MND_ARB_TOKEN_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x03AC) +#define TSDB_CODE_MND_VNODE_NOT_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x03AD) // mnode-dnode-part2 #define TSDB_CODE_MND_TOO_MANY_DNODES TAOS_DEF_ERROR_CODE(0, 0x03B0) @@ -966,6 +968,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_TMQ_REPLAY_NEED_ONE_VGROUP TAOS_DEF_ERROR_CODE(0, 0x4013) #define TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x4014) #define TSDB_CODE_TMQ_NO_TABLE_QUALIFIED TAOS_DEF_ERROR_CODE(0, 0x4015) +#define TSDB_CODE_TMQ_NO_NEED_REBALANCE TAOS_DEF_ERROR_CODE(0, 0x4016) // stream #define TSDB_CODE_STREAM_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x4100) @@ -975,6 +978,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_STREAM_NOT_LEADER TAOS_DEF_ERROR_CODE(0, 0x4105) #define TSDB_CODE_STREAM_CONFLICT_EVENT TAOS_DEF_ERROR_CODE(0, 0x4106) #define TSDB_CODE_STREAM_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x4107) +#define TSDB_CODE_STREAM_INPUTQ_FULL TAOS_DEF_ERROR_CODE(0, 0x4108) // TDLite #define TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS TAOS_DEF_ERROR_CODE(0, 0x5100) diff --git a/include/util/tdef.h b/include/util/tdef.h index 46a0d01457..46c84ab26a 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -332,6 +332,7 @@ typedef enum ELogicConditionType { #define TSDB_MAX_LEARNER_REPLICA 10 #define TSDB_SYNC_LOG_BUFFER_SIZE 4096 #define TSDB_SYNC_LOG_BUFFER_RETENTION 256 +#define TSDB_SYNC_LOG_BUFFER_THRESHOLD (1024 * 1024 * 5) #define TSDB_SYNC_APPLYQ_SIZE_LIMIT 512 #define TSDB_SYNC_NEGOTIATION_WIN 512 diff --git a/include/util/tref.h b/include/util/tref.h index 1520ced14e..f17fb73f3d 100644 --- a/include/util/tref.h +++ b/include/util/tref.h @@ -46,6 +46,7 @@ void *taosAcquireRef(int32_t rsetId, int64_t rid); // release ref, rid is the reference ID returned by taosAddRef // return 0 if success. On error, -1 is returned, and terrno is set appropriately int32_t taosReleaseRef(int32_t rsetId, int64_t rid); +int32_t taosReleaseRefEx(int32_t rsetId, int64_t rid, int32_t* isReleased); // return the first reference if rid is 0, otherwise return the next after current reference. // if return value is NULL, it means list is over(if terrno is set, it means error happens) diff --git a/include/util/tutil.h b/include/util/tutil.h index 6321d2011a..6a8f58e360 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -73,14 +73,14 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar char buf[TSDB_PASSWORD_LEN + 1]; buf[TSDB_PASSWORD_LEN] = 0; - (void)sprintf(buf, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], context.digest[1], - context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], - context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], - context.digest[12], context.digest[13], context.digest[14], context.digest[15]); + (void)sprintf(buf, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], + context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], + context.digest[6], context.digest[7], context.digest[8], context.digest[9], context.digest[10], + context.digest[11], context.digest[12], context.digest[13], context.digest[14], context.digest[15]); (void)memcpy(target, buf, TSDB_PASSWORD_LEN); } -static FORCE_INLINE int32_t taosHashBinary(char* pBuf, int32_t len) { +static FORCE_INLINE int32_t taosHashBinary(char *pBuf, int32_t len) { uint64_t hashVal = MurmurHash3_64(pBuf, len); return sprintf(pBuf, "%" PRIu64, hashVal); } @@ -161,8 +161,7 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, #define TCONTAINER_OF(ptr, type, member) ((type *)((char *)(ptr)-offsetof(type, member))) -#define TAOS_GET_TERRNO(code) \ - (terrno == 0 ? code : terrno) +#define TAOS_GET_TERRNO(code) (terrno == 0 ? code : terrno) #define TAOS_RETURN(CODE) \ do { \ @@ -177,6 +176,15 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, } \ } while (0) +#define TAOS_CHECK_RETURN_WITH_FREE(CMD, PTR) \ + do { \ + int32_t __c = (CMD); \ + if (__c != TSDB_CODE_SUCCESS) { \ + taosMemoryFree(PTR); \ + TAOS_RETURN(__c); \ + } \ + } while (0) + #define TAOS_CHECK_GOTO(CMD, LINO, LABEL) \ do { \ code = (CMD); \ diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 8297ac362e..82ebb5a9fd 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -1636,8 +1636,9 @@ void hbDeregisterConn(STscObj *pTscObj, SClientHbKey connKey) { SClientHbReq *pReq = taosHashAcquire(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); if (pReq) { tFreeClientHbReq(pReq); - if (TSDB_CODE_SUCCESS != taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey))) { - tscError("failed to remove connKey from activeInfo"); + code = taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); + if (TSDB_CODE_SUCCESS != code) { + tscError("hbDeregisterConn taosHashRemove error, code:%s", tstrerror(TAOS_SYSTEM_ERROR(code))); } taosHashRelease(pAppHbMgr->activeInfo, pReq); (void)atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 9b0a9299bf..2a5266b7ed 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -77,7 +77,10 @@ bool chkRequestKilled(void* param) { return killed; } -void cleanupAppInfo() { taosHashCleanup(appInfo.pInstMap); } +void cleanupAppInfo() { + taosHashCleanup(appInfo.pInstMap); + taosHashCleanup(appInfo.pInstMapByClusterId); +} static int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param, SAppInstInfo* pAppInfo, int connType, STscObj** pTscObj); @@ -1162,6 +1165,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { (void)atomic_add_fetch_64((int64_t*)&pActivity->numOfInsertRows, pResult->numOfRows); } } + schedulerFreeJob(&pRequest->body.queryJob, 0); } taosMemoryFree(pResult); @@ -1276,9 +1280,9 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue } if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type) && NULL == pRequest->body.resInfo.execRes.res) { - code = removeMeta(pRequest->pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type)); - if (TSDB_CODE_SUCCESS != code) { - tscError("0x%" PRIx64 " remove meta failed,QID:0x%" PRIx64, pRequest->self, pRequest->requestId); + int ret = removeMeta(pRequest->pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type)); + if (TSDB_CODE_SUCCESS != ret) { + tscError("0x%" PRIx64 " remove meta failed,code:%d,QID:0x%" PRIx64, pRequest->self, ret, pRequest->requestId); } } @@ -2039,7 +2043,7 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) { static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int32_t numOfCols, int32_t* colLength) { int32_t idx = -1; iconv_t conv = taosAcquireConv(&idx, C2M); - if (!conv) return TSDB_CODE_TSC_INTERNAL_ERROR; + if (conv == (iconv_t)-1) return TSDB_CODE_TSC_INTERNAL_ERROR; for (int32_t i = 0; i < numOfCols; ++i) { int32_t type = pResultInfo->fields[i].type; @@ -2236,6 +2240,10 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int } else if (tTagIsJson(data)) { char* jsonString = NULL; parseTagDatatoJson(data, &jsonString); + if(jsonString == NULL) { + tscError("doConvertJson error: parseTagDatatoJson failed"); + return terrno; + } STR_TO_VARSTR(dst, jsonString); taosMemoryFree(jsonString); } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index cc7a7f9489..6957874642 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -73,6 +73,8 @@ void taos_cleanup(void) { tscWarn("failed to cleanup task queue"); } + tmqMgmtClose(); + int32_t id = clientReqRefPool; clientReqRefPool = -1; taosCloseRef(id); @@ -87,9 +89,6 @@ void taos_cleanup(void) { tscDebug("rpc cleanup"); taosConvDestroy(); - - tmqMgmtClose(); - DestroyRegexCache(); tscInfo("all local resources released"); @@ -1316,9 +1315,10 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { if (NEED_CLIENT_HANDLE_ERROR(code)) { tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d,QID:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); - if (TSDB_CODE_SUCCESS != refreshMeta(pRequest->pTscObj, pRequest)) { - tscWarn("0x%" PRIx64 " refresh meta failed, code:%d - %s,QID:0x%" PRIx64, pRequest->self, code, tstrerror(code), - pRequest->requestId); + code = refreshMeta(pRequest->pTscObj, pRequest); + if (code != 0){ + tscWarn("0x%" PRIx64 " refresh meta failed, code:%d - %s,QID:0x%" PRIx64, pRequest->self, code, + tstrerror(code), pRequest->requestId); } pRequest->prevCode = code; doAsyncQuery(pRequest, true); @@ -1525,7 +1525,7 @@ int taos_get_table_vgId(TAOS *taos, const char *db, const char *table, int *vgId conn.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); - SName tableName; + SName tableName = {0}; toName(pTscObj->acctId, db, table, &tableName); SVgroupInfo vgInfo; diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index 83af01a646..aeaa3bef8b 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -68,9 +68,14 @@ static void destroyMonitorClient(void* data) { if (pMonitor == NULL) { return; } - (void)taosTmrStopA(&pMonitor->timer); + if (!taosTmrStopA(&pMonitor->timer)) { + tscError("failed to stop timer, pMonitor:%p", pMonitor); + } taosHashCleanup(pMonitor->counters); - (void)taos_collector_registry_destroy(pMonitor->registry); + int ret = taos_collector_registry_destroy(pMonitor->registry); + if (ret){ + tscError("failed to destroy registry, pMonitor:%p ret:%d", pMonitor, ret); + } taosMemoryFree(pMonitor); } @@ -186,7 +191,10 @@ static void generateClusterReport(taos_collector_registry_t* registry, void* pTr } if (strlen(pCont) != 0 && sendReport(pTransporter, epSet, pCont, MONITOR_TYPE_COUNTER, NULL) == 0) { - (void)taos_collector_registry_clear_batch(registry); + int ret = taos_collector_registry_clear_batch(registry); + if (ret){ + tscError("failed to clear registry, ret:%d", ret); + } } taosMemoryFreeClear(pCont); } @@ -207,7 +215,10 @@ static void reportSendProcess(void* param, void* tmrId) { SEpSet ep = getEpSet_s(&pInst->mgmtEp); generateClusterReport(pMonitor->registry, pInst->pTransporter, &ep); - (void)taosTmrReset(reportSendProcess, pInst->monitorParas.tsMonitorInterval * 1000, param, monitorTimer, &tmrId); + bool reset = taosTmrReset(reportSendProcess, pInst->monitorParas.tsMonitorInterval * 1000, param, monitorTimer, &tmrId); + if (!reset){ + tscError("failed to reset timer, pMonitor:%p", pMonitor); + } taosRUnLockLatch(&monitorLock); } @@ -255,7 +266,11 @@ void monitorCreateClient(int64_t clusterId) { goto fail; } - (void)taos_collector_registry_register_collector(pMonitor->registry, pMonitor->colector); + int r = taos_collector_registry_register_collector(pMonitor->registry, pMonitor->colector); + if (r){ + tscError("failed to register collector, ret:%d", r); + goto fail; + } pMonitor->counters = (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (pMonitor->counters == NULL) { @@ -304,12 +319,18 @@ void monitorCreateClientCounter(int64_t clusterId, const char* name, const char* MonitorClient* pMonitor = *ppMonitor; if (taos_collector_add_metric(pMonitor->colector, newCounter) != 0) { tscError("failed to add metric to collector"); - (void)taos_counter_destroy(newCounter); + int r = taos_counter_destroy(newCounter); + if (r){ + tscError("failed to destroy counter, code: %d", r); + } goto end; } if (taosHashPut(pMonitor->counters, name, strlen(name), &newCounter, POINTER_BYTES) != 0) { tscError("failed to put counter to monitor"); - (void)taos_counter_destroy(newCounter); + int r = taos_counter_destroy(newCounter); + if (r){ + tscError("failed to destroy counter, code: %d", r); + } goto end; } tscInfo("[monitor] monitorCreateClientCounter %" PRIx64 "(%p):%s : %p.", pMonitor->clusterId, pMonitor, name, @@ -374,7 +395,10 @@ static void monitorWriteSlowLog2File(MonitorSlowLogData* slowLogData, char* tmpP SlowLogClient* pClient = taosMemoryCalloc(1, sizeof(SlowLogClient)); if (pClient == NULL) { tscError("failed to allocate memory for slow log client"); - (void)taosCloseFile(&pFile); + int32_t ret = taosCloseFile(&pFile); + if (ret != 0){ + tscError("failed to close file:%p ret:%d", pFile, ret); + } return; } pClient->lastCheckTime = taosGetMonoTimestampMs(); @@ -383,7 +407,10 @@ static void monitorWriteSlowLog2File(MonitorSlowLogData* slowLogData, char* tmpP pClient->pFile = pFile; if (taosHashPut(monitorSlowLogHash, &slowLogData->clusterId, LONG_BYTES, &pClient, POINTER_BYTES) != 0) { tscError("failed to put clusterId:%" PRId64 " to hash table", slowLogData->clusterId); - (void)taosCloseFile(&pFile); + int32_t ret = taosCloseFile(&pFile); + if (ret != 0){ + tscError("failed to close file:%p ret:%d", pFile, ret); + } taosMemoryFree(pClient); return; } @@ -609,7 +636,11 @@ static void processFileRemoved(SlowLogClient* pClient) { tscError("failed to unlock file:%s since %d", pClient->path, terrno); return; } - (void)taosCloseFile(&(pClient->pFile)); + int32_t ret = taosCloseFile(&(pClient->pFile)); + if (ret != 0){ + tscError("failed to close file:%p ret:%d", pClient->pFile, ret); + return; + } TdFilePtr pFile = taosOpenFile(pClient->path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_READ | TD_FILE_TRUNC); @@ -698,7 +729,10 @@ static void monitorSendAllSlowLogFromTempDir(int64_t clusterId) { } if (taosLockFile(pFile) < 0) { tscInfo("failed to lock file:%s since %s, maybe used by other process", filename, terrstr()); - (void)taosCloseFile(&pFile); + int32_t ret = taosCloseFile(&pFile); + if (ret != 0){ + tscError("failed to close file:%p ret:%d", pFile, ret); + } continue; } char* tmp = taosStrdup(filename); @@ -706,7 +740,10 @@ static void monitorSendAllSlowLogFromTempDir(int64_t clusterId) { taosMemoryFree(tmp); } - (void)taosCloseDir(&pDir); + int32_t ret = taosCloseDir(&pDir); + if (ret != 0){ + tscError("failed to close dir, ret:%d", ret); + } } static void* monitorThreadFunc(void* param) { @@ -852,7 +889,9 @@ void monitorClose() { taosHashCleanup(monitorSlowLogHash); taosTmrCleanUp(monitorTimer); taosCloseQueue(monitorQueue); - (void)tsem2_destroy(&monitorSem); + if(tsem2_destroy(&monitorSem) != 0) { + tscError("failed to destroy semaphore"); + } taosWUnLockLatch(&monitorLock); } @@ -874,7 +913,9 @@ int32_t monitorPutData2MonitorQueue(MonitorSlowLogData data) { tscDebug("[monitor] write slow log to queue, clusterId:%" PRIx64 " type:%s, data:%s", slowLogData->clusterId, queueTypeStr[slowLogData->type], slowLogData->data); if (taosWriteQitem(monitorQueue, slowLogData) == 0) { - (void)tsem2_post(&monitorSem); + if(tsem2_post(&monitorSem) != 0) { + tscError("failed to post semaphore"); + } } else { if (taosCloseFile(&(slowLogData->pFile)) != 0) { tscError("failed to close file:%p", slowLogData->pFile); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index f8d443aaad..aef3cef1c5 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -50,7 +50,9 @@ int32_t genericRspCallback(void* param, SDataBuf* pMsg, int32_t code) { if (pRequest->body.queryFp != NULL) { doRequestCallback(pRequest, code); } else { - (void)tsem_post(&pRequest->body.rspSem); + if (tsem_post(&pRequest->body.rspSem) != 0){ + tscError("failed to post semaphore"); + } } return code; } @@ -58,12 +60,10 @@ int32_t genericRspCallback(void* param, SDataBuf* pMsg, int32_t code) { int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = acquireRequest(*(int64_t*)param); if (NULL == pRequest) { - goto End; + goto EXIT; } if (code != TSDB_CODE_SUCCESS) { - setErrno(pRequest, code); - (void)tsem_post(&pRequest->body.rspSem); goto End; } @@ -71,23 +71,17 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { if (NULL == pTscObj->pAppInfo) { code = TSDB_CODE_TSC_DISCONNECTED; - setErrno(pRequest, code); - (void)tsem_post(&pRequest->body.rspSem); goto End; } SConnectRsp connectRsp = {0}; if (tDeserializeSConnectRsp(pMsg->pData, pMsg->len, &connectRsp) != 0) { code = TSDB_CODE_TSC_INVALID_VERSION; - setErrno(pRequest, code); - (void)tsem_post(&pRequest->body.rspSem); goto End; } if ((code = taosCheckVersionCompatibleFromStr(version, connectRsp.sVer, 3)) != 0) { tscError("version not compatible. client version: %s, server version: %s", version, connectRsp.sVer); - setErrno(pRequest, code); - (void)tsem_post(&pRequest->body.rspSem); goto End; } @@ -96,15 +90,11 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { if (delta > timestampDeltaLimit) { code = TSDB_CODE_TIME_UNSYNCED; tscError("time diff:%ds is too big", delta); - setErrno(pRequest, code); - (void)tsem_post(&pRequest->body.rspSem); goto End; } if (connectRsp.epSet.numOfEps == 0) { code = TSDB_CODE_APP_ERROR; - setErrno(pRequest, code); - (void)tsem_post(&pRequest->body.rspSem); goto End; } @@ -177,8 +167,6 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { } else { (void)taosThreadMutexUnlock(&clientHbMgr.lock); code = TSDB_CODE_TSC_DISCONNECTED; - setErrno(pRequest, code); - (void)tsem_post(&pRequest->body.rspSem); goto End; } (void)taosThreadMutexUnlock(&clientHbMgr.lock); @@ -186,13 +174,19 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, connectRsp.clusterId, pTscObj->pAppInfo->numOfConns); - (void)tsem_post(&pRequest->body.rspSem); End: + if (code != 0){ + setErrno(pRequest, code); + } + if (tsem_post(&pRequest->body.rspSem) != 0){ + tscError("failed to post semaphore"); + } if (pRequest) { (void)releaseRequest(pRequest->self); } +EXIT: taosMemoryFree(param); taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pMsg->pData); @@ -245,7 +239,9 @@ int32_t processCreateDbRsp(void* param, SDataBuf* pMsg, int32_t code) { if (pRequest->body.queryFp) { doRequestCallback(pRequest, code); } else { - (void)tsem_post(&pRequest->body.rspSem); + if (tsem_post(&pRequest->body.rspSem) != 0){ + tscError("failed to post semaphore"); + } } return code; } @@ -285,7 +281,9 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { doRequestCallback(pRequest, pRequest->code); } else { - (void)tsem_post(&pRequest->body.rspSem); + if (tsem_post(&pRequest->body.rspSem) != 0){ + tscError("failed to post semaphore"); + } } return code; @@ -363,7 +361,9 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { if (pRequest->body.queryFp != NULL) { doRequestCallback(pRequest, pRequest->code); } else { - (void)tsem_post(&pRequest->body.rspSem); + if (tsem_post(&pRequest->body.rspSem) != 0){ + tscError("failed to post semaphore"); + } } return 0; } @@ -386,7 +386,12 @@ int32_t processCreateSTableRsp(void* param, SDataBuf* pMsg, int32_t code) { SMCreateStbRsp createRsp = {0}; SDecoder coder = {0}; tDecoderInit(&coder, pMsg->pData, pMsg->len); - (void)tDecodeSMCreateStbRsp(&coder, &createRsp); // pMsg->len == 0 + if (pMsg->len > 0){ + code = tDecodeSMCreateStbRsp(&coder, &createRsp); // pMsg->len == 0 + if (code != TSDB_CODE_SUCCESS) { + setErrno(pRequest, code); + } + } tDecoderClear(&coder); pRequest->body.resInfo.execRes.msgType = TDMT_MND_CREATE_STB; @@ -413,7 +418,9 @@ int32_t processCreateSTableRsp(void* param, SDataBuf* pMsg, int32_t code) { doRequestCallback(pRequest, code); } else { - (void)tsem_post(&pRequest->body.rspSem); + if (tsem_post(&pRequest->body.rspSem) != 0){ + tscError("failed to post semaphore"); + } } return code; } @@ -451,14 +458,15 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) { } } -END: taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); if (pRequest->body.queryFp != NULL) { doRequestCallback(pRequest, code); } else { - (void)tsem_post(&pRequest->body.rspSem); + if (tsem_post(&pRequest->body.rspSem) != 0){ + tscError("failed to post semaphore"); + } } return code; } @@ -471,7 +479,12 @@ int32_t processAlterStbRsp(void* param, SDataBuf* pMsg, int32_t code) { SMAlterStbRsp alterRsp = {0}; SDecoder coder = {0}; tDecoderInit(&coder, pMsg->pData, pMsg->len); - (void)tDecodeSMAlterStbRsp(&coder, &alterRsp); // pMsg->len = 0 + if (pMsg->len > 0){ + code = tDecodeSMAlterStbRsp(&coder, &alterRsp); // pMsg->len == 0 + if (code != TSDB_CODE_SUCCESS) { + setErrno(pRequest, code); + } + } tDecoderClear(&coder); pRequest->body.resInfo.execRes.msgType = TDMT_MND_ALTER_STB; @@ -498,7 +511,9 @@ int32_t processAlterStbRsp(void* param, SDataBuf* pMsg, int32_t code) { doRequestCallback(pRequest, code); } else { - (void)tsem_post(&pRequest->body.rspSem); + if (tsem_post(&pRequest->body.rspSem) != 0){ + tscError("failed to post semaphore"); + } } return code; } @@ -649,7 +664,9 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) { if (pRequest->body.queryFp != NULL) { doRequestCallback(pRequest, code); } else { - (void)tsem_post(&pRequest->body.rspSem); + if (tsem_post(&pRequest->body.rspSem) != 0){ + tscError("failed to post semaphore"); + } } return code; } @@ -801,7 +818,9 @@ int32_t processCompactDbRsp(void* param, SDataBuf* pMsg, int32_t code) { if (pRequest->body.queryFp != NULL) { pRequest->body.queryFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, code); } else { - (void)tsem_post(&pRequest->body.rspSem); + if (tsem_post(&pRequest->body.rspSem) != 0){ + tscError("failed to post semaphore"); + } } return code; } diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 0f8f0ec4e7..8a888a2a47 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -417,6 +417,10 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { } char* pJson = NULL; parseTagDatatoJson(pTag, &pJson); + if(pJson == NULL) { + uError("parseTagDatatoJson failed, pJson == NULL"); + goto end; + } cJSON* tag = cJSON_CreateObject(); RAW_NULL_CHECK(tag); STagVal* pTagVal = taosArrayGet(pTagVals, 0); @@ -727,6 +731,10 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { goto end; } parseTagDatatoJson(vAlterTbReq.pTagVal, &buf); + if(buf == NULL) { + uError("parseTagDatatoJson failed, buf == NULL"); + goto end; + } } else { if (vAlterTbReq.tagType == TSDB_DATA_TYPE_VARBINARY) { buf = taosMemoryCalloc(vAlterTbReq.nTagVal * 2 + 2 + 3, 1); @@ -960,6 +968,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { RAW_NULL_CHECK(pCmdMsg.pMsg); if (tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) <= 0) { code = TSDB_CODE_INVALID_PARA; + taosMemoryFree(pCmdMsg.pMsg); goto end; } @@ -971,15 +980,15 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { (void)launchQueryImpl(pRequest, &pQuery, true, NULL); // ignore, because return value is pRequest + taosMemoryFree(pCmdMsg.pMsg); + if (pRequest->code == TSDB_CODE_SUCCESS) { SCatalog* pCatalog = NULL; - // ignore the return value - (void)catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); - (void)catalogRemoveTableMeta(pCatalog, &tableName); + RAW_RETURN_CHECK(catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog)); + RAW_RETURN_CHECK(catalogRemoveTableMeta(pCatalog, &tableName)); } code = pRequest->code; - taosMemoryFree(pCmdMsg.pMsg); end: uDebug(LOG_ID_TAG " create stable return, msg:%s", LOG_ID_VALUE, tstrerror(code)); @@ -1022,8 +1031,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { .requestObjRefId = pRequest->self, .mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)}; SName pName = {0}; - toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, - &pName); // ignore the return value, always return pName + toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName); STableMeta* pTableMeta = NULL; code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { @@ -1064,6 +1072,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { RAW_NULL_CHECK(pCmdMsg.pMsg); if (tSerializeSMDropStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) <= 0) { code = TSDB_CODE_INVALID_PARA; + taosMemoryFree(pCmdMsg.pMsg); goto end; } @@ -1074,15 +1083,14 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { pQuery.stableQuery = true; (void)launchQueryImpl(pRequest, &pQuery, true, NULL); // ignore, because return value is pRequest - + taosMemoryFree(pCmdMsg.pMsg); if (pRequest->code == TSDB_CODE_SUCCESS) { // ignore the error code - (void)catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); - (void)catalogRemoveTableMeta(pCatalog, &tableName); + RAW_RETURN_CHECK(catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog)); + RAW_RETURN_CHECK(catalogRemoveTableMeta(pCatalog, &tableName)); } code = pRequest->code; - taosMemoryFree(pCmdMsg.pMsg); end: uDebug(LOG_ID_TAG " drop stable return, msg:%s", LOG_ID_VALUE, tstrerror(code)); @@ -1230,7 +1238,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { (void)launchQueryImpl(pRequest, pQuery, true, NULL); if (pRequest->code == TSDB_CODE_SUCCESS) { - (void)removeMeta(pTscObj, pRequest->tableList, false); + RAW_RETURN_CHECK(removeMeta(pTscObj, pRequest->tableList, false)); } code = pRequest->code; @@ -1359,7 +1367,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { (void)launchQueryImpl(pRequest, pQuery, true, NULL); if (pRequest->code == TSDB_CODE_SUCCESS) { - (void)removeMeta(pTscObj, pRequest->tableList, false); + RAW_RETURN_CHECK(removeMeta(pTscObj, pRequest->tableList, false)); } code = pRequest->code; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 5a2b2f24cd..082daa805c 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -116,7 +116,7 @@ static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo *conn, const char return TSDB_CODE_SML_INVALID_DATA; } } else { - toName(info->taos->acctId, info->pRequest->pDb, pTabName, &pAuth.tbName); //ignore + toName(info->taos->acctId, info->pRequest->pDb, pTabName, &pAuth.tbName); } pAuth.type = type; @@ -2217,9 +2217,12 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, break; } taosMsleep(100); - (void)refreshMeta(request->pTscObj, request); //ignore return code,try again uInfo("SML:%" PRIx64 " retry:%d/10,ver is old retry or object is creating code:%d, msg:%s", info->id, cnt, code, tstrerror(code)); + code = refreshMeta(request->pTscObj, request); + if (code != 0){ + uInfo("SML:%" PRIx64 " refresh meta error code:%d, msg:%s", info->id, code, tstrerror(code)); + } smlDestroyInfo(info); info = NULL; taos_free_result(request); diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 175628cc7d..d556ada1d2 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -771,7 +771,10 @@ void* stmtBindThreadFunc(void* param) { continue; } - (void)stmtAsyncOutput(pStmt, asyncParam); + int ret = stmtAsyncOutput(pStmt, asyncParam); + if (ret != 0){ + qError("stmtAsyncOutput failed, reason:%s", tstrerror(ret)); + } } qInfo("stmt bind thread stopped"); diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index ce99cd7deb..66430a2647 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -194,6 +194,12 @@ static int32_t stmtUpdateBindInfo(TAOS_STMT2* stmt, STableMeta* pTableMeta, void pStmt->bInfo.tbSuid = pTableMeta->suid; pStmt->bInfo.tbVgId = pTableMeta->vgId; pStmt->bInfo.tbType = pTableMeta->tableType; + + if (!pStmt->bInfo.tagsCached) { + qDestroyBoundColInfo(pStmt->bInfo.boundTags); + taosMemoryFreeClear(pStmt->bInfo.boundTags); + } + pStmt->bInfo.boundTags = tags; pStmt->bInfo.tagsCached = false; tstrncpy(pStmt->bInfo.stbFName, sTableName, sizeof(pStmt->bInfo.stbFName)); @@ -985,10 +991,6 @@ int stmtSetTbTags2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* tags) { return TSDB_CODE_SUCCESS; } - if (pStmt->bInfo.inExecCache) { - return TSDB_CODE_SUCCESS; - } - STableDataCxt** pDataBlock = (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { @@ -996,6 +998,10 @@ int stmtSetTbTags2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* tags) { STMT_ERR_RET(TSDB_CODE_APP_ERROR); } + if (pStmt->bInfo.inExecCache && (!pStmt->sql.autoCreateTbl || (*pDataBlock)->pData->pCreateTbReq)) { + return TSDB_CODE_SUCCESS; + } + tscDebug("start to bind stmt tag values"); STMT_ERR_RET(qBindStmtTagsValue2(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.stbFName, pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf, @@ -1889,7 +1895,12 @@ int stmtGetParamTbName(TAOS_STMT2* stmt, int* nums) { STMT_ERR_RET(stmtParseSql(pStmt)); } - *nums = STMT_TYPE_MULTI_INSERT == pStmt->sql.type ? 1 : 0; + if (TSDB_CODE_TSC_STMT_TBNAME_ERROR == pStmt->errCode) { + *nums = 1; + pStmt->errCode = TSDB_CODE_SUCCESS; + } else { + *nums = STMT_TYPE_MULTI_INSERT == pStmt->sql.type ? 1 : 0; + } return TSDB_CODE_SUCCESS; } diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 0cc38a3a12..8836fce31a 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -807,11 +807,18 @@ static void generateTimedTask(int64_t refId, int32_t type) { if (code == TSDB_CODE_SUCCESS) { *pTaskType = type; if (taosWriteQitem(tmq->delayedTask, pTaskType) == 0) { - (void)tsem2_post(&tmq->rspSem); + if (tsem2_post(&tmq->rspSem) != 0){ + tscError("consumer:0x%" PRIx64 " failed to post sem, type:%d", tmq->consumerId, type); + } + }else{ + taosFreeQitem(pTaskType); } } - (void)taosReleaseRef(tmqMgmt.rsetId, refId); + code = taosReleaseRef(tmqMgmt.rsetId, refId); + if (code != 0){ + tscError("failed to release ref:%"PRId64 ", type:%d, code:%d", refId, type, code); + } } void tmqAssignAskEpTask(void* param, void* tmrId) { @@ -824,8 +831,13 @@ void tmqReplayTask(void* param, void* tmrId) { tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) return; - (void)tsem2_post(&tmq->rspSem); - (void)taosReleaseRef(tmqMgmt.rsetId, refId); + if (tsem2_post(&tmq->rspSem) != 0){ + tscError("consumer:0x%" PRIx64 " failed to post sem, replay", tmq->consumerId); + } + int32_t code = taosReleaseRef(tmqMgmt.rsetId, refId); + if (code != 0){ + tscError("failed to release ref:%"PRId64 ", code:%d", refId, code); + } } void tmqAssignDelayedCommitTask(void* param, void* tmrId) { @@ -835,17 +847,17 @@ void tmqAssignDelayedCommitTask(void* param, void* tmrId) { int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { if (code != 0) { - goto _return; + goto END; } if (pMsg == NULL || param == NULL) { code = TSDB_CODE_INVALID_PARA; - goto _return; + goto END; } SMqHbRsp rsp = {0}; code = tDeserializeSMqHbRsp(pMsg->pData, pMsg->len, &rsp); if (code != 0) { - goto _return; + goto END; } int64_t refId = (int64_t)param; @@ -866,13 +878,15 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { } } taosWUnLockLatch(&tmq->lock); - (void)taosReleaseRef(tmqMgmt.rsetId, refId); + code = taosReleaseRef(tmqMgmt.rsetId, refId); + if (code != 0){ + tscError("failed to release ref:%"PRId64 ", code:%d", refId, code); + } } tDestroySMqHbRsp(&rsp); -_return: - +END: taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); return code; @@ -974,9 +988,15 @@ void tmqSendHbReq(void* param, void* tmrId) { OVER: tDestroySMqHbReq(&req); if (tmrId != NULL) { - (void)taosTmrReset(tmqSendHbReq, tmq->heartBeatIntervalMs, param, tmqMgmt.timer, &tmq->hbLiveTimer); + bool ret = taosTmrReset(tmqSendHbReq, tmq->heartBeatIntervalMs, param, tmqMgmt.timer, &tmq->hbLiveTimer); + if (!ret){ + tscError("failed to reset timer fo tmq hb"); + } + } + int32_t ret = taosReleaseRef(tmqMgmt.rsetId, refId); + if (ret != 0){ + tscError("failed to release ref:%"PRId64 ", code:%d", refId, ret); } - (void)taosReleaseRef(tmqMgmt.rsetId, refId); } static void defaultCommitCbFn(tmq_t* pTmq, int32_t code, void* param) { @@ -995,9 +1015,7 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) { return; } - (void)taosReadAllQitems(pTmq->delayedTask, qall); - - int32_t numOfItems = taosQallItemSize(qall); + int32_t numOfItems = taosReadAllQitems(pTmq->delayedTask, qall); if (numOfItems == 0) { taosFreeQall(qall); return; @@ -1005,9 +1023,7 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) { tscDebug("consumer:0x%" PRIx64 " handle delayed %d tasks before poll data", pTmq->consumerId, numOfItems); int8_t* pTaskType = NULL; - (void)taosGetQitem(qall, (void**)&pTaskType); - - while (pTaskType != NULL) { + while (taosGetQitem(qall, (void**)&pTaskType) != 0) { if (*pTaskType == TMQ_DELAYED_TASK__ASK_EP) { code = askEp(pTmq, NULL, false, false); if (code != 0) { @@ -1015,21 +1031,26 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) { continue; } tscDebug("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId); - (void)taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer, + bool ret = taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer, &pTmq->epTimer); + if (!ret){ + tscError("failed to reset timer fo tmq ask ep"); + } } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { tmq_commit_cb* pCallbackFn = pTmq->commitCb ? pTmq->commitCb : defaultCommitCbFn; asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam); tscDebug("consumer:0x%" PRIx64 " next commit to vnode(s) in %.2fs", pTmq->consumerId, pTmq->autoCommitInterval / 1000.0); - (void)taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer, + bool ret = taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer, &pTmq->commitTimer); + if (!ret){ + tscError("failed to reset timer fo commit"); + } } else { tscError("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType); } taosFreeQitem(pTaskType); - (void)taosGetQitem(qall, (void**)&pTaskType); } taosFreeQall(qall); @@ -1061,26 +1082,18 @@ static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { void tmqClearUnhandleMsg(tmq_t* tmq) { SMqRspWrapper* rspWrapper = NULL; - while (1) { - (void)taosGetQitem(tmq->qall, (void**)&rspWrapper); - if (rspWrapper) { + while (taosGetQitem(tmq->qall, (void**)&rspWrapper) != 0) { tmqFreeRspWrapper(rspWrapper); taosFreeQitem(rspWrapper); - } else { - break; - } } rspWrapper = NULL; - (void)taosReadAllQitems(tmq->mqueue, tmq->qall); - while (1) { - (void)taosGetQitem(tmq->qall, (void**)&rspWrapper); - if (rspWrapper) { - tmqFreeRspWrapper(rspWrapper); - taosFreeQitem(rspWrapper); - } else { - break; - } + if (taosReadAllQitems(tmq->mqueue, tmq->qall) == 0){ + return; + } + while (taosGetQitem(tmq->qall, (void**)&rspWrapper) != 0) { + tmqFreeRspWrapper(rspWrapper); + taosFreeQitem(rspWrapper); } } @@ -1095,7 +1108,9 @@ int32_t tmqSubscribeCb(void* param, SDataBuf* pMsg, int32_t code) { if (pMsg) { taosMemoryFree(pMsg->pEpSet); } - (void)tsem2_post(&pParam->rspSem); + if (tsem2_post(&pParam->rspSem) != 0){ + tscError("failed to post sem, subscribe cb"); + } return 0; } @@ -1154,19 +1169,27 @@ void tmqFreeImpl(void* handle) { } taosFreeQall(tmq->qall); - (void)tsem2_destroy(&tmq->rspSem); + if(tsem2_destroy(&tmq->rspSem) != 0) { + tscError("failed to destroy sem in free tmq"); + } taosArrayDestroyEx(tmq->clientTopics, freeClientTopic); taos_close_internal(tmq->pTscObj); if (tmq->commitTimer) { - (void)taosTmrStopA(&tmq->commitTimer); + if (!taosTmrStopA(&tmq->commitTimer)) { + tscError("failed to stop commit timer"); + } } if (tmq->epTimer) { - (void)taosTmrStopA(&tmq->epTimer); + if (!taosTmrStopA(&tmq->epTimer)) { + tscError("failed to stop ep timer"); + } } if (tmq->hbLiveTimer) { - (void)taosTmrStopA(&tmq->hbLiveTimer); + if (!taosTmrStopA(&tmq->hbLiveTimer)) { + tscError("failed to stop hb timer"); + } } taosMemoryFree(tmq); @@ -1312,7 +1335,6 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { if (code) { terrno = code; tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); - (void)tsem2_destroy(&pTmq->rspSem); SET_ERROR_MSG_TMQ("init tscObj failed") goto _failed; } @@ -1419,7 +1441,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } void* abuf = buf; - (void)tSerializeSCMSubscribeReq(&abuf, &req); + tlen = tSerializeSCMSubscribeReq(&abuf, &req); sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { @@ -1450,8 +1472,12 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { goto FAIL; } - (void)tsem2_wait(¶m.rspSem); - (void)tsem2_destroy(¶m.rspSem); + if (tsem2_wait(¶m.rspSem) != 0){ + tscError("consumer:0x%" PRIx64 ", failed to wait semaphore in subscribe", tmq->consumerId); + } + if(tsem2_destroy(¶m.rspSem) != 0) { + tscError("consumer:0x%" PRIx64 ", failed to destroy semaphore in subscribe", tmq->consumerId); + } if (param.rspErr != 0) { code = param.rspErr; @@ -1656,6 +1682,8 @@ END: (void)strcpy(pRspWrapper->topicName, pParam->topicName); code = taosWriteQitem(tmq->mqueue, pRspWrapper); if (code != 0) { + tmqFreeRspWrapper((SMqRspWrapper*)pRspWrapper); + taosFreeQitem(pRspWrapper); tscError("consumer:0x%" PRIx64 " put poll res into mqueue failed, code:%d", tmq->consumerId, code); } } @@ -1663,10 +1691,17 @@ END: tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d,QID:0x%" PRIx64, tmq ? tmq->consumerId : 0, rspType, vgId, total, requestId); - if (tmq) (void)tsem2_post(&tmq->rspSem); + if (tmq) { + if (tsem2_post(&tmq->rspSem) != 0){ + tscError("failed to post rsp sem, consumer:0x%" PRIx64, tmq->consumerId); + } + } if (pMsg) taosMemoryFreeClear(pMsg->pData); if (pMsg) taosMemoryFreeClear(pMsg->pEpSet); - (void)taosReleaseRef(tmqMgmt.rsetId, refId); + ret = taosReleaseRef(tmqMgmt.rsetId, refId); + if (ret != 0){ + tscError("failed to release ref:%"PRId64 ", code:%d", refId, ret); + } return code; } @@ -1763,7 +1798,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) SHashObj* pVgOffsetHashMap = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); if (pVgOffsetHashMap == NULL) { - (void)taosArrayDestroy(newTopics); + taosArrayDestroy(newTopics); return false; } @@ -1848,10 +1883,10 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl pReq->enableBatchMeta = tmq->enableBatchMeta; } -int32_t tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqMetaRspObj** ppRspObj) { +void tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqMetaRspObj** ppRspObj) { SMqMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqMetaRspObj)); if (pRspObj == NULL) { - return terrno; + return; } pRspObj->resType = RES_TYPE__TMQ_META; tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); @@ -1860,13 +1895,12 @@ int32_t tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqMetaRspObj** (void)memcpy(&pRspObj->metaRsp, &pWrapper->metaRsp, sizeof(SMqMetaRsp)); *ppRspObj = pRspObj; - return 0; } -int32_t tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqBatchMetaRspObj** ppRspObj) { +void tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqBatchMetaRspObj** ppRspObj) { SMqBatchMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqBatchMetaRspObj)); if (pRspObj == NULL) { - return terrno; + return; } pRspObj->common.resType = RES_TYPE__TMQ_BATCH_META; tstrncpy(pRspObj->common.topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); @@ -1876,7 +1910,6 @@ int32_t tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqBatchMet (void)memcpy(&pRspObj->rsp, &pWrapper->batchMetaRsp, sizeof(SMqBatchMetaRsp)); tscDebug("build batchmeta Rsp from wrapper"); *ppRspObj = pRspObj; - return 0; } void changeByteEndian(char* pData) { @@ -1973,31 +2006,29 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg } } -int32_t tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, +void tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObj** ppRspObj) { SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); if (pRspObj == NULL) { - return terrno; + return; } pRspObj->common.resType = RES_TYPE__TMQ; (void)memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp)); tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, &pRspObj->common, &pRspObj->rsp.common); *ppRspObj = pRspObj; - return 0; } -int32_t tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, +void tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqTaosxRspObj** ppRspObj) { SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj)); if (pRspObj == NULL) { - return terrno; + return; } pRspObj->common.resType = RES_TYPE__TMQ_METADATA; (void)memcpy(&pRspObj->rsp, &pWrapper->taosxRsp, sizeof(STaosxRsp)); tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, &pRspObj->common, &pRspObj->rsp.common); *ppRspObj = pRspObj; - return 0; } static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* pVg, int64_t timeout) { @@ -2156,12 +2187,11 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { while (1) { SMqRspWrapper* pRspWrapper = NULL; - (void)taosGetQitem(tmq->qall, (void**)&pRspWrapper); - - if (pRspWrapper == NULL) { - (void)taosReadAllQitems(tmq->mqueue, tmq->qall); - (void)taosGetQitem(tmq->qall, (void**)&pRspWrapper); - if (pRspWrapper == NULL) { + if (taosGetQitem(tmq->qall, (void**)&pRspWrapper) == 0) { + if (taosReadAllQitems(tmq->mqueue, tmq->qall) == 0){ + return NULL; + } + if (taosGetQitem(tmq->qall, (void**)&pRspWrapper) == 0) { return NULL; } } @@ -2239,7 +2269,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { } else { // build rsp int64_t numOfRows = 0; SMqRspObj* pRsp = NULL; - (void)tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp); + tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp); tmq->totalRows += numOfRows; pVg->emptyBlockReceiveTs = 0; if (pRsp && tmq->replayEnable) { @@ -2293,7 +2323,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { pollRspWrapper->metaRsp.head.walsver, pollRspWrapper->metaRsp.head.walever, tmq->consumerId, true); // build rsp SMqMetaRspObj* pRsp = NULL; - (void)tmqBuildMetaRspFromWrapper(pollRspWrapper, &pRsp); + tmqBuildMetaRspFromWrapper(pollRspWrapper, &pRsp); taosMemoryFreeClear(pollRspWrapper->pEpset); taosFreeQitem(pRspWrapper); taosWUnLockLatch(&tmq->lock); @@ -2331,7 +2361,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { pollRspWrapper->batchMetaRsp.head.walsver, pollRspWrapper->batchMetaRsp.head.walever, tmq->consumerId, true); SMqBatchMetaRspObj* pRsp = NULL; - (void)tmqBuildBatchMetaRspFromWrapper(pollRspWrapper, &pRsp); + tmqBuildBatchMetaRspFromWrapper(pollRspWrapper, &pRsp); taosMemoryFreeClear(pollRspWrapper->pEpset); taosFreeQitem(pRspWrapper); taosWUnLockLatch(&tmq->lock); @@ -2381,9 +2411,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { // build rsp int64_t numOfRows = 0; SMqTaosxRspObj* pRsp = NULL; - if (tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp) != 0) { - tscError("consumer:0x%" PRIx64 " build taosx rsp failed, vgId:%d", tmq->consumerId, pVg->vgId); - } + tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp); tmq->totalRows += numOfRows; char buf[TSDB_OFFSET_LEN] = {0}; @@ -2551,7 +2579,10 @@ int32_t tmq_consumer_close(tmq_t* tmq) { } if (code == 0) { - (void)taosRemoveRef(tmqMgmt.rsetId, tmq->refId); + code = taosRemoveRef(tmqMgmt.rsetId, tmq->refId); + if (code != 0){ + tscError("tmq close failed to remove ref:%" PRId64 ", code:%d", tmq->refId, code); + } } return code; } @@ -2709,7 +2740,9 @@ void tmq_commit_async(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* cb, void* static void commitCallBackFn(tmq_t* UNUSED_PARAM(tmq), int32_t code, void* param) { SSyncCommitInfo* pInfo = (SSyncCommitInfo*)param; pInfo->code = code; - (void)tsem2_post(&pInfo->sem); + if (tsem2_post(&pInfo->sem) != 0){ + tscError("failed to post rsp sem in commit cb"); + } } int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { @@ -2738,10 +2771,14 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { asyncCommitFromResult(tmq, pRes, commitCallBackFn, pInfo); } - (void)tsem2_wait(&pInfo->sem); + if (tsem2_wait(&pInfo->sem) != 0){ + tscError("failed to wait sem for sync commit"); + } code = pInfo->code; - (void)tsem2_destroy(&pInfo->sem); + if(tsem2_destroy(&pInfo->sem) != 0) { + tscError("failed to destroy sem for sync commit"); + } taosMemoryFree(pInfo); tscInfo("consumer:0x%" PRIx64 " sync res commit done, code:%s", tmq->consumerId, tstrerror(code)); @@ -2806,12 +2843,16 @@ int32_t tmq_commit_offset_sync(tmq_t* tmq, const char* pTopicName, int32_t vgId, code = asyncCommitOffset(tmq, tname, vgId, &offsetVal, commitCallBackFn, pInfo); if (code == 0) { - (void)tsem2_wait(&pInfo->sem); + if (tsem2_wait(&pInfo->sem) != 0){ + tscError("failed to wait sem for sync commit offset"); + } code = pInfo->code; } if (code == TSDB_CODE_TMQ_SAME_COMMITTED_VALUE) code = TSDB_CODE_SUCCESS; - (void)tsem2_destroy(&pInfo->sem); + if(tsem2_destroy(&pInfo->sem) != 0) { + tscError("failed to destroy sem for sync commit offset"); + } taosMemoryFree(pInfo); tscInfo("consumer:0x%" PRIx64 " sync send commit to vgId:%d, offset:%" PRId64 " code:%s", tmq->consumerId, vgId, @@ -2864,11 +2905,11 @@ end: } int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { - if (param == NULL) { + SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; + if (pParam == NULL) { goto FAIL; } - SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId); if (tmq == NULL) { code = TSDB_CODE_TMQ_CONSUMER_CLOSED; @@ -2906,19 +2947,30 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { tmqFreeRspWrapper((SMqRspWrapper*)pWrapper); taosFreeQitem(pWrapper); } else { - (void)taosWriteQitem(tmq->mqueue, pWrapper); + code = taosWriteQitem(tmq->mqueue, pWrapper); + if (code != 0) { + tmqFreeRspWrapper((SMqRspWrapper*)pWrapper); + taosFreeQitem(pWrapper); + tscError("consumer:0x%" PRIx64 " put ep res into mqueue failed, code:%d", tmq->consumerId, code); + } } } END: - (void)taosReleaseRef(tmqMgmt.rsetId, pParam->refId); - + { + int32_t ret = taosReleaseRef(tmqMgmt.rsetId, pParam->refId); + if (ret != 0){ + tscError("failed to release ref:%"PRId64 ", code:%d", pParam->refId, ret); + } + } FAIL: - if (pParam->sync) { + if (pParam && pParam->sync) { SAskEpInfo* pInfo = pParam->pParam; if (pInfo) { pInfo->code = code; - (void)tsem2_post(&pInfo->sem); + if (tsem2_post(&pInfo->sem) != 0){ + tscError("failed to post rsp sem askep cb"); + } } } @@ -2940,11 +2992,15 @@ int32_t syncAskEp(tmq_t* pTmq) { int32_t code = askEp(pTmq, pInfo, true, false); if (code == 0) { - (void)tsem2_wait(&pInfo->sem); + if (tsem2_wait(&pInfo->sem) != 0){ + tscError("consumer:0x%" PRIx64 ", failed to wait for sem", pTmq->consumerId); + } code = pInfo->code; } - (void)tsem2_destroy(&pInfo->sem); + if(tsem2_destroy(&pInfo->sem) != 0) { + tscError("failed to destroy sem sync ask ep"); + } taosMemoryFree(pInfo); return code; } @@ -3115,7 +3171,9 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { END: pCommon->code = code; if (total == pParam->totalReq) { - (void)tsem2_post(&pCommon->rsp); + if (tsem2_post(&pCommon->rsp) != 0) { + tscError("failed to post semaphore in get wal cb"); + } } if (pMsg) { @@ -3130,8 +3188,10 @@ static void destroyCommonInfo(SMqVgCommon* pCommon) { if (pCommon == NULL) { return; } - (void)taosArrayDestroy(pCommon->pList); - (void)tsem2_destroy(&pCommon->rsp); + taosArrayDestroy(pCommon->pList); + if(tsem2_destroy(&pCommon->rsp) != 0) { + tscError("failed to destroy semaphore for topic:%s", pCommon->pTopicName); + } (void)taosThreadMutexDestroy(&pCommon->mutex); taosMemoryFree(pCommon->pTopicName); taosMemoryFree(pCommon); @@ -3167,7 +3227,9 @@ end: taosMemoryFree(pMsg->pEpSet); } pParam->code = code; - (void)tsem2_post(&pParam->sem); + if (tsem2_post(&pParam->sem) != 0){ + tscError("failed to post semaphore in tmCommittedCb"); + } return code; } @@ -3231,12 +3293,16 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, epSet, NULL, sendInfo); if (code != 0) { - (void)tsem2_destroy(&pParam->sem); + if(tsem2_destroy(&pParam->sem) != 0) { + tscError("failed to destroy semaphore in get committed from server1"); + } taosMemoryFree(pParam); return code; } - (void)tsem2_wait(&pParam->sem); + if (tsem2_wait(&pParam->sem) != 0){ + tscError("failed to wait semaphore in get committed from server"); + } code = pParam->code; if (code == TSDB_CODE_SUCCESS) { if (pParam->vgOffset.offset.val.type == TMQ_OFFSET__LOG) { @@ -3246,7 +3312,9 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep code = TSDB_CODE_TMQ_SNAPSHOT_ERROR; } } - (void)tsem2_destroy(&pParam->sem); + if(tsem2_destroy(&pParam->sem) != 0) { + tscError("failed to destroy semaphore in get committed from server2"); + } taosMemoryFree(pParam); return code; @@ -3517,7 +3585,9 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a } } - (void)tsem2_wait(&pCommon->rsp); + if (tsem2_wait(&pCommon->rsp) != 0){ + tscError("consumer:0x%" PRIx64 " failed to wait sem in get assignment", tmq->consumerId); + } code = pCommon->code; if (code != TSDB_CODE_SUCCESS) { @@ -3580,7 +3650,9 @@ static int32_t tmqSeekCb(void* param, SDataBuf* pMsg, int32_t code) { } SMqSeekParam* pParam = param; pParam->code = code; - (void)tsem2_post(&pParam->sem); + if (tsem2_post(&pParam->sem) != 0){ + tscError("failed to post sem in tmqSeekCb"); + } return 0; } @@ -3677,14 +3749,20 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo); if (code != 0) { - (void)tsem2_destroy(&pParam->sem); + if(tsem2_destroy(&pParam->sem) != 0) { + tscError("consumer:0x%" PRIx64 "destroy rsp sem failed in seek offset", tmq->consumerId); + } taosMemoryFree(pParam); return code; } - (void)tsem2_wait(&pParam->sem); + if (tsem2_wait(&pParam->sem) != 0){ + tscError("consumer:0x%" PRIx64 "wait rsp sem failed in seek offset", tmq->consumerId); + } code = pParam->code; - (void)tsem2_destroy(&pParam->sem); + if(tsem2_destroy(&pParam->sem) != 0) { + tscError("consumer:0x%" PRIx64 "destroy rsp sem failed in seek offset", tmq->consumerId); + } taosMemoryFree(pParam); tscInfo("consumer:0x%" PRIx64 "send seek to vgId:%d, return code:%s", tmq->consumerId, vgId, tstrerror(code)); diff --git a/source/common/src/cos_cp.c b/source/common/src/cos_cp.c index ce5f666f21..adf4160abe 100644 --- a/source/common/src/cos_cp.c +++ b/source/common/src/cos_cp.c @@ -170,7 +170,7 @@ int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) { cp_body = taosMemoryMalloc(size + 1); if (!cp_body) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } int64_t n = taosReadFile(fd, cp_body, size); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 08f6842c4b..7e957357a9 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -124,7 +124,7 @@ int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* buf = taosMemoryRealloc(pColumnInfoData->pData, newSize); if (buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pColumnInfoData->pData = buf; @@ -170,7 +170,7 @@ static int32_t colDataReserve(SColumnInfoData* pColumnInfoData, size_t newSize) if (pColumnInfoData->varmeta.allocLen < newSize) { char* buf = taosMemoryRealloc(pColumnInfoData->pData, newSize); if (buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pColumnInfoData->pData = buf; @@ -388,7 +388,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int if (finalNumOfRows > (*capacity)) { char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); if (p == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } *capacity = finalNumOfRows; @@ -409,7 +409,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int if (pColumnInfoData->varmeta.allocLen < len + oldLen) { char* tmp = taosMemoryRealloc(pColumnInfoData->pData, len + oldLen); if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pColumnInfoData->pData = tmp; @@ -425,14 +425,14 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int // all data may be null, when the pColumnInfoData->info.type == 0, bytes == 0; char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes); if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pColumnInfoData->pData = tmp; if (BitmapLen(numOfRow1) < BitmapLen(finalNumOfRows)) { char* btmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(finalNumOfRows)); if (btmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } uint32_t extend = BitmapLen(finalNumOfRows) - BitmapLen(numOfRow1); memset(btmp + BitmapLen(numOfRow1), 0, extend); @@ -469,7 +469,7 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p if (pColumnInfoData->varmeta.allocLen < newLen) { char* tmp = taosMemoryRealloc(pColumnInfoData->pData, newLen); if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pColumnInfoData->pData = tmp; @@ -545,7 +545,7 @@ int32_t colDataAssignNRows(SColumnInfoData* pDst, int32_t dstIdx, const SColumnI if (pDst->varmeta.allocLen < pDst->varmeta.length + allLen) { char* tmp = taosMemoryRealloc(pDst->pData, pDst->varmeta.length + allLen); if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pDst->pData = tmp; @@ -999,7 +999,7 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { size_t metaSize = pBlock->info.rows * sizeof(int32_t); char* tmp = taosMemoryRealloc(pCol->varmeta.offset, metaSize); // preview calloc is too small if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pCol->varmeta.offset = (int32_t*)tmp; @@ -1017,7 +1017,7 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { if (pCol->varmeta.allocLen < colLength) { char* tmp = taosMemoryRealloc(pCol->pData, colLength); if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pCol->pData = tmp; @@ -1098,7 +1098,7 @@ int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity) if (pCol->varmeta.allocLen < colLength) { char* tmp = taosMemoryRealloc(pCol->pData, colLength); if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pCol->pData = tmp; @@ -1421,7 +1421,6 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { int32_t* index = createTupleIndex(rows); if (index == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } @@ -1544,7 +1543,7 @@ int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockI if (IS_VAR_DATA_TYPE(pColumn->info.type)) { char* tmp = taosMemoryRealloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pColumn->varmeta.offset = (int32_t*)tmp; @@ -1553,7 +1552,7 @@ int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockI // prepare for the null bitmap char* tmp = taosMemoryRealloc(pColumn->nullbitmap, BitmapLen(numOfRows)); if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } int32_t oldLen = BitmapLen(existedRows); @@ -1567,7 +1566,7 @@ int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockI // to MALLOC_ALIGN_BYTES tmp = taosMemoryMallocAlign(MALLOC_ALIGN_BYTES, numOfRows * pColumn->info.bytes); if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } // memset(tmp, 0, numOfRows * pColumn->info.bytes); @@ -2517,24 +2516,25 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf pDataBlock->info.id.groupId, pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version, pDataBlock->info.calWin.skey, pDataBlock->info.calWin.ekey, pDataBlock->info.parTbName); if (len >= size - 1) { - return code; + goto _exit; } for (int32_t j = 0; j < rows; j++) { len += snprintf(dumpBuf + len, size - len, "%s|", flag); if (len >= size - 1) { - return code; + goto _exit; } for (int32_t k = 0; k < colNum; k++) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); if (pColInfoData == NULL) { - return terrno; + code = terrno; + goto _exit; } if (colDataIsNull(pColInfoData, rows, j, NULL) || !pColInfoData->pData) { len += snprintf(dumpBuf + len, size - len, " %15s |", "NULL"); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; continue; } @@ -2542,53 +2542,53 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf switch (pColInfoData->info.type) { case TSDB_DATA_TYPE_TIMESTAMP: memset(pBuf, 0, sizeof(pBuf)); - (void) formatTimestamp(pBuf, *(uint64_t*)var, pColInfoData->info.precision); + (void)formatTimestamp(pBuf, *(uint64_t*)var, pColInfoData->info.precision); len += snprintf(dumpBuf + len, size - len, " %25s |", pBuf); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_TINYINT: len += snprintf(dumpBuf + len, size - len, " %15d |", *(int8_t*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_UTINYINT: len += snprintf(dumpBuf + len, size - len, " %15d |", *(uint8_t*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_SMALLINT: len += snprintf(dumpBuf + len, size - len, " %15d |", *(int16_t*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_USMALLINT: len += snprintf(dumpBuf + len, size - len, " %15d |", *(uint16_t*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_INT: len += snprintf(dumpBuf + len, size - len, " %15d |", *(int32_t*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_UINT: len += snprintf(dumpBuf + len, size - len, " %15u |", *(uint32_t*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_BIGINT: len += snprintf(dumpBuf + len, size - len, " %15" PRId64 " |", *(int64_t*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_UBIGINT: len += snprintf(dumpBuf + len, size - len, " %15" PRIu64 " |", *(uint64_t*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_FLOAT: len += snprintf(dumpBuf + len, size - len, " %15f |", *(float*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_DOUBLE: len += snprintf(dumpBuf + len, size - len, " %15f |", *(double*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_BOOL: len += snprintf(dumpBuf + len, size - len, " %15d |", *(bool*)var); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; break; case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: @@ -2599,24 +2599,33 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf dataSize = TMIN(dataSize, 50); memcpy(pBuf, varDataVal(pData), dataSize); len += snprintf(dumpBuf + len, size - len, " %15s |", pBuf); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; } break; case TSDB_DATA_TYPE_NCHAR: { char* pData = colDataGetVarData(pColInfoData, j); int32_t dataSize = TMIN(sizeof(pBuf), varDataLen(pData)); memset(pBuf, 0, sizeof(pBuf)); - (void)taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf); + code = taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf); + if (code < 0) { + uError("func %s failed to convert to ucs charset since %s", __func__, tstrerror(code)); + goto _exit; + } len += snprintf(dumpBuf + len, size - len, " %15s |", pBuf); - if (len >= size - 1) return 0; + if (len >= size - 1) goto _exit; } break; } } len += snprintf(dumpBuf + len, size - len, "%d\n", j); - if (len >= size - 1) return code; + if (len >= size - 1) goto _exit; } len += snprintf(dumpBuf + len, size - len, "%s |end\n", flag); *pDataBuf = dumpBuf; + dumpBuf = NULL; +_exit: + if (dumpBuf) { + taosMemoryFree(dumpBuf); + } return code; } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 24e669577a..4b44e4af43 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -489,7 +489,7 @@ int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted, colVal = COL_VAL_VALUE(infos[iInfo].columnId, value); } if (taosArrayPush(colValArray, &colVal) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } } @@ -500,7 +500,7 @@ int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted, } if ((taosArrayPush(rowArray, &row)) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } } @@ -702,7 +702,7 @@ static int32_t tRowMergeImpl(SArray *aRowP, STSchema *pTSchema, int32_t iStart, if (pColVal) { if (taosArrayPush(aColVal, pColVal) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } } @@ -1756,7 +1756,7 @@ int32_t tTagToValArray(const STag *pTag, SArray **ppArray) { } (void)tGetTagVal(p + offset, &tv, pTag->flags & TD_TAG_JSON); if (taosArrayPush(*ppArray, &tv) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _err; } } @@ -3251,13 +3251,13 @@ int32_t tRowBuildFromBind2(SBindInfo2 *infos, int32_t numOfInfos, bool infoSorte } if ((bufArray = taosArrayInit(numOfInfos, sizeof(uint8_t *))) == NULL) { taosArrayDestroy(colValArray); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } for (int i = 0; i < numOfInfos; ++i) { if (!taosArrayPush(bufArray, &infos[i].bind->buffer)) { taosArrayDestroy(colValArray); taosArrayDestroy(bufArray); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } @@ -3294,7 +3294,7 @@ int32_t tRowBuildFromBind2(SBindInfo2 *infos, int32_t numOfInfos, bool infoSorte colVal = COL_VAL_VALUE(infos[iInfo].columnId, value); } if (taosArrayPush(colValArray, &colVal) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } } @@ -3305,7 +3305,7 @@ int32_t tRowBuildFromBind2(SBindInfo2 *infos, int32_t numOfInfos, bool infoSorte } if ((taosArrayPush(rowArray, &row)) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } } @@ -3502,7 +3502,7 @@ static int32_t tColDataMerge(SArray **colArr) { dst = taosArrayInit(taosArrayGetSize(src), sizeof(SColData)); if (dst == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } for (int32_t i = 0; i < taosArrayGetSize(src); i++) { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 5b67e1267b..810129e694 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -81,6 +81,7 @@ int32_t tsElectInterval = 25 * 1000; int32_t tsHeartbeatInterval = 1000; int32_t tsHeartbeatTimeout = 20 * 1000; int32_t tsSnapReplMaxWaitN = 128; +int64_t tsLogBufferMemoryAllowed = 0; // bytes // mnode int64_t tsMndSdbWriteDelta = 200; @@ -613,7 +614,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { cfgAddInt32(pCfg, "timeToGetAvailableConn", tsTimeToGetAvailableConn, 20, 1000000, CFG_SCOPE_BOTH, CFG_DYN_NONE)); tsKeepAliveIdle = TRANGE(tsKeepAliveIdle, 1, 72000); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "keepAliveIdle", tsKeepAliveIdle, 1, 7200000, CFG_SCOPE_BOTH, CFG_DYN_ENT_BOTH)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "keepAliveIdle", tsKeepAliveIdle, 1, 7200000, CFG_SCOPE_BOTH, CFG_DYN_NONE)); tsNumOfTaskQueueThreads = tsNumOfCores * 2; tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 16); @@ -702,6 +703,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsQueueMemoryAllowed = tsTotalMemoryKB * 1024 * 0.1; tsQueueMemoryAllowed = TRANGE(tsQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL); + tsLogBufferMemoryAllowed = tsTotalMemoryKB * 1024 * 0.1; + tsLogBufferMemoryAllowed = TRANGE(tsLogBufferMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL); + // clang-format off TAOS_CHECK_RETURN(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, CFG_SCOPE_SERVER, CFG_DYN_NONE)); @@ -736,6 +740,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, (TSDB_SYNC_SNAP_BUFFER_SIZE >> 2), CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "syncLogBufferMemoryAllowed", tsLogBufferMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "arbHeartBeatIntervalSec", tsArbHeartBeatIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "arbCheckSyncIntervalSec", tsArbCheckSyncIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); @@ -970,6 +975,14 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { pItem->stype = stype; } + pItem = cfgGetItem(tsCfg, "syncLogBufferMemoryAllowed"); + if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { + tsLogBufferMemoryAllowed = totalMemoryKB * 1024 * 0.1; + tsLogBufferMemoryAllowed = TRANGE(tsLogBufferMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL); + pItem->i64 = tsLogBufferMemoryAllowed; + pItem->stype = stype; + } + TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -1520,6 +1533,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "syncSnapReplMaxWaitN"); tsSnapReplMaxWaitN = pItem->i32; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "syncLogBufferMemoryAllowed"); + tsLogBufferMemoryAllowed = pItem->i64; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "arbHeartBeatIntervalSec"); tsArbHeartBeatIntervalSec = pItem->i32; @@ -1696,7 +1712,10 @@ int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char * SArray *pArgs) { int32_t code = TSDB_CODE_SUCCESS; - if (tsCfg == NULL) osDefaultInit(); + if (tsCfg == NULL) code = osDefaultInit(); + if (code != 0) { + (void)printf("failed to init os since %s\n", tstrerror(code)); + } SConfig *pCfg = NULL; TAOS_CHECK_RETURN(cfgInit(&pCfg)); @@ -1954,6 +1973,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"randErrorChance", &tsRandErrChance}, {"randErrorDivisor", &tsRandErrDivisor}, {"randErrorScope", &tsRandErrScope}, + {"syncLogBufferMemoryAllowed", &tsLogBufferMemoryAllowed}, {"cacheLazyLoadThreshold", &tsCacheLazyLoadThreshold}, {"checkpointInterval", &tsStreamCheckpointInterval}, diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c index e40db7b4cf..8de557a881 100644 --- a/source/common/src/tmisce.c +++ b/source/common/src/tmisce.c @@ -179,7 +179,7 @@ int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t cap) { int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime) { int32_t code = 0; SJson* pJson = tjsonCreateObject(); - if (pJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (pJson == NULL) return terrno; char tmp[4096] = {0}; @@ -241,7 +241,7 @@ int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t char* pCont = tjsonToString(pJson); if (pCont == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; TAOS_CHECK_GOTO(code, NULL, _exit); goto _exit; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 758a4aeec3..c1d943ab5c 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -10346,7 +10346,11 @@ int32_t tDecodeSMCreateStbRsp(SDecoder *pDecoder, SMCreateStbRsp *pRsp) { } tEndDecode(pDecoder); + return code; + _exit: + tFreeSTableMetaRsp(pRsp->pMeta); + taosMemoryFreeClear(pRsp->pMeta); return code; } @@ -12026,19 +12030,19 @@ int32_t tCloneTbTSMAInfo(STableTSMAInfo *pInfo, STableTSMAInfo **pRes) { *pRet = *pInfo; if (pInfo->pFuncs) { pRet->pFuncs = taosArrayDup(pInfo->pFuncs, NULL); - if (!pRet->pFuncs) code = TSDB_CODE_OUT_OF_MEMORY; + if (!pRet->pFuncs) code = terrno; } if (pInfo->pTags && code == TSDB_CODE_SUCCESS) { pRet->pTags = taosArrayDup(pInfo->pTags, NULL); - if (!pRet->pTags) code = TSDB_CODE_OUT_OF_MEMORY; + if (!pRet->pTags) code = terrno; } if (pInfo->pUsedCols && code == TSDB_CODE_SUCCESS) { pRet->pUsedCols = taosArrayDup(pInfo->pUsedCols, NULL); - if (!pRet->pUsedCols) code = TSDB_CODE_OUT_OF_MEMORY; + if (!pRet->pUsedCols) code = terrno; } if (pInfo->ast && code == TSDB_CODE_SUCCESS) { pRet->ast = taosStrdup(pInfo->ast); - if (!pRet->ast) code = TSDB_CODE_OUT_OF_MEMORY; + if (!pRet->ast) code = terrno; } if (code) { tFreeAndClearTableTSMAInfo(pRet); diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 4bb89464d0..b725514d4f 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -88,6 +88,9 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in #endif void toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName) { + if (pName == NULL){ + return; + } pName->type = TSDB_TABLE_NAME_T; pName->acctId = acctId; snprintf(pName->dbname, sizeof(pName->dbname), "%s", pDbName); @@ -152,7 +155,11 @@ int32_t tNameGetDbName(const SName* name, char* dst) { const char* tNameGetDbNameP(const SName* name) { return &name->dbname[0]; } int32_t tNameGetFullDbName(const SName* name, char* dst) { - return snprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname); + if (name == NULL || dst == NULL) { + return TSDB_CODE_INVALID_PARA; + } + (void)snprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname); + return 0; } bool tNameIsEmpty(const SName* name) { return name->type == 0 || name->acctId == 0; } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 760b86dee1..40b4f863cb 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -482,7 +482,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t r if (isAlloc) { taosMemoryFreeClear(*ppRow); } - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } } diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index d771830d3f..7e8749ef8b 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -1247,7 +1247,7 @@ static int32_t parseTsFormat(const char* formatStr, SArray* formats) { const TSFormatKeyWord* key = keywordSearch(formatStr); if (key) { TSFormatNode format = {.key = key, .type = TS_FORMAT_NODE_TYPE_KEYWORD}; - if (NULL == taosArrayPush(formats, &format)) TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + if (NULL == taosArrayPush(formats, &format)) TAOS_RETURN(terrno); formatStr += key->len; lastOtherFormat = NULL; } else { @@ -1274,7 +1274,7 @@ static int32_t parseTsFormat(const char* formatStr, SArray* formats) { TSFormatNode format = {.type = TS_FORMAT_NODE_TYPE_CHAR, .key = NULL}; format.c = formatStr; format.len = 1; - if (NULL == taosArrayPush(formats, &format)) TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + if (NULL == taosArrayPush(formats, &format)) TAOS_RETURN(terrno); formatStr++; last = taosArrayGetLast(formats); } @@ -1301,7 +1301,7 @@ static int32_t parseTsFormat(const char* formatStr, SArray* formats) { .key = NULL}; format.c = formatStr; format.len = 1; - if (NULL == taosArrayPush(formats, &format)) TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + if (NULL == taosArrayPush(formats, &format)) TAOS_RETURN(terrno); formatStr++; if (format.type == TS_FORMAT_NODE_TYPE_CHAR) lastOtherFormat = taosArrayGetLast(formats); } @@ -1910,7 +1910,7 @@ int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t pr if (!*formats) { *formats = taosArrayInit(8, sizeof(TSFormatNode)); if (!*formats) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } TAOS_CHECK_RETURN(parseTsFormat(format, *formats)); } @@ -1926,7 +1926,7 @@ int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int if (!*formats) { *formats = taosArrayInit(4, sizeof(TSFormatNode)); if (!*formats) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } TAOS_CHECK_RETURN(parseTsFormat(format, *formats)); } @@ -1951,7 +1951,7 @@ int32_t TEST_ts2char(const char* format, int64_t ts, int32_t precision, char* ou SArray* formats = taosArrayInit(4, sizeof(TSFormatNode)); if (!formats) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } TAOS_CHECK_RETURN(parseTsFormat(format, formats)); struct STm tm; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 9c22a11674..f9ae94c53f 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -345,7 +345,7 @@ int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { void *pRsp = rpcMallocCont(rspLen); if (pRsp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; // rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; // return rspMsg.code; } @@ -383,7 +383,7 @@ int32_t dmBuildVariablesBlock(SSDataBlock **ppBlock) { pBlock->pDataBlock = taosArrayInit(pMeta[index].colNum, sizeof(SColumnInfoData)); if (pBlock->pDataBlock == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } @@ -393,7 +393,7 @@ int32_t dmBuildVariablesBlock(SSDataBlock **ppBlock) { colInfoData.info.type = pMeta[index].schema[i].type; colInfoData.info.bytes = pMeta[index].schema[i].bytes; if (taosArrayPush(pBlock->pDataBlock, &colInfoData) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } } @@ -457,7 +457,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); if (pRsp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; dError("failed to retrieve data since %s", tstrerror(code)); blockDataDestroy(pBlock); return code; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index d7170398fb..1561ab0a6b 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -54,7 +54,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { int32_t code = 0; SDnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SDnodeMgmt)); if (pMgmt == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pMgmt->pData = pInput->pData; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index 073ba293e2..a95ec42f7e 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -91,7 +91,7 @@ int32_t mmReadFile(const char *path, SMnodeOpt *pOption) { pData = taosMemoryMalloc(size + 1); if (pData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -134,14 +134,14 @@ static int32_t mmEncodeOption(SJson *pJson, const SMnodeOpt *pOption) { SJson *replicas = tjsonCreateArray(); if (replicas == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if ((code = tjsonAddItemToObject(pJson, "replicas", replicas)) < 0) return code; for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) { SJson *replica = tjsonCreateObject(); if (replica == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } const SReplica *pReplica = pOption->replicas + i; @@ -183,7 +183,7 @@ int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) { // terrno = TSDB_CODE_OUT_OF_MEMORY; pJson = tjsonCreateObject(); if (pJson == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index ad602313e0..e8594130d6 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -71,14 +71,14 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) { int32_t code = 0; pMgmt->writeWroker = taosArrayInit(0, sizeof(SMultiWorker *)); if (pMgmt->writeWroker == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; return code; } for (int32_t i = 0; i < tsNumOfSnodeWriteThreads; i++) { SMultiWorker *pWriteWorker = taosMemoryMalloc(sizeof(SMultiWorker)); if (pWriteWorker == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; return code; } @@ -93,7 +93,7 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) { return code; } if (taosArrayPush(pMgmt->writeWroker, &pWriteWorker) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; return code; } } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index 2f93d51ea2..5fabd4cdde 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -128,7 +128,7 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t pData = taosMemoryMalloc(size + 1); if (pData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -169,7 +169,7 @@ static int32_t vmEncodeVnodeList(SJson *pJson, SVnodeObj **ppVnodes, int32_t num int32_t code = 0; SJson *vnodes = tjsonCreateArray(); if (vnodes == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if ((code = tjsonAddItemToObject(pJson, "vnodes", vnodes)) < 0) { tjsonDelete(vnodes); @@ -181,7 +181,7 @@ static int32_t vmEncodeVnodeList(SJson *pJson, SVnodeObj **ppVnodes, int32_t num if (pVnode == NULL) continue; SJson *vnode = tjsonCreateObject(); - if (vnode == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (vnode == NULL) return terrno; if ((code = tjsonAddDoubleToObject(vnode, "vgId", pVnode->vgId)) < 0) return code; if ((code = tjsonAddDoubleToObject(vnode, "dropped", pVnode->dropped)) < 0) return code; if ((code = tjsonAddDoubleToObject(vnode, "vgVersion", pVnode->vgVersion)) < 0) return code; @@ -221,7 +221,7 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) { // terrno = TSDB_CODE_OUT_OF_MEMORY; pJson = tjsonCreateObject(); if (pJson == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } if ((code = vmEncodeVnodeList(pJson, ppVnodes, numOfVnodes)) != 0) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 2abf292e73..f020ba2f5e 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -921,7 +921,7 @@ int32_t vmProcessArbHeartBeatReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { void *pRsp = rpcMallocCont(rspLen); if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + terrno = terrno; goto _OVER; } diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index fdce9fd4c9..ba0a40e048 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -161,8 +161,7 @@ int32_t dmInitVars(SDnode *pDnode) { pData->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); if (pData->dnodeHash == NULL) { dError("failed to init dnode hash"); - code = TSDB_CODE_OUT_OF_MEMORY; - return terrno = code; + return terrno; } if ((code = dmReadEps(pData)) != 0) { diff --git a/source/dnode/mgmt/node_util/src/dmEps.c b/source/dnode/mgmt/node_util/src/dmEps.c index 982c139c7b..f8f2b3dfca 100644 --- a/source/dnode/mgmt/node_util/src/dmEps.c +++ b/source/dnode/mgmt/node_util/src/dmEps.c @@ -217,7 +217,7 @@ int32_t dmReadEps(SDnodeData *pData) { content = taosMemoryMalloc(size + 1); if (content == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -321,7 +321,7 @@ int32_t dmWriteEps(SDnodeData *pData) { TAOS_CHECK_GOTO(dmInitDndInfo(pData), NULL, _OVER); pJson = tjsonCreateObject(); - if (pJson == NULL) TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER); + if (pJson == NULL) TAOS_CHECK_GOTO(terrno, NULL, _OVER); pData->engineVer = tsVersion; @@ -329,7 +329,7 @@ int32_t dmWriteEps(SDnodeData *pData) { buffer = tjsonToString(pJson); if (buffer == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER); + TAOS_CHECK_GOTO(terrno, NULL, _OVER); } pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); @@ -574,7 +574,7 @@ static int32_t dmDecodeEpPairs(SJson *pJson, SDnodeData *pData) { tjsonGetUInt16ValueFromDouble(dnode, "new_port", pair.newPort, code); if (code < 0) return TSDB_CODE_INVALID_CFG_VALUE; - if (taosArrayPush(pData->oldDnodeEps, &pair) == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (taosArrayPush(pData->oldDnodeEps, &pair) == NULL) return terrno; } return code; @@ -620,7 +620,7 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) { content = taosMemoryMalloc(size + 1); if (content == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -640,7 +640,7 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) { pData->oldDnodeEps = taosArrayInit(1, sizeof(SDnodeEpPair)); if (pData->oldDnodeEps == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; dError("failed to calloc dnodeEp array since %s", strerror(errno)); goto _OVER; } diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index 61a72f2aeb..14154d1a23 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -70,7 +70,7 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) { content = taosMemoryMalloc(size + 1); if (content == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -134,7 +134,7 @@ int32_t dmWriteFile(const char *path, const char *name, bool deployed) { pJson = tjsonCreateObject(); if (pJson == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -226,7 +226,7 @@ static int32_t dmWriteCheckCodeFile(char *file, char *realfile, char *key, bool int32_t len = ENCRYPTED_LEN(sizeof(DM_KEY_INDICATOR)); result = taosMemoryMalloc(len); if (result == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } SCryptOpts opts; @@ -469,7 +469,7 @@ static int32_t dmReadEncryptCodeFile(char *file, char **output) { content = taosMemoryMalloc(size + 1); if (content == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } diff --git a/source/dnode/mnode/impl/inc/mndShow.h b/source/dnode/mnode/impl/inc/mndShow.h index 25268c52bb..26d47d6130 100644 --- a/source/dnode/mnode/impl/inc/mndShow.h +++ b/source/dnode/mnode/impl/inc/mndShow.h @@ -32,6 +32,19 @@ extern "C" { } \ } while (0) +#define RETRIEVE_CHECK_GOTO(CMD, pObj, LINO, LABEL) \ + do { \ + code = (CMD); \ + if (code != TSDB_CODE_SUCCESS) { \ + if (LINO) { \ + *((int32_t *)(LINO)) = __LINE__; \ + } \ + if (pObj) sdbRelease(pSdb, (pObj)); \ + if (pObj) sdbCancelFetch(pSdb, (pObj)); \ + goto LABEL; \ + } \ + } while (0) + int32_t mndInitShow(SMnode *pMnode); void mndCleanupShow(SMnode *pMnode); void mndAddShowRetrieveHandle(SMnode *pMnode, EShowType showType, ShowRetrieveFp fp); diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index db960d790f..bc97d4d4f9 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -37,9 +37,9 @@ int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p void mndFreeStb(SStbObj *pStb); int32_t mndBuildSMCreateStbRsp(SMnode *pMnode, char *dbFName, char *stbFName, void **pCont, int32_t *pLen); -void mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst); -void mndExtractShortDbNameFromStbFullName(const char *stbFullName, char *dst); -void mndExtractShortDbNameFromDbFullName(const char *stbFullName, char *dst); +int32_t mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst); +int32_t mndExtractShortDbNameFromStbFullName(const char *stbFullName, char *dst); +int32_t mndExtractShortDbNameFromDbFullName(const char *stbFullName, char *dst); void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t dstSize); const char *mndGetStbStr(const char *src); diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c index bce8c5901d..82824eb6e8 100644 --- a/source/dnode/mnode/impl/src/mndCompact.c +++ b/source/dnode/mnode/impl/src/mndCompact.c @@ -268,7 +268,11 @@ int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj *pCompac sdbFreeRaw(pVgRaw); TAOS_RETURN(code); } - (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); + + if ((code = sdbSetRawStatus(pVgRaw, SDB_STATUS_READY)) != 0) { + sdbFreeRaw(pVgRaw); + TAOS_RETURN(code); + } rsp->compactId = pCompact->compactId; @@ -357,7 +361,10 @@ static void *mndBuildKillCompactReq(SMnode *pMnode, SVgObj *pVgroup, int32_t *pC pHead->contLen = htonl(contLen); pHead->vgId = htonl(pVgroup->vgId); - (void)tSerializeSVKillCompactReq((char *)pReq + sizeof(SMsgHead), contLen, &req); + if ((contLen = tSerializeSVKillCompactReq((char *)pReq + sizeof(SMsgHead), contLen, &req)) < 0) { + terrno = contLen; + return NULL; + } *pContLen = contLen; return pReq; } @@ -421,7 +428,10 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa mndTransDrop(pTrans); TAOS_RETURN(code); } - (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + if ((code = sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)) != 0) { + mndTransDrop(pTrans); + TAOS_RETURN(code); + } void *pIter = NULL; while (1) { @@ -614,7 +624,10 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) { pHead->contLen = htonl(contLen); pHead->vgId = htonl(pDetail->vgId); - (void)tSerializeSQueryCompactProgressReq((char *)pHead + sizeof(SMsgHead), contLen - sizeof(SMsgHead), &req); + if (tSerializeSQueryCompactProgressReq((char *)pHead + sizeof(SMsgHead), contLen - sizeof(SMsgHead), &req) <= 0) { + sdbRelease(pMnode->pSdb, pDetail); + continue; + } SRpcMsg rpcMsg = {.msgType = TDMT_VND_QUERY_COMPACT_PROGRESS, .contLen = contLen}; @@ -719,7 +732,12 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) { mndTransDrop(pTrans); TAOS_RETURN(code); } - (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + if ((code = sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)) != 0) { + sdbCancelFetch(pMnode->pSdb, pIter); + sdbRelease(pMnode->pSdb, pDetail); + mndTransDrop(pTrans); + TAOS_RETURN(code); + } } sdbRelease(pMnode->pSdb, pDetail); @@ -782,7 +800,12 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) { mndTransDrop(pTrans); TAOS_RETURN(code); } - (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + if ((code = sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED)) != 0) { + sdbCancelFetch(pMnode->pSdb, pIter); + sdbRelease(pMnode->pSdb, pDetail); + mndTransDrop(pTrans); + TAOS_RETURN(code); + } mInfo("compact:%d, add drop compactdetail action", pDetail->compactDetailId); } @@ -809,7 +832,11 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) { mndTransDrop(pTrans); TAOS_RETURN(code); } - (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + if ((code = sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED)) != 0) { + mError("compact:%d, trans:%d, failed to append commit log since %s", compactId, pTrans->id, terrstr()); + mndTransDrop(pTrans); + TAOS_RETURN(code); + } mInfo("compact:%d, add drop compact action", pCompact->compactId); } diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index f71ab95d03..9704ccaa83 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -199,7 +199,7 @@ static void storeOffsetRows(SMnode *pMnode, SMqHbReq *req, SMqConsumerObj *pCons taosWLockLatch(&pSub->lock); SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &pConsumer->consumerId, sizeof(int64_t)); if (pConsumerEp) { - (void)taosArrayDestroy(pConsumerEp->offsetRows); + taosArrayDestroy(pConsumerEp->offsetRows); pConsumerEp->offsetRows = data->offsetRows; data->offsetRows = NULL; } @@ -511,6 +511,11 @@ static int32_t getTopicAddDelete(SMqConsumerObj *pExistedConsumer, SMqConsumerOb } } } + // no topics need to be rebalanced + if (taosArrayGetSize(pConsumerNew->rebNewTopics) == 0 && taosArrayGetSize(pConsumerNew->rebRemovedTopics) == 0) { + code = TSDB_CODE_TMQ_NO_NEED_REBALANCE; + } + END: return code; } @@ -581,6 +586,10 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { if(ubSubscribe){ SMqConsumerObj *pConsumerTmp = NULL; MND_TMQ_RETURN_CHECK(mndAcquireConsumer(pMnode, subscribe.consumerId, &pConsumerTmp)); + if (taosArrayGetSize(pConsumerTmp->assignedTopics) == 0){ + mndReleaseConsumer(pMnode, pConsumerTmp); + goto END; + } mndReleaseConsumer(pMnode, pConsumerTmp); } MND_TMQ_RETURN_CHECK(checkAndSortTopic(pMnode, subscribe.topicNames)); @@ -599,7 +608,7 @@ END: mndTransDrop(pTrans); tDeleteSMqConsumerObj(pConsumerNew); taosArrayDestroyP(subscribe.topicNames, (FDelete)taosMemoryFree); - return code; + return (code == TSDB_CODE_TMQ_NO_NEED_REBALANCE || code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) ? 0 : code; } SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index a432df60a5..b73d4b61a9 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -182,7 +182,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { goto _OVER; } TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pRaw), NULL, _OVER); - (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); + TAOS_CHECK_GOTO(sdbSetRawStatus(pRaw, SDB_STATUS_READY), NULL, _OVER); pRaw = NULL; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); @@ -621,7 +621,10 @@ static int32_t mndUpdateDnodeObj(SMnode *pMnode, SDnodeObj *pDnode) { TAOS_RETURN(terrno); } - (void)tSerializeSDnodeInfoReq(pReq, contLen, &infoReq); + if ((contLen = tSerializeSDnodeInfoReq(pReq, contLen, &infoReq)) <= 0) { + code = contLen; + goto _exit; + } SRpcMsg rpcMsg = {.msgType = TDMT_MND_UPDATE_DNODE_INFO, .pCont = pReq, .contLen = contLen}; TAOS_CHECK_EXIT(tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg)); @@ -661,7 +664,7 @@ static int32_t mndProcessUpdateDnodeInfoReq(SRpcMsg *pReq) { mError("trans:%d, failed to append commit log since %s", pTrans->id, tstrerror(code)); TAOS_CHECK_EXIT(code); } - (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + TAOS_CHECK_EXIT(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)); pCommitRaw = NULL; if ((code = mndTransPrepare(pMnode, pTrans)) != 0) { @@ -874,8 +877,12 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { int32_t contLen = tSerializeSStatusRsp(NULL, 0, &statusRsp); void *pHead = rpcMallocCont(contLen); - (void)tSerializeSStatusRsp(pHead, contLen, &statusRsp); + contLen = tSerializeSStatusRsp(pHead, contLen, &statusRsp); taosArrayDestroy(statusRsp.pDnodeEps); + if (contLen < 0) { + code = contLen; + goto _OVER; + } pReq->info.rspLen = contLen; pReq->info.rsp = pHead; @@ -888,8 +895,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { _OVER: mndReleaseDnode(pMnode, pDnode); taosArrayDestroy(statusReq.pVloads); - (void)mndUpdClusterInfo(pReq); - return code; + return mndUpdClusterInfo(pReq); } static int32_t mndProcessNotifyReq(SRpcMsg *pReq) { @@ -955,7 +961,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC goto _OVER; } TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pRaw), NULL, _OVER); - (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); + TAOS_CHECK_GOTO(sdbSetRawStatus(pRaw, SDB_STATUS_READY), NULL, _OVER); pRaw = NULL; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); @@ -1010,7 +1016,10 @@ static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) { goto _OVER; } - (void)tSerializeSDnodeListRsp(pRsp, rspLen, &rsp); + if ((rspLen = tSerializeSDnodeListRsp(pRsp, rspLen, &rsp)) <= 0) { + code = rspLen; + goto _OVER; + } pReq->info.rspLen = rspLen; pReq->info.rsp = pRsp; @@ -1151,7 +1160,10 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { goto _OVER; } - (void)tSerializeSShowVariablesRsp(pRsp, rspLen, &rsp); + if ((rspLen = tSerializeSShowVariablesRsp(pRsp, rspLen, &rsp)) <= 0) { + code = rspLen; + goto _OVER; + } pReq->info.rspLen = rspLen; pReq->info.rsp = pRsp; @@ -1247,7 +1259,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM goto _OVER; } TAOS_CHECK_GOTO(mndTransAppendRedolog(pTrans, pRaw), NULL, _OVER); - (void)sdbSetRawStatus(pRaw, SDB_STATUS_DROPPING); + TAOS_CHECK_GOTO(sdbSetRawStatus(pRaw, SDB_STATUS_DROPPING), NULL, _OVER); pRaw = NULL; pRaw = mndDnodeActionEncode(pDnode); @@ -1257,7 +1269,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM goto _OVER; } TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pRaw), NULL, _OVER); - (void)sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); + TAOS_CHECK_GOTO(sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED), NULL, _OVER); pRaw = NULL; if (pMObj != NULL) { @@ -1450,7 +1462,10 @@ static int32_t mndSendCfgDnodeReq(SMnode *pMnode, int32_t dnodeId, SDCfgDnodeReq void *pBuf = rpcMallocCont(bufLen); if (pBuf != NULL) { - (void)tSerializeSDCfgDnodeReq(pBuf, bufLen, pDcfgReq); + if ((bufLen = tSerializeSDCfgDnodeReq(pBuf, bufLen, pDcfgReq)) <= 0) { + code = bufLen; + return code; + } mInfo("dnode:%d, send config req to dnode, config:%s value:%s", dnodeId, pDcfgReq->config, pDcfgReq->value); SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen}; code = tmsgSendReq(&epSet, &rpcMsg); @@ -1590,7 +1605,11 @@ static int32_t mndProcessCreateEncryptKeyReqImpl(SRpcMsg *pReq, int32_t dnodeId, void *pBuf = rpcMallocCont(bufLen); if (pBuf != NULL) { - (void)tSerializeSDCfgDnodeReq(pBuf, bufLen, pDcfgReq); + if ((bufLen = tSerializeSDCfgDnodeReq(pBuf, bufLen, pDcfgReq)) <= 0) { + code = bufLen; + sdbRelease(pSdb, pDnode); + goto _exit; + } SRpcMsg rpcMsg = {.msgType = TDMT_DND_CREATE_ENCRYPT_KEY, .pCont = pBuf, .contLen = bufLen}; if (0 == tmsgSendReq(&epSet, &rpcMsg)) { (void)atomic_add_fetch_16(&pMnode->encryptMgmt.nEncrypt, 1); diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 9a617fb894..9e97aa674b 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -375,7 +375,7 @@ static int32_t mndDropFunc(SMnode *pMnode, SRpcMsg *pReq, SFuncObj *pFunc) { goto _OVER; } TAOS_CHECK_GOTO(mndTransAppendRedolog(pTrans, pRedoRaw), NULL, _OVER); - (void)sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); + TAOS_CHECK_GOTO(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING), NULL, _OVER); SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc); if (pUndoRaw == NULL) { @@ -384,7 +384,7 @@ static int32_t mndDropFunc(SMnode *pMnode, SRpcMsg *pReq, SFuncObj *pFunc) { goto _OVER; } TAOS_CHECK_GOTO(mndTransAppendUndolog(pTrans, pUndoRaw), NULL, _OVER); - (void)sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); + TAOS_CHECK_GOTO(sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY), NULL, _OVER); SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc); if (pCommitRaw == NULL) { @@ -393,7 +393,7 @@ static int32_t mndDropFunc(SMnode *pMnode, SRpcMsg *pReq, SFuncObj *pFunc) { goto _OVER; } TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pCommitRaw), NULL, _OVER); - (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + TAOS_CHECK_GOTO(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED), NULL, _OVER); TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); @@ -598,7 +598,10 @@ static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) { goto RETRIEVE_FUNC_OVER; } - (void)tSerializeSRetrieveFuncRsp(pRsp, contLen, &retrieveRsp); + if ((contLen = tSerializeSRetrieveFuncRsp(pRsp, contLen, &retrieveRsp)) <= 0) { + code = contLen; + goto RETRIEVE_FUNC_OVER; + } pReq->info.rsp = pRsp; pReq->info.rspLen = contLen; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index f4c5e9f2a0..5d91b0b0d8 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -91,7 +91,7 @@ int32_t mndInitProfile(SMnode *pMnode) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; // in ms - int32_t checkTime = tsShellActivityTimer * 2 * 1000; + int32_t checkTime = CACHE_OBJ_KEEP_TIME * 1000; pMgmt->connCache = taosCacheInit(TSDB_DATA_TYPE_UINT, checkTime, false, (__cache_free_fn_t)mndFreeConn, "conn"); if (pMgmt->connCache == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index d46726fd31..915741ec49 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -482,7 +482,7 @@ SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) { if ((terrno = tNameFromString(&name, stbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE)) != 0) return NULL; char db[TSDB_TABLE_FNAME_LEN] = {0}; - (void)tNameGetFullDbName(&name, db); + if ((terrno = tNameGetFullDbName(&name, db)) != 0) return NULL; return mndAcquireDb(pMnode, db); } @@ -507,7 +507,9 @@ void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int3 goto _err; } char dbFName[TSDB_DB_FNAME_LEN] = {0}; - (void)tNameGetFullDbName(&name, dbFName); + if ((terrno = tNameGetFullDbName(&name, dbFName)) != 0) { + goto _err; + }; req.name = (char *)tNameGetTableName(&name); req.suid = pStb->uid; @@ -625,8 +627,9 @@ static void *mndBuildVDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead)); - (void)tEncodeSVDropStbReq(&encoder, &req); + terrno = tEncodeSVDropStbReq(&encoder, &req); tEncoderClear(&encoder); + if (terrno != 0) return NULL; *pContLen = contLen; return pHead; @@ -1003,7 +1006,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCrea _OVER: mndTransDrop(pTrans); - (void)mndStbActionDelete(pMnode->pSdb, &stbObj); + if (mndStbActionDelete(pMnode->pSdb, &stbObj) != 0) mError("failed to mndStbActionDelete"); TAOS_RETURN(code); } @@ -1032,6 +1035,7 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) { pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; + int32_t code = 0; SMsgHead *pHead = rpcMallocCont(contLen); if (pHead == NULL) { sdbRelease(pSdb, pVgroup); @@ -1039,12 +1043,16 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) { } pHead->contLen = htonl(contLen); pHead->vgId = htonl(pVgroup->vgId); - (void)tSerializeSVDropTtlTableReq((char *)pHead + sizeof(SMsgHead), reqLen, &ttlReq); + if ((code = tSerializeSVDropTtlTableReq((char *)pHead + sizeof(SMsgHead), reqLen, &ttlReq)) < 0) { + mError("vgId:%d, failed to serialize drop ttl table request since %s", pVgroup->vgId, tstrerror(code)); + sdbRelease(pSdb, pVgroup); + continue; + } SRpcMsg rpcMsg = { .msgType = TDMT_VND_FETCH_TTL_EXPIRED_TBS, .pCont = pHead, .contLen = contLen, .info = pReq->info}; SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup); - int32_t code = tmsgSendReq(&epSet, &rpcMsg); + code = tmsgSendReq(&epSet, &rpcMsg); if (code != 0) { mError("vgId:%d, failed to send drop ttl table request to vnode since 0x%x", pVgroup->vgId, code); } else { @@ -1069,6 +1077,8 @@ static int32_t mndProcessTrimDbTimer(SRpcMsg *pReq) { pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; + int32_t code = 0; + SMsgHead *pHead = rpcMallocCont(contLen); if (pHead == NULL) { sdbCancelFetch(pSdb, pVgroup); @@ -1077,11 +1087,13 @@ static int32_t mndProcessTrimDbTimer(SRpcMsg *pReq) { } pHead->contLen = htonl(contLen); pHead->vgId = htonl(pVgroup->vgId); - (void)tSerializeSVTrimDbReq((char *)pHead + sizeof(SMsgHead), reqLen, &trimReq); + if ((code = tSerializeSVTrimDbReq((char *)pHead + sizeof(SMsgHead), reqLen, &trimReq)) < 0) { + mError("vgId:%d, failed to serialize trim db request since %s", pVgroup->vgId, tstrerror(code)); + } SRpcMsg rpcMsg = {.msgType = TDMT_VND_TRIM, .pCont = pHead, .contLen = contLen}; SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup); - int32_t code = tmsgSendReq(&epSet, &rpcMsg); + code = tmsgSendReq(&epSet, &rpcMsg); if (code != 0) { mError("vgId:%d, timer failed to send vnode-trim request to vnode since 0x%x", pVgroup->vgId, code); } else { @@ -1106,19 +1118,24 @@ static int32_t mndProcessS3MigrateDbTimer(SRpcMsg *pReq) { pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; + int32_t code = 0; + SMsgHead *pHead = rpcMallocCont(contLen); if (pHead == NULL) { - sdbCancelFetch(pSdb, pVgroup); sdbRelease(pSdb, pVgroup); continue; } pHead->contLen = htonl(contLen); pHead->vgId = htonl(pVgroup->vgId); - (void)tSerializeSVS3MigrateDbReq((char *)pHead + sizeof(SMsgHead), reqLen, &s3migrateReq); + if ((code = tSerializeSVS3MigrateDbReq((char *)pHead + sizeof(SMsgHead), reqLen, &s3migrateReq)) < 0) { + mError("vgId:%d, failed to serialize s3migrate db request since %s", pVgroup->vgId, tstrerror(code)); + sdbRelease(pSdb, pVgroup); + continue; + } SRpcMsg rpcMsg = {.msgType = TDMT_VND_S3MIGRATE, .pCont = pHead, .contLen = contLen}; SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup); - int32_t code = tmsgSendReq(&epSet, &rpcMsg); + code = tmsgSendReq(&epSet, &rpcMsg); if (code != 0) { mError("vgId:%d, timer failed to send vnode-s3migrate request to vnode since 0x%x", pVgroup->vgId, code); } else { @@ -2336,11 +2353,13 @@ static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, SStbObj *pObj, void **pCont, i void *cont = taosMemoryMalloc(contLen); tEncoderInit(&ec, cont, contLen); - (void)tEncodeSMAlterStbRsp(&ec, &alterRsp); + code = tEncodeSMAlterStbRsp(&ec, &alterRsp); tEncoderClear(&ec); tFreeSMAlterStbRsp(&alterRsp); + if (code < 0) TAOS_RETURN(code); + *pCont = cont; *pLen = contLen; @@ -2389,7 +2408,7 @@ int32_t mndBuildSMCreateStbRsp(SMnode *pMnode, char *dbFName, char *stbFName, vo void *cont = taosMemoryMalloc(contLen); tEncoderInit(&ec, cont, contLen); - (void)tEncodeSMCreateStbRsp(&ec, &stbRsp); + TAOS_CHECK_GOTO(tEncodeSMCreateStbRsp(&ec, &stbRsp), NULL, _OVER); tEncoderClear(&ec); tFreeSMCreateStbRsp(&stbRsp); @@ -2628,8 +2647,9 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) { if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; SName name = {0}; - // TODO check return value - (void)tNameFromString(&name, alterReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + int32_t ret = 0; + if ((ret = tNameFromString(&name, alterReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE)) != 0) + mError("stb:%s, failed to tNameFromString since %s", alterReq.name, tstrerror(ret)); auditRecord(pReq, pMnode->clusterId, "alterStb", name.dbname, name.tname, alterReq.sql, alterReq.sqlLen); @@ -2916,8 +2936,9 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; SName name = {0}; - // TODO check return value - (void)tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + int32_t ret = 0; + if ((ret = tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE)) != 0) + mError("stb:%s, failed to tNameFromString since %s", dropReq.name, tstrerror(ret)); auditRecord(pReq, pMnode->clusterId, "dropStb", name.dbname, name.tname, dropReq.sql, dropReq.sqlLen); @@ -2968,7 +2989,10 @@ static int32_t mndProcessTableMetaReq(SRpcMsg *pReq) { goto _OVER; } - (void)tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp); + if ((rspLen = tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp)) < 0) { + code = rspLen; + goto _OVER; + } pReq->info.rsp = pRsp; pReq->info.rspLen = rspLen; code = 0; @@ -2995,7 +3019,7 @@ static int32_t mndProcessTableCfgReq(SRpcMsg *pReq) { TAOS_CHECK_GOTO(tDeserializeSTableCfgReq(pReq->pCont, pReq->contLen, &cfgReq), NULL, _OVER); char dbName[TSDB_DB_NAME_LEN] = {0}; - mndExtractShortDbNameFromDbFullName(cfgReq.dbFName, dbName); + TAOS_CHECK_GOTO(mndExtractShortDbNameFromDbFullName(cfgReq.dbFName, dbName), NULL, _OVER); if (0 == strcmp(dbName, TSDB_INFORMATION_SCHEMA_DB)) { mInfo("information_schema table:%s.%s, start to retrieve cfg", cfgReq.dbFName, cfgReq.tbName); TAOS_CHECK_GOTO(mndBuildInsTableCfg(pMnode, cfgReq.dbFName, cfgReq.tbName, &cfgRsp), NULL, _OVER); @@ -3019,7 +3043,10 @@ static int32_t mndProcessTableCfgReq(SRpcMsg *pReq) { goto _OVER; } - (void)tSerializeSTableCfgRsp(pRsp, rspLen, &cfgRsp); + if ((rspLen = tSerializeSTableCfgRsp(pRsp, rspLen, &cfgRsp)) < 0) { + code = rspLen; + goto _OVER; + } pReq->info.rsp = pRsp; pReq->info.rspLen = rspLen; code = 0; @@ -3140,8 +3167,9 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t TAOS_RETURN(code); } - (void)tSerializeSSTbHbRsp(pRsp, rspLen, &hbRsp); + rspLen = tSerializeSSTbHbRsp(pRsp, rspLen, &hbRsp); tFreeSSTbHbRsp(&hbRsp); + if (rspLen < 0) return rspLen; *ppRsp = pRsp; *pRspLen = rspLen; TAOS_RETURN(code); @@ -3175,25 +3203,31 @@ int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs) { TAOS_RETURN(code); } -void mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst) { +int32_t mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst) { SName name = {0}; - (void)tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + TAOS_CHECK_RETURN(tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE)); - (void)tNameGetFullDbName(&name, dst); + TAOS_CHECK_RETURN(tNameGetFullDbName(&name, dst)); + + return 0; } -void mndExtractShortDbNameFromStbFullName(const char *stbFullName, char *dst) { +int32_t mndExtractShortDbNameFromStbFullName(const char *stbFullName, char *dst) { SName name = {0}; - (void)tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + TAOS_CHECK_RETURN(tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE)); - (void)tNameGetDbName(&name, dst); + TAOS_CHECK_RETURN(tNameGetDbName(&name, dst)); + + return 0; } -void mndExtractShortDbNameFromDbFullName(const char *stbFullName, char *dst) { +int32_t mndExtractShortDbNameFromDbFullName(const char *stbFullName, char *dst) { SName name = {0}; - (void)tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB); + TAOS_CHECK_RETURN(tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB)); - (void)tNameGetDbName(&name, dst); + TAOS_CHECK_RETURN(tNameGetDbName(&name, dst)); + + return 0; } void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t dstSize) { @@ -3491,6 +3525,8 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc int32_t numOfRows = 0; SStbObj *pStb = NULL; int32_t cols = 0; + int32_t lino = 0; + int32_t code = 0; SDbObj *pDb = NULL; if (strlen(pShow->db) > 0) { @@ -3515,42 +3551,44 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc cols = 0; SName name = {0}; + char stbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; mndExtractTbNameFromStbFullName(pStb->name, &stbName[VARSTR_HEADER_SIZE], TSDB_TABLE_NAME_LEN); varDataSetLen(stbName, strlen(&stbName[VARSTR_HEADER_SIZE])); - SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)stbName, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)stbName, false), pStb, &lino, _ERROR); char db[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - (void)tNameFromString(&name, pStb->db, T_NAME_ACCT | T_NAME_DB); - (void)tNameGetDbName(&name, varDataVal(db)); + RETRIEVE_CHECK_GOTO(tNameFromString(&name, pStb->db, T_NAME_ACCT | T_NAME_DB), pStb, &lino, _ERROR); + RETRIEVE_CHECK_GOTO(tNameGetDbName(&name, varDataVal(db)), pStb, &lino, _ERROR); varDataSetLen(db, strlen(varDataVal(db))); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)db, false), pStb, &lino, _ERROR); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)db, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->createdTime, false), pStb, &lino, + _ERROR); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->createdTime, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->numOfColumns, false), pStb, &lino, + _ERROR); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->numOfColumns, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->numOfTags, false), pStb, &lino, _ERROR); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->numOfTags, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->updateTime, false); // number of tables + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->updateTime, false), pStb, &lino, + _ERROR); // number of tables pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); if (pStb->commentLen > 0) { char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(comment, pStb->comment); - (void)colDataSetVal(pColInfo, numOfRows, comment, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, comment, false), pStb, &lino, _ERROR); } else if (pStb->commentLen == 0) { char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(comment, ""); - (void)colDataSetVal(pColInfo, numOfRows, comment, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, comment, false), pStb, &lino, _ERROR); } else { colDataSetNULL(pColInfo, numOfRows); } @@ -3560,14 +3598,14 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc varDataSetLen(watermark, strlen(varDataVal(watermark))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)watermark, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)watermark, false), pStb, &lino, _ERROR); char maxDelay[64 + VARSTR_HEADER_SIZE] = {0}; sprintf(varDataVal(maxDelay), "%" PRId64 "a,%" PRId64 "a", pStb->maxdelay[0], pStb->maxdelay[1]); varDataSetLen(maxDelay, strlen(varDataVal(maxDelay))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)maxDelay, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)maxDelay, false), pStb, &lino, _ERROR); char rollup[160 + VARSTR_HEADER_SIZE] = {0}; int32_t rollupNum = (int32_t)taosArrayGetSize(pStb->pFuncs); @@ -3586,7 +3624,7 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc varDataSetLen(rollup, strlen(varDataVal(rollup))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)rollup, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)rollup, false), pStb, &lino, _ERROR); numOfRows++; sdbRelease(pSdb, pStb); @@ -3596,6 +3634,13 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc mndReleaseDb(pMnode, pDb); } + goto _OVER; + +_ERROR: + mError("show:0x%" PRIx64 ", failed to retrieve data at %s:%d since %s", pShow->id, __FUNCTION__, lino, + tstrerror(code)); + +_OVER: pShow->numOfRows += numOfRows; return numOfRows; } @@ -3606,6 +3651,8 @@ static int32_t buildDbColsInfoBlock(const SSDataBlock *p, const SSysTableMeta *p char dName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t numOfRows = p->info.rows; + int32_t lino = 0; + int32_t code = 0; STR_TO_VARSTR(dName, dbName); STR_TO_VARSTR(typeName, "SYSTEM_TABLE"); @@ -3624,20 +3671,20 @@ static int32_t buildDbColsInfoBlock(const SSDataBlock *p, const SSysTableMeta *p for (int32_t j = 0; j < pm->colNum; j++) { // table name SColumnInfoData *pColInfoData = taosArrayGet(p->pDataBlock, 0); - (void)colDataSetVal(pColInfoData, numOfRows, tName, false); + TAOS_CHECK_GOTO(colDataSetVal(pColInfoData, numOfRows, tName, false), &lino, _OVER); // database name pColInfoData = taosArrayGet(p->pDataBlock, 1); - (void)colDataSetVal(pColInfoData, numOfRows, dName, false); + TAOS_CHECK_GOTO(colDataSetVal(pColInfoData, numOfRows, dName, false), &lino, _OVER); pColInfoData = taosArrayGet(p->pDataBlock, 2); - (void)colDataSetVal(pColInfoData, numOfRows, typeName, false); + TAOS_CHECK_GOTO(colDataSetVal(pColInfoData, numOfRows, typeName, false), &lino, _OVER); // col name char colName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(colName, pm->schema[j].name); pColInfoData = taosArrayGet(p->pDataBlock, 3); - (void)colDataSetVal(pColInfoData, numOfRows, colName, false); + TAOS_CHECK_GOTO(colDataSetVal(pColInfoData, numOfRows, colName, false), &lino, _OVER); // col type int8_t colType = pm->schema[j].type; @@ -3652,10 +3699,10 @@ static int32_t buildDbColsInfoBlock(const SSDataBlock *p, const SSysTableMeta *p (int32_t)((pm->schema[j].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } varDataSetLen(colTypeStr, colTypeLen); - (void)colDataSetVal(pColInfoData, numOfRows, (char *)colTypeStr, false); + TAOS_CHECK_GOTO(colDataSetVal(pColInfoData, numOfRows, (char *)colTypeStr, false), &lino, _OVER); pColInfoData = taosArrayGet(p->pDataBlock, 5); - (void)colDataSetVal(pColInfoData, numOfRows, (const char *)&pm->schema[j].bytes, false); + TAOS_CHECK_GOTO(colDataSetVal(pColInfoData, numOfRows, (const char *)&pm->schema[j].bytes, false), &lino, _OVER); for (int32_t k = 6; k <= 8; ++k) { pColInfoData = taosArrayGet(p->pDataBlock, k); colDataSetNULL(pColInfoData, numOfRows); @@ -3664,7 +3711,8 @@ static int32_t buildDbColsInfoBlock(const SSDataBlock *p, const SSysTableMeta *p numOfRows += 1; } } - +_OVER: + mError("failed at %s:%d since %s", __FUNCTION__, lino, tstrerror(code)); return numOfRows; } #define BUILD_COL_FOR_INFO_DB 1 @@ -3712,6 +3760,8 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB SSdb *pSdb = pMnode->pSdb; SStbObj *pStb = NULL; int32_t numOfRows = 0; + int32_t lino = 0; + int32_t code = 0; buildWhichDBs = determineBuildColForWhichDBs(pShow->db); @@ -3771,26 +3821,26 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB mDebug("mndRetrieveStbCol get stable cols, stable name:%s, db:%s", pStb->name, pStb->db); char db[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - (void)tNameFromString(&name, pStb->db, T_NAME_ACCT | T_NAME_DB); - (void)tNameGetDbName(&name, varDataVal(db)); + RETRIEVE_CHECK_GOTO(tNameFromString(&name, pStb->db, T_NAME_ACCT | T_NAME_DB), pStb, &lino, _OVER); + RETRIEVE_CHECK_GOTO(tNameGetDbName(&name, varDataVal(db)), pStb, &lino, _OVER); varDataSetLen(db, strlen(varDataVal(db))); for (int i = 0; i < pStb->numOfColumns; i++) { int32_t cols = 0; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)stbName, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)stbName, false), pStb, &lino, _OVER); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)db, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)db, false), pStb, &lino, _OVER); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, typeName, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, typeName, false), pStb, &lino, _OVER); // col name char colName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(colName, pStb->pColumns[i].name); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, colName, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, colName, false), pStb, &lino, _OVER); // col type int8_t colType = pStb->pColumns[i].type; @@ -3805,10 +3855,11 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB (int32_t)((pStb->pColumns[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } varDataSetLen(colTypeStr, colTypeLen); - (void)colDataSetVal(pColInfo, numOfRows, (char *)colTypeStr, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (char *)colTypeStr, false), pStb, &lino, _OVER); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - (void)colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->pColumns[i].bytes, false); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->pColumns[i].bytes, false), pStb, + &lino, _OVER); while (cols < pShow->numOfColumns) { pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetNULL(pColInfo, numOfRows); @@ -3824,9 +3875,15 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB } } - pShow->numOfRows += numOfRows; mDebug("mndRetrieveStbCol success, rows:%d, pShow->numOfRows:%d", numOfRows, pShow->numOfRows); + goto _OVER; +_ERROR: + mError("failed to mndRetrieveStbCol, rows:%d, pShow->numOfRows:%d, at %s:%d since %s", numOfRows, pShow->numOfRows, + __FUNCTION__, lino, tstrerror(code)); + +_OVER: + pShow->numOfRows += numOfRows; return numOfRows; } @@ -4111,8 +4168,9 @@ static void *mndBuildVDropTbsReq(SMnode *pMnode, const SVgroupInfo *pVgInfo, con void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead)); - (void)tEncodeSVDropTbBatchReq(&encoder, pReq); + int32_t code = tEncodeSVDropTbBatchReq(&encoder, pReq); tEncoderClear(&encoder); + if (code != 0) return NULL; *len = contLen; return pHead; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 672d7aca3e..4ea2431d83 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1142,18 +1142,16 @@ int32_t extractStreamNodeList(SMnode *pMnode) { return taosArrayGetSize(execInfo.pNodeList); } -static bool taskNodeIsUpdated(SMnode *pMnode) { - bool allReady = true; - SArray *pNodeSnapshot = NULL; - - // check if the node update happens or not - streamMutexLock(&execInfo.lock); +static int32_t doCheckForUpdated(SMnode *pMnode, SArray **ppNodeSnapshot) { + bool allReady = false; + bool nodeUpdated = false; + SVgroupChangeInfo changeInfo = {0}; int32_t numOfNodes = extractStreamNodeList(pMnode); + if (numOfNodes == 0) { mDebug("stream task node change checking done, no vgroups exist, do nothing"); execInfo.ts = taosGetTimestampSec(); - streamMutexUnlock(&execInfo.lock); return false; } @@ -1165,43 +1163,46 @@ static bool taskNodeIsUpdated(SMnode *pMnode) { if (pNodeEntry->stageUpdated) { mDebug("stream task not ready due to node update detected, checkpoint not issued"); - streamMutexUnlock(&execInfo.lock); return true; } } - int32_t code = mndTakeVgroupSnapshot(pMnode, &allReady, &pNodeSnapshot); + int32_t code = mndTakeVgroupSnapshot(pMnode, &allReady, ppNodeSnapshot); if (code) { mError("failed to get the vgroup snapshot, ignore it and continue"); } if (!allReady) { mWarn("not all vnodes ready, quit from vnodes status check"); - taosArrayDestroy(pNodeSnapshot); - streamMutexUnlock(&execInfo.lock); return true; } - SVgroupChangeInfo changeInfo = {0}; - code = mndFindChangedNodeInfo(pMnode, execInfo.pNodeList, pNodeSnapshot, &changeInfo); + code = mndFindChangedNodeInfo(pMnode, execInfo.pNodeList, *ppNodeSnapshot, &changeInfo); if (code) { - streamMutexUnlock(&execInfo.lock); - return false; + nodeUpdated = false; + } else { + nodeUpdated = (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0); + if (nodeUpdated) { + mDebug("stream tasks not ready due to node update"); + } } - bool nodeUpdated = (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0); - mndDestroyVgroupChangeInfo(&changeInfo); - taosArrayDestroy(pNodeSnapshot); - - if (nodeUpdated) { - mDebug("stream tasks not ready due to node update"); - } - - streamMutexUnlock(&execInfo.lock); return nodeUpdated; } +// check if the node update happens or not +static bool taskNodeIsUpdated(SMnode *pMnode) { + SArray *pNodeSnapshot = NULL; + + streamMutexLock(&execInfo.lock); + bool updated = doCheckForUpdated(pMnode, &pNodeSnapshot); + streamMutexUnlock(&execInfo.lock); + + taosArrayDestroy(pNodeSnapshot); + return updated; +} + static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) { bool ready = true; if (taskNodeIsUpdated(pMnode)) { @@ -1992,7 +1993,7 @@ static int32_t mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeLis if (pInfo->pUpdateNodeList == NULL || pInfo->pDBMap == NULL) { mndDestroyVgroupChangeInfo(pInfo); - return terrno; + TSDB_CHECK_NULL(NULL, code, lino, _err, terrno); } int32_t numOfNodes = taosArrayGetSize(pPrevNodeList); @@ -2047,6 +2048,7 @@ static int32_t mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeLis return code; _err: + mError("failed to find node change info, code:%s at %s line:%d", tstrerror(code), __func__, lino); mndDestroyVgroupChangeInfo(pInfo); return code; } @@ -2220,7 +2222,11 @@ static int32_t refreshNodeListFromExistedStreams(SMnode *pMnode, SArray *pNodeLi } char buf[256] = {0}; - (void) epsetToStr(&pEntry->epset, buf, tListLen(buf)); // ignore this error since it is only for log file + int32_t ret = epsetToStr(&pEntry->epset, buf, tListLen(buf)); // ignore this error since it is only for log file + if (ret != 0) { // print error and continue + mError("failed to convert epset to str, code:%s", tstrerror(ret)); + } + mDebug("extract nodeInfo from stream obj, nodeId:%d, %s", pEntry->nodeId, buf); } @@ -2230,7 +2236,7 @@ static int32_t refreshNodeListFromExistedStreams(SMnode *pMnode, SArray *pNodeLi return code; } -static int32_t addAllDbsIntoHashmap(SHashObj *pDBMap, SSdb *pSdb) { +static void addAllDbsIntoHashmap(SHashObj *pDBMap, SSdb *pSdb) { void *pIter = NULL; int32_t code = 0; while (1) { @@ -2248,8 +2254,6 @@ static int32_t addAllDbsIntoHashmap(SHashObj *pDBMap, SSdb *pSdb) { mDebug("add Db:%s into Dbs list (total:%d) for kill checkpoint trans", pVgroup->dbName, size); } } - - return code; } // this function runs by only one thread, so it is not multi-thread safe @@ -2310,7 +2314,7 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { mInfo("rollback all stream due to mnode leader/follower switch by using nodeUpdate trans"); updateAllVgroups = true; execInfo.switchFromFollower = false; // reset the flag - (void) addAllDbsIntoHashmap(changeInfo.pDBMap, pMnode->pSdb); + addAllDbsIntoHashmap(changeInfo.pDBMap, pMnode->pSdb); } } diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 43f9d8d055..941956ae2b 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -194,10 +194,13 @@ int32_t mndSendDropOrphanTasksMsg(SMnode *pMnode, SArray *pList) { return terrno; } - (void)tSerializeDropOrphanTaskMsg(pReq, contLen, &msg); + int32_t code = tSerializeDropOrphanTaskMsg(pReq, contLen, &msg); + if (code <= 0) { + mError("failed to serialize the drop orphan task msg, code:%s", tstrerror(code)); + } SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_DROP_ORPHANTASKS, .pCont = pReq, .contLen = contLen}; - int32_t code = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); + code = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); if (code) { mError("failed to put drop-orphan task msg into write queue, code:%s", tstrerror(code)); } else { @@ -216,7 +219,7 @@ int32_t mndProcessResetStatusReq(SRpcMsg *pReq) { mndKillTransImpl(pMnode, pMsg->transId, ""); streamMutexLock(&execInfo.lock); - (void) mndResetChkptReportInfo(execInfo.pChkptStreams, pMsg->streamId); + code = mndResetChkptReportInfo(execInfo.pChkptStreams, pMsg->streamId); // do thing if failed streamMutexUnlock(&execInfo.lock); code = mndGetStreamObj(pMnode, pMsg->streamId, &pStream); @@ -393,7 +396,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { int32_t numOfUpdated = taosArrayGetSize(req.pUpdateNodes); if (numOfUpdated > 0) { mDebug("%d stream node(s) need updated from hbMsg(vgId:%d)", numOfUpdated, req.vgId); - (void) setNodeEpsetExpiredFlag(req.pUpdateNodes); + int32_t unused = setNodeEpsetExpiredFlag(req.pUpdateNodes); } bool snodeChanged = false; diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 1ea33c4918..c52739183e 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -165,7 +165,10 @@ int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray **pList) { } char buf[256] = {0}; - (void)epsetToStr(&entry.epset, buf, tListLen(buf)); + code = epsetToStr(&entry.epset, buf, tListLen(buf)); + if (code != 0) { // print error and continue + mError("failed to convert epset to str, code:%s", tstrerror(code)); + } void *p = taosArrayPush(pVgroupList, &entry); if (p == NULL) { @@ -198,7 +201,10 @@ int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray **pList) { } char buf[256] = {0}; - (void)epsetToStr(&entry.epset, buf, tListLen(buf)); + code = epsetToStr(&entry.epset, buf, tListLen(buf)); + if (code != 0) { // print error and continue + mError("failed to convert epset to str, code:%s", tstrerror(code)); + } void *p = taosArrayPush(pVgroupList, &entry); if (p == NULL) { @@ -424,9 +430,12 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa } char buf[256] = {0}; - (void) epsetToStr(&epset, buf, tListLen(buf)); - mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf); + code = epsetToStr(&epset, buf, tListLen(buf)); + if (code != 0) { // print error and continue + mError("failed to convert epset to str, code:%s", tstrerror(code)); + } + mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf); code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0, TSDB_CODE_VND_INVALID_VGROUP_ID); if (code != 0) { taosMemoryFree(pReq); @@ -639,8 +648,7 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) { void *pBuf = NULL; int32_t len = 0; - (void)streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList); - + bool unusedRet = streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList); int32_t code = doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id); if (code) { return code; @@ -914,8 +922,15 @@ void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { } // 2. remove stream entry in consensus hash table and checkpoint-report hash table - (void) mndClearConsensusCheckpointId(execInfo.pStreamConsensus, pStream->uid); - (void) mndClearChkptReportInfo(execInfo.pChkptStreams, pStream->uid); + code = mndClearConsensusCheckpointId(execInfo.pStreamConsensus, pStream->uid); + if (code) { + mError("failed to clear consensus checkpointId, code:%s", tstrerror(code)); + } + + code = mndClearChkptReportInfo(execInfo.pChkptStreams, pStream->uid); + if (code) { + mError("failed to clear the checkpoint report info, code:%s", tstrerror(code)); + } streamMutexUnlock(&pExecNode->lock); destroyStreamTaskIter(pIter); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 1c6474d787..37a711da41 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -248,7 +248,7 @@ static int32_t processRemovedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash MND_TMQ_RETURN_CHECK(pushVgDataToHash(pConsumerEp->vgs, pHash, *consumerId, pOutput->pSub->key)); } - (void)taosArrayDestroy(pConsumerEp->vgs); + taosArrayDestroy(pConsumerEp->vgs); MND_TMQ_RETURN_CHECK(taosHashRemove(pOutput->pSub->consumerHash, consumerId, sizeof(int64_t))); MND_TMQ_NULL_CHECK(taosArrayPush(pOutput->removedConsumers, consumerId)); actualRemoved++; @@ -682,8 +682,8 @@ END: static void freeRebalanceItem(void *param) { SMqRebInfo *pInfo = param; - (void)taosArrayDestroy(pInfo->newConsumers); - (void)taosArrayDestroy(pInfo->removedConsumers); + taosArrayDestroy(pInfo->newConsumers); + taosArrayDestroy(pInfo->removedConsumers); } // type = 0 remove type = 1 add @@ -738,8 +738,12 @@ static void checkForVgroupSplit(SMnode *pMnode, SMqConsumerObj *pConsumer, SHash } SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVgEp->vgId); if (!pVgroup) { - (void)mndGetOrCreateRebSub(rebSubHash, key, NULL); - mInfo("vnode splitted, vgId:%d rebalance will be triggered", pVgEp->vgId); + code = mndGetOrCreateRebSub(rebSubHash, key, NULL); + if (code != 0){ + mError("failed to mndGetOrCreateRebSub vgroup:%d, error:%s", pVgEp->vgId, tstrerror(code)) + }else{ + mInfo("vnode splitted, vgId:%d rebalance will be triggered", pVgEp->vgId); + } } mndReleaseVgroup(pMnode, pVgroup); } @@ -813,10 +817,10 @@ void mndRebCntDec() { } static void clearRebOutput(SMqRebOutputObj *rebOutput) { - (void)taosArrayDestroy(rebOutput->newConsumers); - (void)taosArrayDestroy(rebOutput->modifyConsumers); - (void)taosArrayDestroy(rebOutput->removedConsumers); - (void)taosArrayDestroy(rebOutput->rebVgs); + taosArrayDestroy(rebOutput->newConsumers); + taosArrayDestroy(rebOutput->modifyConsumers); + taosArrayDestroy(rebOutput->removedConsumers); + taosArrayDestroy(rebOutput->rebVgs); tDeleteSubscribeObj(rebOutput->pSub); taosMemoryFree(rebOutput->pSub); } @@ -858,7 +862,7 @@ static int32_t checkConsumer(SMnode *pMnode, SMqSubscribeObj *pSub) { mError("consumer:0x%" PRIx64 " not exists in sdb for exception", pConsumerEp->consumerId); MND_TMQ_NULL_CHECK(taosArrayAddAll(pSub->unassignedVgs, pConsumerEp->vgs)); - (void)taosArrayDestroy(pConsumerEp->vgs); + taosArrayDestroy(pConsumerEp->vgs); MND_TMQ_RETURN_CHECK(taosHashRemove(pSub->consumerHash, &pConsumerEp->consumerId, sizeof(int64_t))); } END: diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 66c0fec273..e73cc1b5db 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -238,7 +238,9 @@ static int32_t mndPostMgmtCode(SMnode *pMnode, int32_t code) { pMgmt->transSec = 0; pMgmt->transSeq = 0; pMgmt->errCode = code; - (void)tsem_post(&pMgmt->syncSem); + if (tsem_post(&pMgmt->syncSem) < 0) { + mError("trans:%d, failed to post sem", transId); + } if (pMgmt->errCode != 0) { mError("trans:%d, failed to propose since %s, post sem", transId, tstrerror(pMgmt->errCode)); @@ -310,11 +312,15 @@ void mndRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) { } else { mInfo("vgId:1, sync restore finished"); } - (void)mndRefreshUserIpWhiteList(pMnode); + int32_t code = mndRefreshUserIpWhiteList(pMnode); + if (code != 0) { + mError("vgId:1, failed to refresh user ip white list since %s", tstrerror(code)); + mndSetRestored(pMnode, false); + } SyncIndex fsmIndex = mndSyncAppliedIndex(pFsm); if (commitIdx != fsmIndex) { - mError("vgId:1, sync restore finished, but commitIdx:%" PRId64 " is not equal to appliedIdx:%" PRId64, commitIdx, + mError("vgId:1, failed to sync restore, commitIdx:%" PRId64 " is not equal to appliedIdx:%" PRId64, commitIdx, fsmIndex); mndSetRestored(pMnode, false); } @@ -368,7 +374,9 @@ static void mndBecomeFollower(const SSyncFSM *pFsm) { pMgmt->transSec = 0; pMgmt->transSeq = 0; pMgmt->errCode = TSDB_CODE_SYN_NOT_LEADER; - (void)tsem_post(&pMgmt->syncSem); + if (tsem_post(&pMgmt->syncSem) < 0) { + mError("failed to post sem"); + } } (void)taosThreadMutexUnlock(&pMgmt->lock); @@ -387,7 +395,9 @@ static void mndBecomeLearner(const SSyncFSM *pFsm) { pMgmt->transSec = 0; pMgmt->transSeq = 0; pMgmt->errCode = TSDB_CODE_SYN_NOT_LEADER; - (void)tsem_post(&pMgmt->syncSem); + if (tsem_post(&pMgmt->syncSem) < 0) { + mError("failed to post sem"); + } } (void)taosThreadMutexUnlock(&pMgmt->lock); } @@ -487,13 +497,18 @@ int32_t mndInitSync(SMnode *pMnode) { pNode->nodePort = pMgmt->replicas[i].port; tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); pNode->nodeRole = pMgmt->nodeRoles[i]; - (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + if (tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort) != true) { + mError("failed to open sync, tmsgUpdateDnodeInfo is false"); + } mInfo("vgId:1, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, pNode->clusterId); } int32_t code = 0; - (void)tsem_init(&pMgmt->syncSem, 0, 0); + if ((code = tsem_init(&pMgmt->syncSem, 0, 0)) < 0) { + mError("failed to open sync, tsem_init, since %s", tstrerror(code)); + TAOS_RETURN(code); + } pMgmt->sync = syncOpen(&syncInfo, 1); // always check if (pMgmt->sync <= 0) { if (terrno != 0) code = terrno; @@ -511,7 +526,9 @@ void mndCleanupSync(SMnode *pMnode) { syncStop(pMgmt->sync); mInfo("mnode-sync is stopped, id:%" PRId64, pMgmt->sync); - (void)tsem_destroy(&pMgmt->syncSem); + if (tsem_destroy(&pMgmt->syncSem) < 0) { + mError("failed to destroy sem"); + } (void)taosThreadMutexDestroy(&pMgmt->lock); memset(pMgmt, 0, sizeof(SSyncMgmt)); } @@ -526,12 +543,14 @@ void mndSyncCheckTimeout(SMnode *pMnode) { if (delta > MNODE_TIMEOUT_SEC) { mError("trans:%d, failed to propose since timeout, start:%d cur:%d delta:%d seq:%" PRId64, pMgmt->transId, pMgmt->transSec, curSec, delta, pMgmt->transSeq); - pMgmt->transId = 0; - pMgmt->transSec = 0; - pMgmt->transSeq = 0; - terrno = TSDB_CODE_SYN_TIMEOUT; - pMgmt->errCode = TSDB_CODE_SYN_TIMEOUT; - (void)tsem_post(&pMgmt->syncSem); + // pMgmt->transId = 0; + // pMgmt->transSec = 0; + // pMgmt->transSeq = 0; + // terrno = TSDB_CODE_SYN_TIMEOUT; + // pMgmt->errCode = TSDB_CODE_SYN_TIMEOUT; + //if (tsem_post(&pMgmt->syncSem) < 0) { + // mError("failed to post sem"); + //} } else { mDebug("trans:%d, waiting for sync confirm, start:%d cur:%d delta:%d seq:%" PRId64, pMgmt->transId, pMgmt->transSec, curSec, curSec - pMgmt->transSec, pMgmt->transSeq); @@ -572,7 +591,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { mInfo("trans:%d, is proposing and wait sem, seq:%" PRId64, transId, seq); pMgmt->transSeq = seq; (void)taosThreadMutexUnlock(&pMgmt->lock); - (void)tsem_wait(&pMgmt->syncSem); + code = tsem_wait(&pMgmt->syncSem); } else if (code > 0) { mInfo("trans:%d, confirm at once since replica is 1, continue execute", transId); pMgmt->transId = 0; @@ -624,7 +643,9 @@ void mndSyncStop(SMnode *pMnode) { pMgmt->transId = 0; pMgmt->transSec = 0; pMgmt->errCode = TSDB_CODE_APP_IS_STOPPING; - (void)tsem_post(&pMgmt->syncSem); + if (tsem_post(&pMgmt->syncSem) < 0) { + mError("failed to post sem"); + } } (void)taosThreadMutexUnlock(&pMgmt->lock); } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index ed702e7707..78aca0dbee 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -302,7 +302,7 @@ static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic) { taosMemoryFreeClear(pTopic->ast); taosMemoryFreeClear(pTopic->physicalPlan); if (pTopic->schema.nCols) taosMemoryFreeClear(pTopic->schema.pSchema); - (void)taosArrayDestroy(pTopic->ntbColIds); + taosArrayDestroy(pTopic->ntbColIds); return 0; } @@ -467,7 +467,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * MND_TMQ_NULL_CHECK(topicObj.ntbColIds); MND_TMQ_RETURN_CHECK(extractTopicTbInfo(pAst, &topicObj)); if (topicObj.ntbUid == 0) { - (void)taosArrayDestroy(topicObj.ntbColIds); + taosArrayDestroy(topicObj.ntbColIds); topicObj.ntbColIds = NULL; } @@ -505,7 +505,7 @@ END: taosMemoryFreeClear(topicObj.physicalPlan); taosMemoryFreeClear(topicObj.sql); taosMemoryFreeClear(topicObj.ast); - (void)taosArrayDestroy(topicObj.ntbColIds); + taosArrayDestroy(topicObj.ntbColIds); if (topicObj.schema.nCols) { taosMemoryFreeClear(topicObj.schema.pSchema); } @@ -567,9 +567,15 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { { SName dbname = {0}; - (void)tNameFromString(&dbname, createTopicReq.subDbName, T_NAME_ACCT | T_NAME_DB); // ignore error + int32_t ret = tNameFromString(&dbname, createTopicReq.subDbName, T_NAME_ACCT | T_NAME_DB); + if (ret != 0){ + mError("failed to parse db name:%s, ret:%d", createTopicReq.subDbName, ret); + } SName topicName = {0}; - (void)tNameFromString(&topicName, createTopicReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); // ignore error + ret = tNameFromString(&topicName, createTopicReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + if (ret != 0){ + mError("failed to parse topic name:%s, ret:%d", createTopicReq.name, ret); + } auditRecord(pReq, pMnode->clusterId, "createTopic", dbname.dbname, topicName.dbname, createTopicReq.sql, strlen(createTopicReq.sql)); } @@ -735,7 +741,10 @@ END: } SName name = {0}; - (void)tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); // ignore error + int32_t ret = tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + if (ret != 0) { + mError("topic:%s, failed to drop since %s", dropReq.name, tstrerror(ret)); + } auditRecord(pReq, pMnode->clusterId, "dropTopic", name.dbname, name.tname, dropReq.sql, dropReq.sqlLen); tFreeSMDropTopicReq(&dropReq); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 41ddc2ab32..8ca3d59868 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -342,7 +342,8 @@ int32_t mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8 SIpV4Range range = {.ip = 0, .mask = 32}; int32_t code = taosGetIpv4FromFqdn(fqdn, &range.ip); if (code) { - //TODO + mError("failed to get ip from fqdn: %s at line %d since %s", fqdn, lino, tstrerror(code)); + TAOS_RETURN(TSDB_CODE_TSC_INVALID_FQDN); } mDebug("ip-white-list may update for user: %s, fqdn: %s", user, fqdn); SIpWhiteList **ppList = taosHashGet(pIpWhiteTab, user, strlen(user)); @@ -2800,7 +2801,7 @@ static int32_t mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, i void *key = taosHashGetKey(value, &keyLen); char dbName[TSDB_DB_NAME_LEN] = {0}; - mndExtractShortDbNameFromStbFullName(key, dbName); + (void)mndExtractShortDbNameFromStbFullName(key, dbName); char dbNameContent[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(dbNameContent, dbName, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 2f56aac7d6..827f7e2044 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -164,7 +164,7 @@ typedef struct STsdbReader STsdbReader; int32_t tsdbReaderOpen2(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, SSDataBlock *pResBlock, void **ppReader, const char *idstr, SHashObj **pIgnoreTables); int32_t tsdbSetTableList2(STsdbReader *pReader, const void *pTableList, int32_t num); -void tsdbReaderSetId2(STsdbReader *pReader, const char *idstr); +int32_t tsdbReaderSetId(void *pReader, const char *idstr); void tsdbReaderClose2(STsdbReader *pReader); int32_t tsdbNextDataBlock2(STsdbReader *pReader, bool *hasNext); int32_t tsdbRetrieveDatablockSMA2(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 19c5b5d481..69819c87dc 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -314,8 +314,9 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat SReadHandle handle = {.vnode = pVnode, .initTqReader = 1, .skipRollup = 1, .pStateBackend = pStreamState}; initStorageAPI(&handle.api); - pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle, TD_VID(pVnode), 0); - if (!pRSmaInfo->taskInfo[idx]) { + + code = qCreateStreamExecTaskInfo(&pRSmaInfo->taskInfo[idx], param->qmsg[idx], &handle, TD_VID(pVnode), 0); + if (!pRSmaInfo->taskInfo[idx] || (code != 0)) { TAOS_RETURN(TSDB_CODE_RSMA_QTASKINFO_CREATE); } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8448d305a2..77275a200f 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -102,7 +102,6 @@ int32_t tqOpen(const char* path, SVnode* pVnode) { int32_t tqInitialize(STQ* pTq) { int32_t vgId = TD_VID(pTq->pVnode); - int32_t code = streamMetaOpen(pTq->path, pTq, tqBuildStreamTask, tqExpandStreamTask, vgId, -1, tqStartTaskCompleteCallback, &pTq->pStreamMeta); if (code != TSDB_CODE_SUCCESS) { @@ -110,7 +109,6 @@ int32_t tqInitialize(STQ* pTq) { } streamMetaLoadAllTasks(pTq->pStreamMeta); - return tqMetaOpen(pTq); } @@ -344,7 +342,9 @@ int32_t tqProcessPollPush(STQ* pTq, SRpcMsg* pMsg) { .pCont = pHandle->msg->pCont, .contLen = pHandle->msg->contLen, .info = pHandle->msg->info}; - (void)tmsgPutToQueue(&pTq->pVnode->msgCb, QUERY_QUEUE, &msg); + if (tmsgPutToQueue(&pTq->pVnode->msgCb, QUERY_QUEUE, &msg) != 0){ + tqError("vgId:%d tmsgPutToQueue failed, consumer:0x%" PRIx64, vgId, pHandle->consumerId); + } taosMemoryFree(pHandle->msg); pHandle->msg = NULL; } @@ -643,7 +643,6 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg tDecoderInit(&dc, (uint8_t*)msg, msgLen); ret = tDecodeSMqRebVgReq(&dc, &req); - // decode req if (ret < 0) { goto end; } @@ -653,7 +652,10 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg taosRLockLatch(&pTq->lock); STqHandle* pHandle = NULL; - (void)tqMetaGetHandle(pTq, req.subKey, &pHandle); // ignore return code + int32_t code = tqMetaGetHandle(pTq, req.subKey, &pHandle); + if (code != 0){ + tqInfo("vgId:%d, tq process sub req:%s, no such handle, create new one", pTq->pVnode->config.vgId, req.subKey); + } taosRUnLockLatch(&pTq->lock); if (pHandle == NULL) { if (req.oldConsumerId != -1) { @@ -662,6 +664,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } if (req.newConsumerId == -1) { tqError("vgId:%d, tq invalid rebalance request, new consumerId %" PRId64 "", req.vgId, req.newConsumerId); + ret = TSDB_CODE_INVALID_PARA; goto end; } STqHandle handle = {0}; @@ -740,16 +743,25 @@ int32_t tqBuildStreamTask(void* pTqObj, SStreamTask* pTask, int64_t nextProcessV SSchemaWrapper* pschemaWrapper = pOutputInfo->tbSink.pSchemaWrapper; pOutputInfo->tbSink.pTSchema = tBuildTSchema(pschemaWrapper->pSchema, pschemaWrapper->nCols, ver1); if (pOutputInfo->tbSink.pTSchema == NULL) { - return -1; + return terrno; } pOutputInfo->tbSink.pTblInfo = tSimpleHashInit(10240, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); + if (pOutputInfo->tbSink.pTblInfo == NULL) { + tqError("vgId:%d failed init sink tableInfo, code:%s", vgId, tstrerror(terrno)); + return terrno; + } + tSimpleHashSetFreeFp(pOutputInfo->tbSink.pTblInfo, freePtr); } if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { SWalFilterCond cond = {.deleteMsg = 1}; // delete msg also extract from wal files pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, &cond, pTask->id.taskId); + if (pTask->exec.pWalReader == NULL) { + tqError("vgId:%d failed init wal reader, code:%s", vgId, tstrerror(terrno)); + return terrno; + } } streamTaskResetUpstreamStageInfo(pTask); @@ -1002,9 +1014,13 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { } int32_t code = tqStreamTaskProcessRunReq(pTq->pStreamMeta, pMsg, vnodeIsRoleLeader(pTq->pVnode)); + if (code) { + tqError("vgId:%d failed to create task run req, code:%s", TD_VID(pTq->pVnode), tstrerror(code)); + return code; + } // let's continue scan data in the wal files - if (code == 0 && (pReq->reqType >= 0 || pReq->reqType == STREAM_EXEC_T_RESUME_TASK)) { + if (pReq->reqType >= 0 || pReq->reqType == STREAM_EXEC_T_RESUME_TASK) { code = tqScanWalAsync(pTq, false); // it's ok to failed if (code) { tqError("vgId:%d failed to start scan wal file, code:%s", pTq->pStreamMeta->vgId, tstrerror(code)); diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index cccc96d3f0..5ad1680794 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -390,8 +390,8 @@ END: tdbFree(pKey); tdbFree(pVal); int32_t ret = tdbTbcClose(pCur); - if (code == 0 && ret != 0) { - code = ret; + if (ret != 0) { + tqError("failed to close tbc, ret:%d", ret); } return code; } @@ -464,12 +464,10 @@ static int32_t tqMetaRestoreCheckInfo(STQ* pTq) { END: tdbFree(pKey); tdbFree(pVal); - int32_t ret = tdbTbcClose(pCur); - if (code == 0) { - code = ret; + if (ret != 0) { + tqError("failed to close tbc, ret:%d", ret); } - tDeleteSTqCheckInfo(&info); return code; } @@ -534,48 +532,44 @@ END: taosMemoryFree(offset); taosMemoryFree(offsetNew); - // return 0 always, so ignore int32_t ret = tdbTbClose(pExecStore); if (ret != 0) { - tqError("vgId:%d failed to close stream exec store, code:%s", pTq->pStreamMeta->vgId, tstrerror(ret)); + tqError("failed to close tb, ret:%d", ret); } - ret = tdbTbClose(pCheckStore); if (ret != 0) { - tqError("vgId:%d failed to close stream check store, code:%s", pTq->pStreamMeta->vgId, tstrerror(ret)); + tqError("failed to close tb, ret:%d", ret); } - ret = tdbClose(pMetaDB); if (ret != 0) { - tqError("vgId:%d failed to close stream meta db store, code:%s", pTq->pStreamMeta->vgId, tstrerror(ret)); + tqError("failed to close tdb, ret:%d", ret); } return code; } void tqMetaClose(STQ* pTq) { - int32_t code = 0; + int32_t ret = 0; if (pTq->pExecStore) { - code = tdbTbClose(pTq->pExecStore); - if (code) { - tqError("vgId:%d failed to close tq exec store, code:%s", pTq->pStreamMeta->vgId, tstrerror(code)); + ret = tdbTbClose(pTq->pExecStore); + if (ret != 0) { + tqError("failed to close tb, ret:%d", ret); } } if (pTq->pCheckStore) { - code = tdbTbClose(pTq->pCheckStore); - if (code) { - tqError("vgId:%d failed to close tq check store, code:%s", pTq->pStreamMeta->vgId, tstrerror(code)); + ret = tdbTbClose(pTq->pCheckStore); + if (ret != 0) { + tqError("failed to close tb, ret:%d", ret); } } if (pTq->pOffsetStore) { - code = tdbTbClose(pTq->pOffsetStore); - if (code) { - tqError("vgId:%d failed to close tq offset store, code:%s", pTq->pStreamMeta->vgId, tstrerror(code)); + ret = tdbTbClose(pTq->pOffsetStore); + if (ret != 0) { + tqError("failed to close tb, ret:%d", ret); } } - - code = tdbClose(pTq->pMetaDB); - if (code) { - tqError("vgId:%d failed to close tq meta db store, code:%s", pTq->pStreamMeta->vgId, tstrerror(code)); + ret = tdbClose(pTq->pMetaDB); + if (ret != 0) { + tqError("failed to close tdb, ret:%d", ret); } } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index e85c3a5f3a..eb5d6aafe7 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -351,11 +351,8 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con if (data == NULL) { // todo: for all stream in this vnode, keep this offset in the offset files, and wait for a moment, and then // retry - code = TSDB_CODE_OUT_OF_MEMORY; - terrno = code; - tqError("vgId:%d, failed to copy submit data for stream processing, since out of memory", 0); - return code; + return terrno; } (void)memcpy(data, pBody, len); diff --git a/source/dnode/vnode/src/tq/tqSnapshot.c b/source/dnode/vnode/src/tq/tqSnapshot.c index 1abc94f653..12b1e5ec19 100644 --- a/source/dnode/vnode/src/tq/tqSnapshot.c +++ b/source/dnode/vnode/src/tq/tqSnapshot.c @@ -77,7 +77,10 @@ _err: } void tqSnapReaderClose(STqSnapReader** ppReader) { - (void)tdbTbcClose((*ppReader)->pCur); + int32_t ret = tdbTbcClose((*ppReader)->pCur); + if (ret != 0){ + tqError("vgId:%d, vnode snapshot tq reader close failed since %s", TD_VID((*ppReader)->pTq->pVnode), tstrerror(ret)); + } taosMemoryFree(*ppReader); *ppReader = NULL; } diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index a6f3a60563..e467382d89 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -23,7 +23,7 @@ static int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle); static int32_t setWalReaderStartOffset(SStreamTask* pTask, int32_t vgId); static bool handleFillhistoryScanComplete(SStreamTask* pTask, int64_t ver); static bool taskReadyForDataFromWal(SStreamTask* pTask); -static bool doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems); +static int32_t doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems, bool* pSucc); static int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDuration); // extract data blocks(submit/delete) from WAL, and add them into the input queue for all the sources tasks. @@ -40,7 +40,7 @@ int32_t tqScanWal(STQ* pTq) { int32_t code = doScanWalForAllTasks(pMeta, &shouldIdle); if (code) { - tqError("vgId:%d failed to start all tasks, try next time", vgId); + tqError("vgId:%d failed to start all tasks, try next time, code:%s", vgId, tstrerror(code)); return code; } @@ -112,8 +112,8 @@ int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDuration) { if (pMeta->scanInfo.scanTimer == NULL) { pMeta->scanInfo.scanTimer = taosTmrStart(doStartScanWal, idleDuration, pParam, pTimer); } else { - code = taosTmrReset(doStartScanWal, idleDuration, pParam, pTimer, &pMeta->scanInfo.scanTimer); - if (code) { + bool ret = taosTmrReset(doStartScanWal, idleDuration, pParam, pTimer, &pMeta->scanInfo.scanTimer); + if (!ret) { tqError("vgId:%d failed to start scan wal in:%dms", vgId, idleDuration); } } @@ -298,9 +298,11 @@ bool taskReadyForDataFromWal(SStreamTask* pTask) { return true; } -bool doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems) { +int32_t doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems, bool* pSucc) { const char* id = pTask->id.idStr; int32_t numOfNewItems = 0; + int32_t code = 0; + *pSucc = false; while (1) { if ((pTask->info.fillHistory == 1) && pTask->status.appendTranstateBlock) { @@ -309,7 +311,7 @@ bool doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems } SStreamQueueItem* pItem = NULL; - int32_t code = extractMsgFromWal(pTask->exec.pWalReader, (void**)&pItem, maxVer, id); + code = extractMsgFromWal(pTask->exec.pWalReader, (void**)&pItem, maxVer, id); if (code != TSDB_CODE_SUCCESS || pItem == NULL) { // failed, continue int64_t currentVer = walReaderGetCurrentVer(pTask->exec.pWalReader); bool itemInFillhistory = handleFillhistoryScanComplete(pTask, currentVer); @@ -332,10 +334,17 @@ bool doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems break; } } else { - tqTrace("s-task:%s append input queue failed, code:too many items, ver:%" PRId64, id, pTask->chkInfo.nextProcessVer); - code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.nextProcessVer); - if (code) { - tqError("s-task:%s failed to seek ver to:%"PRId64 " in wal", id, pTask->chkInfo.nextProcessVer); + if (code == TSDB_CODE_OUT_OF_MEMORY) { + tqError("s-task:%s failed to put data into inputQ, since out of memory", id); + } else { + tqTrace("s-task:%s append input queue failed, code:inputQ is full, ver:%" PRId64, id, + pTask->chkInfo.nextProcessVer); + code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.nextProcessVer); + if (code) { + tqError("s-task:%s failed to seek ver to:%" PRId64 " in wal", id, pTask->chkInfo.nextProcessVer); + } + + code = 0; // reset the error code } break; @@ -344,7 +353,8 @@ bool doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems } *numOfItems += numOfNewItems; - return numOfNewItems > 0; + *pSucc = (numOfNewItems > 0); + return code; } int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { @@ -363,6 +373,7 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { pTaskList = taosArrayDup(pStreamMeta->pTaskList, NULL); streamMetaWUnLock(pStreamMeta); if (pTaskList == NULL) { + tqError("vgId:%d failed to create task list dup, code:%s", vgId, tstrerror(terrno)); return terrno; } @@ -410,7 +421,8 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { continue; } - bool hasNewData = doPutDataIntoInputQ(pTask, maxVer, &numOfItems); + bool hasNewData = false; + code = doPutDataIntoInputQ(pTask, maxVer, &numOfItems, &hasNewData); streamMutexUnlock(&pTask->lock); if ((numOfItems > 0) || hasNewData) { diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index ff9f1e524e..495fcd771a 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -145,12 +145,13 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, terrno = 0; SMqDataRsp dataRsp = {0}; - int code = tqInitDataRsp(&dataRsp.common, *pOffset); + + int code = tqInitDataRsp(&dataRsp.common, *pOffset); if (code != 0) { goto end; } - qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); + code = qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); code = tqScanData(pTq, pHandle, &dataRsp, pOffset, pRequest); if (code != 0 && terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) { goto end; diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index a11cf770fc..b3bd3052d8 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -36,6 +36,7 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) { int64_t st = taosGetTimestampMs(); int64_t streamId = 0; int32_t taskId = 0; + int32_t code = 0; tqDebug("s-task:%s vgId:%d start to expand stream task", pTask->id.idStr, vgId); @@ -52,7 +53,7 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) { pTask->pState = streamStateOpen(pMeta->path, pTask, streamId, taskId); if (pTask->pState == NULL) { tqError("s-task:%s (vgId:%d) failed to open state for task, expand task failed", pTask->id.idStr, vgId); - return -1; + return terrno; } else { tqDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState); } @@ -75,12 +76,17 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) { initStorageAPI(&handle.api); if (pTask->info.taskLevel == TASK_LEVEL__SOURCE || pTask->info.taskLevel == TASK_LEVEL__AGG) { - pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId, pTask->id.taskId); - if (pTask->exec.pExecutor == NULL) { - tqError("s-task:%s failed to create exec taskInfo, failed to expand task", pTask->id.idStr); - return -1; + code = qCreateStreamExecTaskInfo(&pTask->exec.pExecutor, pTask->exec.qmsg, &handle, vgId, pTask->id.taskId); + if (code) { + tqError("s-task:%s failed to expand task, code:%s", pTask->id.idStr, tstrerror(code)); + return code; + } + + code = qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId); + if (code) { + + return code; } - qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId); } streamSetupScheduleTrigger(pTask); @@ -88,7 +94,7 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) { double el = (taosGetTimestampMs() - st) / 1000.0; tqDebug("s-task:%s vgId:%d expand stream task completed, elapsed time:%.2fsec", pTask->id.idStr, vgId, el); - return TSDB_CODE_SUCCESS; + return code; } void tqSetRestoreVersionInfo(SStreamTask* pTask) { @@ -156,6 +162,13 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM tDecoderClear(&decoder); + int32_t gError = streamGetFatalError(pMeta); + if (gError != 0) { + tqError("vgId:%d global fatal occurs, code:%s, ts:%" PRId64 " func:%s", pMeta->vgId, tstrerror(gError), + pMeta->fatalInfo.ts, pMeta->fatalInfo.func); + return 0; + } + // update the nodeEpset when it exists streamMetaWLock(pMeta); @@ -286,8 +299,11 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM tqDebug("vgId:%d closed tasks:%d, unclosed:%d, all tasks will be started when nodeEp update completed", vgId, updateTasks, (numOfTasks - updateTasks)); } else { - if (streamMetaCommit(pMeta) < 0) { - // persist to disk + if ((code = streamMetaCommit(pMeta)) < 0) { + // always return true + streamMetaWUnLock(pMeta); + taosArrayDestroy(req.pNodeList); + return TSDB_CODE_SUCCESS; } streamMetaClearSetUpdateTaskListComplete(pMeta); @@ -750,8 +766,9 @@ int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored } streamMetaWUnLock(pMeta); + // always return success when handling the requirement issued by mnode during transaction. - return code; + return TSDB_CODE_SUCCESS; } static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { @@ -842,7 +859,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead int32_t idle = taosGetTimestampMs() - execTs; tqDebug("s-task:%s task resume to run after idle for:%dms from:%" PRId64, pTask->id.idStr, idle, execTs); - streamResumeTask(pTask); + code = streamResumeTask(pTask); } else { int8_t status = streamTaskSetSchedStatusInactive(pTask); tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId, @@ -851,7 +868,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead streamMetaReleaseTask(pMeta, pTask); } - return 0; + return code; } SStreamTask* pTask = NULL; @@ -1193,10 +1210,6 @@ int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { ret int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); } -int32_t tqStreamProcessConsensusChkptRsp2(SStreamMeta* pMeta, SRpcMsg* pMsg) { - return doProcessDummyRspMsg(pMeta, pMsg); -} - int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { SMStreamCheckpointReadyRspMsg* pRsp = pMsg->pCont; @@ -1217,14 +1230,13 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t vgId = pMeta->vgId; int32_t code = 0; SStreamTask* pTask = NULL; - SRestoreCheckpointInfo req = {0}; char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t len = pMsg->contLen - sizeof(SMsgHead); int64_t now = taosGetTimestampMs(); + SDecoder decoder; + SRestoreCheckpointInfo req = {0}; - SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msg, len); - if (tDecodeRestoreCheckpointInfo(&decoder, &req) < 0) { tqError("vgId:%d failed to decode set consensus checkpointId req, code:%s", vgId, tstrerror(code)); tDecoderClear(&decoder); @@ -1235,16 +1247,15 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); if (pTask == NULL || (code != 0)) { - tqError( - "vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already", - pMeta->vgId, req.taskId); + tqError("vgId:%d process consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already", + pMeta->vgId, req.taskId); // ignore this code to avoid error code over write int32_t ret = streamMetaAddFailedTask(pMeta, req.streamId, req.taskId); if (ret) { tqError("s-task:0x%x failed add check downstream failed, core:%s", req.taskId, tstrerror(ret)); } - return code; + return 0; } // discard the rsp, since it is expired. @@ -1268,7 +1279,7 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { streamMutexUnlock(&pTask->lock); streamMetaReleaseTask(pMeta, pTask); - return TSDB_CODE_STREAM_INTERNAL_ERROR; + return 0; } SConsenChkptInfo* pConsenInfo = &pTask->status.consenChkptInfo; @@ -1295,10 +1306,13 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { if (pMeta->role == NODE_ROLE_LEADER) { code = tqStreamStartOneTaskAsync(pMeta, pTask->pMsgCb, req.streamId, req.taskId); + if (code) { + tqError("s-task:0x%x vgId:%d failed start task async, code:%s", req.taskId, vgId, tstrerror(code)); + } } else { tqDebug("vgId:%d follower not start task:%s", vgId, pTask->id.idStr); } streamMetaReleaseTask(pMeta, pTask); - return code; + return 0; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 6b61248d3a..9cdd4dcdb5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -2939,17 +2939,19 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pI TSDBROW *max[4] = {0}; int iMax[4] = {-1, -1, -1, -1}; int nMax = 0; - TSKEY maxKey = TSKEY_MIN; + SRowKey maxKey = {.ts = TSKEY_MIN}; for (int i = 0; i < 3; ++i) { if (!pIter->input[i].stop && pIter->input[i].pRow != NULL) { - TSDBKEY key = TSDBROW_KEY(pIter->input[i].pRow); + STsdbRowKey tsdbRowKey = {0}; + tsdbRowGetKey(pIter->input[i].pRow, &tsdbRowKey); // merging & deduplicating on client side - if (maxKey <= key.ts) { - if (maxKey < key.ts) { + int c = tRowKeyCompare(&maxKey, &tsdbRowKey.key); + if (c <= 0) { + if (c < 0) { nMax = 0; - maxKey = key.ts; + maxKey = tsdbRowKey.key; } iMax[nMax] = i; diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index b24aa6fb1d..51109a616b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -243,7 +243,7 @@ static int32_t setTableSchema(SCacheRowsReader* p, uint64_t suid, const char* id code = metaGetTbTSchemaNotNull(p->pVnode->pMeta, suid, -1, 1, &p->pSchema); if (TSDB_CODE_SUCCESS != code) { tsdbWarn("stable:%" PRIu64 " has been dropped, failed to retrieve cached rows, %s", suid, idstr); - if(code != TSDB_CODE_OUT_OF_MEMORY) { + if(code == TSDB_CODE_NOT_FOUND) { return TSDB_CODE_PAR_TABLE_NOT_EXIST; } else { return code; @@ -625,7 +625,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } double cost = (taosGetTimestampUs() - st) / 1000.0; if (cost > tsCacheLazyLoadThreshold) { - pr->lastTs = totalLastTs; + // pr->lastTs = totalLastTs; } } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index e39ac2e45d..731b733b52 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -5986,13 +5986,18 @@ void tsdbUntakeReadSnap2(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proact } // if failed, do nothing -void tsdbReaderSetId2(STsdbReader* pReader, const char* idstr) { +int32_t tsdbReaderSetId(void* p, const char* idstr) { + STsdbReader* pReader = (STsdbReader*) p; taosMemoryFreeClear(pReader->idStr); + pReader->idStr = taosStrdup(idstr); if (pReader->idStr == NULL) { - // no need to do anything + tsdbError("%s failed to build reader id, code:%s", idstr, tstrerror(terrno)); + return terrno; } + pReader->status.fileIter.pSttBlockReader->mergeTree.idStr = pReader->idStr; + return 0; } void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/ } diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index ed1632cb97..02e1a59c22 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -59,7 +59,7 @@ void initTsdbReaderAPI(TsdReader* pReader) { pReader->tsdReaderGetNumOfInMemRows = tsdbGetNumOfRowsInMemTable2; // todo this function should be moved away pReader->tsdSetQueryTableList = tsdbSetTableList2; - pReader->tsdSetReaderTaskId = (void (*)(void*, const char*))tsdbReaderSetId2; + pReader->tsdSetReaderTaskId = tsdbReaderSetId; pReader->tsdSetFilesetDelimited = (void (*)(void*))tsdbSetFilesetDelimited; pReader->tsdSetSetNotifyCb = (void (*)(void*, TsdReaderNotifyCbFn, void*))tsdbReaderSetNotifyCb; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index dff607f464..46094fd713 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -448,6 +448,10 @@ int32_t ctgGetTbTag(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, char* pJson = NULL; parseTagDatatoJson(pTag, &pJson); + if(NULL == pJson) { + taosArrayDestroy(pTagVals); + CTG_ERR_JRET(terrno); + } STagVal tagVal; tagVal.cid = 0; tagVal.type = TSDB_DATA_TYPE_JSON; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 9940474891..520c715b01 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -2093,6 +2093,10 @@ int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* char* pJson = NULL; parseTagDatatoJson(pTag, &pJson); + if (NULL == pJson) { + taosArrayDestroy(pTagVals); + CTG_ERR_JRET(terrno); + } STagVal tagVal; tagVal.cid = 0; tagVal.type = TSDB_DATA_TYPE_JSON; diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 0e54ac77a2..9a72a6d89f 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -3710,6 +3710,9 @@ int32_t ctgGetTbHashVgroupFromCache(SCatalog *pCtg, const SName *pTableName, SVg } *pVgroup = taosMemoryCalloc(1, sizeof(SVgroupInfo)); + if (NULL == *pVgroup) { + CTG_ERR_JRET(terrno); + } CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, NULL, dbCache->vgCache.vgInfo, pTableName, *pVgroup)); _return: diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index a95df06307..0ac606d4d6 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -291,6 +291,10 @@ int32_t ctgdHandleDbgCommand(char *command) { } char *dup = taosStrdup(command); + if (NULL == dup) { + CTG_RET(terrno); + } + char *option = NULL; char *param = NULL; diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 534070c540..25c82b8452 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -2889,7 +2889,7 @@ TEST(apiTest, catalogChkAuth_test) { SUserAuthInfo authInfo = {0}; SUserAuthRes authRes = {0}; TAOS_STRCPY(authInfo.user, ctgTestUsername); - (void)toName(1, ctgTestDbname, ctgTestSTablename, &authInfo.tbName); + toName(1, ctgTestDbname, ctgTestSTablename, &authInfo.tbName); authInfo.type = AUTH_TYPE_READ; bool exists = false; code = catalogChkAuthFromCache(pCtg, &authInfo, &authRes, &exists); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 2f756e8121..eb74b81141 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -565,6 +565,10 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { if (tTagIsJson(pTag)) { char* pJson = NULL; parseTagDatatoJson(pTag, &pJson); + if(NULL == pJson) { + qError("failed to parse tag to json, pJson is NULL"); + return terrno; + } *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s", pJson); taosMemoryFree(pJson); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 162a16b78a..c8180261c0 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -204,28 +204,34 @@ _end: return code; } -void doSetTaskId(SOperatorInfo* pOperator, SStorageAPI* pAPI) { +int32_t doSetTaskId(SOperatorInfo* pOperator, SStorageAPI* pAPI) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { SStreamScanInfo* pStreamScanInfo = pOperator->info; if (pStreamScanInfo->pTableScanOp != NULL) { STableScanInfo* pScanInfo = pStreamScanInfo->pTableScanOp->info; if (pScanInfo->base.dataReader != NULL) { - pAPI->tsdReader.tsdSetReaderTaskId(pScanInfo->base.dataReader, pTaskInfo->id.str); + int32_t code = pAPI->tsdReader.tsdSetReaderTaskId(pScanInfo->base.dataReader, pTaskInfo->id.str); + if (code) { + qError("failed to set reader id for executor, code:%s", tstrerror(code)); + return code; + } } } } else { - doSetTaskId(pOperator->pDownstream[0], pAPI); + return doSetTaskId(pOperator->pDownstream[0], pAPI); } + + return 0; } -void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { +int32_t qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { SExecTaskInfo* pTaskInfo = tinfo; pTaskInfo->id.queryId = queryId; buildTaskId(taskId, queryId, pTaskInfo->id.str); // set the idstr for tsdbReader - doSetTaskId(pTaskInfo->pRoot, &pTaskInfo->storageAPI); + return doSetTaskId(pTaskInfo->pRoot, &pTaskInfo->storageAPI); } int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { @@ -337,33 +343,31 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 return pTaskInfo; } -qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t vgId, int32_t taskId) { +int32_t qCreateStreamExecTaskInfo(qTaskInfo_t* pTaskInfo, void* msg, SReadHandle* readers, int32_t vgId, int32_t taskId) { if (msg == NULL) { - return NULL; + return TSDB_CODE_INVALID_PARA; } + *pTaskInfo = NULL; + SSubplan* pPlan = NULL; int32_t code = qStringToSubplan(msg, &pPlan); if (code != TSDB_CODE_SUCCESS) { - terrno = code; - return NULL; + return code; } - qTaskInfo_t pTaskInfo = NULL; - code = qCreateExecTask(readers, vgId, taskId, pPlan, &pTaskInfo, NULL, 0, NULL, OPTR_EXEC_MODEL_STREAM); + code = qCreateExecTask(readers, vgId, taskId, pPlan, pTaskInfo, NULL, 0, NULL, OPTR_EXEC_MODEL_STREAM); if (code != TSDB_CODE_SUCCESS) { - qDestroyTask(pTaskInfo); - terrno = code; - return NULL; + qDestroyTask(*pTaskInfo); + return code; } - code = qStreamInfoResetTimewindowFilter(pTaskInfo); + code = qStreamInfoResetTimewindowFilter(*pTaskInfo); if (code != TSDB_CODE_SUCCESS) { - qDestroyTask(pTaskInfo); - terrno = code; - return NULL; + qDestroyTask(*pTaskInfo); } - return pTaskInfo; + + return code; } static int32_t filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr, @@ -627,9 +631,13 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, // pSinkParam has been freed during create sinker. code = dsCreateDataSinker(pSinkManager, pSubplan->pDataSink, handle, pSinkParam, (*pTask)->id.str); + if (code) { + qError("s-task:%s failed to create data sinker, code:%s", (*pTask)->id.str, tstrerror(code)); + } } - qDebug("subplan task create completed, TID:0x%" PRIx64 "QID:0x%" PRIx64, taskId, pSubplan->id.queryId); + qDebug("subplan task create completed, TID:0x%" PRIx64 " QID:0x%" PRIx64 " code:%s", taskId, pSubplan->id.queryId, + tstrerror(code)); _error: // if failed to add ref for all tables in this query, abort current query diff --git a/source/libs/executor/src/mergejoinoperator.c b/source/libs/executor/src/mergejoinoperator.c index 4715aa8b96..af5e4ed235 100644 --- a/source/libs/executor/src/mergejoinoperator.c +++ b/source/libs/executor/src/mergejoinoperator.c @@ -149,6 +149,9 @@ int32_t mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* len = varDataTLen(p1); } char* p2 = taosMemoryMalloc(len); + if (NULL == p2) { + MJ_ERR_RET(terrno); + } TAOS_MEMCPY(p2, p1, len); code = colDataSetVal(pDst, numOfRows, p2, false); if (code) { diff --git a/source/libs/executor/src/mergeoperator.c b/source/libs/executor/src/mergeoperator.c index c94a330dbc..49973ac373 100644 --- a/source/libs/executor/src/mergeoperator.c +++ b/source/libs/executor/src/mergeoperator.c @@ -229,7 +229,7 @@ int32_t doSortMerge(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { resetLimitInfoForNextGroup(&pInfo->limitInfo); } - (void)applyLimitOffset(&pInfo->limitInfo, p, pTaskInfo); + bool limitReached = applyLimitOffset(&pInfo->limitInfo, p, pTaskInfo); if (p->info.rows > 0) { break; diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 6fca800ae7..3bdcdd084a 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -394,7 +394,6 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand } } - //pTaskInfo->schemaInfo.qsw = extractQueriedColumnSchema(&pTableScanNode->scan); code = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTableListInfo, pTaskInfo, &pOperator); if (code) { pTaskInfo->code = code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 32967c4a56..8c72742a2d 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -667,8 +667,8 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int val = *pVal; } else { pCache->cacheHit += 1; - STableCachedVal* pVal = taosLRUCacheValue(pCache->pTableMetaEntryCache, h); - val = *pVal; + STableCachedVal* pValTmp = taosLRUCacheValue(pCache->pTableMetaEntryCache, h); + val = *pValTmp; bool bRes = taosLRUCacheRelease(pCache->pTableMetaEntryCache, h, false); qTrace("release LRU cache, res %d", bRes); @@ -720,12 +720,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int if (IS_VAR_DATA_TYPE(((const STagVal*)p)->type)) { taosMemoryFree(data); } - if (code) { - if (freeReader) { - pHandle->api.metaReaderFn.clearReader(&mr); - } - return code; - } + QUERY_CHECK_CODE(code, lino, _end); } else { // todo opt for json tag for (int32_t i = 0; i < pBlock->info.rows; ++i) { code = colDataSetVal(pColInfoData, i, data, false); @@ -4094,8 +4089,8 @@ static void destroyStreamScanOperatorInfo(void* param) { if (param == NULL) { return; } - SStreamScanInfo* pStreamScan = (SStreamScanInfo*)param; + SStreamScanInfo* pStreamScan = (SStreamScanInfo*)param; if (pStreamScan->pTableScanOp && pStreamScan->pTableScanOp->info) { destroyOperator(pStreamScan->pTableScanOp); } @@ -4114,7 +4109,10 @@ static void destroyStreamScanOperatorInfo(void* param) { cleanupExprSupp(&pStreamScan->tbnameCalSup); cleanupExprSupp(&pStreamScan->tagCalSup); - pStreamScan->stateStore.updateInfoDestroy(pStreamScan->pUpdateInfo); + if (pStreamScan->stateStore.updateInfoDestroy) { + pStreamScan->stateStore.updateInfoDestroy(pStreamScan->pUpdateInfo); + } + blockDataDestroy(pStreamScan->pRes); blockDataDestroy(pStreamScan->pUpdateRes); blockDataDestroy(pStreamScan->pDeleteDataRes); @@ -4329,16 +4327,13 @@ int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* } pInfo->pBlockLists = taosArrayInit(4, sizeof(SPackedData)); - if (pInfo->pBlockLists == NULL) { - code = terrno; - goto _error; - } + TSDB_CHECK_NULL(pInfo->pBlockLists, code, lino, _error, terrno); if (pHandle->vnode) { SOperatorInfo* pTableScanOp = NULL; code = createTableScanOperatorInfo(pTableScanNode, pHandle, pTableListInfo, pTaskInfo, &pTableScanOp); if (pTableScanOp == NULL || code != 0) { - qError("createTableScanOperatorInfo error, errorcode: %d", pTaskInfo->code); + qError("createTableScanOperatorInfo error, code:%d", pTaskInfo->code); goto _error; } @@ -4382,6 +4377,7 @@ int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* // set the extract column id to streamHandle pAPI->tqReaderFn.tqReaderSetColIdList(pInfo->tqReader, pColIds); + SArray* tableIdList = NULL; code = extractTableIdList(((STableScanInfo*)(pInfo->pTableScanOp->info))->base.pTableListInfo, &tableIdList); QUERY_CHECK_CODE(code, lino, _error); @@ -4391,9 +4387,11 @@ int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* } else { taosArrayDestroy(pColIds); tableListDestroy(pTableListInfo); - pColIds = NULL; } + // clear the local variable to avoid repeatly free + pColIds = NULL; + // create the pseduo columns info if (pTableScanNode->scan.pScanPseudoCols != NULL) { code = createExprInfo(pTableScanNode->scan.pScanPseudoCols, NULL, &pInfo->pPseudoExpr, &pInfo->numOfPseudoExpr); @@ -4472,6 +4470,10 @@ _error: } if (pInfo != NULL) { + STableScanInfo* p = (STableScanInfo*) pInfo->pTableScanOp->info; + if (p != NULL) { + p->base.pTableListInfo = NULL; + } destroyStreamScanOperatorInfo(pInfo); } diff --git a/source/libs/executor/src/streamfilloperator.c b/source/libs/executor/src/streamfilloperator.c index 3fcba5fb6c..c178f22784 100644 --- a/source/libs/executor/src/streamfilloperator.c +++ b/source/libs/executor/src/streamfilloperator.c @@ -112,6 +112,9 @@ void destroyStreamFillLinearInfo(SStreamFillLinearInfo* pFillLinear) { } void destroyStreamFillInfo(SStreamFillInfo* pFillInfo) { + if (pFillInfo == NULL) { + return; + } if (pFillInfo->type == TSDB_FILL_SET_VALUE || pFillInfo->type == TSDB_FILL_SET_VALUE_F || pFillInfo->type == TSDB_FILL_NULL || pFillInfo->type == TSDB_FILL_NULL_F) { taosMemoryFreeClear(pFillInfo->pResRow->pRowVal); diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 90261a9f8e..60750e285c 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2003,10 +2003,12 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN pInfo->pUpdatedMap = NULL; pInfo->stateStore = pTaskInfo->storageAPI.stateStore; int32_t funResSize = getMaxFunResSize(&pOperator->exprSupp, numOfCols); - pInfo->pState->pFileState = pAPI->stateStore.streamFileStateInit( - tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, compareTs, pInfo->pState, - pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo), pHandle->checkpointId, STREAM_STATE_BUFF_HASH); - QUERY_CHECK_NULL(pInfo->pState->pFileState, code, lino, _error, terrno); + pInfo->pState->pFileState = NULL; + code = + pAPI->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, + compareTs, pInfo->pState, pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo), + pHandle->checkpointId, STREAM_STATE_BUFF_HASH, &pInfo->pState->pFileState); + QUERY_CHECK_CODE(code, lino, _error); pInfo->dataVersion = 0; pInfo->recvGetAll = false; @@ -2188,45 +2190,40 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, in pSup->resultRowSize = (keySize + getResultRowSize(pExpSup->pCtx, numOfOutput)) * ratio; int32_t lino = 0; int32_t code = createSpecialDataBlock(STREAM_CLEAR, &pSup->pScanBlock); - if (code) { - return code; - } + QUERY_CHECK_CODE(code, lino, _end); pSup->gap = gap; pSup->stateKeySize = keySize; pSup->stateKeyType = keyType; pSup->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx)); - if (pSup->pDummyCtx == NULL) { - return terrno; - } + QUERY_CHECK_NULL(pSup->pDummyCtx, code, lino, _end, terrno); pSup->stateStore = *pStore; pSup->pSessionAPI = pApi; initDummyFunction(pSup->pDummyCtx, pExpSup->pCtx, numOfOutput); pSup->pState = taosMemoryCalloc(1, sizeof(SStreamState)); - if (!pSup->pState) { - return terrno; - } + QUERY_CHECK_NULL(pSup->pState, code, lino, _end, terrno); + *(pSup->pState) = *pState; pSup->stateStore.streamStateSetNumber(pSup->pState, -1, tsIndex); int32_t funResSize = getMaxFunResSize(pExpSup, numOfOutput); if (stateType == STREAM_STATE_BUFF_SORT) { - pSup->pState->pFileState = pSup->stateStore.streamFileStateInit( - tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize, funResSize, sesionTs, pSup->pState, - pTwAggSup->deleteMark, taskIdStr, pHandle->checkpointId, stateType); + pSup->pState->pFileState = NULL; + code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize, + funResSize, sesionTs, pSup->pState, pTwAggSup->deleteMark, taskIdStr, + pHandle->checkpointId, stateType, &pSup->pState->pFileState); } else if (stateType == STREAM_STATE_BUFF_HASH_SORT) { - pSup->pState->pFileState = pSup->stateStore.streamFileStateInit( - tsStreamBufferSize, sizeof(SWinKey), pSup->resultRowSize, funResSize, compareTs, pSup->pState, - pTwAggSup->deleteMark, taskIdStr, pHandle->checkpointId, stateType); + pSup->pState->pFileState = NULL; + code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pSup->resultRowSize, funResSize, + compareTs, pSup->pState, pTwAggSup->deleteMark, taskIdStr, + pHandle->checkpointId, stateType, &pSup->pState->pFileState); } - QUERY_CHECK_NULL(pSup->pState->pFileState, code, lino, _end, terrno); + QUERY_CHECK_CODE(code, lino, _end); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pSup->pResultRows = tSimpleHashInit(32, hashFn); - if (!pSup->pResultRows) { - return terrno; - } + QUERY_CHECK_NULL(pSup->pResultRows, code, lino, _end, terrno); for (int32_t i = 0; i < numOfOutput; ++i) { pExpSup->pCtx[i].saveHandle.pState = pSup->pState; @@ -5368,10 +5365,11 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* int32_t funResSize = getMaxFunResSize(pSup, numOfCols); pInfo->stateStore = pTaskInfo->storageAPI.stateStore; - pInfo->pState->pFileState = pTaskInfo->storageAPI.stateStore.streamFileStateInit( + pInfo->pState->pFileState = NULL; + code = pTaskInfo->storageAPI.stateStore.streamFileStateInit( tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, compareTs, pInfo->pState, - pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo), pHandle->checkpointId, STREAM_STATE_BUFF_HASH); - QUERY_CHECK_NULL(pInfo->pState->pFileState, code, lino, _error, terrno); + pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo), pHandle->checkpointId, STREAM_STATE_BUFF_HASH, &pInfo->pState->pFileState); + QUERY_CHECK_CODE(code, lino, _error); setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, pInfo, pTaskInfo); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index bf1153f412..18344d9e79 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1114,6 +1114,10 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, if (tagType == TSDB_DATA_TYPE_JSON) { char* tagJson = NULL; parseTagDatatoJson(tagData, &tagJson); + if (tagJson == NULL) { + code = terrno; + goto _end; + } tagVarChar = taosMemoryMalloc(strlen(tagJson) + VARSTR_HEADER_SIZE); QUERY_CHECK_NULL(tagVarChar, code, lino, _end, terrno); memcpy(varDataVal(tagVarChar), tagJson, strlen(tagJson)); diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c index 763fc6a412..69ce50e150 100644 --- a/source/libs/executor/src/tlinearhash.c +++ b/source/libs/executor/src/tlinearhash.c @@ -150,10 +150,11 @@ static void doRemoveFromBucket(SFilePage* pPage, SLHashNode* pNode, SLHashBucket pBucket->size -= 1; } -static void doTrimBucketPages(SLHashObj* pHashObj, SLHashBucket* pBucket) { +static int32_t doTrimBucketPages(SLHashObj* pHashObj, SLHashBucket* pBucket) { + int32_t code = 0; size_t numOfPages = taosArrayGetSize(pBucket->pPageIdList); if (numOfPages <= 1) { - return; + return code; } int32_t* firstPage = taosArrayGet(pBucket->pPageIdList, 0); @@ -164,11 +165,14 @@ static void doTrimBucketPages(SLHashObj* pHashObj, SLHashBucket* pBucket) { if (pLast->num <= sizeof(SFilePage)) { // this is empty - // TODO check ret - (void)dBufSetBufPageRecycled(pHashObj->pBuf, pLast); + code = dBufSetBufPageRecycled(pHashObj->pBuf, pLast); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed to recycle buf page since %s", __func__, tstrerror(code)); + return code; + } releaseBufPage(pHashObj->pBuf, pFirst); taosArrayRemove(pBucket->pPageIdList, numOfPages - 1); - return; + return code; } char* pStart = pLast->data; @@ -191,8 +195,11 @@ static void doTrimBucketPages(SLHashObj* pHashObj, SLHashBucket* pBucket) { pStart += nodeSize; if (pLast->num <= sizeof(SFilePage)) { // this is empty - // TODO check ret - (void)dBufSetBufPageRecycled(pHashObj->pBuf, pLast); + code = dBufSetBufPageRecycled(pHashObj->pBuf, pLast); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed to recycle buf page since %s", __func__, tstrerror(code)); + return code; + } releaseBufPage(pHashObj->pBuf, pFirst); taosArrayRemove(pBucket->pPageIdList, numOfPages - 1); break; @@ -210,6 +217,7 @@ static void doTrimBucketPages(SLHashObj* pHashObj, SLHashBucket* pBucket) { break; } } + return code; } static int32_t doAddNewBucket(SLHashObj* pHashObj) { @@ -403,7 +411,10 @@ int32_t tHashPut(SLHashObj* pHashObj, const void* key, size_t keyLen, void* data releaseBufPage(pHashObj->pBuf, p); } - doTrimBucketPages(pHashObj, pBucket); + code = doTrimBucketPages(pHashObj, pBucket); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } return code; diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 796ebbeb84..19b825b0ca 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -1308,8 +1308,7 @@ static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, i return terrno; } - // todo - (void)taosSeekCFile(pMemFile->pTdFile, pRegion->fileOffset, SEEK_SET); + TAOS_CHECK_RETURN(taosSeekCFile(pMemFile->pTdFile, pRegion->fileOffset, SEEK_SET)); int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize); int32_t ret = taosReadFromCFile(pRegion->buf, readBytes, 1, pMemFile->pTdFile); diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index d0b3b191ad..c2cd8b8c3d 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -127,7 +127,10 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) { snprintf(dnodeIdEnvItem, 32, "%s=%d", "DNODE_ID", pData->dnodeId); float numCpuCores = 4; - taosGetCpuCores(&numCpuCores, false); + int32_t code = taosGetCpuCores(&numCpuCores, false); + if(code != 0) { + fnError("failed to get cpu cores, code:%d", code); + } numCpuCores = TMAX(numCpuCores, 2); snprintf(thrdPoolSizeEnvItem, 32, "%s=%d", "UV_THREADPOOL_SIZE", (int)numCpuCores * 2); @@ -897,6 +900,9 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo udfCol->colData.fixLenCol.dataLen = colDataGetLength(col, udfBlock->numOfRows); int32_t dataLen = udfCol->colData.fixLenCol.dataLen; udfCol->colData.fixLenCol.data = taosMemoryMalloc(udfCol->colData.fixLenCol.dataLen); + if (NULL == udfCol->colData.fixLenCol.data) { + return terrno; + } char *data = udfCol->colData.fixLenCol.data; memcpy(data, col->pData, dataLen); } diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 6ae718c101..cd54c03a68 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -85,13 +85,19 @@ int32_t udfdCPluginUdfInitLoadAggFuncs(SUdfCPluginCtx *udfCtx, const char *udfNa char mergeFuncName[TSDB_FUNC_NAME_LEN + 7] = {0}; char *mergeSuffix = "_merge"; snprintf(mergeFuncName, sizeof(mergeFuncName), "%s%s", processFuncName, mergeSuffix); - (void)(uv_dlsym(&udfCtx->lib, mergeFuncName, (void **)(&udfCtx->aggMergeFunc))); + int ret = uv_dlsym(&udfCtx->lib, mergeFuncName, (void **)(&udfCtx->aggMergeFunc)); + if (ret != 0) { + fnInfo("uv_dlsym function %s. error: %s", mergeFuncName, uv_strerror(ret)); + } return 0; } int32_t udfdCPluginUdfInit(SScriptUdfInfo *udf, void **pUdfCtx) { int32_t err = 0; SUdfCPluginCtx *udfCtx = taosMemoryCalloc(1, sizeof(SUdfCPluginCtx)); + if (NULL == udfCtx) { + return terrno; + } err = uv_dlopen(udf->path, &udfCtx->lib); if (err != 0) { fnError("can not load library %s. error: %s", udf->path, uv_strerror(err)); @@ -606,6 +612,9 @@ int32_t udfdInitUdf(char *udfName, SUdf *udf) { int32_t udfdNewUdf(SUdf **pUdf, const char *udfName) { SUdf *udfNew = taosMemoryCalloc(1, sizeof(SUdf)); + if (NULL == udfNew) { + return terrno; + } udfNew->refCount = 1; udfNew->lastFetchTime = taosGetTimestampMs(); strncpy(udfNew->name, udfName, TSDB_FUNC_NAME_LEN); @@ -1105,6 +1114,9 @@ int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf) { taosArrayDestroy(retrieveReq.pFuncNames); SUdfdRpcSendRecvInfo *msgInfo = taosMemoryCalloc(1, sizeof(SUdfdRpcSendRecvInfo)); + if(NULL == msgInfo) { + return terrno; + } msgInfo->rpcType = UDFD_RPC_RETRIVE_FUNC; msgInfo->param = udf; if(uv_sem_init(&msgInfo->resultSem, 0) != 0) { diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index 7fbbd90d40..9f738b32e8 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -169,7 +169,8 @@ static EDealRes authSelectImpl(SNode* pNode, void* pContext) { SNode* pTagCond = NULL; STableNode* pTable = (STableNode*)pNode; #ifdef TD_ENTERPRISE - SName name; + SName name = {0}; + toName(pAuthCxt->pParseCxt->acctId, pTable->dbName, pTable->tableName, &name); STableMeta* pTableMeta = NULL; toName(pAuthCxt->pParseCxt->acctId, pTable->dbName, pTable->tableName, &name); int32_t code = getTargetMetaImpl( diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index ff701e7fa2..9f3d5b5a81 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -892,11 +892,11 @@ int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TA if (NULL == tags) { return TSDB_CODE_APP_ERROR; } - + /* if (pDataBlock->pMeta->tableType != TSDB_SUPER_TABLE && pDataBlock->pMeta->tableType != TSDB_CHILD_TABLE) { return TSDB_CODE_TSC_STMT_API_ERROR; } - + */ SSchema* pSchema = getTableTagSchema(pDataBlock->pMeta); if (tags->numOfBound <= 0) { *fieldNum = 0; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 97e10d0d2f..59b5cf53b4 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -525,7 +525,7 @@ static int32_t getTargetMeta(STranslateContext* pCxt, const SName* pName, STable } static int32_t getTableMeta(STranslateContext* pCxt, const char* pDbName, const char* pTableName, STableMeta** pMeta) { - SName name; + SName name = {0}; toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name); return getTargetMeta(pCxt, &name, pMeta, false); } @@ -557,7 +557,7 @@ static int32_t getTableCfg(STranslateContext* pCxt, const SName* pName, STableCf static int32_t refreshGetTableMeta(STranslateContext* pCxt, const char* pDbName, const char* pTableName, STableMeta** pMeta) { SParseContext* pParCxt = pCxt->pParseCxt; - SName name; + SName name = {0}; toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name); int32_t code = TSDB_CODE_SUCCESS; if (pParCxt->async) { @@ -635,7 +635,7 @@ static int32_t getTableHashVgroupImpl(STranslateContext* pCxt, const SName* pNam static int32_t getTableHashVgroup(STranslateContext* pCxt, const char* pDbName, const char* pTableName, SVgroupInfo* pInfo) { - SName name; + SName name = {0}; toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name); return getTableHashVgroupImpl(pCxt, &name, pInfo); } @@ -4686,9 +4686,9 @@ int32_t translateTable(STranslateContext* pCxt, SNode** pTable, SNode* pJoinPare pRealTable->ratio = (NULL != pCxt->pExplainOpt ? pCxt->pExplainOpt->ratio : 1.0); // The SRealTableNode created through ROLLUP already has STableMeta. if (NULL == pRealTable->pMeta) { - SName name; + SName name = {0}; toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name); - code = getTargetMeta( pCxt, &name, &(pRealTable->pMeta), true); + code = getTargetMeta(pCxt, &name, &(pRealTable->pMeta), true); if (TSDB_CODE_SUCCESS != code) { (void)generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code)); return code; @@ -6006,7 +6006,7 @@ static int32_t isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pO *pRet = false; return TSDB_CODE_SUCCESS; } - + SFunctionNode* pTbnameFunc = NULL; SValueNode* pValueNode = NULL; if (nodeType(pOperator->pLeft) == QUERY_NODE_FUNCTION && @@ -6250,7 +6250,7 @@ static void findVgroupsFromEqualTbname(STranslateContext* pCxt, SArray* aTbnames } for (int j = 0; j < nTbls; ++j) { - SName snameTb; + SName snameTb = {0}; char* tbName = taosArrayGetP(aTbnames, j); toName(pCxt->pParseCxt->acctId, dbName, tbName, &snameTb); SVgroupInfo vgInfo = {0}; @@ -6277,7 +6277,7 @@ static void findVgroupsFromEqualTbname(STranslateContext* pCxt, SArray* aTbnames } static int32_t replaceToChildTableQuery(STranslateContext* pCxt, SEqCondTbNameTableInfo* pInfo) { - SName snameTb; + SName snameTb = {0}; int32_t code = 0; SRealTableNode* pRealTable = pInfo->pRealTable; char* tbName = taosArrayGetP(pInfo->aTbnames, 0); @@ -6288,7 +6288,7 @@ static int32_t replaceToChildTableQuery(STranslateContext* pCxt, SEqCondTbNameTa if (NULL == pMeta || TSDB_CHILD_TABLE != pMeta->tableType || pMeta->suid != pRealTable->pMeta->suid) { goto _return; } - + pRealTable->pMeta->uid = pMeta->uid; pRealTable->pMeta->vgId = pMeta->vgId; pRealTable->pMeta->tableType = pMeta->tableType; @@ -6399,11 +6399,11 @@ static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt* } qDebug("before ctbname optimize, code:%d, aTableNum:%d, nTbls:%d, stableQuery:%d", code, aTableNum, nTbls, stableQuery); - + if (TSDB_CODE_SUCCESS == code && 1 == aTableNum && 1 == nTbls && stableQuery && NULL == pInfo->pRealTable->pTsmas) { code = replaceToChildTableQuery(pCxt, pInfo); } - + return code; } @@ -6810,7 +6810,7 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { if (pCxt->pParseCxt && pCxt->pParseCxt->setQueryFp) { (*pCxt->pParseCxt->setQueryFp)(pCxt->pParseCxt->requestRid); } - + if (NULL == pSelect->pFromTable) { return translateSelectWithoutFrom(pCxt, pSelect); } else { @@ -8810,7 +8810,7 @@ static int32_t buildRollupFuncs(SNodeList* pFuncs, SArray** pArray) { static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStmt, SMCreateStbReq* pReq) { int32_t code = TSDB_CODE_SUCCESS; - SName tableName; + SName tableName = {0}; pReq->igExists = pStmt->ignoreExists; pReq->delay1 = pStmt->pOptions->maxDelay1; pReq->delay2 = pStmt->pOptions->maxDelay2; @@ -8885,20 +8885,20 @@ static int32_t doTranslateDropSuperTable(STranslateContext* pCxt, const SName* p static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt) { SDropTableClause* pClause = (SDropTableClause*)nodesListGetNode(pStmt->pTables, 0); - SName tableName; + SName tableName = {0}; if (pStmt->withTsma) return TSDB_CODE_SUCCESS; toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &tableName); - return doTranslateDropSuperTable( pCxt, &tableName, pClause->ignoreNotExists); + return doTranslateDropSuperTable(pCxt, &tableName, pClause->ignoreNotExists); } static int32_t translateDropSuperTable(STranslateContext* pCxt, SDropSuperTableStmt* pStmt) { - SName tableName; + SName tableName = {0}; toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName); return doTranslateDropSuperTable(pCxt, &tableName, pStmt->ignoreNotExists); } static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, SMAlterStbReq* pAlterReq) { - SName tableName; + SName tableName = {0}; toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName); int32_t code = tNameExtractFullName(&tableName, pAlterReq->name); if (TSDB_CODE_SUCCESS != code) return code; @@ -9415,7 +9415,7 @@ static int32_t getSmaIndexAst(STranslateContext* pCxt, SCreateIndexStmt* pStmt, } static int32_t buildCreateSmaReq(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SMCreateSmaReq* pReq) { - SName name; + SName name = {0}; toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name); int32_t code = tNameExtractFullName(&name, pReq->name); if (TSDB_CODE_SUCCESS == code) { @@ -9568,7 +9568,7 @@ static int32_t buildCreateFullTextReq(STranslateContext* pCxt, SCreateIndexStmt* } static int32_t buildCreateTagIndexReq(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SCreateTagIndexReq* pReq) { - SName name; + SName name = {0}; toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name); int32_t code = tNameExtractFullName(&name, pReq->idxName); if (TSDB_CODE_SUCCESS == code) { @@ -9608,7 +9608,7 @@ static int32_t translateCreateFullTextIndex(STranslateContext* pCxt, SCreateInde static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { int32_t code = 0; - SName name; + SName name = {0}; STableMeta* pMeta = NULL; toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); code = getTargetMeta(pCxt, &name, &pMeta, false); @@ -9652,7 +9652,7 @@ static int32_t translateCreateIndex(STranslateContext* pCxt, SCreateIndexStmt* p static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt) { SMDropSmaReq dropSmaReq = {0}; - SName name; + SName name = {0}; toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name); int32_t code = tNameExtractFullName(&name, dropSmaReq.name); if (TSDB_CODE_SUCCESS != code) return code; @@ -9726,7 +9726,7 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS } int32_t code = TSDB_CODE_SUCCESS; - SName name; + SName name = {0}; if ('\0' != pStmt->subSTbName[0]) { pReq->subType = TOPIC_SUB_TYPE__TABLE; toName(pCxt->pParseCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name); @@ -9854,7 +9854,7 @@ static int32_t buildQueryForTableTopic(STranslateContext* pCxt, SCreateTopicStmt .requestId = pParCxt->requestId, .requestObjRefId = pParCxt->requestRid, .mgmtEps = pParCxt->mgmtEpSet}; - SName name; + SName name = {0}; STableMeta* pMeta = NULL; toName(pParCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name); int32_t code = getTargetMeta(pCxt, &name, &pMeta, false); @@ -9978,7 +9978,7 @@ static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt) #ifdef TD_ENTERPRISE if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { int32_t origCode = code; - SName name; + SName name = {0}; toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); SViewMeta* pMeta = NULL; code = getViewMetaFromMetaCache(pCxt, &name, &pMeta); @@ -10096,7 +10096,7 @@ static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pSt #ifdef TD_ENTERPRISE SRealTableNode* pRealTable = (SRealTableNode*)((SSelectStmt*)pStmt->pQuery)->pFromTable; - SName name; + SName name = {0}; STableMeta* pMeta = NULL; int8_t tableType = 0; toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name); @@ -11415,7 +11415,7 @@ static int32_t translateCreateView(STranslateContext* pCxt, SCreateViewStmt* pSt #endif SParseSqlRes res = {.resType = PARSE_SQL_RES_SCHEMA}; - SName name; + SName name = {0}; char dbFName[TSDB_DB_FNAME_LEN]; toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); (void)tNameGetFullDbName(&name, dbFName); @@ -11457,7 +11457,7 @@ static int32_t translateDropView(STranslateContext* pCxt, SDropViewStmt* pStmt) #endif SCMDropViewReq dropReq = {0}; - SName name; + SName name = {0}; int32_t code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); if (TSDB_CODE_SUCCESS == code) { (void)tNameGetFullDbName(&name, dropReq.dbFName); @@ -11578,7 +11578,7 @@ static int32_t translateGrantTagCond(STranslateContext* pCxt, SGrantStmt* pStmt, int32_t code = createRealTableForGrantTable(pStmt, &pTable); if (TSDB_CODE_SUCCESS == code) { - SName name; + SName name = {0}; toName(pCxt->pParseCxt->acctId, pTable->table.dbName, pTable->table.tableName, &name); code = getTargetMeta(pCxt, &name, &(pTable->pMeta), false); if (code) { @@ -11620,7 +11620,7 @@ static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) { req.privileges = pStmt->privileges; #ifdef TD_ENTERPRISE if (0 != pStmt->tabName[0]) { - SName name; + SName name = {0}; STableMeta* pTableMeta = NULL; toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name); code = getTargetMeta(pCxt, &name, &pTableMeta, true); @@ -11656,7 +11656,7 @@ static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) { #ifdef TD_ENTERPRISE if (0 != pStmt->tabName[0]) { - SName name; + SName name = {0}; STableMeta* pTableMeta = NULL; toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name); code = getTargetMeta(pCxt, &name, &pTableMeta, true); @@ -11772,7 +11772,7 @@ static int32_t translateShowCreateTable(STranslateContext* pCxt, SShowCreateTabl } int32_t code = getDBCfg(pCxt, pStmt->dbName, (SDbCfgInfo*)pStmt->pDbCfg); if (TSDB_CODE_SUCCESS == code) { - SName name; + SName name = {0}; toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); code = getTableCfg(pCxt, &name, (STableCfg**)&pStmt->pTableCfg); } @@ -11783,7 +11783,7 @@ static int32_t translateShowCreateView(STranslateContext* pCxt, SShowCreateViewS #ifndef TD_ENTERPRISE return TSDB_CODE_OPS_NOT_SUPPORT; #else - SName name; + SName name = {0}; toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); return getViewMetaFromMetaCache(pCxt, &name, (SViewMeta**)&pStmt->pViewMeta); #endif @@ -12052,7 +12052,7 @@ static int32_t rewriteTSMAFuncs(STranslateContext* pCxt, SCreateTSMAStmt* pStmt, static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStmt, SMCreateSmaReq* pReq, SName* useTbName) { - SName name; + SName name = {0}; SDbCfgInfo pDbInfo = {0}; int32_t code = TSDB_CODE_SUCCESS; toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tsmaName, &name); @@ -12271,7 +12271,7 @@ int32_t translatePostCreateTSMA(SParseContext* pParseCxt, SQuery* pQuery, SSData static int32_t translateDropTSMA(STranslateContext* pCxt, SDropTSMAStmt* pStmt) { int32_t code = TSDB_CODE_SUCCESS; SMDropSmaReq dropReq = {0}; - SName name; + SName name = {0}; STableTSMAInfo* pTsma = NULL; toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tsmaName, &name); code = tNameExtractFullName(&name, dropReq.name); @@ -13178,7 +13178,7 @@ static int32_t rewriteShowVgroups(STranslateContext* pCxt, SQuery* pQuery) { static int32_t checkShowTags(STranslateContext* pCxt, const SShowStmt* pShow) { int32_t code = 0; - SName name; + SName name = {0}; STableMeta* pTableMeta = NULL; toName(pCxt->pParseCxt->acctId, ((SValueNode*)pShow->pDbName)->literal, ((SValueNode*)pShow->pTbName)->literal, &name); code = getTargetMeta(pCxt, &name, &pTableMeta, true); @@ -13539,7 +13539,7 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) { int32_t code = checkCreateTable(pCxt, pStmt, false); SVgroupInfo info = {0}; - SName name; + SName name = {0}; toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); if (TSDB_CODE_SUCCESS == code) { code = getTableHashVgroupImpl(pCxt, &name, &info); @@ -13787,7 +13787,7 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla code = getTableMeta(pCxt, pStmt->useDbName, pStmt->useTableName, &pSuperTableMeta); } if (TSDB_CODE_SUCCESS == code) { - SName name; + SName name = {0}; toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); code = collectUseTable(&name, pCxt->pTargetTables); } @@ -13901,7 +13901,6 @@ _OUT: } typedef struct SParseFileContext { - SHashObj* pTbNameHash; SArray* aTagNames; bool tagNameFilled; STableMeta* pStbMeta; @@ -14036,18 +14035,6 @@ static int32_t parseCsvFile(SMsgBuf* pMsgBuf, SParseContext* pParseCxt, SParseFi code = parseOneStbRow(pMsgBuf, pParFileCxt); - if (TSDB_CODE_SUCCESS == code) { - if (taosHashGet(pParFileCxt->pTbNameHash, pParFileCxt->ctbName.tname, strlen(pParFileCxt->ctbName.tname) + 1) != - NULL) { - taosMemoryFreeClear(pParFileCxt->pTag); - code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_TBNAME_DUPLICATED, pParFileCxt->ctbName.tname); - break; - } - - code = taosHashPut(pParFileCxt->pTbNameHash, pParFileCxt->ctbName.tname, strlen(pParFileCxt->ctbName.tname) + 1, - NULL, 0); - } - if (TSDB_CODE_SUCCESS == code) { code = fillVgroupInfo(pParseCxt, &pParFileCxt->ctbName, &pParFileCxt->vg); } @@ -14087,7 +14074,6 @@ static void destructParseFileContext(SParseFileContext** ppParFileCxt) { SParseFileContext* pParFileCxt = *ppParFileCxt; - taosHashCleanup(pParFileCxt->pTbNameHash); taosArrayDestroy(pParFileCxt->aTagNames); taosMemoryFreeClear(pParFileCxt->pStbMeta); taosArrayDestroy(pParFileCxt->aTagIndexs); @@ -14112,15 +14098,6 @@ static int32_t constructParseFileContext(SCreateSubTableFromFileClause* pStmt, S pParFileCxt->ctbName.acctId = acctId; strcpy(pParFileCxt->ctbName.dbname, pStmt->useDbName); - if (NULL == pParFileCxt->pTbNameHash) { - pParFileCxt->pTbNameHash = - taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); - if (!pParFileCxt->pTbNameHash) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _ERR; - } - } - if (NULL == pParFileCxt->aTagNames) { pParFileCxt->aTagNames = taosArrayInit(8, TSDB_COL_NAME_LEN); if (NULL == pParFileCxt->aTagNames) { @@ -14553,7 +14530,7 @@ static int32_t rewriteDropTable(STranslateContext* pCxt, SQuery* pQuery) { taosHashSetFreeFp(pVgroupHashmap, destroyDropTbReqBatch); FOREACH(pNode, pStmt->pTables) { SDropTableClause* pClause = (SDropTableClause*)pNode; - SName name; + SName name = {0}; toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &name); int32_t code = buildDropTableVgroupHashmap(pCxt, pClause, &name, &tableType, pVgroupHashmap); if (TSDB_CODE_SUCCESS != code) { @@ -14812,7 +14789,7 @@ static int32_t buildRenameColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt } if (TSDB_NORMAL_TABLE == pTableMeta->tableType) { SArray* pTsmas = NULL; - SName tbName; + SName tbName = {0}; int32_t code = 0; toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tbName); if (pCxt->pMetaCache) code = getTableTsmasFromCache(pCxt->pMetaCache, &tbName, &pTsmas); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index b15b884079..540e920326 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1327,6 +1327,10 @@ static int32_t createFillLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect code = createColumnByRewriteExpr(pFill->pWStartTs, &pFill->node.pTargets); } + if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving) { + code = nodesCloneNode(pSelect->pHaving, &pFill->node.pConditions); + } + if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pFill; } else { diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 30902269c3..e08d4cef87 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -514,6 +514,9 @@ end: taosArrayDestroy(pTagVals); if (string == NULL) { string = taosStrdup(TSDB_DATA_NULL_STR_L); + if(string == NULL) { + qError("failed to strdup null string"); + } } *jsonStr = string; } @@ -629,6 +632,7 @@ int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst) { (*pDst)->flags = pSrc->flags; if (pSrc->name) { (*pDst)->name = taosStrdup(pSrc->name); + if (NULL == (*pDst)->name) goto _exit; } (*pDst)->uid = pSrc->uid; (*pDst)->btime = pSrc->btime; @@ -636,21 +640,25 @@ int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst) { (*pDst)->commentLen = pSrc->commentLen; if (pSrc->comment) { (*pDst)->comment = taosStrdup(pSrc->comment); + if (NULL == (*pDst)->comment) goto _exit; } (*pDst)->type = pSrc->type; if (pSrc->type == TSDB_CHILD_TABLE) { if (pSrc->ctb.stbName) { (*pDst)->ctb.stbName = taosStrdup(pSrc->ctb.stbName); + if (NULL == (*pDst)->ctb.stbName) goto _exit; } (*pDst)->ctb.tagNum = pSrc->ctb.tagNum; (*pDst)->ctb.suid = pSrc->ctb.suid; if (pSrc->ctb.tagName) { (*pDst)->ctb.tagName = taosArrayDup(pSrc->ctb.tagName, NULL); + if (NULL == (*pDst)->ctb.tagName) goto _exit; } STag* pTag = (STag*)pSrc->ctb.pTag; if (pTag) { (*pDst)->ctb.pTag = taosMemoryMalloc(pTag->len); + if(NULL == (*pDst)->ctb.pTag) goto _exit; memcpy((*pDst)->ctb.pTag, pTag, pTag->len); } } else { @@ -658,11 +666,17 @@ int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst) { (*pDst)->ntb.schemaRow.version = pSrc->ntb.schemaRow.nCols; if (pSrc->ntb.schemaRow.nCols > 0 && pSrc->ntb.schemaRow.pSchema) { (*pDst)->ntb.schemaRow.pSchema = taosMemoryMalloc(pSrc->ntb.schemaRow.nCols * sizeof(SSchema)); + if (NULL == (*pDst)->ntb.schemaRow.pSchema) goto _exit; memcpy((*pDst)->ntb.schemaRow.pSchema, pSrc->ntb.schemaRow.pSchema, pSrc->ntb.schemaRow.nCols * sizeof(SSchema)); } } return TSDB_CODE_SUCCESS; + +_exit: + tdDestroySVCreateTbReq(*pDst); + taosMemoryFree(*pDst); + return terrno; } void freeDbCfgInfo(SDbCfgInfo* pInfo) { diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index b5b660a51b..8725496ed2 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -85,6 +85,9 @@ int32_t queryBuildTableMetaReqMsg(void *input, char **msg, int32_t msgSize, int3 int32_t bufLen = tSerializeSTableInfoReq(NULL, 0, &infoReq); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeSTableInfoReq(pBuf, bufLen, &infoReq) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -112,6 +115,9 @@ int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *ms int32_t bufLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeSUseDbReq(pBuf, bufLen, &usedbReq) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -133,6 +139,9 @@ int32_t queryBuildQnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t int32_t bufLen = tSerializeSQnodeListReq(NULL, 0, &qnodeListReq); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeSQnodeListReq(pBuf, bufLen, &qnodeListReq) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -154,6 +163,9 @@ int32_t queryBuildDnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t int32_t bufLen = tSerializeSDnodeListReq(NULL, 0, &dnodeListReq); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeSDnodeListReq(pBuf, bufLen, &dnodeListReq) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -174,6 +186,9 @@ int32_t queryBuildGetSerVerMsg(void *input, char **msg, int32_t msgSize, int32_t int32_t bufLen = tSerializeSServerVerReq(NULL, 0, &req); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeSServerVerReq(pBuf, bufLen, &req) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -195,6 +210,9 @@ int32_t queryBuildGetDBCfgMsg(void *input, char **msg, int32_t msgSize, int32_t int32_t bufLen = tSerializeSDbCfgReq(NULL, 0, &dbCfgReq); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeSDbCfgReq(pBuf, bufLen, &dbCfgReq) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -216,6 +234,9 @@ int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t int32_t bufLen = tSerializeSUserIndexReq(NULL, 0, &indexReq); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeSUserIndexReq(pBuf, bufLen, &indexReq) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -247,8 +268,13 @@ int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int3 int32_t bufLen = tSerializeSRetrieveFuncReq(NULL, 0, &funcReq); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + taosArrayDestroy(funcReq.pFuncNames); + return terrno; + } if(tSerializeSRetrieveFuncReq(pBuf, bufLen, &funcReq) < 0) { + taosArrayDestroy(funcReq.pFuncNames); return TSDB_CODE_TSC_INVALID_INPUT; } @@ -270,6 +296,9 @@ int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32 int32_t bufLen = tSerializeSGetUserAuthReq(NULL, 0, &req); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if (tSerializeSGetUserAuthReq(pBuf, bufLen, &req) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -290,6 +319,9 @@ int32_t queryBuildGetTbIndexMsg(void *input, char **msg, int32_t msgSize, int32_ int32_t bufLen = tSerializeSTableIndexReq(NULL, 0, &indexReq); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeSTableIndexReq(pBuf, bufLen, &indexReq) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -314,6 +346,9 @@ int32_t queryBuildGetTbCfgMsg(void *input, char **msg, int32_t msgSize, int32_t int32_t bufLen = tSerializeSTableCfgReq(NULL, 0, &cfgReq); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeSTableCfgReq(pBuf, bufLen, &cfgReq) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -335,6 +370,9 @@ int32_t queryBuildGetViewMetaMsg(void *input, char **msg, int32_t msgSize, int32 int32_t bufLen = tSerializeSViewMetaReq(NULL, 0, &req); void *pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeSViewMetaReq(pBuf, bufLen, &req) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -357,6 +395,9 @@ int32_t queryBuildGetTableTSMAMsg(void *input, char **msg, int32_t msgSize, int3 int32_t bufLen = tSerializeTableTSMAInfoReq(NULL, 0, &req); void * pBuf = (*mallcFp)(bufLen); + if (NULL == pBuf) { + return terrno; + } if(tSerializeTableTSMAInfoReq(pBuf, bufLen, &req) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -379,6 +420,10 @@ int32_t queryBuildGetTSMAMsg(void *input, char **msg, int32_t msgSize, int32_t * int32_t bufLen = tSerializeTableTSMAInfoReq(NULL, 0, &req); void * pBuf = (*mallcFp)(bufLen); + if(pBuf == NULL) + { + return terrno; + } if(tSerializeTableTSMAInfoReq(pBuf, bufLen, &req) < 0) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -396,6 +441,9 @@ int32_t queryBuildGetStreamProgressMsg(void* input, char** msg, int32_t msgSize, int32_t len = tSerializeStreamProgressReq(NULL, 0, input); void* pBuf = (*mallcFp)(len); + if (NULL == pBuf) { + return terrno; + } if(tSerializeStreamProgressReq(pBuf, len, input) < 0) { @@ -666,6 +714,9 @@ int32_t queryProcessGetSerVerRsp(void *output, char *msg, int32_t msgSize) { } *(char **)output = taosStrdup(out.ver); + if (NULL == *(char **)output) { + return terrno; + } return code; } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 8ffc588d41..752bb2c0f5 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -2866,32 +2866,35 @@ _return: } int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray *group) { - size_t groupSize = taosArrayGetSize(group); + size_t groupSize = taosArrayGetSize(group); + int32_t code = TSDB_CODE_SUCCESS; info->groupNum = (uint32_t)groupSize; if (info->groupNum > 0) { info->groups = taosMemoryCalloc(info->groupNum, sizeof(*info->groups)); if (info->groups == NULL) { - info->groupNum = 0; - FLT_ERR_RET(terrno); + FLT_ERR_JRET(terrno); } } for (size_t i = 0; i < groupSize; ++i) { SFilterGroup *pg = taosArrayGet(group, i); if (NULL == pg) { - FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE); + FLT_ERR_JRET(TSDB_CODE_OUT_OF_RANGE); } pg->unitFlags = taosMemoryCalloc(pg->unitNum, sizeof(*pg->unitFlags)); if (pg->unitFlags == NULL) { pg->unitNum = 0; - FLT_ERR_RET(terrno); + FLT_ERR_JRET(terrno); } info->groups[i] = *pg; } - return TSDB_CODE_SUCCESS; + +_return: + info->groupNum = 0; + return code; } int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gResNum) { @@ -2912,6 +2915,8 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gResNum uint32_t uidx = 0; uint32_t code = TSDB_CODE_SUCCESS; SArray *group = taosArrayInit(FILTER_DEFAULT_GROUP_SIZE, sizeof(SFilterGroup)); + SFilterGroup ng = {0}; + if (group == NULL) { FLT_ERR_JRET(terrno); } @@ -2927,11 +2932,8 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gResNum for (int32_t i = 0; i < gResNum; ++i) { res = gRes[i]; - optr = (res->colNum > 1) ? LOGIC_COND_TYPE_AND : LOGIC_COND_TYPE_OR; - SFilterGroup ng = {0}; - for (uint32_t m = 0; m < res->colNum; ++m) { colInfo = &res->colInfo[res->colIdx[m]]; if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { @@ -2944,28 +2946,16 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gResNum for (int32_t n = 0; n < usize; ++n) { SFilterUnit *u = (SFilterUnit *)taosArrayGetP((SArray *)colInfo->info, n); if (NULL == u) { - code = TSDB_CODE_OUT_OF_RANGE; - break; + FLT_ERR_JRET(TSDB_CODE_OUT_OF_RANGE); } code = filterAddUnitFromUnit(info, &oinfo, u, &uidx); - if (TSDB_CODE_SUCCESS != code) { - break; - } + FLT_ERR_JRET(code); code = filterAddUnitToGroup(&ng, uidx); - if (TSDB_CODE_SUCCESS != code) { - break; - } - } - if (TSDB_CODE_SUCCESS != code) { - break; + FLT_ERR_JRET(code); } continue; } - if (TSDB_CODE_SUCCESS != code) { - filterFreeGroup((void*)&ng); - FLT_ERR_JRET(code); - } if (colInfo->type != RANGE_TYPE_MR_CTX) { fltError("filterRewrite get invalid col type : %d", colInfo->type); @@ -2973,24 +2963,28 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gResNum } code = filterAddGroupUnitFromCtx(info, &oinfo, colInfo->info, res->colIdx[m], &ng, optr, group); - if (TSDB_CODE_SUCCESS != code) { - filterFreeGroup((void*)&ng); - FLT_ERR_JRET(code); - } + FLT_ERR_JRET(code); } if (ng.unitNum > 0) { if (NULL == taosArrayPush(group, &ng)) { - filterFreeGroup((void*)&ng); FLT_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } + ng = (SFilterGroup){0}; } } FLT_ERR_JRET(filterConvertGroupFromArray(info, group)); -_return: taosArrayDestroy(group); + filterFreeInfo(&oinfo); + return 0; + +_return: + filterFreeGroup((void*)&ng); + + taosArrayDestroyEx(group, filterFreeGroup); + filterFreeInfo(&oinfo); FLT_RET(code); diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index a3be94d4c8..0f3d1bfa81 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -572,6 +572,7 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask); int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType, void *param); int32_t schAcquireJob(int64_t refId, SSchJob **ppJob); int32_t schReleaseJob(int64_t refId); +int32_t schReleaseJobEx(int64_t refId, int32_t* released); void schFreeFlowCtrl(SSchJob *pJob); int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel); int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 40391cea7e..7ff3d1386c 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -267,6 +267,9 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD if (pMsg->pData) { SDecoder coder = {0}; SSubmitRsp2 *rsp = taosMemoryMalloc(sizeof(*rsp)); + if (NULL == rsp) { + SCH_ERR_JRET(terrno); + } tDecoderInit(&coder, pMsg->pData, msgSize); code = tDecodeSSubmitRsp2(&coder, rsp); tDecoderClear(&coder); @@ -957,6 +960,9 @@ int32_t schUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, SQueryNodeAddr *addr pMsgSendInfo->target.type = TARGET_TYPE_VNODE; pMsgSendInfo->target.vgId = addr->nodeId; pMsgSendInfo->target.dbFName = taosStrdup(pTask->plan->dbFName); + if (NULL == pMsgSendInfo->target.dbFName) { + return terrno; + } } return TSDB_CODE_SUCCESS; diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 612486c806..14447ba856 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -41,6 +41,15 @@ FORCE_INLINE int32_t schReleaseJob(int64_t refId) { return taosReleaseRef(schMgmt.jobRef, refId); } +FORCE_INLINE int32_t schReleaseJobEx(int64_t refId, int32_t* released) { + if (0 == refId) { + return TSDB_CODE_SUCCESS; + } + + qDebug("sch release ex jobId:0x%" PRIx64, refId); + return taosReleaseRefEx(schMgmt.jobRef, refId, released); +} + int32_t schDumpEpSet(SEpSet *pEpSet, char** ppRes) { *ppRes = NULL; if (NULL == pEpSet) { @@ -196,7 +205,7 @@ void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask) { SCH_TASK_ELOG("fail to get the %dth condidateAddr in task, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); return; } - + SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; @@ -341,7 +350,7 @@ void schFreeRpcCtx(SRpcCtx *pCtx) { void schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask) { *pTask = NULL; - + int32_t s = taosHashGetSize(pTaskList); if (s <= 0) { return; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index a62c59e547..7bd40500ec 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -119,7 +119,7 @@ int32_t schedulerGetTasksStatus(int64_t jobId, SArray *pSub) { qError("failed to get task %d, total: %d", m, pLevel->taskNum); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + SQuerySubDesc subDesc = {0}; subDesc.tid = pTask->taskId; TAOS_STRCPY(subDesc.status, jobTaskStatusStr(pTask->status)); @@ -179,8 +179,11 @@ void schedulerFreeJob(int64_t *jobId, int32_t errCode) { SCH_JOB_DLOG("start to free job 0x%" PRIx64 ", code:%s", *jobId, tstrerror(errCode)); (void)schHandleJobDrop(pJob, errCode); // ignore any error - (void)schReleaseJob(*jobId); // ignore error - *jobId = 0; + int32_t released = false; + (void)schReleaseJobEx(*jobId, &released); // ignore error + if (released) { + *jobId = 0; + } } void schedulerDestroy(void) { diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 3799c5a62b..2cebeb489c 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -238,7 +238,7 @@ void initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t up int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32_t upstreamTaskId, int32_t childId, int64_t checkpointId, SRpcMsg* pMsg); -void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock); +int32_t flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock); int32_t streamCreateSinkResTrigger(SStreamTrigger** pTrigger, int32_t triggerType, int32_t trigger); #ifdef __cplusplus diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 893b82cf52..befd9ee037 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -2573,11 +2573,15 @@ void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId) { } STaskDbWrapper* taskDbOpenImpl(const char* key, char* statePath, char* dbPath) { - char* err = NULL; - char** cfNames = NULL; - size_t nCf = 0; + char* err = NULL; + char** cfNames = NULL; + size_t nCf = 0; + int32_t code = 0; + int32_t lino = 0; STaskDbWrapper* pTaskDb = taosMemoryCalloc(1, sizeof(STaskDbWrapper)); + TSDB_CHECK_NULL(pTaskDb, code, lino, _EXIT, terrno); + pTaskDb->idstr = key ? taosStrdup(key) : NULL; pTaskDb->path = statePath ? taosStrdup(statePath) : NULL; @@ -2592,6 +2596,7 @@ STaskDbWrapper* taskDbOpenImpl(const char* key, char* statePath, char* dbPath) { pTaskDb->db = rocksdb_open(pTaskDb->pCfOpts[0], dbPath, &err); if (pTaskDb->db == NULL) { stError("%s open state-backend failed, reason:%s", key, err); + code = TSDB_CODE_STREAM_INTERNAL_ERROR; goto _EXIT; } @@ -2608,11 +2613,12 @@ STaskDbWrapper* taskDbOpenImpl(const char* key, char* statePath, char* dbPath) { cfNames = rocksdb_list_column_families(pTaskDb->dbOpt, dbPath, &nCf, &err); if (err != NULL) { stError("%s failed to create column-family, %s, %" PRIzu ", reason:%s", key, dbPath, nCf, err); + code = TSDB_CODE_STREAM_INTERNAL_ERROR; goto _EXIT; } } - if (taskDbOpenCfs(pTaskDb, dbPath, cfNames, nCf) != 0) { + if ((code = taskDbOpenCfs(pTaskDb, dbPath, cfNames, nCf)) != 0) { goto _EXIT; } @@ -2625,6 +2631,8 @@ STaskDbWrapper* taskDbOpenImpl(const char* key, char* statePath, char* dbPath) { return pTaskDb; _EXIT: + stError("%s taskDb open failed, %s at line:%d code:%s", key, __func__, lino, tstrerror(code)); + taskDbDestroy(pTaskDb, false); if (err) taosMemoryFree(err); if (cfNames) rocksdb_list_column_families_destroy(cfNames, nCf); diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 540199bb90..2688617823 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -21,7 +21,7 @@ #define CHECK_NOT_RSP_DURATION 10 * 1000 // 10 sec static void processDownstreamReadyRsp(SStreamTask* pTask); -static void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId); +static int32_t addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId); static void rspMonitorFn(void* param, void* tmrId); static void streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutputInfo, int64_t startTs); static int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id); @@ -226,13 +226,13 @@ int32_t streamTaskProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* stError("s-task:%s vgId:%d self vnode-transfer/leader-change/restart detected, old stage:%" PRId64 ", current stage:%" PRId64 ", not check wait for downstream task nodeUpdate, and all tasks restart", id, pRsp->upstreamNodeId, pRsp->oldStage, pTask->pMeta->stage); - addIntoNodeUpdateList(pTask, pRsp->upstreamNodeId); + code = addIntoNodeUpdateList(pTask, pRsp->upstreamNodeId); } else { stError( "s-task:%s downstream taskId:0x%x (vgId:%d) not leader, self dispatch epset needs to be updated, not check " "downstream again, nodeUpdate needed", id, pRsp->downstreamTaskId, pRsp->downstreamNodeId); - addIntoNodeUpdateList(pTask, pRsp->downstreamNodeId); + code = addIntoNodeUpdateList(pTask, pRsp->downstreamNodeId); } streamMetaAddFailedTaskSelf(pTask, now); @@ -258,6 +258,11 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa } void* buf = rpcMallocCont(sizeof(SMsgHead) + len); + if (buf == NULL) { + stError("s-task:0x%x vgId:%d failed prepare msg, %s at line:%d code:%s", taskId, pMeta->vgId, __func__, __LINE__, tstrerror(code)); + return terrno; + } + ((SMsgHead*)buf)->vgId = htonl(vgId); void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); @@ -268,7 +273,7 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = *pRpcInfo}; tmsgSendRsp(&rspMsg); - code = (code >= 0)? 0:code; + code = TMIN(code, 0); return code; } @@ -298,13 +303,8 @@ void streamTaskStartMonitorCheckRsp(SStreamTask* pTask) { int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s start check-rsp monitor, ref:%d ", pTask->id.idStr, ref); - - if (pInfo->checkRspTmr == NULL) { - pInfo->checkRspTmr = taosTmrStart(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer); - } else { - streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, - "check-status-monitor"); - } + streamTmrStart(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, + "check-status-monitor"); streamMutexUnlock(&pInfo->checkInfoLock); } @@ -371,12 +371,14 @@ void processDownstreamReadyRsp(SStreamTask* pTask) { } } -void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { +int32_t addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { int32_t vgId = pTask->pMeta->vgId; + int32_t code = 0;; + bool existed = false; streamMutexLock(&pTask->lock); + int32_t num = taosArrayGetSize(pTask->outputInfo.pNodeEpsetUpdateList); - bool existed = false; for (int i = 0; i < num; ++i) { SDownstreamTaskEpset* p = taosArrayGet(pTask->outputInfo.pNodeEpsetUpdateList, i); if (p == NULL) { @@ -391,15 +393,19 @@ void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { if (!existed) { SDownstreamTaskEpset t = {.nodeId = nodeId}; - void* p = taosArrayPush(pTask->outputInfo.pNodeEpsetUpdateList, &t); + + void* p = taosArrayPush(pTask->outputInfo.pNodeEpsetUpdateList, &t); if (p == NULL) { - // todo let's retry + code = terrno; + stError("s-task:%s vgId:%d failed to update epset, code:%s", pTask->id.idStr, vgId, tstrerror(code)); + } else { + stInfo("s-task:%s vgId:%d downstream nodeId:%d needs to be updated, total needs updated:%d", pTask->id.idStr, + vgId, t.nodeId, (num + 1)); } - stInfo("s-task:%s vgId:%d downstream nodeId:%d needs to be updated, total needs updated:%d", pTask->id.idStr, vgId, - t.nodeId, (num + 1)); } streamMutexUnlock(&pTask->lock); + return code; } void streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutputInfo, int64_t startTs) { @@ -629,6 +635,7 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) { const char* id = pTask->id.idStr; int32_t vgId = pTask->pMeta->vgId; int32_t numOfTimeout = taosArrayGetSize(pTimeoutList); + int32_t code = 0; pInfo->timeoutStartTs = taosGetTimestampMs(); for (int32_t i = 0; i < numOfTimeout; ++i) { @@ -640,14 +647,13 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) { int32_t taskId = *px; SDownstreamStatusInfo* p = NULL; findCheckRspStatus(pInfo, taskId, &p); + if (p != NULL) { if (p->status != -1 || p->rspTs != 0) { - stError("s-task:%s invalid rsp record entry, index:%d, status:%d, rspTs:%" PRId64, pTask->id.idStr, i, - p->status, p->rspTs); + stError("s-task:%s invalid rsp record entry, index:%d, status:%d, rspTs:%" PRId64, id, i, p->status, p->rspTs); continue; } - - int32_t code = doSendCheckMsg(pTask, p); + code = doSendCheckMsg(pTask, p); } } @@ -666,7 +672,7 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) { SDownstreamStatusInfo* p = NULL; findCheckRspStatus(pInfo, *pTaskId, &p); if (p != NULL) { - addIntoNodeUpdateList(pTask, p->vgId); + code = addIntoNodeUpdateList(pTask, p->vgId); stDebug("s-task:%s vgId:%d downstream task:0x%x (vgId:%d) timeout more than 100sec, add into nodeUpate list", id, vgId, p->taskId, p->vgId); } @@ -849,7 +855,7 @@ void rspMonitorFn(void* param, void* tmrId) { handleTimeoutDownstreamTasks(pTask, pTimeoutList); } - streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, + streamTmrStart(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, "check-status-monitor"); streamMutexUnlock(&pInfo->checkInfoLock); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index b0f6f45110..769116264d 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -209,29 +209,18 @@ int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStream return code; } -int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { - SSDataBlock* pDataBlock = taosArrayGet(pBlock->blocks, 0); - if (pDataBlock == NULL) { - return TSDB_CODE_INVALID_PARA; - } - - int64_t checkpointId = pDataBlock->info.version; - int32_t transId = pDataBlock->info.window.skey; - const char* id = pTask->id.idStr; - int32_t code = TSDB_CODE_SUCCESS; - int32_t vgId = pTask->pMeta->vgId; - int32_t taskLevel = pTask->info.taskLevel; +static int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t checkpointId, SStreamDataBlock* pBlock, + int32_t transId) { + int32_t code = 0; + int32_t vgId = pTask->pMeta->vgId; + int32_t taskLevel = pTask->info.taskLevel; + const char* id = pTask->id.idStr; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; - - streamMutexLock(&pTask->lock); if (pTask->chkInfo.checkpointId > checkpointId) { stError("s-task:%s vgId:%d current checkpointId:%" PRId64 " recv expired checkpoint-trigger block, checkpointId:%" PRId64 " transId:%d, discard", id, vgId, pTask->chkInfo.checkpointId, checkpointId, transId); - streamMutexUnlock(&pTask->lock); - - streamFreeQitem((SStreamQueueItem*)pBlock); return code; } @@ -239,37 +228,33 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock stError("s-task:%s vgId:%d checkpointId:%" PRId64 " transId:%d, has been marked failed, failedId:%" PRId64 " discard the checkpoint-trigger block", id, vgId, checkpointId, transId, pActiveInfo->failedId); - streamMutexUnlock(&pTask->lock); - - streamFreeQitem((SStreamQueueItem*)pBlock); return code; } if (pTask->chkInfo.checkpointId == checkpointId) { { // send checkpoint-ready msg to upstream - SRpcMsg msg = {0}; + SRpcMsg msg = {0}; SStreamUpstreamEpInfo* pInfo = NULL; streamTaskGetUpstreamTaskEpInfo(pTask, pBlock->srcTaskId, &pInfo); if (pInfo == NULL) { - streamMutexUnlock(&pTask->lock); return TSDB_CODE_STREAM_TASK_NOT_EXIST; } code = initCheckpointReadyMsg(pTask, pInfo->nodeId, pBlock->srcTaskId, pInfo->childId, checkpointId, &msg); if (code == TSDB_CODE_SUCCESS) { code = tmsgSendReq(&pInfo->epSet, &msg); + if (code) { + stError("s-task:%s vgId:%d failed send chkpt-ready msg to upstream, code:%s", id, vgId, tstrerror(code)); + } } } stWarn( - "s-task:%s vgId:%d recv already finished checkpoint msg, send checkpoint-ready to upstream:0x%x to resume the " - "interrupted checkpoint", + "s-task:%s vgId:%d recv already finished checkpoint-trigger, send checkpoint-ready to upstream:0x%x to resume " + "the interrupted checkpoint", id, vgId, pBlock->srcTaskId); streamTaskOpenUpstreamInput(pTask, pBlock->srcTaskId); - streamMutexUnlock(&pTask->lock); - - streamFreeQitem((SStreamQueueItem*)pBlock); return code; } @@ -278,9 +263,6 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock stError("s-task:%s vgId:%d active checkpointId:%" PRId64 ", recv invalid checkpoint-trigger checkpointId:%" PRId64 " discard", id, vgId, pActiveInfo->activeId, checkpointId); - streamMutexUnlock(&pTask->lock); - - streamFreeQitem((SStreamQueueItem*)pBlock); return code; } else { // checkpointId == pActiveInfo->activeId if (pActiveInfo->allUpstreamTriggerRecv == 1) { @@ -288,8 +270,6 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock "s-task:%s vgId:%d all upstream checkpoint-trigger recv, discard this checkpoint-trigger, " "checkpointId:%" PRId64 " transId:%d", id, vgId, checkpointId, transId); - streamMutexUnlock(&pTask->lock); - streamFreeQitem((SStreamQueueItem*)pBlock); return code; } @@ -298,7 +278,6 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock for (int32_t i = 0; i < taosArrayGetSize(pActiveInfo->pReadyMsgList); ++i) { STaskCheckpointReadyInfo* p = taosArrayGet(pActiveInfo->pReadyMsgList, i); if (p == NULL) { - streamMutexUnlock(&pTask->lock); return TSDB_CODE_INVALID_PARA; } @@ -306,9 +285,6 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock stWarn("s-task:%s repeatly recv checkpoint-source msg from task:0x%x vgId:%d, checkpointId:%" PRId64 ", prev recvTs:%" PRId64 " discard", pTask->id.idStr, p->upstreamTaskId, p->upstreamNodeId, p->checkpointId, p->recvTs); - - streamMutexUnlock(&pTask->lock); - streamFreeQitem((SStreamQueueItem*)pBlock); return code; } } @@ -316,7 +292,33 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock } } + return 0; +} + +int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { + int64_t checkpointId = 0; + int32_t transId = 0; + const char* id = pTask->id.idStr; + int32_t code = TSDB_CODE_SUCCESS; + int32_t vgId = pTask->pMeta->vgId; + int32_t taskLevel = pTask->info.taskLevel; + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + + SSDataBlock* pDataBlock = taosArrayGet(pBlock->blocks, 0); + if (pDataBlock == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + checkpointId = pDataBlock->info.version; + transId = pDataBlock->info.window.skey; + + streamMutexLock(&pTask->lock); + code = doCheckBeforeHandleChkptTrigger(pTask, checkpointId, pBlock, transId); streamMutexUnlock(&pTask->lock); + if (code) { + streamFreeQitem((SStreamQueueItem*)pBlock); + return code; + } stDebug("s-task:%s vgId:%d start to handle the checkpoint-trigger block, checkpointId:%" PRId64 " ver:%" PRId64 ", transId:%d current active checkpointId:%" PRId64, @@ -345,12 +347,8 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s start checkpoint-trigger monitor in 10s, ref:%d ", pTask->id.idStr, ref); streamMetaAcquireOneTask(pTask); - - if (pTmrInfo->tmrHandle == NULL) { - pTmrInfo->tmrHandle = taosTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer); - } else { - streamTmrReset(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "trigger-recv-monitor"); - } + streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, + "trigger-recv-monitor"); pTmrInfo->launchChkptId = pActiveInfo->activeId; } else { // already launched, do nothing stError("s-task:%s previous checkpoint-trigger monitor tmr is set, not start new one", pTask->id.idStr); @@ -365,7 +363,11 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock // The transfer of state may generate new data that need to dispatch to downstream tasks, // Otherwise, those new generated data by executors that is kept in outputQ, may be lost if this program crashed // before the next checkpoint. - flushStateDataInExecutor(pTask, (SStreamQueueItem*)pBlock); + code = flushStateDataInExecutor(pTask, (SStreamQueueItem*)pBlock); + if (code) { + streamFreeQitem((SStreamQueueItem*)pBlock); + return code; + } if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH) { stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId); @@ -398,7 +400,10 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock code = streamTaskBuildCheckpoint(pTask); // todo: not handle error yet } else { // source & agg tasks need to forward the checkpoint msg downwards stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, forwards to downstream", id, num); - flushStateDataInExecutor(pTask, (SStreamQueueItem*)pBlock); + code = flushStateDataInExecutor(pTask, (SStreamQueueItem*)pBlock); + if (code) { + return code; + } // Put the checkpoint-trigger block into outputQ, to make sure all blocks with less version have been handled by // this task already. And then, dispatch check point msg to all downstream tasks @@ -669,10 +674,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV } streamMetaWLock(pMeta); - - if (streamMetaCommit(pMeta) < 0) { - // persist to disk - } + code = streamMetaCommit(pMeta); return TSDB_CODE_SUCCESS; } @@ -887,48 +889,11 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { return code; } -void checkpointTriggerMonitorFn(void* param, void* tmrId) { - SStreamTask* pTask = param; - int32_t vgId = pTask->pMeta->vgId; - int64_t now = taosGetTimestampMs(); - const char* id = pTask->id.idStr; - +static int32_t doChkptStatusCheck(SStreamTask* pTask) { + const char* id = pTask->id.idStr; + int32_t vgId = pTask->pMeta->vgId; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; - SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptTriggerMsgTmr; - - if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { - int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - stError("s-task:%s source task should not start the checkpoint-trigger monitor fn, ref:%d quit", id, ref); - streamMetaReleaseTask(pTask->pMeta, pTask); - return; - } - - // check the status every 100ms - if (streamTaskShouldStop(pTask)) { - int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); - stDebug("s-task:%s vgId:%d quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); - streamMetaReleaseTask(pTask->pMeta, pTask); - return; - } - - if (++pTmrInfo->activeCounter < 50) { - streamTmrReset(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "trigger-recv-monitor"); - return; - } - - pTmrInfo->activeCounter = 0; - stDebug("s-task:%s vgId:%d checkpoint-trigger monitor in tmr, ts:%" PRId64, id, vgId, now); - - streamMutexLock(&pTask->lock); - SStreamTaskState pState = streamTaskGetStatus(pTask); - if (pState.state != TASK_STATUS__CK) { - int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); - stDebug("s-task:%s vgId:%d not in checkpoint status, quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); - - streamMutexUnlock(&pTask->lock); - streamMetaReleaseTask(pTask->pMeta, pTask); - return; - } + SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptTriggerMsgTmr; // checkpoint-trigger recv flag is set, quit if (pActiveInfo->allUpstreamTriggerRecv) { @@ -936,48 +901,44 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { stDebug("s-task:%s vgId:%d all checkpoint-trigger recv, quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); - streamMutexUnlock(&pTask->lock); - streamMetaReleaseTask(pTask->pMeta, pTask); - return; - } - - streamMutexUnlock(&pTask->lock); - - streamMutexLock(&pActiveInfo->lock); - - // send msg to retrieve checkpoint trigger msg - SArray* pList = pTask->upstreamInfo.pList; - SArray* pNotSendList = taosArrayInit(4, sizeof(SStreamUpstreamEpInfo)); - if (pNotSendList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - stDebug("s-task:%s start to triggerMonitor, reason:%s", id, tstrerror(terrno)); - streamMutexUnlock(&pActiveInfo->lock); - - stDebug("s-task:%s start to monitor checkpoint-trigger in 10s", id); - streamTmrReset(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "trigger-recv-monitor"); - return; +// streamMutexUnlock(&pTask->lock); +// streamMetaReleaseTask(pTask->pMeta, pTask); + return -1; } if ((pTmrInfo->launchChkptId != pActiveInfo->activeId) || (pActiveInfo->activeId == 0)) { - streamMutexUnlock(&pActiveInfo->lock); int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); stWarn("s-task:%s vgId:%d checkpoint-trigger retrieve by previous checkpoint procedure, checkpointId:%" PRId64 ", quit, ref:%d", id, vgId, pTmrInfo->launchChkptId, ref); - streamMetaReleaseTask(pTask->pMeta, pTask); - return; +// streamMutexUnlock(&pActiveInfo->lock); +// streamMetaReleaseTask(pTask->pMeta, pTask); + return -1; } // active checkpoint info is cleared for now if ((pActiveInfo->activeId == 0) || (pActiveInfo->transId == 0) || (pTask->chkInfo.startTs == 0)) { - streamMutexUnlock(&pActiveInfo->lock); int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); stWarn("s-task:%s vgId:%d active checkpoint may be cleared, quit from retrieve checkpoint-trigger send tmr, ref:%d", id, vgId, ref); - streamMetaReleaseTask(pTask->pMeta, pTask); - return; +// streamMutexUnlock(&pActiveInfo->lock); +// streamMetaReleaseTask(pTask->pMeta, pTask); + return -1; + } + + return 0; +} + +static int32_t doFindNotSendUpstream(SStreamTask* pTask, SArray* pList, SArray** ppNotSendList) { + const char* id = pTask->id.idStr; + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + + SArray* pNotSendList = taosArrayInit(4, sizeof(SStreamUpstreamEpInfo)); + if (pNotSendList == NULL) { + stDebug("s-task:%s start to triggerMonitor, reason:%s", id, tstrerror(terrno)); + return terrno; } for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) { @@ -1001,13 +962,87 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { void* px = taosArrayPush(pNotSendList, pInfo); if (px == NULL) { stError("s-task:%s failed to record not send info, code: out of memory", id); + taosArrayDestroy(pNotSendList); + return terrno; } } } + *ppNotSendList = pNotSendList; + return 0; +} + +void checkpointTriggerMonitorFn(void* param, void* tmrId) { + SStreamTask* pTask = param; + int32_t vgId = pTask->pMeta->vgId; + int64_t now = taosGetTimestampMs(); + const char* id = pTask->id.idStr; + + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptTriggerMsgTmr; + + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); + stError("s-task:%s source task should not start the checkpoint-trigger monitor fn, ref:%d quit", id, ref); + streamMetaReleaseTask(pTask->pMeta, pTask); + return; + } + + // check the status every 100ms + if (streamTaskShouldStop(pTask)) { + int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); + stDebug("s-task:%s vgId:%d quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); + streamMetaReleaseTask(pTask->pMeta, pTask); + return; + } + + if (++pTmrInfo->activeCounter < 50) { + streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "trigger-recv-monitor"); + return; + } + + pTmrInfo->activeCounter = 0; + stDebug("s-task:%s vgId:%d checkpoint-trigger monitor in tmr, ts:%" PRId64, id, vgId, now); + + streamMutexLock(&pTask->lock); + SStreamTaskState state = streamTaskGetStatus(pTask); + streamMutexUnlock(&pTask->lock); + + if (state.state != TASK_STATUS__CK) { + int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); + stDebug("s-task:%s vgId:%d status:%s not in checkpoint status, quit from monitor checkpoint-trigger, ref:%d", id, + vgId, state.name, ref); + streamMetaReleaseTask(pTask->pMeta, pTask); + return; + } + + streamMutexLock(&pActiveInfo->lock); + + int32_t code = doChkptStatusCheck(pTask); + if (code) { + streamMutexUnlock(&pTask->lock); + streamMetaReleaseTask(pTask->pMeta, pTask); + return; + } + + // send msg to retrieve checkpoint trigger msg + SArray* pList = pTask->upstreamInfo.pList; + SArray* pNotSendList = NULL; + + code = doFindNotSendUpstream(pTask, pList, &pNotSendList); + if (code) { + int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); + stDebug("s-task:%s failed to find not send upstream, code:%s, out of tmr, ref:%d", id, tstrerror(code), ref); + streamMutexUnlock(&pActiveInfo->lock); + streamMetaReleaseTask(pTask->pMeta, pTask); + + taosArrayDestroy(pNotSendList); + return; + } + // do send retrieve checkpoint trigger msg to upstream int32_t size = taosArrayGetSize(pNotSendList); - int32_t code = doSendRetrieveTriggerMsg(pTask, pNotSendList); + code = doSendRetrieveTriggerMsg(pTask, pNotSendList); if (code) { stError("s-task:%s vgId:%d failed to retrieve trigger msg, code:%s", pTask->id.idStr, vgId, tstrerror(code)); } @@ -1017,7 +1052,7 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { // check every 100ms if (size > 0) { stDebug("s-task:%s start to monitor checkpoint-trigger in 10s", id); - streamTmrReset(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "trigger-recv-monitor"); + streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "trigger-recv-monitor"); } else { int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); stDebug("s-task:%s all checkpoint-trigger recved, quit from monitor checkpoint-trigger tmr, ref:%d", id, ref); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 5f7b8dd39e..0f0015c7d9 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -634,12 +634,8 @@ static void doMonitorDispatchData(void* param, void* tmrId) { void streamStartMonitorDispatchData(SStreamTask* pTask, int64_t waitDuration) { int32_t vgId = pTask->pMeta->vgId; - if (pTask->msgInfo.pRetryTmr != NULL) { - streamTmrReset(doMonitorDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pRetryTmr, vgId, - "dispatch-monitor-tmr"); - } else { - pTask->msgInfo.pRetryTmr = taosTmrStart(doMonitorDispatchData, waitDuration, pTask, streamTimer); - } + streamTmrStart(doMonitorDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pRetryTmr, vgId, + "dispatch-monitor"); } int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, SSDataBlock* pDataBlock, int64_t groupId, @@ -888,77 +884,42 @@ int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32 return TSDB_CODE_SUCCESS; } -static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { - SStreamTask* pTask = param; - int32_t vgId = pTask->pMeta->vgId; - const char* id = pTask->id.idStr; +static int32_t doTaskChkptStatusCheck(SStreamTask* pTask, int32_t num) { SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptReadyMsgTmr; + const char* id = pTask->id.idStr; + int32_t vgId = pTask->pMeta->vgId; - // check the status every 100ms - if (streamTaskShouldStop(pTask)) { - int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); - stDebug("s-task:%s vgId:%d status:stop, quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); - streamMetaReleaseTask(pTask->pMeta, pTask); - return; - } - - if (++pTmrInfo->activeCounter < 50) { - streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, - "chkpt-ready-monitor"); - return; - } - - pTmrInfo->activeCounter = 0; - stDebug("s-task:%s in sending checkpoint-ready msg monitor tmr", id); - - streamMutexLock(&pTask->lock); - SStreamTaskState pState = streamTaskGetStatus(pTask); - if (pState.state != TASK_STATUS__CK) { - int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); - stDebug("s-task:%s vgId:%d status:%s not in checkpoint, quit from monitor checkpoint-ready send, ref:%d", id, vgId, - pState.name, ref); - streamMutexUnlock(&pTask->lock); - streamMetaReleaseTask(pTask->pMeta, pTask); - return; - } - streamMutexUnlock(&pTask->lock); - - streamMutexLock(&pActiveInfo->lock); - - SArray* pList = pActiveInfo->pReadyMsgList; - int32_t num = taosArrayGetSize(pList); if (pTmrInfo->launchChkptId != pActiveInfo->activeId) { - streamMutexUnlock(&pActiveInfo->lock); int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); stWarn("s-task:%s vgId:%d ready-msg send tmr launched by previous checkpoint procedure, checkpointId:%" PRId64 ", quit, ref:%d", id, vgId, pTmrInfo->launchChkptId, ref); - - streamMetaReleaseTask(pTask->pMeta, pTask); - return; + return -1; } // active checkpoint info is cleared for now if ((pActiveInfo->activeId == 0) || (pActiveInfo->transId == 0) || (num == 0) || (pTask->chkInfo.startTs == 0)) { - streamMutexUnlock(&pActiveInfo->lock); int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); stWarn("s-task:%s vgId:%d active checkpoint may be cleared, quit from readyMsg send tmr, ref:%d", id, vgId, ref); - - streamMetaReleaseTask(pTask->pMeta, pTask); - return; + return -1; } - SArray* pNotRspList = taosArrayInit(4, sizeof(int32_t)); - if (taosArrayGetSize(pTask->upstreamInfo.pList) != num) { - streamMutexUnlock(&pActiveInfo->lock); int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); stWarn("s-task:%s vgId:%d upstream number:%d not equals sent readyMsg:%d, quit from readyMsg send tmr, ref:%d", id, vgId, (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList), num, ref); + return -1; + } - streamMetaReleaseTask(pTask->pMeta, pTask); - return; + return 0; +} + +static int32_t doFindNotConfirmUpstream(SArray** ppNotRspList, SArray* pList, int32_t num, int32_t vgId, int32_t level, + const char* id) { + SArray* pTmp = taosArrayInit(4, sizeof(int32_t)); + if (pTmp == NULL) { + return terrno; } for (int32_t i = 0; i < num; ++i) { @@ -971,63 +932,138 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { continue; } - void* p = taosArrayPush(pNotRspList, &pInfo->upstreamTaskId); + void* p = taosArrayPush(pTmp, &pInfo->upstreamTaskId); if (p == NULL) { stError("s-task:%s vgId:%d failed to record not rsp task, code: out of memory", id, vgId); + return terrno; } else { stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId, - pTask->info.taskLevel, pInfo->upstreamTaskId); + level, pInfo->upstreamTaskId); } } - int32_t checkpointId = pActiveInfo->activeId; + *ppNotRspList = pTmp; + return 0; +} - int32_t notRsp = taosArrayGetSize(pNotRspList); - if (notRsp > 0) { // send checkpoint-ready msg again - for (int32_t i = 0; i < taosArrayGetSize(pNotRspList); ++i) { - int32_t* pTaskId = taosArrayGet(pNotRspList, i); - if (pTaskId == NULL) { +static void doSendChkptReadyMsg(SStreamTask* pTask, SArray* pNotRspList, int64_t checkpointId, SArray* pReadyList) { + int32_t code = 0; + int32_t num = taosArrayGetSize(pReadyList); + const char* id = pTask->id.idStr; + + for (int32_t i = 0; i < taosArrayGetSize(pNotRspList); ++i) { + int32_t* pTaskId = taosArrayGet(pNotRspList, i); + if (pTaskId == NULL) { + continue; + } + + for (int32_t j = 0; j < num; ++j) { + STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pReadyList, j); + if (pReadyInfo == NULL) { continue; } - for (int32_t j = 0; j < num; ++j) { - STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pList, j); - if (pReadyInfo == NULL) { - continue; - } + if (*pTaskId == pReadyInfo->upstreamTaskId) { // send msg again - if (*pTaskId == pReadyInfo->upstreamTaskId) { // send msg again - - SRpcMsg msg = {0}; - int32_t code = initCheckpointReadyMsg(pTask, pReadyInfo->upstreamNodeId, pReadyInfo->upstreamTaskId, - pReadyInfo->childId, checkpointId, &msg); + SRpcMsg msg = {0}; + code = initCheckpointReadyMsg(pTask, pReadyInfo->upstreamNodeId, pReadyInfo->upstreamTaskId, + pReadyInfo->childId, checkpointId, &msg); + if (code == TSDB_CODE_SUCCESS) { + code = tmsgSendReq(&pReadyInfo->upstreamNodeEpset, &msg); if (code == TSDB_CODE_SUCCESS) { - code = tmsgSendReq(&pReadyInfo->upstreamNodeEpset, &msg); - if (code == TSDB_CODE_SUCCESS) { - stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x again", id, pTask->info.taskLevel, - pReadyInfo->upstreamTaskId); - } else { - stError("s-task:%s failed to send checkpoint-ready msg, try nex time in 10s", id); - } + stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x again", id, pTask->info.taskLevel, + pReadyInfo->upstreamTaskId); } else { - stError("s-task:%s failed to prepare the checkpoint-ready msg, try nex time in 10s", id); + stError("s-task:%s failed to send checkpoint-ready msg, try nex time in 10s", id); } + } else { + stError("s-task:%s failed to prepare the checkpoint-ready msg, try nex time in 10s", id); } } } + } +} - streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, +static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { + SStreamTask* pTask = param; + int32_t vgId = pTask->pMeta->vgId; + const char* id = pTask->id.idStr; + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptReadyMsgTmr; + SArray* pNotRspList = NULL; + + // check the status every 100ms + if (streamTaskShouldStop(pTask)) { + int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); + stDebug("s-task:%s vgId:%d status:stop, quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); + streamMetaReleaseTask(pTask->pMeta, pTask); + return; + } + + if (++pTmrInfo->activeCounter < 50) { + streamTmrStart(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, + "chkpt-ready-monitor"); + return; + } + + // reset tmr + pTmrInfo->activeCounter = 0; + stDebug("s-task:%s in sending checkpoint-ready msg monitor tmr", id); + + streamMutexLock(&pTask->lock); + SStreamTaskState state = streamTaskGetStatus(pTask); + streamMutexUnlock(&pTask->lock); + + // 1. check status in the first place + if (state.state != TASK_STATUS__CK) { + int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); + stDebug("s-task:%s vgId:%d status:%s not in checkpoint, quit from monitor checkpoint-ready, ref:%d", id, vgId, + state.name, ref); + streamMetaReleaseTask(pTask->pMeta, pTask); + return; + } + + streamMutexLock(&pActiveInfo->lock); + + SArray* pList = pActiveInfo->pReadyMsgList; + int32_t num = taosArrayGetSize(pList); + int32_t code = doTaskChkptStatusCheck(pTask, num); + if (code) { + streamMutexUnlock(&pActiveInfo->lock); + streamMetaReleaseTask(pTask->pMeta, pTask); + return; + } + + code = doFindNotConfirmUpstream(&pNotRspList, pList, num, vgId, pTask->info.taskLevel, id); + if (code) { + int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); + stError("s-task:%s failed to find not rsp checkpoint-ready downstream, code:%s, out of tmr, ref:%d", id, + tstrerror(code), ref); + streamMutexUnlock(&pActiveInfo->lock); + streamMetaReleaseTask(pTask->pMeta, pTask); + + taosArrayDestroy(pNotRspList); + return; + } + + int32_t checkpointId = pActiveInfo->activeId; + int32_t notRsp = taosArrayGetSize(pNotRspList); + doSendChkptReadyMsg(pTask, pNotRspList, checkpointId, pList); + + if (notRsp > 0) { // send checkpoint-ready msg again + streamTmrStart(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "chkpt-ready-monitor"); streamMutexUnlock(&pActiveInfo->lock); } else { int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); stDebug( - "s-task:%s vgId:%d recv of checkpoint-ready msg confirmed by all upstream task(s), clear checkpoint-ready msg " - "and quit from timer, ref:%d", + "s-task:%s vgId:%d checkpoint-ready msg confirmed by all upstream task(s), clear checkpoint-ready msg and quit " + "from timer, ref:%d", id, vgId, ref); streamClearChkptReadyMsg(pActiveInfo); streamMutexUnlock(&pActiveInfo->lock); + // release should be the last execution, since pTask may be destroy after it immidiately. streamMetaReleaseTask(pTask->pMeta, pTask); } @@ -1085,12 +1121,8 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) { stDebug("s-task:%s start checkpoint-ready monitor in 10s, ref:%d ", pTask->id.idStr, ref); streamMetaAcquireOneTask(pTask); - if (pTmrInfo->tmrHandle == NULL) { - pTmrInfo->tmrHandle = taosTmrStart(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer); - } else { - streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, - "chkpt-ready-monitor"); - } + streamTmrStart(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, + "chkpt-ready-monitor"); // mark the timer monitor checkpointId pTmrInfo->launchChkptId = pActiveInfo->activeId; diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 5f462f62f9..3a27234ba9 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -24,7 +24,7 @@ #define FILL_HISTORY_TASK_EXEC_INTERVAL 5000 // 5 sec static int32_t streamTransferStateDoPrepare(SStreamTask* pTask); -static void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks); +static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks); bool streamTaskShouldStop(const SStreamTask* pTask) { SStreamTaskState pState = streamTaskGetStatus(pTask); @@ -95,7 +95,7 @@ static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray* return code; } -void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks) { +int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks) { int32_t code = TSDB_CODE_SUCCESS; void* pExecutor = pTask->exec.pExecutor; int32_t size = 0; @@ -112,7 +112,7 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to if (streamTaskShouldStop(pTask) || (pRes == NULL)) { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - return; + return code; } SSDataBlock* output = NULL; @@ -122,8 +122,13 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to qResetTaskInfoCode(pExecutor); } - stError("unexpected stream execution, s-task:%s since %s", pTask->id.idStr, tstrerror(code)); - continue; + if (code == TSDB_CODE_OUT_OF_MEMORY || code == TSDB_CODE_INVALID_PARA || code == TSDB_CODE_FILE_CORRUPTED) { + stFatal("s-task:%s failed to continue execute since %s", pTask->id.idStr, tstrerror(code)); + return code; + } else { + qResetTaskCode(pExecutor); + continue; + } } if (output == NULL) { @@ -194,7 +199,7 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to code = doDumpResult(pTask, pItem, pRes, size, totalSize, totalBlocks); // todo: here we need continue retry to put it into output buffer if (code != TSDB_CODE_SUCCESS) { - return; + return code; } pRes = NULL; @@ -208,6 +213,8 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to } else { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); } + + return code; } // todo contiuous try to create result blocks @@ -627,7 +634,7 @@ static void doRecordThroughput(STaskExecStatisInfo* pInfo, int64_t totalBlocks, } } -static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, int32_t num) { +static int32_t doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, int32_t num) { const char* id = pTask->id.idStr; int32_t blockSize = 0; int64_t st = taosGetTimestampMs(); @@ -635,23 +642,28 @@ static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, i int64_t ver = pInfo->processedVer; int64_t totalSize = 0; int32_t totalBlocks = 0; + int32_t code = 0; stDebug("s-task:%s start to process batch blocks, num:%d, type:%s", id, num, streamQueueItemGetTypeStr(pBlock->type)); - int32_t code = doSetStreamInputBlock(pTask, pBlock, &ver, id); + code = doSetStreamInputBlock(pTask, pBlock, &ver, id); if (code) { stError("s-task:%s failed to set input block, not exec for these blocks", id); - return; + return code; + } + + code = streamTaskExecImpl(pTask, pBlock, &totalSize, &totalBlocks); + if (code) { + return code; } - streamTaskExecImpl(pTask, pBlock, &totalSize, &totalBlocks); doRecordThroughput(&pTask->execInfo, totalBlocks, totalSize, blockSize, st, pTask->id.idStr); // update the currentVer if processing the submit blocks. if (!(pInfo->checkpointVer <= pInfo->nextProcessVer && ver >= pInfo->checkpointVer)) { stError("s-task:%s invalid info, checkpointVer:%" PRId64 ", nextProcessVer:%" PRId64 " currentVer:%" PRId64, id, pInfo->checkpointVer, pInfo->nextProcessVer, ver); - return; + return code; } if (ver != pInfo->processedVer) { @@ -660,9 +672,11 @@ static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, i id, pInfo->processedVer, ver, pInfo->nextProcessVer, pInfo->checkpointVer); pInfo->processedVer = ver; } + + return code; } -void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock) { +int32_t flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock) { const char* id = pTask->id.idStr; // 1. transfer the ownership of executor state @@ -703,7 +717,12 @@ void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointB } // 2. flush data in executor to K/V store, which should be completed before do checkpoint in the K/V. - doStreamTaskExecImpl(pTask, pCheckpointBlock, 1); + int32_t code = doStreamTaskExecImpl(pTask, pCheckpointBlock, 1); + if(code) { + stError("s-task:%s failed to exec stream task before checkpoint, code:%s", id, tstrerror(code)); + } + + return code; } /** @@ -712,6 +731,7 @@ void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointB */ static int32_t doStreamExecTask(SStreamTask* pTask) { const char* id = pTask->id.idStr; + int32_t code = 0; // merge multiple input data if possible in the input queue. stDebug("s-task:%s start to extract data block from inputQ", id); @@ -784,9 +804,9 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { if (type == STREAM_INPUT__DATA_BLOCK) { pTask->execInfo.sink.dataSize += blockSize; stDebug("s-task:%s sink task start to sink %d blocks, size:%.2fKiB", id, numOfBlocks, SIZE_IN_KiB(blockSize)); - int32_t code = doOutputResultBlockImpl(pTask, (SStreamDataBlock*)pInput); + code = doOutputResultBlockImpl(pTask, (SStreamDataBlock*)pInput); if (code != TSDB_CODE_SUCCESS) { - // todo handle error. + return code; } double el = (taosGetTimestampMs() - st) / 1000.0; @@ -801,17 +821,19 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { } if (type != STREAM_INPUT__CHECKPOINT) { - doStreamTaskExecImpl(pTask, pInput, numOfBlocks); + code = doStreamTaskExecImpl(pTask, pInput, numOfBlocks); streamFreeQitem(pInput); + if (code) { + return code; + } } else { // todo other thread may change the status // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed. streamMutexLock(&pTask->lock); SStreamTaskState pState = streamTaskGetStatus(pTask); if (pState.state == TASK_STATUS__CK) { stDebug("s-task:%s checkpoint block received, set status:%s", id, pState.name); - int32_t code = streamTaskBuildCheckpoint(pTask); // ignore this error msg, and continue + code = streamTaskBuildCheckpoint(pTask); // ignore this error msg, and continue } else { // todo refactor - int32_t code = 0; if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { code = streamTaskSendCheckpointSourceRsp(pTask); } else { @@ -827,7 +849,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { streamMutexUnlock(&pTask->lock); streamFreeQitem(pInput); - return 0; + return code; } } } @@ -858,21 +880,21 @@ bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus) { } } -void streamResumeTask(SStreamTask* pTask) { +int32_t streamResumeTask(SStreamTask* pTask) { const char* id = pTask->id.idStr; int32_t code = 0; if (pTask->status.schedStatus != TASK_SCHED_STATUS__ACTIVE) { - stError("s-task:%s invalid sched status:%d, not resume task", id, pTask->status.schedStatus); - return; + stError("s-task:%s invalid sched status:%d, not resume task", pTask->id.idStr, pTask->status.schedStatus); + return code; } while (1) { code = doStreamExecTask(pTask); if (code) { stError("s-task:%s failed to exec stream task, code:%s", id, tstrerror(code)); + return code; } - // check if continue streamMutexLock(&pTask->lock); @@ -888,7 +910,7 @@ void streamResumeTask(SStreamTask* pTask) { stDebug("s-task:%s exec completed, status:%s, sched-status:%d, lastExecTs:%" PRId64, id, p, pTask->status.schedStatus, pTask->status.lastExecTs); - return; + return code; } else { // check if this task needs to be idle for a while if (pTask->status.schedIdleTime > 0) { @@ -896,28 +918,31 @@ void streamResumeTask(SStreamTask* pTask) { streamMutexUnlock(&pTask->lock); setLastExecTs(pTask, taosGetTimestampMs()); - return; + return code; } } streamMutexUnlock(&pTask->lock); } + + return code; } int32_t streamExecTask(SStreamTask* pTask) { // this function may be executed by multi-threads, so status check is required. const char* id = pTask->id.idStr; + int32_t code = 0; int8_t schedStatus = streamTaskSetSchedStatusActive(pTask); if (schedStatus == TASK_SCHED_STATUS__WAITING) { - streamResumeTask(pTask); + code = streamResumeTask(pTask); } else { char* p = streamTaskGetStatus(pTask).name; stDebug("s-task:%s already started to exec by other thread, status:%s, sched-status:%d", id, p, pTask->status.schedStatus); } - return 0; + return code; } int32_t streamTaskReleaseState(SStreamTask* pTask) { diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c index 703e6a3256..72d2e89936 100644 --- a/source/libs/stream/src/streamHb.c +++ b/source/libs/stream/src/streamHb.c @@ -279,7 +279,7 @@ void streamMetaHbToMnode(void* param, void* tmrId) { } if (!waitForEnoughDuration(pMeta->pHbInfo)) { - streamTmrReset(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr, vgId, + streamTmrStart(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr, vgId, "meta-hb-tmr"); code = taosReleaseRef(streamMetaId, rid); @@ -301,7 +301,7 @@ void streamMetaHbToMnode(void* param, void* tmrId) { } streamMetaRUnLock(pMeta); - streamTmrReset(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr, pMeta->vgId, + streamTmrStart(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr, pMeta->vgId, "meta-hb-tmr"); code = taosReleaseRef(streamMetaId, rid); @@ -317,7 +317,7 @@ int32_t createMetaHbInfo(int64_t* pRid, SMetaHbInfo** pRes) { return terrno; } - pInfo->hbTmr = taosTmrStart(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, pRid, streamTimer); + streamTmrStart(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, pRid, streamTimer, &pInfo->hbTmr, 0, "stream-hb"); pInfo->tickCounter = 0; pInfo->msgSendTs = -1; pInfo->hbCount = 0; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index a5cd9bf404..f2b89bb6aa 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -368,8 +368,9 @@ void streamMetaRemoveDB(void* arg, char* key) { int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, FTaskExpand expandTaskFn, int32_t vgId, int64_t stage, startComplete_fn_t fn, SStreamMeta** p) { - int32_t code = 0; QRY_PARAM_CHECK(p); + int32_t code = 0; + int32_t lino = 0; SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta)); if (pMeta == NULL) { @@ -379,23 +380,18 @@ int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, int32_t len = strlen(path) + 64; char* tpath = taosMemoryCalloc(1, len); - if (tpath == NULL) { - code = terrno; - goto _err; - } + TSDB_CHECK_NULL(tpath, code, lino, _err, terrno); sprintf(tpath, "%s%s%s", path, TD_DIRSEP, "stream"); pMeta->path = tpath; code = streamMetaOpenTdb(pMeta); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); if ((code = streamMetaMayCvtDbFormat(pMeta)) < 0) { stError("vgId:%d convert sub info format failed, open stream meta failed, reason: %s", pMeta->vgId, tstrerror(terrno)); - goto _err; + TSDB_CHECK_CODE(code, lino, _err); } if ((code = streamMetaBegin(pMeta) < 0)) { @@ -405,28 +401,17 @@ int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, _hash_fn_t fp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR); pMeta->pTasksMap = taosHashInit(64, fp, true, HASH_NO_LOCK); - if (pMeta->pTasksMap == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } + TSDB_CHECK_NULL(pMeta->pTasksMap, code, lino, _err, terrno); pMeta->updateInfo.pTasks = taosHashInit(64, fp, false, HASH_NO_LOCK); - if (pMeta->updateInfo.pTasks == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } + TSDB_CHECK_NULL(pMeta->updateInfo.pTasks, code, lino, _err, terrno); code = streamMetaInitStartInfo(&pMeta->startInfo); - if (code) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); // task list pMeta->pTaskList = taosArrayInit(4, sizeof(SStreamTaskId)); - if (pMeta->pTaskList == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } + TSDB_CHECK_NULL(pMeta->pTaskList, code, lino, _err, terrno); pMeta->scanInfo.scanCounter = 0; pMeta->vgId = vgId; @@ -440,59 +425,44 @@ int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, pMeta->startInfo.completeFn = fn; pMeta->pTaskDbUnique = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + TSDB_CHECK_NULL(pMeta->pTaskDbUnique, code, lino, _err, terrno); pMeta->numOfPausedTasks = 0; pMeta->numOfStreamTasks = 0; pMeta->closeFlag = false; stInfo("vgId:%d open stream meta succ, latest checkpoint:%" PRId64 ", stage:%" PRId64, vgId, pMeta->chkpId, stage); - pMeta->rid = taosAddRef(streamMetaId, pMeta); // set the attribute when running on Linux OS TdThreadRwlockAttr attr; code = taosThreadRwlockAttrInit(&attr); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); #ifdef LINUX code = pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); #endif code = taosThreadRwlockInit(&pMeta->lock, &attr); - if (code) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); code = taosThreadRwlockAttrDestroy(&attr); - if (code) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); memcpy(pRid, &pMeta->rid, sizeof(pMeta->rid)); code = metaRefMgtAdd(pMeta->vgId, pRid); - if (code) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); code = createMetaHbInfo(pRid, &pMeta->pHbInfo); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } - - pMeta->qHandle = taosInitScheduler(32, 1, "stream-chkp", NULL); + TSDB_CHECK_CODE(code, lino, _err); code = bkdMgtCreate(tpath, (SBkdMgt**)&pMeta->bkdChkptMgt); - if (code != 0) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _err); code = taosThreadMutexInit(&pMeta->backendMutex, NULL); + TSDB_CHECK_CODE(code, lino, _err); *p = pMeta; return code; @@ -526,9 +496,10 @@ _err: if (pMeta->startInfo.pReadyTaskSet) taosHashCleanup(pMeta->startInfo.pReadyTaskSet); if (pMeta->startInfo.pFailedTaskSet) taosHashCleanup(pMeta->startInfo.pFailedTaskSet); if (pMeta->bkdChkptMgt) bkdMgtDestroy(pMeta->bkdChkptMgt); + taosMemoryFree(pMeta); - stError("failed to open stream meta, reason:%s", tstrerror(terrno)); + stError("vgId:%d failed to open stream meta, at line:%d reason:%s", vgId, lino, tstrerror(code)); return code; } @@ -655,9 +626,6 @@ void streamMetaCloseImpl(void* arg) { taosMemoryFree(pMeta->path); streamMutexDestroy(&pMeta->backendMutex); - taosCleanUpScheduler(pMeta->qHandle); - taosMemoryFree(pMeta->qHandle); - bkdMgtDestroy(pMeta->bkdChkptMgt); pMeta->role = NODE_ROLE_UNINIT; @@ -955,32 +923,38 @@ int32_t streamMetaBegin(SStreamMeta* pMeta) { streamMetaWLock(pMeta); int32_t code = tdbBegin(pMeta->db, &pMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + if (code) { + streamSetFatalError(pMeta, code, __func__, __LINE__); + } streamMetaWUnLock(pMeta); return code; } int32_t streamMetaCommit(SStreamMeta* pMeta) { - int32_t code = 0; - code = tdbCommit(pMeta->db, pMeta->txn); + int32_t code = tdbCommit(pMeta->db, pMeta->txn); if (code != 0) { - stError("vgId:%d failed to commit stream meta", pMeta->vgId); - return code; + streamSetFatalError(pMeta, code, __func__, __LINE__); + stFatal("vgId:%d failed to commit stream meta, code:%s, line:%d", pMeta->vgId, tstrerror(code), + pMeta->fatalInfo.line); } code = tdbPostCommit(pMeta->db, pMeta->txn); if (code != 0) { - stError("vgId:%d failed to do post-commit stream meta", pMeta->vgId); + streamSetFatalError(pMeta, code, __func__, __LINE__); + stFatal("vgId:%d failed to do post-commit stream meta, code:%s, line:%d", pMeta->vgId, tstrerror(code), + pMeta->fatalInfo.line); return code; } code = tdbBegin(pMeta->db, &pMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); if (code != 0) { - stError("vgId:%d failed to begin trans", pMeta->vgId); - return code; + streamSetFatalError(pMeta, code, __func__, __LINE__); + stFatal("vgId:%d failed to begin trans, code:%s, line:%d", pMeta->vgId, tstrerror(code), pMeta->fatalInfo.line); + } else { + stDebug("vgId:%d stream meta file commit completed", pMeta->vgId); } - stDebug("vgId:%d stream meta file commit completed", pMeta->vgId); return code; } @@ -1274,7 +1248,7 @@ void streamMetaNotifyClose(SStreamMeta* pMeta) { void streamMetaStartHb(SStreamMeta* pMeta) { int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); if (pRid == NULL) { - stError("vgId:%d failed to prepare the metaHb to mnode, hbMsg will not started, code: out of memory", pMeta->vgId); + stFatal("vgId:%d failed to prepare the metaHb to mnode, hbMsg will not started, code: out of memory", pMeta->vgId); return; } @@ -1287,40 +1261,6 @@ void streamMetaStartHb(SStreamMeta* pMeta) { streamMetaHbToMnode(pRid, NULL); } -void streamMetaRLock(SStreamMeta* pMeta) { - // stTrace("vgId:%d meta-rlock", pMeta->vgId); - int32_t code = taosThreadRwlockRdlock(&pMeta->lock); - if (code) { - stError("vgId:%d meta-rlock failed, code:%s", pMeta->vgId, tstrerror(code)); - } -} - -void streamMetaRUnLock(SStreamMeta* pMeta) { - // stTrace("vgId:%d meta-runlock", pMeta->vgId); - int32_t code = taosThreadRwlockUnlock(&pMeta->lock); - if (code != TSDB_CODE_SUCCESS) { - stError("vgId:%d meta-runlock failed, code:%s", pMeta->vgId, tstrerror(code)); - } else { - // stTrace("vgId:%d meta-runlock completed", pMeta->vgId); - } -} - -void streamMetaWLock(SStreamMeta* pMeta) { - // stTrace("vgId:%d meta-wlock", pMeta->vgId); - int32_t code = taosThreadRwlockWrlock(&pMeta->lock); - if (code) { - stError("vgId:%d failed to apply wlock, code:%s", pMeta->vgId, tstrerror(code)); - } -} - -void streamMetaWUnLock(SStreamMeta* pMeta) { - // stTrace("vgId:%d meta-wunlock", pMeta->vgId); - int32_t code = taosThreadRwlockUnlock(&pMeta->lock); - if (code) { - stError("vgId:%d failed to apply wunlock, code:%s", pMeta->vgId, tstrerror(code)); - } -} - int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pList) { QRY_PARAM_CHECK(pList); @@ -1424,60 +1364,6 @@ int32_t streamMetaResetTaskStatus(SStreamMeta* pMeta) { return 0; } -int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { - int32_t code = TSDB_CODE_SUCCESS; - int64_t now = taosGetTimestampMs(); - int64_t startTs = 0; - bool hasFillhistoryTask = false; - STaskId hId = {0}; - - stDebug("vgId:%d add start failed task:0x%x", pMeta->vgId, taskId); - - streamMetaRLock(pMeta); - - STaskId id = {.streamId = streamId, .taskId = taskId}; - SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); - - if (ppTask != NULL) { - startTs = (*ppTask)->taskCheckInfo.startTs; - hasFillhistoryTask = HAS_RELATED_FILLHISTORY_TASK(*ppTask); - hId = (*ppTask)->hTaskInfo.id; - - streamMetaRUnLock(pMeta); - - // add the failed task info, along with the related fill-history task info into tasks list. - code = streamMetaAddTaskLaunchResult(pMeta, streamId, taskId, startTs, now, false); - if (hasFillhistoryTask) { - code = streamMetaAddTaskLaunchResult(pMeta, hId.streamId, hId.taskId, startTs, now, false); - } - } else { - streamMetaRUnLock(pMeta); - - stError("failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped", - streamId, taskId, pMeta->vgId); - code = TSDB_CODE_STREAM_TASK_NOT_EXIST; - } - - return code; -} - -void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs) { - int32_t startTs = pTask->execInfo.checkTs; - int32_t code = streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, startTs, failedTs, false); - if (code) { - stError("s-task:%s failed to add self task failed to start, code:%s", pTask->id.idStr, tstrerror(code)); - } - - // automatically set the related fill-history task to be failed. - if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - STaskId* pId = &pTask->hTaskInfo.id; - code = streamMetaAddTaskLaunchResult(pTask->pMeta, pId->streamId, pId->taskId, startTs, failedTs, false); - if (code) { - stError("s-task:0x%" PRIx64 " failed to add self task failed to start, code:%s", pId->taskId, tstrerror(code)); - } - } -} - void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SStreamTask* pHTask, int32_t transId, int64_t startTs) { const char* id = pTask->id.idStr; diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index cfa49fd92e..6c3685bb0f 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -287,7 +287,7 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) "s-task:%s inputQ is full, capacity(size:%d num:%dMiB), current(blocks:%d, size:%.2fMiB) stop to push data", pTask->id.idStr, STREAM_TASK_QUEUE_CAPACITY, STREAM_TASK_QUEUE_CAPACITY_IN_SIZE, total, size); streamDataSubmitDestroy(px); - return TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_STREAM_INPUTQ_FULL; } int32_t msgLen = px->submit.msgLen; @@ -312,7 +312,7 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) stTrace("s-task:%s input queue is full, capacity:%d size:%d MiB, current(blocks:%d, size:%.2fMiB) abort", pTask->id.idStr, STREAM_TASK_QUEUE_CAPACITY, STREAM_TASK_QUEUE_CAPACITY_IN_SIZE, total, size); streamFreeQitem(pItem); - return TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_STREAM_INPUTQ_FULL; } int32_t code = taosWriteQitem(pQueue, pItem); diff --git a/source/libs/stream/src/streamSched.c b/source/libs/stream/src/streamSched.c index f5b5524026..bc55019197 100644 --- a/source/libs/stream/src/streamSched.c +++ b/source/libs/stream/src/streamSched.c @@ -52,8 +52,10 @@ void streamSetupScheduleTrigger(SStreamTask* pTask) { int32_t ref = atomic_add_fetch_32(&pTask->refCnt, 1); stDebug("s-task:%s setup scheduler trigger, ref:%d delay:%" PRId64 " ms", id, ref, pTask->info.delaySchedParam); - pTask->schedInfo.pDelayTimer = taosTmrStart(streamTaskSchedHelper, (int32_t)delay, pTask, streamTimer); + streamTmrStart(streamTaskSchedHelper, delay, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, + pTask->pMeta->vgId, "sched-tmr"); pTask->schedInfo.status = TASK_TRIGGER_STATUS__INACTIVE; + } } int32_t streamTrySchedExec(SStreamTask* pTask) { @@ -101,13 +103,8 @@ void streamTaskResumeInFuture(SStreamTask* pTask) { // add one ref count for task streamMetaAcquireOneTask(pTask); - - if (pTask->schedInfo.pIdleTimer == NULL) { - pTask->schedInfo.pIdleTimer = taosTmrStart(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer); - } else { - streamTmrReset(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer, - &pTask->schedInfo.pIdleTimer, pTask->pMeta->vgId, "resume-task-tmr"); - } + streamTmrStart(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer, &pTask->schedInfo.pIdleTimer, + pTask->pMeta->vgId, "resume-task-tmr"); } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -150,7 +147,7 @@ void streamTaskSchedHelper(void* param, void* tmrId) { stTrace("s-task:%s in scheduler, trigger status:%d, next:%dms", id, status, nextTrigger); if (streamTaskShouldStop(pTask) || streamTaskShouldPause(pTask)) { - stDebug("s-task:%s jump out of schedTimer", id); + stDebug("s-task:%s should stop, jump out of schedTimer", id); return; } @@ -164,19 +161,16 @@ void streamTaskSchedHelper(void* param, void* tmrId) { if (code) { stError("s-task:%s failed to prepare retrieve data trigger, code:%s, try again in %dms", id, tstrerror(code), nextTrigger); - streamTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, - "sched-run-tmr"); terrno = code; - return; + goto _end; } atomic_store_8(&pTask->schedInfo.status, TASK_TRIGGER_STATUS__INACTIVE); code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTrigger); if (code != TSDB_CODE_SUCCESS) { - streamTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, - "sched-run-tmr"); - return; + stError("s-task:%s failed to put retrieve block into trigger, code:%s", pTask->id.idStr, tstrerror(code)); + goto _end; } code = streamTrySchedExec(pTask); @@ -186,5 +180,7 @@ void streamTaskSchedHelper(void* param, void* tmrId) { } } - streamTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, "sched-run-tmr"); +_end: + streamTmrStart(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, + "sched-run-tmr"); } diff --git a/source/libs/stream/src/streamStartHistory.c b/source/libs/stream/src/streamStartHistory.c index 21f0168434..1290aef6a3 100644 --- a/source/libs/stream/src/streamStartHistory.c +++ b/source/libs/stream/src/streamStartHistory.c @@ -80,6 +80,7 @@ int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated) { } void streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration) { + int32_t vgId = pTask->pMeta->vgId; int32_t numOfTicks = idleDuration / SCANHISTORY_IDLE_TIME_SLICE; if (numOfTicks <= 0) { numOfTicks = 1; @@ -100,14 +101,8 @@ void streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration) { int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s scan-history resumed in %.2fs, ref:%d", pTask->id.idStr, numOfTicks * 0.1, ref); - - if (pTask->schedHistoryInfo.pTimer == NULL) { - pTask->schedHistoryInfo.pTimer = - taosTmrStart(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer); - } else { - streamTmrReset(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, - &pTask->schedHistoryInfo.pTimer, pTask->pMeta->vgId, " start-history-task-tmr"); - } + streamTmrStart(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, + &pTask->schedHistoryInfo.pTimer, vgId, "history-task"); } int32_t streamTaskStartScanHistory(SStreamTask* pTask) { @@ -337,7 +332,7 @@ void doRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, i stDebug("s-task:%s status:%s failed to launch fill-history task:0x%x, retry launch:%dms, retryCount:%d", pTask->id.idStr, p, hTaskId, pHTaskInfo->waitInterval, pHTaskInfo->retryTimes); - streamTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamTimer, &pHTaskInfo->pTimer, + streamTmrStart(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamTimer, &pHTaskInfo->pTimer, pTask->pMeta->vgId, " start-history-task-tmr"); } } @@ -391,7 +386,7 @@ void tryLaunchHistoryTask(void* param, void* tmrId) { pHTaskInfo->tickCount -= 1; if (pHTaskInfo->tickCount > 0) { - streamTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamTimer, &pHTaskInfo->pTimer, + streamTmrStart(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamTimer, &pHTaskInfo->pTimer, pTask->pMeta->vgId, " start-history-task-tmr"); streamMetaReleaseTask(pMeta, pTask); return; @@ -519,7 +514,7 @@ int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) { } stDebug("s-task:%s set timer active flag, task timer not null", idStr); - streamTmrReset(tryLaunchHistoryTask, WAIT_FOR_MINIMAL_INTERVAL, pInfo, streamTimer, &pTask->hTaskInfo.pTimer, + streamTmrStart(tryLaunchHistoryTask, WAIT_FOR_MINIMAL_INTERVAL, pInfo, streamTimer, &pTask->hTaskInfo.pTimer, pTask->pMeta->vgId, " start-history-task-tmr"); } @@ -621,8 +616,8 @@ void doExecScanhistoryInFuture(void* param, void* tmrId) { // release the task. streamMetaReleaseTask(pTask->pMeta, pTask); } else { - streamTmrReset(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, - &pTask->schedHistoryInfo.pTimer, pTask->pMeta->vgId, " start-history-task-tmr"); + streamTmrStart(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, + &pTask->schedHistoryInfo.pTimer, pTask->pMeta->vgId, " start-history-task-tmr"); } } diff --git a/source/libs/stream/src/streamStartTask.c b/source/libs/stream/src/streamStartTask.c index 98c6534b46..0858f57414 100644 --- a/source/libs/stream/src/streamStartTask.c +++ b/source/libs/stream/src/streamStartTask.c @@ -30,8 +30,8 @@ typedef struct STaskInitTs { } STaskInitTs; static int32_t prepareBeforeStartTasks(SStreamMeta* pMeta, SArray** pList, int64_t now); -static bool allCheckDownstreamRsp(SStreamMeta* pMeta, STaskStartInfo* pStartInfo, int32_t numOfTotal); -static void displayStatusInfo(SStreamMeta* pMeta, SHashObj* pTaskSet, bool succ); +static bool allCheckDownstreamRsp(SStreamMeta* pMeta, STaskStartInfo* pStartInfo, int32_t numOfTotal); +static void displayStatusInfo(SStreamMeta* pMeta, SHashObj* pTaskSet, bool succ); // restore the checkpoint id by negotiating the latest consensus checkpoint id int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { @@ -505,3 +505,57 @@ void streamTaskSetReqConsenChkptId(SStreamTask* pTask, int64_t ts) { stDebug("s-task:%s set req consen-checkpointId flag, prev transId:%d, ts:%" PRId64, pTask->id.idStr, prevTrans, ts); } + +int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { + int32_t code = TSDB_CODE_SUCCESS; + int64_t now = taosGetTimestampMs(); + int64_t startTs = 0; + bool hasFillhistoryTask = false; + STaskId hId = {0}; + + stDebug("vgId:%d add start failed task:0x%x", pMeta->vgId, taskId); + + streamMetaRLock(pMeta); + + STaskId id = {.streamId = streamId, .taskId = taskId}; + SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); + + if (ppTask != NULL) { + startTs = (*ppTask)->taskCheckInfo.startTs; + hasFillhistoryTask = HAS_RELATED_FILLHISTORY_TASK(*ppTask); + hId = (*ppTask)->hTaskInfo.id; + + streamMetaRUnLock(pMeta); + + // add the failed task info, along with the related fill-history task info into tasks list. + code = streamMetaAddTaskLaunchResult(pMeta, streamId, taskId, startTs, now, false); + if (hasFillhistoryTask) { + code = streamMetaAddTaskLaunchResult(pMeta, hId.streamId, hId.taskId, startTs, now, false); + } + } else { + streamMetaRUnLock(pMeta); + + stError("failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped", + streamId, taskId, pMeta->vgId); + code = TSDB_CODE_STREAM_TASK_NOT_EXIST; + } + + return code; +} + +void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs) { + int32_t startTs = pTask->execInfo.checkTs; + int32_t code = streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, startTs, failedTs, false); + if (code) { + stError("s-task:%s failed to add self task failed to start, code:%s", pTask->id.idStr, tstrerror(code)); + } + + // automatically set the related fill-history task to be failed. + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + STaskId* pId = &pTask->hTaskInfo.id; + code = streamMetaAddTaskLaunchResult(pTask->pMeta, pId->streamId, pId->taskId, startTs, failedTs, false); + if (code) { + stError("s-task:0x%" PRIx64 " failed to add self task failed to start, code:%s", pId->taskId, tstrerror(code)); + } + } +} diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 598b563df3..19db90bc69 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -101,8 +101,10 @@ int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; + SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState)); stDebug("open stream state %p, %s", pState, path); + if (pState == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; QUERY_CHECK_CODE(code, lino, _end); @@ -136,7 +138,7 @@ SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, i _end: if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + qError("0x%x %s failed at line %d since %s", taskId, __func__, lino, tstrerror(code)); } return NULL; } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 244c4aaf88..bac382bae9 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -489,8 +489,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i STaskOutputInfo* pOutputInfo = &pTask->outputInfo; pOutputInfo->pTokenBucket = taosMemoryCalloc(1, sizeof(STokenBucket)); if (pOutputInfo->pTokenBucket == NULL) { - stError("s-task:%s failed to prepare the tokenBucket, code:%s", pTask->id.idStr, - tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + stError("s-task:%s failed to prepare the tokenBucket, code:%s", pTask->id.idStr, tstrerror(terrno)); return terrno; } @@ -804,8 +803,8 @@ bool streamTaskSetSchedStatusWait(SStreamTask* pTask) { pTask->status.schedStatus = TASK_SCHED_STATUS__WAITING; ret = true; } - streamMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); return ret; } diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index 06286479a3..3709c4dfbd 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -491,8 +491,8 @@ static void keepPrevInfo(SStreamTaskSM* pSM) { int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param) { SStreamTask* pTask = pSM->pTask; - const char* id = pTask->id.idStr; - int32_t code = 0; + const char* id = pTask->id.idStr; + int32_t code = 0; // do update the task status streamMutexLock(&pTask->lock); diff --git a/source/libs/stream/src/streamTimer.c b/source/libs/stream/src/streamTimer.c index 5e12f51c9d..8b77fe7cb1 100644 --- a/source/libs/stream/src/streamTimer.c +++ b/source/libs/stream/src/streamTimer.c @@ -40,11 +40,23 @@ int32_t streamTimerGetInstance(tmr_h* pTmr) { return TSDB_CODE_SUCCESS; } -void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId, +void streamTmrStart(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* pParam, void* pHandle, tmr_h* pTmrId, int32_t vgId, const char* pMsg) { - bool ret = taosTmrReset(fp, mseconds, param, handle, pTmrId); - if (ret) { + if (*pTmrId == NULL) { + *pTmrId = taosTmrStart(fp, mseconds, pParam, pHandle); + if (*pTmrId == NULL) { + stError("vgId:%d start %s tmr failed, code:%s", vgId, pMsg, tstrerror(terrno)); + return; + } + } else { + bool ret = taosTmrReset(fp, mseconds, pParam, pHandle, pTmrId); + if (ret) { + stError("vgId:%d start %s tmr failed, code:%s", vgId, pMsg, tstrerror(terrno)); + return; + } } + + stDebug("vgId:%d start %s tmr succ", vgId, pMsg); } void streamTmrStop(tmr_h tmrId) { diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index e1472aa9dd..155af0f2bf 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -344,7 +344,7 @@ bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* p void** pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); uint64_t index = ((uint64_t)tableId) % pInfo->numBuckets; TSKEY maxTs = *(TSKEY*)taosArrayGet(pInfo->pTsBuckets, index); - if (ts < maxTs - pInfo->watermark && maxTs != INT64_MIN) { + if (maxTs != INT64_MIN && ts < maxTs - pInfo->watermark) { // this window has been closed. if (pInfo->pCloseWinSBF) { code = tScalableBfPut(pInfo->pCloseWinSBF, pInfo->pKeyBuff, buffLen, &res); @@ -584,6 +584,8 @@ int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) { int32_t sBfSize = 0; if (tDecodeI32(&decoder, &sBfSize) < 0) return -1; pInfo->pTsSBFs = taosArrayInit(sBfSize, sizeof(void*)); + QUERY_CHECK_NULL(pInfo->pTsSBFs, code, lino, _error, terrno); + for (int32_t i = 0; i < sBfSize; i++) { SScalableBf* pSBf = NULL; code = tScalableBfDecode(&decoder, &pSBf); diff --git a/source/libs/stream/src/streamUtil.c b/source/libs/stream/src/streamUtil.c index b79ca32ff3..4c481e6041 100644 --- a/source/libs/stream/src/streamUtil.c +++ b/source/libs/stream/src/streamUtil.c @@ -35,3 +35,55 @@ void streamMutexDestroy(TdThreadMutex *pMutex) { stError("%p mutex destroy, code:%s", pMutex, tstrerror(code)); } } + +void streamMetaRLock(SStreamMeta* pMeta) { + // stTrace("vgId:%d meta-rlock", pMeta->vgId); + int32_t code = taosThreadRwlockRdlock(&pMeta->lock); + if (code) { + stError("vgId:%d meta-rlock failed, code:%s", pMeta->vgId, tstrerror(code)); + } +} + +void streamMetaRUnLock(SStreamMeta* pMeta) { + // stTrace("vgId:%d meta-runlock", pMeta->vgId); + int32_t code = taosThreadRwlockUnlock(&pMeta->lock); + if (code != TSDB_CODE_SUCCESS) { + stError("vgId:%d meta-runlock failed, code:%s", pMeta->vgId, tstrerror(code)); + } else { + // stTrace("vgId:%d meta-runlock completed", pMeta->vgId); + } +} + +void streamMetaWLock(SStreamMeta* pMeta) { + // stTrace("vgId:%d meta-wlock", pMeta->vgId); + int32_t code = taosThreadRwlockWrlock(&pMeta->lock); + if (code) { + stError("vgId:%d failed to apply wlock, code:%s", pMeta->vgId, tstrerror(code)); + } +} + +void streamMetaWUnLock(SStreamMeta* pMeta) { + // stTrace("vgId:%d meta-wunlock", pMeta->vgId); + int32_t code = taosThreadRwlockUnlock(&pMeta->lock); + if (code) { + stError("vgId:%d failed to apply wunlock, code:%s", pMeta->vgId, tstrerror(code)); + } +} + +void streamSetFatalError(SStreamMeta* pMeta, int32_t code, const char* funcName, int32_t lino) { + int32_t oldCode = atomic_val_compare_exchange_32(&pMeta->fatalInfo.code, 0, code); + if (oldCode == 0) { + pMeta->fatalInfo.ts = taosGetTimestampMs(); + pMeta->fatalInfo.threadId = taosGetSelfPthreadId(); + tstrncpy(pMeta->fatalInfo.func, funcName, tListLen(pMeta->fatalInfo.func)); + pMeta->fatalInfo.line = lino; + stInfo("vgId:%d set fatal error, code:%s %s line:%d", pMeta->vgId, tstrerror(code), funcName, lino); + } else { + stFatal("vgId:%d existed fatal error:%s, ts:%" PRId64 " failed to set new fatal error code:%s", pMeta->vgId, + tstrerror(pMeta->fatalInfo.code), pMeta->fatalInfo.ts, tstrerror(code)); + } +} + +int32_t streamGetFatalError(const SStreamMeta* pMeta) { + return atomic_load_32((volatile int32_t*) &pMeta->fatalInfo.code); +} diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index 0a330da4fa..923aaa945b 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -132,9 +132,9 @@ static void streamFileStateEncode(TSKEY* pKey, void** pVal, int32_t* pLen) { int32_t tmp = taosEncodeFixedI64(&buff, *pKey); } -SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, - GetTsFun fp, void* pFile, TSKEY delMark, const char* taskId, int64_t checkpointId, - int8_t type) { +int32_t streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, GetTsFun fp, + void* pFile, TSKEY delMark, const char* taskId, int64_t checkpointId, int8_t type, + SStreamFileState** ppFileState) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; if (memSize <= 0) { @@ -219,12 +219,13 @@ SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_ // todo(liuyao) optimize if (type == STREAM_STATE_BUFF_HASH) { - recoverSnapshot(pFileState, checkpointId); + code = recoverSnapshot(pFileState, checkpointId); } else if (type == STREAM_STATE_BUFF_SORT) { - recoverSesssion(pFileState, checkpointId); + code = recoverSesssion(pFileState, checkpointId); } else if (type == STREAM_STATE_BUFF_HASH_SORT) { - recoverFillSnapshot(pFileState, checkpointId); + code = recoverFillSnapshot(pFileState, checkpointId); } + QUERY_CHECK_CODE(code, lino, _end); void* valBuf = NULL; int32_t len = 0; @@ -235,14 +236,14 @@ SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_ qDebug("===stream===flushMark read:%" PRId64, pFileState->flushMark); } taosMemoryFreeClear(valBuf); + (*ppFileState) = pFileState; _end: if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); streamFileStateDestroy(pFileState); - pFileState = NULL; + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } - return pFileState; + return code; } void destroyRowBuffPos(SRowBuffPos* pPos) { @@ -866,8 +867,10 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { return code; } -void recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { +int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t winRes = TSDB_CODE_SUCCESS; if (pFileState->maxTs != INT64_MIN) { int64_t mark = (INT64_MIN + pFileState->deleteMark >= pFileState->maxTs) ? INT64_MIN @@ -878,7 +881,7 @@ void recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileState->pFileStore, INT64_MAX); int32_t recoverNum = TMIN(MIN_NUM_OF_RECOVER_ROW_BUFF, pFileState->maxRowCount); - while (code == TSDB_CODE_SUCCESS) { + while (winRes == TSDB_CODE_SUCCESS) { if (pFileState->curRowCount >= recoverNum) { break; } @@ -886,22 +889,34 @@ void recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { void* pVal = NULL; int32_t vlen = 0; SSessionKey key = {0}; - code = streamStateSessionGetKVByCur_rocksdb(pCur, &key, &pVal, &vlen); - if (code != TSDB_CODE_SUCCESS) { + winRes = streamStateSessionGetKVByCur_rocksdb(pCur, &key, &pVal, &vlen); + if (winRes != TSDB_CODE_SUCCESS) { break; } + + if (vlen != pFileState->rowSize) { + code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + QUERY_CHECK_CODE(code, lino, _end); + } + SRowBuffPos* pPos = createSessionWinBuff(pFileState, &key, pVal, &vlen); - code = putSessionWinResultBuff(pFileState, pPos); - if (code != TSDB_CODE_SUCCESS) { + winRes = putSessionWinResultBuff(pFileState, pPos); + if (winRes != TSDB_CODE_SUCCESS) { break; } - code = streamStateSessionCurPrev_rocksdb(pCur); + winRes = streamStateSessionCurPrev_rocksdb(pCur); + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } streamStateFreeCur(pCur); + return code; } -void recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { +int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; int32_t winCode = TSDB_CODE_SUCCESS; @@ -956,6 +971,7 @@ _end: qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } streamStateFreeCur(pCur); + return code; } int32_t streamFileStateGetSelectRowSize(SStreamFileState* pFileState) { return pFileState->selectivityRowSize; } @@ -997,10 +1013,11 @@ int32_t recoverFillSnapshot(SStreamFileState* pFileState, int64_t ckId) { SStreamStateCur* pCur = streamStateFillSeekToLast_rocksdb(pFileState->pFileStore); if (pCur == NULL) { - return -1; + return code; } int32_t recoverNum = TMIN(MIN_NUM_OF_RECOVER_ROW_BUFF, pFileState->maxRowCount); - while (code == TSDB_CODE_SUCCESS) { + int32_t winRes = TSDB_CODE_SUCCESS; + while (winRes == TSDB_CODE_SUCCESS) { if (pFileState->curRowCount >= recoverNum) { break; } @@ -1008,8 +1025,8 @@ int32_t recoverFillSnapshot(SStreamFileState* pFileState, int64_t ckId) { void* pVal = NULL; int32_t vlen = 0; SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); - code = streamStateFillGetKVByCur_rocksdb(pCur, pNewPos->pKey, (const void**)&pVal, &vlen); - if (code != TSDB_CODE_SUCCESS || isFlushedState(pFileState, pFileState->getTs(pNewPos->pKey), 0)) { + winRes = streamStateFillGetKVByCur_rocksdb(pCur, pNewPos->pKey, (const void**)&pVal, &vlen); + if (winRes != TSDB_CODE_SUCCESS || isFlushedState(pFileState, pFileState->getTs(pNewPos->pKey), 0)) { destroyRowBuffPos(pNewPos); SListNode* pNode = tdListPopTail(pFileState->usedBuffs); taosMemoryFreeClear(pNode); @@ -1020,8 +1037,8 @@ int32_t recoverFillSnapshot(SStreamFileState* pFileState, int64_t ckId) { memcpy(pNewPos->pRowBuff, pVal, vlen); taosMemoryFreeClear(pVal); pNewPos->beFlushed = true; - code = tSimpleHashPut(pFileState->rowStateBuff, pNewPos->pKey, pFileState->keyLen, &pNewPos, POINTER_BYTES); - if (code != TSDB_CODE_SUCCESS) { + winRes = tSimpleHashPut(pFileState->rowStateBuff, pNewPos->pKey, pFileState->keyLen, &pNewPos, POINTER_BYTES); + if (winRes != TSDB_CODE_SUCCESS) { destroyRowBuffPos(pNewPos); break; } @@ -1029,7 +1046,7 @@ int32_t recoverFillSnapshot(SStreamFileState* pFileState, int64_t ckId) { } streamStateFreeCur(pCur); - return TSDB_CODE_SUCCESS; + return code; } int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen, diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 009854b45b..0b653ddbe9 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -21,7 +21,7 @@ extern "C" { #endif #include "sync.h" -#include "taosdef.h" +#include "tglobal.h" #include "trpc.h" #include "ttimer.h" diff --git a/source/libs/sync/inc/syncPipeline.h b/source/libs/sync/inc/syncPipeline.h index ea85b796d5..427a3690f2 100644 --- a/source/libs/sync/inc/syncPipeline.h +++ b/source/libs/sync/inc/syncPipeline.h @@ -54,6 +54,7 @@ typedef struct SSyncLogBuffer { int64_t matchIndex; int64_t endIndex; int64_t size; + int64_t bytes; TdThreadMutex mutex; TdThreadMutexAttr attr; int64_t totalIndex; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 8e709f8809..8081de60c9 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -3294,11 +3294,11 @@ _out:; // proceed match index, with replicating on needed SyncIndex matchIndex = syncLogBufferProceed(ths->pLogBuf, ths, NULL, "Append"); - if(pEntry != NULL) + if (pEntry != NULL) sTrace("vgId:%d, append raft entry. index:%" PRId64 ", term:%" PRId64 " pBuf: [%" PRId64 " %" PRId64 " %" PRId64 - ", %" PRId64 ")", - ths->vgId, pEntry->index, pEntry->term, ths->pLogBuf->startIndex, ths->pLogBuf->commitIndex, - ths->pLogBuf->matchIndex, ths->pLogBuf->endIndex); + ", %" PRId64 ")", + ths->vgId, pEntry->index, pEntry->term, ths->pLogBuf->startIndex, ths->pLogBuf->commitIndex, + ths->pLogBuf->matchIndex, ths->pLogBuf->endIndex); if (code == 0 && ths->state == TAOS_SYNC_STATE_ASSIGNED_LEADER) { (void)syncNodeUpdateAssignedCommitIndex(ths, matchIndex); @@ -3449,8 +3449,8 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { pMsgReply->startTime = ths->startTime; pMsgReply->timeStamp = tsMs; - sGTrace("vgId:%d, process sync-heartbeat msg from dnode:%d, cluster:%d, Msgterm:%" PRId64 " currentTerm:%" PRId64, ths->vgId, - DID(&(pMsg->srcId)), CID(&(pMsg->srcId)), pMsg->term, currentTerm); + sGTrace("vgId:%d, process sync-heartbeat msg from dnode:%d, cluster:%d, Msgterm:%" PRId64 " currentTerm:%" PRId64, + ths->vgId, DID(&(pMsg->srcId)), CID(&(pMsg->srcId)), pMsg->term, currentTerm); if (pMsg->term > currentTerm && ths->state == TAOS_SYNC_STATE_LEARNER) { raftStoreSetTerm(ths, pMsg->term); diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 86d4d83d29..d042bf8ade 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -28,6 +28,8 @@ #include "syncUtil.h" #include "syncVoteMgr.h" +static int64_t tsLogBufferMemoryUsed = 0; // total bytes of vnode log buffer + static bool syncIsMsgBlock(tmsg_t type) { return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || (type == TDMT_VND_UPDATE_TAG_VAL) || (type == TDMT_VND_ALTER_CONFIRM); @@ -101,6 +103,10 @@ int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt SSyncLogBufEntry tmp = {.pItem = pEntry, .prevLogIndex = pMatch->index, .prevLogTerm = pMatch->term}; pBuf->entries[index % pBuf->size] = tmp; pBuf->endIndex = index + 1; + if (pNode->vgId > 1) { + pBuf->bytes += pEntry->bytes; + (void)atomic_add_fetch_64(&tsLogBufferMemoryUsed, (int64_t)pEntry->bytes); + } (void)taosThreadMutexUnlock(&pBuf->mutex); TAOS_CHECK_RETURN(syncLogBufferValidate(pBuf)); @@ -260,6 +266,10 @@ int32_t syncLogBufferInitWithoutLock(SSyncLogBuffer* pBuf, SSyncNode* pNode) { SSyncLogBufEntry tmp = {.pItem = pEntry, .prevLogIndex = -1, .prevLogTerm = -1}; pBuf->entries[index % pBuf->size] = tmp; taken = true; + if (pNode->vgId > 1) { + pBuf->bytes += pEntry->bytes; + (void)atomic_add_fetch_64(&tsLogBufferMemoryUsed, (int64_t)pEntry->bytes); + } } if (index < toIndex) { @@ -286,6 +296,10 @@ int32_t syncLogBufferInitWithoutLock(SSyncLogBuffer* pBuf, SSyncNode* pNode) { } SSyncLogBufEntry tmp = {.pItem = pDummy, .prevLogIndex = commitIndex - 1, .prevLogTerm = commitTerm}; pBuf->entries[(commitIndex + pBuf->size) % pBuf->size] = tmp; + if (pNode->vgId > 1) { + pBuf->bytes += pDummy->bytes; + (void)atomic_add_fetch_64(&tsLogBufferMemoryUsed, (int64_t)pDummy->bytes); + } if (index < toIndex) { pBuf->entries[(index + 1) % pBuf->size].prevLogIndex = commitIndex; @@ -330,6 +344,7 @@ int32_t syncLogBufferReInit(SSyncLogBuffer* pBuf, SSyncNode* pNode) { (void)memset(&pBuf->entries[(index + pBuf->size) % pBuf->size], 0, sizeof(pBuf->entries[0])); } pBuf->startIndex = pBuf->commitIndex = pBuf->matchIndex = pBuf->endIndex = 0; + pBuf->bytes = 0; int32_t code = syncLogBufferInitWithoutLock(pBuf, pNode); if (code < 0) { sError("vgId:%d, failed to re-initialize sync log buffer since %s.", pNode->vgId, tstrerror(code)); @@ -470,8 +485,12 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt goto _out; } SSyncLogBufEntry tmp = {.pItem = pEntry, .prevLogIndex = prevIndex, .prevLogTerm = prevTerm}; - pEntry = NULL; pBuf->entries[index % pBuf->size] = tmp; + if (pNode->vgId > 1) { + pBuf->bytes += pEntry->bytes; + (void)atomic_add_fetch_64(&tsLogBufferMemoryUsed, (int64_t)pEntry->bytes); + } + pEntry = NULL; // update end index pBuf->endIndex = TMAX(index + 1, pBuf->endIndex); @@ -846,14 +865,34 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm } // recycle + bool isVnode = pNode->vgId > 1; SyncIndex until = pBuf->commitIndex - TSDB_SYNC_LOG_BUFFER_RETENTION; - for (SyncIndex index = pBuf->startIndex; index < until; index++) { - SSyncRaftEntry* pEntry = pBuf->entries[(index + pBuf->size) % pBuf->size].pItem; - if (pEntry == NULL) return TSDB_CODE_SYN_INTERNAL_ERROR; + do { + if ((pBuf->startIndex >= pBuf->commitIndex) || + !((pBuf->startIndex < until) || (isVnode && pBuf->bytes >= TSDB_SYNC_LOG_BUFFER_THRESHOLD && + atomic_load_64(&tsLogBufferMemoryUsed) >= tsLogBufferMemoryAllowed))) { + break; + } + SSyncRaftEntry* pEntry = pBuf->entries[(pBuf->startIndex + pBuf->size) % pBuf->size].pItem; + if (pEntry == NULL) { + sError("vgId:%d, invalid log entry to recycle. index:%" PRId64 ", startIndex:%" PRId64 ", until:%" PRId64 + ", commitIndex:%" PRId64 ", endIndex:%" PRId64 ", term:%" PRId64, + pNode->vgId, pEntry->index, pBuf->startIndex, until, pBuf->commitIndex, pBuf->endIndex, pEntry->term); + return TSDB_CODE_SYN_INTERNAL_ERROR; + } + if (isVnode) { + pBuf->bytes -= pEntry->bytes; + (void)atomic_sub_fetch_64(&tsLogBufferMemoryUsed, (int64_t)pEntry->bytes); + } + sDebug("vgId:%d, recycle log entry. index:%" PRId64 ", startIndex:%" PRId64 ", until:%" PRId64 + ", commitIndex:%" PRId64 ", endIndex:%" PRId64 ", term:%" PRId64 ", entry bytes:%u, buf bytes:%" PRId64 + ", used:%" PRId64 ", allowed:%" PRId64, + pNode->vgId, pEntry->index, pBuf->startIndex, until, pBuf->commitIndex, pBuf->endIndex, pEntry->term, + pEntry->bytes, pBuf->bytes, atomic_load_64(&tsLogBufferMemoryUsed), tsLogBufferMemoryAllowed); syncEntryDestroy(pEntry); - (void)memset(&pBuf->entries[(index + pBuf->size) % pBuf->size], 0, sizeof(pBuf->entries[0])); - pBuf->startIndex = index + 1; - } + (void)memset(&pBuf->entries[(pBuf->startIndex + pBuf->size) % pBuf->size], 0, sizeof(pBuf->entries[0])); + ++pBuf->startIndex; + } while (true); code = 0; _out: @@ -1324,6 +1363,7 @@ void syncLogBufferClear(SSyncLogBuffer* pBuf) { (void)memset(&pBuf->entries[(index + pBuf->size) % pBuf->size], 0, sizeof(pBuf->entries[0])); } pBuf->startIndex = pBuf->commitIndex = pBuf->matchIndex = pBuf->endIndex = 0; + pBuf->bytes = 0; (void)taosThreadMutexUnlock(&pBuf->mutex); } diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index ff40616d70..93c214545f 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -2479,7 +2479,10 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { if (c > 0) { pBtc->idx += 1; } - (void)tdbBtcMoveDownward(pBtc); + if (tdbBtcMoveDownward(pBtc) < 0) { + tdbError("tdb/btc-move-to: btc move downward failed."); + return TSDB_CODE_FAILED; + } } } diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index e0e2a0c323..f62d728511 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -616,7 +616,7 @@ static void httpHandleReq(SHttpMsg* msg) { int32_t fd = taosCreateSocketWithTimeout(5000); if (fd < 0) { tError("http-report failed to open socket, dst:%s:%d, chanId:%" PRId64 ", seq:%" PRId64 ", reason:%s", cli->addr, - cli->port, chanId, cli->seq, tstrerror(TAOS_SYSTEM_ERROR(errno))); + cli->port, chanId, cli->seq, tstrerror(terrno)); destroyHttpClient(cli); (void)taosReleaseRef(httpRefMgt, chanId); return; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index a8f1d26f15..087e82d0ec 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1403,7 +1403,7 @@ static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd) { int32_t fd = taosCreateSocketWithTimeout(TRANS_CONN_TIMEOUT * 10); if (fd == -1) { tError("%s conn %p failed to create socket, reason:%s", transLabel(pTransInst), conn, - tstrerror(TAOS_SYSTEM_ERROR(errno))); + tstrerror(terrno)); cliHandleFastFail(conn, -1); return; } @@ -1883,9 +1883,9 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { int32_t fd = taosCreateSocketWithTimeout(TRANS_CONN_TIMEOUT * 10); if (fd == -1) { tGError("%s conn %p failed to create socket, reason:%s", transLabel(pTransInst), conn, - tstrerror(TAOS_SYSTEM_ERROR(errno))); + tstrerror(terrno)); cliHandleExcept(conn, -1); - errno = 0; + terrno = 0; return; } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 5f1ae02166..0dd55aacdb 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -684,7 +684,7 @@ _err: int64_t walGetVerRetention(SWal* pWal, int64_t bytes) { int64_t ver = -1; int64_t totSize = 0; - (void)taosThreadMutexLock(&pWal->mutex); + (void)taosThreadRwlockRdlock(&pWal->mutex); int32_t fileIdx = taosArrayGetSize(pWal->fileInfoSet); while (--fileIdx) { SWalFileInfo* pInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); @@ -694,7 +694,7 @@ int64_t walGetVerRetention(SWal* pWal, int64_t bytes) { } totSize += pInfo->fileSize; } - (void)taosThreadMutexUnlock(&pWal->mutex); + (void)taosThreadRwlockUnlock(&pWal->mutex); return ver + 1; } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index e1d31ce113..3bb4b9d747 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -21,11 +21,11 @@ #include "walInt.h" typedef struct { - int8_t stop; - int8_t inited; - uint32_t seq; - int32_t refSetId; - TdThread thread; + int8_t stop; + int8_t inited; + uint32_t seq; + int32_t refSetId; + TdThread thread; stopDnodeFn stopDnode; } SWalMgmt; @@ -81,6 +81,15 @@ void walCleanUp() { } } +static int32_t walInitLock(SWal *pWal) { + TdThreadRwlockAttr attr; + (void)taosThreadRwlockAttrInit(&attr); + (void)taosThreadRwlockAttrSetKindNP(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); + (void)taosThreadRwlockInit(&pWal->mutex, &attr); + (void)taosThreadRwlockAttrDestroy(&attr); + return 0; +} + SWal *walOpen(const char *path, SWalCfg *pCfg) { SWal *pWal = taosMemoryCalloc(1, sizeof(SWal)); if (pWal == NULL) { @@ -88,7 +97,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { return NULL; } - if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) { + if (walInitLock(pWal) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); taosMemoryFree(pWal); return NULL; @@ -179,7 +188,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { _err: taosArrayDestroy(pWal->fileInfoSet); taosHashCleanup(pWal->pRefHash); - TAOS_UNUSED(taosThreadMutexDestroy(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockDestroy(&pWal->mutex)); taosMemoryFreeClear(pWal); return NULL; @@ -215,15 +224,15 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { int32_t walPersist(SWal *pWal) { int32_t code = 0; - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex)); code = walSaveMeta(pWal); - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(code); } void walClose(SWal *pWal) { - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex)); (void)walSaveMeta(pWal); TAOS_UNUSED(taosCloseFile(&pWal->pLogFile)); pWal->pLogFile = NULL; @@ -243,7 +252,7 @@ void walClose(SWal *pWal) { } taosHashCleanup(pWal->pRefHash); pWal->pRefHash = NULL; - (void)taosThreadMutexUnlock(&pWal->mutex); + (void)taosThreadRwlockUnlock(&pWal->mutex); if (pWal->cfg.level == TAOS_WAL_SKIP) { wInfo("vgId:%d, remove all wals, path:%s", pWal->cfg.vgId, pWal->path); @@ -258,7 +267,7 @@ static void walFreeObj(void *wal) { SWal *pWal = wal; wDebug("vgId:%d, wal:%p is freed", pWal->cfg.vgId, pWal); - (void)taosThreadMutexDestroy(&pWal->mutex); + (void)taosThreadRwlockDestroy(&pWal->mutex); taosMemoryFreeClear(pWal); } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index aed7db327f..321a47d678 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -115,9 +115,9 @@ void walReaderValidVersionRange(SWalReader *pReader, int64_t *sver, int64_t *eve void walReaderVerifyOffset(SWalReader *pWalReader, STqOffsetVal *pOffset) { // if offset version is small than first version , let's seek to first version - TAOS_UNUSED(taosThreadMutexLock(&pWalReader->pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockRdlock(&pWalReader->pWal->mutex)); int64_t firstVer = walGetFirstVer((pWalReader)->pWal); - TAOS_UNUSED(taosThreadMutexUnlock(&pWalReader->pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWalReader->pWal->mutex)); if (pOffset->version < firstVer) { pOffset->version = firstVer; @@ -201,21 +201,29 @@ static int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) { // bsearch in fileSet SWalFileInfo tmpInfo; tmpInfo.firstVer = ver; - SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); - if (pRet == NULL) { + TAOS_UNUSED(taosThreadRwlockRdlock(&pWal->mutex)); + SWalFileInfo *gloablPRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); + if (gloablPRet == NULL) { wError("failed to find WAL log file with ver:%" PRId64, ver); - + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER); } - + SWalFileInfo *pRet = taosMemoryMalloc(sizeof(SWalFileInfo)); + if (pRet == NULL) { + wError("failed to allocate memory for localRet"); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } + TAOS_MEMCPY(pRet, gloablPRet, sizeof(SWalFileInfo)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); if (pReader->curFileFirstVer != pRet->firstVer) { // error code was set inner - TAOS_CHECK_RETURN(walReadChangeFile(pReader, pRet->firstVer)); + TAOS_CHECK_RETURN_WITH_FREE(walReadChangeFile(pReader, pRet->firstVer), pRet); } // error code was set inner - TAOS_CHECK_RETURN(walReadSeekFilePos(pReader, pRet->firstVer, ver)); - + TAOS_CHECK_RETURN_WITH_FREE(walReadSeekFilePos(pReader, pRet->firstVer, ver), pRet); + taosMemoryFree(pRet); wDebug("vgId:%d, wal version reset from %" PRId64 " to %" PRId64, pReader->pWal->cfg.vgId, pReader->curVersion, ver); pReader->curVersion = ver; diff --git a/source/libs/wal/src/walRef.c b/source/libs/wal/src/walRef.c index 579921a7e0..bf24ed89fb 100644 --- a/source/libs/wal/src/walRef.c +++ b/source/libs/wal/src/walRef.c @@ -61,34 +61,34 @@ int32_t walSetRefVer(SWalRef *pRef, int64_t ver) { SWal *pWal = pRef->pWal; wDebug("vgId:%d, wal ref version %" PRId64 ", refId %" PRId64, pWal->cfg.vgId, ver, pRef->refId); if (pRef->refVer != ver) { - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex)); if (ver < pWal->vers.firstVer || ver > pWal->vers.lastVer) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER); } pRef->refVer = ver; - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); } TAOS_RETURN(TSDB_CODE_SUCCESS); } void walRefFirstVer(SWal *pWal, SWalRef *pRef) { - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockRdlock(&pWal->mutex)); pRef->refVer = pWal->vers.firstVer; - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); wDebug("vgId:%d, wal ref version %" PRId64 " for first", pWal->cfg.vgId, pRef->refVer); } void walRefLastVer(SWal *pWal, SWalRef *pRef) { - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockRdlock(&pWal->mutex)); pRef->refVer = pWal->vers.lastVer; - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); wDebug("vgId:%d, wal ref version %" PRId64 " for last", pWal->cfg.vgId, pRef->refVer); } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 224a7dc711..a5105fc107 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -23,7 +23,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { int32_t code = 0; - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex)); wInfo("vgId:%d, restore from snapshot, version %" PRId64, pWal->cfg.vgId, ver); @@ -34,7 +34,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { SWalRef *pRef = *(SWalRef **)pIter; if (pRef->refVer != -1 && pRef->refVer <= ver) { taosHashCancelIterate(pWal->pRefHash, pIter); - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TSDB_CODE_FAILED); } @@ -51,7 +51,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); if (taosRemoveFile(fnameStr) < 0) { wError("vgId:%d, restore from snapshot, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(terrno); } @@ -60,7 +60,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); if (taosRemoveFile(fnameStr) < 0) { wError("vgId:%d, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(terrno); } @@ -81,7 +81,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { pWal->vers.snapshotVer = ver; pWal->vers.verInSnapshotting = -1; - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -160,12 +160,12 @@ static int64_t walChangeWrite(SWal *pWal, int64_t ver) { } int32_t walRollback(SWal *pWal, int64_t ver) { - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex)); wInfo("vgId:%d, wal rollback for version %" PRId64, pWal->cfg.vgId, ver); int64_t code; char fnameStr[WAL_FILE_LEN]; if (ver > pWal->vers.lastVer || ver <= pWal->vers.commitVer || ver <= pWal->vers.snapshotVer) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER); } @@ -175,7 +175,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // change current files code = walChangeWrite(pWal, ver); if (code < 0) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(code); } @@ -198,21 +198,21 @@ int32_t walRollback(SWal *pWal, int64_t ver) { TAOS_UNUSED(taosCloseFile(&pWal->pIdxFile)); TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND); if (pIdxFile == NULL) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(terrno); } int64_t idxOff = walGetVerIdxOffset(pWal, ver); code = taosLSeekFile(pIdxFile, idxOff, SEEK_SET); if (code < 0) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(terrno); } // read idx file and get log file pos SWalIdxEntry entry; if (taosReadFile(pIdxFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(terrno); } @@ -223,14 +223,14 @@ int32_t walRollback(SWal *pWal, int64_t ver) { wDebug("vgId:%d, wal truncate file %s", pWal->cfg.vgId, fnameStr); if (pLogFile == NULL) { // TODO - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(terrno); } code = taosLSeekFile(pLogFile, entry.offset, SEEK_SET); if (code < 0) { // TODO - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(terrno); } @@ -238,19 +238,19 @@ int32_t walRollback(SWal *pWal, int64_t ver) { SWalCkHead head; int64_t size = taosReadFile(pLogFile, &head, sizeof(SWalCkHead)); if (size != sizeof(SWalCkHead)) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(terrno); } code = walValidHeadCksum(&head); if (code != 0) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED); } if (head.head.version != ver) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED); } @@ -258,13 +258,13 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // truncate old files code = taosFtruncateFile(pLogFile, entry.offset); if (code < 0) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(code); } code = taosFtruncateFile(pIdxFile, idxOff); if (code < 0) { - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(code); } @@ -278,13 +278,13 @@ int32_t walRollback(SWal *pWal, int64_t ver) { code = walSaveMeta(pWal); if (code < 0) { wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr()); - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(code); } // unlock - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -375,7 +375,7 @@ int32_t walBeginSnapshot(SWal *pWal, int64_t ver, int64_t logRetention) { TAOS_RETURN(TSDB_CODE_FAILED); } - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex)); pWal->vers.verInSnapshotting = ver; pWal->vers.logRetention = logRetention; @@ -391,7 +391,7 @@ int32_t walBeginSnapshot(SWal *pWal, int64_t ver, int64_t logRetention) { } _exit: - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(code); } @@ -399,7 +399,7 @@ _exit: int32_t walEndSnapshot(SWal *pWal) { int32_t code = 0, lino = 0; - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex)); int64_t ver = pWal->vers.verInSnapshotting; wDebug("vgId:%d, wal end snapshot for version %" PRId64 ", log retention %" PRId64 " first ver %" PRId64 @@ -508,7 +508,7 @@ int32_t walEndSnapshot(SWal *pWal) { taosArrayClear(pWal->toDeleteFiles); _exit: - taosThreadMutexUnlock(&pWal->mutex); + taosThreadRwlockUnlock(&pWal->mutex); if (code) { wError("vgId:%d, %s failed at line %d since %s", pWal->cfg.vgId, __func__, lino, tstrerror(code)); @@ -717,7 +717,7 @@ int32_t walAppendLog(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syn int32_t bodyLen) { int32_t code = 0, lino = 0; - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex)); if (index != pWal->vers.lastVer + 1) { TAOS_CHECK_GOTO(TSDB_CODE_WAL_INVALID_VER, &lino, _exit); @@ -736,7 +736,7 @@ _exit: wError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); return code; } @@ -747,7 +747,7 @@ int32_t walFsync(SWal *pWal, bool forceFsync) { return code; } - TAOS_UNUSED(taosThreadMutexLock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex)); if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); if (taosFsyncFile(pWal->pLogFile) < 0) { @@ -756,7 +756,7 @@ int32_t walFsync(SWal *pWal, bool forceFsync) { code = terrno; } } - TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); + TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); return code; } diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index e06c1a6aac..04747cacd4 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -109,8 +109,8 @@ void taosRemoveDir(const char *dirname) { } } - (void)taosCloseDir(&pDir); - (void)rmdir(dirname); + TAOS_UNUSED(taosCloseDir(&pDir)); + TAOS_UNUSED(rmdir(dirname)); // printf("dir:%s is removed\n", dirname); return; @@ -374,7 +374,7 @@ int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen) { bool taosIsDir(const char *dirname) { TdDirPtr pDir = taosOpenDir(dirname); if (pDir != NULL) { - (void)taosCloseDir(&pDir); + TAOS_SKIP_ERROR(taosCloseDir(&pDir)); return true; } return false; diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index c0273e4a6f..f2c90e778d 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -50,7 +50,10 @@ int32_t osDefaultInit() { taosSeedRand(taosSafeRand()); taosGetSystemLocale(tsLocale, tsCharset); - taosGetSystemTimezone(tsTimezoneStr, &tsTimezone); + code = taosGetSystemTimezone(tsTimezoneStr, &tsTimezone); + if(code != 0) { + return code; + } if (strlen(tsTimezoneStr) > 0) { // ignore empty timezone if ((code = taosSetSystemTimezone(tsTimezoneStr, tsTimezoneStr, &tsDaylight, &tsTimezone)) != TSDB_CODE_SUCCESS) return code; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 0e5b6b71a1..40f48af266 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -1535,8 +1535,9 @@ int taosSeekCFile(FILE *file, int64_t offset, int whence) { int code = fseeko(file, offset, whence); if (-1 == code) { terrno = TAOS_SYSTEM_ERROR(errno); + code = terrno; } - return terrno; + return code; #endif } diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c index da4e8dfb9d..b99017782b 100644 --- a/source/os/src/osRand.c +++ b/source/os/src/osRand.c @@ -69,7 +69,7 @@ uint32_t taosSafeRand(void) { if (len < 0) { seed = (int)taosGetTimestampSec(); } - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); } return (uint32_t)seed; diff --git a/source/os/src/osSignal.c b/source/os/src/osSignal.c index 7060b408ca..f341f5422d 100644 --- a/source/os/src/osSignal.c +++ b/source/os/src/osSignal.c @@ -31,23 +31,35 @@ int32_t taosSetSignal(int32_t signum, FSignalHandler sigfp) { // SIGHUP doesn't exist in windows, we handle it in the way of ctrlhandler if (signum == SIGHUP) { - SetConsoleCtrlHandler((PHANDLER_ROUTINE)sigfp, TRUE); + if(SetConsoleCtrlHandler((PHANDLER_ROUTINE)sigfp, TRUE) == 0) { + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); + return terrno; + } } else { - signal(signum, (FWinSignalHandler)sigfp); + if(signal(signum, (FWinSignalHandler)sigfp) == SIG_ERR) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } } return 0; } int32_t taosIgnSignal(int32_t signum) { if (signum == SIGUSR1 || signum == SIGHUP) return 0; - signal(signum, SIG_IGN); + if(signal(signum, SIG_IGN) == SIG_ERR) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } return 0; } int32_t taosDflSignal(int32_t signum) { if (signum == SIGUSR1 || signum == SIGHUP) return 0; - signal(signum, SIG_DFL); + if(signal(signum, SIG_DFL) == SIG_ERR) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } return 0; } diff --git a/source/os/src/osSleep.c b/source/os/src/osSleep.c index f72805b15d..67a621e8b7 100644 --- a/source/os/src/osSleep.c +++ b/source/os/src/osSleep.c @@ -37,7 +37,7 @@ void taosMsleep(int32_t ms) { #ifdef WINDOWS Sleep(ms); #else - (void)usleep(ms * 1000); + TAOS_SKIP_ERROR(usleep(ms * 1000)); #endif } @@ -49,10 +49,15 @@ void taosUsleep(int32_t us) { interval.QuadPart = (10 * us); timer = CreateWaitableTimer(NULL, TRUE, NULL); - SetWaitableTimer(timer, &interval, 0, NULL, NULL, 0); - WaitForSingleObject(timer, INFINITE); - CloseHandle(timer); + if (timer == NULL) { + return; + } + if (!SetWaitableTimer(timer, &interval, 0, NULL, NULL, 0)) { + return; + } + TAOS_SKIP_ERROR(WaitForSingleObject(timer, INFINITE)); + TAOS_SKIP_ERROR(CloseHandle(timer)); #else - (void)usleep(us); + TAOS_SKIP_ERROR(usleep(us)); #endif } diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 4d650715f9..2065ba68fe 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -120,7 +120,12 @@ int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t int32_t taosCloseSocketNoCheck1(SocketFd fd) { #ifdef WINDOWS - return closesocket(fd); + int ret = closesocket(fd); + if (ret == SOCKET_ERROR) { + int errorCode = WSAGetLastError(); + return terrno = TAOS_SYSTEM_WINSOCKET_ERROR(errorCode); + } + return 0; #else int32_t code = close(fd); if (-1 == code) { @@ -295,14 +300,18 @@ int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void } #endif - return setsockopt(pSocket->fd, level, optname, optval, optlen); +int ret = setsockopt(pSocket->fd, level, optname, optval, optlen); +if (ret == SOCKET_ERROR) { + int errorCode = WSAGetLastError(); + return terrno = TAOS_SYSTEM_WINSOCKET_ERROR(errorCode); +} #else int32_t code = setsockopt(pSocket->fd, level, optname, optval, (int)optlen); if (-1 == code) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } - return code; + return 0; #endif } @@ -320,19 +329,6 @@ int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void #endif -uint32_t taosInetAddr(const char *ipAddr) { -#ifdef WINDOWS - uint32_t value; - int32_t ret = inet_pton(AF_INET, ipAddr, &value); - if (ret <= 0) { - return INADDR_NONE; - } else { - return value; - } -#else - return inet_addr(ipAddr); -#endif -} const char *taosInetNtoa(struct in_addr ipInt, char *dstStr, int32_t len) { const char* r = inet_ntop(AF_INET, &ipInt, dstStr, len); if (NULL == r) { @@ -770,10 +766,7 @@ bool taosValidIpAndPort(uint32_t ip, uint16_t port) { TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); if (pSocket == NULL) { - code = terrno; - (void)taosCloseSocketNoCheck1(fd); - terrno = code; - + TAOS_SKIP_ERROR(taosCloseSocketNoCheck1(fd)); return false; } pSocket->refId = 0; @@ -782,22 +775,18 @@ bool taosValidIpAndPort(uint32_t ip, uint16_t port) { /* set REUSEADDR option, so the portnumber can be re-used */ reuse = 1; if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { - code = terrno; - (void)taosCloseSocket(&pSocket); - terrno = code; - + TAOS_SKIP_ERROR(taosCloseSocket(&pSocket)); return false; } /* bind socket to server address */ if (-1 == bind(pSocket->fd, (struct sockaddr *)&serverAdd, sizeof(serverAdd))) { - code = TAOS_SYSTEM_ERROR(errno); - (void)taosCloseSocket(&pSocket); - terrno = code; + terrno = TAOS_SYSTEM_ERROR(errno); + TAOS_SKIP_ERROR(taosCloseSocket(&pSocket)); return false; } - (void)taosCloseSocket(&pSocket); + TAOS_SKIP_ERROR(taosCloseSocket(&pSocket)); return true; } @@ -945,8 +934,7 @@ int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip) { int iResult; iResult = WSAStartup(MAKEWORD(2, 2), &wsaData); if (iResult != 0) { - // printf("WSAStartup failed: %d\n", iResult); - return 0xFFFFFFFF; + return TAOS_SYSTEM_WINSOCKET_ERROR(WSAGetLastError()); } #endif @@ -1010,7 +998,7 @@ int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip) { #endif *ip = 0xFFFFFFFF; - return 0xFFFFFFFF; + return TSDB_CODE_RPC_FQDN_ERROR; } #endif } @@ -1023,7 +1011,7 @@ int32_t taosGetFqdn(char *fqdn) { iResult = WSAStartup(MAKEWORD(2, 2), &wsaData); if (iResult != 0) { // printf("WSAStartup failed: %d\n", iResult); - return 1; + return TAOS_SYSTEM_WINSOCKET_ERROR(WSAGetLastError()); } #endif char hostname[1024]; @@ -1079,8 +1067,8 @@ int32_t taosGetFqdn(char *fqdn) { int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { - fprintf(stderr, "failed to get fqdn, code:%d, hostname:%s, reason:%s\n", ret, hostname, gai_strerror(ret)); - return -1; + //fprintf(stderr, "failed to get fqdn, code:%d, hostname:%s, reason:%s\n", ret, hostname, gai_strerror(ret)); + return TAOS_SYSTEM_WINSOCKET_ERROR(WSAGetLastError()); } strcpy(fqdn, result->ai_canonname); freeaddrinfo(result); @@ -1142,13 +1130,13 @@ int32_t taosCreateSocketWithTimeout(uint32_t timeout) { if ((fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP)) == INVALID_SOCKET) { terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno; } #if defined(WINDOWS) if (0 != setsockopt(fd, IPPROTO_TCP, TCP_MAXRT, (char *)&timeout, sizeof(timeout))) { taosCloseSocketNoCheck1(fd); - return -1; + return TAOS_SYSTEM_WINSOCKET_ERROR(WSAGetLastError()); } #elif defined(_TD_DARWIN_64) // invalid config @@ -1160,10 +1148,9 @@ int32_t taosCreateSocketWithTimeout(uint32_t timeout) { #else // Linux like systems uint32_t conn_timeout_ms = timeout; if (-1 == setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, (char *)&conn_timeout_ms, sizeof(conn_timeout_ms))) { - int32_t code = TAOS_SYSTEM_ERROR(errno); - (void)taosCloseSocketNoCheck1(fd); - terrno = code; - return -1; + terrno = TAOS_SYSTEM_ERROR(errno); + TAOS_SKIP_ERROR(taosCloseSocketNoCheck1(fd)); + return terrno; } #endif diff --git a/source/os/src/osString.c b/source/os/src/osString.c index ffc64f3493..68392b5050 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -216,14 +216,14 @@ int32_t taosConvInit(void) { for (int32_t i = 0; i < gConvMaxNum[M2C]; ++i) { gConv[M2C][i].conv = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); - if ((iconv_t)-1 == gConv[M2C][i].conv || (iconv_t)0 == gConv[M2C][i].conv) { + if ((iconv_t)-1 == gConv[M2C][i].conv) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } } for (int32_t i = 0; i < gConvMaxNum[1 - M2C]; ++i) { gConv[1 - M2C][i].conv = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC); - if ((iconv_t)-1 == gConv[1 - M2C][i].conv || (iconv_t)0 == gConv[1 - M2C][i].conv) { + if ((iconv_t)-1 == gConv[1 - M2C][i].conv) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } @@ -251,13 +251,13 @@ iconv_t taosAcquireConv(int32_t *idx, ConvType type) { *idx = -1; if (type == M2C) { iconv_t c = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); - if ((iconv_t)-1 == c || (iconv_t)0 == c) { + if ((iconv_t)-1 == c) { terrno = TAOS_SYSTEM_ERROR(errno); } return c; } else { iconv_t c = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC); - if ((iconv_t)-1 == c || (iconv_t)0 == c) { + if ((iconv_t)-1 == c) { terrno = TAOS_SYSTEM_ERROR(errno); } return c; @@ -289,7 +289,11 @@ iconv_t taosAcquireConv(int32_t *idx, ConvType type) { } *idx = startId; - return gConv[type][startId].conv; + if ((iconv_t)0 == gConv[type][startId].conv) { + return (iconv_t)-1; + } else { + return gConv[type][startId].conv; + } } void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type) { @@ -312,7 +316,7 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4 int32_t idx = -1; iconv_t conv = taosAcquireConv(&idx, M2C); - if ((iconv_t)-1 == conv || (iconv_t)0 == conv) { + if ((iconv_t)-1 == conv) { return false; } @@ -350,8 +354,8 @@ int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) { int32_t idx = -1; int32_t code = 0; iconv_t conv = taosAcquireConv(&idx, C2M); - if ((iconv_t)-1 == conv || (iconv_t)0 == conv) { - return TSDB_CODE_APP_ERROR; + if ((iconv_t)-1 == conv) { + return terrno; } size_t ucs4_input_len = ucs4_max_len; diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 36b1d7b60d..4d97232806 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -93,6 +93,10 @@ LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) { path[len - 1] = 'p'; HANDLE file = CreateFile(path, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (file == INVALID_HANDLE_VALUE) { + FreeLibrary(dll); + return EXCEPTION_CONTINUE_SEARCH; + } MINIDUMP_EXCEPTION_INFORMATION mei; mei.ThreadId = GetCurrentThreadId(); @@ -191,10 +195,11 @@ static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { &cpuInfo->si, &cpuInfo->st, &cpuInfo->guest, &cpuInfo->guest_nice); if (EOF == code) { terrno = TAOS_SYSTEM_ERROR(errno); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); return terrno; } - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); #endif return 0; @@ -263,9 +268,9 @@ bool taosCheckSystemIsLittleEnd() { void taosGetSystemInfo() { #ifdef WINDOWS - taosGetCpuCores(&tsNumOfCores, false); - taosGetTotalMemory(&tsTotalMemoryKB); - taosGetCpuUsage(NULL, NULL); + TAOS_SKIP_ERROR(taosGetCpuCores(&tsNumOfCores, false)); + TAOS_SKIP_ERROR(taosGetTotalMemory(&tsTotalMemoryKB)); + TAOS_SKIP_ERROR(taosGetCpuUsage(NULL, NULL)); #elif defined(_TD_DARWIN_64) long physical_pages = sysconf(_SC_PHYS_PAGES); long page_size = sysconf(_SC_PAGESIZE); @@ -274,10 +279,10 @@ void taosGetSystemInfo() { tsNumOfCores = sysconf(_SC_NPROCESSORS_ONLN); #else taosGetProcIOnfos(); - (void)taosGetCpuCores(&tsNumOfCores, false); - (void)taosGetTotalMemory(&tsTotalMemoryKB); - (void)taosGetCpuUsage(NULL, NULL); - (void)taosGetCpuInstructions(&tsSSE42Supported, &tsAVXSupported, &tsAVX2Supported, &tsFMASupported, &tsAVX512Supported); + TAOS_SKIP_ERROR(taosGetCpuCores(&tsNumOfCores, false)); + TAOS_SKIP_ERROR(taosGetTotalMemory(&tsTotalMemoryKB)); + TAOS_SKIP_ERROR(taosGetCpuUsage(NULL, NULL)); + TAOS_SKIP_ERROR(taosGetCpuInstructions(&tsSSE42Supported, &tsAVXSupported, &tsAVX2Supported, &tsFMASupported, &tsAVX512Supported)); #endif } @@ -313,11 +318,11 @@ int32_t taosGetEmail(char *email, int32_t maxLen) { if (taosReadFile(pFile, (void *)email, maxLen) < 0) { int32_t code = terrno; - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); return code; } - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); return 0; #endif @@ -332,7 +337,10 @@ bool getWinVersionReleaseName(char *releaseName, int32_t maxLen) { UINT uLen; VS_FIXEDFILEINFO *pFileInfo; - GetWindowsDirectory(szFileName, MAX_PATH); + int ret = GetWindowsDirectory(szFileName, MAX_PATH); + if (ret == 0) { + return false; + } wsprintf(szFileName, L"%s%s", szFileName, L"\\explorer.exe"); dwLen = GetFileVersionInfoSize(szFileName, &dwHandle); if (dwLen == 0) { @@ -372,12 +380,12 @@ int32_t taosGetOsReleaseName(char *releaseName, char* sName, char* ver, int32_t if(sName) snprintf(sName, maxLen, "macOS"); if (sysctl(osversion_name, 2, osversion, &osversion_len, NULL, 0) == -1) { - return -1; + return TAOS_SYSTEM_ERROR(errno); } uint32_t major, minor; - if (sscanf(osversion, "%u.%u", &major, &minor) != 2) { - return -1; + if (sscanf(osversion, "%u.%u", &major, &minor) == EOF) { + return TAOS_SYSTEM_ERROR(errno); } if (major >= 20) { major -= 9; // macOS 11 and newer @@ -432,8 +440,11 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { #ifdef WINDOWS char value[100]; DWORD bufferSize = sizeof(value); - RegGetValue(HKEY_LOCAL_MACHINE, "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", "ProcessorNameString", + LSTATUS ret = RegGetValue(HKEY_LOCAL_MACHINE, "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", "ProcessorNameString", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); + if (ret != ERROR_SUCCESS) { + return TAOS_SYSTEM_ERROR(ret); + } tstrncpy(cpuModel, value, maxLen); SYSTEM_INFO si; memset(&si, 0, sizeof(SYSTEM_INFO)); @@ -695,7 +706,7 @@ int32_t taosGetTotalMemory(int64_t *totalKB) { MEMORYSTATUSEX memsStat; memsStat.dwLength = sizeof(memsStat); if (!GlobalMemoryStatusEx(&memsStat)) { - return -1; + return TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); } *totalKB = memsStat.ullTotalPhys / 1024; @@ -704,6 +715,9 @@ int32_t taosGetTotalMemory(int64_t *totalKB) { return 0; #else *totalKB = (int64_t)(sysconf(_SC_PHYS_PAGES) * tsPageSizeKB); + if(*totalKB <= 0) { + return TAOS_SYSTEM_ERROR(errno); + } return 0; #endif } @@ -748,7 +762,7 @@ int32_t taosGetProcMemory(int64_t *usedKB) { char tmp[10]; (void)sscanf(line, "%s %" PRId64, tmp, usedKB); - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); return 0; #endif @@ -759,7 +773,7 @@ int32_t taosGetSysMemory(int64_t *usedKB) { MEMORYSTATUSEX memsStat; memsStat.dwLength = sizeof(memsStat); if (!GlobalMemoryStatusEx(&memsStat)) { - return -1; + return TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); } int64_t nMemFree = memsStat.ullAvailPhys / 1024; @@ -772,6 +786,9 @@ int32_t taosGetSysMemory(int64_t *usedKB) { return 0; #else *usedKB = sysconf(_SC_AVPHYS_PAGES) * tsPageSizeKB; + if(*usedKB <= 0) { + return TAOS_SYSTEM_ERROR(errno); + } return 0; #endif } @@ -831,7 +848,7 @@ int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int if (write_bytes) *write_bytes = 0; return 0; } - return -1; + return TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); #elif defined(_TD_DARWIN_64) if (rchars) *rchars = 0; if (wchars) *wchars = 0; @@ -1021,7 +1038,10 @@ void taosKillSystem() { int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { #ifdef WINDOWS GUID guid; - CoCreateGuid(&guid); + HRESULT h = CoCreateGuid(&guid); + if (h != S_OK) { + return TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); + } snprintf(uid, uidlen, "%08X-%04X-%04X-%02X%02X-%02X%02X%02X%02X%02X%02X", guid.Data1, guid.Data2, guid.Data3, guid.Data4[0], guid.Data4[1], guid.Data4[2], guid.Data4[3], guid.Data4[4], guid.Data4[5], guid.Data4[6], guid.Data4[7]); @@ -1045,7 +1065,7 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { return terrno; } else { len = taosReadFile(pFile, uid, uidlen); - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); if (len < 0) { return len; } @@ -1087,7 +1107,7 @@ char *taosGetCmdlineByPID(int pid) { cmdline[n] = 0; - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); } else { cmdline[0] = 0; } @@ -1225,14 +1245,13 @@ SysNameInfo taosGetSysNameInfo() { } char localHostName[512]; - taosGetlocalhostname(localHostName, 512); + TAOS_SKIP_ERROR(taosGetlocalhostname(localHostName, 512)); TdCmdPtr pCmd = taosOpenCmd("scutil --get LocalHostName"); tstrncpy(info.nodename, localHostName, sizeof(info.nodename)); return info; #else SysNameInfo info = {0}; - struct utsname uts; if (!uname(&uts)) { tstrncpy(info.sysname, uts.sysname, sizeof(info.sysname)); @@ -1268,7 +1287,7 @@ bool taosCheckCurrentInDll() { } #ifdef _TD_DARWIN_64 -int taosGetMaclocalhostnameByCommand(char *hostname, size_t maxLen) { +int32_t taosGetMaclocalhostnameByCommand(char *hostname, size_t maxLen) { TdCmdPtr pCmd = taosOpenCmd("scutil --get LocalHostName"); if (pCmd != NULL) { if (taosGetsCmd(pCmd, maxLen - 1, hostname) > 0) { @@ -1280,10 +1299,10 @@ int taosGetMaclocalhostnameByCommand(char *hostname, size_t maxLen) { } taosCloseCmd(&pCmd); } - return -1; + return TAOS_SYSTEM_ERROR(errno); } -int getMacLocalHostNameBySCD(char *hostname, size_t maxLen) { +int32_t getMacLocalHostNameBySCD(char *hostname, size_t maxLen) { SCDynamicStoreRef store = SCDynamicStoreCreate(NULL, CFSTR(""), NULL, NULL); CFStringRef hostname_cfstr = SCDynamicStoreCopyLocalHostName(store); if (hostname_cfstr != NULL) { @@ -1297,7 +1316,7 @@ int getMacLocalHostNameBySCD(char *hostname, size_t maxLen) { } #endif -int taosGetlocalhostname(char *hostname, size_t maxLen) { +int32_t taosGetlocalhostname(char *hostname, size_t maxLen) { #ifdef _TD_DARWIN_64 int res = getMacLocalHostNameBySCD(hostname, maxLen); if (res != 0) { diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index a8a9ff681b..fe52369a53 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -31,7 +31,7 @@ void WINAPI windowsServiceCtrlHandle(DWORD request) { ServiceStatus.dwCurrentState = SERVICE_STOP_PENDING; if (!SetServiceStatus(hServiceStatusHandle, &ServiceStatus)) { DWORD nError = GetLastError(); - printf("failed to send stopped status to windows service: %d", nError); + fprintf(stderr, "failed to send stopped status to windows service: %d", nError); } break; default: @@ -50,19 +50,19 @@ void WINAPI mainWindowsService(int argc, char** argv) { hServiceStatusHandle = RegisterServiceCtrlHandler("taosd", &windowsServiceCtrlHandle); if (hServiceStatusHandle == 0) { DWORD nError = GetLastError(); - printf("failed to register windows service ctrl handler: %d", nError); + fprintf(stderr, "failed to register windows service ctrl handler: %d", nError); } ServiceStatus.dwCurrentState = SERVICE_RUNNING; if (SetServiceStatus(hServiceStatusHandle, &ServiceStatus)) { DWORD nError = GetLastError(); - printf("failed to send running status to windows service: %d", nError); + fprintf(stderr, "failed to send running status to windows service: %d", nError); } if (mainWindowsFunc != NULL) mainWindowsFunc(argc, argv); ServiceStatus.dwCurrentState = SERVICE_STOPPED; if (!SetServiceStatus(hServiceStatusHandle, &ServiceStatus)) { DWORD nError = GetLastError(); - printf("failed to send stopped status to windows service: %d", nError); + fprintf(stderr, "failed to send stopped status to windows service: %d", nError); } } void stratWindowsService(MainWindows mainWindows) { @@ -140,17 +140,27 @@ void taosCloseDll(void* handle) { } #endif -int taosSetConsoleEcho(bool on) { +int32_t taosSetConsoleEcho(bool on) { #if defined(WINDOWS) HANDLE hStdin = GetStdHandle(STD_INPUT_HANDLE); + if (hStdin == INVALID_HANDLE_VALUE) { + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); + return terrno; + } DWORD mode = 0; - GetConsoleMode(hStdin, &mode); + if(!GetConsoleMode(hStdin, &mode)){ + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); + return terrno; + } if (on) { mode |= ENABLE_ECHO_INPUT; } else { mode &= ~ENABLE_ECHO_INPUT; } - SetConsoleMode(hStdin, mode); + if(!SetConsoleMode(hStdin, mode)) { + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); + return terrno; + } return 0; #else diff --git a/source/os/src/osTimer.c b/source/os/src/osTimer.c index 6e5d9844a9..fe3a0dcf95 100644 --- a/source/os/src/osTimer.c +++ b/source/os/src/osTimer.c @@ -80,7 +80,7 @@ void taos_block_sigalrm(void) { static void taosDeleteTimer(void *tharg) { timer_t *pTimer = tharg; - (void)timer_delete(*pTimer); + TAOS_SKIP_ERROR(timer_delete(*pTimer)); } static TdThread timerThread; diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index 8fdd296655..89ced69f97 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -811,7 +811,11 @@ int32_t taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, i #elif defined(_TD_DARWIN_64) - setenv("TZ", buf, 1); + code = setenv("TZ", buf, 1); + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } tzset(); int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); *tsTimezone = tz; @@ -839,13 +843,17 @@ int32_t taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, i return code; } -void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { +int32_t taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { + int32_t code = 0; #ifdef WINDOWS char value[100]; char keyPath[100]; DWORD bufferSize = sizeof(value); - RegGetValue(HKEY_LOCAL_MACHINE, "SYSTEM\\CurrentControlSet\\Control\\TimeZoneInformation", "TimeZoneKeyName", + LONG result = RegGetValue(HKEY_LOCAL_MACHINE, "SYSTEM\\CurrentControlSet\\Control\\TimeZoneInformation", "TimeZoneKeyName", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); + if (result != ERROR_SUCCESS) { + return TAOS_SYSTEM_WINAPI_ERROR(result); + } strcpy(outTimezoneStr, "not configured"); *tsTimezone = 0; if (bufferSize > 0) { @@ -854,7 +862,10 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { strcpy(outTimezoneStr, win_tz[i][1]); bufferSize = sizeof(value); sprintf(keyPath, "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Time Zones\\%s", value); - RegGetValue(HKEY_LOCAL_MACHINE, keyPath, "Display", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); + result = RegGetValue(HKEY_LOCAL_MACHINE, keyPath, "Display", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); + if (result != ERROR_SUCCESS) { + return TAOS_SYSTEM_WINAPI_ERROR(result); + } if (bufferSize > 0) { // value[4] = (value[4] == '+' ? '-' : '+'); sprintf(outTimezoneStr, "%s (UTC, %c%c%c%c%c)", outTimezoneStr, value[4], value[5], value[6], value[8], @@ -865,39 +876,28 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { } } } + return 0; #elif defined(_TD_DARWIN_64) char buf[4096] = {0}; char *tz = NULL; { int n = readlink("/etc/localtime", buf, sizeof(buf)); if (n < 0) { - printf("read /etc/localtime error, reason:%s\n", strerror(errno)); - return; + return TSDB_CODE_TIME_ERROR; } buf[n] = '\0'; char *zi = strstr(buf, "zoneinfo"); if (!zi) { - printf("parsing /etc/localtime failed\n"); - return; + return TSDB_CODE_TIME_ERROR; } tz = zi + strlen("zoneinfo") + 1; - // for (int i = n - 1; i >= 0; --i) { - // if (buf[i] == '/') { - // if (tz) { - // tz = buf + i + 1; - // break; - // } - // tz = buf + i + 1; - // } - // } - // if (!tz || 0 == strchr(tz, '/')) { - // printf("parsing /etc/localtime failed\n"); - // return; - // } - - setenv("TZ", tz, 1); + code = setenv("TZ", tz, 1); + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } tzset(); } @@ -908,7 +908,9 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { */ time_t tx1 = taosGetTimestampSec(); struct tm tm1; - taosLocalTime(&tx1, &tm1, NULL); + if (taosLocalTime(&tx1, &tm1, NULL) == NULL) { + return TSDB_CODE_TIME_ERROR; + } daylight = tm1.tm_isdst; isdst_now = tm1.tm_isdst; @@ -920,6 +922,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { */ snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], -timezone / 3600); + return 0; #else char buf[4096] = {0}; @@ -927,8 +930,6 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { { int n = readlink("/etc/localtime", buf, sizeof(buf)-1); if (n < 0) { - (void)printf("read /etc/localtime error, reason:%s\n", strerror(errno)); - if (taosCheckExistFile("/etc/timezone")) { /* * NOTE: do not remove it. @@ -937,7 +938,9 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { */ time_t tx1 = taosGetTimestampSec(); struct tm tm1; - (void)taosLocalTime(&tx1, &tm1, NULL); + if(taosLocalTime(&tx1, &tm1, NULL) == NULL) { + return TSDB_CODE_TIME_ERROR; + } /* load time zone string from /etc/timezone */ // FILE *f = fopen("/etc/timezone", "r"); errno = 0; @@ -946,12 +949,11 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { if (pFile != NULL) { int len = taosReadFile(pFile, buf, 64); if (len < 0) { - (void)taosCloseFile(&pFile); - (void)printf("read /etc/timezone error, reason:%s\n", strerror(errno)); - return; + TAOS_UNUSED(taosCloseFile(&pFile)); + return TSDB_CODE_TIME_ERROR; } - (void)taosCloseFile(&pFile); + TAOS_UNUSED(taosCloseFile(&pFile)); buf[sizeof(buf) - 1] = 0; char *lineEnd = strstr(buf, "\n"); @@ -961,7 +963,11 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables if (strlen(buf) > 0) { - (void)setenv("TZ", buf, 1); + code = setenv("TZ", buf, 1); + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } } } // get and set default timezone @@ -986,34 +992,23 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { (void)snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); } else { - (void)printf("There is not /etc/timezone.\n"); + return TSDB_CODE_TIME_ERROR; } - return; + return 0; } buf[n] = '\0'; char *zi = strstr(buf, "zoneinfo"); if (!zi) { - (void)printf("parsing /etc/localtime failed\n"); - return; + return TSDB_CODE_TIME_ERROR; } tz = zi + strlen("zoneinfo") + 1; - // for (int i = n - 1; i >= 0; --i) { - // if (buf[i] == '/') { - // if (tz) { - // tz = buf + i + 1; - // break; - // } - // tz = buf + i + 1; - // } - // } - // if (!tz || 0 == strchr(tz, '/')) { - // printf("parsing /etc/localtime failed"); - // return; - // } - - (void)setenv("TZ", tz, 1); + code = setenv("TZ", tz, 1); + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } tzset(); } @@ -1024,7 +1019,9 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { */ time_t tx1 = taosGetTimestampSec(); struct tm tm1; - (void)taosLocalTime(&tx1, &tm1, NULL); + if(taosLocalTime(&tx1, &tm1, NULL) == NULL) { + return TSDB_CODE_TIME_ERROR; + } isdst_now = tm1.tm_isdst; /* @@ -1035,5 +1032,6 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { */ (void)snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], -timezone / 3600); + return 0; #endif } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 5ceec33831..df104508da 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -112,6 +112,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FAILED_TO_CONNECT_S3, "Failed to connect to TAOS_DEFINE_ERROR(TSDB_CODE_MSG_PREPROCESSED, "Message has been processed in preprocess") TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_BUFFER, "Out of buffer") TAOS_DEFINE_ERROR(TSDB_CODE_INTERNAL_ERROR, "Internal error") +TAOS_DEFINE_ERROR(TSDB_CODE_TIME_ERROR, "Internal error in time") //client TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION, "Invalid operation") @@ -278,6 +279,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_MNODES, "The replica of mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_ARBGROUP_ALREADY_EXIST, "Arbitrator group already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ARBGROUP_NOT_EXIST, "Arbitrator group not there") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ARB_TOKEN_MISMATCH, "Arbitrator token mismatch") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VNODE_NOT_OFFLINE, "Vnode is not offline on this restoring dnode") // mnode-dnode-part2 TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DNODES, "Too many dnodes") @@ -806,6 +808,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_SAME_COMMITTED_VALUE, "Same committed valu TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_REPLAY_NEED_ONE_VGROUP, "Replay need only one vgroup if subscribe super table") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT, "Replay is disabled if subscribe db or stable") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_NO_TABLE_QUALIFIED, "No table qualified for query") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_NO_NEED_REBALANCE, "No need rebalance") // stream TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_NOT_EXIST, "Stream task not exist") @@ -815,6 +818,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_IVLD_STATUS, "Invalid task status TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_CONFLICT_EVENT, "Stream conflict event") TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_INTERNAL_ERROR, "Stream internal error") TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_NOT_LEADER, "Stream task not on leader vnode") +TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_INPUTQ_FULL, "Task input queue is full") // TDLite TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS, "Invalid TDLite open flags") diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index bd22a4e42b..7d905e843d 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -215,7 +215,10 @@ int32_t taosInitSlowLog() { int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) { if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0; - TAOS_CHECK_RETURN(osUpdate()); + int32_t code = osUpdate(); + if (code != 0) { + uError("failed to update os info, reason:%s", tstrerror(code)); + } TAOS_CHECK_RETURN(taosInitNormalLog(logName, maxFiles)); if (tsc){ diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index f22233a757..ffb11b156d 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -316,7 +316,7 @@ static char* evictBufPage(SDiskbasedBuf* pBuf) { } terrno = 0; - (void)tdListPopNode(pBuf->lruList, pn); + pn = tdListPopNode(pBuf->lruList, pn); SPageInfo* d = *(SPageInfo**)pn->data; @@ -337,7 +337,7 @@ static int32_t lruListPushFront(SList* pList, SPageInfo* pi) { } static void lruListMoveToFront(SList* pList, SPageInfo* pi) { - (void)tdListPopNode(pList, pi->pn); + pi->pn = tdListPopNode(pList, pi->pn); tdListPrependNode(pList, pi->pn); } @@ -474,8 +474,11 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId) { pBuf->totalBufSize += pBuf->pageSize; } else { taosMemoryFree(availablePage); - (void)taosArrayPop(pBuf->pIdList); - (void)tSimpleHashRemove(pBuf->all, pageId, sizeof(int32_t)); + SPageInfo **pLast = taosArrayPop(pBuf->pIdList); + int32_t ret = tSimpleHashRemove(pBuf->all, pageId, sizeof(int32_t)); + if (ret != TSDB_CODE_SUCCESS) { + uError("%s failed to clear pageId %d from buf hash-set since %s", __func__, *pageId, tstrerror(ret)); + } taosMemoryFree(pi); terrno = code; return NULL; @@ -544,6 +547,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { int32_t code = lruListPushFront(pBuf->lruList, *pi); if (TSDB_CODE_SUCCESS != code) { taosMemoryFree((*pi)->pData); + (*pi)->pData = NULL; terrno = code; return NULL; } @@ -554,7 +558,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { int32_t code = loadPageFromDisk(pBuf, *pi); if (code != 0) { taosMemoryFree((*pi)->pData); - + (*pi)->pData = NULL; terrno = code; return NULL; } diff --git a/source/util/src/tref.c b/source/util/src/tref.c index a55578d6c2..685fe01fee 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -56,7 +56,7 @@ static void taosLockList(int64_t *lockedBy); static void taosUnlockList(int64_t *lockedBy); static void taosIncRsetCount(SRefSet *pSet); static void taosDecRsetCount(SRefSet *pSet); -static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove); +static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove, int32_t* isReleased); int32_t taosOpenRef(int32_t max, RefFp fp) { SRefNode **nodeList; @@ -182,7 +182,7 @@ int64_t taosAddRef(int32_t rsetId, void *p) { return rid; } -int32_t taosRemoveRef(int32_t rsetId, int64_t rid) { return taosDecRefCount(rsetId, rid, 1); } +int32_t taosRemoveRef(int32_t rsetId, int64_t rid) { return taosDecRefCount(rsetId, rid, 1, NULL); } // if rid is 0, return the first p in hash list, otherwise, return the next after current rid void *taosAcquireRef(int32_t rsetId, int64_t rid) { @@ -251,7 +251,8 @@ void *taosAcquireRef(int32_t rsetId, int64_t rid) { return p; } -int32_t taosReleaseRef(int32_t rsetId, int64_t rid) { return taosDecRefCount(rsetId, rid, 0); } +int32_t taosReleaseRef(int32_t rsetId, int64_t rid) { return taosDecRefCount(rsetId, rid, 0, NULL); } +int32_t taosReleaseRefEx(int32_t rsetId, int64_t rid, int32_t* isReleased) { return taosDecRefCount(rsetId, rid, 0, isReleased); } // if rid is 0, return the first p in hash list, otherwise, return the next after current rid void *taosIterateRef(int32_t rsetId, int64_t rid) { @@ -384,7 +385,7 @@ int32_t taosListRef() { return num; } -static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove) { +static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove, int32_t* isReleased) { int32_t hash; SRefSet *pSet; SRefNode *pNode; @@ -458,6 +459,10 @@ static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove) { taosDecRsetCount(pSet); } + if (isReleased) { + *isReleased = released; + } + return code; } diff --git a/source/util/src/tunit.c b/source/util/src/tunit.c index 57226878dd..e73045cc89 100644 --- a/source/util/src/tunit.c +++ b/source/util/src/tunit.c @@ -23,14 +23,16 @@ #define UNIT_ONE_PEBIBYTE (UNIT_ONE_TEBIBYTE * UNIT_SIZE_CONVERT_FACTOR) #define UNIT_ONE_EXBIBYTE (UNIT_ONE_PEBIBYTE * UNIT_SIZE_CONVERT_FACTOR) -static int32_t parseCfgIntWithUnit(const char* str, double* res) { - double val, temp = (double)INT64_MAX; +static int32_t parseCfgIntWithUnit(const char* str, int64_t* res) { + double val = 0, temp = (double)INT64_MAX; char* endPtr; + bool useDouble = false; errno = 0; - val = taosStr2Int64(str, &endPtr, 0); + int64_t int64Val = taosStr2Int64(str, &endPtr, 0); if (*endPtr == '.' || errno == ERANGE) { errno = 0; val = taosStr2Double(str, &endPtr); + useDouble = true; } if (endPtr == str || errno == ERANGE || isnan(val)) { return terrno = TSDB_CODE_INVALID_CFG_VALUE; @@ -67,23 +69,33 @@ static int32_t parseCfgIntWithUnit(const char* str, double* res) { default: return terrno = TSDB_CODE_INVALID_CFG_VALUE; } + endPtr++; + if ((val > 0 && val > temp) || (val < 0 && val < -temp)) { return terrno = TSDB_CODE_OUT_OF_RANGE; } - endPtr++; val *= factor; + int64Val *= factor; } while (isspace((unsigned char)*endPtr)) endPtr++; if (*endPtr) { return terrno = TSDB_CODE_INVALID_CFG_VALUE; } - val = rint(val); - *res = val; + if (useDouble) { + val = rint(val); + if ((val > 0 && val >= (double)INT64_MAX) || (val < 0 && val <= (double)INT64_MIN)) { + return terrno = TSDB_CODE_OUT_OF_RANGE; + } else { + *res = (int64_t)val; + } + } else { + *res = int64Val; + } return TSDB_CODE_SUCCESS; } int32_t taosStrHumanToInt64(const char* str, int64_t* out) { - double res; + int64_t res; int32_t code = parseCfgIntWithUnit(str, &res); if (code == TSDB_CODE_SUCCESS) *out = (int64_t)res; return code; @@ -109,7 +121,7 @@ void taosInt64ToHumanStr(int64_t val, char* outStr) { #endif int32_t taosStrHumanToInt32(const char* str, int32_t* out) { - double res; + int64_t res; int32_t code = parseCfgIntWithUnit(str, &res); if (code == TSDB_CODE_SUCCESS) { if (res < INT32_MIN || res > INT32_MAX) { diff --git a/tests/army/query/accuracy/test_ts5400.py b/tests/army/query/accuracy/test_ts5400.py new file mode 100644 index 0000000000..5263df8014 --- /dev/null +++ b/tests/army/query/accuracy/test_ts5400.py @@ -0,0 +1,51 @@ +import taos +import socket +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.eos import * +from frame.server.dnodes import * + +class TDTestCase(TBase): + """Add test case to cover TS-5400 + """ + updatecfgDict = { + "timezone": "UTC" + } + + def init(self, conn, logSql, replicaVar=1): + host = socket.gethostname() + con = taos.connect(host=f"{host}", config=tdDnodes.getSimCfgPath(), timezone='UTC') + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(con.cursor()) + + def prepare_data(self): + # prepare data for TS-5400 + tdSql.execute("create database db_ts5400 BUFFER 512 CACHESIZE 1024 CACHEMODEL 'both' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 KEEP 365000d;") + tdSql.execute("use db_ts5400;") + #tdSql.execute("create stable st(ts TIMESTAMP ENCODE 'delta-i' COMPRESS 'lz4' LEVEL 'medium', `uk` VARCHAR(64) ENCODE 'disabled' COMPRESS 'lz4' LEVEL 'medium' PRIMARY KEY ) tags(ta int,tb int,tc int);") + tdSql.execute("create stable st(ts TIMESTAMP, `uk` VARCHAR(64)) tags(ta int,tb int,tc int);") + tdSql.execute("create table t1 using st tags(1,1,1);") + tdSql.execute("insert into t1 values ('1970-01-29 05:04:53.000','22:: ');") + + def test_ts5400(self): + self.prepare_data() + tdSql.execute("use db_ts5400;") + tdSql.query("select _wstart, count(*) from st interval(1y);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '1970-01-01 00:00:00.000') + tdSql.checkData(0, 1, 1) + + def run(self): + self.test_ts5400() + + def stop(self): + tdSql.execute("drop database db_ts5400;") + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/army/query/function/ans/rand.csv b/tests/army/query/function/ans/rand.csv new file mode 100644 index 0000000000..40e20c5ba4 --- /dev/null +++ b/tests/army/query/function/ans/rand.csv @@ -0,0 +1,21 @@ +0.663936012733698 +0.840187717154710 +0.840187717154710 +0.700976369297587 +0.561380175203728 +0.916457875592847 +0.274745596235034 +0.135438768721856 +0.486904139391568 +0.352760728612896 +0.206965447965528 +0.419929514834624 +0.419929514834624 +0.419929514834624 +0.419929514834624 +0.419929514834624 +0.419929514834624 +0.419929514834624 +0.419929514834624 +0.419929514834624 +0.419929514834624 \ No newline at end of file diff --git a/tests/army/query/function/in/rand.in b/tests/army/query/function/in/rand.in new file mode 100644 index 0000000000..185a76d6f9 --- /dev/null +++ b/tests/army/query/function/in/rand.in @@ -0,0 +1,3 @@ +select RAND(1245); +select RAND(id) from ts_4893.d0 limit 10; +select RAND(id) from ts_4893.d0 order by id desc limit 10; \ No newline at end of file diff --git a/tests/army/query/function/test_function.py b/tests/army/query/function/test_function.py index 4981e93563..18a0d46711 100644 --- a/tests/army/query/function/test_function.py +++ b/tests/army/query/function/test_function.py @@ -509,31 +509,98 @@ class TDTestCase(TBase): tdSql.error( "select * from (select to_iso8601(ts, timezone()), timezone() from meters order by ts desc) limit 1000;", expectErrInfo="Not supported timzone format") # TS-5340 + + def test_rand(self): + self.test_normal_query("rand") + + tdSql.query("select rand();") + tdSql.checkRows(1) + tdSql.checkCols(1) + self.check_result_in_range(0, 0) + + tdSql.query("select rand(null);") + tdSql.checkRows(1) + tdSql.checkCols(1) + self.check_result_in_range(0, 0) + + tdSql.query("select rand() from (select 1) t limit 1;") + tdSql.checkRows(1) + tdSql.checkCols(1) + self.check_result_in_range(0, 0) + + tdSql.query("select rand(id) from ts_4893.d0 limit 100;") + tdSql.checkRows(100) + tdSql.checkCols(1) + for i in range(len(tdSql.res)): + self.check_result_in_range(i, 0) + + tdSql.query("select rand(id) from ts_4893.meters limit 100;") + tdSql.checkRows(100) + tdSql.checkCols(1) + for i in range(len(tdSql.res)): + self.check_result_in_range(i, 0) + + tdSql.query("select rand(123), rand(123);") + tdSql.checkRows(1) + tdSql.checkCols(2) + if tdSql.res[0][0] != tdSql.res[0][1]: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, tdSql.sql, tdSql.res[0][0], tdSql.res[0][1]) + tdLog.exit("%s(%d) failed: sql:%s data1:%s ne data2:%s" % args) + + def check_result_in_range(self, row, col): + res = tdSql.res[row][col] + if res < 0 or res >= 1: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, tdSql.sql, row, col, res) + tdLog.exit("%s(%d) failed: sql:%s row:%s col:%s data:%s lt 0 or ge 1" % args) + + def test_max(self): + self.test_normal_query("max") + + tdSql.query("select max(null) from ts_4893.meters;") + tdSql.checkRows(1) + tdSql.checkCols(1) + tdSql.checkData(0, 0, 'None') + + tdSql.query("select max(id) from ts_4893.meters;") + tdSql.checkRows(1) + + tdSql.query("select max(name) from ts_4893.meters;") + tdSql.checkRows(1) + + tdSql.query("select max(current) from ts_4893.meters;") + tdSql.checkRows(1) + + tdSql.query("select max(nch1) from ts_4893.meters;") + tdSql.checkRows(1) + + tdSql.query("select max(var1) from ts_4893.meters;") + tdSql.checkRows(1) + def test_min(self): self.test_normal_query("min") - tdSql.query("select min(var1), min(id) from ts_4893.d0;") + tdSql.query("select min(null) from ts_4893.meters;") tdSql.checkRows(1) - tdSql.checkData(0, 0, 'abc一二三abc一二三abc') - tdSql.checkData(0, 1, 0) - def test_max(self): - self.test_normal_query("max") - tdSql.query("select max(var1), max(id) from ts_4893.d0;") - tdSql.checkRows(1) - tdSql.checkData(0, 0, '一二三四五六七八九十') - tdSql.checkData(0, 1, 9999) - def test_rand(self): - tdSql.query("select rand();") + tdSql.checkCols(1) + tdSql.checkData(0, 0, 'None') + + tdSql.query("select min(id) from ts_4893.meters;") tdSql.checkRows(1) - tdSql.query("select rand(1);") + tdSql.query("select min(name) from ts_4893.meters;") tdSql.checkRows(1) - tdSql.query("select rand(1) from ts_4893.meters limit 10;") - tdSql.checkRows(10) + tdSql.query("select min(current) from ts_4893.meters;") + tdSql.checkRows(1) + + tdSql.query("select min(nch1) from ts_4893.meters;") + tdSql.checkRows(1) + + tdSql.query("select min(var1) from ts_4893.meters;") + tdSql.checkRows(1) - tdSql.query("select rand(id) from ts_4893.d0 limit 10;") - tdSql.checkRows(10) # run def run(self): tdLog.debug(f"start to excute {__file__}") @@ -576,8 +643,8 @@ class TDTestCase(TBase): self.test_varpop() # select function - self.test_min() self.test_max() + self.test_min() # error function self.test_error() diff --git a/tests/army/query/sys/tb_perf_queries_exist_test.py b/tests/army/query/sys/tb_perf_queries_exist_test.py new file mode 100644 index 0000000000..e6afc0bec6 --- /dev/null +++ b/tests/army/query/sys/tb_perf_queries_exist_test.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.autogen import * + +''' + TS-5349: https://jira.taosdata.com:18080/browse/TS-5349 + 查询 performance_schema.perf_queries 后, 再查询 information_schema.perf_queries, + 正常情况下在 information_schema 中不存在表 perf_queries +''' + +class TDTestCase(TBase): + + def run(self): + tdSql.query("select * from performance_schema.perf_queries;") + tdLog.info("Table [perf_queries] exist in schema [performance_schema]") + + tdSql.error("select * from information_schema.perf_queries;") + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ed7a09f648..87bac4cce3 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -27,6 +27,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f cluster/clusterBasic.py -N 5 ,,y,army,./pytest.sh python3 ./test.py -f query/query_basic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/accuracy/test_query_accuracy.py +,,y,army,./pytest.sh python3 ./test.py -f query/accuracy/test_ts5400.py ,,y,army,./pytest.sh python3 ./test.py -f insert/insert_basic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f cluster/splitVgroupByLearner.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f authorith/authBasic.py -N 3 @@ -42,6 +43,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f query/fill/fill_compare_asc_desc.py ,,y,army,./pytest.sh python3 ./test.py -f query/last/test_last.py ,,y,army,./pytest.sh python3 ./test.py -f query/window/base.py +,,y,army,./pytest.sh python3 ./test.py -f query/sys/tb_perf_queries_exist_test.py -N 3 # # system test diff --git a/tests/system-test/0-others/show.py b/tests/system-test/0-others/show.py index 64696c5e6d..032e5eebe1 100644 --- a/tests/system-test/0-others/show.py +++ b/tests/system-test/0-others/show.py @@ -284,6 +284,20 @@ class TDTestCase: for error_val in error_vals: tdSql.error(f'ALTER DNODE 1 "{var}" "{error_val}"') + var = 'randErrorDivisor' + vals = ['9223372036854775807', '9223372036854775807.1', '9223372036854775806', '9223372036854775808', '9223372036854775808.1', '9223372036854775807.0', '9223372036854775806.1'] + expected_vals = ['9223372036854775807', 'err', '9223372036854775806', 'err', 'err', 'err', 'err'] + for val_str, expected_val in zip(vals, expected_vals): + sql = f'ALTER dnode 1 "{var}" "{val_str}"' + if expected_val == 'err': + tdSql.error(sql) + else: + tdSql.execute(sql, queryTimes=1) + actual_val = self.get_variable(var, False) + if expected_val != actual_val: + tdLog.exit(f"failed to set local {var} to {expected_val} actually {actual_val}") + + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__)