diff --git a/.gitignore b/.gitignore index 8f461f2b02..4e47039628 100644 --- a/.gitignore +++ b/.gitignore @@ -162,3 +162,4 @@ geos_c.h source/libs/parser/src/sql.c include/common/ttokenauto.h !packaging/smokeTest/pytest_require.txt +tdengine-test-dir diff --git a/docs/en/07-develop/01-connect.md b/docs/en/07-develop/01-connect.md index 97e96f98a4..66bd9b6b16 100644 --- a/docs/en/07-develop/01-connect.md +++ b/docs/en/07-develop/01-connect.md @@ -164,9 +164,6 @@ If you are using Maven to manage your project, simply add the following dependen pip3 install taospy[ws] ``` - - - - **Installation Verification** @@ -199,8 +196,8 @@ import taosws + - Edit `go.mod` to add the `driver-go` dependency. diff --git a/docs/en/14-reference/03-taos-sql/03-table.md b/docs/en/14-reference/03-taos-sql/03-table.md index 030d88cf63..3c047eb19e 100644 --- a/docs/en/14-reference/03-taos-sql/03-table.md +++ b/docs/en/14-reference/03-taos-sql/03-table.md @@ -194,7 +194,7 @@ alter_table_option: { ### Modify Subtable Tag Value ```sql -ALTER TABLE tb_name SET tag tag_name=new_tag_value; +ALTER TABLE tb_name SET TAG tag_name1=new_tag_value1,tag_name2=new_tag_value2...; ``` ### Modify Table Lifespan diff --git a/docs/en/14-reference/03-taos-sql/32-compress.md b/docs/en/14-reference/03-taos-sql/32-compress.md index efe1959b1a..8bb2ae31e3 100644 --- a/docs/en/14-reference/03-taos-sql/32-compress.md +++ b/docs/en/14-reference/03-taos-sql/32-compress.md @@ -28,13 +28,14 @@ In this document, it specifically refers to the internal levels of the second-le - Default compression algorithms list and applicable range for each data type -| Data Type | Available Encoding Algorithms | Default Encoding Algorithm | Available Compression Algorithms|Default Compression Algorithm| Default Compression Level| -| :-----------:|:----------:|:-------:|:-------:|:----------:|:----:| -| tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium| -| bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium| -|float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|lz4| medium| -|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| medium| -|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| medium| +| Data Type |Available Encoding Algorithms | Default Encoding Algorithm | Available Compression Algorithms | Default Compression Algorithm | Default Compression Level | +|:------------------------------------:|:----------------:|:-----------:|:--------------------:|:----:|:------:| +| int/uint | simple8b | simple8b | lz4/zlib/zstd/xz | lz4 | medium | +| tinyint/untinyint/smallint/usmallint | simple8b | simple8b | lz4/zlib/zstd/xz | zlib | medium | +| bigint/ubigint/timestamp | simple8b/delta-i | delta-i | lz4/zlib/zstd/xz | lz4 | medium | +| float/double | delta-d | delta-d | lz4/zlib/zstd/xz/tsz | lz4 | medium | +| binary/nchar | disabled | disabled | lz4/zlib/zstd/xz | zstd | medium | +| bool | bit-packing | bit-packing | lz4/zlib/zstd/xz | zstd | medium | ## SQL Syntax diff --git a/docs/zh/07-develop/01-connect/index.md b/docs/zh/07-develop/01-connect/index.md index 94f55967ec..2381c49d93 100644 --- a/docs/zh/07-develop/01-connect/index.md +++ b/docs/zh/07-develop/01-connect/index.md @@ -37,7 +37,7 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 关键不同点在于： -1. 使用 原生连接，需要保证客户端的驱动程序 taosc 和服务端的 TDengine 版本配套。 +1. 使用 原生连接，需要保证客户端的驱动程序 taosc 和服务端的 TDengine 版本保持一致。 2. 使用 REST 连接，用户无需安装客户端驱动程序 taosc，具有跨平台易用的优势，但是无法体验数据订阅和二进制数据类型等功能。另外与 原生连接 和 WebSocket 连接相比，REST连接的性能最低。REST 接口是无状态的。在使用 REST 连接时，需要在 SQL 中指定表、超级表的数据库名称。 3. 使用 WebSocket 连接，用户也无需安装客户端驱动程序 taosc。 4. 连接云服务实例，必须使用 REST 连接 或 WebSocket 连接。 diff --git a/docs/zh/14-reference/01-components/12-tdinsight/assets/Alert-Used-Only.webp b/docs/zh/14-reference/01-components/12-tdinsight/assets/Alert-Used-Only.webp new file mode 100644 index 0000000000..8824ee5889 Binary files /dev/null and b/docs/zh/14-reference/01-components/12-tdinsight/assets/Alert-Used-Only.webp differ diff --git a/docs/zh/14-reference/01-components/12-tdinsight/assets/Alert-rules.webp b/docs/zh/14-reference/01-components/12-tdinsight/assets/Alert-rules.webp new file mode 100644 index 0000000000..e9f91c0488 Binary files /dev/null and b/docs/zh/14-reference/01-components/12-tdinsight/assets/Alert-rules.webp differ diff --git a/docs/zh/14-reference/01-components/12-tdinsight/assets/TDengine-Alert.webp b/docs/zh/14-reference/01-components/12-tdinsight/assets/TDengine-Alert.webp new file mode 100644 index 0000000000..a0d9b695f4 Binary files /dev/null and b/docs/zh/14-reference/01-components/12-tdinsight/assets/TDengine-Alert.webp differ diff --git a/docs/zh/14-reference/01-components/12-tdinsight/index.mdx b/docs/zh/14-reference/01-components/12-tdinsight/index.mdx index e5f44a1080..0f6ea46ffd 100644 --- a/docs/zh/14-reference/01-components/12-tdinsight/index.mdx +++ b/docs/zh/14-reference/01-components/12-tdinsight/index.mdx @@ -145,6 +145,44 @@ TDinsight 仪表盘旨在提供 TDengine 相关资源的使用情况和状态， 还有上述分类的细分维度折线图。 +### 预配置告警规则自动导入 + +涛思总结用户使用经验，整理出 14 个常用的告警规则（alert rule），能够对集群关键指标进行监测并及时上报指标异常、超限等告警信息。 +从 TDengine-server 3.3.4.3 版本（tdengine-datasource 3.6.3）开始，TDengine Datasource 支持预配置告警规则自动导入功能，用户可将 14 个告警规则一键导入 Grafana（11 及以上版本），直接使用。 +预配置告警规则导入方法如下图所示，在 tdengine-datasource setting 界面，打开 “Load Tengine Alert” 开关，点击 “Save & test” 按钮后，插件会自动加载上述告警规则, 规则会放入以数据源名称 + “-alert” 的 grafana 告警目录中。如不需要，关闭 “Load TDengine Alert” 开关，点击 “Clear TDengine Alert” 旁边的按钮则会清除此数据源已导入的所有告警规则。 + +![TDengine Alert](./assets/TDengine-Alert.webp) + +导入后，点击 Grafana 左侧 “Alert rules” ，可查看当前所有告警规则。 +用户只需配置联络点（Contact points），即可获取告警通知。联络点配置方法见[告警配置](https://docs.taosdata.com/third-party/visual/grafana/#%E5%91%8A%E8%AD%A6%E9%85%8D%E7%BD%AE)。 + +![Alert-rules](./assets/Alert-rules.webp) + +14 个告警规则具体配置如下： + +| 规则名称| 规则阈值| 无监控数据时的行为 | 数据扫描间隔 |持续时间 | 执行SQL | +| ------ | --------- | ---------------- | ----------- |------- |----------------------| +|dnode 节点的CPU负载|均值 > 80%|触发告警|5分钟|5分钟 |`select now(), dnode_id, last(cpu_system) as cup_use from log.taosd_dnodes_info where _ts >= (now- 5m) and _ts < now partition by dnode_id having first(_ts) > 0 `| +|dnode 节点的的内存 |均值 > 60%|触发告警|5分钟|5分钟|`select now(), dnode_id, last(mem_engine) / last(mem_total) * 100 as taosd from log.taosd_dnodes_info where _ts >= (now- 5m) and _ts 80%|触发告警|5分钟|5分钟|`select now(), dnode_id, data_dir_level, data_dir_name, last(used) / last(total) * 100 as used from log.taosd_dnodes_data_dirs where _ts >= (now - 5m) and _ts < now partition by dnode_id, data_dir_level, data_dir_name`| +|集群授权到期 |< 60天|触发告警|1天|0秒|`select now(), cluster_id, last(grants_expire_time) / 86400 as expire_time from log.taosd_cluster_info where _ts >= (now - 24h) and _ts < now partition by cluster_id having first(_ts) > 0 `| +|测点数达到授权测点数|>= 90%|触发告警|1天|0秒|`select now(), cluster_id, CASE WHEN max(grants_timeseries_total) > 0.0 THEN max(grants_timeseries_used) /max(grants_timeseries_total) * 100.0 ELSE 0.0 END AS result from log.taosd_cluster_info where _ts >= (now - 30s) and _ts < now partition by cluster_id having timetruncate(first(_ts), 1m) > 0`| +|查询并发请求数 | > 100|不触发报警|1分钟|0秒|`select now() as ts, count(*) as slow_count from performance_schema.perf_queries`| +|慢查询执行最长时间 （无时间窗口） |> 300秒|不触发报警|1分钟|0秒|`select now() as ts, count(*) as slow_count from performance_schema.perf_queries where exec_usec>300000000`| +|dnode下线 |total != alive|触发告警|30秒|0秒|`select now(), cluster_id, last(dnodes_total) - last(dnodes_alive) as dnode_offline from log.taosd_cluster_info where _ts >= (now -30s) and _ts < now partition by cluster_id having first(_ts) > 0`| +|vnode下线 |total != alive|触发告警|30秒|0秒|`select now(), cluster_id, last(vnodes_total) - last(vnodes_alive) as vnode_offline from log.taosd_cluster_info where _ts >= (now - 30s) and _ts < now partition by cluster_id having first(_ts) > 0 `| +|数据删除请求数 |> 0|不触发报警|30秒|0秒|``select now(), count(`count`) as `delete_count` from log.taos_sql_req where sql_type = 'delete' and _ts >= (now -30s) and _ts < now``| +|Adapter RESTful 请求失败 |> 5|不触发报警|30秒|0秒|``select now(), sum(`fail`) as `Failed` from log.adapter_requests where req_type=0 and ts >= (now -30s) and ts < now``| +|Adapter WebSocket 请求失败 |> 5|不触发报警|30秒|0秒|``select now(), sum(`fail`) as `Failed` from log.adapter_requests where req_type=1 and ts >= (now -30s) and ts < now``| +|dnode 数据上报缺少 |< 3|触发告警|180秒|0秒|`select now(), cluster_id, count(*) as dnode_report from log.taosd_cluster_info where _ts >= (now -180s) and _ts < now partition by cluster_id having timetruncate(first(_ts), 1h) > 0`| +|dnode 重启 |max(update_time) > last(update_time)|触发告警|90秒|0秒|`select now(), dnode_id, max(uptime) - last(uptime) as dnode_restart from log.taosd_dnodes_info where _ts >= (now - 90s) and _ts < now partition by dnode_id`| + +用户可参考上述告警规则，根据自己业务需求进行修改与完善。 +Grafana7.5 及以下版本，Dashboards 与 Alert rules 功能合在一起，而之后的新版本两个功能是分开的。为兼容 Grafana7.5 及以下版本，TDinsight 面板中增加了 Alert Used Only 面板，仅 Grafana7.5 及以下版本需要使用。 + +![Alert Used Only](./assets/Alert-Used-Only.webp) + + ## 升级 下面三种方式都可以进行升级： - 用图形界面，若有新版本，可以在 ”TDengine Datasource“ 插件页面点击 update 升级。 @@ -155,10 +193,11 @@ TDinsight 仪表盘旨在提供 TDengine 相关资源的使用情况和状态， 针对不同的安装方式，卸载时： - 用图形界面，在 ”TDengine Datasource“ 插件页面点击 ”Uninstall“ 卸载。 - 通过 `TDinsight.sh` 脚本安装的 TDinsight，可以使用命令行 `TDinsight.sh -R` 清理相关资源。 -- 手动安装的 TDinsight，要完全卸载，需要清理以下内容： +- 手动安装的 TDinsight，要完全卸载，需要按照顺序清理以下内容： 1. Grafana 中的 TDinsight Dashboard。 - 2. Grafana 中的 Data Source 数据源。 - 3. 从插件安装目录删除 `tdengine-datasource` 插件。 + 2. Grafana 中的 Alert rules 告警规则。 + 3. Grafana 中的 Data Source 数据源。 + 4. 从插件安装目录删除 `tdengine-datasource` 插件。 ## 附录 diff --git a/docs/zh/14-reference/03-taos-sql/03-table.md b/docs/zh/14-reference/03-taos-sql/03-table.md index 4bf1bd5b87..a9f9ddb517 100644 --- a/docs/zh/14-reference/03-taos-sql/03-table.md +++ b/docs/zh/14-reference/03-taos-sql/03-table.md @@ -195,7 +195,7 @@ alter_table_option: { ### 修改子表标签值 ``` -ALTER TABLE tb_name SET TAG tag_name=new_tag_value; +ALTER TABLE tb_name SET TAG tag_name1=new_tag_value1,tag_name2=new_tag_value2...; ``` ### 修改表生命周期 diff --git a/docs/zh/14-reference/03-taos-sql/32-compress.md b/docs/zh/14-reference/03-taos-sql/32-compress.md index 51650c9123..2ef6e9e06a 100644 --- a/docs/zh/14-reference/03-taos-sql/32-compress.md +++ b/docs/zh/14-reference/03-taos-sql/32-compress.md @@ -29,14 +29,15 @@ description: 可配置压缩算法 - 各个数据类型的默认压缩算法列表和适用范围 -| 数据类型 | 可选编码算法 | 编码算法默认值 | 可选压缩算法|压缩算法默认值| 压缩等级默认值| -| :-----------:|:----------:|:-------:|:-------:|:----------:|:----:| -| int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium| -| tinyint/untinyint/smallint/usmallint | simple8b| simple8b | lz4/zlib/zstd/xz| zlib | medium| -| bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium| -|float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|lz4| medium| -|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| zstd| medium| -|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| zstd| medium| +| 数据类型 | 可选编码算法 | 编码算法默认值 | 可选压缩算法 | 压缩算法默认值 |压缩等级默认值| +|:------------------------------------:|:----------------:|:-----------:|:--------------------:|:----:|:------:| +| int/uint | simple8b | simple8b | lz4/zlib/zstd/xz | lz4 | medium | +| tinyint/untinyint/smallint/usmallint | simple8b | simple8b | lz4/zlib/zstd/xz | zlib | medium | +| bigint/ubigint/timestamp | simple8b/delta-i | delta-i | lz4/zlib/zstd/xz | lz4 | medium | +| float/double | delta-d | delta-d | lz4/zlib/zstd/xz/tsz | lz4 | medium | +| binary/nchar | disabled | disabled | lz4/zlib/zstd/xz | zstd | medium | +| bool | bit-packing | bit-packing | lz4/zlib/zstd/xz | zstd | medium | + ## SQL 语法 diff --git a/include/os/osAtomic.h b/include/os/osAtomic.h index 97bb6f53ba..d595c687b7 100644 --- a/include/os/osAtomic.h +++ b/include/os/osAtomic.h @@ -48,13 +48,13 @@ void atomic_store_8(int8_t volatile *ptr, int8_t val); void atomic_store_16(int16_t volatile *ptr, int16_t val); void atomic_store_32(int32_t volatile *ptr, int32_t val); void atomic_store_64(int64_t volatile *ptr, int64_t val); -double atomic_store_double(double volatile *ptr, double val); +void atomic_store_double(double volatile *ptr, double val); void atomic_store_ptr(void *ptr, void *val); int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val); int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val); int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val); int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val); -double atomic_exchange_double(double volatile *ptr, int64_t val); +double atomic_exchange_double(double volatile *ptr, double val); void *atomic_exchange_ptr(void *ptr, void *val); int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval); int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval); @@ -71,7 +71,7 @@ int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val); double atomic_fetch_add_double(double volatile *ptr, double val); -void *atomic_fetch_add_ptr(void *ptr, void *val); +void *atomic_fetch_add_ptr(void *ptr, int64_t val); int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val); @@ -82,37 +82,37 @@ int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val); double atomic_fetch_sub_double(double volatile *ptr, double val); -void *atomic_fetch_sub_ptr(void *ptr, void *val); +void *atomic_fetch_sub_ptr(void *ptr, int64_t val); int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val); int64_t atomic_and_fetch_64(int64_t volatile *ptr, int64_t val); -void *atomic_and_fetch_ptr(void *ptr, void *val); +void *atomic_and_fetch_ptr(void *ptr, int64_t val); int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_and_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_and_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_and_64(int64_t volatile *ptr, int64_t val); -void *atomic_fetch_and_ptr(void *ptr, void *val); +void *atomic_fetch_and_ptr(void *ptr, int64_t val); int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_or_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_or_fetch_32(int32_t volatile *ptr, int32_t val); int64_t atomic_or_fetch_64(int64_t volatile *ptr, int64_t val); -void *atomic_or_fetch_ptr(void *ptr, void *val); +void *atomic_or_fetch_ptr(void *ptr, int64_t val); int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_or_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_or_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_or_64(int64_t volatile *ptr, int64_t val); -void *atomic_fetch_or_ptr(void *ptr, void *val); +void *atomic_fetch_or_ptr(void *ptr, int64_t val); int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_xor_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_xor_fetch_32(int32_t volatile *ptr, int32_t val); int64_t atomic_xor_fetch_64(int64_t volatile *ptr, int64_t val); -void *atomic_xor_fetch_ptr(void *ptr, void *val); +void *atomic_xor_fetch_ptr(void *ptr, int64_t val); int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_xor_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val); -void *atomic_fetch_xor_ptr(void *ptr, void *val); +void *atomic_fetch_xor_ptr(void *ptr, int64_t val); #ifdef _MSC_VER #define tmemory_barrier(order) MemoryBarrier() diff --git a/packaging/delete_ref_lock.py b/packaging/delete_ref_lock.py index cf0e4cdd05..7200246a8a 100644 --- a/packaging/delete_ref_lock.py +++ b/packaging/delete_ref_lock.py @@ -1,59 +1,80 @@ import subprocess import re -# 执行 git fetch 命令并捕获输出 def git_fetch(): result = subprocess.run(['git', 'fetch'], capture_output=True, text=True) return result -# 解析分支名称 +def git_prune(): + # git remote prune origin + print("git remote prune origin") + result = subprocess.run(['git', 'remote', 'prune', 'origin'], capture_output=True, text=True) + return result + def parse_branch_name_type1(error_output): - # 使用正则表达式匹配 'is at' 前的分支名称 + # error: cannot lock ref 'refs/remotes/origin/fix/3.0/TD-32817': is at 7af5 but expected eaba + # match the branch name before ‘is at’ with a regular expression match = re.search(r"error: cannot lock ref '(refs/remotes/origin/[^']+)': is at", error_output) if match: return match.group(1) return None -# 解析第二种错误中的分支名称 def parse_branch_name_type2(error_output): - # 使用正则表达式匹配 'exists' 前的第一个引号内的分支名称 + # match the branch name before ‘exists; cannot create’ with a regular expression match = re.search(r"'(refs/remotes/origin/[^']+)' exists;", error_output) if match: return match.group(1) return None -# 执行 git update-ref -d 命令 +# parse branch name from error output of git remote prune origin +def parse_branch_name_type3(error_output): + # match the branch name before the first single quote before 'Unable to' with a regular expression + # git error: could not delete references: cannot lock ref 'refs/remotes/origin/test/3.0/TS-4893': Unable to create 'D:/workspace/main/TDinternal/community/.git/refs/remotes/origin/test/3.0/TS-4893.lock': File exists + match = re.search(r"references: cannot lock ref '(refs/remotes/origin/[^']+)': Unable to", error_output) + if match: + return match.group(1) + return None + + +# execute git update-ref -d to delete the ref def git_update_ref(branch_name): if branch_name: subprocess.run(['git', 'update-ref', '-d', f'{branch_name}'], check=True) -# 解析错误类型并执行相应的修复操作 +# parse error type and execute corresponding repair operation def handle_error(error_output): - # 错误类型1：本地引用的提交ID与远程不一致 - if "is at" in error_output and "but expected" in error_output: - branch_name = parse_branch_name_type1(error_output) - if branch_name: - print(f"Detected error type 1, attempting to delete ref for branch: {branch_name}") - git_update_ref(branch_name) - else: - print("Error parsing branch name for type 1.") - # 错误类型2：尝试创建新的远程引用时，本地已经存在同名的引用 - elif "exists; cannot create" in error_output: - branch_name = parse_branch_name_type2(error_output) - if branch_name: - print(f"Detected error type 2, attempting to delete ref for branch: {branch_name}") - git_update_ref(branch_name) - else: - print("Error parsing branch name for type 2.") + error_types = [ + ("is at", "but expected", parse_branch_name_type1, "type 1"), + ("exists; cannot create", None, parse_branch_name_type2, "type 2"), + ("Unable to create", "File exists", parse_branch_name_type3, "type 3") + ] + + for error_type in error_types: + if error_type[0] in error_output and (error_type[1] is None or error_type[1] in error_output): + branch_name = error_type[2](error_output) + if branch_name: + print(f"Detected error {error_type[3]}, attempting to delete ref for branch: {branch_name}") + git_update_ref(branch_name) + else: + print(f"Error parsing branch name for {error_type[3]}.") + break -# 主函数 def main(): fetch_result = git_fetch() - if fetch_result.returncode != 0: # 如果 git fetch 命令失败 + if fetch_result.returncode != 0: error_output = fetch_result.stderr handle_error(error_output) else: print("Git fetch successful.") + prune_result = git_prune() + print(prune_result.returncode) + if prune_result.returncode != 0: + error_output = prune_result.stderr + print(error_output) + handle_error(error_output) + else: + print("Git prune successful.") + if __name__ == "__main__": main() \ No newline at end of file diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 9ea865e523..76aec26ff4 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -52,6 +52,22 @@ #define TMQ_META_VERSION "1.0" +static bool tmqAddJsonObjectItem(cJSON *object, const char *string, cJSON *item){ + bool ret = cJSON_AddItemToObject(object, string, item); + if (!ret){ + cJSON_Delete(item); + } + return ret; +} +static bool tmqAddJsonArrayItem(cJSON *array, cJSON *item){ + bool ret = cJSON_AddItemToArray(array, item); + if (!ret){ + cJSON_Delete(item); + } + return ret; +} + + static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen); static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); } static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t, @@ -68,41 +84,43 @@ static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sche cJSON* type = cJSON_CreateString("create"); RAW_NULL_CHECK(type); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "type", type)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "type", type)); cJSON* tableType = cJSON_CreateString(t == TSDB_NORMAL_TABLE ? "normal" : "super"); RAW_NULL_CHECK(tableType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableType", tableType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableType", tableType)); cJSON* tableName = cJSON_CreateString(name); RAW_NULL_CHECK(tableName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableName", tableName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableName", tableName)); cJSON* columns = cJSON_CreateArray(); RAW_NULL_CHECK(columns); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "columns", columns)); + for (int i = 0; i < schemaRow->nCols; i++) { cJSON* column = cJSON_CreateObject(); RAW_NULL_CHECK(column); + RAW_FALSE_CHECK(tmqAddJsonArrayItem(columns, column)); SSchema* s = schemaRow->pSchema + i; cJSON* cname = cJSON_CreateString(s->name); RAW_NULL_CHECK(cname); - RAW_FALSE_CHECK(cJSON_AddItemToObject(column, "name", cname)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(column, "name", cname)); cJSON* ctype = cJSON_CreateNumber(s->type); RAW_NULL_CHECK(ctype); - RAW_FALSE_CHECK(cJSON_AddItemToObject(column, "type", ctype)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(column, "type", ctype)); if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_VARBINARY || s->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = s->bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(column, "length", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(column, "length", cbytes)); } else if (s->type == TSDB_DATA_TYPE_NCHAR) { int32_t length = (s->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(column, "length", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(column, "length", cbytes)); } cJSON* isPk = cJSON_CreateBool(s->flags & COL_IS_KEY); RAW_NULL_CHECK(isPk); - RAW_FALSE_CHECK(cJSON_AddItemToObject(column, "isPrimarykey", isPk)); - RAW_FALSE_CHECK(cJSON_AddItemToArray(columns, column)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(column, "isPrimarykey", isPk)); if (pColCmprRow == NULL) { continue; @@ -124,44 +142,44 @@ static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sche cJSON* encodeJson = cJSON_CreateString(encode); RAW_NULL_CHECK(encodeJson); - RAW_FALSE_CHECK(cJSON_AddItemToObject(column, "encode", encodeJson)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(column, "encode", encodeJson)); cJSON* compressJson = cJSON_CreateString(compress); RAW_NULL_CHECK(compressJson); - RAW_FALSE_CHECK(cJSON_AddItemToObject(column, "compress", compressJson)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(column, "compress", compressJson)); cJSON* levelJson = cJSON_CreateString(level); RAW_NULL_CHECK(levelJson); - RAW_FALSE_CHECK(cJSON_AddItemToObject(column, "level", levelJson)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(column, "level", levelJson)); } - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "columns", columns)); cJSON* tags = cJSON_CreateArray(); RAW_NULL_CHECK(tags); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tags", tags)); + for (int i = 0; schemaTag && i < schemaTag->nCols; i++) { cJSON* tag = cJSON_CreateObject(); RAW_NULL_CHECK(tag); + RAW_FALSE_CHECK(tmqAddJsonArrayItem(tags, tag)); SSchema* s = schemaTag->pSchema + i; cJSON* tname = cJSON_CreateString(s->name); RAW_NULL_CHECK(tname); - RAW_FALSE_CHECK(cJSON_AddItemToObject(tag, "name", tname)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(tag, "name", tname)); cJSON* ttype = cJSON_CreateNumber(s->type); RAW_NULL_CHECK(ttype); - RAW_FALSE_CHECK(cJSON_AddItemToObject(tag, "type", ttype)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(tag, "type", ttype)); if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_VARBINARY || s->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = s->bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(tag, "length", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(tag, "length", cbytes)); } else if (s->type == TSDB_DATA_TYPE_NCHAR) { int32_t length = (s->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(tag, "length", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(tag, "length", cbytes)); } - RAW_FALSE_CHECK(cJSON_AddItemToArray(tags, tag)); } - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tags", tags)); end: *pJson = json; @@ -175,7 +193,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) { RAW_NULL_CHECK(encodeStr); cJSON* encodeJson = cJSON_CreateString(encodeStr); RAW_NULL_CHECK(encodeJson); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "encode", encodeJson)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "encode", encodeJson)); return code; } uint8_t compress = COMPRESS_L2_TYPE_U32(para); @@ -184,7 +202,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) { RAW_NULL_CHECK(compressStr); cJSON* compressJson = cJSON_CreateString(compressStr); RAW_NULL_CHECK(compressJson); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "compress", compressJson)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "compress", compressJson)); return code; } uint8_t level = COMPRESS_L2_TYPE_LEVEL_U32(para); @@ -193,7 +211,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) { RAW_NULL_CHECK(levelStr); cJSON* levelJson = cJSON_CreateString(levelStr); RAW_NULL_CHECK(levelJson); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "level", levelJson)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "level", levelJson)); return code; } @@ -214,19 +232,19 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** RAW_NULL_CHECK(json); cJSON* type = cJSON_CreateString("alter"); RAW_NULL_CHECK(type); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "type", type)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "type", type)); SName name = {0}; RAW_RETURN_CHECK(tNameFromString(&name, req.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE)); cJSON* tableType = cJSON_CreateString("super"); RAW_NULL_CHECK(tableType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableType", tableType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableType", tableType)); cJSON* tableName = cJSON_CreateString(name.tname); RAW_NULL_CHECK(tableName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableName", tableName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableName", tableName)); cJSON* alterType = cJSON_CreateNumber(req.alterType); RAW_NULL_CHECK(alterType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "alterType", alterType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "alterType", alterType)); switch (req.alterType) { case TSDB_ALTER_TABLE_ADD_TAG: case TSDB_ALTER_TABLE_ADD_COLUMN: { @@ -234,22 +252,22 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** RAW_NULL_CHECK(field); cJSON* colName = cJSON_CreateString(field->name); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); cJSON* colType = cJSON_CreateNumber(field->type); RAW_NULL_CHECK(colType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colType", colType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colType", colType)); if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY || field->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = field->bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } else if (field->type == TSDB_DATA_TYPE_NCHAR) { int32_t length = (field->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } break; } @@ -258,22 +276,22 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** RAW_NULL_CHECK(field); cJSON* colName = cJSON_CreateString(field->name); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); cJSON* colType = cJSON_CreateNumber(field->type); RAW_NULL_CHECK(colType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colType", colType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colType", colType)); if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY || field->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = field->bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } else if (field->type == TSDB_DATA_TYPE_NCHAR) { int32_t length = (field->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } RAW_RETURN_CHECK(setCompressOption(json, field->compress)); break; @@ -284,7 +302,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** RAW_NULL_CHECK(field); cJSON* colName = cJSON_CreateString(field->name); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); break; } case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: @@ -293,21 +311,21 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** RAW_NULL_CHECK(field); cJSON* colName = cJSON_CreateString(field->name); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); cJSON* colType = cJSON_CreateNumber(field->type); RAW_NULL_CHECK(colType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colType", colType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colType", colType)); if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY || field->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = field->bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } else if (field->type == TSDB_DATA_TYPE_NCHAR) { int32_t length = (field->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } break; } @@ -319,10 +337,10 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** RAW_NULL_CHECK(newField); cJSON* colName = cJSON_CreateString(oldField->name); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); cJSON* colNewName = cJSON_CreateString(newField->name); RAW_NULL_CHECK(colNewName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colNewName", colNewName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colNewName", colNewName)); break; } case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS: { @@ -330,7 +348,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** RAW_NULL_CHECK(field); cJSON* colName = cJSON_CreateString(field->name); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); RAW_RETURN_CHECK(setCompressOption(json, field->bytes)); break; } @@ -391,51 +409,47 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { int64_t id = pCreateReq->uid; uint8_t tagNum = pCreateReq->ctb.tagNum; int32_t code = 0; - cJSON* tags = NULL; + SArray* pTagVals = NULL; + char* pJson = NULL; + cJSON* tableName = cJSON_CreateString(name); RAW_NULL_CHECK(tableName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableName", tableName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableName", tableName)); cJSON* using = cJSON_CreateString(sname); RAW_NULL_CHECK(using); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "using", using)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "using", using)); cJSON* tagNumJson = cJSON_CreateNumber(tagNum); RAW_NULL_CHECK(tagNumJson); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tagNum", tagNumJson)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tagNum", tagNumJson)); - tags = cJSON_CreateArray(); + cJSON* tags = cJSON_CreateArray(); RAW_NULL_CHECK(tags); - SArray* pTagVals = NULL; + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tags", tags)); RAW_RETURN_CHECK(tTagToValArray(pTag, &pTagVals)); - if (tTagIsJson(pTag)) { STag* p = (STag*)pTag; if (p->nTag == 0) { uError("p->nTag == 0"); goto end; } - char* pJson = NULL; parseTagDatatoJson(pTag, &pJson, NULL); - if (pJson == NULL) { - uError("parseTagDatatoJson failed, pJson == NULL"); - goto end; - } + RAW_NULL_CHECK(pJson); cJSON* tag = cJSON_CreateObject(); RAW_NULL_CHECK(tag); + RAW_FALSE_CHECK(tmqAddJsonArrayItem(tags, tag)); STagVal* pTagVal = taosArrayGet(pTagVals, 0); RAW_NULL_CHECK(pTagVal); char* ptname = taosArrayGet(tagName, 0); RAW_NULL_CHECK(ptname); cJSON* tname = cJSON_CreateString(ptname); RAW_NULL_CHECK(tname); - RAW_FALSE_CHECK(cJSON_AddItemToObject(tag, "name", tname)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(tag, "name", tname)); cJSON* ttype = cJSON_CreateNumber(TSDB_DATA_TYPE_JSON); RAW_NULL_CHECK(ttype); - RAW_FALSE_CHECK(cJSON_AddItemToObject(tag, "type", ttype)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(tag, "type", ttype)); cJSON* tvalue = cJSON_CreateString(pJson); RAW_NULL_CHECK(tvalue); - RAW_FALSE_CHECK(cJSON_AddItemToObject(tag, "value", tvalue)); - RAW_FALSE_CHECK(cJSON_AddItemToArray(tags, tag)); - taosMemoryFree(pJson); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(tag, "value", tvalue)); goto end; } @@ -444,36 +458,34 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { RAW_NULL_CHECK(pTagVal); cJSON* tag = cJSON_CreateObject(); RAW_NULL_CHECK(tag); + RAW_FALSE_CHECK(tmqAddJsonArrayItem(tags, tag)); char* ptname = taosArrayGet(tagName, i); RAW_NULL_CHECK(ptname); cJSON* tname = cJSON_CreateString(ptname); RAW_NULL_CHECK(tname); - RAW_FALSE_CHECK(cJSON_AddItemToObject(tag, "name", tname)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(tag, "name", tname)); cJSON* ttype = cJSON_CreateNumber(pTagVal->type); RAW_NULL_CHECK(ttype); - RAW_FALSE_CHECK(cJSON_AddItemToObject(tag, "type", ttype)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(tag, "type", ttype)); cJSON* tvalue = NULL; if (IS_VAR_DATA_TYPE(pTagVal->type)) { - char* buf = NULL; int64_t bufSize = 0; if (pTagVal->type == TSDB_DATA_TYPE_VARBINARY) { bufSize = pTagVal->nData * 2 + 2 + 3; } else { bufSize = pTagVal->nData + 3; } - buf = taosMemoryCalloc(bufSize, 1); - + char* buf = taosMemoryCalloc(bufSize, 1); RAW_NULL_CHECK(buf); - if (!buf) goto end; if (dataConverToStr(buf, bufSize, pTagVal->type, pTagVal->pData, pTagVal->nData, NULL) != TSDB_CODE_SUCCESS) { taosMemoryFree(buf); goto end; } tvalue = cJSON_CreateString(buf); - RAW_NULL_CHECK(tvalue); taosMemoryFree(buf); + RAW_NULL_CHECK(tvalue); } else { double val = 0; GET_TYPED_DATA(val, double, pTagVal->type, &pTagVal->i64); @@ -481,12 +493,11 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { RAW_NULL_CHECK(tvalue); } - RAW_FALSE_CHECK(cJSON_AddItemToObject(tag, "value", tvalue)); - RAW_FALSE_CHECK(cJSON_AddItemToArray(tags, tag)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(tag, "value", tvalue)); } end: - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tags", tags)); + taosMemoryFree(pJson); taosArrayDestroy(pTagVals); } @@ -497,22 +508,23 @@ static void buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs, cJSO RAW_NULL_CHECK(json); cJSON* type = cJSON_CreateString("create"); RAW_NULL_CHECK(type); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "type", type)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "type", type)); cJSON* tableType = cJSON_CreateString("child"); RAW_NULL_CHECK(tableType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableType", tableType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableType", tableType)); buildChildElement(json, pCreateReq); cJSON* createList = cJSON_CreateArray(); RAW_NULL_CHECK(createList); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "createList", createList)); + for (int i = 0; nReqs > 1 && i < nReqs; i++) { cJSON* create = cJSON_CreateObject(); RAW_NULL_CHECK(create); buildChildElement(create, pCreateReq + i); - RAW_FALSE_CHECK(cJSON_AddItemToArray(createList, create)); + RAW_FALSE_CHECK(tmqAddJsonArrayItem(createList, create)); } - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "createList", createList)); end: *pJson = json; @@ -619,62 +631,62 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { RAW_NULL_CHECK(json); cJSON* type = cJSON_CreateString("alter"); RAW_NULL_CHECK(type); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "type", type)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "type", type)); cJSON* tableType = cJSON_CreateString(vAlterTbReq.action == TSDB_ALTER_TABLE_UPDATE_TAG_VAL || vAlterTbReq.action == TSDB_ALTER_TABLE_UPDATE_MULTI_TAG_VAL ? "child" : "normal"); RAW_NULL_CHECK(tableType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableType", tableType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableType", tableType)); cJSON* tableName = cJSON_CreateString(vAlterTbReq.tbName); RAW_NULL_CHECK(tableName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableName", tableName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableName", tableName)); cJSON* alterType = cJSON_CreateNumber(vAlterTbReq.action); RAW_NULL_CHECK(alterType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "alterType", alterType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "alterType", alterType)); switch (vAlterTbReq.action) { case TSDB_ALTER_TABLE_ADD_COLUMN: { cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); cJSON* colType = cJSON_CreateNumber(vAlterTbReq.type); RAW_NULL_CHECK(colType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colType", colType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colType", colType)); if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY || vAlterTbReq.type == TSDB_DATA_TYPE_VARBINARY || vAlterTbReq.type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = vAlterTbReq.bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } else if (vAlterTbReq.type == TSDB_DATA_TYPE_NCHAR) { int32_t length = (vAlterTbReq.bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } break; } case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: { cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); cJSON* colType = cJSON_CreateNumber(vAlterTbReq.type); RAW_NULL_CHECK(colType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colType", colType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colType", colType)); if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY || vAlterTbReq.type == TSDB_DATA_TYPE_VARBINARY || vAlterTbReq.type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = vAlterTbReq.bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } else if (vAlterTbReq.type == TSDB_DATA_TYPE_NCHAR) { int32_t length = (vAlterTbReq.bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } RAW_RETURN_CHECK(setCompressOption(json, vAlterTbReq.compress)); break; @@ -682,43 +694,43 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { case TSDB_ALTER_TABLE_DROP_COLUMN: { cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); break; } case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: { cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); cJSON* colType = cJSON_CreateNumber(vAlterTbReq.colModType); RAW_NULL_CHECK(colType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colType", colType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colType", colType)); if (vAlterTbReq.colModType == TSDB_DATA_TYPE_BINARY || vAlterTbReq.colModType == TSDB_DATA_TYPE_VARBINARY || vAlterTbReq.colModType == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = vAlterTbReq.colModBytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } else if (vAlterTbReq.colModType == TSDB_DATA_TYPE_NCHAR) { int32_t length = (vAlterTbReq.colModBytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } break; } case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: { cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); cJSON* colNewName = cJSON_CreateString(vAlterTbReq.colNewName); RAW_NULL_CHECK(colNewName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colNewName", colNewName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colNewName", colNewName)); break; } case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: { cJSON* tagName = cJSON_CreateString(vAlterTbReq.tagName); RAW_NULL_CHECK(tagName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", tagName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", tagName)); bool isNull = vAlterTbReq.isNull; if (vAlterTbReq.tagType == TSDB_DATA_TYPE_JSON) { @@ -757,12 +769,12 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { cJSON* colValue = cJSON_CreateString(buf); taosMemoryFree(buf); RAW_NULL_CHECK(colValue); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colValue", colValue)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colValue", colValue)); } cJSON* isNullCJson = cJSON_CreateBool(isNull); RAW_NULL_CHECK(isNullCJson); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colValueNull", isNullCJson)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colValueNull", isNullCJson)); break; } case TSDB_ALTER_TABLE_UPDATE_MULTI_TAG_VAL: { @@ -774,14 +786,17 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { cJSON* tags = cJSON_CreateArray(); RAW_NULL_CHECK(tags); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tags", tags)); + for (int32_t i = 0; i < nTags; i++) { cJSON* member = cJSON_CreateObject(); RAW_NULL_CHECK(member); + RAW_FALSE_CHECK(tmqAddJsonArrayItem(tags, member)); SMultiTagUpateVal* pTagVal = taosArrayGet(vAlterTbReq.pMultiTag, i); cJSON* tagName = cJSON_CreateString(pTagVal->tagName); RAW_NULL_CHECK(tagName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(member, "colName", tagName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(member, "colName", tagName)); if (pTagVal->tagType == TSDB_DATA_TYPE_JSON) { uError("processAlterTable isJson false"); @@ -789,14 +804,13 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { } bool isNull = pTagVal->isNull; if (!isNull) { - char* buf = NULL; int64_t bufSize = 0; if (pTagVal->tagType == TSDB_DATA_TYPE_VARBINARY) { bufSize = pTagVal->nTagVal * 2 + 2 + 3; } else { bufSize = pTagVal->nTagVal + 3; } - buf = taosMemoryCalloc(bufSize, 1); + char* buf = taosMemoryCalloc(bufSize, 1); RAW_NULL_CHECK(buf); if (dataConverToStr(buf, bufSize, pTagVal->tagType, pTagVal->pTagVal, pTagVal->nTagVal, NULL) != TSDB_CODE_SUCCESS) { @@ -806,21 +820,19 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { cJSON* colValue = cJSON_CreateString(buf); taosMemoryFree(buf); RAW_NULL_CHECK(colValue); - RAW_FALSE_CHECK(cJSON_AddItemToObject(member, "colValue", colValue)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(member, "colValue", colValue)); } cJSON* isNullCJson = cJSON_CreateBool(isNull); RAW_NULL_CHECK(isNullCJson); - RAW_FALSE_CHECK(cJSON_AddItemToObject(member, "colValueNull", isNullCJson)); - RAW_FALSE_CHECK(cJSON_AddItemToArray(tags, member)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(member, "colValueNull", isNullCJson)); } - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tags", tags)); break; } case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS: { cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); RAW_RETURN_CHECK(setCompressOption(json, vAlterTbReq.compress)); break; } @@ -858,13 +870,13 @@ static void processDropSTable(SMqMetaRsp* metaRsp, cJSON** pJson) { RAW_NULL_CHECK(json); cJSON* type = cJSON_CreateString("drop"); RAW_NULL_CHECK(type); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "type", type)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "type", type)); cJSON* tableType = cJSON_CreateString("super"); RAW_NULL_CHECK(tableType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableType", tableType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableType", tableType)); cJSON* tableName = cJSON_CreateString(req.name); RAW_NULL_CHECK(tableName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableName", tableName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableName", tableName)); end: uDebug("processDropSTable return"); @@ -897,10 +909,10 @@ static void processDeleteTable(SMqMetaRsp* metaRsp, cJSON** pJson) { RAW_NULL_CHECK(json); cJSON* type = cJSON_CreateString("delete"); RAW_NULL_CHECK(type); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "type", type)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "type", type)); cJSON* sqlJson = cJSON_CreateString(sql); RAW_NULL_CHECK(sqlJson); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", sqlJson)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "sql", sqlJson)); end: uDebug("processDeleteTable return"); @@ -928,16 +940,17 @@ static void processDropTable(SMqMetaRsp* metaRsp, cJSON** pJson) { RAW_NULL_CHECK(json); cJSON* type = cJSON_CreateString("drop"); RAW_NULL_CHECK(type); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "type", type)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "type", type)); cJSON* tableNameList = cJSON_CreateArray(); RAW_NULL_CHECK(tableNameList); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableNameList", tableNameList)); + for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { SVDropTbReq* pDropTbReq = req.pReqs + iReq; cJSON* tableName = cJSON_CreateString(pDropTbReq->name); RAW_NULL_CHECK(tableName); - RAW_FALSE_CHECK(cJSON_AddItemToArray(tableNameList, tableName)); + RAW_FALSE_CHECK(tmqAddJsonArrayItem(tableNameList, tableName)); } - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableNameList", tableNameList)); end: uDebug("processDropTable return"); @@ -2183,6 +2196,8 @@ static void processBatchMetaToJson(SMqBatchMetaRsp* pMsgRsp, char** string) { RAW_FALSE_CHECK(cJSON_AddStringToObject(pJson, "tmq_meta_version", TMQ_META_VERSION)); cJSON* pMetaArr = cJSON_CreateArray(); RAW_NULL_CHECK(pMetaArr); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(pJson, "metas", pMetaArr)); + int32_t num = taosArrayGetSize(rsp.batchMetaReq); for (int32_t i = 0; i < num; i++) { int32_t* len = taosArrayGet(rsp.batchMetaLen, i); @@ -2198,10 +2213,9 @@ static void processBatchMetaToJson(SMqBatchMetaRsp* pMsgRsp, char** string) { cJSON* pItem = NULL; processSimpleMeta(&metaRsp, &pItem); tDeleteMqMetaRsp(&metaRsp); - RAW_FALSE_CHECK(cJSON_AddItemToArray(pMetaArr, pItem)); + RAW_FALSE_CHECK(tmqAddJsonArrayItem(pMetaArr, pItem)); } - RAW_FALSE_CHECK(cJSON_AddItemToObject(pJson, "metas", pMetaArr)); tDeleteMqBatchMetaRsp(&rsp); char* fullStr = cJSON_PrintUnformatted(pJson); cJSON_Delete(pJson); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index d0872009fd..64a11fbe5b 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -797,7 +797,7 @@ static int32_t mndGetStreamNameFromSmaName(char *streamName, char *smaName) { if (TSDB_CODE_SUCCESS != code) { return code; } - sprintf(streamName, "%d.%s", n.acctId, n.tname); + snprintf(streamName, TSDB_TABLE_FNAME_LEN,"%d.%s", n.acctId, n.tname); return TSDB_CODE_SUCCESS; } @@ -1222,7 +1222,7 @@ static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp memcpy(rsp->dbFName, pSma->db, sizeof(pSma->db)); memcpy(rsp->tblFName, pSma->stb, sizeof(pSma->stb)); - strcpy(rsp->indexType, TSDB_INDEX_TYPE_SMA); + tstrncpy(rsp->indexType, TSDB_INDEX_TYPE_SMA, TSDB_INDEX_TYPE_LEN); SNodeList *pList = NULL; int32_t extOffset = 0; @@ -1255,8 +1255,8 @@ int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool return TSDB_CODE_SUCCESS; } - strcpy(rsp->dbFName, pStb->db); - strcpy(rsp->tbName, pStb->name + strlen(pStb->db) + 1); + tstrncpy(rsp->dbFName, pStb->db, TSDB_DB_FNAME_LEN); + tstrncpy(rsp->tbName, pStb->name + strlen(pStb->db) + 1, TSDB_TABLE_NAME_LEN); rsp->suid = pStb->uid; rsp->version = pStb->smaVer; mndReleaseStb(pMnode, pStb); @@ -1632,7 +1632,7 @@ static int32_t mndCreateTSMABuildCreateStreamReq(SCreateTSMACxt *pCxt) { f.bytes = pExprNode->resType.bytes; f.type = pExprNode->resType.type; f.flags = COL_SMA_ON; - strcpy(f.name, pExprNode->userAlias); + tstrncpy(f.name, pExprNode->userAlias, TSDB_COL_NAME_LEN); if (NULL == taosArrayPush(pCxt->pCreateStreamReq->pCols, &f)) { code = terrno; break; @@ -1735,7 +1735,7 @@ static int32_t mndCreateTSMATxnPrepare(SCreateTSMACxt* pCxt) { } dropStbReq.igNotExists = true; - strncpy(dropStbReq.name, pCxt->targetStbFullName, TSDB_TABLE_FNAME_LEN); + tstrncpy(dropStbReq.name, pCxt->targetStbFullName, TSDB_TABLE_FNAME_LEN); dropStbUndoAction.epSet = createStreamRedoAction.epSet; dropStbUndoAction.acceptableCode = TSDB_CODE_MND_STB_NOT_EXIST; dropStbUndoAction.retryCode = TSDB_CODE_MND_STREAM_MUST_BE_DELETED; @@ -1836,7 +1836,7 @@ static int32_t mndTSMAGenerateOutputName(const char* tsmaName, char* streamName, if (TSDB_CODE_SUCCESS != code) { return code; } - sprintf(streamName, "%d.%s", smaName.acctId, smaName.tname); + snprintf(streamName, TSDB_TABLE_FNAME_LEN, "%d.%s", smaName.acctId, smaName.tname); snprintf(targetStbName, TSDB_TABLE_FNAME_LEN, "%s"TSMA_RES_STB_POSTFIX, tsmaName); return TSDB_CODE_SUCCESS; } @@ -2486,7 +2486,7 @@ static int32_t mndGetSomeTsmas(SMnode* pMnode, STableTSMAInfoRsp* pRsp, tsmaFilt mndReleaseStb(pMnode, pStb); TAOS_RETURN(code); } - sprintf(streamName, "%d.%s", smaName.acctId, smaName.tname); + snprintf(streamName, TSDB_TABLE_FNAME_LEN, "%d.%s", smaName.acctId, smaName.tname); pStream = NULL; code = mndAcquireStream(pMnode, streamName, &pStream); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 5d41e1506c..0a107518df 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1931,11 +1931,6 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange sdbCancelFetch(pSdb, pIter); return terrno = code; } - - code = mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_UPDATE_NAME, pStream->uid); - if (code) { - mError("failed to register trans, transId:%d, and continue", pTrans->id); - } } if (!includeAllNodes) { @@ -1951,6 +1946,12 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange mDebug("stream:0x%" PRIx64 " %s involved node changed, create update trans, transId:%d", pStream->uid, pStream->name, pTrans->id); + // NOTE: for each stream, we register one trans entry for task update + code = mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_UPDATE_NAME, pStream->uid); + if (code) { + mError("failed to register trans, transId:%d, and continue", pTrans->id); + } + code = mndStreamSetUpdateEpsetAction(pMnode, pStream, pChangeInfo, pTrans); // todo: not continue, drop all and retry again diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index 905a73ad48..a1e104aeca 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -35,7 +35,11 @@ int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt) size_t keyLen = 0; void *pIter = NULL; SArray *pList = taosArrayInit(4, sizeof(SKeyInfo)); - int32_t num = 0; + int32_t numOfChkpt = 0; + + if (pNumOfActiveChkpt != NULL) { + *pNumOfActiveChkpt = 0; + } if (pList == NULL) { return terrno; @@ -50,15 +54,15 @@ int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt) void *pKey = taosHashGetKey(pEntry, &keyLen); // key is the name of src/dst db name SKeyInfo info = {.pKey = pKey, .keyLen = keyLen}; - mDebug("transId:%d %s startTs:%" PRId64 " cleared since finished", pEntry->transId, pEntry->name, - pEntry->startTime); + mDebug("transId:%d stream:0x%" PRIx64 " %s startTs:%" PRId64 " cleared since finished", pEntry->transId, + pEntry->streamId, pEntry->name, pEntry->startTime); void* p = taosArrayPush(pList, &info); if (p == NULL) { return terrno; } } else { if (strcmp(pEntry->name, MND_STREAM_CHECKPOINT_NAME) == 0) { - num++; + numOfChkpt++; } mndReleaseTrans(pMnode, pTrans); } @@ -78,48 +82,34 @@ int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt) } } - mDebug("clear %d finished stream-trans, remained:%d, active checkpoint trans:%d", size, - taosHashGetSize(execInfo.transMgmt.pDBTrans), num); + mDebug("clear %d finished stream-trans, active trans:%d, active checkpoint trans:%d", size, + taosHashGetSize(execInfo.transMgmt.pDBTrans), numOfChkpt); taosArrayDestroy(pList); if (pNumOfActiveChkpt != NULL) { - *pNumOfActiveChkpt = num; + *pNumOfActiveChkpt = numOfChkpt; } return 0; } -// * Transactions of different streams are not related. Here only check the conflict of transaction for a given stream. -// For a given stream: -// 1. checkpoint trans is conflict with any other trans except for the drop and reset trans. -// 2. create/drop/reset/update trans are conflict with any other trans. -int32_t mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName, bool lock) { - if (lock) { - streamMutexLock(&execInfo.lock); - } - +static int32_t doStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName) { int32_t num = taosHashGetSize(execInfo.transMgmt.pDBTrans); if (num <= 0) { - if (lock) { - streamMutexUnlock(&execInfo.lock); - } return 0; } + // if any task updates exist, any other stream trans are not allowed to be created int32_t code = mndStreamClearFinishedTrans(pMnode, NULL); if (code) { - mError("failed to clear finish trans, code:%s", tstrerror(code)); + mError("failed to clear finish trans, code:%s, and continue", tstrerror(code)); } SStreamTransInfo *pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId)); if (pEntry != NULL) { SStreamTransInfo tInfo = *pEntry; - if (lock) { - streamMutexUnlock(&execInfo.lock); - } - if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0) { if ((strcmp(pTransName, MND_STREAM_DROP_NAME) != 0) && (strcmp(pTransName, MND_STREAM_TASK_RESET_NAME) != 0) && (strcmp(pTransName, MND_STREAM_RESTART_NAME) != 0)) { @@ -141,11 +131,25 @@ int32_t mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char mDebug("stream:0x%" PRIx64 " no conflict trans existed, continue create trans", streamId); } + return TSDB_CODE_SUCCESS; +} + +// * Transactions of different streams are not related. Here only check the conflict of transaction for a given stream. +// For a given stream: +// 1. checkpoint trans is conflict with any other trans except for the drop and reset trans. +// 2. create/drop/reset/update trans are conflict with any other trans. +int32_t mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName, bool lock) { + if (lock) { + streamMutexLock(&execInfo.lock); + } + + int32_t code = doStreamTransConflictCheck(pMnode, streamId, pTransName); + if (lock) { streamMutexUnlock(&execInfo.lock); } - return 0; + return code; } int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId) { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 4bc538531b..718b7d0df6 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -52,10 +52,17 @@ static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans, bool t static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans, bool topHalf); static bool mndTransPerformFinishStage(SMnode *pMnode, STrans *pTrans, bool topHalf); -static bool mndCannotExecuteTransAction(SMnode *pMnode, bool topHalf) { - return (!pMnode->deploy && !mndIsLeader(pMnode)) || !topHalf; +static inline bool mndTransIsInSyncContext(bool topHalf) { return !topHalf; } + +static bool mndCannotExecuteTrans(SMnode *pMnode, bool topHalf) { + bool isLeader = mndIsLeader(pMnode); + bool ret = (!pMnode->deploy && !isLeader) || mndTransIsInSyncContext(topHalf); + if (ret) mDebug("cannot execute trans action, deploy:%d, isLeader:%d, topHalf:%d", pMnode->deploy, isLeader, topHalf); + return ret; } +static inline char *mndStrExecutionContext(bool topHalf) { return topHalf ? "transContext" : "syncContext"; } + static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans); static int32_t mndProcessTransTimer(SRpcMsg *pReq); static int32_t mndProcessTtl(SRpcMsg *pReq); @@ -1335,7 +1342,7 @@ static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransActi // execute in trans context static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { if (pAction->msgSent) return 0; - if (mndCannotExecuteTransAction(pMnode, topHalf)) { + if (mndCannotExecuteTrans(pMnode, topHalf)) { TAOS_RETURN(TSDB_CODE_MND_TRANS_CTX_SWITCH); } @@ -1481,8 +1488,8 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans, bool topHalf) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions, topHalf); if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS && code != TSDB_CODE_MND_TRANS_CTX_SWITCH) { - mError("trans:%d, failed to execute redoActions since:%s, code:0x%x, topHalf(TransContext):%d", pTrans->id, - terrstr(), terrno, topHalf); + mError("trans:%d, failed to execute redoActions since:%s, code:0x%x, in %s", pTrans->id, terrstr(), terrno, + mndStrExecutionContext(topHalf)); } return code; } @@ -1490,8 +1497,8 @@ static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans, bool t static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans, bool topHalf) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions, topHalf); if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS && code != TSDB_CODE_MND_TRANS_CTX_SWITCH) { - mError("trans:%d, failed to execute undoActions since %s. topHalf(TransContext):%d", pTrans->id, terrstr(), - topHalf); + mError("trans:%d, failed to execute undoActions since %s. in %s", pTrans->id, terrstr(), + mndStrExecutionContext(topHalf)); } return code; } @@ -1499,8 +1506,8 @@ static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans, bool t static int32_t mndTransExecuteCommitActions(SMnode *pMnode, STrans *pTrans, bool topHalf) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->commitActions, topHalf); if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS && code != TSDB_CODE_MND_TRANS_CTX_SWITCH) { - mError("trans:%d, failed to execute commitActions since %s. topHalf(TransContext):%d", pTrans->id, terrstr(), - topHalf); + mError("trans:%d, failed to execute commitActions since %s. in %s", pTrans->id, terrstr(), + mndStrExecutionContext(topHalf)); } return code; } @@ -1520,7 +1527,7 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr for (int32_t action = pTrans->actionPos; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pActions, action); - mInfo("trans:%d, current action:%d, stage:%s, actionType(0:log,1:msg):%d", pTrans->id, pTrans->actionPos, + mInfo("trans:%d, current action:%d, stage:%s, actionType(1:msg,2:log):%d", pTrans->id, pTrans->actionPos, mndTransStr(pAction->stage), pAction->actionType); code = mndTransExecSingleAction(pMnode, pTrans, pAction, topHalf); @@ -1551,11 +1558,11 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr } mndSetTransLastAction(pTrans, pAction); - if (mndCannotExecuteTransAction(pMnode, topHalf)) { + if (mndCannotExecuteTrans(pMnode, topHalf)) { pTrans->lastErrorNo = code; pTrans->code = code; - mInfo("trans:%d, %s:%d, topHalf(TransContext):%d, not execute next action, code:%s", pTrans->id, - mndTransStr(pAction->stage), action, topHalf, tstrerror(code)); + mInfo("trans:%d, %s:%d, cannot execute next action in %s, code:%s", pTrans->id, mndTransStr(pAction->stage), + action, mndStrExecutionContext(topHalf), tstrerror(code)); break; } @@ -1656,20 +1663,21 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool code = mndTransExecuteRedoActions(pMnode, pTrans, topHalf); } - if (mndCannotExecuteTransAction(pMnode, topHalf)) { + if (code != 0 && code != TSDB_CODE_MND_TRANS_CTX_SWITCH && mndTransIsInSyncContext(topHalf)) { pTrans->lastErrorNo = code; pTrans->code = code; - bool continueExec = true; - if (code != 0 && code != TSDB_CODE_MND_TRANS_CTX_SWITCH) { - taosMsleep(100); - continueExec = true; - } else { - continueExec = false; + mInfo( + "trans:%d, failed to execute, will retry redo action stage in 100 ms , in %s, " + "continueExec:%d, code:%s", + pTrans->id, mndStrExecutionContext(topHalf), continueExec, tstrerror(code)); + taosMsleep(100); + return true; + } else { + if (mndCannotExecuteTrans(pMnode, topHalf)) { + mInfo("trans:%d, cannot continue to execute redo action stage in %s, continueExec:%d, code:%s", pTrans->id, + mndStrExecutionContext(topHalf), continueExec, tstrerror(code)); + return false; } - mInfo("trans:%d, cannot execute redo action stage, topHalf(TransContext):%d, continueExec:%d, code:%s", pTrans->id, - topHalf, continueExec, tstrerror(code)); - - return continueExec; } terrno = code; @@ -1712,9 +1720,9 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool return continueExec; } -// in trans context +// execute in trans context static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { - if (mndCannotExecuteTransAction(pMnode, topHalf)) return false; + if (mndCannotExecuteTrans(pMnode, topHalf)) return false; bool continueExec = true; int32_t code = mndTransCommit(pMnode, pTrans); @@ -1768,7 +1776,7 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans, bool code = mndTransExecuteUndoActions(pMnode, pTrans, topHalf); } - if (mndCannotExecuteTransAction(pMnode, topHalf)) return false; + if (mndCannotExecuteTrans(pMnode, topHalf)) return false; terrno = code; if (code == 0) { @@ -1789,7 +1797,7 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans, bool // in trans context static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { - if (mndCannotExecuteTransAction(pMnode, topHalf)) return false; + if (mndCannotExecuteTrans(pMnode, topHalf)) return false; bool continueExec = true; int32_t code = mndTransRollback(pMnode, pTrans); @@ -1806,8 +1814,9 @@ static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans, bool to return continueExec; } +// excute in trans context static bool mndTransPerformPreFinishStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { - if (mndCannotExecuteTransAction(pMnode, topHalf)) return false; + if (mndCannotExecuteTrans(pMnode, topHalf)) return false; bool continueExec = true; int32_t code = mndTransPreFinish(pMnode, pTrans); @@ -1850,8 +1859,8 @@ void mndTransExecuteImp(SMnode *pMnode, STrans *pTrans, bool topHalf) { bool continueExec = true; while (continueExec) { - mInfo("trans:%d, continue to execute, stage:%s createTime:%" PRId64 " topHalf(TransContext):%d", pTrans->id, - mndTransStr(pTrans->stage), pTrans->createdTime, topHalf); + mInfo("trans:%d, continue to execute stage:%s in %s, createTime:%" PRId64 "", pTrans->id, + mndTransStr(pTrans->stage), mndStrExecutionContext(topHalf), pTrans->createdTime); pTrans->lastExecTime = taosGetTimestampMs(); switch (pTrans->stage) { case TRN_STAGE_PREPARE: diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index bbc58004d9..5a515b1440 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -254,11 +254,12 @@ static void tdRSmaTaskInit(SStreamMeta *pMeta, SRSmaInfoItem *pItem, SStreamTask } static void tdRSmaTaskRemove(SStreamMeta *pMeta, int64_t streamId, int32_t taskId) { + streamMetaWLock(pMeta); + int32_t code = streamMetaUnregisterTask(pMeta, streamId, taskId); if (code != 0) { smaError("vgId:%d, rsma task:%" PRIi64 ",%d drop failed since %s", pMeta->vgId, streamId, taskId, tstrerror(code)); } - streamMetaWLock(pMeta); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); if (streamMetaCommit(pMeta) < 0) { // persist to disk diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index a234777441..bb50499ea8 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1122,91 +1122,76 @@ _OVER: return code; } +// always return success to mnode +//todo: handle failure of build and send msg to mnode +static void doSendChkptSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, int32_t code, + int32_t taskId) { + SRpcMsg rsp = {0}; + int32_t ret = streamTaskBuildCheckpointSourceRsp(pReq, pRpcInfo, &rsp, code); + if (ret) { // suppress the error in build checkpoint source rsp + tqError("s-task:0x%x failed to build checkpoint-source rsp, code:%s", taskId, tstrerror(ret)); + } + tmsgSendRsp(&rsp); // error occurs +} + // no matter what kinds of error happened, make sure the mnode will receive the success execution code. int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) { - int32_t vgId = TD_VID(pTq->pVnode); - SStreamMeta* pMeta = pTq->pStreamMeta; - char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t len = pMsg->contLen - sizeof(SMsgHead); - int32_t code = 0; + int32_t vgId = TD_VID(pTq->pVnode); + SStreamMeta* pMeta = pTq->pStreamMeta; + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + int32_t code = 0; + SStreamCheckpointSourceReq req = {0}; + SDecoder decoder = {0}; + SStreamTask* pTask = NULL; + int64_t checkpointId = 0; // disable auto rsp to mnode pRsp->info.handle = NULL; - SStreamCheckpointSourceReq req = {0}; - SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msg, len); if (tDecodeStreamCheckpointSourceReq(&decoder, &req) < 0) { code = TSDB_CODE_MSG_DECODE_ERROR; tDecoderClear(&decoder); tqError("vgId:%d failed to decode checkpoint-source msg, code:%s", vgId, tstrerror(code)); - - SRpcMsg rsp = {0}; - int32_t ret = streamTaskBuildCheckpointSourceRsp(&req, &pMsg->info, &rsp, TSDB_CODE_SUCCESS); - if (ret) { // suppress the error in build checkpointsource rsp - tqError("s-task:0x%x failed to build checkpoint-source rsp, code:%s", req.taskId, tstrerror(code)); - } - - tmsgSendRsp(&rsp); // error occurs - return TSDB_CODE_SUCCESS; // always return success to mnode, todo: handle failure of build and send msg to mnode + doSendChkptSourceRsp(&req, &pMsg->info, TSDB_CODE_SUCCESS, req.taskId); + return TSDB_CODE_SUCCESS; // always return success to mnode, } + tDecoderClear(&decoder); if (!vnodeIsRoleLeader(pTq->pVnode)) { tqDebug("vgId:%d not leader, ignore checkpoint-source msg, s-task:0x%x", vgId, req.taskId); - SRpcMsg rsp = {0}; - int32_t ret = streamTaskBuildCheckpointSourceRsp(&req, &pMsg->info, &rsp, TSDB_CODE_SUCCESS); - if (ret) { // suppress the error in build checkpointsource rsp - tqError("s-task:0x%x failed to build checkpoint-source rsp, code:%s", req.taskId, tstrerror(code)); - } - - tmsgSendRsp(&rsp); // error occurs - return TSDB_CODE_SUCCESS; // always return success to mnode, todo: handle failure of build and send msg to mnode + doSendChkptSourceRsp(&req, &pMsg->info, TSDB_CODE_SUCCESS, req.taskId); + return TSDB_CODE_SUCCESS; // always return success to mnode } if (!pTq->pVnode->restored) { tqDebug("vgId:%d checkpoint-source msg received during restoring, checkpointId:%" PRId64 ", transId:%d s-task:0x%x ignore it", vgId, req.checkpointId, req.transId, req.taskId); - SRpcMsg rsp = {0}; - int32_t ret = streamTaskBuildCheckpointSourceRsp(&req, &pMsg->info, &rsp, TSDB_CODE_SUCCESS); - if (ret) { // suppress the error in build checkpointsource rsp - tqError("s-task:0x%x failed to build checkpoint-source rsp, code:%s", req.taskId, tstrerror(code)); - } - - tmsgSendRsp(&rsp); // error occurs - return TSDB_CODE_SUCCESS; // always return success to mnode, , todo: handle failure of build and send msg to mnode + doSendChkptSourceRsp(&req, &pMsg->info, TSDB_CODE_SUCCESS, req.taskId); + return TSDB_CODE_SUCCESS; // always return success to mnode } - SStreamTask* pTask = NULL; code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); if (pTask == NULL || code != 0) { tqError("vgId:%d failed to find s-task:0x%x, ignore checkpoint msg. checkpointId:%" PRId64 " transId:%d it may have been destroyed", vgId, req.taskId, req.checkpointId, req.transId); - SRpcMsg rsp = {0}; - int32_t ret = streamTaskBuildCheckpointSourceRsp(&req, &pMsg->info, &rsp, TSDB_CODE_SUCCESS); - if (ret) { // suppress the error in build checkpointsource rsp - tqError("s-task:%s failed to build checkpoint-source rsp, code:%s", pTask->id.idStr, tstrerror(code)); - } - tmsgSendRsp(&rsp); // error occurs + doSendChkptSourceRsp(&req, &pMsg->info, TSDB_CODE_SUCCESS, req.taskId); return TSDB_CODE_SUCCESS; } if (pTask->status.downstreamReady != 1) { - streamTaskSetFailedChkptInfo(pTask, req.transId, req.checkpointId); // record the latest failed checkpoint id + // record the latest failed checkpoint id + streamTaskSetFailedChkptInfo(pTask, req.transId, req.checkpointId); tqError("s-task:%s not ready for checkpoint, since downstream not ready, ignore this checkpointId:%" PRId64 ", transId:%d set it failed", pTask->id.idStr, req.checkpointId, req.transId); + streamMetaReleaseTask(pMeta, pTask); - - SRpcMsg rsp = {0}; - int32_t ret = streamTaskBuildCheckpointSourceRsp(&req, &pMsg->info, &rsp, TSDB_CODE_SUCCESS); - if (ret) { // suppress the error in build checkpointsource rsp - tqError("s-task:%s failed to build checkpoint-source rsp, code:%s", pTask->id.idStr, tstrerror(code)); - } - - tmsgSendRsp(&rsp); // error occurs + doSendChkptSourceRsp(&req, &pMsg->info, TSDB_CODE_SUCCESS, req.taskId); return TSDB_CODE_SUCCESS; // todo retry handle error } @@ -1221,14 +1206,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) streamMutexUnlock(&pTask->lock); streamMetaReleaseTask(pMeta, pTask); - - SRpcMsg rsp = {0}; - int32_t ret = streamTaskBuildCheckpointSourceRsp(&req, &pMsg->info, &rsp, TSDB_CODE_SUCCESS); - if (ret) { // suppress the error in build checkpointsource rsp - tqError("s-task:%s failed to build checkpoint-source rsp, code:%s", pTask->id.idStr, tstrerror(code)); - } - - tmsgSendRsp(&rsp); // error occurs + doSendChkptSourceRsp(&req, &pMsg->info, TSDB_CODE_SUCCESS, req.taskId); return TSDB_CODE_SUCCESS; } } else { @@ -1240,7 +1218,6 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) // check if the checkpoint msg already sent or not. if (status == TASK_STATUS__CK) { - int64_t checkpointId = 0; streamTaskGetActiveCheckpointInfo(pTask, NULL, &checkpointId); tqWarn("s-task:%s repeatly recv checkpoint-source msg checkpointId:%" PRId64 @@ -1249,7 +1226,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) streamMutexUnlock(&pTask->lock); streamMetaReleaseTask(pMeta, pTask); - + doSendChkptSourceRsp(&req, &pMsg->info, TSDB_CODE_SYN_PROPOSE_NOT_READY, req.taskId); return TSDB_CODE_SUCCESS; } else { // checkpoint already finished, and not in checkpoint status if (req.checkpointId <= pTask->chkInfo.checkpointId) { @@ -1259,15 +1236,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) streamMutexUnlock(&pTask->lock); streamMetaReleaseTask(pMeta, pTask); - - SRpcMsg rsp = {0}; - int32_t ret = streamTaskBuildCheckpointSourceRsp(&req, &pMsg->info, &rsp, TSDB_CODE_SUCCESS); - if (ret) { // suppress the error in build checkpointsource rsp - tqError("s-task:%s failed to build checkpoint-source rsp, code:%s", pTask->id.idStr, tstrerror(code)); - } - - tmsgSendRsp(&rsp); // error occurs - + doSendChkptSourceRsp(&req, &pMsg->info, TSDB_CODE_SUCCESS, req.taskId); return TSDB_CODE_SUCCESS; } } @@ -1278,7 +1247,9 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) if (code) { qError("s-task:%s (vgId:%d) failed to process checkpoint-source req, code:%s", pTask->id.idStr, vgId, tstrerror(code)); - return code; + streamMetaReleaseTask(pMeta, pTask); + doSendChkptSourceRsp(&req, &pMsg->info, TSDB_CODE_SUCCESS, req.taskId); + return TSDB_CODE_SUCCESS; } if (req.mndTrigger) { @@ -1293,13 +1264,8 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) code = streamAddCheckpointSourceRspMsg(&req, &pMsg->info, pTask); if (code != TSDB_CODE_SUCCESS) { - SRpcMsg rsp = {0}; - int32_t ret = streamTaskBuildCheckpointSourceRsp(&req, &pMsg->info, &rsp, TSDB_CODE_SUCCESS); - if (ret) { // suppress the error in build checkpointsource rsp - tqError("s-task:%s failed to build checkpoint-source rsp, code:%s", pTask->id.idStr, tstrerror(code)); - } - tmsgSendRsp(&rsp); // error occurs - return TSDB_CODE_SUCCESS; + streamTaskSetCheckpointFailed(pTask); // set the checkpoint failed + doSendChkptSourceRsp(&req, &pMsg->info, TSDB_CODE_SUCCESS, req.taskId); } streamMetaReleaseTask(pMeta, pTask); diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 3f4ff7f3d9..5405ace89b 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -13,9 +13,7 @@ * along with this program. If not, see . */ -#include #include "tcommon.h" -#include "tmsg.h" #include "tq.h" #define IS_NEW_SUBTB_RULE(_t) (((_t)->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER) && ((_t)->subtableWithoutMd5 != 1)) @@ -50,7 +48,7 @@ static int32_t doPutSinkTableInfoIntoCache(SSHashObj* pSinkTableMap, STableSinkI static bool doGetSinkTableInfoFromCache(SSHashObj* pTableInfoMap, uint64_t groupId, STableSinkInfo** pInfo); static int32_t doRemoveSinkTableInfoInCache(SSHashObj* pSinkTableMap, uint64_t groupId, const char* id); static int32_t checkTagSchema(SStreamTask* pTask, SVnode* pVnode); -static void reubuildAndSendMultiResBlock(SStreamTask* pTask, const SArray* pBlocks, SVnode* pVnode, int64_t earlyTs); +static void rebuildAndSendMultiResBlock(SStreamTask* pTask, const SArray* pBlocks, SVnode* pVnode, int64_t earlyTs); static int32_t handleResultBlockMsg(SStreamTask* pTask, SSDataBlock* pDataBlock, int32_t index, SVnode* pVnode, int64_t earlyTs); static int32_t doWaitForDstTableDropped(SVnode* pVnode, SStreamTask* pTask, const char* dstTableName); @@ -1187,7 +1185,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { return; } - reubuildAndSendMultiResBlock(pTask, pBlocks, pVnode, earlyTs); + rebuildAndSendMultiResBlock(pTask, pBlocks, pVnode, earlyTs); } } @@ -1290,7 +1288,7 @@ int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* return TSDB_CODE_SUCCESS; } -void reubuildAndSendMultiResBlock(SStreamTask* pTask, const SArray* pBlocks, SVnode* pVnode, int64_t earlyTs) { +void rebuildAndSendMultiResBlock(SStreamTask* pTask, const SArray* pBlocks, SVnode* pVnode, int64_t earlyTs) { int32_t code = 0; const char* id = pTask->id.idStr; int32_t vgId = pTask->pMeta->vgId; diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 29372c5da7..bc7e2e28e3 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -17,19 +17,20 @@ #include "vnd.h" #define MAX_REPEAT_SCAN_THRESHOLD 3 -#define SCAN_WAL_IDLE_DURATION 100 +#define SCAN_WAL_IDLE_DURATION 500 // idle for 500ms to do next wal scan typedef struct SBuildScanWalMsgParam { int64_t metaId; int32_t numOfTasks; } SBuildScanWalMsgParam; -static int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle); +static int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta); static int32_t setWalReaderStartOffset(SStreamTask* pTask, int32_t vgId); static bool handleFillhistoryScanComplete(SStreamTask* pTask, int64_t ver); static bool taskReadyForDataFromWal(SStreamTask* pTask); static int32_t doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems, bool* pSucc); static int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDuration); +static int32_t doScanWalAsync(STQ* pTq, bool ckPause); // extract data blocks(submit/delete) from WAL, and add them into the input queue for all the sources tasks. int32_t tqScanWal(STQ* pTq) { @@ -37,12 +38,11 @@ int32_t tqScanWal(STQ* pTq) { int32_t vgId = pMeta->vgId; int64_t st = taosGetTimestampMs(); int32_t numOfTasks = 0; - bool shouldIdle = true; tqDebug("vgId:%d continue to check if data in wal are available, scanCounter:%d", vgId, pMeta->scanInfo.scanCounter); // check all tasks - int32_t code = doScanWalForAllTasks(pMeta, &shouldIdle); + int32_t code = doScanWalForAllTasks(pMeta); if (code) { tqError("vgId:%d failed to start all tasks, try next time, code:%s", vgId, tstrerror(code)); return code; @@ -133,10 +133,9 @@ int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDuration) { } int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { - int32_t vgId = TD_VID(pTq->pVnode); SStreamMeta* pMeta = pTq->pStreamMeta; bool alreadyRestored = pTq->pVnode->restored; - int32_t numOfTasks = 0; + int32_t code = 0; // do not launch the stream tasks, if it is a follower or not restored vnode. if (!(vnodeIsRoleLeader(pTq->pVnode) && alreadyRestored)) { @@ -144,47 +143,8 @@ int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { } streamMetaWLock(pMeta); - - numOfTasks = taosArrayGetSize(pMeta->pTaskList); - if (numOfTasks == 0) { - tqDebug("vgId:%d no stream tasks existed to run", vgId); - streamMetaWUnLock(pMeta); - return 0; - } - - if (pMeta->startInfo.startAllTasks) { - tqTrace("vgId:%d in restart procedure, not scan wal", vgId); - streamMetaWUnLock(pMeta); - return 0; - } - - pMeta->scanInfo.scanCounter += 1; - if (pMeta->scanInfo.scanCounter > MAX_REPEAT_SCAN_THRESHOLD) { - pMeta->scanInfo.scanCounter = MAX_REPEAT_SCAN_THRESHOLD; - } - - if (pMeta->scanInfo.scanCounter > 1) { - tqDebug("vgId:%d wal read task has been launched, remain scan times:%d", vgId, pMeta->scanInfo.scanCounter); - streamMetaWUnLock(pMeta); - return 0; - } - - int32_t numOfPauseTasks = pMeta->numOfPausedTasks; - if (ckPause && numOfTasks == numOfPauseTasks) { - tqDebug("vgId:%d ignore all submit, all streams had been paused, reset the walScanCounter", vgId); - - // reset the counter value, since we do not launch the scan wal operation. - pMeta->scanInfo.scanCounter = 0; - streamMetaWUnLock(pMeta); - return 0; - } - - tqDebug("vgId:%d create msg to start wal scan to launch stream tasks, numOfTasks:%d, vnd restored:%d", vgId, - numOfTasks, alreadyRestored); - - int32_t code = streamTaskSchedTask(&pTq->pVnode->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA); + code = doScanWalAsync(pTq, ckPause); streamMetaWUnLock(pMeta); - return code; } @@ -368,11 +328,8 @@ int32_t doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfIt return code; } -int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { - *pScanIdle = true; - bool noDataInWal = true; +int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta) { int32_t vgId = pStreamMeta->vgId; - int32_t numOfTasks = taosArrayGetSize(pStreamMeta->pTaskList); if (numOfTasks == 0) { return TSDB_CODE_SUCCESS; @@ -410,8 +367,6 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { continue; } - *pScanIdle = false; - // seek the stored version and extract data from WAL code = setWalReaderStartOffset(pTask, vgId); if (code != TSDB_CODE_SUCCESS) { @@ -437,7 +392,6 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { streamMutexUnlock(&pTask->lock); if ((numOfItems > 0) || hasNewData) { - noDataInWal = false; code = streamTrySchedExec(pTask); if (code != TSDB_CODE_SUCCESS) { streamMetaReleaseTask(pStreamMeta, pTask); @@ -449,11 +403,47 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { streamMetaReleaseTask(pStreamMeta, pTask); } - // all wal are checked, and no new data available in wal. - if (noDataInWal) { - *pScanIdle = true; - } - taosArrayDestroy(pTaskList); return TSDB_CODE_SUCCESS; } + +int32_t doScanWalAsync(STQ* pTq, bool ckPause) { + SStreamMeta* pMeta = pTq->pStreamMeta; + bool alreadyRestored = pTq->pVnode->restored; + int32_t vgId = pMeta->vgId; + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + + if (numOfTasks == 0) { + tqDebug("vgId:%d no stream tasks existed to run", vgId); + return 0; + } + + if (pMeta->startInfo.startAllTasks) { + tqTrace("vgId:%d in restart procedure, not scan wal", vgId); + return 0; + } + + pMeta->scanInfo.scanCounter += 1; + if (pMeta->scanInfo.scanCounter > MAX_REPEAT_SCAN_THRESHOLD) { + pMeta->scanInfo.scanCounter = MAX_REPEAT_SCAN_THRESHOLD; + } + + if (pMeta->scanInfo.scanCounter > 1) { + tqDebug("vgId:%d wal read task has been launched, remain scan times:%d", vgId, pMeta->scanInfo.scanCounter); + return 0; + } + + int32_t numOfPauseTasks = pMeta->numOfPausedTasks; + if (ckPause && numOfTasks == numOfPauseTasks) { + tqDebug("vgId:%d ignore all submit, all streams had been paused, reset the walScanCounter", vgId); + + // reset the counter value, since we do not launch the scan wal operation. + pMeta->scanInfo.scanCounter = 0; + return 0; + } + + tqDebug("vgId:%d create msg to start wal scan to launch stream tasks, numOfTasks:%d, vnd restored:%d", vgId, + numOfTasks, alreadyRestored); + + return streamTaskSchedTask(&pTq->pVnode->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA); +} diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 6e27677b54..06b7b33cd8 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -718,8 +718,6 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen } } - streamMetaWUnLock(pMeta); - // drop the related fill-history task firstly if (hTaskId.taskId != 0 && hTaskId.streamId != 0) { tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId); @@ -737,7 +735,6 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen } // commit the update - streamMetaWLock(pMeta); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); tqDebug("vgId:%d task:0x%x dropped, remain tasks:%d", vgId, pReq->taskId, numOfTasks); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 328c2c1585..d15bc68832 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -977,6 +977,11 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { extern int32_t vnodeAsyncRetention(SVnode *pVnode, int64_t now); static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { + if (!pVnode->restored) { + vInfo("vgId:%d, ignore trim req during restoring. ver:%" PRId64, TD_VID(pVnode), ver); + return 0; + } + int32_t code = 0; SVTrimDbReq trimReq = {0}; diff --git a/source/libs/executor/src/streamfilloperator.c b/source/libs/executor/src/streamfilloperator.c index 846d064fa8..ffaa62721e 100644 --- a/source/libs/executor/src/streamfilloperator.c +++ b/source/libs/executor/src/streamfilloperator.c @@ -987,7 +987,14 @@ _end: } void resetStreamFillSup(SStreamFillSupporter* pFillSup) { - tSimpleHashClear(pFillSup->pResMap); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + SSHashObj* pNewMap = tSimpleHashInit(16, hashFn); + if (pNewMap != NULL) { + tSimpleHashCleanup(pFillSup->pResMap); + pFillSup->pResMap = pNewMap; + } else { + tSimpleHashClear(pFillSup->pResMap); + } pFillSup->hasDelete = false; } void resetStreamFillInfo(SStreamFillOperatorInfo* pInfo) { @@ -1406,6 +1413,7 @@ static int32_t doStreamForceFillNext(SOperatorInfo* pOperator, SSDataBlock** ppR } pInfo->stateStore.streamStateClearExpiredState(pInfo->pState); + resetStreamFillInfo(pInfo); setStreamOperatorCompleted(pOperator); (*ppRes) = NULL; goto _end; @@ -1477,6 +1485,7 @@ static int32_t doStreamForceFillNext(SOperatorInfo* pOperator, SSDataBlock** ppR if ((*ppRes) == NULL) { pInfo->stateStore.streamStateClearExpiredState(pInfo->pState); + resetStreamFillInfo(pInfo); setStreamOperatorCompleted(pOperator); } diff --git a/source/libs/executor/src/streamintervalsliceoperator.c b/source/libs/executor/src/streamintervalsliceoperator.c index cd02f07143..d038e4d82c 100644 --- a/source/libs/executor/src/streamintervalsliceoperator.c +++ b/source/libs/executor/src/streamintervalsliceoperator.c @@ -120,6 +120,36 @@ void initIntervalSlicePoint(SStreamAggSupporter* pAggSup, STimeWindow* pTWin, in pPoint->pLastRow = POINTER_SHIFT(pPoint->pFinished, sizeof(bool)); } +int32_t getIntervalSlicePrevStateBuf(SStreamAggSupporter* pAggSup, SInterval* pInterval, SWinKey* pCurKey, + SInervalSlicePoint* pPrevPoint) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SWinKey prevKey = {.groupId = pCurKey->groupId}; + SET_WIN_KEY_INVALID(prevKey.ts); + int32_t prevVLen = 0; + int32_t prevWinCode = TSDB_CODE_SUCCESS; + code = pAggSup->stateStore.streamStateGetPrev(pAggSup->pState, pCurKey, &prevKey, (void**)&pPrevPoint->pResPos, + &prevVLen, &prevWinCode); + QUERY_CHECK_CODE(code, lino, _end); + + if (prevWinCode == TSDB_CODE_SUCCESS) { + STimeWindow prevSTW = {.skey = prevKey.ts}; + prevSTW.ekey = taosTimeGetIntervalEnd(prevSTW.skey, pInterval); + initIntervalSlicePoint(pAggSup, &prevSTW, pCurKey->groupId, pPrevPoint); + qDebug("===stream=== set stream twa prev point buf.ts:%" PRId64 ", groupId:%" PRIu64 ", res:%d", + pPrevPoint->winKey.win.skey, pPrevPoint->winKey.groupId, prevWinCode); + } else { + SET_WIN_KEY_INVALID(pPrevPoint->winKey.win.skey); + SET_WIN_KEY_INVALID(pPrevPoint->winKey.win.ekey); + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; +} + static int32_t getIntervalSliceCurStateBuf(SStreamAggSupporter* pAggSup, SInterval* pInterval, bool needPrev, STimeWindow* pTWin, int64_t groupId, SInervalSlicePoint* pCurPoint, SInervalSlicePoint* pPrevPoint, int32_t* pWinCode) { int32_t code = TSDB_CODE_SUCCESS; @@ -136,24 +166,8 @@ static int32_t getIntervalSliceCurStateBuf(SStreamAggSupporter* pAggSup, SInterv initIntervalSlicePoint(pAggSup, pTWin, groupId, pCurPoint); if (needPrev) { - SWinKey prevKey = {.groupId = groupId}; - SET_WIN_KEY_INVALID(prevKey.ts); - int32_t prevVLen = 0; - int32_t prevWinCode = TSDB_CODE_SUCCESS; - code = pAggSup->stateStore.streamStateGetPrev(pAggSup->pState, &curKey, &prevKey, (void**)&pPrevPoint->pResPos, - &prevVLen, &prevWinCode); + code = getIntervalSlicePrevStateBuf(pAggSup, pInterval, &curKey, pPrevPoint); QUERY_CHECK_CODE(code, lino, _end); - - if (prevWinCode == TSDB_CODE_SUCCESS) { - STimeWindow prevSTW = {.skey = prevKey.ts}; - prevSTW.ekey = taosTimeGetIntervalEnd(prevSTW.skey, pInterval); - initIntervalSlicePoint(pAggSup, &prevSTW, groupId, pPrevPoint); - qDebug("===stream=== set stream twa prev point buf.ts:%" PRId64 ", groupId:%" PRIu64 ", res:%d", pPrevPoint->winKey.win.skey, - pPrevPoint->winKey.groupId, prevWinCode); - } else { - SET_WIN_KEY_INVALID(pPrevPoint->winKey.win.skey); - SET_WIN_KEY_INVALID(pPrevPoint->winKey.win.ekey); - } } _end: @@ -265,13 +279,15 @@ static int32_t doStreamIntervalSliceAggImpl(SOperatorInfo* pOperator, SSDataBloc STimeWindow curWin = getActiveTimeWindow(NULL, pResultRowInfo, curTs, &pInfo->interval, TSDB_ORDER_ASC); while (1) { - if (curTs > pInfo->endTs) { - break; - } - int32_t winCode = TSDB_CODE_SUCCESS; - code = getIntervalSliceCurStateBuf(&pInfo->streamAggSup, &pInfo->interval, pInfo->hasInterpoFunc, &curWin, groupId, &curPoint, &prevPoint, &winCode); - QUERY_CHECK_CODE(code, lino, _end); + if (curTs <= pInfo->endTs) { + code = getIntervalSliceCurStateBuf(&pInfo->streamAggSup, &pInfo->interval, pInfo->hasInterpoFunc, &curWin, groupId, &curPoint, &prevPoint, &winCode); + QUERY_CHECK_CODE(code, lino, _end); + } else if (pInfo->hasInterpoFunc) { + SWinKey curKey = {.ts = curWin.skey, .groupId = groupId}; + code = getIntervalSlicePrevStateBuf(&pInfo->streamAggSup, &pInfo->interval, &curKey, &prevPoint); + QUERY_CHECK_CODE(code, lino, _end); + } if (pInfo->hasInterpoFunc && IS_VALID_WIN_KEY(prevPoint.winKey.win.skey) && isInterpoWindowFinished(&prevPoint) == false) { code = setIntervalSliceOutputBuf(&prevPoint, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); @@ -288,6 +304,12 @@ static int32_t doStreamIntervalSliceAggImpl(SOperatorInfo* pOperator, SSDataBloc code = saveWinResult(&prevKey, prevPoint.pResPos, pInfo->pUpdatedMap); QUERY_CHECK_CODE(code, lino, _end); setInterpoWindowFinished(&prevPoint); + } else if (IS_VALID_WIN_KEY(prevPoint.winKey.win.skey)) { + releaseOutputBuf(pInfo->streamAggSup.pState, prevPoint.pResPos, &pInfo->streamAggSup.stateStore); + } + + if (curTs > pInfo->endTs) { + break; } code = setIntervalSliceOutputBuf(&curPoint, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); @@ -300,7 +322,7 @@ static int32_t doStreamIntervalSliceAggImpl(SOperatorInfo* pOperator, SSDataBloc forwardRows = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, curWin.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); int32_t prevEndPos = (forwardRows - 1) + startPos; - if (pInfo->hasInterpoFunc && winCode != TSDB_CODE_SUCCESS) { + if (pInfo->hasInterpoFunc) { int32_t endRowId = getQualifiedRowNumDesc(pSup, pBlock, tsCols, prevEndPos, false); TSKEY endRowTs = tsCols[endRowId]; transBlockToSliceResultRow(pBlock, endRowId, endRowTs, curPoint.pLastRow, 0, NULL, NULL, pInfo->pOffsetInfo); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 1839e664b3..95a332ac05 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -4952,7 +4952,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .paramInfoPattern = 0, .outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_ALL_TYPE}}, .translateFunc = translateSelectValue, - .getEnvFunc = getSelectivityFuncEnv, + .getEnvFunc = getGroupKeyFuncEnv, .initFunc = functionSetup, .processFunc = groupConstValueFunction, .finalizeFunc = groupConstValueFinalize, diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 6ead98df0f..27c642470d 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -26,9 +26,9 @@ (pDst)->fldname = (pSrc)->fldname; \ } while (0) -#define COPY_CHAR_ARRAY_FIELD(fldname) \ - do { \ - strcpy((pDst)->fldname, (pSrc)->fldname); \ +#define COPY_CHAR_ARRAY_FIELD(fldname) \ + do { \ + tstrncpy((pDst)->fldname, (pSrc)->fldname, sizeof((pDst)->fldname)); \ } while (0) #define COPY_OBJECT_FIELD(fldname, size) \ diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 6dd55412d6..847b921ddf 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -2341,7 +2341,7 @@ char* nodesGetStrValueFromNode(SValueNode* pNode) { return NULL; } - sprintf(buf, "%s", pNode->datum.b ? "true" : "false"); + snprintf(buf, MAX_NUM_STR_SIZE, "%s", pNode->datum.b ? "true" : "false"); return buf; } case TSDB_DATA_TYPE_TINYINT: @@ -2354,7 +2354,7 @@ char* nodesGetStrValueFromNode(SValueNode* pNode) { return NULL; } - sprintf(buf, "%" PRId64, pNode->datum.i); + snprintf(buf, MAX_NUM_STR_SIZE, "%" PRId64, pNode->datum.i); return buf; } case TSDB_DATA_TYPE_UTINYINT: @@ -2366,7 +2366,7 @@ char* nodesGetStrValueFromNode(SValueNode* pNode) { return NULL; } - sprintf(buf, "%" PRIu64, pNode->datum.u); + snprintf(buf, MAX_NUM_STR_SIZE, "%" PRIu64, pNode->datum.u); return buf; } case TSDB_DATA_TYPE_FLOAT: @@ -2376,7 +2376,7 @@ char* nodesGetStrValueFromNode(SValueNode* pNode) { return NULL; } - sprintf(buf, "%e", pNode->datum.d); + snprintf(buf, MAX_NUM_STR_SIZE, "%e", pNode->datum.d); return buf; } case TSDB_DATA_TYPE_NCHAR: @@ -2532,7 +2532,7 @@ static EDealRes doCollect(SCollectColumnsCxt* pCxt, SColumnNode* pCol, SNode* pN } if (pCol->projRefIdx > 0) { len = taosHashBinary(name, strlen(name)); - len += sprintf(name + len, "_%d", pCol->projRefIdx); + len += tsnprintf(name + len, TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN - len, "_%d", pCol->projRefIdx); } SNode** pNodeFound = taosHashGet(pCxt->pColHash, name, len); if (pNodeFound == NULL) { @@ -2952,17 +2952,16 @@ int32_t nodesValueNodeToVariant(const SValueNode* pNode, SVariant* pVal) { case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_GEOMETRY: - pVal->pz = taosMemoryMalloc(pVal->nLen + 1); + pVal->pz = taosMemoryCalloc(1, pVal->nLen + 1); if (pVal->pz) { - memcpy(pVal->pz, pNode->datum.p, pVal->nLen); - pVal->pz[pVal->nLen] = 0; + memcpy(pVal->pz, pNode->datum.p, varDataTLen(pNode->datum.p)); } else { code = terrno; } break; case TSDB_DATA_TYPE_JSON: pVal->nLen = getJsonValueLen(pNode->datum.p); - pVal->pz = taosMemoryMalloc(pVal->nLen); + pVal->pz = taosMemoryCalloc(1, pVal->nLen); if (pVal->pz) { memcpy(pVal->pz, pNode->datum.p, pVal->nLen); } else { diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 022dcea462..5ecef23de7 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -142,13 +142,13 @@ static int32_t parseEndpoint(SAstCreateContext* pCxt, const SToken* pEp, char* p (void)strdequote(ep); (void)strtrim(ep); if (NULL == pPort) { - strcpy(pFqdn, ep); + tstrncpy(pFqdn, ep, TSDB_FQDN_LEN); return TSDB_CODE_SUCCESS; } char* pColon = strchr(ep, ':'); if (NULL == pColon) { *pPort = tsServerPort; - strcpy(pFqdn, ep); + tstrncpy(pFqdn, ep, TSDB_FQDN_LEN); return TSDB_CODE_SUCCESS; } strncpy(pFqdn, ep, pColon - ep); @@ -320,12 +320,12 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { if (nodesIsExprNode(pRealizedExpr)) { SExprNode* pExpr = (SExprNode*)pRealizedExpr; if (QUERY_NODE_COLUMN == nodeType(pExpr)) { - strcpy(pExpr->aliasName, ((SColumnNode*)pExpr)->colName); - strcpy(pExpr->userAlias, ((SColumnNode*)pExpr)->colName); + tstrncpy(pExpr->aliasName, ((SColumnNode*)pExpr)->colName, TSDB_COL_NAME_LEN); + tstrncpy(pExpr->userAlias, ((SColumnNode*)pExpr)->colName, TSDB_COL_NAME_LEN); } else if (pRawExpr->isPseudoColumn) { // all pseudo column are translate to function with same name - strcpy(pExpr->userAlias, ((SFunctionNode*)pExpr)->functionName); - strcpy(pExpr->aliasName, ((SFunctionNode*)pExpr)->functionName); + tstrncpy(pExpr->userAlias, ((SFunctionNode*)pExpr)->functionName, TSDB_COL_NAME_LEN); + tstrncpy(pExpr->aliasName, ((SFunctionNode*)pExpr)->functionName, TSDB_COL_NAME_LEN); } else { int32_t len = TMIN(sizeof(pExpr->aliasName) - 1, pRawExpr->n); @@ -333,9 +333,9 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { // Len of pRawExpr->p could be larger than len of aliasName[TSDB_COL_NAME_LEN]. // If aliasName is truncated, hash value of aliasName could be the same. uint64_t hashVal = MurmurHash3_64(pRawExpr->p, pRawExpr->n); - sprintf(pExpr->aliasName, "%" PRIu64, hashVal); + snprintf(pExpr->aliasName, TSDB_COL_NAME_LEN, "%" PRIu64, hashVal); strncpy(pExpr->userAlias, pRawExpr->p, len); - pExpr->userAlias[len] = '\0'; + pExpr->userAlias[len] = 0; } } pRawExpr->pNode = NULL; @@ -948,11 +948,11 @@ SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pL goto _err; } if ('+' == pVal->literal[0]) { - sprintf(pNewLiteral, "-%s", pVal->literal + 1); + snprintf(pNewLiteral, strlen(pVal->literal) + 2, "-%s", pVal->literal + 1); } else if ('-' == pVal->literal[0]) { - sprintf(pNewLiteral, "%s", pVal->literal + 1); + snprintf(pNewLiteral, strlen(pVal->literal) + 2, "%s", pVal->literal + 1); } else { - sprintf(pNewLiteral, "-%s", pVal->literal); + snprintf(pNewLiteral, strlen(pVal->literal) + 2, "-%s", pVal->literal); } taosMemoryFree(pVal->literal); pVal->literal = pNewLiteral; @@ -1025,7 +1025,7 @@ static SNode* createPrimaryKeyCol(SAstCreateContext* pCxt, const SToken* pFuncNa CHECK_MAKE_NODE(pCol); pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; if (NULL == pFuncName) { - strcpy(pCol->colName, ROWTS_PSEUDO_COLUMN_NAME); + tstrncpy(pCol->colName, ROWTS_PSEUDO_COLUMN_NAME, TSDB_COL_NAME_LEN); } else { strncpy(pCol->colName, pFuncName->z, pFuncName->n); } @@ -1058,7 +1058,7 @@ SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType d CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&func); CHECK_MAKE_NODE(func); - strcpy(func->functionName, "cast"); + tstrncpy(func->functionName, "cast", TSDB_FUNC_NAME_LEN); func->node.resType = dt; if (TSDB_DATA_TYPE_VARCHAR == dt.type || TSDB_DATA_TYPE_GEOMETRY == dt.type || TSDB_DATA_TYPE_VARBINARY == dt.type) { func->node.resType.bytes = func->node.resType.bytes + VARSTR_HEADER_SIZE; @@ -1082,7 +1082,7 @@ SNode* createPositionFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SNode* CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&func); CHECK_MAKE_NODE(func); - strcpy(func->functionName, "position"); + tstrncpy(func->functionName, "position", TSDB_FUNC_NAME_LEN); pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr); CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr2); @@ -1100,7 +1100,7 @@ SNode* createTrimFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, ETrimType t CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&func); CHECK_MAKE_NODE(func); - strcpy(func->functionName, "trim"); + tstrncpy(func->functionName, "trim", TSDB_FUNC_NAME_LEN); func->trimType = type; pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr); CHECK_PARSER_STATUS(pCxt); @@ -1117,7 +1117,7 @@ SNode* createTrimFunctionNodeExt(SAstCreateContext* pCxt, SNode* pExpr, SNode* p CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&func); CHECK_MAKE_NODE(func); - strcpy(func->functionName, "trim"); + tstrncpy(func->functionName, "trim", TSDB_FUNC_NAME_LEN); func->trimType = type; pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr); CHECK_PARSER_STATUS(pCxt); @@ -1137,7 +1137,7 @@ SNode* createSubstrFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SNode* pE CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&func); CHECK_MAKE_NODE(func); - strcpy(func->functionName, "substr"); + tstrncpy(func->functionName, "substr", TSDB_FUNC_NAME_LEN); pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr); CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr2); @@ -1155,7 +1155,7 @@ SNode* createSubstrFunctionNodeExt(SAstCreateContext* pCxt, SNode* pExpr, SNode* CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&func); CHECK_MAKE_NODE(func); - strcpy(func->functionName, "substr"); + tstrncpy(func->functionName, "substr", TSDB_FUNC_NAME_LEN); pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr); CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr2); @@ -1239,10 +1239,10 @@ SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, SToken* pT taosRandStr(tempTable->table.tableAlias, 8); } if (QUERY_NODE_SELECT_STMT == nodeType(pSubquery)) { - strcpy(((SSelectStmt*)pSubquery)->stmtName, tempTable->table.tableAlias); + tstrncpy(((SSelectStmt*)pSubquery)->stmtName, tempTable->table.tableAlias, TSDB_TABLE_NAME_LEN); ((SSelectStmt*)pSubquery)->isSubquery = true; } else if (QUERY_NODE_SET_OPERATOR == nodeType(pSubquery)) { - strcpy(((SSetOperator*)pSubquery)->stmtName, tempTable->table.tableAlias); + tstrncpy(((SSetOperator*)pSubquery)->stmtName, tempTable->table.tableAlias, TSDB_TABLE_NAME_LEN); } return (SNode*)tempTable; _err: @@ -1447,7 +1447,7 @@ SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues) { fill->pWStartTs = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&(fill->pWStartTs)); CHECK_MAKE_NODE(fill->pWStartTs); - strcpy(((SFunctionNode*)fill->pWStartTs)->functionName, "_wstart"); + tstrncpy(((SFunctionNode*)fill->pWStartTs)->functionName, "_wstart", TSDB_FUNC_NAME_LEN); return (SNode*)fill; _err: nodesDestroyNode((SNode*)fill); @@ -1779,7 +1779,7 @@ SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* setSubquery(setOp->pLeft); setOp->pRight = pRight; setSubquery(setOp->pRight); - sprintf(setOp->stmtName, "%p", setOp); + snprintf(setOp->stmtName, TSDB_TABLE_NAME_LEN, "%p", setOp); return (SNode*)setOp; _err: nodesDestroyNode(pLeft); @@ -2337,8 +2337,8 @@ SNode* createCreateTableStmt(SAstCreateContext* pCxt, bool ignoreExists, SNode* SCreateTableStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_CREATE_TABLE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); + tstrncpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); pStmt->ignoreExists = ignoreExists; pStmt->pCols = pCols; pStmt->pTags = pTags; @@ -2359,10 +2359,10 @@ SNode* createCreateSubTableClause(SAstCreateContext* pCxt, bool ignoreExists, SN SCreateSubTableClause* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_CREATE_SUBTABLE_CLAUSE, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); - strcpy(pStmt->useDbName, ((SRealTableNode*)pUseRealTable)->table.dbName); - strcpy(pStmt->useTableName, ((SRealTableNode*)pUseRealTable)->table.tableName); + tstrncpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); + tstrncpy(pStmt->useDbName, ((SRealTableNode*)pUseRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->useTableName, ((SRealTableNode*)pUseRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); pStmt->ignoreExists = ignoreExists; pStmt->pSpecificTags = pSpecificTags; pStmt->pValsOfTags = pValsOfTags; @@ -2385,8 +2385,8 @@ SNode* createCreateSubTableFromFileClause(SAstCreateContext* pCxt, bool ignoreEx SCreateSubTableFromFileClause* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_CREATE_SUBTABLE_FROM_FILE_CLAUSE, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); - strcpy(pStmt->useDbName, ((SRealTableNode*)pUseRealTable)->table.dbName); - strcpy(pStmt->useTableName, ((SRealTableNode*)pUseRealTable)->table.tableName); + tstrncpy(pStmt->useDbName, ((SRealTableNode*)pUseRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->useTableName, ((SRealTableNode*)pUseRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); pStmt->ignoreExists = ignoreExists; pStmt->pSpecificTags = pSpecificTags; if (TK_NK_STRING == pFilePath->type) { @@ -2420,8 +2420,8 @@ SNode* createDropTableClause(SAstCreateContext* pCxt, bool ignoreNotExists, SNod SDropTableClause* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_DROP_TABLE_CLAUSE, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); + tstrncpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); pStmt->ignoreNotExists = ignoreNotExists; nodesDestroyNode(pRealTable); return (SNode*)pStmt; @@ -2448,8 +2448,8 @@ SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool withOpt, bool igno SDropSuperTableStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_DROP_SUPER_TABLE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); + tstrncpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); pStmt->ignoreNotExists = ignoreNotExists; pStmt->withOpt = withOpt; nodesDestroyNode(pRealTable); @@ -2460,8 +2460,8 @@ _err: } static SNode* createAlterTableStmtFinalize(SNode* pRealTable, SAlterTableStmt* pStmt) { - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); + tstrncpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); nodesDestroyNode(pRealTable); return (SNode*)pStmt; } @@ -2780,8 +2780,8 @@ SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* SShowCreateTableStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(type, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); + tstrncpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); nodesDestroyNode(pRealTable); return (SNode*)pStmt; _err: @@ -2794,8 +2794,8 @@ SNode* createShowCreateViewStmt(SAstCreateContext* pCxt, ENodeType type, SNode* SShowCreateViewStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(type, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->viewName, ((SRealTableNode*)pRealTable)->table.tableName); + tstrncpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->viewName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); nodesDestroyNode(pRealTable); return (SNode*)pStmt; _err: @@ -2808,8 +2808,8 @@ SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable SShowTableDistributedStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); + tstrncpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); nodesDestroyNode(pRealTable); return (SNode*)pStmt; _err: @@ -2965,7 +2965,7 @@ SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const ST pCxt->errCode = nodesMakeNode(QUERY_NODE_CREATE_USER_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName); - strcpy(pStmt->password, password); + tstrncpy(pStmt->password, password, TSDB_USET_PASSWORD_LEN); pStmt->sysinfo = sysinfo; pStmt->createDb = createDb; pStmt->isImport = is_import; @@ -2987,7 +2987,7 @@ SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t al char password[TSDB_USET_PASSWORD_LEN] = {0}; SToken* pVal = pAlterInfo; CHECK_NAME(checkPassword(pCxt, pVal, password)); - strcpy(pStmt->password, password); + tstrncpy(pStmt->password, password, TSDB_USET_PASSWORD_LEN); break; } case TSDB_ALTER_USER_ENABLE: { @@ -3297,8 +3297,8 @@ SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, pStmt->withMeta = withMeta; pStmt->pWhere = pWhere; - strcpy(pStmt->subDbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->subSTbName, ((SRealTableNode*)pRealTable)->table.tableName); + tstrncpy(pStmt->subDbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->subSTbName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); nodesDestroyNode(pRealTable); return (SNode*)pStmt; _err: @@ -3411,8 +3411,8 @@ SNode* createDescribeStmt(SAstCreateContext* pCxt, SNode* pRealTable) { SDescribeStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_DESCRIBE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); + tstrncpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); nodesDestroyNode(pRealTable); return (SNode*)pStmt; _err: @@ -3492,8 +3492,8 @@ SNode* createCreateViewStmt(SAstCreateContext* pCxt, bool orReplace, SNode* pVie } pStmt->pQuerySql = tstrdup(pAs->z + i); CHECK_OUT_OF_MEM(pStmt->pQuerySql); - strcpy(pStmt->dbName, ((SViewNode*)pView)->table.dbName); - strcpy(pStmt->viewName, ((SViewNode*)pView)->table.tableName); + tstrncpy(pStmt->dbName, ((SViewNode*)pView)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->viewName, ((SViewNode*)pView)->table.tableName, TSDB_VIEW_NAME_LEN); nodesDestroyNode(pView); pStmt->orReplace = orReplace; pStmt->pQuery = pQuery; @@ -3511,8 +3511,8 @@ SNode* createDropViewStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pCxt->errCode = nodesMakeNode(QUERY_NODE_DROP_VIEW_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); pStmt->ignoreNotExists = ignoreNotExists; - strcpy(pStmt->dbName, ((SViewNode*)pView)->table.dbName); - strcpy(pStmt->viewName, ((SViewNode*)pView)->table.tableName); + tstrncpy(pStmt->dbName, ((SViewNode*)pView)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->viewName, ((SViewNode*)pView)->table.tableName, TSDB_VIEW_NAME_LEN); nodesDestroyNode(pView); return (SNode*)pStmt; _err: @@ -3597,8 +3597,8 @@ SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken pCxt->errCode = nodesMakeNode(QUERY_NODE_CREATE_STREAM_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->streamName, pStreamName); - strcpy(pStmt->targetDbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->targetTabName, ((SRealTableNode*)pRealTable)->table.tableName); + tstrncpy(pStmt->targetDbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->targetTabName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); nodesDestroyNode(pRealTable); pStmt->ignoreExists = ignoreExists; pStmt->pOptions = (SStreamOptions*)pOptions; @@ -3852,9 +3852,9 @@ SNode* createInsertStmt(SAstCreateContext* pCxt, SNode* pTable, SNodeList* pCols pStmt->pCols = pCols; pStmt->pQuery = pQuery; if (QUERY_NODE_SELECT_STMT == nodeType(pQuery)) { - strcpy(((SSelectStmt*)pQuery)->stmtName, ((STableNode*)pTable)->tableAlias); + tstrncpy(((SSelectStmt*)pQuery)->stmtName, ((STableNode*)pTable)->tableAlias, TSDB_TABLE_NAME_LEN); } else if (QUERY_NODE_SET_OPERATOR == nodeType(pQuery)) { - strcpy(((SSetOperator*)pQuery)->stmtName, ((STableNode*)pTable)->tableAlias); + tstrncpy(((SSetOperator*)pQuery)->stmtName, ((STableNode*)pTable)->tableAlias, TSDB_TABLE_NAME_LEN); } return (SNode*)pStmt; _err: diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 45a7876819..3aa60c283e 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -480,8 +480,8 @@ static int32_t collectMetaKeyFromExplain(SCollectMetaKeyCxt* pCxt, SExplainStmt* static int32_t collectMetaKeyFromDescribe(SCollectMetaKeyCxt* pCxt, SDescribeStmt* pStmt) { SName name = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId}; - strcpy(name.dbname, pStmt->dbName); - strcpy(name.tname, pStmt->tableName); + tstrncpy(name.dbname, pStmt->dbName, TSDB_DB_NAME_LEN); + tstrncpy(name.tname, pStmt->tableName, TSDB_TABLE_NAME_LEN); int32_t code = catalogRemoveTableMeta(pCxt->pParseCxt->pCatalog, &name); #ifdef TD_ENTERPRISE if (TSDB_CODE_SUCCESS == code) { @@ -781,8 +781,8 @@ static int32_t collectMetaKeyFromShowCreateDatabase(SCollectMetaKeyCxt* pCxt, SS static int32_t collectMetaKeyFromShowCreateTable(SCollectMetaKeyCxt* pCxt, SShowCreateTableStmt* pStmt) { SName name = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId}; - strcpy(name.dbname, pStmt->dbName); - strcpy(name.tname, pStmt->tableName); + tstrncpy(name.dbname, pStmt->dbName, TSDB_DB_NAME_LEN); + tstrncpy(name.tname, pStmt->tableName, TSDB_TABLE_NAME_LEN); int32_t code = catalogRemoveTableMeta(pCxt->pParseCxt->pCatalog, &name); if (TSDB_CODE_SUCCESS == code) { code = reserveTableCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache); @@ -799,8 +799,8 @@ static int32_t collectMetaKeyFromShowCreateTable(SCollectMetaKeyCxt* pCxt, SShow static int32_t collectMetaKeyFromShowCreateView(SCollectMetaKeyCxt* pCxt, SShowCreateViewStmt* pStmt) { SName name = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId}; - strcpy(name.dbname, pStmt->dbName); - strcpy(name.tname, pStmt->viewName); + tstrncpy(name.dbname, pStmt->dbName, TSDB_DB_NAME_LEN); + tstrncpy(name.tname, pStmt->viewName, TSDB_TABLE_NAME_LEN); char dbFName[TSDB_DB_FNAME_LEN]; (void)tNameGetFullDbName(&name, dbFName); int32_t code = catalogRemoveViewMeta(pCxt->pParseCxt->pCatalog, dbFName, 0, pStmt->viewName, 0); @@ -841,8 +841,8 @@ static int32_t collectMetaKeyFromInsert(SCollectMetaKeyCxt* pCxt, SInsertStmt* p static int32_t collectMetaKeyFromShowBlockDist(SCollectMetaKeyCxt* pCxt, SShowTableDistributedStmt* pStmt) { SName name = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId}; - strcpy(name.dbname, pStmt->dbName); - strcpy(name.tname, pStmt->tableName); + tstrncpy(name.dbname, pStmt->dbName, TSDB_DB_NAME_LEN); + tstrncpy(name.tname, pStmt->tableName, TSDB_TABLE_NAME_LEN); int32_t code = catalogRemoveTableMeta(pCxt->pParseCxt->pCatalog, &name); if (TSDB_CODE_SUCCESS == code) { code = collectMetaKeyFromRealTableImpl(pCxt, pStmt->dbName, pStmt->tableName, AUTH_TYPE_READ); diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index c8dfc87602..87ad0d62b7 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -134,8 +134,8 @@ EDealRes rewriteAuthTable(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { SColumnNode* pCol = (SColumnNode*)pNode; SAuthRewriteCxt* pCxt = (SAuthRewriteCxt*)pContext; - strcpy(pCol->tableName, pCxt->pTarget->tableName); - strcpy(pCol->tableAlias, pCxt->pTarget->tableAlias); + tstrncpy(pCol->tableName, pCxt->pTarget->tableName, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->tableAlias, pCxt->pTarget->tableAlias, TSDB_TABLE_NAME_LEN); } return DEAL_RES_CONTINUE; diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index 911cb2a915..99be9b07b0 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -178,14 +178,14 @@ static EDealRes doFindAndReplaceNode(SNode** pNode, void* pContext) { SCalcConstContext* pCxt = pContext; if (pCxt->replaceCxt.pTarget == *pNode) { char aliasName[TSDB_COL_NAME_LEN] = {0}; - strcpy(aliasName, ((SExprNode*)*pNode)->aliasName); + tstrncpy(aliasName, ((SExprNode*)*pNode)->aliasName, TSDB_COL_NAME_LEN); nodesDestroyNode(*pNode); *pNode = NULL; pCxt->code = nodesCloneNode(pCxt->replaceCxt.pNew, pNode); if (NULL == *pNode) { return DEAL_RES_ERROR; } - strcpy(((SExprNode*)*pNode)->aliasName, aliasName); + tstrncpy(((SExprNode*)*pNode)->aliasName, aliasName, TSDB_COL_NAME_LEN); pCxt->replaceCxt.replaced = true; return DEAL_RES_END; @@ -228,14 +228,14 @@ static int32_t calcConstProject(SCalcConstContext* pCxt, SNode* pProject, bool d SAssociationNode* pAssNode = taosArrayGet(pAssociation, i); SNode** pCol = pAssNode->pPlace; if (*pCol == pAssNode->pAssociationNode) { - strcpy(aliasName, ((SExprNode*)*pCol)->aliasName); + tstrncpy(aliasName, ((SExprNode*)*pCol)->aliasName, TSDB_COL_NAME_LEN); SArray* pOrigAss = NULL; TSWAP(((SExprNode*)*pCol)->pAssociation, pOrigAss); nodesDestroyNode(*pCol); *pCol = NULL; code = nodesCloneNode(*pNew, pCol); if (TSDB_CODE_SUCCESS == code) { - strcpy(((SExprNode*)*pCol)->aliasName, aliasName); + tstrncpy(((SExprNode*)*pCol)->aliasName, aliasName, TSDB_COL_NAME_LEN); TSWAP(pOrigAss, ((SExprNode*)*pCol)->pAssociation); } taosArrayDestroy(pOrigAss); diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 0b79741da1..9e6abe1517 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -927,7 +927,7 @@ int32_t buildBoundFields(int32_t numOfBound, int16_t* boundColumns, SSchema* pSc for (int32_t i = 0; i < numOfBound; ++i) { schema = &pSchema[boundColumns[i]]; - strcpy((*fields)[i].name, schema->name); + tstrncpy((*fields)[i].name, schema->name, 65); (*fields)[i].type = schema->type; (*fields)[i].bytes = schema->bytes; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 035d114252..a36d44db7a 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1305,15 +1305,15 @@ static bool hasPkInTable(const STableMeta* pTableMeta) { static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* pColSchema, int32_t tagFlag, SColumnNode* pCol) { - strcpy(pCol->dbName, pTable->table.dbName); - strcpy(pCol->tableAlias, pTable->table.tableAlias); - strcpy(pCol->tableName, pTable->table.tableName); - strcpy(pCol->colName, pColSchema->name); + tstrncpy(pCol->dbName, pTable->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pCol->tableAlias, pTable->table.tableAlias, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->tableName, pTable->table.tableName, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->colName, pColSchema->name, TSDB_COL_NAME_LEN); if ('\0' == pCol->node.aliasName[0]) { - strcpy(pCol->node.aliasName, pColSchema->name); + tstrncpy(pCol->node.aliasName, pColSchema->name, TSDB_COL_NAME_LEN); } if ('\0' == pCol->node.userAlias[0]) { - strcpy(pCol->node.userAlias, pColSchema->name); + tstrncpy(pCol->node.userAlias, pColSchema->name, TSDB_COL_NAME_LEN); } pCol->tableId = pTable->pMeta->uid; pCol->tableType = pTable->pMeta->tableType; @@ -1344,7 +1344,7 @@ static int32_t setColumnInfoByExpr(STempTableNode* pTable, SExprNode* pExpr, SCo return terrno; } - strcpy(pCol->tableAlias, pTable->table.tableAlias); + tstrncpy(pCol->tableAlias, pTable->table.tableAlias, TSDB_TABLE_NAME_LEN); pCol->isPrimTs = isPrimaryKeyImpl((SNode*)pExpr); pCol->colId = pCol->isPrimTs ? PRIMARYKEY_TIMESTAMP_COL_ID : 0; if (QUERY_NODE_COLUMN == nodeType(pExpr)) { @@ -1352,12 +1352,12 @@ static int32_t setColumnInfoByExpr(STempTableNode* pTable, SExprNode* pExpr, SCo // strcpy(pCol->dbName, ((SColumnNode*)pExpr)->dbName); // strcpy(pCol->tableName, ((SColumnNode*)pExpr)->tableName); } - strcpy(pCol->colName, pExpr->aliasName); + tstrncpy(pCol->colName, pExpr->aliasName, TSDB_COL_NAME_LEN); if ('\0' == pCol->node.aliasName[0]) { - strcpy(pCol->node.aliasName, pCol->colName); + tstrncpy(pCol->node.aliasName, pExpr->aliasName, TSDB_COL_NAME_LEN); } if ('\0' == pCol->node.userAlias[0]) { - strcpy(pCol->node.userAlias, pExpr->userAlias); + tstrncpy(pCol->node.userAlias, pExpr->aliasName, TSDB_COL_NAME_LEN); } pCol->node.resType = pExpr->resType; return TSDB_CODE_SUCCESS; @@ -1645,7 +1645,7 @@ static void biMakeAliasNameInMD5(char* pExprStr, int32_t len, char* pAlias) { tMD5Final(&ctx); char* p = pAlias; for (uint8_t i = 0; i < tListLen(ctx.digest); ++i) { - sprintf(p, "%02x", ctx.digest[i]); + snprintf(p, len + 1 - 2 * i, "%02x", ctx.digest[i]); p += 2; } } @@ -1684,7 +1684,7 @@ static int32_t biMakeTbnameProjectAstNode(char* funcName, char* tableAlias, SNod if (TSDB_CODE_SUCCESS == code) { snprintf(tbNameFunc->node.userAlias, sizeof(tbNameFunc->node.userAlias), (tableAlias) ? "%s.tbname" : "%stbname", (tableAlias) ? tableAlias : ""); - strncpy(tbNameFunc->node.aliasName, tbNameFunc->functionName, TSDB_COL_NAME_LEN); + tstrncpy(tbNameFunc->node.aliasName, tbNameFunc->functionName, TSDB_COL_NAME_LEN); if (funcName == NULL) { *pOutNode = (SNode*)tbNameFunc; return code; @@ -1702,7 +1702,7 @@ static int32_t biMakeTbnameProjectAstNode(char* funcName, char* tableAlias, SNod if (tsKeepColumnName) { snprintf(multiResFunc->node.userAlias, sizeof(tbNameFunc->node.userAlias), (tableAlias) ? "%s.tbname" : "%stbname", (tableAlias) ? tableAlias : ""); - strcpy(multiResFunc->node.aliasName, tbNameFunc->functionName); + tstrncpy(multiResFunc->node.aliasName, tbNameFunc->functionName, TSDB_COL_NAME_LEN); } else { snprintf(multiResFunc->node.userAlias, sizeof(multiResFunc->node.userAlias), tableAlias ? "%s(%s.tbname)" : "%s(%stbname)", funcName, tableAlias ? tableAlias : ""); @@ -1829,8 +1829,8 @@ int32_t biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode, bool* pRe (SNode**)&tbnameFuncNode); if (TSDB_CODE_SUCCESS != code) return code; tbnameFuncNode->node.resType = pCol->node.resType; - strcpy(tbnameFuncNode->node.aliasName, pCol->node.aliasName); - strcpy(tbnameFuncNode->node.userAlias, pCol->node.userAlias); + tstrncpy(tbnameFuncNode->node.aliasName, pCol->node.aliasName, TSDB_COL_NAME_LEN); + tstrncpy(tbnameFuncNode->node.userAlias, pCol->node.userAlias, TSDB_COL_NAME_LEN); nodesDestroyNode(*ppNode); *ppNode = (SNode*)tbnameFuncNode; @@ -2836,7 +2836,8 @@ static int32_t translateMultiResFunc(STranslateContext* pCxt, SFunctionNode* pFu } } if (tsKeepColumnName && 1 == LIST_LENGTH(pFunc->pParameterList) && !pFunc->node.asAlias && !pFunc->node.asParam) { - strcpy(pFunc->node.userAlias, ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->userAlias); + tstrncpy(pFunc->node.userAlias, ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->userAlias, + TSDB_COL_NAME_LEN); } return TSDB_CODE_SUCCESS; } @@ -2911,8 +2912,8 @@ static int32_t rewriteFuncToValue(STranslateContext* pCxt, char** pLiteral, SNod if (TSDB_CODE_SUCCESS != code) { return code; } - strcpy(pVal->node.aliasName, ((SExprNode*)*pNode)->aliasName); - strcpy(pVal->node.userAlias, ((SExprNode*)*pNode)->userAlias); + tstrncpy(pVal->node.aliasName, ((SExprNode*)*pNode)->aliasName, TSDB_COL_NAME_LEN); + tstrncpy(pVal->node.userAlias, ((SExprNode*)*pNode)->userAlias, TSDB_COL_NAME_LEN); pVal->node.resType = ((SExprNode*)*pNode)->resType; if (NULL == pLiteral || NULL == *pLiteral) { pVal->isNull = true; @@ -3027,10 +3028,10 @@ static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode if (0 != LIST_LENGTH(pFunc->pParameterList)) { SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); pCol->node.resType = pOldExpr->resType; - strcpy(pCol->tableAlias, pVal->literal); - strcpy(pCol->colName, pFunc->functionName); - strcpy(pCol->node.aliasName, pCol->colName); - strcpy(pCol->node.userAlias, pCol->colName); + tstrncpy(pCol->node.aliasName, pVal->literal, TSDB_COL_NAME_LEN); + tstrncpy(pCol->colName, pFunc->functionName, TSDB_COL_NAME_LEN); + tstrncpy(pCol->node.aliasName, pCol->colName, TSDB_COL_NAME_LEN); + tstrncpy(pCol->node.userAlias, pCol->colName, TSDB_COL_NAME_LEN); nodesDestroyNode(*ppNode); *ppNode = (SNode*)pCol; @@ -3038,9 +3039,9 @@ static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode } } pCol->node.resType = pOldExpr->resType; - strcpy(pCol->node.aliasName, pOldExpr->aliasName); - strcpy(pCol->node.userAlias, pOldExpr->userAlias); - strcpy(pCol->colName, pOldExpr->aliasName); + tstrncpy(pCol->node.aliasName, pOldExpr->aliasName, TSDB_COL_NAME_LEN); + tstrncpy(pCol->node.userAlias, pOldExpr->userAlias, TSDB_COL_NAME_LEN); + tstrncpy(pCol->colName, pOldExpr->aliasName, TSDB_COL_NAME_LEN); nodesDestroyNode(*ppNode); *ppNode = (SNode*)pCol; @@ -3299,7 +3300,7 @@ static int32_t createCastFunc(STranslateContext* pCxt, SNode* pExpr, SDataType d if (TSDB_CODE_SUCCESS != code) { return code; } - strcpy(pFunc->functionName, "cast"); + tstrncpy(pFunc->functionName, "cast", TSDB_FUNC_NAME_LEN); pFunc->node.resType = dt; if (TSDB_CODE_SUCCESS != nodesListMakeAppend(&pFunc->pParameterList, pExpr)) { nodesDestroyNode((SNode*)pFunc); @@ -3515,9 +3516,9 @@ static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode pCxt->errCode = code; return DEAL_RES_ERROR; } - strcpy(pFunc->functionName, "_select_value"); - strcpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName); - strcpy(pFunc->node.userAlias, ((SExprNode*)*pNode)->userAlias); + tstrncpy(pFunc->functionName, "_select_value", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName, TSDB_COL_NAME_LEN); + tstrncpy(pFunc->node.userAlias, ((SExprNode*)*pNode)->userAlias, TSDB_COL_NAME_LEN); pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode); if (TSDB_CODE_SUCCESS == pCxt->errCode) { pCxt->errCode = getFuncInfo(pCxt, pFunc); @@ -3539,9 +3540,9 @@ static EDealRes rewriteExprToGroupKeyFunc(STranslateContext* pCxt, SNode** pNode return DEAL_RES_ERROR; } - strcpy(pFunc->functionName, "_group_key"); - strcpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName); - strcpy(pFunc->node.userAlias, ((SExprNode*)*pNode)->userAlias); + tstrncpy(pFunc->functionName, "_group_key", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName, TSDB_COL_NAME_LEN); + tstrncpy(pFunc->node.userAlias, ((SExprNode*)*pNode)->userAlias, TSDB_COL_NAME_LEN); pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode); if (TSDB_CODE_SUCCESS == pCxt->errCode) { *pNode = (SNode*)pFunc; @@ -3558,9 +3559,9 @@ static EDealRes rewriteExprToSelectTagFunc(STranslateContext* pCxt, SNode** pNod return DEAL_RES_ERROR; } - strcpy(pFunc->functionName, "_group_const_value"); - strcpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName); - strcpy(pFunc->node.userAlias, ((SExprNode*)*pNode)->userAlias); + tstrncpy(pFunc->functionName, "_group_const_value", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName, TSDB_COL_NAME_LEN); + tstrncpy(pFunc->node.userAlias, ((SExprNode*)*pNode)->userAlias, TSDB_COL_NAME_LEN); pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode); if (TSDB_CODE_SUCCESS == pCxt->errCode) { *pNode = (SNode*)pFunc; @@ -4334,7 +4335,7 @@ static int32_t setTableTsmas(STranslateContext* pCxt, SName* pName, SRealTableNo int32_t len = tsnprintf(buf, TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN, "%s.%s_%s", pTsma->dbFName, pTsma->name, pRealTable->table.tableName); len = taosCreateMD5Hash(buf, len); - strncpy(tsmaTargetTbName.tname, buf, MD5_OUTPUT_LEN); + tstrncpy(tsmaTargetTbName.tname, buf, TSDB_TABLE_NAME_LEN); code = collectUseTable(&tsmaTargetTbName, pCxt->pTargetTables); if (TSDB_CODE_SUCCESS == code) code = catalogGetCachedTableHashVgroup(pCxt->pParseCxt->pCatalog, &tsmaTargetTbName, &vgInfo, &exists); @@ -4378,7 +4379,7 @@ static int32_t setTableTsmas(STranslateContext* pCxt, SName* pName, SRealTableNo } } if (code == TSDB_CODE_SUCCESS) { - sprintf(ctbInfo.tableName, "%s", tsmaTargetTbName.tname); + snprintf(ctbInfo.tableName, TSDB_TABLE_NAME_LEN, "%s", tsmaTargetTbName.tname); ctbInfo.uid = pTableMeta->uid; taosMemoryFree(pTableMeta); } else if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { @@ -4440,10 +4441,9 @@ static EDealRes doTranslateTbName(SNode** pNode, void* pContext) { return DEAL_RES_ERROR; } varDataSetLen(pVal->datum.p, tbLen); - strncpy(varDataVal(pVal->datum.p), pVal->literal, tbLen); - strcpy(pVal->node.userAlias, pFunc->node.userAlias); - strcpy(pVal->node.aliasName, pFunc->node.aliasName); - + tstrncpy(varDataVal(pVal->datum.p), pVal->literal, tbLen + 1); + tstrncpy(pVal->node.userAlias, pFunc->node.userAlias, TSDB_COL_NAME_LEN); + tstrncpy(pVal->node.aliasName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); nodesDestroyNode(*pNode); *pNode = (SNode*)pVal; } @@ -4508,9 +4508,9 @@ static int32_t addPrimJoinEqCond(SNode** pCond, SRealTableNode* leftTable, SReal pLeft->tableId = pLMeta->uid; pLeft->colId = pLMeta->schema[0].colId; pLeft->colType = COLUMN_TYPE_COLUMN; - strcpy(pLeft->tableName, leftTable->table.tableName); - strcpy(pLeft->tableAlias, leftTable->table.tableAlias); - strcpy(pLeft->colName, pLMeta->schema[0].name); + tstrncpy(pLeft->tableName, leftTable->table.tableName, TSDB_TABLE_NAME_LEN); + tstrncpy(pLeft->tableAlias, leftTable->table.tableAlias, TSDB_TABLE_NAME_LEN); + tstrncpy(pLeft->colName, pLMeta->schema[0].name, TSDB_COL_NAME_LEN); pOp->pLeft = (SNode*)pLeft; @@ -4525,9 +4525,9 @@ static int32_t addPrimJoinEqCond(SNode** pCond, SRealTableNode* leftTable, SReal pRight->tableId = pRMeta->uid; pRight->colId = pRMeta->schema[0].colId; pRight->colType = COLUMN_TYPE_COLUMN; - strcpy(pRight->tableName, rightTable->table.tableName); - strcpy(pRight->tableAlias, rightTable->table.tableAlias); - strcpy(pRight->colName, pRMeta->schema[0].name); + tstrncpy(pRight->tableName, rightTable->table.tableName, TSDB_TABLE_NAME_LEN); + tstrncpy(pRight->tableAlias, rightTable->table.tableAlias, TSDB_TABLE_NAME_LEN); + tstrncpy(pRight->colName, pRMeta->schema[0].name, TSDB_COL_NAME_LEN); pOp->pRight = (SNode*)pRight; @@ -5130,29 +5130,29 @@ static int32_t createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr, SNo pFunc->node.resType = pExpr->resType; pFunc->funcId = pSrcFunc->funcId; pFunc->funcType = pSrcFunc->funcType; - strcpy(pFunc->functionName, pSrcFunc->functionName); + tstrncpy(pFunc->functionName, pSrcFunc->functionName, TSDB_FUNC_NAME_LEN); char buf[TSDB_FUNC_NAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN + TSDB_NAME_DELIMITER_LEN + 3] = {0}; int32_t len = 0; if (QUERY_NODE_COLUMN == nodeType(pExpr)) { SColumnNode* pCol = (SColumnNode*)pExpr; if (tsKeepColumnName) { - strcpy(pFunc->node.userAlias, pCol->colName); - strcpy(pFunc->node.aliasName, pCol->colName); + tstrncpy(pFunc->node.userAlias, pCol->colName, TSDB_COL_NAME_LEN); + tstrncpy(pFunc->node.aliasName, pCol->colName, TSDB_COL_NAME_LEN); } else { len = tsnprintf(buf, sizeof(buf) - 1, "%s(%s.%s)", pSrcFunc->functionName, pCol->tableAlias, pCol->colName); (void)taosHashBinary(buf, len); - strncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN - 1); + tstrncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN); len = tsnprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pCol->colName); // note: userAlias could be truncated here - strncpy(pFunc->node.userAlias, buf, TSDB_COL_NAME_LEN - 1); + tstrncpy(pFunc->node.userAlias, buf, TSDB_COL_NAME_LEN); } } else { len = tsnprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pExpr->aliasName); (void)taosHashBinary(buf, len); - strncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN - 1); + tstrncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN); len = tsnprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pExpr->userAlias); // note: userAlias could be truncated here - strncpy(pFunc->node.userAlias, buf, TSDB_COL_NAME_LEN - 1); + tstrncpy(pFunc->node.userAlias, buf, TSDB_COL_NAME_LEN); } *ppNodeOut = (SNode*)pFunc; return code; @@ -5481,9 +5481,9 @@ static int32_t rewriteProjectAlias(SNodeList* pProjectionList) { FOREACH(pProject, pProjectionList) { SExprNode* pExpr = (SExprNode*)pProject; if ('\0' == pExpr->userAlias[0]) { - strcpy(pExpr->userAlias, pExpr->aliasName); + tstrncpy(pExpr->userAlias, pExpr->aliasName, TSDB_COL_NAME_LEN); } - sprintf(pExpr->aliasName, "#expr_%d", no++); + snprintf(pExpr->aliasName, TSDB_COL_NAME_LEN,"#expr_%d", no++); } return TSDB_CODE_SUCCESS; } @@ -5669,7 +5669,11 @@ static int32_t getTimeRange(SNode** pPrimaryKeyCond, STimeWindow* pTimeRange, bo int32_t code = scalarCalculateConstants(*pPrimaryKeyCond, &pNew); if (TSDB_CODE_SUCCESS == code) { *pPrimaryKeyCond = pNew; - code = filterGetTimeRange(*pPrimaryKeyCond, pTimeRange, pIsStrict); + if (nodeType(pNew) == QUERY_NODE_VALUE) { + *pTimeRange = TSWINDOW_INITIALIZER; + } else { + code = filterGetTimeRange(*pPrimaryKeyCond, pTimeRange, pIsStrict); + } } return code; } @@ -6225,7 +6229,7 @@ static int32_t createDefaultFillNode(STranslateContext* pCxt, SNode** pOutput) { return code; } pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; - strcpy(pCol->colName, ROWTS_PSEUDO_COLUMN_NAME); + tstrncpy(pCol->colName, ROWTS_PSEUDO_COLUMN_NAME, TSDB_COL_NAME_LEN); pFill->pWStartTs = (SNode*)pCol; *pOutput = (SNode*)pFill; @@ -6830,7 +6834,7 @@ static int32_t replaceToChildTableQuery(STranslateContext* pCxt, SEqCondTbNameTa int32_t len = tsnprintf(buf, TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN, "%s.%s_%s", pTsma->dbFName, pTsma->name, pRealTable->table.tableName); len = taosCreateMD5Hash(buf, len); - strncpy(tsmaTargetTbName.tname, buf, MD5_OUTPUT_LEN); + tstrncpy(tsmaTargetTbName.tname, buf, TSDB_TABLE_NAME_LEN); STsmaTargetTbInfo ctbInfo = {0}; if (!pRealTable->tsmaTargetTbInfo) { pRealTable->tsmaTargetTbInfo = taosArrayInit(pRealTable->pTsmas->size, sizeof(STsmaTargetTbInfo)); @@ -6839,7 +6843,7 @@ static int32_t replaceToChildTableQuery(STranslateContext* pCxt, SEqCondTbNameTa break; } } - sprintf(ctbInfo.tableName, "%s", tsmaTargetTbName.tname); + snprintf(ctbInfo.tableName, TSDB_TABLE_NAME_LEN, "%s", tsmaTargetTbName.tname); ctbInfo.uid = pMeta->uid; if (NULL == taosArrayPush(pRealTable->tsmaTargetTbInfo, &ctbInfo)) { @@ -6904,7 +6908,7 @@ static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt* code = terrno; break; } - sprintf(pNewTbName, "%s.%s_%s", pTsma->dbFName, pTsma->name, pTbName); + snprintf(pNewTbName, TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN + 1, "%s.%s_%s", pTsma->dbFName, pTsma->name, pTbName); int32_t len = taosCreateMD5Hash(pNewTbName, strlen(pNewTbName)); } if (TSDB_CODE_SUCCESS == code) { @@ -6993,7 +6997,7 @@ static int32_t createPrimaryKeyColByTable(STranslateContext* pCxt, STableNode* p return code; } pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; - strcpy(pCol->colName, ROWTS_PSEUDO_COLUMN_NAME); + tstrncpy(pCol->colName, ROWTS_PSEUDO_COLUMN_NAME, TSDB_COL_NAME_LEN); bool found = false; code = findAndSetColumn(pCxt, &pCol, pTable, &found, true); if (TSDB_CODE_SUCCESS != code || !found) { @@ -7092,7 +7096,7 @@ static int32_t createPkColByTable(STranslateContext* pCxt, SRealTableNode* pTabl return code; } pCol->colId = pTable->pMeta->schema[1].colId; - strcpy(pCol->colName, pTable->pMeta->schema[1].name); + tstrncpy(pCol->colName, pTable->pMeta->schema[1].name, TSDB_COL_NAME_LEN); bool found = false; code = findAndSetColumn(pCxt, &pCol, (STableNode*)pTable, &found, true); if (TSDB_CODE_SUCCESS != code || !found) { @@ -8558,7 +8562,7 @@ static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray, bool calB field.bytes = pCol->dataType.bytes; } - strcpy(field.name, pCol->colName); + tstrncpy(field.name, pCol->colName, TSDB_COL_NAME_LEN); if (pCol->pOptions) { setColEncode(&field.compress, columnEncodeVal(((SColumnOptions*)pCol->pOptions)->encode)); setColCompress(&field.compress, columnCompressVal(((SColumnOptions*)pCol->pOptions)->compress)); @@ -8596,7 +8600,7 @@ static int32_t tagDefNodeToField(SNodeList* pList, SArray** pArray, bool calByte } else { field.bytes = pCol->dataType.bytes; } - strcpy(field.name, pCol->colName); + tstrncpy(field.name, pCol->colName, TSDB_COL_NAME_LEN); if (pCol->sma) { field.flags |= COL_SMA_ON; } @@ -9001,7 +9005,7 @@ static void toSchema(const SColumnDefNode* pCol, col_id_t colId, SSchema* pSchem pSchema->type = pCol->dataType.type; pSchema->bytes = calcTypeBytes(pCol->dataType); pSchema->flags = flags; - strcpy(pSchema->name, pCol->colName); + tstrncpy(pSchema->name, pCol->colName, TSDB_COL_NAME_LEN); } typedef struct SSampleAstInfo { @@ -9038,8 +9042,8 @@ static int32_t addWstartToSampleProjects(SNodeList* pProjectionList) { if (NULL == pFunc) { return code; } - strcpy(pFunc->functionName, "_wstart"); - strcpy(pFunc->node.userAlias, "_wstart"); + tstrncpy(pFunc->functionName, "_wstart", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.userAlias, "_wstart", TSDB_FUNC_NAME_LEN); return nodesListPushFront(pProjectionList, (SNode*)pFunc); } @@ -9049,8 +9053,8 @@ static int32_t addWendToSampleProjects(SNodeList* pProjectionList) { if (NULL == pFunc) { return code; } - strcpy(pFunc->functionName, "_wend"); - strcpy(pFunc->node.userAlias, "_wend"); + tstrncpy(pFunc->functionName, "_wend", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.userAlias, "_wend", TSDB_FUNC_NAME_LEN); return nodesListAppend(pProjectionList, (SNode*)pFunc); } @@ -9060,8 +9064,8 @@ static int32_t addWdurationToSampleProjects(SNodeList* pProjectionList) { if (NULL == pFunc) { return code; } - strcpy(pFunc->functionName, "_wduration"); - strcpy(pFunc->node.userAlias, "_wduration"); + tstrncpy(pFunc->functionName, "_wduration", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.userAlias, "_wduration", TSDB_FUNC_NAME_LEN); return nodesListAppend(pProjectionList, (SNode*)pFunc); } @@ -9081,7 +9085,7 @@ static int32_t buildProjectsForSampleAst(SSampleAstInfo* pInfo, SNodeList** pLis SNode* pProject = NULL; if (pProjectionTotalLen) *pProjectionTotalLen = 0; FOREACH(pProject, pProjectionList) { - sprintf(((SExprNode*)pProject)->aliasName, "#%p", pProject); + snprintf(((SExprNode*)pProject)->aliasName, TSDB_COL_NAME_LEN, "#%p", pProject); if (pProjectionTotalLen) *pProjectionTotalLen += ((SExprNode*)pProject)->resType.bytes; } *pList = pProjectionList; @@ -9114,7 +9118,7 @@ static int32_t buildIntervalForSampleAst(SSampleAstInfo* pInfo, SNode** pOutput) return code; } ((SColumnNode*)pInterval->pCol)->colId = PRIMARYKEY_TIMESTAMP_COL_ID; - strcpy(((SColumnNode*)pInterval->pCol)->colName, ROWTS_PSEUDO_COLUMN_NAME); + tstrncpy(((SColumnNode*)pInterval->pCol)->colName, ROWTS_PSEUDO_COLUMN_NAME, TSDB_COL_NAME_LEN); *pOutput = (SNode*)pInterval; return TSDB_CODE_SUCCESS; } @@ -9126,7 +9130,7 @@ static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, ch if (NULL == pSelect) { return code; } - sprintf(pSelect->stmtName, "%p", pSelect); + snprintf(pSelect->stmtName, TSDB_TABLE_NAME_LEN, "%p", pSelect); code = buildTableForSampleAst(pInfo, &pSelect->pFromTable); if (TSDB_CODE_SUCCESS == code) { @@ -9194,7 +9198,7 @@ static int32_t createColumnFromDef(SColumnDefNode* pDef, SNode** ppCol) { if (NULL == pCol) { return code; } - strcpy(pCol->colName, pDef->colName); + tstrncpy(pCol->colName, pDef->colName, TSDB_COL_NAME_LEN); *ppCol = (SNode*)pCol; return code; } @@ -9287,9 +9291,9 @@ static int32_t createTbnameFunction(SFunctionNode** ppFunc) { if (NULL == pFunc) { return code; } - strcpy(pFunc->functionName, "tbname"); - strcpy(pFunc->node.aliasName, "tbname"); - strcpy(pFunc->node.userAlias, "tbname"); + tstrncpy(pFunc->functionName, "tbname", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.aliasName, "tbname", TSDB_COL_NAME_LEN); + tstrncpy(pFunc->node.userAlias, "tbname", TSDB_COL_NAME_LEN); *ppFunc = pFunc; return code; } @@ -9486,7 +9490,7 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: { TAOS_FIELD field = {.type = pStmt->dataType.type, .bytes = calcTypeBytes(pStmt->dataType)}; - strcpy(field.name, pStmt->colName); + tstrncpy(field.name, pStmt->colName, 65); if (NULL == taosArrayPush(pAlterReq->pFields, &field)) { return terrno; } @@ -9495,12 +9499,12 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* case TSDB_ALTER_TABLE_UPDATE_TAG_NAME: case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: { TAOS_FIELD oldField = {0}; - strcpy(oldField.name, pStmt->colName); + tstrncpy(oldField.name, pStmt->colName, 65); if (NULL == taosArrayPush(pAlterReq->pFields, &oldField)) { return terrno; } TAOS_FIELD newField = {0}; - strcpy(newField.name, pStmt->newColName); + tstrncpy(newField.name, pStmt->newColName, 65); if (NULL == taosArrayPush(pAlterReq->pFields, &newField)) { return terrno; } @@ -9508,7 +9512,7 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* } case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS: { TAOS_FIELD field = {0}; - strcpy(field.name, pStmt->colName); + tstrncpy(field.name, pStmt->colName, 65); if (!checkColumnEncode(pStmt->pColOptions->encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; if (!checkColumnCompress(pStmt->pColOptions->compress)) return TSDB_CODE_TSC_COMPRESS_PARAM_ERROR; if (!checkColumnLevel(pStmt->pColOptions->compressLevel)) return TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR; @@ -9531,7 +9535,7 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* if (!pAlterReq->pFields) return terrno; SFieldWithOptions field = {.type = pStmt->dataType.type, .bytes = calcTypeBytes(pStmt->dataType)}; // TAOS_FIELD field = {.type = pStmt->dataType.type, .bytes = calcTypeBytes(pStmt->dataType)}; - strcpy(field.name, pStmt->colName); + tstrncpy(field.name, pStmt->colName, TSDB_COL_NAME_LEN); if (pStmt->pColOptions != NULL) { if (!checkColumnEncodeOrSetDefault(pStmt->dataType.type, pStmt->pColOptions->encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; @@ -9772,12 +9776,12 @@ static int32_t translateCreateUser(STranslateContext* pCxt, SCreateUserStmt* pSt if ((code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "sysinfo", pStmt->sysinfo, 0, 1, false))) { return code; } - strcpy(createReq.user, pStmt->userName); + tstrncpy(createReq.user, pStmt->userName, TSDB_USER_LEN); createReq.createType = 0; createReq.superUser = 0; createReq.sysInfo = pStmt->sysinfo; createReq.enable = 1; - strcpy(createReq.pass, pStmt->password); + tstrncpy(createReq.pass, pStmt->password, TSDB_USET_PASSWORD_LEN); createReq.isImport = pStmt->isImport; createReq.createDb = pStmt->createDb; @@ -9816,7 +9820,7 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt if ((code = checkAlterUser(pCxt, pStmt))) { return code; } - strcpy(alterReq.user, pStmt->userName); + tstrncpy(alterReq.user, pStmt->userName, TSDB_USER_LEN); alterReq.alterType = pStmt->alterType; alterReq.superUser = 0; alterReq.enable = pStmt->enable; @@ -9842,7 +9846,7 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt static int32_t translateDropUser(STranslateContext* pCxt, SDropUserStmt* pStmt) { SDropUserReq dropReq = {0}; - strcpy(dropReq.user, pStmt->userName); + tstrncpy(dropReq.user, pStmt->userName, TSDB_USER_LEN); int32_t code = buildCmdMsg(pCxt, TDMT_MND_DROP_USER, (FSerializeFunc)tSerializeSDropUserReq, &dropReq); tFreeSDropUserReq(&dropReq); @@ -9888,7 +9892,7 @@ static int32_t translateUpdateAnode(STranslateContext* pCxt, SUpdateAnodeStmt* p static int32_t translateCreateDnode(STranslateContext* pCxt, SCreateDnodeStmt* pStmt) { SCreateDnodeReq createReq = {0}; - strcpy(createReq.fqdn, pStmt->fqdn); + tstrncpy(createReq.fqdn, pStmt->fqdn, TSDB_FQDN_LEN); createReq.port = pStmt->port; int32_t code = buildCmdMsg(pCxt, TDMT_MND_CREATE_DNODE, (FSerializeFunc)tSerializeSCreateDnodeReq, &createReq); @@ -9899,7 +9903,7 @@ static int32_t translateCreateDnode(STranslateContext* pCxt, SCreateDnodeStmt* p static int32_t translateDropDnode(STranslateContext* pCxt, SDropDnodeStmt* pStmt) { SDropDnodeReq dropReq = {0}; dropReq.dnodeId = pStmt->dnodeId; - strcpy(dropReq.fqdn, pStmt->fqdn); + tstrncpy(dropReq.fqdn, pStmt->fqdn, TSDB_FQDN_LEN); dropReq.port = pStmt->port; dropReq.force = pStmt->force; dropReq.unsafe = pStmt->unsafe; @@ -9912,8 +9916,8 @@ static int32_t translateDropDnode(STranslateContext* pCxt, SDropDnodeStmt* pStmt static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pStmt) { SMCfgDnodeReq cfgReq = {0}; cfgReq.dnodeId = pStmt->dnodeId; - strcpy(cfgReq.config, pStmt->config); - strcpy(cfgReq.value, pStmt->value); + tstrncpy(cfgReq.config, pStmt->config, TSDB_DNODE_CONFIG_LEN); + tstrncpy(cfgReq.value, pStmt->value, TSDB_DNODE_VALUE_LEN); int32_t code = 0; if (0 == strncasecmp(cfgReq.config, "encrypt_key", 12)) { @@ -9960,8 +9964,8 @@ static int32_t translateRestoreDnode(STranslateContext* pCxt, SRestoreComponentN static int32_t translateAlterCluster(STranslateContext* pCxt, SAlterClusterStmt* pStmt) { SMCfgClusterReq cfgReq = {0}; - strcpy(cfgReq.config, pStmt->config); - strcpy(cfgReq.value, pStmt->value); + tstrncpy(cfgReq.config, pStmt->config, TSDB_DNODE_CONFIG_LEN); + tstrncpy(cfgReq.value, pStmt->value, TSDB_CLUSTER_VALUE_LEN); int32_t code = buildCmdMsg(pCxt, TDMT_MND_CONFIG_CLUSTER, (FSerializeFunc)tSerializeSMCfgClusterReq, &cfgReq); tFreeSMCfgClusterReq(&cfgReq); @@ -10437,9 +10441,9 @@ static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt* if (NULL == col) { return code; } - strcpy(col->colName, column->name); - strcpy(col->node.aliasName, col->colName); - strcpy(col->node.userAlias, col->colName); + tstrncpy(col->colName, column->name, TSDB_COL_NAME_LEN); + tstrncpy(col->node.aliasName, col->colName, TSDB_COL_NAME_LEN); + tstrncpy(col->node.userAlias, col->colName, TSDB_COL_NAME_LEN); code = addTagList(&colCxt.pTags, (SNode*)col); if (TSDB_CODE_SUCCESS != code) { nodesDestroyNode((SNode*)col); @@ -10476,9 +10480,9 @@ static int32_t buildQueryForTableTopic(STranslateContext* pCxt, SCreateTopicStmt if (TSDB_CODE_SUCCESS == code) { code = nodesMakeNode(QUERY_NODE_REAL_TABLE, (SNode**)&realTable); if (realTable) { - strcpy(realTable->table.dbName, pStmt->subDbName); - strcpy(realTable->table.tableName, pStmt->subSTbName); - strcpy(realTable->table.tableAlias, pStmt->subSTbName); + tstrncpy(realTable->table.dbName, pStmt->subDbName, TSDB_DB_NAME_LEN); + tstrncpy(realTable->table.tableName, pStmt->subSTbName, TSDB_TABLE_NAME_LEN); + tstrncpy(realTable->table.tableAlias, pStmt->subSTbName, TSDB_TABLE_NAME_LEN); code = createSelectStmtImpl(true, pProjection, (SNode*)realTable, NULL, ppSelect); } if (TSDB_CODE_SUCCESS == code) { @@ -10551,7 +10555,7 @@ static int32_t translateDropCGroup(STranslateContext* pCxt, SDropCGroupStmt* pSt if (TSDB_CODE_SUCCESS != code) return code; (void)tNameGetFullDbName(&name, dropReq.topic); dropReq.igNotExists = pStmt->ignoreNotExists; - strcpy(dropReq.cgroup, pStmt->cgroup); + tstrncpy(dropReq.cgroup, pStmt->cgroup, TSDB_CGROUP_LEN); return buildCmdMsg(pCxt, TDMT_MND_TMQ_DROP_CGROUP, (FSerializeFunc)tSerializeSMDropCgroupReq, &dropReq); } @@ -10666,7 +10670,7 @@ static int32_t translateKillCompact(STranslateContext* pCxt, SKillStmt* pStmt) { static int32_t translateKillQuery(STranslateContext* pCxt, SKillQueryStmt* pStmt) { SKillQueryReq killReq = {0}; - strcpy(killReq.queryStrId, pStmt->queryId); + tstrncpy(killReq.queryStrId, pStmt->queryId, TSDB_QUERY_ID_LEN); return buildCmdMsg(pCxt, TDMT_MND_KILL_QUERY, (FSerializeFunc)tSerializeSKillQueryReq, &killReq); } @@ -10722,7 +10726,7 @@ static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pSt static void getSourceDatabase(SNode* pStmt, int32_t acctId, char* pDbFName) { SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId}; - strcpy(name.dbname, ((SRealTableNode*)(((SSelectStmt*)pStmt)->pFromTable))->table.dbName); + tstrncpy(name.dbname, ((SRealTableNode*)(((SSelectStmt*)pStmt)->pFromTable))->table.dbName, TSDB_DB_NAME_LEN); (void)tNameGetFullDbName(&name, pDbFName); } @@ -10781,7 +10785,7 @@ static int32_t addIrowTsToCreateStreamQueryImpl(STranslateContext* pCxt, SSelect SColumnDefNode* pColDef = NULL; code = nodesMakeNode(QUERY_NODE_COLUMN_DEF, (SNode**)&pColDef); if (TSDB_CODE_SUCCESS == code) { - strcpy(pColDef->colName, pFunc->node.aliasName); + tstrncpy(pColDef->colName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); pColDef->dataType = pFunc->node.resType; pColDef->sma = true; code = setColumnDefNodePrimaryKey(pColDef, false); @@ -10805,8 +10809,8 @@ static int32_t addWstartTsToCreateStreamQueryImpl(STranslateContext* pCxt, SSele if (NULL == pFunc) { return code; } - strcpy(pFunc->functionName, "_wstart"); - strcpy(pFunc->node.userAlias, "_irowts"); + tstrncpy(pFunc->functionName, "_wstart", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.userAlias, "_irowts", TSDB_COL_NAME_LEN); char* defaultName[] = {"_wstart", "ts", NULL}; getStreamQueryFirstProjectAliasName(pUserAliasSet, pFunc->node.aliasName, sizeof(pFunc->node.aliasName), defaultName); code = getFuncInfo(pCxt, pFunc); @@ -10818,7 +10822,7 @@ static int32_t addWstartTsToCreateStreamQueryImpl(STranslateContext* pCxt, SSele SColumnDefNode* pColDef = NULL; code = nodesMakeNode(QUERY_NODE_COLUMN_DEF, (SNode**)&pColDef); if (TSDB_CODE_SUCCESS == code) { - strcpy(pColDef->colName, pFunc->node.aliasName); + tstrncpy(pColDef->colName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); pColDef->dataType = pFunc->node.resType; pColDef->sma = true; code = setColumnDefNodePrimaryKey(pColDef, false); @@ -10978,7 +10982,7 @@ static int32_t addColDefNodeByProj(SNodeList** ppCols, const SNode* pProject, in SColumnDefNode* pColDef = NULL; int32_t code = nodesMakeNode(QUERY_NODE_COLUMN_DEF, (SNode**)&pColDef); if (TSDB_CODE_SUCCESS != code) return code; - strcpy(pColDef->colName, pExpr->userAlias); + tstrncpy(pColDef->colName, pExpr->userAlias, TSDB_COL_NAME_LEN); pColDef->dataType = pExpr->resType; pColDef->sma = flags & COL_SMA_ON; code = setColumnDefNodePrimaryKey(pColDef, flags & COL_IS_KEY); @@ -11279,7 +11283,7 @@ static int32_t adjustDataTypeOfProjections(STranslateContext* pCxt, const STable SColumnDefNode* pColDef = NULL; int32_t code = nodesMakeNode(QUERY_NODE_COLUMN_DEF, (SNode**)&pColDef); if (TSDB_CODE_SUCCESS != code) return code; - strcpy(pColDef->colName, pSchema->name); + tstrncpy(pColDef->colName, pSchema->name, TSDB_COL_NAME_LEN); pColDef->dataType = dt; pColDef->sma = pSchema->flags & COL_SMA_ON; code = setColumnDefNodePrimaryKey(pColDef, pSchema->flags & COL_IS_KEY); @@ -11322,7 +11326,7 @@ static int32_t addProjToProjColPos(STranslateContext* pCxt, const SSchema* pSche if (!dataTypeEqual(&dt, &((SExprNode*)pNewProj)->resType)) { SNode* pFunc = NULL; code = createCastFunc(pCxt, pNewProj, dt, &pFunc); - strcpy(((SExprNode*)pFunc)->userAlias, ((SExprNode*)pNewProj)->userAlias); + tstrncpy(((SExprNode*)pFunc)->userAlias, ((SExprNode*)pNewProj)->userAlias, TSDB_COL_NAME_LEN); pNewProj = pFunc; } if (TSDB_CODE_SUCCESS == code) { @@ -11416,9 +11420,6 @@ static int32_t adjustOrderOfProjections(STranslateContext* pCxt, SNodeList** ppC code = nodesMakeList(&pNewProjections); if (TSDB_CODE_SUCCESS != code) return code; code = nodesMakeList(&pNewCols); - if (TSDB_CODE_SUCCESS != code) { - code = code; - } for (int32_t i = 0; TSDB_CODE_SUCCESS == code && i < num; ++i) { SProjColPos* pPos = taosArrayGet(pProjColPos, i); code = nodesListStrictAppend(pNewProjections, pPos->pProj); @@ -11524,9 +11525,6 @@ static int32_t adjustOrderOfTags(STranslateContext* pCxt, SNodeList* pTags, cons int32_t numOfTags = getNumOfTags(pMeta); const SSchema* pTagsSchema = getTableTagSchema(pMeta); code = nodesMakeList(&pNewTagExprs); - if (NULL == pNewTagExprs) { - code = code; - } for (int32_t i = 0; TSDB_CODE_SUCCESS == code && i < numOfTags; ++i) { const SSchema* pTagSchema = pTagsSchema + i; if (indexOfBoundTags < numOfBoundTags) { @@ -11891,8 +11889,8 @@ static int32_t buildCreateStreamReq(STranslateContext* pCxt, SCreateStreamStmt* (void)tNameGetFullDbName(&name, pReq->name); if ('\0' != pStmt->targetTabName[0]) { - strcpy(name.dbname, pStmt->targetDbName); - strcpy(name.tname, pStmt->targetTabName); + tstrncpy(name.dbname, pStmt->targetDbName, TSDB_DB_NAME_LEN); + tstrncpy(name.tname, pStmt->targetTabName, TSDB_TABLE_NAME_LEN); code = tNameExtractFullName(&name, pReq->targetStbFullName); } if (TSDB_CODE_SUCCESS == code) { @@ -12120,7 +12118,7 @@ static int32_t translateCreateView(STranslateContext* pCxt, SCreateViewStmt* pSt pStmt->createReq.precision = res.schemaRes.precision; pStmt->createReq.numOfCols = res.schemaRes.numOfCols; pStmt->createReq.pSchema = res.schemaRes.pSchema; - strncpy(pStmt->createReq.name, pStmt->viewName, sizeof(pStmt->createReq.name) - 1); + tstrncpy(pStmt->createReq.name, pStmt->viewName, TSDB_VIEW_NAME_LEN); tstrncpy(pStmt->createReq.dbFName, dbFName, sizeof(pStmt->createReq.dbFName)); snprintf(pStmt->createReq.fullname, sizeof(pStmt->createReq.fullname) - 1, "%s.%s", pStmt->createReq.dbFName, pStmt->viewName); @@ -12149,7 +12147,7 @@ static int32_t translateDropView(STranslateContext* pCxt, SDropViewStmt* pStmt) int32_t code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); if (TSDB_CODE_SUCCESS == code) { (void)tNameGetFullDbName(&name, dropReq.dbFName); - strncpy(dropReq.name, pStmt->viewName, sizeof(dropReq.name) - 1); + tstrncpy(dropReq.name, pStmt->viewName, TSDB_VIEW_NAME_LEN); snprintf(dropReq.fullname, sizeof(dropReq.fullname) - 1, "%s.%s", dropReq.dbFName, dropReq.name); dropReq.sql = (char*)pCxt->pParseCxt->pSql; if (NULL == dropReq.sql) { @@ -12216,7 +12214,7 @@ static int32_t translateCreateFunction(STranslateContext* pCxt, SCreateFunctionS } SCreateFuncReq req = {0}; - strcpy(req.name, pStmt->funcName); + tstrncpy(req.name, pStmt->funcName, TSDB_FUNC_NAME_LEN); req.orReplace = pStmt->orReplace; req.igExists = pStmt->ignoreExists; req.funcType = pStmt->isAgg ? TSDB_FUNC_TYPE_AGGREGATE : TSDB_FUNC_TYPE_SCALAR; @@ -12235,7 +12233,7 @@ static int32_t translateCreateFunction(STranslateContext* pCxt, SCreateFunctionS static int32_t translateDropFunction(STranslateContext* pCxt, SDropFunctionStmt* pStmt) { SDropFuncReq req = {0}; - strcpy(req.name, pStmt->funcName); + tstrncpy(req.name, pStmt->funcName, TSDB_FUNC_NAME_LEN); req.igNotExists = pStmt->ignoreNotExists; return buildCmdMsg(pCxt, TDMT_MND_DROP_FUNC, (FSerializeFunc)tSerializeSDropFuncReq, &req); } @@ -12246,9 +12244,9 @@ static int32_t createRealTableForGrantTable(SGrantStmt* pStmt, SRealTableNode** if (NULL == pRealTable) { return code; } - strcpy(pRealTable->table.dbName, pStmt->objName); - strcpy(pRealTable->table.tableName, pStmt->tabName); - strcpy(pRealTable->table.tableAlias, pStmt->tabName); + tstrncpy(pRealTable->table.dbName, pStmt->objName, TSDB_DB_NAME_LEN); + tstrncpy(pRealTable->table.tableName, pStmt->tabName, TSDB_TABLE_NAME_LEN); + tstrncpy(pRealTable->table.tableAlias, pStmt->tabName, TSDB_TABLE_NAME_LEN); *pTable = pRealTable; return TSDB_CODE_SUCCESS; } @@ -12323,9 +12321,9 @@ static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) { } #endif - strcpy(req.user, pStmt->userName); - sprintf(req.objname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName); - sprintf(req.tabName, "%s", pStmt->tabName); + tstrncpy(req.user, pStmt->userName, TSDB_USER_LEN); + snprintf(req.objname, TSDB_DB_FNAME_LEN, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName); + snprintf(req.tabName, TSDB_TABLE_NAME_LEN, "%s", pStmt->tabName); if (!req.isView) { code = translateGrantTagCond(pCxt, pStmt, &req); } @@ -12359,9 +12357,9 @@ static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) { } #endif - strcpy(req.user, pStmt->userName); - sprintf(req.objname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName); - sprintf(req.tabName, "%s", pStmt->tabName); + tstrncpy(req.user, pStmt->userName, TSDB_USER_LEN); + snprintf(req.objname, TSDB_DB_FNAME_LEN, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName); + snprintf(req.tabName, TSDB_TABLE_NAME_LEN, "%s", pStmt->tabName); code = buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req); tFreeSAlterUserReq(&req); return code; @@ -12377,7 +12375,7 @@ static int32_t translateBalanceVgroup(STranslateContext* pCxt, SBalanceVgroupStm static int32_t translateBalanceVgroupLeader(STranslateContext* pCxt, SBalanceVgroupLeaderStmt* pStmt) { SBalanceVgroupLeaderReq req = {0}; req.vgId = pStmt->vgId; - strcpy(req.db, pStmt->dbName); + tstrncpy(req.db, pStmt->dbName, TSDB_DB_FNAME_LEN); int32_t code = buildCmdMsg(pCxt, TDMT_MND_BALANCE_VGROUP_LEADER, (FSerializeFunc)tSerializeSBalanceVgroupLeaderReq, &req); tFreeSBalanceVgroupLeaderReq(&req); @@ -12544,14 +12542,14 @@ static int32_t buildTSMAAstStreamSubTable(SCreateTSMAStmt* pStmt, SMCreateSmaReq code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pVal); if (TSDB_CODE_SUCCESS != code) goto _end; - sprintf(pMd5Func->functionName, "%s", "md5"); - sprintf(pConcatFunc->functionName, "%s", "concat"); + snprintf(pMd5Func->functionName, TSDB_FUNC_NAME_LEN, "%s", "md5"); + snprintf(pConcatFunc->functionName, TSDB_FUNC_NAME_LEN, "%s", "concat"); pVal->literal = taosMemoryMalloc(TSDB_TABLE_FNAME_LEN + 1); if (!pVal->literal) { code = terrno; goto _end; } - sprintf(pVal->literal, "%s_", pReq->name); + snprintf(pVal->literal, TSDB_TABLE_FNAME_LEN + 1, "%s_", pReq->name); pVal->node.resType.type = TSDB_DATA_TYPE_VARCHAR; pVal->node.resType.bytes = strlen(pVal->literal); code = nodesListMakeAppend(&pConcatFunc->pParameterList, (SNode*)pVal); @@ -12595,10 +12593,8 @@ static int32_t buildTSMAAst(STranslateContext* pCxt, SCreateTSMAStmt* pStmt, SMC // append partition by tbname code = createTbnameFunction(&pTbnameFunc); if (pTbnameFunc) { - sprintf(pTbnameFunc->node.userAlias, "tbname"); + snprintf(pTbnameFunc->node.userAlias, TSDB_COL_NAME_LEN, "tbname"); code = nodesListMakeStrictAppend(&info.pPartitionByList, (SNode*)pTbnameFunc); - } else { - code = code; } } if (TSDB_CODE_SUCCESS == code) { @@ -12663,7 +12659,7 @@ static int32_t createColumnBySchema(const SSchema* pSchema, SColumnNode** ppCol) (*ppCol)->colId = pSchema->colId; (*ppCol)->node.resType.type = pSchema->type; (*ppCol)->node.resType.bytes = pSchema->bytes; - strcpy((*ppCol)->colName, pSchema->name); + tstrncpy((*ppCol)->colName, pSchema->name, TSDB_COL_NAME_LEN); return TSDB_CODE_SUCCESS; } @@ -13213,9 +13209,9 @@ static int32_t extractQueryResultSchema(const SNodeList* pProjections, int32_t* } (*pSchema)[index].colId = index + 1; if ('\0' != pExpr->userAlias[0]) { - strcpy((*pSchema)[index].name, pExpr->userAlias); + tstrncpy((*pSchema)[index].name, pExpr->userAlias, TSDB_COL_NAME_LEN); } else { - strcpy((*pSchema)[index].name, pExpr->aliasName); + tstrncpy((*pSchema)[index].name, pExpr->aliasName, TSDB_COL_NAME_LEN); } index += 1; } @@ -13233,7 +13229,7 @@ static int32_t extractExplainResultSchema(int32_t* numOfCols, SSchema** pSchema) } (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = TSDB_EXPLAIN_RESULT_ROW_SIZE; - strcpy((*pSchema)[0].name, TSDB_EXPLAIN_RESULT_COLUMN_NAME); + tstrncpy((*pSchema)[0].name, TSDB_EXPLAIN_RESULT_COLUMN_NAME, TSDB_COL_NAME_LEN); return TSDB_CODE_SUCCESS; } @@ -13247,32 +13243,32 @@ static int32_t extractDescribeResultSchema(STableMeta* pMeta, int32_t* numOfCols (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = DESCRIBE_RESULT_FIELD_LEN; - strcpy((*pSchema)[0].name, "field"); + tstrncpy((*pSchema)[0].name, "field", TSDB_COL_NAME_LEN); (*pSchema)[1].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[1].bytes = DESCRIBE_RESULT_TYPE_LEN; - strcpy((*pSchema)[1].name, "type"); + tstrncpy((*pSchema)[1].name, "type", TSDB_COL_NAME_LEN); (*pSchema)[2].type = TSDB_DATA_TYPE_INT; (*pSchema)[2].bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes; - strcpy((*pSchema)[2].name, "length"); + tstrncpy((*pSchema)[2].name, "length", TSDB_COL_NAME_LEN); (*pSchema)[3].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[3].bytes = DESCRIBE_RESULT_NOTE_LEN; - strcpy((*pSchema)[3].name, "note"); + tstrncpy((*pSchema)[3].name, "note", TSDB_COL_NAME_LEN); if (pMeta && useCompress(pMeta->tableType)) { (*pSchema)[4].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[4].bytes = DESCRIBE_RESULT_COPRESS_OPTION_LEN; - strcpy((*pSchema)[4].name, "encode"); + tstrncpy((*pSchema)[4].name, "encode", TSDB_COL_NAME_LEN); (*pSchema)[5].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[5].bytes = DESCRIBE_RESULT_COPRESS_OPTION_LEN; - strcpy((*pSchema)[5].name, "compress"); + tstrncpy((*pSchema)[5].name, "compress", TSDB_COL_NAME_LEN); (*pSchema)[6].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[6].bytes = DESCRIBE_RESULT_COPRESS_OPTION_LEN; - strcpy((*pSchema)[6].name, "level"); + tstrncpy((*pSchema)[6].name, "level", TSDB_COL_NAME_LEN); } return TSDB_CODE_SUCCESS; @@ -13287,11 +13283,11 @@ static int32_t extractShowCreateDatabaseResultSchema(int32_t* numOfCols, SSchema (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = TSDB_DB_NAME_LEN; - strcpy((*pSchema)[0].name, "Database"); + tstrncpy((*pSchema)[0].name, "Database", TSDB_COL_NAME_LEN); (*pSchema)[1].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[1].bytes = TSDB_MAX_BINARY_LEN; - strcpy((*pSchema)[1].name, "Create Database"); + tstrncpy((*pSchema)[1].name, "Create Database", TSDB_COL_NAME_LEN); return TSDB_CODE_SUCCESS; } @@ -13305,11 +13301,11 @@ static int32_t extractShowCreateTableResultSchema(int32_t* numOfCols, SSchema** (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = SHOW_CREATE_TB_RESULT_FIELD1_LEN; - strcpy((*pSchema)[0].name, "Table"); + tstrncpy((*pSchema)[0].name, "Table", TSDB_COL_NAME_LEN); (*pSchema)[1].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[1].bytes = SHOW_CREATE_TB_RESULT_FIELD2_LEN; - strcpy((*pSchema)[1].name, "Create Table"); + tstrncpy((*pSchema)[1].name, "Create Table", TSDB_COL_NAME_LEN); return TSDB_CODE_SUCCESS; } @@ -13323,11 +13319,11 @@ static int32_t extractShowCreateViewResultSchema(int32_t* numOfCols, SSchema** p (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = SHOW_CREATE_VIEW_RESULT_FIELD1_LEN; - strcpy((*pSchema)[0].name, "View"); + tstrncpy((*pSchema)[0].name, "View", TSDB_COL_NAME_LEN); (*pSchema)[1].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[1].bytes = SHOW_CREATE_VIEW_RESULT_FIELD2_LEN; - strcpy((*pSchema)[1].name, "Create View"); + tstrncpy((*pSchema)[1].name, "Create View", TSDB_COL_NAME_LEN); return TSDB_CODE_SUCCESS; } @@ -13341,19 +13337,19 @@ static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pS (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = TSDB_CONFIG_OPTION_LEN; - strcpy((*pSchema)[0].name, "name"); + tstrncpy((*pSchema)[0].name, "name", TSDB_COL_NAME_LEN); (*pSchema)[1].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[1].bytes = TSDB_CONFIG_PATH_LEN; - strcpy((*pSchema)[1].name, "value"); + tstrncpy((*pSchema)[1].name, "value", TSDB_COL_NAME_LEN); (*pSchema)[2].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[2].bytes = TSDB_CONFIG_SCOPE_LEN; - strcpy((*pSchema)[2].name, "scope"); + tstrncpy((*pSchema)[2].name, "scope", TSDB_COL_NAME_LEN); (*pSchema)[3].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[3].bytes = TSDB_CONFIG_INFO_LEN; - strcpy((*pSchema)[3].name, "info"); + tstrncpy((*pSchema)[3].name, "info", TSDB_COL_NAME_LEN); return TSDB_CODE_SUCCESS; } @@ -13367,15 +13363,15 @@ static int32_t extractCompactDbResultSchema(int32_t* numOfCols, SSchema** pSchem (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = COMPACT_DB_RESULT_FIELD1_LEN; - strcpy((*pSchema)[0].name, "result"); + tstrncpy((*pSchema)[0].name, "result", TSDB_COL_NAME_LEN); (*pSchema)[1].type = TSDB_DATA_TYPE_INT; (*pSchema)[1].bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes; - strcpy((*pSchema)[1].name, "id"); + tstrncpy((*pSchema)[1].name, "id", TSDB_COL_NAME_LEN); (*pSchema)[2].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[2].bytes = COMPACT_DB_RESULT_FIELD3_LEN; - strcpy((*pSchema)[2].name, "reason"); + tstrncpy((*pSchema)[2].name, "reason", TSDB_COL_NAME_LEN); return TSDB_CODE_SUCCESS; } @@ -13420,7 +13416,7 @@ static int32_t createStarCol(SNode** ppNode) { if (NULL == pCol) { return code; } - strcpy(pCol->colName, "*"); + tstrncpy(pCol->colName, "*", TSDB_COL_NAME_LEN); *ppNode = (SNode*)pCol; return code; } @@ -13474,7 +13470,7 @@ static int32_t createSimpleSelectStmtImpl(const char* pDb, const char* pTable, S if (NULL == pSelect) { return code; } - sprintf(pSelect->stmtName, "%p", pSelect); + snprintf(pSelect->stmtName, TSDB_TABLE_NAME_LEN, "%p", pSelect); SRealTableNode* pRealTable = NULL; code = nodesMakeNode(QUERY_NODE_REAL_TABLE, (SNode**)&pRealTable); @@ -13570,8 +13566,8 @@ static int32_t createParOperatorNode(EOperatorType opType, const char* pLeftCol, nodesDestroyNode((SNode*)pOper); return code; } - strcpy(((SColumnNode*)pOper->pLeft)->colName, pLeftCol); - strcpy(((SColumnNode*)pOper->pRight)->colName, pRightCol); + tstrncpy(((SColumnNode*)pOper->pLeft)->colName, pLeftCol, TSDB_COL_NAME_LEN); + tstrncpy(((SColumnNode*)pOper->pRight)->colName, pRightCol, TSDB_COL_NAME_LEN); *ppResOp = (SNode*)pOper; return TSDB_CODE_SUCCESS; @@ -13838,7 +13834,7 @@ static int32_t createShowCondition(const SShowStmt* pShow, SSelectStmt* pSelect) } if (NULL != pShow->pDbName) { - strcpy(((SRealTableNode*)pSelect->pFromTable)->qualDbName, ((SValueNode*)pShow->pDbName)->literal); + tstrncpy(((SRealTableNode*)pSelect->pFromTable)->qualDbName, ((SValueNode*)pShow->pDbName)->literal, TSDB_DB_NAME_LEN); } return TSDB_CODE_SUCCESS; @@ -13911,7 +13907,7 @@ static int32_t createTagsFunction(SFunctionNode** ppNode) { if (NULL == pFunc) { return code; } - strcpy(pFunc->functionName, "_tags"); + tstrncpy(pFunc->functionName, "_tags", TSDB_FUNC_NAME_LEN); *ppNode = pFunc; return code; } @@ -14009,8 +14005,8 @@ static int32_t createBlockDistInfoFunc(SFunctionNode** ppNode) { return code; } - strcpy(pFunc->functionName, "_block_dist_info"); - strcpy(pFunc->node.aliasName, "_block_dist_info"); + tstrncpy(pFunc->functionName, "_block_dist_info", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.aliasName, "_block_dist_info", TSDB_COL_NAME_LEN); *ppNode = pFunc; return code; } @@ -14022,8 +14018,8 @@ static int32_t createBlockDistFunc(SFunctionNode** ppNode) { return code; } - strcpy(pFunc->functionName, "_block_dist"); - strcpy(pFunc->node.aliasName, "_block_dist"); + tstrncpy(pFunc->functionName, "_block_dist", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.aliasName, "_block_dist", TSDB_COL_NAME_LEN); SFunctionNode* pFuncNew = NULL; code = createBlockDistInfoFunc(&pFuncNew); if (TSDB_CODE_SUCCESS == code) { @@ -14121,7 +14117,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* SVgroupCreateTableBatch* pBatch) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId}; - (void)strcpy(name.dbname, pStmt->dbName); + tstrncpy(name.dbname, pStmt->dbName, TSDB_DB_NAME_LEN); (void)tNameGetFullDbName(&name, dbFName); SVCreateTbReq req = {0}; @@ -14177,7 +14173,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* ++index; } pBatch->info = *pVgroupInfo; - (void)strcpy(pBatch->dbName, pStmt->dbName); + tstrncpy(pBatch->dbName, pStmt->dbName, TSDB_DB_NAME_LEN); pBatch->req.pArray = taosArrayInit(1, sizeof(struct SVCreateTbReq)); if (NULL == pBatch->req.pArray) { tdDestroySVCreateTbReq(&req); @@ -14353,7 +14349,7 @@ static int32_t addCreateTbReqIntoVgroup(SHashObj* pVgroupHashmap, const char* db if (pTableBatch == NULL) { SVgroupCreateTableBatch tBatch = {0}; tBatch.info = *pVgInfo; - strcpy(tBatch.dbName, dbName); + tstrncpy(tBatch.dbName, dbName, TSDB_DB_NAME_LEN); tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq)); if (!tBatch.req.pArray) { @@ -14856,7 +14852,7 @@ static int32_t constructParseFileContext(SCreateSubTableFromFileClause* pStmt, S pParFileCxt->pTag = NULL; pParFileCxt->ctbName.type = TSDB_TABLE_NAME_T; pParFileCxt->ctbName.acctId = acctId; - strcpy(pParFileCxt->ctbName.dbname, pStmt->useDbName); + tstrncpy(pParFileCxt->ctbName.dbname, pStmt->useDbName, TSDB_DB_NAME_LEN); if (NULL == pParFileCxt->aTagNames) { pParFileCxt->aTagNames = taosArrayInit(8, TSDB_COL_NAME_LEN); @@ -14930,8 +14926,8 @@ static int32_t prepareReadCsvFile(STranslateContext* pCxt, SCreateSubTableFromFi } pCreateInfo->ignoreExists = pCreateStmt->ignoreExists; - strncpy(pCreateInfo->useDbName, pCreateStmt->useDbName, TSDB_DB_NAME_LEN); - strncpy(pCreateInfo->useTableName, pCreateStmt->useTableName, TSDB_TABLE_NAME_LEN); + tstrncpy(pCreateInfo->useDbName, pCreateStmt->useDbName, TSDB_DB_NAME_LEN); + tstrncpy(pCreateInfo->useTableName, pCreateStmt->useTableName, TSDB_TABLE_NAME_LEN); } { @@ -16131,8 +16127,8 @@ static int32_t createParCaseWhenNode(SNode* pCase, SNodeList* pWhenThenList, SNo pCaseWhen->pWhenThenList = pWhenThenList; pCaseWhen->pElse = pElse; if (pAias) { - strcpy(pCaseWhen->node.aliasName, pAias); - strcpy(pCaseWhen->node.userAlias, pAias); + tstrncpy(pCaseWhen->node.aliasName, pAias, TSDB_COL_NAME_LEN); + tstrncpy(pCaseWhen->node.userAlias, pAias, TSDB_COL_NAME_LEN); } *ppResCaseWhen = (SNode*)pCaseWhen; return TSDB_CODE_SUCCESS; @@ -16145,9 +16141,9 @@ static int32_t createParFunctionNode(const char* pFunName, const char* pAias, SN if (TSDB_CODE_SUCCESS != code) { return code; } - strcpy(pFunc->functionName, pFunName); - strcpy(pFunc->node.aliasName, pAias); - strcpy(pFunc->node.userAlias, pAias); + tstrncpy(pFunc->functionName, pFunName, TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.aliasName, pAias, TSDB_COL_NAME_LEN); + tstrncpy(pFunc->node.userAlias, pAias, TSDB_COL_NAME_LEN); pFunc->pParameterList = pParameterList; *ppResFunc = (SNode*)pFunc; return TSDB_CODE_SUCCESS; @@ -16172,7 +16168,7 @@ static int32_t createParTempTableNode(SSelectStmt* pSubquery, SNode** ppResTempT } pTempTable->pSubquery = (SNode*)pSubquery; taosRandStr(pTempTable->table.tableAlias, 8); - strcpy(pSubquery->stmtName, pTempTable->table.tableAlias); + tstrncpy(pSubquery->stmtName, pTempTable->table.tableAlias, TSDB_TABLE_NAME_LEN); pSubquery->isSubquery = true; *ppResTempTable = (SNode*)pTempTable; return TSDB_CODE_SUCCESS; @@ -16588,7 +16584,7 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) { pStmt->pProjectionList = pProjList; pStmt->pFromTable = pTempTblNode; - sprintf(pStmt->stmtName, "%p", pStmt); + snprintf(pStmt->stmtName, TSDB_TABLE_NAME_LEN, "%p", pStmt); nodesDestroyNode(pQuery->pRoot); pQuery->pRoot = (SNode*)pStmt; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 6ebf767eed..8c9fd89db6 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -250,7 +250,7 @@ int32_t generateSyntaxErrMsgExt(SMsgBuf* pBuf, int32_t errCode, const char* pFor int32_t buildInvalidOperationMsg(SMsgBuf* pBuf, const char* msg) { if (pBuf->buf) { - strncpy(pBuf->buf, msg, pBuf->len); + tstrncpy(pBuf->buf, msg, pBuf->len); } return TSDB_CODE_TSC_INVALID_OPERATION; @@ -277,7 +277,7 @@ int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* } char buf[64] = {0}; // only extract part of sql string - strncpy(buf, sourceStr, tListLen(buf) - 1); + tstrncpy(buf, sourceStr, tListLen(buf)); if (additionalInfo != NULL) { snprintf(pBuf->buf, pBuf->len, msgFormat2, buf, additionalInfo); @@ -454,7 +454,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi continue; } STagVal val = {0}; - // strcpy(val.colName, colName); + // TSDB_DB_FNAME_LENme, colName); val.pKey = jsonKey; retCode = taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES); // add key to hash to remove dumplicate, value is useless @@ -596,15 +596,15 @@ int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type, char* pStr, bool isView) { - return sprintf(pStr, "%s*%d*%s*%s*%d*%d", pUser, acctId, pDb, (NULL == pTable || '\0' == pTable[0]) ? "``" : pTable, - type, isView); + return snprintf(pStr, USER_AUTH_KEY_MAX_LEN, "%s*%d*%s*%s*%d*%d", pUser, acctId, pDb, + (NULL == pTable || '\0' == pTable[0]) ? "``" : pTable, type, isView); } static int32_t getIntegerFromAuthStr(const char* pStart, char** pNext) { char* p = strchr(pStart, '*'); char buf[10] = {0}; if (NULL == p) { - strcpy(buf, pStart); + tstrncpy(buf, pStart, 10); *pNext = NULL; } else { strncpy(buf, pStart, p - pStart); @@ -616,7 +616,7 @@ static int32_t getIntegerFromAuthStr(const char* pStart, char** pNext) { static void getStringFromAuthStr(const char* pStart, char* pStr, char** pNext) { char* p = strchr(pStart, '*'); if (NULL == p) { - strcpy(pStr, pStart); + tstrncpy(pStr, pStart, strlen(pStart) + 1); *pNext = NULL; } else { strncpy(pStr, pStart, p - pStart); @@ -708,7 +708,7 @@ static int32_t buildTableReqFromDb(SHashObj* pDbsHash, SArray** pDbs) { SParseTablesMetaReq* p = taosHashIterate(pDbsHash, NULL); while (NULL != p) { STablesReq req = {0}; - strcpy(req.dbFName, p->dbFName); + tstrncpy(req.dbFName, p->dbFName, TSDB_DB_FNAME_LEN); int32_t code = buildTableReq(p->pTables, &req.pTables); if (TSDB_CODE_SUCCESS == code) { if (NULL == taosArrayPush(*pDbs, &req)) { @@ -832,7 +832,7 @@ int32_t createSelectStmtImpl(bool isDistinct, SNodeList* pProjectionList, SNode* select->isDistinct = isDistinct; select->pProjectionList = pProjectionList; select->pFromTable = pTable; - sprintf(select->stmtName, "%p", select); + snprintf(select->stmtName, TSDB_TABLE_NAME_LEN, "%p", select); select->timeLineResMode = select->isDistinct ? TIME_LINE_NONE : TIME_LINE_GLOBAL; select->timeLineCurMode = TIME_LINE_GLOBAL; select->onlyHasKeepOrderFunc = true; diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index d9783fb9a3..6d2822f038 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -221,7 +221,7 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam, vo static EDealRes rewriteQueryExprAliasImpl(SNode* pNode, void* pContext) { if (nodesIsExprNode(pNode) && QUERY_NODE_COLUMN != nodeType(pNode)) { - sprintf(((SExprNode*)pNode)->aliasName, "#%d", *(int32_t*)pContext); + snprintf(((SExprNode*)pNode)->aliasName, TSDB_COL_NAME_LEN, "#%d", *(int32_t*)pContext); ++(*(int32_t*)pContext); } return DEAL_RES_CONTINUE; @@ -528,9 +528,6 @@ int32_t qStmtBindParams2(SQuery* pQuery, TAOS_STMT2_BIND* pParams, int32_t colId nodesDestroyNode(pQuery->pRoot); pQuery->pRoot = NULL; code = nodesCloneNode(pQuery->pPrepareRoot, &pQuery->pRoot); - if (NULL == pQuery->pRoot) { - code = code; - } } if (TSDB_CODE_SUCCESS == code) { rewriteExprAlias(pQuery->pRoot); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index a9c3beb025..363aa71479 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -119,9 +119,9 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { } SExprNode* pToBeRewrittenExpr = (SExprNode*)(*pNode); pCol->node.resType = pToBeRewrittenExpr->resType; - strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName); - strcpy(pCol->node.userAlias, ((SExprNode*)pExpr)->userAlias); - strcpy(pCol->colName, ((SExprNode*)pExpr)->aliasName); + tstrncpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName, TSDB_COL_NAME_LEN); + tstrncpy(pCol->node.userAlias, ((SExprNode*)pExpr)->userAlias, TSDB_COL_NAME_LEN); + tstrncpy(pCol->colName, ((SExprNode*)pExpr)->aliasName, TSDB_COL_NAME_LEN); pCol->node.projIdx = ((SExprNode*)(*pNode))->projIdx; if (QUERY_NODE_FUNCTION == nodeType(pExpr)) { setColumnInfo((SFunctionNode*)pExpr, pCol, pCxt->isPartitionBy); @@ -150,7 +150,7 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { case QUERY_NODE_LOGIC_CONDITION: case QUERY_NODE_FUNCTION: { if ('\0' == ((SExprNode*)pNode)->aliasName[0]) { - sprintf(((SExprNode*)pNode)->aliasName, "#expr_%p", pNode); + snprintf(((SExprNode*)pNode)->aliasName, TSDB_COL_NAME_LEN, "#expr_%p", pNode); } return DEAL_RES_IGNORE_CHILD; } @@ -311,12 +311,12 @@ static SNode* createFirstCol(SRealTableNode* pTable, const SSchema* pSchema) { pCol->tableId = pTable->pMeta->uid; pCol->colId = pSchema->colId; pCol->colType = COLUMN_TYPE_COLUMN; - strcpy(pCol->tableAlias, pTable->table.tableAlias); - strcpy(pCol->tableName, pTable->table.tableName); + tstrncpy(pCol->tableAlias, pTable->table.tableAlias, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->tableName, pTable->table.tableName, TSDB_TABLE_NAME_LEN); pCol->isPk = pSchema->flags & COL_IS_KEY; pCol->tableHasPk = hasPkInTable(pTable->pMeta); pCol->numOfPKs = pTable->pMeta->tableInfo.numOfPKs; - strcpy(pCol->colName, pSchema->name); + tstrncpy(pCol->colName, pSchema->name, TSDB_COL_NAME_LEN); return (SNode*)pCol; } @@ -392,8 +392,8 @@ static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealT pScan->scanRange = TSWINDOW_INITIALIZER; pScan->tableName.type = TSDB_TABLE_NAME_T; pScan->tableName.acctId = pCxt->pPlanCxt->acctId; - strcpy(pScan->tableName.dbname, pRealTable->table.dbName); - strcpy(pScan->tableName.tname, pRealTable->table.tableName); + tstrncpy(pScan->tableName.dbname, pRealTable->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pScan->tableName.tname, pRealTable->table.tableName, TSDB_TABLE_NAME_LEN); pScan->showRewrite = pCxt->pPlanCxt->showRewrite; pScan->ratio = pRealTable->ratio; pScan->dataRequired = FUNC_DATA_REQUIRED_DATA_LOAD; @@ -785,12 +785,12 @@ static int32_t addWinJoinPrimKeyToAggFuncs(SSelectStmt* pSelect, SNodeList** pLi } SSchema* pColSchema = &pProbeTable->pMeta->schema[0]; - strcpy(pCol->dbName, pProbeTable->table.dbName); - strcpy(pCol->tableAlias, pProbeTable->table.tableAlias); - strcpy(pCol->tableName, pProbeTable->table.tableName); - strcpy(pCol->colName, pColSchema->name); - strcpy(pCol->node.aliasName, pColSchema->name); - strcpy(pCol->node.userAlias, pColSchema->name); + tstrncpy(pCol->dbName, pProbeTable->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pCol->tableAlias, pProbeTable->table.tableAlias, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->tableName, pProbeTable->table.tableName, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->colName, pColSchema->name, TSDB_COL_NAME_LEN); + tstrncpy(pCol->node.aliasName, pColSchema->name, TSDB_COL_NAME_LEN); + tstrncpy(pCol->node.userAlias, pColSchema->name, TSDB_COL_NAME_LEN); pCol->tableId = pProbeTable->pMeta->uid; pCol->tableType = pProbeTable->pMeta->tableType; pCol->colId = pColSchema->colId; @@ -1557,7 +1557,7 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect if (TSDB_CODE_SUCCESS == code) { pSort->pSortKeys = NULL; code = nodesCloneList(pSelect->pOrderByList, &pSort->pSortKeys); - if (NULL != pSort->pSortKeys) { + if (TSDB_CODE_SUCCESS == code) { SNode* pNode = NULL; SOrderByExprNode* firstSortKey = (SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0); if (isPrimaryKeySort(pSelect->pOrderByList)) pSort->node.outputTsOrder = firstSortKey->order; @@ -1624,10 +1624,7 @@ static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSel pProject->pProjections = NULL; code = nodesCloneList(pSelect->pProjectionList, &pProject->pProjections); - if (NULL == pProject->pProjections) { - code = code; - } - strcpy(pProject->stmtName, pSelect->stmtName); + tstrncpy(pProject->stmtName, pSelect->stmtName, TSDB_TABLE_NAME_LEN); if (TSDB_CODE_SUCCESS == code) { code = createColumnByProjections(pCxt, pSelect->stmtName, pSelect->pProjectionList, &pProject->node.pTargets); @@ -2117,7 +2114,7 @@ static int32_t createVnodeModifLogicNodeByDelete(SLogicPlanContext* pCxt, SDelet pModify->tableId = pRealTable->pMeta->uid; pModify->tableType = pRealTable->pMeta->tableType; snprintf(pModify->tableName, sizeof(pModify->tableName), "%s", pRealTable->table.tableName); - strcpy(pModify->tsColName, pRealTable->pMeta->schema->name); + tstrncpy(pModify->tsColName, pRealTable->pMeta->schema->name, TSDB_COL_NAME_LEN); pModify->deleteTimeRange = pDelete->timeRange; pModify->pAffectedRows = NULL; code = nodesCloneNode(pDelete->pCountFunc, &pModify->pAffectedRows); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 8bcd2a9619..00d897cf8b 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -188,7 +188,7 @@ static EDealRes optRebuildTbanme(SNode** pNode, void* pContext) { *(int32_t*)pContext = code; return DEAL_RES_ERROR; } - strcpy(pFunc->functionName, "tbname"); + tstrncpy(pFunc->functionName, "tbname", TSDB_FUNC_NAME_LEN); pFunc->funcType = FUNCTION_TYPE_TBNAME; pFunc->node.resType = ((SColumnNode*)*pNode)->node.resType; nodesDestroyNode(*pNode); @@ -1302,9 +1302,7 @@ static int32_t pdcJoinAddPreFilterColsToTarget(SOptimizeContext* pCxt, SJoinLogi SNodeList* pCondCols = NULL; code = nodesMakeList(&pCondCols); SNodeList* pTargets = NULL; - if (NULL == pCondCols) { - code = code; - } else { + if (TSDB_CODE_SUCCESS == code) { code = nodesCollectColumnsFromNode(pJoin->pColOnCond, NULL, COLLECT_COL_TYPE_ALL, &pCondCols); } if (TSDB_CODE_SUCCESS == code) { @@ -2959,9 +2957,9 @@ static int32_t smaIndexOptCreateSmaCol(SNode* pFunc, uint64_t tableId, int32_t c pCol->tableType = TSDB_SUPER_TABLE; pCol->colId = colId; pCol->colType = COLUMN_TYPE_COLUMN; - strcpy(pCol->colName, ((SExprNode*)pFunc)->aliasName); + tstrncpy(pCol->colName, ((SExprNode*)pFunc)->aliasName, TSDB_COL_NAME_LEN); pCol->node.resType = ((SExprNode*)pFunc)->resType; - strcpy(pCol->node.aliasName, ((SExprNode*)pFunc)->aliasName); + tstrncpy(pCol->node.aliasName, ((SExprNode*)pFunc)->aliasName, TSDB_COL_NAME_LEN); *ppNode = pCol; return code; } @@ -3030,7 +3028,7 @@ static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNo } SExprNode exprNode; exprNode.resType = ((SExprNode*)pWsNode)->resType; - sprintf(exprNode.aliasName, "#expr_%d", index + 1); + snprintf(exprNode.aliasName, TSDB_COL_NAME_LEN, "#expr_%d", index + 1); SColumnNode* pkNode = NULL; code = smaIndexOptCreateSmaCol((SNode*)&exprNode, tableId, PRIMARYKEY_TIMESTAMP_COL_ID, &pkNode); if (TSDB_CODE_SUCCESS != code) { @@ -3200,7 +3198,7 @@ static void partTagsSetAlias(char* pAlias, const char* pTableAlias, const char* int32_t len = tsnprintf(name, TSDB_COL_FNAME_LEN, "%s.%s", pTableAlias, pColName); (void)taosHashBinary(name, len); - strncpy(pAlias, name, TSDB_COL_NAME_LEN - 1); + tstrncpy(pAlias, name, TSDB_COL_NAME_LEN); } static int32_t partTagsCreateWrapperFunc(const char* pFuncName, SNode* pNode, SFunctionNode** ppNode) { @@ -3218,7 +3216,7 @@ static int32_t partTagsCreateWrapperFunc(const char* pFuncName, SNode* pNode, SF SColumnNode* pCol = (SColumnNode*)pNode; partTagsSetAlias(pFunc->node.aliasName, pCol->tableAlias, pCol->colName); } else { - strcpy(pFunc->node.aliasName, ((SExprNode*)pNode)->aliasName); + tstrncpy(pFunc->node.aliasName, ((SExprNode*)pNode)->aliasName, TSDB_COL_NAME_LEN); } code = nodesCloneNode(pNode, &pNew); if (TSDB_CODE_SUCCESS == code) { @@ -3507,7 +3505,7 @@ static EDealRes eliminateProjOptRewriteScanTableAlias(SNode* pNode, void* pConte if (QUERY_NODE_COLUMN == nodeType(pNode)) { SColumnNode* pCol = (SColumnNode*)pNode; RewriteTableAliasCxt* pCtx = (RewriteTableAliasCxt*)pContext; - strncpy(pCol->tableAlias, pCtx->newTableAlias, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->tableAlias, pCtx->newTableAlias, TSDB_TABLE_NAME_LEN); } return DEAL_RES_CONTINUE; } @@ -3761,7 +3759,7 @@ static int32_t rewriteTailOptCreateProjectExpr(SFunctionNode* pFunc, SNode** ppN if (NULL == pExpr) { return code; } - strcpy(((SExprNode*)pExpr)->aliasName, pFunc->node.aliasName); + tstrncpy(((SExprNode*)pExpr)->aliasName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); *ppNode = pExpr; return code; } @@ -3917,15 +3915,15 @@ static int32_t rewriteUniqueOptCreateFirstFunc(SFunctionNode* pSelectValue, SNod return code; } - strcpy(pFunc->functionName, "first"); + tstrncpy(pFunc->functionName, "first", TSDB_FUNC_NAME_LEN); if (NULL != pSelectValue) { - strcpy(pFunc->node.aliasName, pSelectValue->node.aliasName); + tstrncpy(pFunc->node.aliasName, pSelectValue->node.aliasName, TSDB_COL_NAME_LEN); } else { int64_t pointer = (int64_t)pFunc; char name[TSDB_FUNC_NAME_LEN + TSDB_POINTER_PRINT_BYTES + TSDB_NAME_DELIMITER_LEN + 1] = {0}; int32_t len = tsnprintf(name, sizeof(name) - 1, "%s.%" PRId64 "", pFunc->functionName, pointer); (void)taosHashBinary(name, len); - strncpy(pFunc->node.aliasName, name, TSDB_COL_NAME_LEN - 1); + tstrncpy(pFunc->node.aliasName, name, TSDB_COL_NAME_LEN); } SNode* pNew = NULL; code = nodesCloneNode(pCol, &pNew); @@ -4023,15 +4021,15 @@ static int32_t rewriteUniqueOptCreateProjectCol(SFunctionNode* pFunc, SNode** pp if (FUNCTION_TYPE_UNIQUE == pFunc->funcType) { SExprNode* pExpr = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0); if (QUERY_NODE_COLUMN == nodeType(pExpr)) { - strcpy(pCol->tableAlias, ((SColumnNode*)pExpr)->tableAlias); - strcpy(pCol->colName, ((SColumnNode*)pExpr)->colName); + tstrncpy(pCol->tableAlias, ((SColumnNode*)pExpr)->tableAlias, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->colName, ((SColumnNode*)pExpr)->colName, TSDB_COL_NAME_LEN); } else { - strcpy(pCol->colName, pExpr->aliasName); + tstrncpy(pCol->colName, pExpr->aliasName, TSDB_COL_NAME_LEN); } } else { - strcpy(pCol->colName, pFunc->node.aliasName); + tstrncpy(pCol->colName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); } - strcpy(pCol->node.aliasName, pFunc->node.aliasName); + tstrncpy(pCol->node.aliasName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); *ppNode = (SNode*)pCol; return code; } @@ -4405,7 +4403,7 @@ static int32_t lastRowScanBuildFuncTypes(SScanLogicNode* pScan, SColumnNode* pCo return terrno; } pFuncTypeParam->pCol->colId = pColNode->colId; - strcpy(pFuncTypeParam->pCol->name, pColNode->colName); + tstrncpy(pFuncTypeParam->pCol->name, pColNode->colName, TSDB_COL_NAME_LEN); if (NULL == taosArrayPush(pScan->pFuncTypes, pFuncTypeParam)) { taosMemoryFree(pFuncTypeParam); return terrno; @@ -4482,15 +4480,15 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic if (TSDB_CODE_SUCCESS != code) { break; } - sprintf(((SColumnNode*)newColNode)->colName, "#dup_col.%p", newColNode); - sprintf(((SColumnNode*)pParamNode)->colName, "#dup_col.%p", newColNode); + snprintf(((SColumnNode*)newColNode)->colName, TSDB_COL_NAME_LEN, "#dup_col.%p", newColNode); + snprintf(((SColumnNode*)pParamNode)->colName, TSDB_COL_NAME_LEN, "#dup_col.%p", newColNode); if (FUNCTION_TYPE_LAST_ROW == funcType && ((SColumnNode*)pParamNode)->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { if (!adjLastRowTsColName) { adjLastRowTsColName = true; - strncpy(tsColName, ((SColumnNode*)pParamNode)->colName, TSDB_COL_NAME_LEN); + tstrncpy(tsColName, ((SColumnNode*)pParamNode)->colName, TSDB_COL_NAME_LEN); } else { - strncpy(((SColumnNode*)pParamNode)->colName, tsColName, TSDB_COL_NAME_LEN); + tstrncpy(((SColumnNode*)pParamNode)->colName, tsColName, TSDB_COL_NAME_LEN); nodesDestroyNode(newColNode); continue; } @@ -4614,7 +4612,7 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic ((cxt.pLastCols->length == 1 && nodesEqualNode((SNode*)pPKTsCol, nodesListGetNode(cxt.pLastCols, 0))) || (pScan->node.pTargets->length == 2 && cxt.pkBytes > 0))) { // when select last(ts),tbname,ts from ..., we add another ts to targets - sprintf(pPKTsCol->colName, "#sel_val.%p", pPKTsCol); + snprintf(pPKTsCol->colName, TSDB_COL_NAME_LEN, "#sel_val.%p", pPKTsCol); SNode* pNew = NULL; code = nodesCloneNode((SNode*)pPKTsCol, &pNew); if (TSDB_CODE_SUCCESS == code) { @@ -4631,7 +4629,7 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic if (pNonPKCol && cxt.pLastCols->length == 1 && nodesEqualNode((SNode*)pNonPKCol, nodesListGetNode(cxt.pLastCols, 0))) { // when select last(c1), c1 from ..., we add c1 to targets - sprintf(pNonPKCol->colName, "#sel_val.%p", pNonPKCol); + snprintf(pNonPKCol->colName, TSDB_COL_NAME_LEN, "#sel_val.%p", pNonPKCol); SNode* pNew = NULL; code = nodesCloneNode((SNode*)pNonPKCol, &pNew); if (TSDB_CODE_SUCCESS == code) { @@ -5403,10 +5401,10 @@ static bool tbCntScanOptIsEligibleLogicCond(STbCntScanOptInfo* pInfo, SLogicCond } if (!hasDbCond && 0 == strcmp(pCol->colName, "db_name")) { hasDbCond = true; - strcpy(pInfo->table.dbname, pVal->literal); + tstrncpy(pInfo->table.dbname, pVal->literal, TSDB_DB_NAME_LEN); } else if (!hasStbCond && 0 == strcmp(pCol->colName, "stable_name")) { hasStbCond = true; - strcpy(pInfo->table.tname, pVal->literal); + tstrncpy(pInfo->table.tname, pVal->literal, TSDB_TABLE_NAME_LEN); } else { return false; } @@ -5470,8 +5468,8 @@ static int32_t tbCntScanOptCreateTableCountFunc(SNode** ppNode) { if (NULL == pFunc) { return code; } - strcpy(pFunc->functionName, "_table_count"); - strcpy(pFunc->node.aliasName, "_table_count"); + tstrncpy(pFunc->functionName, "_table_count", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.aliasName, "_table_count", TSDB_COL_NAME_LEN); code = fmGetFuncInfo(pFunc, NULL, 0); if (TSDB_CODE_SUCCESS != code) { nodesDestroyNode((SNode*)pFunc); @@ -5483,8 +5481,8 @@ static int32_t tbCntScanOptCreateTableCountFunc(SNode** ppNode) { static int32_t tbCntScanOptRewriteScan(STbCntScanOptInfo* pInfo) { pInfo->pScan->scanType = SCAN_TYPE_TABLE_COUNT; - strcpy(pInfo->pScan->tableName.dbname, pInfo->table.dbname); - strcpy(pInfo->pScan->tableName.tname, pInfo->table.tname); + tstrncpy(pInfo->pScan->tableName.dbname, pInfo->table.dbname, TSDB_DB_NAME_LEN); + tstrncpy(pInfo->pScan->tableName.tname, pInfo->table.tname, TSDB_TABLE_NAME_LEN); NODES_DESTORY_LIST(pInfo->pScan->node.pTargets); NODES_DESTORY_LIST(pInfo->pScan->pScanCols); NODES_DESTORY_NODE(pInfo->pScan->node.pConditions); @@ -5534,8 +5532,8 @@ static int32_t tbCntScanOptCreateSumFunc(SFunctionNode* pCntFunc, SNode* pParam, if (NULL == pFunc) { return code; } - strcpy(pFunc->functionName, "sum"); - strcpy(pFunc->node.aliasName, pCntFunc->node.aliasName); + tstrncpy(pFunc->functionName, "sum", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.aliasName, pCntFunc->node.aliasName, TSDB_COL_NAME_LEN); code = createColumnByRewriteExpr(pParam, &pFunc->pParameterList); if (TSDB_CODE_SUCCESS == code) { code = fmGetFuncInfo(pFunc, NULL, 0); @@ -6312,29 +6310,22 @@ static int32_t stbJoinOptCreateDynQueryCtrlNode(SLogicNode* pRoot, SLogicNode* p if (TSDB_CODE_SUCCESS == code) { pDynCtrl->node.pChildren = NULL; code = nodesMakeList(&pDynCtrl->node.pChildren); - if (NULL == pDynCtrl->node.pChildren) { - code = code; - } } if (TSDB_CODE_SUCCESS == code) { pDynCtrl->stbJoin.pVgList = NULL; code = nodesMakeList(&pDynCtrl->stbJoin.pVgList); - if (NULL == pDynCtrl->stbJoin.pVgList) { - code = code; - } } if (TSDB_CODE_SUCCESS == code) { pDynCtrl->stbJoin.pUidList = NULL; code = nodesMakeList(&pDynCtrl->stbJoin.pUidList); - if (NULL == pDynCtrl->stbJoin.pUidList) { - code = code; - } } SJoinLogicNode* pHJoin = (SJoinLogicNode*)pPrev; - code = nodesListStrictAppend(pDynCtrl->stbJoin.pUidList, nodesListGetNode(pHJoin->node.pTargets, 0)); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppend(pDynCtrl->stbJoin.pUidList, nodesListGetNode(pHJoin->node.pTargets, 0)); + } if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pDynCtrl->stbJoin.pUidList, nodesListGetNode(pHJoin->node.pTargets, 2)); } @@ -6353,9 +6344,6 @@ static int32_t stbJoinOptCreateDynQueryCtrlNode(SLogicNode* pRoot, SLogicNode* p if (TSDB_CODE_SUCCESS == code) { pDynCtrl->node.pTargets = NULL; code = nodesCloneList(pPost->pTargets, &pDynCtrl->node.pTargets); - if (!pDynCtrl->node.pTargets) { - code = code; - } } } @@ -7117,10 +7105,9 @@ int32_t tsmaOptCreateTsmaScanCols(const STSMAOptUsefulTsma* pTsma, const SNodeLi pCol->tableType = TSDB_SUPER_TABLE; pCol->tableId = pTsma->targetTbUid; pCol->colType = COLUMN_TYPE_COLUMN; - strcpy(pCol->tableName, pTsma->targetTbName); - strcpy(pCol->dbName, pTsma->pTsma->targetDbFName); - strcpy(pCol->colName, pFunc->node.aliasName); - strcpy(pCol->node.aliasName, pFunc->node.aliasName); + tstrncpy(pCol->tableName, pTsma->targetTbName, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->colName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); + tstrncpy(pCol->node.aliasName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); pCol->node.resType.type = TSDB_DATA_TYPE_BINARY; code = nodesListMakeStrictAppend(&pScanCols, (SNode*)pCol); } @@ -7144,8 +7131,8 @@ static int32_t tsmaOptRewriteTag(const STSMAOptCtx* pTsmaOptCtx, const STSMAOptU for (int32_t i = 0; i < pTsma->pTsma->pTags->size; ++i) { const SSchema* pSchema = taosArrayGet(pTsma->pTsma->pTags, i); if (strcmp(pTagCol->colName, pSchema->name) == 0) { - strcpy(pTagCol->tableName, pTsma->targetTbName); - strcpy(pTagCol->tableAlias, pTsma->targetTbName); + tstrncpy(pTagCol->tableName, pTsma->targetTbName, TSDB_TABLE_NAME_LEN); + tstrncpy(pTagCol->tableAlias, pTsma->targetTbName, TSDB_TABLE_NAME_LEN); pTagCol->tableId = pTsma->targetTbUid; pTagCol->tableType = TSDB_SUPER_TABLE; pTagCol->colId = pSchema->colId; @@ -7170,8 +7157,8 @@ static int32_t tsmaOptRewriteTbname(const STSMAOptCtx* pTsmaOptCtx, SNode** pTbN nodesDestroyNode(*pTbNameNode); SColumnNode* pCol = (SColumnNode*)pRewrittenFunc; const SSchema* pSchema = taosArrayGet(pTsma->pTsma->pTags, pTsma->pTsma->pTags->size - 1); - strcpy(pCol->tableName, pTsma->targetTbName); - strcpy(pCol->tableAlias, pTsma->targetTbName); + tstrncpy(pCol->tableName, pTsma->targetTbName, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->tableAlias, pTsma->targetTbName, TSDB_TABLE_NAME_LEN); pCol->tableId = pTsma->targetTbUid; pCol->tableType = TSDB_SUPER_TABLE; pCol->colId = pSchema->colId; @@ -7296,7 +7283,7 @@ static int32_t tsmaOptRewriteScan(STSMAOptCtx* pTsmaOptCtx, SScanLogicNode* pNew if (code == TSDB_CODE_SUCCESS) { pNewScan->stableId = pTsma->pTsma->destTbUid; pNewScan->tableId = pTsma->targetTbUid; - strcpy(pNewScan->tableName.tname, pTsma->targetTbName); + tstrncpy(pNewScan->tableName.tname, pTsma->targetTbName, TSDB_TABLE_NAME_LEN); } if (code == TSDB_CODE_SUCCESS) { code = tsmaOptRewriteNodeList(pNewScan->pScanPseudoCols, pTsmaOptCtx, pTsma, true, true); @@ -7349,12 +7336,12 @@ static int32_t tsmaOptCreateWStart(int8_t precision, SFunctionNode** pWStartOut) if (NULL == pWStart) { return code; } - strcpy(pWStart->functionName, "_wstart"); + tstrncpy(pWStart->functionName, "_wstart", TSDB_FUNC_NAME_LEN); int64_t pointer = (int64_t)pWStart; char name[TSDB_COL_NAME_LEN + TSDB_POINTER_PRINT_BYTES + TSDB_NAME_DELIMITER_LEN + 1] = {0}; int32_t len = tsnprintf(name, sizeof(name) - 1, "%s.%" PRId64 "", pWStart->functionName, pointer); (void)taosHashBinary(name, len); - strncpy(pWStart->node.aliasName, name, TSDB_COL_NAME_LEN - 1); + tstrncpy(pWStart->node.aliasName, name, TSDB_COL_NAME_LEN); pWStart->node.resType.precision = precision; code = fmGetFuncInfo(pWStart, NULL, 0); @@ -7457,12 +7444,12 @@ static int32_t tsmaOptGeneratePlan(STSMAOptCtx* pTsmaOptCtx) { for (int32_t j = 0; j < pTsmaOptCtx->pScan->pTsmas->size; ++j) { if (taosArrayGetP(pTsmaOptCtx->pScan->pTsmas, j) == pTsma->pTsma) { const STsmaTargetTbInfo* ptbInfo = taosArrayGet(pTsmaOptCtx->pScan->pTsmaTargetTbInfo, j); - strcpy(pTsma->targetTbName, ptbInfo->tableName); + tstrncpy(pTsma->targetTbName, ptbInfo->tableName, TSDB_TABLE_NAME_LEN); pTsma->targetTbUid = ptbInfo->uid; } } } else { - strcpy(pTsma->targetTbName, pTsma->pTsma->targetTb); + tstrncpy(pTsma->targetTbName, pTsma->pTsma->targetTb, TSDB_TABLE_NAME_LEN); pTsma->targetTbUid = pTsma->pTsma->destTbUid; } } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index b2e36aeda6..ca3187e04f 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -40,9 +40,9 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int if (!*ppKey) { return terrno; } - strcat(*ppKey, pStmtName); - strcat(*ppKey, "."); - strcat(*ppKey, pCol->node.aliasName); + TAOS_STRNCAT(*ppKey, pStmtName, TSDB_TABLE_NAME_LEN); + TAOS_STRNCAT(*ppKey, ".", 2); + TAOS_STRNCAT(*ppKey, pCol->node.aliasName, TSDB_COL_NAME_LEN); *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); return code; } else { @@ -50,7 +50,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int if (!*ppKey) { return terrno; } - strcat(*ppKey, pCol->node.aliasName); + TAOS_STRNCAT(*ppKey, pCol->node.aliasName, TSDB_COL_NAME_LEN); *pLen = strlen(*ppKey); return code; } @@ -60,7 +60,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int if (!*ppKey) { return terrno; } - strcat(*ppKey, pCol->colName); + TAOS_STRNCAT(*ppKey, pCol->colName, TSDB_COL_NAME_LEN); *pLen = strlen(*ppKey); return code; } @@ -69,9 +69,9 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int if (!*ppKey) { return terrno; } - strcat(*ppKey, pCol->tableAlias); - strcat(*ppKey, "."); - strcat(*ppKey, pCol->colName); + TAOS_STRNCAT(*ppKey, pCol->tableAlias, TSDB_TABLE_NAME_LEN); + TAOS_STRNCAT(*ppKey, ".", 2); + TAOS_STRNCAT(*ppKey, pCol->colName, TSDB_COL_NAME_LEN); *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); return code; } else if (QUERY_NODE_FUNCTION == nodeType(pNode)) { @@ -84,9 +84,9 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int if (!*ppKey) { return terrno; } - strcat(*ppKey, pStmtName); - strcat(*ppKey, "."); - strcat(*ppKey, ((SExprNode*)pNode)->aliasName); + TAOS_STRNCAT(*ppKey, pStmtName, TSDB_TABLE_NAME_LEN); + TAOS_STRNCAT(*ppKey, ".", 2); + TAOS_STRNCAT(*ppKey, ((SExprNode*)pNode)->aliasName, TSDB_COL_NAME_LEN); *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); return code; } @@ -94,9 +94,9 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int if (!*ppKey) { return terrno; } - strcat(*ppKey, pVal->literal); - strcat(*ppKey, "."); - strcat(*ppKey, ((SExprNode*)pNode)->aliasName); + TAOS_STRNCAT(*ppKey, pVal->literal, strlen(pVal->literal)); + TAOS_STRNCAT(*ppKey, ".", 2); + TAOS_STRNCAT(*ppKey, ((SExprNode*)pNode)->aliasName, TSDB_COL_NAME_LEN); *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); return code; } @@ -108,9 +108,9 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int if (!*ppKey) { return terrno; } - strcat(*ppKey, pStmtName); - strcat(*ppKey, "."); - strcat(*ppKey, ((SExprNode*)pNode)->aliasName); + TAOS_STRNCAT(*ppKey, pStmtName, TSDB_TABLE_NAME_LEN); + TAOS_STRNCAT(*ppKey, ".", 2); + TAOS_STRNCAT(*ppKey, ((SExprNode*)pNode)->aliasName, TSDB_COL_NAME_LEN); *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); return code; } @@ -119,7 +119,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int if (!*ppKey) { return terrno; } - strcat(*ppKey, ((SExprNode*)pNode)->aliasName); + TAOS_STRNCAT(*ppKey, ((SExprNode*)pNode)->aliasName, TSDB_COL_NAME_LEN); *pLen = strlen(*ppKey); return code; } @@ -228,7 +228,7 @@ static int32_t buildDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SD code = putSlotToHash(name, len, pDataBlockDesc->dataBlockId, slotId, pNode, pHash); if (TSDB_CODE_SUCCESS == code) { if (nodeType(pNode) == QUERY_NODE_COLUMN && ((SColumnNode*)pNode)->resIdx > 0) { - sprintf(name + strlen(name), "_%d", ((SColumnNode*)pNode)->resIdx); + snprintf(name + strlen(name), 16, "_%d", ((SColumnNode*)pNode)->resIdx); code = putSlotToHash(name, strlen(name), pDataBlockDesc->dataBlockId, slotId, pNode, pProjIdxDescHash); } } @@ -395,7 +395,7 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) { } SSlotIndex* pIndex = NULL; if (((SColumnNode*)pNode)->projRefIdx > 0) { - sprintf(name + strlen(name), "_%d", ((SColumnNode*)pNode)->projRefIdx); + snprintf(name + strlen(name), 16, "_%d", ((SColumnNode*)pNode)->projRefIdx); pIndex = taosHashGet(pCxt->pLeftProjIdxHash, name, strlen(name)); if (!pIndex) { pIndex = taosHashGet(pCxt->pRightProdIdxHash, name, strlen(name)); @@ -566,9 +566,6 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SSubplan* pS if (TSDB_CODE_SUCCESS == code && NULL != pScanLogicNode->pScanPseudoCols) { pScanPhysiNode->pScanPseudoCols = NULL; code = nodesCloneList(pScanLogicNode->pScanPseudoCols, &pScanPhysiNode->pScanPseudoCols); - if (NULL == pScanPhysiNode->pScanPseudoCols) { - code = code; - } } if (TSDB_CODE_SUCCESS == code) { @@ -588,9 +585,6 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SSubplan* pS if (NULL != pScanLogicNode->pTagCond) { pSubplan->pTagCond = NULL; code = nodesCloneNode(pScanLogicNode->pTagCond, &pSubplan->pTagCond); - if (NULL == pSubplan->pTagCond) { - code = code; - } } } @@ -598,9 +592,6 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SSubplan* pS if (NULL != pScanLogicNode->pTagIndexCond) { pSubplan->pTagIndexCond = NULL; code = nodesCloneNode(pScanLogicNode->pTagIndexCond, &pSubplan->pTagIndexCond); - if (NULL == pSubplan->pTagIndexCond) { - code = code; - } } } @@ -1690,11 +1681,11 @@ static EDealRes collectAndRewrite(SRewritePrecalcExprsCxt* pCxt, SNode** pNode) SExprNode* pRewrittenExpr = (SExprNode*)pExpr; pCol->node.resType = pRewrittenExpr->resType; if ('\0' != pRewrittenExpr->aliasName[0]) { - strcpy(pCol->colName, pRewrittenExpr->aliasName); + tstrncpy(pCol->colName, pRewrittenExpr->aliasName, TSDB_COL_NAME_LEN); } else { snprintf(pRewrittenExpr->aliasName, sizeof(pRewrittenExpr->aliasName), "#expr_%d_%d", pCxt->planNodeId, pCxt->rewriteId); - strcpy(pCol->colName, pRewrittenExpr->aliasName); + tstrncpy(pCol->colName, pRewrittenExpr->aliasName, TSDB_COL_NAME_LEN); } nodesDestroyNode(*pNode); *pNode = (SNode*)pCol; @@ -1715,7 +1706,7 @@ static int32_t rewriteValueToOperator(SRewritePrecalcExprsCxt* pCxt, SNode** pNo } SValueNode* pVal = (SValueNode*)*pNode; pOper->node.resType = pVal->node.resType; - strcpy(pOper->node.aliasName, pVal->node.aliasName); + tstrncpy(pOper->node.aliasName, pVal->node.aliasName, TSDB_COL_NAME_LEN); pOper->opType = OP_TYPE_ASSIGN; pOper->pRight = *pNode; *pNode = (SNode*)pOper; @@ -1976,9 +1967,6 @@ static int32_t createInterpFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pCh pInterpFunc->rangeInterval = pFuncLogicNode->rangeInterval; pInterpFunc->rangeIntervalUnit = pFuncLogicNode->rangeIntervalUnit; code = nodesCloneNode(pFuncLogicNode->pFillValues, &pInterpFunc->pFillValues); - if (TSDB_CODE_SUCCESS != code) { - code = code; - } } if (TSDB_CODE_SUCCESS == code) { @@ -2894,7 +2882,7 @@ static int32_t createQueryInserter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNod pInserter->tableId = pModify->tableId; pInserter->stableId = pModify->stableId; pInserter->tableType = pModify->tableType; - strcpy(pInserter->tableName, pModify->tableName); + tstrncpy(pInserter->tableName, pModify->tableName, TSDB_TABLE_NAME_LEN); pInserter->explain = (QUERY_NODE_EXPLAIN_STMT == nodeType(pCxt->pPlanCxt->pAstRoot) ? true : false); if (pModify->pVgroupList) { pInserter->vgId = pModify->pVgroupList->vgroups[0].vgId; @@ -2906,9 +2894,6 @@ static int32_t createQueryInserter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNod if (TSDB_CODE_SUCCESS == code) { pInserter->sink.pInputDataBlockDesc = NULL; code = nodesCloneNode((SNode*)pSubplan->pNode->pOutputDataBlockDesc, (SNode**)&pInserter->sink.pInputDataBlockDesc); - if (NULL == pInserter->sink.pInputDataBlockDesc) { - code = code; - } } if (TSDB_CODE_SUCCESS == code) { @@ -2947,8 +2932,8 @@ static int32_t createDataDeleter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pDeleter->tableId = pModify->tableId; pDeleter->tableType = pModify->tableType; - strcpy(pDeleter->tableFName, pModify->tableName); - strcpy(pDeleter->tsColName, pModify->tsColName); + tstrncpy(pDeleter->tableFName, pModify->tableName, TSDB_TABLE_NAME_LEN); + tstrncpy(pDeleter->tsColName, pModify->tsColName, TSDB_COL_NAME_LEN); pDeleter->deleteTimeRange = pModify->deleteTimeRange; code = setNodeSlotId(pCxt, pRoot->pOutputDataBlockDesc->dataBlockId, -1, pModify->pAffectedRows, @@ -2962,9 +2947,6 @@ static int32_t createDataDeleter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* if (TSDB_CODE_SUCCESS == code) { pDeleter->sink.pInputDataBlockDesc = NULL; code = nodesCloneNode((SNode*)pRoot->pOutputDataBlockDesc, (SNode**)&pDeleter->sink.pInputDataBlockDesc); - if (NULL == pDeleter->sink.pInputDataBlockDesc) { - code = code; - } } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index e960c0ff5d..5c2d1efd7b 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -428,12 +428,12 @@ static int32_t stbSplAppendWStart(SNodeList* pFuncs, int32_t* pIndex, uint8_t pr if (NULL == pWStart) { return code; } - strcpy(pWStart->functionName, "_wstart"); + tstrncpy(pWStart->functionName, "_wstart", TSDB_FUNC_NAME_LEN); int64_t pointer = (int64_t)pWStart; char name[TSDB_COL_NAME_LEN + TSDB_POINTER_PRINT_BYTES + TSDB_NAME_DELIMITER_LEN + 1] = {0}; int32_t len = tsnprintf(name, sizeof(name) - 1, "%s.%" PRId64 "", pWStart->functionName, pointer); (void)taosHashBinary(name, len); - strncpy(pWStart->node.aliasName, name, TSDB_COL_NAME_LEN - 1); + tstrncpy(pWStart->node.aliasName, name, TSDB_COL_NAME_LEN); pWStart->node.resType.precision = precision; code = fmGetFuncInfo(pWStart, NULL, 0); @@ -460,12 +460,12 @@ static int32_t stbSplAppendWEnd(SWindowLogicNode* pWin, int32_t* pIndex) { if (NULL == pWEnd) { return code; } - strcpy(pWEnd->functionName, "_wend"); + tstrncpy(pWEnd->functionName, "_wend", TSDB_FUNC_NAME_LEN); int64_t pointer = (int64_t)pWEnd; char name[TSDB_COL_NAME_LEN + TSDB_POINTER_PRINT_BYTES + TSDB_NAME_DELIMITER_LEN + 1] = {0}; int32_t len = tsnprintf(name, sizeof(name) - 1, "%s.%" PRId64 "", pWEnd->functionName, pointer); (void)taosHashBinary(name, len); - strncpy(pWEnd->node.aliasName, name, TSDB_COL_NAME_LEN - 1); + tstrncpy(pWEnd->node.aliasName, name, TSDB_COL_NAME_LEN); code = fmGetFuncInfo(pWEnd, NULL, 0); if (TSDB_CODE_SUCCESS == code) { @@ -1135,8 +1135,8 @@ static int32_t stbSplAggNodeCreateMerge(SSplitContext* pCtx, SStableSplitInfo* p if (!nodesEqualNode(pParam, (SNode*)pCol)) continue; // use the colName of group_key func to make sure finding the right slot id for merge keys. - strcpy(pCol->colName, pFunc->node.aliasName); - strcpy(pCol->node.aliasName, pFunc->node.aliasName); + tstrncpy(pCol->colName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); + tstrncpy(pCol->node.aliasName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); memset(pCol->tableAlias, 0, TSDB_TABLE_NAME_LEN); break; } @@ -1265,15 +1265,15 @@ static int32_t stbSplCreateColumnNode(SExprNode* pExpr, SNode** ppNode) { return code; } if (QUERY_NODE_COLUMN == nodeType(pExpr)) { - strcpy(pCol->dbName, ((SColumnNode*)pExpr)->dbName); - strcpy(pCol->tableName, ((SColumnNode*)pExpr)->tableName); - strcpy(pCol->tableAlias, ((SColumnNode*)pExpr)->tableAlias); - strcpy(pCol->colName, ((SColumnNode*)pExpr)->colName); + tstrncpy(pCol->dbName, ((SColumnNode*)pExpr)->dbName, TSDB_DB_NAME_LEN); + tstrncpy(pCol->tableName, ((SColumnNode*)pExpr)->tableName, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->tableAlias, ((SColumnNode*)pExpr)->tableAlias, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->colName, ((SColumnNode*)pExpr)->colName, TSDB_COL_NAME_LEN); } else { - strcpy(pCol->colName, pExpr->aliasName); + tstrncpy(pCol->colName, pExpr->aliasName, TSDB_COL_NAME_LEN); } - strcpy(pCol->node.aliasName, pExpr->aliasName); - strcpy(pCol->node.userAlias, pExpr->userAlias); + tstrncpy(pCol->node.aliasName, pExpr->aliasName, TSDB_COL_NAME_LEN); + tstrncpy(pCol->node.userAlias, pExpr->userAlias, TSDB_COL_NAME_LEN); pCol->node.resType = pExpr->resType; *ppNode = (SNode*)pCol; return code; @@ -1358,9 +1358,6 @@ static int32_t stbSplCreatePartSortNode(SSortLogicNode* pSort, SLogicNode** pOut int32_t code = TSDB_CODE_SUCCESS; SSortLogicNode* pPartSort = NULL; code = nodesCloneNode((SNode*)pSort, (SNode**)&pPartSort); - if (NULL == pPartSort) { - code = code; - } SNodeList* pMergeKeys = NULL; if (TSDB_CODE_SUCCESS == code) { @@ -1541,9 +1538,6 @@ static int32_t stbSplCreateMergeScanNode(SScanLogicNode* pScan, SLogicNode** pOu int32_t code = TSDB_CODE_SUCCESS; SScanLogicNode* pMergeScan = NULL; code = nodesCloneNode((SNode*)pScan, (SNode**)&pMergeScan); - if (NULL == pMergeScan) { - code = code; - } SNodeList* pMergeKeys = NULL; if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index fd8670c23e..f03e2d8ab0 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -68,14 +68,14 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) { return DEAL_RES_ERROR; } pCol->node.resType = pExpr->resType; - strcpy(pCol->colName, pExpr->aliasName); + tstrncpy(pCol->colName, pExpr->aliasName, TSDB_COL_NAME_LEN); if (QUERY_NODE_FUNCTION == nodeType(pNode)) { SFunctionNode* pFunc = (SFunctionNode*)pNode; if (pFunc->funcType == FUNCTION_TYPE_TBNAME) { SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); if (NULL != pVal) { - strcpy(pCol->tableAlias, pVal->literal); - strcpy(pCol->tableName, pVal->literal); + tstrncpy(pCol->tableAlias, pVal->literal, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->tableName, pVal->literal, TSDB_TABLE_NAME_LEN); } } } @@ -636,9 +636,9 @@ SFunctionNode* createGroupKeyAggFunc(SColumnNode* pGroupCol) { SFunctionNode* pFunc = NULL; int32_t code = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&pFunc); if (pFunc) { - strcpy(pFunc->functionName, "_group_key"); - strcpy(pFunc->node.aliasName, pGroupCol->node.aliasName); - strcpy(pFunc->node.userAlias, pGroupCol->node.userAlias); + tstrncpy(pFunc->functionName, "_group_key", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.aliasName, pGroupCol->node.aliasName, TSDB_COL_NAME_LEN); + tstrncpy(pFunc->node.userAlias, pGroupCol->node.userAlias, TSDB_COL_NAME_LEN); SNode* pNew = NULL; code = nodesCloneNode((SNode*)pGroupCol, &pNew); if (TSDB_CODE_SUCCESS == code) { @@ -655,7 +655,7 @@ SFunctionNode* createGroupKeyAggFunc(SColumnNode* pGroupCol) { char name[TSDB_FUNC_NAME_LEN + TSDB_NAME_DELIMITER_LEN + TSDB_POINTER_PRINT_BYTES + 1] = {0}; int32_t len = tsnprintf(name, sizeof(name) - 1, "%s.%p", pFunc->functionName, pFunc); (void)taosHashBinary(name, len); - strncpy(pFunc->node.aliasName, name, TSDB_COL_NAME_LEN - 1); + tstrncpy(pFunc->node.aliasName, name, TSDB_COL_NAME_LEN); } } if (TSDB_CODE_SUCCESS != code) { diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index ebb769f054..8c6db7b8ce 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1249,7 +1249,6 @@ EDealRes sclRewriteFunction(SNode **pNode, SScalarCtx *ctx) { ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } - res->node.resType.bytes = varDataTLen(output.columnData->pData); (void)memcpy(res->datum.p, output.columnData->pData, varDataTLen(output.columnData->pData)); } else { ctx->code = nodesSetValueNodeValue(res, output.columnData->pData); diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 9be0e3fc40..37249b5418 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -1129,7 +1129,11 @@ int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask) { int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { // SCH_ERR_JRET(schSetTaskCandidateAddrs(pJob, pTask)); if (NULL == schMgmt.queryMgmt) { - SCH_ERR_RET(qWorkerInit(NODE_TYPE_CLIENT, CLIENT_HANDLE, (void **)&schMgmt.queryMgmt, NULL)); + void* p = NULL; + SCH_ERR_RET(qWorkerInit(NODE_TYPE_CLIENT, CLIENT_HANDLE, &p, NULL)); + if (atomic_val_compare_exchange_ptr(&schMgmt.queryMgmt, NULL, p)) { + qWorkerDestroy(&p); + } } SArray *explainRes = NULL; diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 427733e9ec..e4f921f04a 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -37,7 +37,7 @@ extern "C" { #define META_HB_CHECK_INTERVAL 200 #define META_HB_SEND_IDLE_COUNTER 25 // send hb every 5 sec #define STREAM_TASK_KEY_LEN ((sizeof(int64_t)) << 1) -#define STREAM_TASK_QUEUE_CAPACITY 20480 +#define STREAM_TASK_QUEUE_CAPACITY 5120 #define STREAM_TASK_QUEUE_CAPACITY_IN_SIZE (30) // clang-format off diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index ce7dc79a17..381b852e4e 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -4378,7 +4378,6 @@ void streamStateParTagSeekKeyNext_rocksdb(SStreamState* pState, const int64_t gr } int32_t streamStateParTagGetKVByCur_rocksdb(SStreamStateCur* pCur, int64_t* pGroupId, const void** pVal, int32_t* pVLen) { - stDebug("streamStateFillGetKVByCur_rocksdb"); if (!pCur) { return -1; } diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 64194e349c..01cd201a1b 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -613,7 +613,6 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV { // destroy the related fill-history tasks // drop task should not in the meta-lock, and drop the related fill-history task now - streamMetaWUnLock(pMeta); if (pReq->dropRelHTask) { code = streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); @@ -621,7 +620,6 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV id, vgId, pReq->taskId, numOfTasks); } - streamMetaWLock(pMeta); if (pReq->dropRelHTask) { code = streamMetaCommit(pMeta); } @@ -656,9 +654,11 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV pInfo->processedVer <= pReq->checkpointVer); if (!valid) { - stFatal("invalid checkpoint id check, current checkpointId:%" PRId64 " checkpointVer:%" PRId64 - " processedVer:%" PRId64 " req checkpointId:%" PRId64 " checkpointVer:%" PRId64, - pInfo->checkpointId, pInfo->checkpointVer, pInfo->processedVer, pReq->checkpointId, pReq->checkpointVer); + stFatal("s-task:%s invalid checkpointId update info recv, current checkpointId:%" PRId64 " checkpointVer:%" PRId64 + " processedVer:%" PRId64 " req checkpointId:%" PRId64 " checkpointVer:%" PRId64 " discard it", + id, pInfo->checkpointId, pInfo->checkpointVer, pInfo->processedVer, pReq->checkpointId, + pReq->checkpointVer); + streamMutexUnlock(&pTask->lock); return TSDB_CODE_STREAM_INTERNAL_ERROR; } @@ -695,8 +695,6 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV return TSDB_CODE_SUCCESS; } - streamMetaWUnLock(pMeta); - // drop task should not in the meta-lock, and drop the related fill-history task now if (pReq->dropRelHTask) { code = streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId); @@ -705,9 +703,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV (int32_t)pReq->hTaskId, numOfTasks); } - streamMetaWLock(pMeta); code = streamMetaCommit(pMeta); - return TSDB_CODE_SUCCESS; } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 318720b5b0..85f287f301 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -522,7 +522,10 @@ static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int if (pItem->type == STREAM_INPUT__GET_RES) { const SStreamTrigger* pTrigger = (const SStreamTrigger*)pInput; code = qSetMultiStreamInput(pExecutor, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); - + if (pTask->info.trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) { + stDebug("s-task:%s set force_window_close as source block, skey:%"PRId64, id, pTrigger->pBlock->info.window.skey); + (*pVer) = pTrigger->pBlock->info.window.skey; + } } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)pInput; code = qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); @@ -671,7 +674,7 @@ static int32_t doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock doRecordThroughput(&pTask->execInfo, totalBlocks, totalSize, blockSize, st, pTask->id.idStr); - // update the currentVer if processing the submit blocks. + // update the currentVer if processing the submitted blocks. if (!(pInfo->checkpointVer <= pInfo->nextProcessVer && ver >= pInfo->checkpointVer)) { stError("s-task:%s invalid info, checkpointVer:%" PRId64 ", nextProcessVer:%" PRId64 " currentVer:%" PRId64, id, pInfo->checkpointVer, pInfo->nextProcessVer, ver); @@ -688,6 +691,34 @@ static int32_t doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock return code; } +// do nothing after sync executor state to storage backend, untill checkpoint is completed. +static int32_t doHandleChkptBlock(SStreamTask* pTask) { + int32_t code = 0; + const char* id = pTask->id.idStr; + + streamMutexLock(&pTask->lock); + SStreamTaskState pState = streamTaskGetStatus(pTask); + if (pState.state == TASK_STATUS__CK) { // todo other thread may change the status + stDebug("s-task:%s checkpoint block received, set status:%s", id, pState.name); + code = streamTaskBuildCheckpoint(pTask); // ignore this error msg, and continue + } else { // todo refactor + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + code = streamTaskSendCheckpointSourceRsp(pTask); + } else { + code = streamTaskSendCheckpointReadyMsg(pTask); + } + + if (code != TSDB_CODE_SUCCESS) { + // todo: let's retry send rsp to upstream/mnode + stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%d, code:%s", id, 0, + tstrerror(code)); + } + } + + streamMutexUnlock(&pTask->lock); + return code; +} + int32_t flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock) { const char* id = pTask->id.idStr; @@ -832,36 +863,16 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { } } - if (type != STREAM_INPUT__CHECKPOINT) { + if (type == STREAM_INPUT__CHECKPOINT) { + code = doHandleChkptBlock(pTask); + streamFreeQitem(pInput); + return code; + } else { code = doStreamTaskExecImpl(pTask, pInput, numOfBlocks); streamFreeQitem(pInput); if (code) { return code; } - } else { // todo other thread may change the status - // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed. - streamMutexLock(&pTask->lock); - SStreamTaskState pState = streamTaskGetStatus(pTask); - if (pState.state == TASK_STATUS__CK) { - stDebug("s-task:%s checkpoint block received, set status:%s", id, pState.name); - code = streamTaskBuildCheckpoint(pTask); // ignore this error msg, and continue - } else { // todo refactor - if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { - code = streamTaskSendCheckpointSourceRsp(pTask); - } else { - code = streamTaskSendCheckpointReadyMsg(pTask); - } - - if (code != TSDB_CODE_SUCCESS) { - // todo: let's retry send rsp to upstream/mnode - stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%d, code:%s", id, 0, - tstrerror(code)); - } - } - - streamMutexUnlock(&pTask->lock); - streamFreeQitem(pInput); - return code; } } } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index a6f87711bf..23a98ef3ae 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -501,8 +501,6 @@ _err: void streamMetaInitBackend(SStreamMeta* pMeta) { pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId, pMeta->vgId); if (pMeta->streamBackend == NULL) { - streamMetaWUnLock(pMeta); - while (1) { streamMetaWLock(pMeta); pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId, pMeta->vgId); @@ -908,8 +906,6 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t int32_t code = 0; STaskId id = {.streamId = streamId, .taskId = taskId}; - streamMetaWLock(pMeta); - code = streamMetaAcquireTaskUnsafe(pMeta, &id, &pTask); if (code == 0) { // desc the paused task counter @@ -958,10 +954,8 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t } streamMetaReleaseTask(pMeta, pTask); - streamMetaWUnLock(pMeta); } else { stDebug("vgId:%d failed to find the task:0x%x, it may have been dropped already", vgId, taskId); - streamMetaWUnLock(pMeta); } return 0; diff --git a/source/libs/stream/src/streamSched.c b/source/libs/stream/src/streamSched.c index 56bb7437c1..1f76f349ae 100644 --- a/source/libs/stream/src/streamSched.c +++ b/source/libs/stream/src/streamSched.c @@ -211,6 +211,7 @@ static int32_t doCreateForceWindowTrigger(SStreamTask* pTask, int32_t* pNextTrig const char* id = pTask->id.idStr; int8_t precision = pTask->info.interval.precision; SStreamTrigger* pTrigger = NULL; + bool isFull = false; while (1) { code = streamCreateForcewindowTrigger(&pTrigger, pTask->info.delaySchedParam, &pTask->info.interval, @@ -233,7 +234,6 @@ static int32_t doCreateForceWindowTrigger(SStreamTask* pTask, int32_t* pNextTrig // check whether the time window gaps exist or not int64_t now = taosGetTimestamp(precision); - int64_t ekey = pTrigger->pBlock->info.window.skey + pTask->info.interval.interval; // there are gaps, needs to be filled STimeWindow w = pTrigger->pBlock->info.window; @@ -245,13 +245,18 @@ static int32_t doCreateForceWindowTrigger(SStreamTask* pTask, int32_t* pNextTrig } pTask->status.latestForceWindow = w; - if (ekey + pTask->info.watermark + pTask->info.interval.interval > now) { - int64_t prev = convertTimePrecision(*pNextTrigger, precision, TSDB_TIME_PRECISION_MILLI); + isFull = streamQueueIsFull(pTask->inputq.queue); + + if ((w.ekey + pTask->info.watermark + pTask->info.interval.interval > now) || isFull) { + int64_t prev = convertTimePrecision(*pNextTrigger, precision, TSDB_TIME_PRECISION_MILLI); + if (!isFull) { + *pNextTrigger = w.ekey + pTask->info.watermark + pTask->info.interval.interval - now; + } - *pNextTrigger = ekey + pTask->info.watermark + pTask->info.interval.interval - now; *pNextTrigger = convertTimePrecision(*pNextTrigger, precision, TSDB_TIME_PRECISION_MILLI); - stDebug("s-task:%s generate %d time window(s), trigger delay adjust from %" PRId64 " to %d", id, num, prev, - *pNextTrigger); + pTask->chkInfo.nextProcessVer = w.ekey + pTask->info.interval.interval; + stDebug("s-task:%s generate %d time window(s), trigger delay adjust from %" PRId64 " to %d, set ver:%" PRId64, id, + num, prev, *pNextTrigger, pTask->chkInfo.nextProcessVer); return code; } else { stDebug("s-task:%s gap exist for force_window_close, current force_window_skey:%" PRId64, id, w.skey); @@ -308,7 +313,7 @@ void streamTaskSchedHelper(void* param, void* tmrId) { } if (streamTaskGetStatus(pTask).state == TASK_STATUS__CK) { - nextTrigger = TRIGGER_RECHECK_INTERVAL; // retry in 10 seec + nextTrigger = TRIGGER_RECHECK_INTERVAL; // retry in 10 sec stDebug("s-task:%s in checkpoint procedure, not retrieve result, next:%dms", id, TRIGGER_RECHECK_INTERVAL); } else { if (pTask->info.trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE && pTask->info.taskLevel == TASK_LEVEL__SOURCE) { diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index dcabadb8bd..05edad0f5f 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -947,6 +947,7 @@ int32_t recoverSession(SStreamFileState* pFileState, int64_t ckId) { } SRowBuffPos* pPos = createSessionWinBuff(pFileState, &key, pVal, &vlen); + pPos->beUsed = false; winRes = putSessionWinResultBuff(pFileState, pPos); if (winRes != TSDB_CODE_SUCCESS) { break; @@ -1008,6 +1009,7 @@ int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { memcpy(pNewPos->pRowBuff, pVal, vlen); taosMemoryFreeClear(pVal); pNewPos->beFlushed = true; + pNewPos->beUsed = false; qDebug("===stream=== read checkpoint state from disc. %s", __func__); code = tSimpleHashPut(pFileState->rowStateBuff, pNewPos->pKey, pFileState->keyLen, &pNewPos, POINTER_BYTES); if (code != TSDB_CODE_SUCCESS) { @@ -1090,6 +1092,7 @@ int32_t recoverFillSnapshot(SStreamFileState* pFileState, int64_t ckId) { if (vlen != pFileState->rowSize) { qError("row size mismatch, expect:%d, actual:%d", pFileState->rowSize, vlen); + destroyRowBuffPos(pNewPos); code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; taosMemoryFreeClear(pVal); QUERY_CHECK_CODE(code, lino, _end); @@ -1098,6 +1101,7 @@ int32_t recoverFillSnapshot(SStreamFileState* pFileState, int64_t ckId) { memcpy(pNewPos->pRowBuff, pVal, vlen); taosMemoryFreeClear(pVal); pNewPos->beFlushed = true; + pNewPos->beUsed = false; qDebug("===stream=== read checkpoint state from disc. %s", __func__); winRes = tSimpleHashPut(pFileState->rowStateBuff, pNewPos->pKey, pFileState->keyLen, &pNewPos, POINTER_BYTES); if (winRes != TSDB_CODE_SUCCESS) { @@ -1228,6 +1232,8 @@ void setFillInfo(SStreamFileState* pFileState) { } void clearExpiredState(SStreamFileState* pFileState) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SSHashObj* pSearchBuff = pFileState->searchBuff; void* pIte = NULL; int32_t iter = 0; @@ -1246,6 +1252,13 @@ void clearExpiredState(SStreamFileState* pFileState) { } taosArrayRemoveBatch(pWinStates, 0, size - 1, NULL); } + code = clearRowBuff(pFileState); + QUERY_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } int32_t getStateSearchRowBuff(SStreamFileState* pFileState, const SWinKey* pKey, void** pVal, int32_t* pVLen, diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 0fe074084f..ed9eb4b224 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1464,7 +1464,7 @@ int32_t syncNodeRestore(SSyncNode* pSyncNode) { // if (endIndex != lastVer + 1) return TSDB_CODE_SYN_INTERNAL_ERROR; pSyncNode->commitIndex = TMAX(pSyncNode->commitIndex, commitIndex); - sInfo("vgId:%d, restore sync until commitIndex:%" PRId64, pSyncNode->vgId, pSyncNode->commitIndex); + sInfo("vgId:%d, restore began, and keep syncing until commitIndex:%" PRId64, pSyncNode->vgId, pSyncNode->commitIndex); if (pSyncNode->fsmState != SYNC_FSM_STATE_INCOMPLETE && (code = syncLogBufferCommit(pSyncNode->pLogBuf, pSyncNode, pSyncNode->commitIndex)) < 0) { diff --git a/source/os/src/osAtomic.c b/source/os/src/osAtomic.c index c891ee4579..5da2307cb3 100644 --- a/source/os/src/osAtomic.c +++ b/source/os/src/osAtomic.c @@ -19,7 +19,6 @@ typedef union { volatile int64_t i; volatile double d; - //double d; } double_number; #ifdef WINDOWS @@ -345,7 +344,7 @@ void atomic_store_64(int64_t volatile* ptr, int64_t val) { #endif } -double atomic_store_double(double volatile *ptr, double val){ +void atomic_store_double(double volatile* ptr, double val) { for (;;) { double_number old_num = {0}; old_num.d = *ptr; // current old value @@ -354,9 +353,9 @@ double atomic_store_double(double volatile *ptr, double val){ new_num.d = val; double_number ret_num = {0}; - ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i); + ret_num.i = atomic_val_compare_exchange_64((volatile int64_t*)ptr, old_num.i, new_num.i); - if (ret_num.i == old_num.i) return ret_num.d; + if (ret_num.i == old_num.i) return; } } @@ -414,16 +413,16 @@ int64_t atomic_exchange_64(int64_t volatile* ptr, int64_t val) { #endif } -double atomic_exchange_double(double volatile *ptr, int64_t val){ +double atomic_exchange_double(double volatile* ptr, double val) { for (;;) { double_number old_num = {0}; old_num.d = *ptr; // current old value double_number new_num = {0}; - int64_t iNew = val; + new_num.d = val; double_number ret_num = {0}; - ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i); + ret_num.i = atomic_val_compare_exchange_64((volatile int64_t*)ptr, old_num.i, new_num.i); if (ret_num.i == old_num.i) { return ret_num.d; @@ -589,7 +588,7 @@ int64_t atomic_fetch_add_64(int64_t volatile* ptr, int64_t val) { #endif } -double atomic_fetch_add_double(double volatile *ptr, double val){ +double atomic_fetch_add_double(double volatile* ptr, double val) { for (;;) { double_number old_num = {0}; old_num.d = *ptr; // current old value @@ -598,13 +597,13 @@ double atomic_fetch_add_double(double volatile *ptr, double val){ new_num.d = old_num.d + val; double_number ret_num = {0}; - ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i); + ret_num.i = atomic_val_compare_exchange_64((volatile int64_t*)ptr, old_num.i, new_num.i); if (ret_num.i == old_num.i) return ret_num.d; } } -void* atomic_fetch_add_ptr(void* ptr, void* val) { +void* atomic_fetch_add_ptr(void* ptr, int64_t val) { #ifdef WINDOWS return _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) @@ -710,7 +709,7 @@ int64_t atomic_fetch_sub_64(int64_t volatile* ptr, int64_t val) { #endif } -double atomic_fetch_sub_double(double volatile *ptr, double val){ +double atomic_fetch_sub_double(double volatile* ptr, double val) { for (;;) { double_number old_num = {0}; old_num.d = *ptr; // current old value @@ -719,13 +718,13 @@ double atomic_fetch_sub_double(double volatile *ptr, double val){ new_num.d = old_num.d - val; double_number ret_num = {0}; - ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i); + ret_num.i = atomic_val_compare_exchange_64((volatile int64_t*)ptr, old_num.i, new_num.i); if (ret_num.i == old_num.i) return ret_num.d; } } -void* atomic_fetch_sub_ptr(void* ptr, void* val) { +void* atomic_fetch_sub_ptr(void* ptr, int64_t val) { #ifdef WINDOWS return interlocked_fetch_sub_ptr(ptr, val); #elif defined(_TD_NINGSI_60) @@ -777,7 +776,7 @@ int64_t atomic_and_fetch_64(int64_t volatile* ptr, int64_t val) { #endif } -void* atomic_and_fetch_ptr(void* ptr, void* val) { +void* atomic_and_fetch_ptr(void* ptr, int64_t val) { #ifdef WINDOWS return interlocked_and_fetch_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) @@ -829,7 +828,7 @@ int64_t atomic_fetch_and_64(int64_t volatile* ptr, int64_t val) { #endif } -void* atomic_fetch_and_ptr(void* ptr, void* val) { +void* atomic_fetch_and_ptr(void* ptr, int64_t val) { #ifdef WINDOWS return interlocked_fetch_and_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) @@ -881,7 +880,7 @@ int64_t atomic_or_fetch_64(int64_t volatile* ptr, int64_t val) { #endif } -void* atomic_or_fetch_ptr(void* ptr, void* val) { +void* atomic_or_fetch_ptr(void* ptr, int64_t val) { #ifdef WINDOWS return interlocked_or_fetch_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) @@ -933,7 +932,7 @@ int64_t atomic_fetch_or_64(int64_t volatile* ptr, int64_t val) { #endif } -void* atomic_fetch_or_ptr(void* ptr, void* val) { +void* atomic_fetch_or_ptr(void* ptr, int64_t val) { #ifdef WINDOWS return interlocked_fetch_or_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) @@ -985,7 +984,7 @@ int64_t atomic_xor_fetch_64(int64_t volatile* ptr, int64_t val) { #endif } -void* atomic_xor_fetch_ptr(void* ptr, void* val) { +void* atomic_xor_fetch_ptr(void* ptr, int64_t val) { #ifdef WINDOWS return interlocked_xor_fetch_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) @@ -1037,7 +1036,7 @@ int64_t atomic_fetch_xor_64(int64_t volatile* ptr, int64_t val) { #endif } -void* atomic_fetch_xor_ptr(void* ptr, void* val) { +void* atomic_fetch_xor_ptr(void* ptr, int64_t val) { #ifdef WINDOWS return interlocked_fetch_xor_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 25245d2df9..410f9da623 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -39,7 +39,7 @@ enum { WRDE_SYNTAX /* Shell syntax error. */ }; -int wordexp(char *words, wordexp_t *pwordexp, int flags) { +int32_t wordexp(char *words, wordexp_t *pwordexp, int32_t flags) { pwordexp->we_offs = 0; pwordexp->we_wordc = 1; pwordexp->we_wordv[0] = pwordexp->wordPos; @@ -197,14 +197,10 @@ int32_t taosMulMkDir(const char *dirname) { } } - if (code < 0 && errno == EEXIST) { - return 0; - } - - return code; + return 0; } -int32_t taosMulModeMkDir(const char *dirname, int mode, bool checkAccess) { +int32_t taosMulModeMkDir(const char *dirname, int32_t mode, bool checkAccess) { if (dirname == NULL || strlen(dirname) >= TDDIRMAXLEN) { terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -326,7 +322,7 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { int32_t days = (int32_t)(TABS(sec - fileSec) / 86400 + 1); if (days > keepDays) { TAOS_UNUSED(taosRemoveFile(filename)); - uInfo("file:%s is removed, days:%d keepDays:%d, sed:%" PRId64, filename, days, keepDays, fileSec); + // printf("file:%s is removed, days:%d keepDays:%d, sed:%" PRId64, filename, days, keepDays, fileSec); } else { // printf("file:%s won't be removed, days:%d keepDays:%d", filename, days, keepDays); } @@ -340,7 +336,9 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { OS_PARAM_CHECK(dirname); OS_PARAM_CHECK(outname); - wordexp_t full_path; + if (dirname[0] == 0) return 0; + + wordexp_t full_path = {0}; int32_t code = wordexp(dirname, &full_path, 0); switch (code) { case 0: @@ -357,13 +355,12 @@ int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { } wordfree(&full_path); - return 0; } int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen) { OS_PARAM_CHECK(dirname); - OS_PARAM_CHECK(realPath); + char tmp[PATH_MAX] = {0}; #ifdef WINDOWS if (_fullpath(tmp, dirname, maxlen) != NULL) { @@ -377,12 +374,14 @@ int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen) { tstrncpy(realPath, tmp, maxlen); } return 0; + } else { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } + } else { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; } - - terrno = TAOS_SYSTEM_ERROR(errno); - - return terrno; } bool taosIsDir(const char *dirname) { @@ -454,7 +453,7 @@ TdDirPtr taosOpenDir(const char *dirname) { } #ifdef WINDOWS - char szFind[MAX_PATH]; //这是要找的 + char szFind[MAX_PATH]; // 这是要找的 HANDLE hFind; TdDirPtr pDir = taosMemoryMalloc(sizeof(TdDir)); @@ -462,7 +461,7 @@ TdDirPtr taosOpenDir(const char *dirname) { return NULL; } - snprintf(szFind, sizeof(szFind), "%s%s", dirname, "\\*.*"); //利用通配符找这个目录下的所以文件，包括目录 + snprintf(szFind, sizeof(szFind), "%s%s", dirname, "\\*.*"); // 利用通配符找这个目录下的所以文件，包括目录 pDir->hFind = FindFirstFile(szFind, &(pDir->dirEntry.findFileData)); if (INVALID_HANDLE_VALUE == pDir->hFind) { @@ -585,14 +584,14 @@ void taosGetCwd(char *buf, int32_t len) { #endif } -int taosGetDirSize(const char *path, int64_t *size) { - int32_t code; +int32_t taosGetDirSize(const char *path, int64_t *size) { + int32_t code = 0; + char fullPath[PATH_MAX + 100] = {0}; + TdDirPtr pDir = taosOpenDir(path); if (pDir == NULL) { return code = terrno; } - int32_t nBytes = 0; - char fullPath[1024] = {0}; int64_t totalSize = 0; TdDirEntryPtr de = NULL; @@ -601,31 +600,23 @@ int taosGetDirSize(const char *path, int64_t *size) { if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) { continue; } - nBytes = snprintf(fullPath, sizeof(fullPath), "%s%s%s", path, TD_DIRSEP, name); - if (nBytes <= 0 || nBytes >= sizeof(fullPath)) { - TAOS_UNUSED(taosCloseDir(&pDir)); - return TSDB_CODE_OUT_OF_RANGE; - } + (void)snprintf(fullPath, sizeof(fullPath), "%s%s%s", path, TD_DIRSEP, name); int64_t subSize = 0; if (taosIsDir(fullPath)) { code = taosGetDirSize(fullPath, &subSize); - if (code != 0) { - TAOS_UNUSED(taosCloseDir(&pDir)); - return code; - } } else { code = taosStatFile(fullPath, &subSize, NULL, NULL); - if (code != 0) { - TAOS_UNUSED(taosCloseDir(&pDir)); - return code; - } } + + if (code != 0) goto _OVER; + totalSize += subSize; fullPath[0] = 0; } +_OVER: *size = totalSize; TAOS_UNUSED(taosCloseDir(&pDir)); - return 0; + return code; } diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 28072f334a..3051005789 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -233,15 +233,19 @@ int32_t taosBlockSIGPIPE() { } int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip) { + int32_t code = 0; OS_PARAM_CHECK(fqdn); OS_PARAM_CHECK(ip); + int64_t limitMs = 1000; + int64_t st = taosGetTimestampMs(), cost = 0; #ifdef WINDOWS // Initialize Winsock WSADATA wsaData; int iResult; iResult = WSAStartup(MAKEWORD(2, 2), &wsaData); if (iResult != 0) { - return TAOS_SYSTEM_WINSOCKET_ERROR(WSAGetLastError()); + code = TAOS_SYSTEM_WINSOCKET_ERROR(WSAGetLastError()); + goto _err; } #endif @@ -260,12 +264,12 @@ int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip) { inRetry = true; continue; } else if (EAI_SYSTEM == ret) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; + code = TAOS_SYSTEM_ERROR(errno); + goto _err; } - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; + code = TAOS_SYSTEM_ERROR(errno); + goto _err; } struct sockaddr *sa = result->ai_addr; @@ -275,8 +279,7 @@ int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip) { *ip = ia.s_addr; freeaddrinfo(result); - - return 0; + goto _err; } #else struct addrinfo hints = {0}; @@ -292,7 +295,7 @@ int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip) { struct in_addr ia = si->sin_addr; *ip = ia.s_addr; freeaddrinfo(result); - return 0; + goto _err; } else { #ifdef EAI_SYSTEM if (ret == EAI_SYSTEM) { @@ -305,9 +308,16 @@ int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip) { #endif *ip = 0xFFFFFFFF; - return TSDB_CODE_RPC_FQDN_ERROR; + code = TSDB_CODE_RPC_FQDN_ERROR; + goto _err; } #endif +_err: + cost = taosGetTimestampMs() - st; + if (cost >= limitMs) { + uWarn("get ip from fqdn:%s, cost:%" PRId64 "ms", fqdn, cost); + } + return code; } int32_t taosGetFqdn(char *fqdn) { diff --git a/source/os/test/CMakeLists.txt b/source/os/test/CMakeLists.txt index cc7110517f..13fea463f7 100644 --- a/source/os/test/CMakeLists.txt +++ b/source/os/test/CMakeLists.txt @@ -71,6 +71,9 @@ add_test( COMMAND osTimeTests ) + +if(TD_LINUX) + add_executable(osAtomicTests "osAtomicTests.cpp") target_link_libraries(osAtomicTests os util gtest_main) add_test( @@ -78,6 +81,22 @@ add_test( COMMAND osAtomicTests ) +add_executable(osDirTests "osDirTests.cpp") +target_link_libraries(osDirTests os util gtest_main) +add_test( + NAME osDirTests + COMMAND osDirTests +) + +add_executable(osEnvTests "osEnvTests.cpp") +target_link_libraries(osEnvTests os util gtest_main) +add_test( + NAME osEnvTests + COMMAND osEnvTests +) + +endif() + add_executable(osSemaphoreTests "osSemaphoreTests.cpp") target_link_libraries(osSemaphoreTests os util gtest_main) add_test( diff --git a/source/os/test/osAtomicTests.cpp b/source/os/test/osAtomicTests.cpp index eebc1574c3..170ba1cc7d 100644 --- a/source/os/test/osAtomicTests.cpp +++ b/source/os/test/osAtomicTests.cpp @@ -29,87 +29,434 @@ #include "os.h" #include "tlog.h" -TEST(osAtomicTests, Exchange16) { - int16_t value = 123; - int16_t new_value = 456; - int16_t result = atomic_exchange_16(&value, new_value); - EXPECT_EQ(result, 123); - EXPECT_EQ(value, 456); +TEST(osAtomicTests, atomic_load) { + int8_t result8 = 0, value8 = 8; + int16_t result16 = 0, value16 = 16; + int32_t result32 = 0, value32 = 32; + int64_t result64 = 0, value64 = 64; + void* resultp = &result64; + void* valuep = &value64; + + result8 = atomic_load_8(&value8); + result16 = atomic_load_16(&value16); + result32 = atomic_load_32(&value32); + result64 = atomic_load_64(&value64); + resultp = atomic_load_ptr(&valuep); + + EXPECT_EQ(result8, 8); + EXPECT_EQ(result16, 16); + EXPECT_EQ(result32, 32); + EXPECT_EQ(result64, 64); + EXPECT_EQ(resultp, &value64); + EXPECT_EQ(value8, 8); + EXPECT_EQ(value16, 16); + EXPECT_EQ(value32, 32); + EXPECT_EQ(value64, 64); + EXPECT_EQ(valuep, &value64); } -TEST(osAtomicTests, Exchange32) { - int32_t value = 123; - int32_t new_value = 456; - int32_t result = atomic_exchange_32(&value, new_value); - EXPECT_EQ(result, 123); - EXPECT_EQ(value, 456); +TEST(osAtomicTests, atomic_store) { + int8_t result8 = 0, value8 = 8; + int16_t result16 = 0, value16 = 16; + int32_t result32 = 0, value32 = 32; + int64_t result64 = 0, value64 = 64; + double resultd = 0, valued = 64; + void* resultp = &result64; + void* valuep = &value64; + + atomic_store_8(&result8, value8); + atomic_store_16(&result16, value16); + atomic_store_32(&result32, value32); + atomic_store_64(&result64, value64); + atomic_store_double(&resultd, valued); + atomic_store_ptr(&resultp, valuep); + + EXPECT_EQ(result8, 8); + EXPECT_EQ(result16, 16); + EXPECT_EQ(result32, 32); + EXPECT_EQ(result64, 64); + EXPECT_DOUBLE_EQ(resultd, 64); + EXPECT_EQ(resultp, &value64); } -TEST(osAtomicTests, Exchange64) { - int64_t value = 123; - int64_t new_value = 456; - int64_t result = atomic_exchange_64(&value, new_value); - EXPECT_EQ(result, 123); - EXPECT_EQ(value, 456); +TEST(osAtomicTests, atomic_exchange) { + int8_t result8 = 0, value8 = 8, newval8 = 18; + int16_t result16 = 0, value16 = 16, newval16 = 116; + int32_t result32 = 0, value32 = 32, newval32 = 132; + int64_t result64 = 0, value64 = 64, newval64 = 164; + double resultd = 0, valued = 64, newvald = 164; + void* valuep = &value64; + void* newvalp = &newval64; + void* resultp = &result64; + + result8 = atomic_exchange_8(&value8, newval8); + result16 = atomic_exchange_16(&value16, newval16); + result32 = atomic_exchange_32(&value32, newval32); + result64 = atomic_exchange_64(&value64, newval64); + resultd = atomic_exchange_double(&valued, newvald); + resultp = atomic_exchange_ptr(&valuep, newvalp); + + EXPECT_EQ(value8, 18); + EXPECT_EQ(value16, 116); + EXPECT_EQ(value32, 132); + EXPECT_EQ(value64, 164); + EXPECT_DOUBLE_EQ(valued, 164); + EXPECT_EQ(valuep, &newval64); + + EXPECT_EQ(result8, 8); + EXPECT_EQ(result16, 16); + EXPECT_EQ(result32, 32); + EXPECT_EQ(result64, 64); + EXPECT_DOUBLE_EQ(resultd, 64); + EXPECT_EQ(resultp, &value64); } -TEST(osAtomicTests, ExchangePtr) { - int value1 = 123; - int value2 = 456; - int* ptr = &value1; - int* result = (int*)atomic_exchange_ptr(&ptr, &value2); - EXPECT_EQ(result, &value1); - EXPECT_EQ(*ptr, 456); +TEST(osAtomicTests, atomic_val_compare_exchange) { + int8_t result8 = 0, value8 = 8, oldval8 = 8, newval8 = 18; + int16_t result16 = 0, value16 = 16, oldval16 = 16, newval16 = 116; + int32_t result32 = 0, value32 = 32, oldval32 = 32, newval32 = 132; + int64_t result64 = 0, value64 = 64, oldval64 = 64, newval64 = 164; + void* resultp = NULL; + void* valuep = &value64; + void* oldvalp = &value64; + void* newvalp = &newval64; + + result8 = atomic_val_compare_exchange_8(&value8, oldval8, newval8); + result16 = atomic_val_compare_exchange_16(&value16, oldval16, newval16); + result32 = atomic_val_compare_exchange_32(&value32, oldval32, newval32); + result64 = atomic_val_compare_exchange_64(&value64, oldval64, newval64); + resultp = atomic_val_compare_exchange_ptr(&valuep, oldvalp, newvalp); + + EXPECT_EQ(result8, 8); + EXPECT_EQ(value8, 18); + EXPECT_EQ(result16, 16); + EXPECT_EQ(value16, 116); + EXPECT_EQ(result32, 32); + EXPECT_EQ(value32, 132); + EXPECT_EQ(result64, 64); + EXPECT_EQ(value64, 164); + EXPECT_EQ(resultp, &value64); + EXPECT_EQ(valuep, &newval64); + + oldval8 = 9; + oldval16 = 99; + oldval32 = 999; + oldval64 = 9999; + oldvalp = NULL; + + result8 = atomic_val_compare_exchange_8(&value8, oldval8, newval8); + result16 = atomic_val_compare_exchange_16(&value16, oldval16, newval16); + result32 = atomic_val_compare_exchange_32(&value32, oldval32, newval32); + result64 = atomic_val_compare_exchange_64(&value64, oldval64, newval64); + resultp = atomic_val_compare_exchange_ptr(&valuep, oldvalp, newvalp); + + EXPECT_EQ(result8, 18); + EXPECT_EQ(value8, 18); + EXPECT_EQ(result16, 116); + EXPECT_EQ(value16, 116); + EXPECT_EQ(result32, 132); + EXPECT_EQ(value32, 132); + EXPECT_EQ(result64, 164); + EXPECT_EQ(value64, 164); + EXPECT_EQ(resultp, &newval64); + EXPECT_EQ(valuep, &newval64); } -TEST(osAtomicTests, ValCompareExchange8) { - int8_t value = 12; - int8_t oldval = 12; - int8_t newval = 45; - int8_t result = atomic_val_compare_exchange_8(&value, oldval, newval); - EXPECT_EQ(result, 12); - EXPECT_EQ(value, 45); +TEST(osAtomicTests, atomic_add_fetch) { + int8_t result8 = 0, value8 = 8; + int16_t result16 = 0, value16 = 16; + int32_t result32 = 0, value32 = 32; + int64_t result64 = 0, value64 = 64; - oldval = 78; - result = atomic_val_compare_exchange_8(&value, oldval, newval); - EXPECT_EQ(result, 45); - EXPECT_EQ(value, 45); + int64_t valuex = 128; + int64_t* valuep = &valuex; + int64_t resultx = 0; + + result8 = atomic_add_fetch_8(&value8, 10); + result16 = atomic_add_fetch_16(&value16, 10); + result32 = atomic_add_fetch_32(&value32, 10); + result64 = atomic_add_fetch_64(&value64, 10); + resultx = (int64_t)atomic_add_fetch_ptr(valuep, 10); + + EXPECT_EQ(result8, 18); + EXPECT_EQ(value8, 18); + EXPECT_EQ(result16, 26); + EXPECT_EQ(value16, 26); + EXPECT_EQ(result32, 42); + EXPECT_EQ(value32, 42); + EXPECT_EQ(result64, 74); + EXPECT_EQ(value64, 74); + EXPECT_EQ(resultx, 138); + EXPECT_EQ(*valuep, 138); + EXPECT_EQ(valuex, 138); } -TEST(osAtomicTests, ValCompareExchange16) { - int16_t value = 123; - int16_t oldval = 123; - int16_t newval = 456; - int16_t result = atomic_val_compare_exchange_16(&value, oldval, newval); - EXPECT_EQ(result, 123); - EXPECT_EQ(value, 456); +TEST(osAtomicTests, atomic_fetch_add) { + int8_t result8 = 0, value8 = 8; + int16_t result16 = 0, value16 = 16; + int32_t result32 = 0, value32 = 32; + int64_t result64 = 0, value64 = 64; + double resultd = 0, valued = 64; - oldval = 789; - result = atomic_val_compare_exchange_16(&value, oldval, newval); - EXPECT_EQ(result, 456); - EXPECT_EQ(value, 456); + int64_t valuex = 128; + int64_t* valuep = &valuex; + int64_t resultx = 0; + + result8 = atomic_fetch_add_8(&value8, 10); + result16 = atomic_fetch_add_16(&value16, 10); + result32 = atomic_fetch_add_32(&value32, 10); + result64 = atomic_fetch_add_64(&value64, 10); + resultd = atomic_fetch_add_double(&valued, 10); + resultx = (int64_t)atomic_fetch_add_ptr(valuep, 10); + + EXPECT_EQ(result8, 8); + EXPECT_EQ(value8, 18); + EXPECT_EQ(result16, 16); + EXPECT_EQ(value16, 26); + EXPECT_EQ(result32, 32); + EXPECT_EQ(value32, 42); + EXPECT_EQ(result64, 64); + EXPECT_EQ(value64, 74); + EXPECT_DOUBLE_EQ(resultd, 64); + EXPECT_DOUBLE_EQ(valued, 74); + EXPECT_EQ(resultx, 128); + EXPECT_EQ(*valuep, 138); + EXPECT_EQ(valuex, 138); } -TEST(osAtomicTests, TestAtomicExchange8) { - volatile int8_t value = 42; - int8_t new_value = 100; - int8_t old_value = atomic_exchange_8(&value, new_value); - EXPECT_EQ(old_value, 42); - EXPECT_EQ(value, new_value); +TEST(osAtomicTests, atomic_sub_fetch) { + int8_t result8 = 0, value8 = 8; + int16_t result16 = 0, value16 = 16; + int32_t result32 = 0, value32 = 32; + int64_t result64 = 0, value64 = 64; + + int64_t valuex = 128; + int64_t* valuep = &valuex; + int64_t resultx = 0; + + result8 = atomic_sub_fetch_8(&value8, 10); + result16 = atomic_sub_fetch_16(&value16, 10); + result32 = atomic_sub_fetch_32(&value32, 10); + result64 = atomic_sub_fetch_64(&value64, 10); + resultx = (int64_t)atomic_sub_fetch_ptr(valuep, 10); + + EXPECT_EQ(result8, -2); + EXPECT_EQ(value8, -2); + EXPECT_EQ(result16, 6); + EXPECT_EQ(value16, 6); + EXPECT_EQ(result32, 22); + EXPECT_EQ(value32, 22); + EXPECT_EQ(result64, 54); + EXPECT_EQ(value64, 54); + EXPECT_EQ(resultx, 118); + EXPECT_EQ(*valuep, 118); + EXPECT_EQ(valuex, 118); } -TEST(osAtomicTests, TestAtomicAddFetch16) { - volatile int16_t value = 42; - int16_t increment = 10; - int16_t new_value = atomic_add_fetch_16(&value, increment); - EXPECT_EQ(new_value, 52); - EXPECT_EQ(value, 52); +TEST(osAtomicTests, atomic_fetch_sub) { + int8_t result8 = 0, value8 = 8; + int16_t result16 = 0, value16 = 16; + int32_t result32 = 0, value32 = 32; + int64_t result64 = 0, value64 = 64; + double resultd = 0, valued = 64; + + int64_t valuex = 128; + int64_t* valuep = &valuex; + int64_t resultx = 0; + + result8 = atomic_fetch_sub_8(&value8, 10); + result16 = atomic_fetch_sub_16(&value16, 10); + result32 = atomic_fetch_sub_32(&value32, 10); + result64 = atomic_fetch_sub_64(&value64, 10); + resultd = atomic_fetch_sub_double(&valued, 10); + resultx = (int64_t)atomic_fetch_sub_ptr(valuep, 10); + + EXPECT_EQ(result8, 8); + EXPECT_EQ(value8, -2); + EXPECT_EQ(result16, 16); + EXPECT_EQ(value16, 6); + EXPECT_EQ(result32, 32); + EXPECT_EQ(value32, 22); + EXPECT_EQ(result64, 64); + EXPECT_EQ(value64, 54); + EXPECT_DOUBLE_EQ(resultd, 64); + EXPECT_DOUBLE_EQ(valued, 54); + EXPECT_EQ(resultx, 128); + EXPECT_EQ(*valuep, 118); + EXPECT_EQ(valuex, 118); } -//TEST(osAtomicTests, AddFetchPtr) { -// uintptr_t val = 0; -// uintptr_t* ptr = &val; -// uintptr_t ret = atomic_add_fetch_ptr(ptr, 10); -// EXPECT_EQ(ret, 10); -// EXPECT_EQ(val, 10); -//} +TEST(osAtomicTests, atomic_and_fetch) { + int8_t result8 = 0, value8 = 3; + int16_t result16 = 0, value16 = 3; + int32_t result32 = 0, value32 = 3; + int64_t result64 = 0, value64 = 3; + + int64_t valuex = 3; + int64_t* valuep = &valuex; + int64_t resultx = 0; + + result8 = atomic_and_fetch_8(&value8, 5); + result16 = atomic_and_fetch_16(&value16, 5); + result32 = atomic_and_fetch_32(&value32, 5); + result64 = atomic_and_fetch_64(&value64, 5); + resultx = (int64_t)atomic_and_fetch_ptr(valuep, 5); + + EXPECT_EQ(result8, 1); + EXPECT_EQ(value8, 1); + EXPECT_EQ(result16, 1); + EXPECT_EQ(value16, 1); + EXPECT_EQ(result32, 1); + EXPECT_EQ(value32, 1); + EXPECT_EQ(result64, 1); + EXPECT_EQ(value64, 1); + EXPECT_EQ(resultx, 1); + EXPECT_EQ(*valuep, 1); + EXPECT_EQ(valuex, 1); +} + +TEST(osAtomicTests, atomic_fetch_and) { + int8_t result8 = 0, value8 = 3; + int16_t result16 = 0, value16 = 3; + int32_t result32 = 0, value32 = 3; + int64_t result64 = 0, value64 = 3; + + int64_t valuex = 3; + int64_t* valuep = &valuex; + int64_t resultx = 0; + + result8 = atomic_fetch_and_8(&value8, 5); + result16 = atomic_fetch_and_16(&value16, 5); + result32 = atomic_fetch_and_32(&value32, 5); + result64 = atomic_fetch_and_64(&value64, 5); + resultx = (int64_t)atomic_fetch_and_ptr(valuep, 5); + + EXPECT_EQ(result8, 3); + EXPECT_EQ(value8, 1); + EXPECT_EQ(result16, 3); + EXPECT_EQ(value16, 1); + EXPECT_EQ(result32, 3); + EXPECT_EQ(value32, 1); + EXPECT_EQ(result64, 3); + EXPECT_EQ(value64, 1); + EXPECT_EQ(resultx, 3); + EXPECT_EQ(*valuep, 1); + EXPECT_EQ(valuex, 1); +} + +TEST(osAtomicTests, atomic_or_fetch) { + int8_t result8 = 0, value8 = 3; + int16_t result16 = 0, value16 = 3; + int32_t result32 = 0, value32 = 3; + int64_t result64 = 0, value64 = 3; + + int64_t valuex = 3; + int64_t* valuep = &valuex; + int64_t resultx = 0; + + result8 = atomic_or_fetch_8(&value8, 5); + result16 = atomic_or_fetch_16(&value16, 5); + result32 = atomic_or_fetch_32(&value32, 5); + result64 = atomic_or_fetch_64(&value64, 5); + resultx = (int64_t)atomic_or_fetch_ptr(valuep, 5); + + EXPECT_EQ(result8, 7); + EXPECT_EQ(value8, 7); + EXPECT_EQ(result16, 7); + EXPECT_EQ(value16, 7); + EXPECT_EQ(result32, 7); + EXPECT_EQ(value32, 7); + EXPECT_EQ(result64, 7); + EXPECT_EQ(value64, 7); + EXPECT_EQ(resultx, 7); + EXPECT_EQ(*valuep, 7); + EXPECT_EQ(valuex, 7); +} + +TEST(osAtomicTests, atomic_fetch_or) { + int8_t result8 = 0, value8 = 3; + int16_t result16 = 0, value16 = 3; + int32_t result32 = 0, value32 = 3; + int64_t result64 = 0, value64 = 3; + + int64_t valuex = 3; + int64_t* valuep = &valuex; + int64_t resultx = 0; + + result8 = atomic_fetch_or_8(&value8, 5); + result16 = atomic_fetch_or_16(&value16, 5); + result32 = atomic_fetch_or_32(&value32, 5); + result64 = atomic_fetch_or_64(&value64, 5); + resultx = (int64_t)atomic_fetch_or_ptr(valuep, 5); + + EXPECT_EQ(result8, 3); + EXPECT_EQ(value8, 7); + EXPECT_EQ(result16, 3); + EXPECT_EQ(value16, 7); + EXPECT_EQ(result32, 3); + EXPECT_EQ(value32, 7); + EXPECT_EQ(result64, 3); + EXPECT_EQ(value64, 7); + EXPECT_EQ(resultx, 3); + EXPECT_EQ(*valuep, 7); + EXPECT_EQ(valuex, 7); +} + + +TEST(osAtomicTests, atomic_xor_fetch) { + int8_t result8 = 0, value8 = 3; + int16_t result16 = 0, value16 = 3; + int32_t result32 = 0, value32 = 3; + int64_t result64 = 0, value64 = 3; + + int64_t valuex = 3; + int64_t* valuep = &valuex; + int64_t resultx = 0; + + result8 = atomic_xor_fetch_8(&value8, 5); + result16 = atomic_xor_fetch_16(&value16, 5); + result32 = atomic_xor_fetch_32(&value32, 5); + result64 = atomic_xor_fetch_64(&value64, 5); + resultx = (int64_t)atomic_xor_fetch_ptr(valuep, 5); + + EXPECT_EQ(result8, 6); + EXPECT_EQ(value8, 6); + EXPECT_EQ(result16, 6); + EXPECT_EQ(value16, 6); + EXPECT_EQ(result32, 6); + EXPECT_EQ(value32, 6); + EXPECT_EQ(result64, 6); + EXPECT_EQ(value64, 6); + EXPECT_EQ(resultx, 6); + EXPECT_EQ(*valuep, 6); + EXPECT_EQ(valuex, 6); +} + +TEST(osAtomicTests, atomic_fetch_xor) { + int8_t result8 = 0, value8 = 3; + int16_t result16 = 0, value16 = 3; + int32_t result32 = 0, value32 = 3; + int64_t result64 = 0, value64 = 3; + + int64_t valuex = 3; + int64_t* valuep = &valuex; + int64_t resultx = 0; + + result8 = atomic_fetch_xor_8(&value8, 5); + result16 = atomic_fetch_xor_16(&value16, 5); + result32 = atomic_fetch_xor_32(&value32, 5); + result64 = atomic_fetch_xor_64(&value64, 5); + resultx = (int64_t)atomic_fetch_xor_ptr(valuep, 5); + + EXPECT_EQ(result8, 3); + EXPECT_EQ(value8, 6); + EXPECT_EQ(result16, 3); + EXPECT_EQ(value16, 6); + EXPECT_EQ(result32, 3); + EXPECT_EQ(value32, 6); + EXPECT_EQ(result64, 3); + EXPECT_EQ(value64, 6); + EXPECT_EQ(resultx, 3); + EXPECT_EQ(*valuep, 6); + EXPECT_EQ(valuex, 6); +} \ No newline at end of file diff --git a/source/os/test/osDirTests.cpp b/source/os/test/osDirTests.cpp new file mode 100644 index 0000000000..1dba7299be --- /dev/null +++ b/source/os/test/osDirTests.cpp @@ -0,0 +1,464 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + +#include "os.h" +#include "tlog.h" + +TEST(osDirTests, taosRemoveDir) { + int32_t ret = 0; + + const char* testDir = "/tmp/tdengine-test-dir"; + if (taosDirExist(testDir)) { + taosRemoveDir(testDir); + } + + taosRemoveDir(testDir); + + ret = taosMkDir(testDir); + EXPECT_EQ(ret, 0); + + const char* testFile = "/tmp/tdengine-test-dir/test-file"; + TdFilePtr testFilePtr = taosCreateFile(testFile, TD_FILE_CREATE); + EXPECT_NE(testFilePtr, nullptr); + ret = taosCloseFile(&testFilePtr); + EXPECT_EQ(ret, 0); + + const char* testDir2 = "/tmp/tdengine-test-dir/test-dir2"; + ret = taosMkDir(testDir2); + EXPECT_EQ(ret, 0); + + const char* testFile2 = "/tmp/tdengine-test-dir/test-dir2/test-file2"; + TdFilePtr testFilePtr2 = taosCreateFile(testFile2, TD_FILE_CREATE); + EXPECT_NE(testFilePtr2, nullptr); + ret = taosCloseFile(&testFilePtr2); + EXPECT_EQ(ret, 0); + + taosRemoveDir(testDir); + + bool exist = taosDirExist(testDir); + EXPECT_EQ(exist, false); + + taosRemoveDir("/tmp/tdengine-test-dir"); +} + +TEST(osDirTests, taosDirExist) { + const char* dir1 = NULL; + bool exist = taosDirExist(dir1); + EXPECT_EQ(exist, false); + + char dir2[2048] = {0}; + for (int32_t i = 0; i < 2047; ++i) { + dir2[i] = 1; + } + exist = taosDirExist(dir2); + EXPECT_EQ(exist, false); + + taosRemoveDir("/tmp/tdengine-test-dir"); +} + +TEST(osDirTests, taosMulMkDir) { + int32_t ret = 0; + + const char* dir1 = NULL; + ret = taosMulMkDir(dir1); + EXPECT_EQ(ret, -1); + + char dir2[2048] = {0}; + for (int32_t i = 0; i < 2047; ++i) { + dir2[i] = '1'; + } + ret = taosMulMkDir(dir2); + EXPECT_EQ(ret, -1); + + const char* dir3 = "/tmp/tdengine-test-dir/1/2/3/4"; + taosRemoveDir(dir3); + ret = taosMulMkDir(dir3); + EXPECT_EQ(ret, 0); + taosRemoveDir(dir3); + + const char* dir4 = "./tdengine-test-dir/1/2/3/4"; + taosRemoveDir(dir4); + ret = taosMulMkDir(dir4); + EXPECT_EQ(ret, 0); + taosRemoveDir(dir4); + + const char* dir5 = "tdengine-test-dir/1/2/3/4"; + taosRemoveDir(dir5); + ret = taosMulMkDir(dir5); + EXPECT_EQ(ret, 0); + ret = taosMulMkDir(dir5); + EXPECT_EQ(ret, 0); + taosRemoveDir(dir5); + + const char* testFile = "/tmp/tdengine-test-dir/test-file"; + TdFilePtr testFilePtr = taosCreateFile(testFile, TD_FILE_CREATE); + EXPECT_NE(testFilePtr, nullptr); + ret = taosCloseFile(&testFilePtr); + EXPECT_EQ(ret, 0); + char dir6[2048] = {0}; + strcpy(dir6, "/tmp/tdengine-test-dir/test-file/1/2/3/4"); + ret = taosMulMkDir(dir6); + EXPECT_NE(ret, 0); + + taosRemoveDir("/tmp/tdengine-test-dir"); +} + +TEST(osDirTests, taosMulModeMkDir) { + int32_t ret = 0; + + const char* dir1 = NULL; + ret = taosMulModeMkDir(dir1, 777, true); + EXPECT_NE(ret, 0); + + char dir2[2048] = {0}; + for (int32_t i = 0; i < 2047; ++i) { + dir2[i] = '1'; + } + ret = taosMulModeMkDir(dir2, 777, true); + EXPECT_NE(ret, 0); + + const char* dir3 = "/tmp/tdengine-test-dir/1/2/3/4"; + taosRemoveDir(dir3); + ret = taosMulMkDir(dir3); + EXPECT_EQ(ret, 0); + ret = taosMulModeMkDir(dir3, 777, true); + EXPECT_EQ(ret, 0); + ret = taosMulModeMkDir(dir3, 777, false); + EXPECT_EQ(ret, 0); + ret = taosMulModeMkDir(dir3, 999, true); + EXPECT_EQ(ret, 0); + ret = taosMulModeMkDir(dir3, 999, false); + EXPECT_EQ(ret, 0); + taosRemoveDir(dir3); + + const char* dir4 = "./tdengine-test-dir/1/2/3/4"; + taosRemoveDir(dir4); + ret = taosMulModeMkDir(dir4, 777, true); + EXPECT_EQ(ret, 0); + ret = taosMulModeMkDir(dir4, 777, false); + EXPECT_EQ(ret, 0); + taosRemoveDir(dir4); + + const char* dir5 = "tdengine-test-dir/1/2/3/4"; + taosRemoveDir(dir5); + ret = taosMulModeMkDir(dir5, 777, true); + EXPECT_EQ(ret, 0); + ret = taosMulModeMkDir(dir5, 777, false); + EXPECT_EQ(ret, 0); + ret = taosMulModeMkDir(dir5, 777, true); + EXPECT_EQ(ret, 0); + ret = taosMulModeMkDir(dir5, 777, false); + EXPECT_EQ(ret, 0); + taosRemoveDir(dir5); + + const char* testFile = "/tmp/tdengine-test-dir/test-file"; + TdFilePtr testFilePtr = taosCreateFile(testFile, TD_FILE_CREATE); + EXPECT_NE(testFilePtr, nullptr); + ret = taosCloseFile(&testFilePtr); + EXPECT_EQ(ret, 0); + char dir6[2048] = {0}; + strcpy(dir6, "/tmp/tdengine-test-dir/test-file/1/2/3/4"); + ret = taosMulModeMkDir(dir6, 777, true); + EXPECT_NE(ret, 0); + + const char* dir7 = "tdengine-test-dir/1/2/3/5"; + taosRemoveDir(dir7); + ret = taosMulModeMkDir(dir7, 999, true); + EXPECT_EQ(ret, 0); + ret = taosMulModeMkDir(dir7, 999, false); + EXPECT_EQ(ret, 0); + + taosRemoveDir("/tmp/tdengine-test-dir"); +} + +TEST(osDirTests, taosRemoveOldFiles) { + int32_t ret = 0; + const char* testDir = "/tmp/tdengine-test-dir"; + if (taosDirExist(testDir)) { + taosRemoveDir(testDir); + } + taosRemoveDir(testDir); + ret = taosMkDir(testDir); + EXPECT_EQ(ret, 0); + + const char* testFile = "/tmp/tdengine-test-dir/test-file"; + TdFilePtr testFilePtr = taosCreateFile(testFile, TD_FILE_CREATE); + EXPECT_NE(testFilePtr, nullptr); + ret = taosCloseFile(&testFilePtr); + EXPECT_EQ(ret, 0); + + taosRemoveOldFiles(testFile, 10); + + const char* testDir2 = "/tmp/tdengine-test-dir/test-dir2"; + ret = taosMkDir(testDir2); + EXPECT_EQ(ret, 0); + + const char* testFile3 = "/tmp/tdengine-test-dir/log.1433726073.gz"; + TdFilePtr testFilePtr3 = taosCreateFile(testFile3, TD_FILE_CREATE); + EXPECT_NE(testFilePtr3, nullptr); + ret = taosCloseFile(&testFilePtr3); + EXPECT_EQ(ret, 0); + + const char* testFile4 = "/tmp/tdengine-test-dir/log.80.gz"; + TdFilePtr testFilePtr4 = taosCreateFile(testFile4, TD_FILE_CREATE); + EXPECT_NE(testFilePtr4, nullptr); + ret = taosCloseFile(&testFilePtr4); + EXPECT_EQ(ret, 0); + + char testFile5[1024]; + snprintf(testFile5, 1024, "/tmp/tdengine-test-dir/log.%d.gz", taosGetTimestampSec()); + TdFilePtr testFilePtr5 = taosCreateFile(testFile5, TD_FILE_CREATE); + EXPECT_NE(testFilePtr5, nullptr); + ret = taosCloseFile(&testFilePtr5); + EXPECT_EQ(ret, 0); + + const char* testFile6 = "/tmp/tdengine-test-dir/log.1433726073.gz"; + TdFilePtr testFilePtr6 = taosCreateFile(testFile6, TD_FILE_CREATE); + EXPECT_NE(testFilePtr6, nullptr); + ret = taosCloseFile(&testFilePtr6); + EXPECT_EQ(ret, 0); + + taosRemoveOldFiles(testDir, 10); + + bool exist = taosDirExist(testDir); + EXPECT_EQ(exist, true); + + taosRemoveDir("/tmp/tdengine-test-dir"); +} + +TEST(osDirTests, taosExpandDir) { + int32_t ret = 0; + const char* testDir = "/tmp/tdengine-test-dir"; + ret = taosMkDir(testDir); + EXPECT_EQ(ret, 0); + + char fullpath[1024] = {0}; + ret = taosExpandDir(testDir, NULL, 1024); + EXPECT_NE(ret, 0); + ret = taosExpandDir(NULL, fullpath, 1024); + EXPECT_NE(ret, 0); + ret = taosExpandDir(testDir, fullpath, 1024); + EXPECT_EQ(ret, 0); + + ret = taosExpandDir("/x123", fullpath, 1024); + EXPECT_EQ(ret, 0); + + ret = taosExpandDir("", fullpath, 1024); + EXPECT_EQ(ret, 0); + + char dir2[2048] = {0}; + for (int32_t i = 0; i < 2047; ++i) { + dir2[i] = '1'; + } + ret = taosExpandDir(dir2, fullpath, 1024); + EXPECT_EQ(ret, 0); + + taosRemoveDir("/tmp/tdengine-test-dir"); +} + +TEST(osDirTests, taosRealPath) { + int32_t ret = 0; + char testDir[PATH_MAX * 2] = "/tmp/tdengine-test-dir"; + ret = taosMkDir(testDir); + EXPECT_EQ(ret, 0); + + char fullpath[PATH_MAX * 2] = {0}; + + ret = taosRealPath(testDir, NULL, PATH_MAX * 2); + EXPECT_EQ(ret, 0); + + ret = taosRealPath(NULL, fullpath, PATH_MAX * 2); + EXPECT_NE(ret, 0); + + ret = taosRealPath(testDir, fullpath, PATH_MAX * 2); + EXPECT_EQ(ret, 0); + + ret = taosRealPath(testDir, fullpath, 12); + EXPECT_NE(ret, 0); + + ret = taosRealPath("/c/d", fullpath, 1024); + EXPECT_NE(ret, 0); + + taosRemoveDir("/tmp/tdengine-test-dir"); +} + +TEST(osDirTests, taosIsDir) { + bool ret = taosIsDir("/c/d"); + EXPECT_EQ(ret, false); +} + +TEST(osDirTests, taosDirName) { + char* ret = taosDirName(NULL); + EXPECT_EQ(ret, nullptr); + + char name1[24] = "xyz"; + ret = taosDirName(name1); + EXPECT_NE(ret, nullptr); + EXPECT_EQ(name1[0], 0); + + char name2[24] = "/root/xyz"; + ret = taosDirName(name2); + EXPECT_NE(ret, nullptr); + EXPECT_STREQ(ret, "/root"); +} + +TEST(osDirTests, taosDirEntryBaseName) { + char* ret = taosDirEntryBaseName(NULL); + EXPECT_EQ(ret, nullptr); + + char name1[12] = "/"; + ret = taosDirEntryBaseName(name1); + EXPECT_STREQ(ret, "/"); + + char name2[12] = "/root/"; + ret = taosDirEntryBaseName(name2); + EXPECT_STREQ(ret, "root"); + + char name3[12] = "/root"; + ret = taosDirEntryBaseName(name3); + EXPECT_STREQ(ret, "root"); + + char name4[12] = "root"; + ret = taosDirEntryBaseName(name4); + EXPECT_STREQ(ret, "root"); +} + +TEST(osDirTests, taosOpenDir) { + TdDirPtr ret = taosOpenDir(NULL); + EXPECT_EQ(ret, nullptr); +} + +TEST(osDirTests, taosReadDir) { + TdDirEntryPtr rddir = taosReadDir(NULL); + EXPECT_EQ(rddir, nullptr); + + int32_t ret = 0; + const char* testDir = "/tmp/tdengine-test-dir"; + ret = taosMkDir(testDir); + EXPECT_EQ(ret, 0); + + const char* testFile = "/tmp/tdengine-test-dir/test-file"; + TdFilePtr testFilePtr = taosCreateFile(testFile, TD_FILE_CREATE); + EXPECT_NE(testFilePtr, nullptr); + ret = taosCloseFile(&testFilePtr); + EXPECT_EQ(ret, 0); + + TdDirPtr dir = taosOpenDir(testFile); + EXPECT_EQ(dir, nullptr); + + const char* testDir2 = "/tmp/tdengine-test-dir/test-dir2"; + ret = taosMkDir(testDir2); + EXPECT_EQ(ret, 0); + + const char* testFile2 = "/tmp/tdengine-test-dir/test-dir2/test-file2"; + TdFilePtr testFilePtr2 = taosCreateFile(testFile2, TD_FILE_CREATE); + EXPECT_NE(testFilePtr2, nullptr); + ret = taosCloseFile(&testFilePtr2); + EXPECT_EQ(ret, 0); + + dir = taosOpenDir(testFile); + EXPECT_EQ(dir, nullptr); + + rddir = taosReadDir(dir); + EXPECT_EQ(rddir, nullptr); + + dir = taosOpenDir(testDir); + EXPECT_NE(dir, nullptr); + + rddir = taosReadDir(dir); + EXPECT_NE(rddir, nullptr); + + bool entry = taosDirEntryIsDir(NULL); + EXPECT_EQ(entry, false); + + char* entryname = taosGetDirEntryName(NULL); + EXPECT_EQ(entryname, nullptr); + + entryname = taosGetDirEntryName(rddir); + EXPECT_NE(entryname, nullptr); + + int32_t code = taosCloseDir(NULL); + EXPECT_NE(code, 0); + + code = taosCloseDir(&dir); + EXPECT_EQ(code, 0); + + taosRemoveDir("/tmp/tdengine-test-dir"); +} + +TEST(osDirTests, taosGetDirSize) { + TdDirEntryPtr rddir = taosReadDir(NULL); + EXPECT_EQ(rddir, nullptr); + + int32_t ret = 0; + const char* testDir = "/tmp/tdengine-test-dir"; + ret = taosMkDir(testDir); + EXPECT_EQ(ret, 0); + + const char* testFile = "/tmp/tdengine-test-dir/test-file"; + TdFilePtr testFilePtr = taosCreateFile(testFile, TD_FILE_CREATE); + EXPECT_NE(testFilePtr, nullptr); + ret = taosCloseFile(&testFilePtr); + EXPECT_EQ(ret, 0); + + TdDirPtr dir = taosOpenDir(testFile); + EXPECT_EQ(dir, nullptr); + + const char* testDir2 = "/tmp/tdengine-test-dir/test-dir2"; + ret = taosMkDir(testDir2); + EXPECT_EQ(ret, 0); + + const char* testFile2 = "/tmp/tdengine-test-dir/test-dir2/test-file2"; + TdFilePtr testFilePtr2 = taosCreateFile(testFile2, TD_FILE_CREATE); + EXPECT_NE(testFilePtr2, nullptr); + ret = taosCloseFile(&testFilePtr2); + EXPECT_EQ(ret, 0); + + int64_t size = -1; + ret = taosGetDirSize(testFile, &size); + EXPECT_NE(ret, 0); + + ret = taosGetDirSize(testDir, &size); + EXPECT_EQ(ret, 0); + + taosRemoveDir("/tmp/tdengine-test-dir"); + + taosRemoveDir("./tdengine-test-dir/1/2/3/5"); + taosRemoveDir("./tdengine-test-dir/1/2/3/4"); + taosRemoveDir("./tdengine-test-dir/1/2/3"); + taosRemoveDir("./tdengine-test-dir/1/2"); + taosRemoveDir("./tdengine-test-dir/1"); + taosRemoveDir("./tdengine-test-dir/"); + taosRemoveDir("tdengine-test-dir/1/2/3/5"); + taosRemoveDir("tdengine-test-dir/1/2/3/4"); + taosRemoveDir("tdengine-test-dir/1/2/3"); + taosRemoveDir("tdengine-test-dir/1/2"); + taosRemoveDir("tdengine-test-dir/1"); + taosRemoveDir("tdengine-test-dir/"); + taosRemoveDir("tdengine-test-dir/"); +} diff --git a/source/os/test/osEnvTests.cpp b/source/os/test/osEnvTests.cpp new file mode 100644 index 0000000000..d5b3cac8a6 --- /dev/null +++ b/source/os/test/osEnvTests.cpp @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + +#include "os.h" +#include "tlog.h" + +TEST(osEnvTests, osDefaultInit) { + int32_t ret = 0; + + strcpy(tsTimezoneStr, ""); + strcpy(tsTempDir, ""); + tsNumOfCores = 0; + ret = osDefaultInit(); + EXPECT_EQ(ret, 0); + + strcpy(tsTempDir, "/tmp"); + ret = osDefaultInit(); + EXPECT_EQ(ret, 0); + + osCleanup(); +} + +TEST(osEnvTests, osUpdate) { + int32_t ret = 0; + + strcpy(tsLogDir, ""); + strcpy(tsDataDir, ""); + strcpy(tsTempDir, ""); + + ret = osUpdate(); + EXPECT_EQ(ret, 0); +} + +TEST(osEnvTests, osSufficient) { + bool ret = 0; + + tsLogSpace.size.avail = 10000; + tsDataSpace.size.avail = 10000; + tsTempSpace.size.avail = 10000; + tsLogSpace.reserved = 2000; + tsDataSpace.reserved = 2000; + tsDataSpace.reserved = 2000; + + ret = osLogSpaceAvailable(); + EXPECT_EQ(ret, true); + + ret = osTempSpaceAvailable(); + EXPECT_EQ(ret, true); + + ret = osDataSpaceAvailable(); + EXPECT_EQ(ret, true); + + ret = osLogSpaceSufficient(); + EXPECT_EQ(ret, true); + + ret = osDataSpaceSufficient(); + EXPECT_EQ(ret, true); + + ret = osTempSpaceSufficient(); + EXPECT_EQ(ret, true); + + osSetSystemLocale(NULL, NULL); + osSetSystemLocale(NULL, NULL); + osSetSystemLocale("1", "2"); + + osSetProcPath(1, NULL); + osSetProcPath(0, (char **)&ret); +} \ No newline at end of file diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index e801008970..55da8c0929 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -33,6 +33,13 @@ ENDIF() INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/src/util/inc) INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/include/common) +add_executable(heapTest "heapTest.cpp") +target_link_libraries(heapTest os util gtest_main) +add_test( + NAME heapTest + COMMAND heapTest +) + # arrayTest add_executable(arrayTest "arrayTest.cpp") target_link_libraries(arrayTest os util gtest_main) diff --git a/source/util/test/heapTest.cpp b/source/util/test/heapTest.cpp index 51eeb26ed3..fd61e5032b 100644 --- a/source/util/test/heapTest.cpp +++ b/source/util/test/heapTest.cpp @@ -20,7 +20,7 @@ int32_t heapCompare(const HeapNode* a, const HeapNode* b) { return 1; } -TEST(TD_UTIL_HEAP_TEST, heapTest) { +TEST(heapTest, heapTest) { Heap* heap = heapCreate(heapCompare); ASSERT_TRUE(heap != NULL); ASSERT_EQ(0, heapSize(heap)); diff --git a/tests/ci/container_build_newmachine.sh b/tests/ci/container_build_newmachine.sh index 3c87cf156f..369429b99f 100755 --- a/tests/ci/container_build_newmachine.sh +++ b/tests/ci/container_build_newmachine.sh @@ -80,7 +80,7 @@ else fi -mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan +rm -rf ${REP_REAL_PATH}/debug date ret=$? diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index a635ae680a..47a6b418f6 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1082,6 +1082,7 @@ ,,n,system-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insertMix.py -N 3 ,,n,system-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/stt.py -N 3 ,,n,system-test,python3 ./test.py -f eco-system/meta/database/keep_time_offset.py +,,y,system-test,./pytest.sh python3 ./test.py -f eco-system/manager/schema_change.py -N 3 -M 3 #tsim test ,,y,script,./test.sh -f tsim/query/timeline.sim diff --git a/tests/run_all_ci_cases.sh b/tests/run_all_ci_cases.sh index 486c47ff4c..959af66d19 100644 --- a/tests/run_all_ci_cases.sh +++ b/tests/run_all_ci_cases.sh @@ -27,7 +27,6 @@ function runCasesOneByOne () { || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_ALLCI_REPORT end_time=`date +%s` echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_ALLCI_REPORT - if $case failed elif [[ "$line" == *"$2"* ]]; then if [[ "$cmd" == *"pytest.sh"* ]]; then diff --git a/tests/script/tsim/stream/streamTwaFwcInterval.sim b/tests/script/tsim/stream/streamTwaFwcInterval.sim index 8640650310..5151d7caff 100644 --- a/tests/script/tsim/stream/streamTwaFwcInterval.sim +++ b/tests/script/tsim/stream/streamTwaFwcInterval.sim @@ -289,6 +289,64 @@ if $data51 != $query1_data51 then goto loop3 endi +print ======step3 +sql create database test3 vgroups 1; +sql use test3; + +sql create stable st(ts timestamp, a int, b int , c int)tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams3 trigger force_window_close IGNORE EXPIRED 1 IGNORE UPDATE 1 into streamt3 as select _wstart, twa(a), ta from st partition by tbname,ta interval(10s); + +run tsim/stream/checkTaskStatus.sim + +sql insert into t1 values(now + 3000a,1,1,1); +sql flush database test; +sql insert into t1 values(now + 3001a,10,10,10); +sql insert into t1 values(now + 13s,50,50,50); + +sleep 1000 + +print sql select _wstart, twa(a), ta from st partition by tbname,ta interval(10s) order by 1; +sql select _wstart, twa(a), ta from st partition by tbname,ta interval(10s) order by 1; + +$query_data01 = $data01 + +print $data00 $data01 $data02 $data03 $data04 +print $data10 $data11 $data12 $data13 $data14 +print $data20 $data21 $data22 $data23 $data24 +print $data30 $data31 $data32 $data33 $data34 +print $data40 $data41 $data42 $data43 $data44 +print $data50 $data51 $data52 $data53 $data54 + + +$loop_count = 0 +loop4: + +sleep 2000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +print 2 sql select * from streamt3 order by 1; +sql select * from streamt3 order by 1; + +print $data00 $data01 $data02 $data03 $data04 +print $data10 $data11 $data12 $data13 $data14 +print $data20 $data21 $data22 $data23 $data24 +print $data30 $data31 $data32 $data33 $data34 +print $data40 $data41 $data42 $data43 $data44 +print $data50 $data51 $data52 $data53 $data54 + +if $data01 != $query_data01 then + print ======data01======$data01 + print ====query_data01=$query_data01 + goto loop4 +endi + print end system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/system-test/2-query/and_or_for_byte.py b/tests/system-test/2-query/and_or_for_byte.py index ca9c1f2bef..22ec7ef29b 100644 --- a/tests/system-test/2-query/and_or_for_byte.py +++ b/tests/system-test/2-query/and_or_for_byte.py @@ -515,6 +515,18 @@ class TDTestCase: tdSql.query(f"select t1 from {dbname}.stb1 where abs(c1+t1)=1") tdSql.checkRows(1) tdSql.checkData(0,0,0) + tdSql.query(f"select * from {dbname}.stb1") + rows = tdSql.queryRows + tdSql.query(f"select t1 from {dbname}.stb1 where abs(c1+t1)=1 or (1<2)") + tdSql.checkRows(rows) + tdSql.query(f"select t1 from {dbname}.stb1 where abs(c1+t1)=1 and (1<2)") + tdSql.checkRows(1) + tdSql.checkData(0,0,0) + tdSql.query(f"select t1 from {dbname}.stb1 where abs(c1+t1)=1 and (1>2)") + tdSql.checkRows(0) + tdSql.query(f"select t1 from {dbname}.stb1 where abs(c1+t1)=1 or (1>2)") + tdSql.checkRows(1) + tdSql.checkData(0,0,0) tdSql.query( f"select abs(c1+t1)*t1 from {dbname}.stb1 where abs(c1)/floor(abs(ceil(t1))) ==1") diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index 74f5e86267..f3c5bb171c 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -451,7 +451,74 @@ class TDTestCase: sql = "select avg(c1), concat(t9,t10) from db.stb group by concat(t9,t10), concat(t9,t10),tbname" tdSql.query(sql, queryTimes=1) tdSql.checkRows(5) + + def test_TS_5727(self): + tdSql.execute(f" use {self.dbname} ") + stableName = "test5727" + sql = f"CREATE STABLE {self.dbname}.{stableName} (`ts` TIMESTAMP, `WaterConsumption` FLOAT, \ + `ElecConsumption` INT, `Status` BOOL, `status2` BOOL, `online` BOOL) \ + TAGS (`ActivationTime` TIMESTAMP, `ProductId` INT, \ + `ProductMac` VARCHAR(24), `location` INT)" + tdSql.execute(sql) + + sql = f'CREATE TABLE {self.dbname}.`d00` USING {self.dbname}.{stableName} \ + (`ActivationTime`, `ProductId`, `ProductMac`, `location`) \ + TAGS (1733124710578, 1001, "00:11:22:33:44:55", 100000)' + tdSql.execute(sql) + sql = f'CREATE TABLE {self.dbname}.`d01` USING {self.dbname}.{stableName} \ + (`ActivationTime`, `ProductId`, `ProductMac`, `location`) \ + TAGS (1733124723572, 1002, "00:12:22:33:44:55", 200000)' + tdSql.execute(sql) + sql = f'CREATE TABLE {self.dbname}.`d02` USING {self.dbname}.{stableName} \ + (`ActivationTime`, `ProductId`, `ProductMac`, `location`) \ + TAGS (1733124730908, 1003, "00:11:23:33:44:55", 100000)' + tdSql.execute(sql) + + sql = f'insert into {self.dbname}.d00 values(now - 2s, 5, 5, true, true, false);' + tdSql.execute(sql) + sql = f'insert into {self.dbname}.d01 values(now - 1s, 6, 5, true, true, true);' + tdSql.execute(sql) + sql = f'insert into {self.dbname}.d02 values(now, 6, 7, true, true, true);' + tdSql.execute(sql) + + sql = f'select `location`, tbname from {self.dbname}.{stableName} where ts < now group by tbname order by tbname;' + tdSql.query(sql) + tdSql.checkRows(3) + tdSql.checkData(0, 0, 100000) + tdSql.checkData(1, 0, 200000) + tdSql.checkData(2, 0, 100000) + tdSql.checkData(0, 1, "d00") + tdSql.checkData(1, 1, "d01") + tdSql.checkData(2, 1, "d02") + + sql = f'select tbname,last(online) as online,location from {self.dbname}.{stableName} where ts < now group by tbname order by tbname;' + tdSql.query(sql) + tdSql.checkRows(3) + tdSql.checkData(0, 0, "d00") + tdSql.checkData(1, 0, "d01") + tdSql.checkData(2, 0, "d02") + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(0, 2, 100000) + tdSql.checkData(1, 2, 200000) + tdSql.checkData(2, 2, 100000) + + sql = f'select location,tbname,last_row(online) as online from {self.dbname}.{stableName} where ts < now group by tbname order by tbname;' + tdSql.query(sql) + tdSql.checkRows(3) + tdSql.checkData(0, 0, 100000) + tdSql.checkData(1, 0, 200000) + tdSql.checkData(2, 0, 100000) + tdSql.checkData(0, 1, "d00") + tdSql.checkData(1, 1, "d01") + tdSql.checkData(2, 1, "d02") + tdSql.checkData(0, 2, False) + tdSql.checkData(1, 2, True) + tdSql.checkData(2, 2, True) + + def run(self): tdSql.prepare() self.prepare_db() @@ -493,6 +560,8 @@ class TDTestCase: # self.test_groupby('group', 5, 5) self.test_error() + + self.test_TS_5727() def stop(self): diff --git a/tests/system-test/2-query/max_partition.py b/tests/system-test/2-query/max_partition.py index c4635dcf50..824488652f 100644 --- a/tests/system-test/2-query/max_partition.py +++ b/tests/system-test/2-query/max_partition.py @@ -64,6 +64,10 @@ class TDTestCase: tdSql.query(f"select tbname , max(c1) from {dbname}.sub_stb_1 where c1 is null group by c1 order by c1 desc ") tdSql.checkRows(1) tdSql.checkData(0,0,"sub_stb_1") + tdSql.query(f"select tbname , max(c1) from {dbname}.sub_stb_1 group by c1 order by c1 desc ") + rows = tdSql.queryRows + tdSql.query(f"select tbname , max(c1) from {dbname}.sub_stb_1 where c1 is null or (1<2) group by c1 order by c1 desc ") + tdSql.checkRows(rows) tdSql.query(f"select max(c1) ,c2 ,t2,tbname from {dbname}.stb group by abs(c1) order by abs(c1)") tdSql.checkRows(self.row_nums+1) diff --git a/tests/system-test/2-query/normal.py b/tests/system-test/2-query/normal.py index 161a9e610d..32357d2a37 100644 --- a/tests/system-test/2-query/normal.py +++ b/tests/system-test/2-query/normal.py @@ -75,6 +75,25 @@ class TDTestCase: tdSql.checkData(1, 1, 3) tdSql.checkData(2, 1, 9) + tdSql.query(f"select * from {dbname}.stb_1 order by ts") + rows = tdSql.queryRows + tdSql.query(f"select * from {dbname}.stb_1 where col1 in (1, 2) or (1<2) order by ts") + tdSql.checkRows(rows) + + tdSql.query(f"select * from (select * from {dbname}.stb_1 where col1 in (1, 9, 3) or (1<2) order by ts)") + tdSql.checkRows(rows) + + tdSql.query(f"select * from {dbname}.stb_1 where col1 in (1, 2) and (1<2) order by ts") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query(f"select * from {dbname}.stb_1 where col1 in (1, 9, 3) and (1<2) order by ts") + tdSql.checkRows(3) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 3) + tdSql.checkData(2, 1, 9) + tdSql.query(f"select * from {dbname}.stb_1 where col1 in (1, 9, 3, 'xy') order by ts") tdSql.checkRows(3) tdSql.checkData(0, 1, 1) @@ -160,7 +179,7 @@ class TDTestCase: tdSql.prepare() self.timeZoneTest() - # self.inAndNotinTest() + self.inAndNotinTest() def stop(self): diff --git a/tests/system-test/2-query/not.py b/tests/system-test/2-query/not.py index 1254226db3..9f4813bebc 100644 --- a/tests/system-test/2-query/not.py +++ b/tests/system-test/2-query/not.py @@ -28,6 +28,10 @@ class TDTestCase: for type_name in stype: tdsql.execute(f"drop table if exists {dbname}.{stbname}") tdsql.execute(f"create table if not exists {dbname}.{stbname} (ts timestamp, v1 {type_name}) tags(t1 {type_name})") + + tdsql.query(f"select t1, * from {dbname}.{stbname} where t1 not in (1, 2) or (1<2) order by t1") + tdsql.checkRows(0) + tdsql.execute(f"insert into {dbname}.sub_1 using {dbname}.{stbname} tags(1) values({self.ts}, 10)") tdsql.execute(f"insert into {dbname}.sub_2 using {dbname}.{stbname} tags(2) values({self.ts + 1000}, 20)") tdsql.execute(f"insert into {dbname}.sub_3 using {dbname}.{stbname} tags(3) values({self.ts + 2000}, 30)") @@ -36,6 +40,11 @@ class TDTestCase: tdsql.query(f"select t1, * from {dbname}.{stbname} where t1 not in (1, 2) order by t1") tdsql.checkRows(1) tdsql.checkData(0, 0, 3) + tdsql.query(f"select * from {dbname}.{stbname} where t1 not in (1, 2) or (1<2) order by t1") + tdsql.checkRows(3) + tdsql.checkData(0, 1, 10) + tdsql.checkData(1, 1, 20) + tdsql.checkData(2, 1, 30) # Test case 2: NOT BETWEEN tdsql.query(f"select * from {dbname}.{stbname} where v1 not between 10 and 20 order by t1") diff --git a/tests/system-test/2-query/union.py b/tests/system-test/2-query/union.py index 5104489592..24ccac5d90 100644 --- a/tests/system-test/2-query/union.py +++ b/tests/system-test/2-query/union.py @@ -426,6 +426,15 @@ class TDTestCase: tdLog.printNoPrefix("==========step4:after wal, all check again ") self.all_test() + self.test_TD_33137() + + def test_TD_33137(self): + sql = "select 'asd' union all select 'asdasd'" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(2) + sql = "select db_name `TABLE_CAT`, '' `TABLE_SCHEM`, stable_name `TABLE_NAME`, 'TABLE' `TABLE_TYPE`, table_comment `REMARKS` from information_schema.ins_stables union all select db_name `TABLE_CAT`, '' `TABLE_SCHEM`, table_name `TABLE_NAME`, case when `type`='SYSTEM_TABLE' then 'TABLE' when `type`='NORMAL_TABLE' then 'TABLE' when `type`='CHILD_TABLE' then 'TABLE' else 'UNKNOWN' end `TABLE_TYPE`, table_comment `REMARKS` from information_schema.ins_tables union all select db_name `TABLE_CAT`, '' `TABLE_SCHEM`, view_name `TABLE_NAME`, 'VIEW' `TABLE_TYPE`, NULL `REMARKS` from information_schema.ins_views" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(48) def stop(self): tdSql.close() diff --git a/tests/system-test/eco-system/manager/schema_change.py b/tests/system-test/eco-system/manager/schema_change.py index 400d2b100b..8eef773d48 100644 --- a/tests/system-test/eco-system/manager/schema_change.py +++ b/tests/system-test/eco-system/manager/schema_change.py @@ -133,8 +133,17 @@ class TDTestCase: if len(tags) < 10: return - sel_cols = random.sample(columns, random.randint(2,int(len(columns)/10))) - sel_tags = random.sample(tags, random.randint(1,int(len(tags)/10))) + # cmax + cmax = int(len(columns)/10) + if cmax <=2 : + cmax = 3 + sel_cols = random.sample(columns, random.randint(2, cmax)) + + # tmax + tmax = int(len(tags)/10) + if tmax <=1 : + tmax = 2 + sel_tags = random.sample(tags, random.randint(1, tmax)) field_cols = ",".join(sel_cols) field_tags = ",".join(sel_tags) @@ -217,18 +226,43 @@ class TDTestCase: #time.sleep(0.3) + # create db and stb + def create(self, db, stb, cols, tags): + # create db + sql = f"create database {db} ;" + tdSql.execute(sql) + + # switch db + tdSql.execute(f"use {db};") + + # cols + sql1 = "" + for k, v in cols.items(): + sql1 += f",{k} {v}" + # tags + sql2 = "" + for k, v in tags.items(): + if sql2 == "": + sql2 = f"{k} {v}" + else: + sql2 += f",{k} {v}" + + # create stb + sql = f"create table {db}.{stb}(ts timestamp {sql1}) tags({sql2})" + tdSql.execute(sql) + # run def run(self): # seed random.seed(int(time.time())) self.dbname = "schema_change" - # switch db - tdSql.execute(f"use {self.dbname};") + # create db + self.create(self.dbname, "meters", self.column_dict, self.tag_dict) + # change meters - self.change_schema(1000000) - + self.change_schema(1000) def stop(self):