xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into feat/TS-4994-3.0

This commit is contained in:
Hongze Cheng 2024-11-14 14:16:09 +08:00
parent 80f3b01fc5 a546250239
commit 1f51f480af
70 changed files with 5397 additions and 2898 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
Jenkinsfile2
View File

@ -332,16 +332,6 @@ def pre_test_win(){
cd %WIN_COMMUNITY_ROOT%
git submodule update --init --recursive
'''
bat '''
cd %WIN_CONNECTOR_ROOT%
git branch
git reset --hard
git pull
'''
bat '''
cd %WIN_CONNECTOR_ROOT%
git log -5
'''
}
def pre_test_build_win() {
@ -363,12 +353,9 @@ def pre_test_build_win() {
time /t
'''
bat '''
cd %WIN_CONNECTOR_ROOT%
python.exe -m pip install --upgrade pip
python -m pip uninstall taospy -y
python -m pip install taospy==2.7.16
python -m pip uninstall taos-ws-py -y
python -m pip install taos-ws-py==0.3.3
cd %WIN_COMMUNITY_ROOT%/tests/ci
pip3 install taospy==2.7.16
pip3 install taos-ws-py==0.3.3
xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32
'''
return 1
@ -387,7 +374,6 @@ def run_win_ctest() {
def run_win_test() {
bat '''
echo "windows test ..."
cd %WIN_CONNECTOR_ROOT%
xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32
ls -l C:\\Windows\\System32\\taos.dll
time /t
@ -476,7 +462,6 @@ pipeline {
WIN_INTERNAL_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\TDinternal"
WIN_COMMUNITY_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\TDinternal\\community"
WIN_SYSTEM_TEST_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\TDinternal\\community\\tests\\system-test"
WIN_CONNECTOR_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\taos-connector-python"
}
steps {
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {

2
cmake/cmake.version
View File

@ -2,7 +2,7 @@
IF (DEFINED VERNUMBER)
SET(TD_VER_NUMBER ${VERNUMBER})
ELSE ()
SET(TD_VER_NUMBER "3.3.4.0.alpha")
SET(TD_VER_NUMBER "3.3.4.3.alpha")
ENDIF ()
IF (DEFINED VERCOMPATIBLE)

2
docs/en/14-reference/03-taos-sql/10-function.md
View File

@ -1149,7 +1149,7 @@ TOP(expr, k)
UNIQUE(expr)
```
**Description**: The values that occur the first time in the specified column. The effect is similar to `distinct` keyword. For a table with composite primary key, only the data with the smallest primary key value is returned.
**Description**: Return the unique values of this column. The effect is similar to `distinct` keyword. Return the row with the earliest timestamp for duplicate data. For a table with composite primary key, only the data with the smallest primary key value is returned.
**Return value type**:Same as the data type of the column being operated upon

4
docs/en/28-releases/01-tdengine.md
View File

@ -20,6 +20,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://t
import Release from "/components/ReleaseV3";
## 3.3.4.3
<Release type="tdengine" version="3.3.4.3" />
## 3.3.3.0
<Release type="tdengine" version="3.3.3.0" />

10
docs/zh/06-advanced/05-data-in/07-mqtt.md
View File

@ -63,7 +63,7 @@ TDengine 可以通过 MQTT 连接器从 MQTT 代理订阅数据并将其写入 T
在 **Clean Session** 中，选择是否清除会话。默认值为 true。
在 **订阅主题及 QoS 配置** 中填写要消费的 Topic 名称。使用如下格式设置： `topic1::0,topic2::1`。
在 **订阅主题及 QoS 配置** 中填写要消费的 Topic 名称和 QoS。使用如下格式设置： `{topic_name}::{qos}`（如：`my_topic::0`）。MQTT 协议 5.0 支持共享订阅，可以通过多个客户端订阅同一个 Topic 实现负载均衡，使用如下格式： `$share/{group_name}/{topic_name}::{qos}`，其中，`$share` 是固定前缀，表示启用共享订阅，`group_name` 是分组名称，类似 kafka 的消费者组。
点击 **检查连通性** 按钮，检查数据源是否可用。
@ -146,7 +146,13 @@ json 数据支持 JSONObject 或者 JSONArray，使用 json 解析器可以解
### 7. 高级选项
在 **日志级别** 下拉列表中选择日志级别。有五个选项：`TRACE`、`DEBUG`、`INFO`、`WARN`、`ERROR`。 默认值为 INFO。
在 **消息等待队列大小** 中填写接收 MQTT 消息的缓存队列大小，当队列满时，新到达的数据会直接丢弃。可设置为 0，即不缓存。
在 **处理批次上限** 中填写可以同时进行数据处理流程的批次数量，当到达此上限后，不再从消息缓存队列中获取消息，会导致缓存队列的消息积压，最小值为 1。
在 **批次大小** 中填写每次发送给数据处理流程的消息数量，和 **批次延时** 配合使用，当读取的 MQTT 消息数量达到批次大小时，就算 **批次延时** 没有到达也立即向数据处理流程发送数据，最小值为 1。
在 **批次延时** 中填写每次生成批次消息的超时时间（单位：毫秒），从每批次接收到的第一个消息开始算起，和 **批次大小** 配合使用，当读取消息到达超时时间时，就算 **批次大小** 不满足数量也立即向数据处理流程发送数据，最小值为 1。
当 **保存原始数据时**，以下2个参数配置生效。

BIN
docs/zh/06-advanced/05-data-in/mqtt-14.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 40 KiB

After

Width:  |  Height:  |  Size: 62 KiB

24
docs/zh/06-advanced/06-TDgpt/02-management.md
View File

@ -4,10 +4,10 @@ sidebar_label: "安装部署"
---
### 环境准备
为了使用 TDgpt 的高级时序数据分析功能功能，需要在 TDengine 集群中安装部署 AI node（Anode）。ANode 可以运行在 Linux/Windows/Mac 等操作系统之上。请确保安装部署 Anode之前，系统中已经具备 3.10 及以上版本的Python环境，以及相应的 Python 包自动安装组件 Pip，否则无法正常安装 Anode。
为了使用 TDgpt 的高级时序数据分析功能功能，需要在 TDengine 集群中安装部署 AI node（Anode）。Anode 可以运行在 Linux/Windows/Mac 等操作系统之上。请确保安装部署 Anode之前，系统中已经具备 3.10 及以上版本的Python环境，以及相应的 Python 包自动安装组件 Pip，否则无法正常安装 Anode。
### 安装及卸载
不同操作系统上安装及部署操作有细微的差异，主要是安装/卸载操作、安装路径、Anode服务的启停等几个方面。下面将以 Linux 系统为例，说明安装部署的整个流程。使用 Linux 环境下的安装包 TDengine-enterprise-anode-1.x.x.tar.gz 可进行 ANode 的安装部署工作，使用如下命令：
不同操作系统上安装及部署操作有细微的差异，主要是安装/卸载操作、安装路径、Anode服务的启停等几个方面。下面将以 Linux 系统为例，说明安装部署的整个流程。使用 Linux 环境下的安装包 TDengine-enterprise-anode-1.x.x.tar.gz 可进行 Anode 的安装部署工作，使用如下命令：
```bash
tar -xzvf TDengine-enterprise-anode-1.0.0.tar.gz
@ -15,11 +15,11 @@ cd TDengine-enterprise-anode-1.0.0
sudo ./install.sh
```
在安装完成 ANode 之后，执行命令 `rmtaosanode` 即可已经安装的 Anode。
ANode 使用 Python 虚拟环境运行，避免影响安装环境中现有的 Python 库。安装后的默认 Python 虚拟环境目录位于 `/var/lib/taos/taosanode/venv/`。为了避免反复安装虚拟环境带来的开销，卸载 ANode 执行的命令 `rmtaosanode` 并不会自动删除该虚拟环境，如果您确认不需要 Python 的虚拟环境，手动删除即可。
在安装完成 Anode 之后，执行命令 `rmtaosanode` 即可已经安装的 Anode。
Anode 使用 Python 虚拟环境运行，避免影响安装环境中现有的 Python 库。安装后的默认 Python 虚拟环境目录位于 `/var/lib/taos/taosanode/venv/`。为了避免反复安装虚拟环境带来的开销，卸载 Anode 执行的命令 `rmtaosanode` 并不会自动删除该虚拟环境，如果您确认不需要 Python 的虚拟环境，手动删除即可。
### 启停服务
在 Linux 系统中，安装 ANode 以后可以使用 `systemd` 来管理 ANode 服务。使用如下命令可以启动/停止/检查状态。
在 Linux 系统中，安装 Anode 以后可以使用 `systemd` 来管理 Anode 服务。使用如下命令可以启动/停止/检查状态。
```bash
systemctl start taosanoded
@ -39,7 +39,7 @@ systemctl status taosanoded
#### 配置说明
Anode 提供的服务使用 uWSGI 驱动，因此 ANode 和 uWSGI 的配置信息共同存放在相同的配置文件 `taosanode.ini`，该配置文件默认位于 `/etc/taos/`目录下，其具体内容及说明如下：
Anode 提供的服务使用 uWSGI 驱动，因此 Anode 和 uWSGI 的配置信息共同存放在相同的配置文件 `taosanode.ini`，该配置文件默认位于 `/etc/taos/`目录下，其具体内容及说明如下：
```ini
[uwsgi]
@ -90,14 +90,14 @@ Anode 运行配置主要是以下：
- log-level: app-log文件的日志级别
### ANode 基本操作
#### 创建 ANode
### Anode 基本操作
#### 创建 Anode
```sql
CREATE ANODE {node_url}
```
node_url 是提供服务的 ANode 的 IP 和 PORT, 例如：`create anode 'http://localhost:6050'`。启动 ANode 以后如果不注册到 TDengine 集群中，则无法提供正常的服务。不建议 ANode 注册到两个或多个集群中。
node_url 是提供服务的 Anode 的 IP 和 PORT, 例如：`create anode 'http://localhost:6050'`。启动 Anode 以后如果不注册到 TDengine 集群中，则无法提供正常的服务。不建议 Anode 注册到两个或多个集群中。
#### 查看 ANode
#### 查看 Anode
列出集群中所有的数据分析节点，包括其 `FQDN`, `PORT`, `STATUS`。
```sql
SHOW ANODES;
@ -115,8 +115,8 @@ UPDATE ANODE {node_id}
UPDATE ALL ANODES
```
#### 删除 ANode
#### 删除 Anode
```sql
DROP ANODE {anode_id}
```
删除 ANode 只是将 ANode 从 TDengine 集群中删除，管理 ANode 的启停仍然需要使用`systemctl`命令。
删除 Anode 只是将 Anode 从 TDengine 集群中删除，管理 Anode 的启停仍然需要使用`systemctl`命令。

4
docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/index.md
View File

@ -51,12 +51,12 @@ algo=expr1
```SQL
--- 使用 iqr 算法进行异常检测，检测列 i32 列。
SELECT _wstart, _wend, SUM(i32)
FROM ai.atb
FROM foo
ANOMALY_WINDOW(i32, "algo=iqr");
--- 使用 ksigma 算法进行异常检测，输入参数 k 值为 2，检测列 i32 列
SELECT _wstart, _wend, SUM(i32)
FROM ai.atb
FROM foo
ANOMALY_WINDOW(i32, "algo=ksigma,k=2");
```

24
docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md
View File

@ -84,10 +84,32 @@ class _MyForecastService(AbstractForecastService):
SELECT COUNT(*) FROM foo ANOMALY_DETECTION(col, 'algo=myad')
```
将该文件保存在 `./taosanalytics/algo/fc/` 目录下，然后重启 taosanode 服务。通过执行 `SHOW ANODES FULL` 能够看到新加入的算法，然后就可以通过 SQL 语句调用该预测算法。
将该文件保存在 `./taosanalytics/algo/fc/` 目录下，然后重启 taosanode 服务。通过执行 `SHOW ANODES FULL` 能够看到新加入的算法，通过 SQL 语句调用该预测算法。
```SQL
--- 对 col 列进行异常检测，通过指定 algo 参数为 myfc 来调用新添加的预测类
SELECT _flow, _fhigh, _frowts, FORECAST(col_name, "algo=myfc")
FROM foo;
```
### 单元测试
在测试目录`taosanalytics/test`中的 forecast_test.py 中增加单元测试用例或添加新的测试文件。单元测试依赖 Python Unit test 包。
```python
def test_myfc(self):
""" 测试 myfc 类 """
s = loader.get_service("myfc")
# 设置用于预测分析的数据
s.set_input_list(self.get_input_list())
# 检查预测结果应该全部为 1
r = s.set_params(
{"fc_rows": 10, "start_ts": 171000000, "time_step": 86400 * 30, "start_p": 0}
)
r = s.execute()
expected_list = [1] * 10
self.assertEqlist(r["res"][0], expected_list)
```

7
docs/zh/06-advanced/06-TDgpt/index.md
View File

@ -5,15 +5,14 @@ title: TDgpt
import TDgpt from './pic/data-analysis.png';
## 概述
TDgpt 是 TDengine Enterprise 中针对时序数据提供高级分析功能的企业级组件，能够独立于 TDengine 主进程部署和运行，不消耗和占用 TDengine 主进程的资源，通过内置接口向 TDengine 提供运行时动态扩展的高级时序数据分析功能。TDgpt 具有服务无状态、功能易扩展、快速弹性部署、应用轻量化、高安全性等特点。
TDgpt 运行在部署于 TDengine 集群中的 AI Node (ANode)中。每个 TDengine 集群中可以部署一个或若干个 ANode 节点，不同的 ANode 节点之间不相关，无同步或协同的要求。ANode 注册到 TDengine 集群以后，就可以通过内部接口提供服务。TDgpt 提供的高级时序数据分析服务可分为时序数据异常检测和时序数据预测分析两个类别。
TDgpt 运行在部署于 TDengine 集群中的 AI Node (Anode)中。每个 TDengine 集群中可以部署一个或若干个 Anode 节点，不同的 Anode 节点之间不相关，无同步或协同的要求。Anode 注册到 TDengine 集群以后，就可以通过内部接口提供服务。TDgpt 提供的高级时序数据分析服务可分为时序数据异常检测和时序数据预测分析两个类别。
下图是部署了 TDgpt 的 TDengine 集群示意图。
下图是部署 TDgpt 的 TDengine 集群示意图。
<img src={TDgpt} width="560" alt="TDgpt架构图" />
通过注册指令将 ANode 注册到 MNode 中以后，就加入到 TDengine 集群，并可被查询引擎动态调用执行。在查询处理过程中，查询引擎根据生成的物理执行计划，**按需**向 ANode 请求高级时序数据分析服务。用户可通过SQL语句与 ANode 节点交互，并使用其提供的全部分析服务。需要注意的是 ANode 不直接接受用户的数据分析请求。同时 ANode 提供高效的动态注册机制，其注册和卸载过程完全不影响 TDengine 集群的服务，只影响提供对应的查询服务能力。
通过注册指令将 Anode 注册到 Mnode 中以后，就加入到 TDengine 集群，并可被查询引擎动态调用执行。在查询处理过程中，查询引擎根据生成的物理执行计划，**按需**向 Anode 请求高级时序数据分析服务。用户可通过SQL语句与 Anode 节点交互，并使用其提供的全部分析服务。需要注意的是 Anode 不直接接受用户的数据分析请求。同时 Anode 提供高效的动态注册机制，其注册和卸载过程完全不影响 TDengine 集群的服务，只影响提供对应的查询服务能力。
TDgpt 提供的高级数据分析功能分为时序数据异常检测和时序数据预测。
- 时序数据异常检测的结果采用异常窗口的形式提供，即分析系统自动将算法检测到的连续异常数据以时间窗口的形式返回，其使用方式与 TDengine 中其他类型的时间窗口（例如状态窗口、事件窗口）类似。特别地，可以将异常数据窗口视作为一种特殊的**事件窗口（Event Window）**，因此状态窗口可使用的所有查询操作均可应用在异常窗口上。

4
docs/zh/08-operation/06-gui.md
View File

@ -165,6 +165,10 @@ toc_max_heading_level: 4
第一步 填写添加新主题需要的信息，点击“创建”按钮；
![topic-03-addTopicWizard.jpeg](./pic/topic-03-addTopicWizard.jpeg "添加新主题 Wizard 页面")
如上图，您可以选择是否 “同步 meta”。如果同步 meta 信息，则可以订阅到 meta 信息，比如增加或者删除超级表。
您需要根据使用场景来选择是否开启，如果您引用 taos 连接器编写业务代码订阅 topic，则不能开启“同步 meta”，只能订阅数据；如果您创建 topic 在 explorer 配置同步任务使用，则可以开启“同步 meta”。
第二步 页面出现以下记录，则证明创建成功。
![topic-05-addTopicSucc1.jpeg](./pic/topic-05-addTopicSucc1.jpeg "查看已创建的流计算")

14
docs/zh/08-operation/19-debug.md Normal file
View File

@ -0,0 +1,14 @@
---
sidebar_label: 分析调试
title: 分析调试
toc_max_heading_level: 4
---
为了更好的分析调试 TDengine ，推荐开发者在操作系统中安装以下分析调试工具：
## gdb
GDB（GNU Debugger）是一个功能强大的命令行调试器，广泛用于调试 C、C++ 和其他编程语言的程序。
## valgrind
valgrind 是一个用于内存调试、内存泄漏检测和性能分析的工具框架。Valgrind 提供了一组工具，帮助开发者检测和修复程序中的内存错误、线程错误和性能问题。
## bpftrace
bpftrace 是一个高级的动态跟踪工具，基于 eBPF（Extended Berkeley Packet Filter）技术，用于在 Linux 系统上进行性能分析和故障排除。
## perf
perf 是一个强大的 Linux 性能分析工具。它提供了对系统和应用程序的详细性能分析，帮助开发者和系统管理员识别和解决性能瓶颈。

BIN
docs/zh/08-operation/pic/topic-03-addTopicWizard.jpeg
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 275 KiB

After

Width:  |  Height:  |  Size: 83 KiB

315
docs/zh/14-reference/01-components/01-taosd.md
View File

@ -26,42 +26,66 @@ taosd 命令行参数如下
:::
### 连接相关
| 参数名称 | 参数说明 |
| :--------------------: | :-------------------------------------------------------------------------------------: |
| firstEp | taosd 启动时，主动连接的集群中首个 dnode 的 end point，缺省值：localhost:6030 |
| secondEp | taosd 启动时，如果 firstEp 连接不上，尝试连接集群中第二个 dnode 的 endpoint，缺省值：无 |
| fqdn | 启动 taosd 后所监听的服务地址，缺省值：所在服务器上配置的第一个 hostname |
| serverPort | 启动 taosd 后所监听的端口，缺省值：6030 |
| numOfRpcSessions | 允许一个 dnode 能发起的最大连接数，取值范围 100-100000，缺省值：30000 |
| timeToGetAvailableConn | 获得可用连接的最长等待时间，取值范围 10-50000000，单位为毫秒，缺省值：500000 |
|参数名称|支持版本|参数含义|
|-----------------------|-----------|-|
|firstEp | |taosd 启动时，主动连接的集群中首个 dnode 的 end point，默认值 localhost:6030|
|secondEp | |taosd 启动时，如果 firstEp 连接不上，尝试连接集群中第二个 dnode 的 endpoint，无默认值|
|fqdn | |taosd 监听的服务地址，默认为所在服务器上配置的第一个 hostname|
|serverPort | |taosd 监听的端口，默认值 6030|
|compressMsgSize | |是否对 RPC 消息进行压缩；-1：所有消息都不压缩；0：所有消息都压缩；N (N>0)：只有大于 N 个字节的消息才压缩；默认值 -1|
|shellActivityTimer | |客户端向 mnode 发送心跳的时长，单位为秒，取值范围 1-120，默认值 3|
|numOfRpcSessions | |RPC 支持的最大连接数，取值范围 100-100000，默认值 30000|
|numOfRpcThreads | |RPC 线程数目，默认值为 CPU 核数的一半|
|numOfTaskQueueThreads | |dnode 处理 RPC 消息的线程数|
|statusInterval | |dnode 与 mnode 之间的心跳间隔|
|rpcQueueMemoryAllowed | |dnode 允许的 rpc 消息占用的内存最大值，单位 bytes，取值范围 104857600-INT64_MAX，默认值 服务器内存的 1/10 |
|resolveFQDNRetryTime | |FQDN 解析失败时的重试次数|
|timeToGetAvailableConn | |获得可用连接的最长等待时间，取值范围 10-50000000，单位为毫秒，默认值 500000|
|maxShellConns | |允许创建的最大链接数|
|maxRetryWaitTime | |重连最大超时时间|
|shareConnLimit |3.3.4.3 之后|内部参数，一个链接可以共享的查询数目，取值范围 1-256，默认值 10|
|readTimeout |3.3.4.3 之后|内部参数，最小超时时间，取值范围 64-604800，单位为秒，默认值 900|
### 监控相关
| 参数名称 | 参数说明 |
| :----------------: | :------------------------------------------------------------------------------------: |
| monitor | 是否收集监控数据并上报，0: 关闭；1:打开；缺省值：0 |
| monitorFqdn | taosKeeper 服务所在服务器的 FQDN，缺省值：无 |
| monitorPort | taosKeeper 服务所监听的端口号，缺省值：6043 |
| monitorInternal | 监控数据库记录系统参数（CPU/内存）的时间间隔，单位是秒，取值范围 1-200000 ，缺省值：30 |
| telemetryReporting | 是否上传 telemetry，0: 不上传，1：上传，缺省值：1 |
| crashReporting | 是否上传 crash 信息；0: 不上传，1: 上传；缺省值： 1 |
|参数名称|支持版本|参数含义|
|-----------------------|-----------|-|
|monitor | |是否收集监控数据并上报，0：关闭；1:打开；默认值 0|
|monitorFqdn | |taosKeeper 服务所在服务器的 FQDN，默认值 无|
|monitorPort | |taosKeeper 服务所监听的端口号，默认值 6043|
|monitorInterval | |监控数据库记录系统参数（CPU/内存）的时间间隔，单位是秒，取值范围 1-200000 ，默认值 30|
|monitorMaxLogs | |缓存的待上报日志条数|
|monitorComp | |是否采用压缩方式上报监控日志时|
|monitorLogProtocol | |是否打印监控日志|
|monitorForceV2 | |是否使用 V2 版本协议上报|
|telemetryReporting | |是否上传 telemetry，0：不上传，1：上传，默认值 1|
|telemetryServer | |telemetry 服务器地址|
|telemetryPort | |telemetry 服务器端口编号|
|telemetryInterval | |telemetry 上传时间间隔，单位为秒，默认 43200|
|crashReporting | |是否上传 crash 信息；0：不上传，1：上传；默认值 1|
### 查询相关
| 参数名称 | 参数说明 |
| :--------------------: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |
| queryPolicy | 查询策略，1: 只使用 vnode，不使用 qnode; 2: 没有扫描算子的子任务在 qnode 执行，带扫描算子的子任务在 vnode 执行; 3: vnode 只运行扫描算子，其余算子均在 qnode 执行 ；4: 使用客户端聚合模式；缺省值：1 |
| maxNumOfDistinctRes | 允许返回的 distinct 结果最大行数，默认值 10 万，最大允许值 1 亿 |
| countAlwaysReturnValue | count/hyperloglog函数在输入数据为空或者NULL的情况下是否返回值，0: 返回空行，1: 返回；该参数设置为 1 时，如果查询中含有 INTERVAL 子句或者该查询使用了TSMA时, 且相应的组或窗口内数据为空或者NULL， 对应的组或窗口将不返回查询结果. 注意此参数客户端和服务端值应保持一致. |
|参数名称|支持版本|参数含义|
|------------------------|-----------|-|
|countAlwaysReturnValue | |count/hyperloglog 函数在输入数据为空或者 NULL 的情况下是否返回值；0：返回空行，1：返回；默认值 1；该参数设置为 1 时，如果查询中含有 INTERVAL 子句或者该查询使用了 TSMA 时，且相应的组或窗口内数据为空或者 NULL，对应的组或窗口将不返回查询结果；注意此参数客户端和服务端值应保持一致|
|tagFilterCache | |是否缓存标签过滤结果|
|maxNumOfDistinctRes | |允许返回的 distinct 结果最大行数，默认值 10 万，最大允许值 1 亿|
|queryBufferSize | |暂不生效|
|queryRspPolicy | |查询响应策略|
|filterScalarMode | |强制使用标量过滤模式，0：关闭；1：开启，默认值 0|
|queryPlannerTrace | |内部参数，查询计划是否输出详细日志|
|queryNodeChunkSize | |内部参数，查询计划的块大小|
|queryUseNodeAllocator | |内部参数，查询计划的分配方法|
|queryMaxConcurrentTables| |内部参数，查询计划的并发数目|
|queryRsmaTolerance | |内部参数，用于判定查询哪一级 rsma 数据时的容忍时间，单位为毫秒|
|enableQueryHb | |内部参数，是否发送查询心跳消息|
|pqSortMemThreshold | |内部参数，排序使用的内存阈值|
### 区域相关
| 参数名称 | 参数说明 |
| :------: | :------------------------------------------------------------------------------------------------------: |
| timezone | 时区，缺省值：当前服务器所配置的时区 |
| locale | 系统区位信息及编码格式 ，缺省值：系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过 API 设置 |
| charset | 字符集编码，缺省值：系统自动获取 |
|参数名称|支持版本|参数含义|
|-----------------|-----------|-|
|timezone | |时区；缺省从系统中动态获取当前的时区设置|
|locale | |系统区位信息及编码格式，缺省从系统中获取|
|charset | |字符集编码，缺省从系统中获取|
:::info
1. 为应对多时区的数据写入和查询问题，TDengine 采用 Unix 时间戳(Unix Timestamp)来记录和存储时间戳。Unix 时间戳的特点决定了任一时刻不论在任何时区，产生的时间戳均一致。需要注意的是，Unix 时间戳是在客户端完成转换和记录。为了确保客户端其他形式的时间转换为正确的 Unix 时间戳，需要设置正确的时区。
@ -101,7 +125,7 @@ SELECT count(*) FROM table_name WHERE TS<1554984068000;
客户端的输入的字符均采用操作系统当前默认的编码格式，在 Linux/macOS 系统上多为 UTF-8，部分中文系统编码则可能是 GB18030 或 GBK 等。在 docker 环境中默认的编码是 POSIX。在中文版 Windows 系统中，编码则是 CP936。客户端需要确保正确设置自己所使用的字符集，即客户端运行的操作系统当前编码字符集，才能保证 nchar 中的数据正确转换为 UCS4-LE 编码格式。
在 Linux/macOS 中 locale 的命名规则为: \<语言>_\<地区>.\<字符集编码> 如：zh_CN.UTF-8，zh 代表中文，CN 代表大陆地区，UTF-8 表示字符集。字符集编码为客户端正确解析本地字符串提供编码转换的说明。Linux/macOS 可以通过设置 locale 来确定系统的字符编码，由于 Windows 使用的 locale 中不是 POSIX 标准的 locale 格式，因此在 Windows 下需要采用另一个配置参数 charset 来指定字符编码。在 Linux/macOS 中也可以使用 charset 来指定字符编码。
在 Linux/macOS 中 locale 的命名规则为：\<语言>_\<地区>.\<字符集编码> 如：zh_CN.UTF-8，zh 代表中文，CN 代表大陆地区，UTF-8 表示字符集。字符集编码为客户端正确解析本地字符串提供编码转换的说明。Linux/macOS 可以通过设置 locale 来确定系统的字符编码，由于 Windows 使用的 locale 中不是 POSIX 标准的 locale 格式，因此在 Windows 下需要采用另一个配置参数 charset 来指定字符编码。在 Linux/macOS 中也可以使用 charset 来指定字符编码。
3. 如果配置文件中不设置 charset，在 Linux/macOS 中，taos 在启动时候，自动读取系统当前的 locale 信息，并从 locale 信息中解析提取 charset 编码格式。如果自动读取 locale 信息失败，则尝试读取 charset 配置，如果读取 charset 配置也失败，则中断启动过程。
@ -139,73 +163,148 @@ charset 的有效值是 UTF-8。
:::
### 存储相关
| 参数名称 | 参数说明 |
| :--------------: | :--------------------------------------------------------------------: |
| dataDir | 数据文件目录，所有的数据文件都将写入该目录，缺省值：/var/lib/taos |
| tempDir | 指定所有系统运行过程中的临时文件生成的目录，缺省值：/tmp |
| minimalTmpDirGB | tempDir 所指定的临时文件目录所需要保留的最小空间，单位 GB，缺省值: 1 |
| minimalDataDirGB | dataDir 指定的时序数据存储目录所需要保留的最小空间，单位 GB，缺省值: 2 |
|参数名称|支持版本|参数含义|
|--------------------|-----------|-|
|dataDir | |数据文件目录，所有的数据文件都将写入该目录，默认值 /var/lib/taos|
|tempDir | |指定所有系统运行过程中的临时文件生成的目录，默认值 /tmp|
|minimalDataDirGB | |dataDir 指定的时序数据存储目录所需要保留的最小空间，单位 GB，默认值 2|
|minimalTmpDirGB | |tempDir 所指定的临时文件目录所需要保留的最小空间，单位 GB，默认值 1|
|minDiskFreeSize |3.1.1.0 之后|当某块磁盘上的可用空间小于等于这个阈值时，该磁盘将不再被选择用于生成新的数据文件，单位为字节，取值范围 52428800-1073741824，默认值为 52428800；企业版参数|
|s3MigrateIntervalSec|3.3.4.3 之后|本地数据文件自动上传 S3 的触发周期，单位为秒。最小值：600；最大值：100000。默认值 3600；企业版参数|
|s3MigrateEnabled |3.3.4.3 之后|是否自动进行 S3 迁移，默认值为 0，表示关闭自动 S3 迁移，可配置为 1；企业版参数|
|s3Accesskey |3.3.4.3 之后|冒号分隔的用户 SecretId:SecretKey，例如 AKIDsQmwsfKxTo2A6nGVXZN0UlofKn6JRRSJ:lIdoy99ygEacU7iHfogaN2Xq0yumSm1E；企业版参数|
|s3Endpoint |3.3.4.3 之后|用户所在地域的 COS 服务域名，支持 http 和 https，bucket 的区域需要与 endpoint 保持一致，否则无法访问；企业版参数|
|s3BucketName |3.3.4.3 之后|存储桶名称，减号后面是用户注册 COS 服务的 AppId，其中 AppId 是 COS 特有，AWS 和阿里云都没有，配置时需要作为 bucket name 的一部分，使用减号分隔；参数值均为字符串类型，但不需要引号；例如 test0711-1309024725；企业版参数|
|s3PageCacheSize |3.3.4.3 之后|S3 page cache 缓存页数目，取值范围 4-1048576，单位为页，默认值 4096；企业版参数|
|s3UploadDelaySec |3.3.4.3 之后|data 文件持续多长时间不再变动后上传至 S3，取值范围 1-2592000 (30天），单位为秒，默认值 60；企业版参数|
|cacheLazyLoadThreshold | |内部参数，缓存的装载策略|
### 集群相关
|参数名称|支持版本|参数含义|
|--------------------------|-----------|-|
|supportVnodes | |dnode 支持的最大 vnode 数目，取值范围 0-4096，默认值 CPU 核数的 2 倍 + 5|
|numOfCommitThreads | |落盘线程的最大数量，取值范围 0-1024，默认值为 4|
|numOfMnodeReadThreads | |mnode 的 Read 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）|
|numOfVnodeQueryThreads | |vnode 的 Query 线程数目，取值范围 0-1024，默认值为 CPU 核数的两倍（不超过 16）|
|numOfVnodeFetchThreads | |vnode 的 Fetch 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）|
|numOfVnodeRsmaThreads | |vnode 的 Rsma 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）|
|numOfQnodeQueryThreads | |qnode 的 Query 线程数目，取值范围 0-1024，默认值为 CPU 核数的两倍（不超过 16）|
|numOfSnodeSharedThreads | |snode 的共享线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不小于 2，不超过 4）|
|numOfSnodeUniqueThreads | |snode 的独占线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不小于 2，不超过 4）|
|ratioOfVnodeStreamThreads | |流计算使用 vnode 线程的比例，取值范围 0.01-4，默认值 4|
|ttlUnit | |ttl 参数的单位，取值范围 1-31572500，单位为秒，默认值 86400|
|ttlPushInterval | |ttl 检测超时频率，取值范围 1-100000，单位为秒，默认值 10|
|ttlChangeOnWrite | |ttl 到期时间是否伴随表的修改操作改变；0：不改变，1：改变；默认值为 0|
|ttlBatchDropNum | |ttl 一批删除子表的数目，最小值为 0，默认值 10000|
|retentionSpeedLimitMB | |数据在不同级别硬盘上迁移时的速度限制，取值范围 0-1024，单位 MB，默认值 0，表示不限制|
|maxTsmaNum | |集群内可创建的TSMA个数；取值范围 0-3；默认值 3|
|tmqMaxTopicNum | |订阅最多可建立的 topic 数量；取值范围 1-10000；默认值为 20|
|tmqRowSize | |订阅数据块的最大记录条数，取值范围 1-1000000，默认值 4096|
|audit | |审计功能开关；企业版参数|
|auditInterval | |审计数据上报的时间间隔；企业版参数|
|auditCreateTable | |是否针对创建子表开启申计功能；企业版参数|
|encryptAlgorithm | |数据加密算法；企业版参数|
|encryptScope | |加密范围；企业版参数|
|enableWhiteList | |白名单功能开关；企业版参数|
|syncLogBufferMemoryAllowed| |一个 dnode 允许的 sync 日志缓存消息占用的内存最大值，单位 bytes，取值范围 104857600-INT64_MAX，默认值 服务器内存的 1/10，3.1.3.2/3.3.2.13 版本开始生效 |
|syncElectInterval | |内部参数，用于同步模块调试|
|syncHeartbeatInterval | |内部参数，用于同步模块调试|
|syncHeartbeatTimeout | |内部参数，用于同步模块调试|
|syncSnapReplMaxWaitN | |内部参数，用于同步模块调试|
|syncSnapReplMaxWaitN | |内部参数，用于同步模块调试|
|arbHeartBeatIntervalSec | |内部参数，用于同步模块调试|
|arbCheckSyncIntervalSec | |内部参数，用于同步模块调试|
|arbSetAssignedTimeoutSec | |内部参数，用于同步模块调试|
|mndSdbWriteDelta | |内部参数，用于 mnode 模块调试|
|mndLogRetention | |内部参数，用于 mnode 模块调试|
|skipGrant | |内部参数，用于授权检查|
|trimVDbIntervalSec | |内部参数，用于删除过期数据|
|ttlFlushThreshold | |内部参数，ttl 定时器的频率|
|compactPullupInterval | |内部参数，数据重整定时器的频率|
|walFsyncDataSizeLimit | |内部参数，WAL 进行 FSYNC 的阈值|
|transPullupInterval | |内部参数，mnode 执行事务的重试间隔|
|mqRebalanceInterval | |内部参数，消费者再平衡的时间间隔|
|uptimeInterval | |内部参数，用于记录系统启动时间|
|timeseriesThreshold | |内部参数，用于统计用量|
|udf | |是否启动 UDF 服务；0：不启动，1：启动；默认值为 0 |
|udfdResFuncs | |内部参数，用于 UDF 结果集设置|
|udfdLdLibPath | |内部参数，表示 UDF 装载的库路径|
| 参数名称 | 参数说明 |
| :-----------: | :-------------------------------------------------------------------------: |
| supportVnodes | dnode 支持的最大 vnode 数目，取值范围：0-4096，缺省值： CPU 核数的 2 倍 + 5 |
### 内存相关
| 参数名称 | 参数说明 |
| :----------------: | :---------------------------------------------: |
| rpcQueueMemoryAllowed | 一个 dnode 允许的 rpc 消息占用的内存最大值，单位 bytes，取值范围：104857600-INT64_MAX，缺省值：服务器内存的 1/10 |
| syncLogBufferMemoryAllowed | 一个 dnode 允许的 sync 日志缓存消息占用的内存最大值，单位 bytes，取值范围：104857600-INT64_MAX，缺省值：服务器内存的 1/10，3.1.3.2/3.3.2.13 版本开始生效 |
### 性能调优
| 参数名称 | 参数说明 |
| :----------------: | :---------------------------------------------: |
| numOfCommitThreads | 落盘线程的最大数量，取值范围 0-1024，缺省值为 4 |
### 流计算参数
|参数名称|支持版本|参数含义|
|-----------------------|-----------|-|
|disableStream | |流计算的启动开关|
|streamBufferSize | |控制内存中窗口状态缓存的大小，默认值为 128MB|
|streamAggCnt | |内部参数，并发进行聚合计算的数目|
|checkpointInterval | |内部参数，checkponit 同步间隔|
|concurrentCheckpoint | |内部参数，是否并发检查 checkpoint|
|maxStreamBackendCache | |内部参数，流计算使用的最大缓存|
|streamSinkDataRate | |内部参数，用于控制流计算结果的写入速度|
### 日志相关
|参数名称|支持版本|参数含义|
|----------------|-----------|-|
|logDir | |日志文件目录，运行日志将写入该目录，默认值 /var/log/taos|
|minimalLogDirGB | |日志文件夹所在磁盘可用空间大小小于该值时，停止写日志，单位 GB，默认值 1|
|numOfLogLines | |单个日志文件允许的最大行数，默认值 10,000,000|
|asyncLog | |日志写入模式，0：同步，1：异步，默认值 1|
|logKeepDays | |日志文件的最长保存时间，单位：天，默认值 0，意味着无限保存，日志文件不会被重命名，也不会有新的日志文件滚动产生，但日志文件的内容有可能会不断滚动，取决于日志文件大小的设置；当设置为大于 0 的值时，当日志文件大小达到设置的上限时会被重命名为 taosdlog.yyy，其中 yyy 为日志文件最后修改的时间戳，并滚动产生新的日志文件|
|slowLogThreshold|3.3.3.0 之后|慢查询门限值，大于等于门限值认为是慢查询，单位秒，默认值 3 |
|slowLogMaxLen |3.3.3.0 之后|慢查询日志最大长度，取值范围 1-16384，默认值 4096|
|slowLogScope |3.3.3.0 之后|慢查询记录类型，取值范围 ALL/QUERY/INSERT/OTHERS/NONE，默认值 QUERY|
|slowLogExceptDb |3.3.3.0 之后|指定的数据库不上报慢查询，仅支持配置换一个数据库|
|debugFlag | |运行日志开关，131（输出错误和警告日志），135（输出错误、警告和调试日志），143（输出错误、警告、调试和跟踪日志）；默认值 131 或 135 （取决于不同模块）|
|tmrDebugFlag | |定时器模块的日志开关，取值范围同上|
|uDebugFlag | |共用功能模块的日志开关，取值范围同上|
|rpcDebugFlag | |rpc 模块的日志开关，取值范围同上|
|qDebugFlag | |query 模块的日志开关，取值范围同上|
|dDebugFlag | |dnode 模块的日志开关，取值范围同上|
|vDebugFlag | |vnode 模块的日志开关，取值范围同上|
|mDebugFlag | |mnode 模块的日志开关，取值范围同上|
|azDebugFlag |3.3.4.3 之后|S3 模块的日志开关，取值范围同上|
|sDebugFlag | |sync 模块的日志开关，取值范围同上|
|tsdbDebugFlag | |tsdb 模块的日志开关，取值范围同上|
|tqDebugFlag | |tq 模块的日志开关，取值范围同上|
|fsDebugFlag | |fs 模块的日志开关，取值范围同上|
|udfDebugFlag | |udf 模块的日志开关，取值范围同上|
|smaDebugFlag | |sma 模块的日志开关，取值范围同上|
|idxDebugFlag | |index 模块的日志开关，取值范围同上|
|tdbDebugFlag | |tdb 模块的日志开关，取值范围同上|
|metaDebugFlag | |meta 模块的日志开关，取值范围同上|
|stDebugFlag | |stream 模块的日志开关，取值范围同上|
|sndDebugFlag | |snode 模块的日志开关，取值范围同上|
| 参数名称 | 参数说明 |
| :--------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------------: |
| logDir | 日志文件目录，运行日志将写入该目录，缺省值：/var/log/taos |
| minimalLogDirGB | 当日志文件夹所在磁盘可用空间大小小于该值时，停止写日志，单位GB，缺省值：1 |
| numOfLogLines | 单个日志文件允许的最大行数，缺省值：10,000,000 |
| asyncLog | 日志写入模式，0: 同步，1: 异步，缺省值: 1 |
| logKeepDays | 日志文件的最长保存时间 ，单位：天，缺省值：0，意味着无限保存，日志文件不会被重命名，也不会有新的日志文件滚动产生，但日志文件的内容有可能会不断滚动，取决于日志文件大小的设置；当设置为大于0 的值时，当日志文件大小达到设置的上限时会被重命名为 taosdlog.xxx，其中 xxx 为日志文件最后修改的时间戳，并滚动产生新的日志文件 |
| slowLogThreshold | 慢查询门限值，大于等于门限值认为是慢查询，单位秒，默认值: 3 |
| slowLogScope | 定启动记录哪些类型的慢查询，可选值：ALL, QUERY, INSERT, OHTERS, NONE; 默认值：ALL |
| debugFlag | 运行日志开关，131（输出错误和警告日志），135（输出错误、警告和调试日志），143（输出错误、警告、调试和跟踪日志）; 默认值：131 或 135 （取决于不同模块） |
| tmrDebugFlag | 定时器模块的日志开关，取值范围同上 |
| uDebugFlag | 共用功能模块的日志开关，取值范围同上 |
| rpcDebugFlag | rpc 模块的日志开关，取值范围同上 |
| cDebugFlag | 客户端模块的日志开关，取值范围同上 |
| jniDebugFlag | jni 模块的日志开关，取值范围同上 |
| qDebugFlag | query 模块的日志开关，取值范围同上 |
| dDebugFlag | dnode 模块的日志开关，取值范围同上，缺省值 135 |
| vDebugFlag | vnode 模块的日志开关，取值范围同上 |
| mDebugFlag | mnode 模块的日志开关，取值范围同上 |
| wDebugFlag | wal 模块的日志开关，取值范围同上 |
| sDebugFlag | sync 模块的日志开关，取值范围同上 |
| tsdbDebugFlag | tsdb 模块的日志开关，取值范围同上 |
| tqDebugFlag | tq 模块的日志开关，取值范围同上 |
| fsDebugFlag | fs 模块的日志开关，取值范围同上 |
| udfDebugFlag | udf 模块的日志开关，取值范围同上 |
| smaDebugFlag | sma 模块的日志开关，取值范围同上 |
| idxDebugFlag | index 模块的日志开关，取值范围同上 |
| tdbDebugFlag | tdb 模块的日志开关，取值范围同上 |
### 调试相关
|参数名称|支持版本|参数含义|
|--------------------|-----------|-|
|enableCoreFile | |crash 时是否生成 core 文件，0：不生成，1：生成；默认值 1|
|configDir | |配置文件所在目录|
|scriptDir | |内部测试工具的脚本目录|
|assert | |断言控制开关，默认值 0|
|randErrorChance | |内部参数，用于随机失败测试|
|randErrorDivisor | |内部参数，用于随机失败测试|
|randErrorScope | |内部参数，用于随机失败测试|
|safetyCheckLevel | |内部参数，用于随机失败测试|
|experimental | |内部参数，用于一些实验特性|
|simdEnable |3.3.4.3 之后|内部参数，用于测试 SIMD 加速|
|AVX512Enable |3.3.4.3 之后|内部参数，用于测试 AVX512 加速|
|rsyncPort | |内部参数，用于调试流计算|
|snodeAddress | |内部参数，用于调试流计算|
|checkpointBackupDir | |内部参数，用于恢复 snode 数据|
|enableAuditDelete | |内部参数，用于测试审计功能|
|slowLogThresholdTest| |内部参数，用于测试慢日志|
### 压缩参数
| 参数名称 | 参数说明 |
|:-------------:|:----------------------------------------------------------------:|
| compressMsgSize | 是否对 RPC 消息进行压缩；-1: 所有消息都不压缩; 0: 所有消息都压缩; N (N>0): 只有大于 N 个字节的消息才压缩；缺省值 -1 |
| fPrecision | 设置 float 类型浮点数压缩精度 ，取值范围：0.1 ~ 0.00000001 ，默认值 0.00000001 , 小于此值的浮点数尾数部分将被截断 |
|dPrecision | 设置 double 类型浮点数压缩精度 , 取值范围：0.1 ~ 0.0000000000000001 ， 缺省值 0.0000000000000001 ， 小于此值的浮点数尾数部分将被截取 |
|lossyColumn | 对 float 和/或 double 类型启用 TSZ 有损压缩；取值范围： float, double, none；缺省值: none，表示关闭无损压缩。**注意：此参数在 3.3.0.0 及更高版本中不再使用** |
|ifAdtFse | 在启用 TSZ 有损压缩时，使用 FSE 算法替换 HUFFMAN 算法， FSE 算法压缩速度更快，但解压稍慢，追求压缩速度可选用此算法; 0: 关闭，1：打开；默认值为 0 |
|参数名称|支持版本|参数含义|
|------------|-----------|-|
|fPrecision | |设置 float 类型浮点数压缩精度 ，取值范围 0.1 ~ 0.00000001 ，默认值 0.00000001 , 小于此值的浮点数尾数部分将被截断|
|dPrecision | |设置 double 类型浮点数压缩精度 , 取值范围 0.1 ~ 0.0000000000000001 ， 默认值 0.0000000000000001 ， 小于此值的浮点数尾数部分将被截取|
|lossyColumn |3.3.0.0 之前|对 float 和/或 double 类型启用 TSZ 有损压缩；取值范围 float/double/none；默认值 none，表示关闭无损压缩|
|ifAdtFse | |在启用 TSZ 有损压缩时，使用 FSE 算法替换 HUFFMAN 算法，FSE 算法压缩速度更快，但解压稍慢，追求压缩速度可选用此算法；0：关闭，1：打开；默认值为 0|
|maxRange | |内部参数，用于有损压缩设置|
|curRange | |内部参数，用于有损压缩设置|
|compressor | |内部参数，用于有损压缩设置|
**补充说明**
1. 在 3.2.0.0 ~ 3.3.0.0（不包含）版本生效，启用该参数后不能回退到升级前的版本
@ -220,16 +319,6 @@ lossyColumns float|double
02/22 10:49:27.607990 00002933 UTL lossyColumns float|double
```
### 其他参数
| 参数名称 | 参数说明 |
| :--------------: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |
| enableCoreFile | crash 时是否生成 core 文件；0: 不生成，1：生成；默认值为 1; 不同的启动方式，生成 core 文件的目录如下：1、systemctl start taosd 启动：生成的 core 在根目录下 <br/> 2、手动启动，就在 taosd 执行目录下。 |
| udf | 是否启动 UDF 服务；0: 不启动，1：启动；默认值为 0 |
| ttlChangeOnWrite | ttl 到期时间是否伴随表的修改操作改变; 0: 不改变，1：改变；默认值为 0 |
| tmqMaxTopicNum | 订阅最多可建立的 topic 数量; 取值范围 1-10000；缺省值为20 |
| maxTsmaNum | 集群内可创建的TSMA个数；取值范围：0-3；缺省值为 3 |
## taosd 监控指标
@ -282,7 +371,7 @@ taosd 会将监控指标上报给 taosKeeper，这些监控指标会被 taosKeep
| :------------- | :-------- | :------ | :--------------------------------------------- |
| \_ts | TIMESTAMP | | timestamp |
| tables\_num | DOUBLE | | vgroup 中 table 数量 |
| status | DOUBLE | | vgroup 状态, 取值范围：unsynced = 0, ready = 1 |
| status | DOUBLE | | vgroup 状态, 取值范围 unsynced = 0, ready = 1 |
| vgroup\_id | VARCHAR | TAG | vgroup id |
| database\_name | VARCHAR | TAG | vgroup 所属的 database 名字 |
| cluster\_id | VARCHAR | TAG | cluster id |
@ -311,10 +400,10 @@ taosd 会将监控指标上报给 taosKeeper，这些监控指标会被 taosKeep
| io\_write\_disk | DOUBLE | | 磁盘 io 吞吐率，从 `/proc/<taosd_pid>/io` 中读取的 write_bytes。单位 byte/s |
| vnodes\_num | DOUBLE | | dnode 上 vnodes 数量 |
| masters | DOUBLE | | dnode 上 master node 数量 |
| has\_mnode | DOUBLE | | dnode 是否包含 mnode，取值范围：包含=1,不包含=0 |
| has\_qnode | DOUBLE | | dnode 是否包含 qnode，取值范围：包含=1,不包含=0 |
| has\_snode | DOUBLE | | dnode 是否包含 snode，取值范围：包含=1,不包含=0 |
| has\_bnode | DOUBLE | | dnode 是否包含 bnode，取值范围：包含=1,不包含=0 |
| has\_mnode | DOUBLE | | dnode 是否包含 mnode，取值范围 包含=1,不包含=0 |
| has\_qnode | DOUBLE | | dnode 是否包含 qnode，取值范围 包含=1,不包含=0 |
| has\_snode | DOUBLE | | dnode 是否包含 snode，取值范围 包含=1,不包含=0 |
| has\_bnode | DOUBLE | | dnode 是否包含 bnode，取值范围 包含=1,不包含=0 |
| error\_log\_count | DOUBLE | | error 总数 |
| info\_log\_count | DOUBLE | | info 总数 |
| debug\_log\_count | DOUBLE | | debug 总数 |
@ -330,7 +419,7 @@ taosd 会将监控指标上报给 taosKeeper，这些监控指标会被 taosKeep
| field | type | is\_tag | comment |
| :---------- | :-------- | :------ | :--------------------------------------- |
| \_ts | TIMESTAMP | | timestamp |
| status | DOUBLE | | dnode 状态,取值范围：ready=1，offline =0 |
| status | DOUBLE | | dnode 状态,取值范围 ready=1，offline =0 |
| dnode\_id | VARCHAR | TAG | dnode id |
| dnode\_ep | VARCHAR | TAG | dnode endpoint |
| cluster\_id | VARCHAR | TAG | cluster id |
@ -373,7 +462,7 @@ taosd 会将监控指标上报给 taosKeeper，这些监控指标会被 taosKeep
| field | type | is\_tag | comment |
| :---------- | :-------- | :------ | :------------------------------------------------------------------------------------------------------- |
| \_ts | TIMESTAMP | | timestamp |
| role | DOUBLE | | mnode 角色， 取值范围：offline = 0,follower = 100,candidate = 101,leader = 102,error = 103,learner = 104 |
| role | DOUBLE | | mnode 角色， 取值范围 offline = 0,follower = 100,candidate = 101,leader = 102,error = 103,learner = 104 |
| mnode\_id | VARCHAR | TAG | master node id |
| mnode\_ep | VARCHAR | TAG | master node endpoint |
| cluster\_id | VARCHAR | TAG | cluster id |
@ -385,7 +474,7 @@ taosd 会将监控指标上报给 taosKeeper，这些监控指标会被 taosKeep
| field | type | is\_tag | comment |
| :------------- | :-------- | :------ | :------------------------------------------------------------------------------------------------------ |
| \_ts | TIMESTAMP | | timestamp |
| vnode\_role | DOUBLE | | vnode 角色，取值范围：offline = 0,follower = 100,candidate = 101,leader = 102,error = 103,learner = 104 |
| vnode\_role | DOUBLE | | vnode 角色，取值范围 offline = 0,follower = 100,candidate = 101,leader = 102,error = 103,learner = 104 |
| vgroup\_id | VARCHAR | TAG | dnode id |
| dnode\_id | VARCHAR | TAG | dnode id |
| database\_name | VARCHAR | TAG | vgroup 所属的 database 名字 |
@ -399,9 +488,9 @@ taosd 会将监控指标上报给 taosKeeper，这些监控指标会被 taosKeep
| :---------- | :-------- | :------ | :--------------------------------------- |
| \_ts | TIMESTAMP | | timestamp |
| count | DOUBLE | | sql 数量 |
| result | VARCHAR | TAG | sql的执行结果，取值范围：Success, Failed |
| result | VARCHAR | TAG | sql的执行结果，取值范围 Success, Failed |
| username | VARCHAR | TAG | 执行sql的user name |
| sql\_type | VARCHAR | TAG | sql类型，取值范围：inserted_rows |
| sql\_type | VARCHAR | TAG | sql类型，取值范围 inserted_rows |
| dnode\_id | VARCHAR | TAG | dnode id |
| dnode\_ep | VARCHAR | TAG | dnode endpoint |
| vgroup\_id | VARCHAR | TAG | dnode id |
@ -415,9 +504,9 @@ taosd 会将监控指标上报给 taosKeeper，这些监控指标会被 taosKeep
| :---------- | :-------- | :------ | :---------------------------------------- |
| \_ts | TIMESTAMP | | timestamp |
| count | DOUBLE | | sql 数量 |
| result | VARCHAR | TAG | sql的执行结果，取值范围：Success, Failed |
| result | VARCHAR | TAG | sql的执行结果，取值范围 Success, Failed |
| username | VARCHAR | TAG | 执行sql的user name |
| sql\_type | VARCHAR | TAG | sql类型，取值范围：select, insert，delete |
| sql\_type | VARCHAR | TAG | sql类型，取值范围 select, insert，delete |
| cluster\_id | VARCHAR | TAG | cluster id |
### taos\_slow\_sql 表
@ -428,9 +517,9 @@ taosd 会将监控指标上报给 taosKeeper，这些监控指标会被 taosKeep
| :---------- | :-------- | :------ | :---------------------------------------------------- |
| \_ts | TIMESTAMP | | timestamp |
| count | DOUBLE | | sql 数量 |
| result | VARCHAR | TAG | sql的执行结果，取值范围：Success, Failed |
| result | VARCHAR | TAG | sql的执行结果，取值范围 Success, Failed |
| username | VARCHAR | TAG | 执行sql的user name |
| duration | VARCHAR | TAG | sql执行耗时，取值范围：3-10s,10-100s,100-1000s,1000s- |
| duration | VARCHAR | TAG | sql执行耗时，取值范围 3-10s,10-100s,100-1000s,1000s- |
| cluster\_id | VARCHAR | TAG | cluster id |
## 日志相关

127
docs/zh/14-reference/01-components/02-taosc.md
View File

@ -8,38 +8,101 @@ TDengine 客户端驱动提供了应用编程所需要的全部 API，并且在
## 配置参数
| 参数名称 | 参数含义 |
|:-----------:|:----------------------------------------------------------:|
|firstEp | taos 启动时，主动连接的集群中首个 dnode 的 endpoint，缺省值：hostname:6030，若无法获取该服务器的 hostname，则赋值为 localhost |
|secondEp | 启动时，如果 firstEp 连接不上，尝试连接集群中第二个 dnode 的 endpoint，没有缺省值 |
|numOfRpcSessions | 一个客户端能创建的最大连接数，取值范围：10-50000000(单位为毫秒)；缺省值：500000 |
|telemetryReporting | 是否上传 telemetry，0: 不上传，1： 上传；缺省值：1 |
|crashReporting | 是否上传 telemetry，0: 不上传，1： 上传；缺省值：1 |
|queryPolicy | 查询语句的执行策略，1: 只使用 vnode，不使用 qnode; 2: 没有扫描算子的子任务在 qnode 执行，带扫描算子的子任务在 vnode 执行; 3: vnode 只运行扫描算子，其余算子均在 qnode 执行 ；缺省值：1 |
|querySmaOptimize | sma index 的优化策略，0: 表示不使用 sma index，永远从原始数据进行查询; 1: 表示使用 sma index，对符合的语句，直接从预计算的结果进行查询；缺省值：0 |
|keepColumnName | Last、First、LastRow 函数查询且未指定别名时，自动设置别名为列名（不含函数名），因此 order by 子句如果引用了该列名将自动引用该列对应的函数; 1: 表示自动设置别名为列名(不包含函数名), 0: 表示不自动设置别名; 缺省值: 0 |
|countAlwaysReturnValue | count/hyperloglog函数在输入数据为空或者NULL的情况下是否返回值; 0：返回空行，1：返回; 缺省值 1; 该参数设置为 1 时，如果查询中含有 INTERVAL 子句或者该查询使用了TSMA时, 且相应的组或窗口内数据为空或者NULL， 对应的组或窗口将不返回查询结果. 注意此参数客户端和服务端值应保持一致. |
|multiResultFunctionStarReturnTags | 查询超级表时，last(\*)/last_row(\*)/first(\*) 是否返回标签列；查询普通表、子表时，不受该参数影响; 0：不返回标签列，1：返回标签列 ; 缺省值: 0; 该参数设置为 0 时，last(\*)/last_row(\*)/first(\*) 只返回超级表的普通列；为 1 时，返回超级表的普通列和标签列 |
|maxTsmaCalcDelay| 查询时客户端可允许的tsma计算延迟, 若tsma的计算延迟大于配置值, 则该TSMA将不会被使用.; 取值范围: 600s - 86400s, 即10分钟-1小时 ; 缺省值：600 秒|
|tsmaDataDeleteMark |TSMA计算的历史数据中间结果保存时间, 单位为毫秒; 取值范围：>= 3600000, 即大于等于1h; 缺省值: 86400000, 即1d |
|timezone | 时区; 缺省从系统中动态获取当前的时区设置 |
|locale | 系统区位信息及编码格式, 缺省从系统中获取 |
|charset | 字符集编码，缺省从系统中获取 |
|metaCacheMaxSize | 指定单个客户端元数据缓存大小的最大值, 单位 MB; 缺省值 -1，表示无限制 |
|logDir | 日志文件目录，客户端运行日志将写入该目录, 缺省值: /var/log/taos |
|minimalLogDirGB | 当日志文件夹所在磁盘可用空间大小小于该值时，停止写日志; 缺省值 1 |
|numOfLogLines | 单个日志文件允许的最大行数; 缺省值 10,000,000 |
|asyncLog | 是否异步写入日志，0：同步；1：异步；缺省值：1 |
|logKeepDays | 日志文件的最长保存时间; 缺省值: 0，表示无限保存; 大于 0 时，日志文件会被重命名为 taosdlog.xxx，其中 xxx 为日志文件最后修改的时间戳|
|smlChildTableName | schemaless 自定义的子表名的 key, 无缺省值 |
|smlAutoChildTableNameDelimiter | schemaless tag之间的连接符，连起来作为子表名，无缺省值 |
|smlTagName | schemaless tag 为空时默认的 tag 名字, 缺省值 "_tag_null" |
|smlTsDefaultName | schemaless自动建表的时间列名字通过该配置设置, 缺省值 "_ts" |
|smlDot2Underline | schemaless 把超级表名中的 dot 转成下划线 |
|enableCoreFile | crash 时是否生成 core 文件，0: 不生成， 1： 生成；缺省值：1 |
|enableScience | 是否开启科学计数法显示浮点数; 0: 不开始, 1: 开启；缺省值：1 |
|compressMsgSize | 是否对 RPC 消息进行压缩; -1: 所有消息都不压缩; 0: 所有消息都压缩; N (N>0): 只有大于 N 个字节的消息才压缩; 缺省值 -1|
|queryTableNotExistAsEmpty | 查询表不存在时是否返回空结果集; false: 返回错误; true: 返回空结果集; 缺省值 false|
### 连接相关
|参数名称|支持版本|参数含义|
|----------------------|-----------|-|
|firstEp | |启动时，主动连接的集群中首个 dnode 的 endpoint，缺省值：hostname:6030，若无法获取该服务器的 hostname，则赋值为 localhost|
|secondEp | |启动时，如果 firstEp 连接不上，尝试连接集群中第二个 dnode 的 endpoint，没有缺省值|
|compressMsgSize | |是否对 RPC 消息进行压缩；-1：所有消息都不压缩；0：所有消息都压缩；N (N>0)：只有大于 N 个字节的消息才压缩；缺省值 -1|
|shellActivityTimer | |客户端向 mnode 发送心跳的时长，单位为秒，取值范围 1-120，默认值 3|
|numOfRpcSessions | |RPC 支持的最大连接数，取值范围 100-100000，缺省值 30000|
|numOfRpcThreads | |RPC 线程数目，默认值为 CPU 核数的一半|
|timeToGetAvailableConn| |获得可用连接的最长等待时间，取值范围 10-50000000，单位为毫秒，缺省值 500000|
|useAdapter | |内部参数，是否使用 taosadapter，影响 CSV 文件导入|
|shareConnLimit |3.3.4.3 之后|内部参数，一个链接可以共享的查询数目，取值范围 1-256，默认值 10|
|readTimeout |3.3.4.3 之后|内部参数，最小超时时间，取值范围 64-604800，单位为秒，默认值 900|
### 查询相关
|参数名称|支持版本|参数含义|
|---------------------------------|-----------|-|
|countAlwaysReturnValue | |count/hyperloglog 函数在输入数据为空或者 NULL 的情况下是否返回值；0：返回空行，1：返回；默认值 1；该参数设置为 1 时，如果查询中含有 INTERVAL 子句或者该查询使用了 TSMA 时，且相应的组或窗口内数据为空或者 NULL，对应的组或窗口将不返回查询结果；注意此参数客户端和服务端值应保持一致|
|keepColumnName | |Last、First、LastRow 函数查询且未指定别名时，自动设置别名为列名（不含函数名），因此 order by 子句如果引用了该列名将自动引用该列对应的函数；1：表示自动设置别名为列名(不包含函数名)，0：表示不自动设置别名；缺省值：0|
|multiResultFunctionStarReturnTags|3.3.3.0 以后|查询超级表时，last(\*)/last_row(\*)/first(\*) 是否返回标签列；查询普通表、子表时，不受该参数影响；0：不返回标签列，1：返回标签列；缺省值：0；该参数设置为 0 时，last(\*)/last_row(\*)/first(\*) 只返回超级表的普通列；为 1 时，返回超级表的普通列和标签列|
|
| |指定单个客户端元数据缓存大小的最大值，单位 MB；缺省值 -1，表示无限制|
|maxTsmaCalcDelay | |查询时客户端可允许的 tsma 计算延迟，若 tsma 的计算延迟大于配置值，则该 TSMA 将不会被使用；取值范围 600s - 86400s，即 10 分钟 - 1 小时；缺省值：600 秒|
|tsmaDataDeleteMark | |TSMA 计算的历史数据中间结果保存时间，单位为毫秒；取值范围 >= 3600000，即大于等于1h；缺省值：86400000，即 1d |
|queryPolicy | |查询语句的执行策略，1：只使用 vnode，不使用 qnode；2：没有扫描算子的子任务在 qnode 执行，带扫描算子的子任务在 vnode 执行；3：vnode 只运行扫描算子，其余算子均在 qnode 执行；缺省值：1|
|queryTableNotExistAsEmpty | |查询表不存在时是否返回空结果集；false：返回错误；true：返回空结果集；缺省值 false|
|querySmaOptimize | |sma index 的优化策略，0：表示不使用 sma index，永远从原始数据进行查询；1：表示使用 sma index，对符合的语句，直接从预计算的结果进行查询；缺省值：0|
|queryPlannerTrace | |内部参数，查询计划是否输出详细日志|
|queryNodeChunkSize | |内部参数，查询计划的块大小|
|queryUseNodeAllocator | |内部参数，查询计划的分配方法|
|queryMaxConcurrentTables | |内部参数，查询计划的并发数目|
|enableQueryHb | |内部参数，是否发送查询心跳消息|
|minSlidingTime | |内部参数，sliding 的最小允许值|
|minIntervalTime | |内部参数，interval 的最小允许值|
### 写入相关
|参数名称|支持版本|参数含义|
|------------------------------|-----------|-|
|smlChildTableName | |schemaless 自定义的子表名的 key，无缺省值|
|smlAutoChildTableNameDelimiter| |schemaless tag 之间的连接符，连起来作为子表名，无缺省值|
|smlTagName | |schemaless tag 为空时默认的 tag 名字，缺省值 "_tag_null"|
|smlTsDefaultName | |schemaless 自动建表的时间列名字通过该配置设置，缺省值 "_ts"|
|smlDot2Underline | |schemaless 把超级表名中的 dot 转成下划线|
|maxInsertBatchRows | |内部参数，一批写入的最大条数|
### 区域相关
|参数名称|支持版本|参数含义|
|-----------------|-----------|-|
|timezone | |时区；缺省从系统中动态获取当前的时区设置|
|locale | |系统区位信息及编码格式，缺省从系统中获取|
|charset | |字符集编码，缺省从系统中获取|
### 存储相关
|参数名称|支持版本|参数含义|
|-----------------|-----------|-|
|tempDir | |指定所有运行过程中的临时文件生成的目录，Linux 平台默认值为 /tmp|
|minimalTmpDirGB | |tempDir 所指定的临时文件目录所需要保留的最小空间，单位 GB，缺省值：1|
### 日志相关
|参数名称|支持版本|参数含义|
|-----------------|-----------|-|
|logDir | |日志文件目录，运行日志将写入该目录，缺省值：/var/log/taos|
|minimalLogDirGB | |日志文件夹所在磁盘可用空间大小小于该值时，停止写日志，单位 GB，缺省值：1|
|numOfLogLines | |单个日志文件允许的最大行数，缺省值：10,000,000|
|asyncLog | |日志写入模式，0：同步，1：异步，缺省值：1|
|logKeepDays | |日志文件的最长保存时间，单位：天，缺省值：0，意味着无限保存，日志文件不会被重命名，也不会有新的日志文件滚动产生，但日志文件的内容有可能会不断滚动，取决于日志文件大小的设置；当设置为大于 0 的值时，当日志文件大小达到设置的上限时会被重命名为 taoslogx.yyy，其中 yyy 为日志文件最后修改的时间戳，并滚动产生新的日志文件|
|debugFlag | |运行日志开关，131（输出错误和警告日志），135（输出错误、警告和调试日志），143（输出错误、警告、调试和跟踪日志）；默认值 131 或 135 （取决于不同模块）|
|tmrDebugFlag | |定时器模块的日志开关，取值范围同上|
|uDebugFlag | |共用功能模块的日志开关，取值范围同上|
|rpcDebugFlag | |rpc 模块的日志开关，取值范围同上|
|jniDebugFlag | |jni 模块的日志开关，取值范围同上|
|qDebugFlag | |query 模块的日志开关，取值范围同上|
|cDebugFlag | |客户端模块的日志开关，取值范围同上|
|simDebugFlag | |内部参数，测试工具的日志开关，取值范围同上|
|tqClientDebugFlag|3.3.4.3 之后|客户端模块的日志开关，取值范围同上|
### 调试相关
|参数名称|支持版本|参数含义|
|-----------------|-----------|-|
|crashReporting | |是否上传 crash 到 telemetry，0：不上传，1：上传；缺省值：1|
|enableCoreFile | |crash 时是否生成 core 文件，0：不生成，1：生成；缺省值：1|
|assert | |断言控制开关，缺省值：0|
|configDir | |配置文件所在目录|
|scriptDir | |内部参数，测试用例的目录|
|randErrorChance |3.3.3.0 之后|内部参数，用于随机失败测试|
|randErrorDivisor |3.3.3.0 之后|内部参数，用于随机失败测试|
|randErrorScope |3.3.3.0 之后|内部参数，用于随机失败测试|
|safetyCheckLevel |3.3.3.0 之后|内部参数，用于随机失败测试|
|simdEnable |3.3.4.3 之后|内部参数，用于测试 SIMD 加速|
|AVX512Enable |3.3.4.3 之后|内部参数，用于测试 AVX512 加速|
### SHELL 相关
|参数名称|支持版本|参数含义|
|-----------------|-----------|-|
|enableScience | |是否开启科学计数法显示浮点数；0：不开始，1：开启；缺省值：1|
## API

2
docs/zh/14-reference/03-taos-sql/10-function.md
View File

@ -1988,7 +1988,7 @@ TOP(expr, k)
UNIQUE(expr)
```
**功能说明**：返回该列数据首次出现的值。该函数功能与 distinct 相似。对于存在复合主键的表的查询，若最小时间戳的数据有多条，则只有对应的复合主键最小的数据被返回。
**功能说明**：返回该列数据去重后的值。该函数功能与 distinct 相似。对于相同的数据，返回时间戳最小的一条，对于存在复合主键的表的查询，若最小时间戳的数据有多条，则只有对应的复合主键最小的数据被返回。
**返回数据类型**：同应用的字段。

705
docs/zh/14-reference/03-taos-sql/20-keywords.md
View File

@ -11,337 +11,470 @@ description: TDengine 保留关键字的详细列表
关键字列表如下：
### A
- ABORT
- ACCOUNT
- ACCOUNTS
- ADD
- AFTER
- AGGREGATE
- ALIVE
- ALL
- ALTER
- ANALYZE
- AND
- APPS
- AS
- ASC
- AT_ONCE
- ATTACH
|关键字|说明|
|----------------------|-|
| ABORT | |
| ACCOUNT | |
| ACCOUNTS | |
| ADD | |
| AFTER | |
| AGGREGATE | |
| ALIAS | |
| ALIVE | |
| ALL | |
| ALTER | |
| ANALYZE | 3.3.4.3 及后续版本 |
| AND | |
| ANODE | 3.3.4.3 及后续版本 |
| ANODES | 3.3.4.3 及后续版本 |
| ANOMALY_WINDOW | 3.3.4.3 及后续版本 |
| ANTI | |
| APPS | |
| ARBGROUPS | |
| ARROW | |
| AS | |
| ASC | |
| ASOF | |
| AT_ONCE | |
| ATTACH | |
### B
- BALANCE
- BEFORE
- BEGIN
- BETWEEN
- BIGINT
- BINARY
- BITAND
- BITNOT
- BITOR
- BLOCKS
- BNODE
- BNODES
- BOOL
- BUFFER
- BUFSIZE
- BY
|关键字|说明|
|----------------------|-|
| BALANCE | |
| BEFORE | |
| BEGIN | |
| BETWEEN | |
| BIGINT | |
| BIN | |
| BINARY | |
| BITAND | |
| BITAND | |
| BITNOT | |
| BITOR | |
| BLOB | |
| BLOCKS | |
| BNODE | |
| BNODES | |
| BOOL | |
| BOTH | |
| BUFFER | |
| BUFSIZE | |
| BWLIMIT | |
| BY | |
### C
- CACHE
- CACHEMODEL
- CACHESIZE
- CASCADE
- CAST
- CHANGE
- CLIENT_VERSION
- CLUSTER
- COLON
- COLUMN
- COMMA
- COMMENT
- COMP
- COMPACT
- CONCAT
- CONFLICT
- CONNECTION
- CONNECTIONS
- CONNS
- CONSUMER
- CONSUMERS
- CONTAINS
- COPY
- COUNT
- CREATE
- CURRENT_USER
|关键字|说明|
|----------------------|-|
| CACHE | |
| CACHEMODEL | |
| CACHESIZE | |
| CASE | |
| CAST | |
| CHANGE | |
| CHILD | |
| CLIENT_VERSION | |
| CLUSTER | |
| COLON | |
| COLUMN | |
| COMMA | |
| COMMENT | |
| COMP | |
| COMPACT | |
| COMPACTS | |
| CONCAT | |
| CONFLICT | |
| CONNECTION | |
| CONNECTIONS | |
| CONNS | |
| CONSUMER | |
| CONSUMERS | |
| CONTAINS | |
| COPY | |
| COUNT | |
| COUNT_WINDOW | |
| CREATE | |
| CREATEDB | |
| CURRENT_USER | |
### D
- DATABASE
- DATABASES
- DBS
- DEFERRED
- DELETE
- DELIMITERS
- DESC
- DESCRIBE
- DETACH
- DISTINCT
- DISTRIBUTED
- DIVIDE
- DNODE
- DNODES
- DOT
- DOUBLE
- DROP
- DURATION
|关键字|说明|
|----------------------|-|
| DATABASE | |
| DATABASES | |
| DBS | |
| DECIMAL | |
| DEFERRED | |
| DELETE | |
| DELETE_MARK | |
| DELIMITERS | |
| DESC | |
| DESCRIBE | |
| DETACH | |
| DISTINCT | |
| DISTRIBUTED | |
| DIVIDE | |
| DNODE | |
| DNODES | |
| DOT | |
| DOUBLE | |
| DROP | |
| DURATION | |
### E
- EACH
- ENABLE
- END
- EVERY
- EXISTS
- EXPIRED
- EXPLAIN
|关键字|说明|
|----------------------|-|
| EACH | |
| ELSE | |
| ENABLE | |
| ENCRYPT_ALGORITHM | |
| ENCRYPT_KEY | |
| ENCRYPTIONS | |
| END | |
| EQ | |
| EVENT_WINDOW | |
| EVERY | |
| EXCEPT | |
| EXISTS | |
| EXPIRED | |
| EXPLAIN | |
### F
- FAIL
- FILE
- FILL
- FIRST
- FLOAT
- FLUSH
- FOR
- FROM
- FUNCTION
- FUNCTIONS
|关键字|说明|
|----------------------|-|
| FAIL | |
| FHIGH | 3.3.4.3 及后续版本 |
| FILE | |
| FILL | |
| FILL_HISTORY | |
| FIRST | |
| FLOAT | |
| FLOW | 3.3.4.3 及后续版本 |
| FLUSH | |
| FOR | |
| FORCE | |
| FORCE_WINDOW_CLOSE | 3.3.4.3 及后续版本 |
| FROM | |
| FROWTS | 3.3.4.3 及后续版本 |
| FULL | |
| FUNCTION | |
| FUNCTIONS | |
### G
- GLOB
- GRANT
- GRANTS
- GROUP
|关键字|说明|
|----------------------|-|
| GE | |
| GEOMETRY | |
| GLOB | |
| GRANT | |
| GRANTS | |
| GROUP | |
| GT | |
### H
- HAVING
- HOST
|关键字|说明|
|----------------------|-|
| HAVING | |
| HEX | |
| HOST | |
### I
- ID
- IF
- IGNORE
- IMMEDIATE
- IMPORT
- IN
- INDEX
- INDEXES
- INITIALLY
- INNER
- INSERT
- INSTEAD
- INT
- INTEGER
- INTERVAL
- INTO
- IS
- IS NULL
|关键字|说明|
|----------------------|-|
| ID | |
| IF | |
| IGNORE | |
| ILLEGAL | |
| IMMEDIATE | |
| IMPORT | |
| IN | |
| INDEX | |
| INDEXES | |
| INITIALLY | |
| INNER | |
| INSERT | |
| INSTEAD | |
| INT | |
| INTEGER | |
| INTERSECT | |
| INTERVAL | |
| INTO | |
| IPTOKEN | |
| IROWTS | |
| IS | |
| IS_IMPORT | |
| ISFILLED | |
| ISNULL | |
### J
- JOIN
- JSON
|关键字|说明|
|----------------------|-|
| JLIMIT | |
| JOIN | |
| JSON | |
### K
- KEEP
- KEY
- KILL
|关键字|说明|
|----------------------|-|
| KEEP | |
| KEEP_TIME_OFFSET | |
| KEY | |
| KILL | |
### L
- LAST
- LAST_ROW
- LICENCES
- LIKE
- LIMIT
- LINEAR
- LOCAL
|关键字|说明|
|----------------------|-|
| LANGUAGE | |
| LAST | |
| LAST_ROW | |
| LE | |
| LEADER | |
| LEADING | |
| LEFT | |
| LICENCES | |
| LIKE | |
| LIMIT | |
| LINEAR | |
| LOCAL | |
| LOGS | |
| LP | |
| LSHIFT | |
| LT | |
### M
- MATCH
- MAX_DELAY
- BWLIMIT
- MAXROWS
- MAX_SPEED
- MERGE
- META
- MINROWS
- MINUS
- MNODE
- MNODES
- MODIFY
- MODULES
|关键字|说明|
|----------------------|-|
| MACHINES | |
| MATCH | |
| MAX_DELAY | |
| MAXROWS | |
| MEDIUMBLOB | |
| MERGE | |
| META | |
| MINROWS | |
| MINUS | |
| MNODE | |
| MNODES | |
| MODIFY | |
| MODULES | |
### N
- NCHAR
- NEXT
- NMATCH
- NONE
- NOT
- NOT NULL
- NOW
- NULL
- NULLS
|关键字|说明|
|----------------------|-|
| NCHAR | |
| NE | |
| NEXT | |
| NMATCH | |
| NONE | |
| NORMAL | |
| NOT | |
| NOTNULL | |
| NOW | |
| NULL | |
| NULL_F | |
| NULLS | |
### O
- OF
- OFFSET
- ON
- OR
- ORDER
- OUTPUTTYPE
|关键字|说明|
|----------------------|-|
| OF | |
| OFFSET | |
| ON | |
| ONLY | |
| OR | |
| ORDER | |
| OUTER | |
| OUTPUTTYPE | |
### P
- PAGES
- PAGESIZE
- PARTITIONS
- PASS
- PLUS
- PORT
- PPS
- PRECISION
- PREV
- PRIVILEGE
|关键字|说明|
|----------------------|-|
| PAGES | |
| PAGESIZE | |
| PARTITION | |
| PASS | |
| PAUSE | |
| PI | |
| PLUS | |
| PORT | |
| POSITION | |
| PPS | |
| PRECISION | |
| PREV | |
| PRIMARY | |
| PRIVILEGE | |
| PRIVILEGES | |
### Q
- QNODE
- QNODES
- QTIME
- QUERIES
- QUERY
|关键字|说明|
|----------------------|-|
| QDURATION | |
| QEND | |
| QNODE | |
| QNODES | |
| QSTART | |
| QTAGS | |
| QTIME | |
| QUERIES | |
| QUERY | |
| QUESTION | |
### R
- RAISE
- RANGE
- RATIO
- READ
- REDISTRIBUTE
- RENAME
- REPLACE
- REPLICA
- RESET
- RESTRICT
- RETENTIONS
- REVOKE
- ROLLUP
- ROW
|关键字|说明|
|----------------------|-|
| RAISE | |
| RAND | |
| RANGE | |
| RATIO | |
| READ | |
| RECURSIVE | |
| REDISTRIBUTE | |
| REM | |
| REPLACE | |
| REPLICA | |
| RESET | |
| RESTORE | |
| RESTRICT | |
| RESUME | |
| RETENTIONS | |
| REVOKE | |
| RIGHT | |
| ROLLUP | |
| ROW | |
| ROWTS | |
| RP | |
| RSHIFT | |
### S
- SCHEMALESS
- SCORES
- SELECT
- SEMI
- SERVER_STATUS
- SERVER_VERSION
- SESSION
- SET
- SHOW
- SINGLE_STABLE
- SLIDING
- SLIMIT
- SMA
- SMALLINT
- SNODE
- SNODES
- SOFFSET
- SPLIT
- STABLE
- STABLES
- START
- STATE
- STATE_WINDOW
- STATEMENT
- STORAGE
- STREAM
- STREAMS
- STRICT
- STRING
- SUBSCRIPTIONS
- SYNCDB
- SYSINFO
|关键字|说明|
|----------------------|-|
| S3_CHUNKPAGES | |
| S3_COMPACT | |
| S3_KEEPLOCAL | |
| SCHEMALESS | |
| SCORES | |
| SELECT | |
| SEMI | |
| SERVER_STATUS | |
| SERVER_VERSION | |
| SESSION | |
| SET | |
| SHOW | |
| SINGLE_STABLE | |
| SLASH | |
| SLIDING | |
| SLIMIT | |
| SMA | |
| SMALLINT | |
| SMIGRATE | |
| SNODE | |
| SNODES | |
| SOFFSET | |
| SPLIT | |
| STABLE | |
| STABLES | |
| STAR | |
| START | |
| STATE | |
| STATE_WINDOW | |
| STATEMENT | |
| STORAGE | |
| STREAM | |
| STREAMS | |
| STRICT | |
| STRING | |
| STT_TRIGGER | |
| SUBSCRIBE | |
| SUBSCRIPTIONS | |
| SUBSTR | |
| SUBSTRING | |
| SUBTABLE | |
| SYSINFO | |
| SYSTEM | |
### T
- TABLE
- TABLES
- TAG
- TAGS
- TBNAME
- TIMES
- TIMESTAMP
- TIMEZONE
- TINYINT
- TO
- TODAY
- TOPIC
- TOPICS
- TRANSACTION
- TRANSACTIONS
- TRIGGER
- TRIM
- TSERIES
- TTL
|关键字|说明|
|----------------------|-|
| TABLE | |
| TABLE_PREFIX | |
| TABLE_SUFFIX | |
| TABLES | |
| TAG | |
| TAGS | |
| TBNAME | |
| THEN | |
| TIMES | |
| TIMESTAMP | |
| TIMEZONE | |
| TINYINT | |
| TO | |
| TODAY | |
| TOPIC | |
| TOPICS | |
| TRAILING | |
| TRANSACTION | |
| TRANSACTIONS | |
| TRIGGER | |
| TRIM | |
| TSDB_PAGESIZE | |
| TSERIES | |
| TSMA | |
| TSMAS | |
| TTL | |
### U
- UNION
- UNSIGNED
- UPDATE
- USE
- USER
- USERS
- USING
|关键字|说明|
|----------------------|-|
| UNION | |
| UNSAFE | |
| UNSIGNED | |
| UNTREATED | |
| UPDATE | |
| USE | |
| USER | |
| USERS | |
| USING | |
### V
|关键字|说明|
|----------------------|-|
| VALUE | |
| VALUE_F | |
| VALUES | |
| VARBINARY | |
| VARCHAR | |
| VARIABLE | |
| VARIABLES | |
| VERBOSE | |
| VGROUP | |
| VGROUPS | |
| VIEW | |
| VIEWS | |
| VNODE | |
| VNODES | |
- VALUE
- VALUES
- VARCHAR
- VARIABLE
- VARIABLES
- VERBOSE
- VGROUP
- VGROUPS
- VIEW
- VNODES
### W
- WAL
- WAL_FSYNC_PERIOD
- WAL_LEVEL
- WAL_RETENTION_PERIOD
- WAL_RETENTION_SIZE
- WATERMARK
- WHERE
- WINDOW_CLOSE
- WITH
- WRITE
|关键字|说明|
|----------------------|-|
| WAL | |
| WAL_FSYNC_PERIOD | |
| WAL_LEVEL | |
| WAL_RETENTION_PERIOD | |
| WAL_RETENTION_SIZE | |
| WAL_ROLL_PERIOD | |
| WAL_SEGMENT_SIZE | |
| WATERMARK | |
| WDURATION | |
| WEND | |
| WHEN | |
| WHERE | |
| WINDOW | |
| WINDOW_CLOSE | |
| WINDOW_OFFSET | |
| WITH | |
| WRITE | |
| WSTART | |
### \_

4
docs/zh/28-releases/01-tdengine.md
View File

@ -24,6 +24,10 @@ TDengine 3.x 各版本安装包下载链接如下：
import Release from "/components/ReleaseV3";
## 3.3.4.3
<Release type="tdengine" version="3.3.4.3" />
## 3.3.3.0
<Release type="tdengine" version="3.3.3.0" />

63
docs/zh/28-releases/03-notes/3.3.4.3.md Normal file
View File

@ -0,0 +1,63 @@
---
title: 3.3.4.3 版本说明
sidebar_label: 3.3.4.3
description: 3.3.4.3 版本说明
---
### 新特性
1. 新功能：流计算的 TWA 函数支持时间驱动的结果推送模式
1. 新功能：流计算的 Interp 函数支持时间驱动的结果推送模式
1. 新功能：支持微软对象存储
### 优化
1. 优化：提升并发大查询时节点之间互相拉数据的效率
1. 优化：支持使用 AVX2 和 AVX512 对 double 、timestamp 和 bigint 类型进行解码优化
1. 优化：调整 case when 语句的结果类型判断方法
1. 优化：顺序执行 compact 和 split vgroup操作时的日志错误提示
1. 优化：提升查询 “select ... from ... where ts in (...)” 的数据扫描速度
1. 优化：增加了流计算的兼容性保证机制，避免后续函数变更产生新的兼容性问题，之前版本的流计算必须重建
1. 优化：提升 taosX 在交叉写入场景下的数据同步性能
1. 优化：支持关闭整数/浮点数类型的编码
1. 优化：多副本流计算中必须使用 snode
1. 优化：客户端生成唯一 ID 标识每一个查询任务，避免重复 ID 导致的内存损坏
1. 优化：加快数据库的创建时间
1. 优化：修改 s3MigrateEnabled 默认值为0
1. 优化：支持在审计数据库中记录删除操作
1. 优化：支持在指定的 dnode 中创建数据库 [企业版]
1. 优化：调整删除超级表数据列时的报错信息
### 修复
1. 修复：last_row 查询性能在 3.3.3.0 中大幅下降的问题
1. 修复：WAL 条目不完整时 taosd 无法启动的问题
1. 修复: partition by 常量时查询结果错误的问题
1. 修复：标量函数包含 _wstart 且填充方式为 prev 时计算结果错误
1. 修复：Windows 平台下的时区设置问题
1. 修复：空数据库进行 compact 操作时，事务无法结束【企业版】
1. 修复：事务冲突的逻辑错误
1. 修复：管理节点某些错误会导致事务无法停止
1. 修复：管理节点某些错误会导致事务无法停止
1. 修复：dnode 数据清空后 taosc 重试错误的问题
1. 修复：Data Compact 被异常终止后，中间文件未被清理
1. 修复：新增列后，Kafka 连接器的 earliest 模式消费不到新列数据
1. 修复：interp 函数在 fill(prev) 时行为不正确
1. 修复：TSMA 在高频元数据操作时异常停止的问题
1. 修复：show create stable 语句执行结果的标签显示错误
1. 修复：Percentile 函数在大数据量查询时会崩溃。
1. 修复：partition by 和 having 联合使用时的语法错误问题
1. 修复：interp 在 partition by tbname,c1 时 tbname 为空的问题
1. 修复：通过 stmt 写入非法布尔数值时 taosd 可能 crash
1. 修复：库符号 version 与使用相同符号的库冲突的问题
1. 修复：在 windows 平台下 JDBC 驱动的句柄数持续升高问题
1. 修复：3.3.3.1 升级至 3.3.4.0 偶现的启动失败问题
1. 修复：Windows 平台重复增删表的内存泄漏
1. 修复：无法限制并发拉起 checkpoint 数量导致流计算消耗资源过多
1. 修复：并发查询时的 too many session 问题
1. 修复：Windows 平台下 taos shell 在慢查询场景中崩溃的问题
1. 修复：当打开 dnode日志时，加密数据库无法恢复的问题
1. 修复：由于 mnode 同步超时，进而导致 taosd 无法启动的问题
1. 修复：由于在快照同步过程中整理文件组数据的速度过慢，从而导致 Vnode（虚拟节点）无法恢复的问题
1. 修复：通过行协议向字符串类型的字段中写入带转义符的数据时，taosd 会崩溃
1. 修复：Error Code 逻辑处理错误导致的元数据文件损坏
1. 修复：查询语句中包含多个 “not” 条件语句嵌套时，未设置标量模式导致查询错误
1. 修复：vnode 统计信息上报超时导致的 dnode offline 问题
1. 修复：在不支持 avx 指令集的服务器上，taosd 启动失败问题
1. 修复：taosX 数据迁移容错处理 0x09xx 错误码

1
docs/zh/28-releases/03-notes/index.md
View File

@ -3,5 +3,6 @@ title: 版本说明
sidebar_label: 版本说明
description: 各版本版本说明
---
[3.3.4.3](./3.3.4.3)
[3.3.3.0](./3.3.3.0)
[3.3.2.0](./3.3.2.0)

29
include/libs/function/tudf.h
View File

@ -61,6 +61,35 @@ extern "C" {
} \
} while (0)
#define TAOS_UDF_CHECK_PTR_RCODE(...) \
do { \
const void *ptrs[] = {__VA_ARGS__}; \
for (int i = 0; i < sizeof(ptrs) / sizeof(ptrs[0]); ++i) { \
if (ptrs[i] == NULL) { \
fnError("udfd %dth parameter invalid, NULL PTR.line:%d", i, __LINE__); \
return TSDB_CODE_INVALID_PARA; \
} \
} \
} while (0)
#define TAOS_UDF_CHECK_PTR_RVOID(...) \
do { \
const void *ptrs[] = {__VA_ARGS__}; \
for (int i = 0; i < sizeof(ptrs) / sizeof(ptrs[0]); ++i) { \
if (ptrs[i] == NULL) { \
fnError("udfd %dth parameter invalid, NULL PTR.line:%d", i, __LINE__); \
return; \
} \
} \
} while (0)
#define TAOS_UDF_CHECK_CONDITION(o, code) \
do { \
if ((o) == false) { \
fnError("Condition not met.line:%d", __LINE__); \
return code; \
} \
} while (0)
// low level APIs
/**

8
include/os/os.h
View File

@ -137,6 +137,14 @@ extern threadlocal bool tsEnableRandErr;
terrno = _code; \
}
#define OS_PARAM_CHECK(_o) \
do { \
if ((_o) == NULL) { \
terrno = TSDB_CODE_INVALID_PARA; \
return terrno; \
} \
} while (0)
#ifdef __cplusplus
}
#endif

1
include/os/osSemaphore.h
View File

@ -104,6 +104,7 @@ int64_t taosGetPthreadId(TdThread thread);
void taosResetPthread(TdThread *thread);
bool taosComparePthread(TdThread first, TdThread second);
int32_t taosGetPId();
int32_t taosGetPIdByName(const char* name, int32_t* pPId);
int32_t taosGetAppName(char *name, int32_t *len);
#ifdef __cplusplus

2
include/os/osSystem.h
View File

@ -48,8 +48,6 @@ void taosCloseCmd(TdCmdPtr *ppCmd);
void *taosLoadDll(const char *filename);
void *taosLoadSym(void *handle, char *name);
void taosCloseDll(void *handle);
int32_t taosSetConsoleEcho(bool on);

2
include/util/tjson.h
View File

@ -25,7 +25,7 @@ extern "C" {
#define tjsonGetNumberValue(pJson, pName, val, code) \
do { \
uint64_t _tmp = 0; \
int64_t _tmp = 0; \
code = tjsonGetBigIntValue(pJson, pName, &_tmp); \
val = _tmp; \
} while (0)

22
include/util/tutil.h
View File

@ -120,6 +120,18 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
}
}
/*
* LIKELY and UNLIKELY macros for branch predication hints. Use them judiciously
* only in very hot code paths. Misuse or abuse can lead to performance degradation.
*/
#if __GNUC__ >= 3
#define LIKELY(x) __builtin_expect((x) != 0, 1)
#define UNLIKELY(x) __builtin_expect((x) != 0, 0)
#else
#define LIKELY(x) ((x) != 0)
#define UNLIKELY(x) ((x) != 0)
#endif
#define TAOS_CHECK_ERRNO(CODE) \
do { \
terrno = (CODE); \
@ -131,7 +143,7 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
#define TSDB_CHECK_CODE(CODE, LINO, LABEL) \
do { \
if (TSDB_CODE_SUCCESS != (CODE)) { \
if (UNLIKELY(TSDB_CODE_SUCCESS != (CODE))) { \
LINO = __LINE__; \
goto LABEL; \
} \
@ -139,15 +151,17 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
#define QUERY_CHECK_CODE TSDB_CHECK_CODE
#define QUERY_CHECK_CONDITION(condition, CODE, LINO, LABEL, ERRNO) \
if (!condition) { \
#define TSDB_CHECK_CONDITION(condition, CODE, LINO, LABEL, ERRNO) \
if (UNLIKELY(!(condition))) { \
(CODE) = (ERRNO); \
(LINO) = __LINE__; \
goto LABEL; \
}
#define QUERY_CHECK_CONDITION TSDB_CHECK_CONDITION
#define TSDB_CHECK_NULL(ptr, CODE, LINO, LABEL, ERRNO) \
if ((ptr) == NULL) { \
if (UNLIKELY((ptr) == NULL)) { \
(CODE) = (ERRNO); \
(LINO) = __LINE__; \
goto LABEL; \

16
source/common/src/rsync.c
View File

@ -119,11 +119,21 @@ static int32_t execCommand(char* command) {
}
void stopRsync() {
int32_t code =
int32_t pid = 0;
int32_t code = 0;
char buf[128] = {0};
#ifdef WINDOWS
system("taskkill /f /im rsync.exe");
code = system("taskkill /f /im rsync.exe");
#else
system("pkill rsync");
code = taosGetPIdByName("rsync", &pid);
if (code == 0) {
int32_t ret = tsnprintf(buf, tListLen(buf), "kill -9 %d", pid);
if (ret > 0) {
uInfo("kill rsync program pid:%d", pid);
code = system(buf);
}
}
#endif
if (code != 0) {

37
source/common/src/tglobal.c
View File

@ -59,7 +59,6 @@ int32_t tsNumOfRpcSessions = 30000;
int32_t tsShareConnLimit = 10;
int32_t tsReadTimeout = 900;
int32_t tsTimeToGetAvailableConn = 500000;
int32_t tsKeepAliveIdle = 60;
int32_t tsNumOfCommitThreads = 2;
int32_t tsNumOfTaskQueueThreads = 16;
@ -522,7 +521,7 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input
int32_t taosAddClientLogCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddDir(pCfg, "configDir", configDir, CFG_SCOPE_BOTH, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddDir(pCfg, "scriptDir", configDir, CFG_SCOPE_BOTH, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddDir(pCfg, "scriptDir", configDir, CFG_SCOPE_CLIENT, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddDir(pCfg, "logDir", tsLogDir, CFG_SCOPE_BOTH, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(
@ -530,13 +529,14 @@ int32_t taosAddClientLogCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "asyncLog", tsAsyncLog, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, CFG_SCOPE_BOTH, CFG_DYN_ENT_BOTH));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "simDebugFlag", simDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "simDebugFlag", simDebugFlag, 0, 255, CFG_SCOPE_CLIENT, CFG_DYN_BOTH));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqClientDebugFlag", tqClientDebugFlag, 0, 255, CFG_SCOPE_CLIENT, CFG_DYN_SERVER));
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
@ -549,7 +549,6 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqClientDebugFlag", tqClientDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
@ -590,17 +589,18 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(
cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "keepColumnName", tsKeepColumnName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "smlChildTableName", tsSmlChildTableName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "smlAutoChildTableNameDelimiter", tsSmlAutoChildTableNameDelimiter,
CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "smlTagName", tsSmlTagName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "smlTsDefaultName", tsSmlTsDefaultName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "smlDot2Underline", tsSmlDot2Underline, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, CFG_SCOPE_CLIENT, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, CFG_SCOPE_CLIENT,
CFG_DYN_CLIENT) != 0);
TAOS_CHECK_RETURN(
cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT));
cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_SERVER, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX,
@ -630,15 +630,12 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(
cfgAddInt32(pCfg, "timeToGetAvailableConn", tsTimeToGetAvailableConn, 20, 1000000, CFG_SCOPE_BOTH, CFG_DYN_NONE));
tsKeepAliveIdle = TRANGE(tsKeepAliveIdle, 1, 72000);
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "keepAliveIdle", tsKeepAliveIdle, 1, 7200000, CFG_SCOPE_BOTH, CFG_DYN_NONE));
tsNumOfTaskQueueThreads = tsNumOfCores * 2;
tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 16);
TAOS_CHECK_RETURN(
cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "multiResultFunctionStarReturnTags", tsMultiResultFunctionStarReturnTags,
CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
@ -727,8 +724,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddString(pCfg, "encryptScope", tsEncryptScope, CFG_SCOPE_SERVER, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, CFG_SCOPE_SERVER, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));
@ -746,7 +742,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE));
@ -782,12 +778,12 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "auditInterval", tsAuditInterval, 500, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH, CFG_DYN_ENT_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_BOTH, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_SERVER, CFG_DYN_BOTH));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "rsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_BOTH, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "rsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "snodeAddress", tsSnodeAddress, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
@ -1293,9 +1289,6 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "timeToGetAvailableConn");
tsTimeToGetAvailableConn = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "keepAliveIdle");
tsKeepAliveIdle = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "experimental");
tsExperimental = pItem->bval;
@ -2030,7 +2023,6 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) {
{"cacheLazyLoadThreshold", &tsCacheLazyLoadThreshold},
{"checkpointInterval", &tsStreamCheckpointInterval},
{"keepAliveIdle", &tsKeepAliveIdle},
{"logKeepDays", &tsLogKeepDays},
{"maxStreamBackendCache", &tsMaxStreamBackendCache},
{"mqRebalanceInterval", &tsMqRebalanceInterval},
@ -2288,7 +2280,6 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) {
{"crashReporting", &tsEnableCrashReport},
{"enableQueryHb", &tsEnableQueryHb},
{"keepColumnName", &tsKeepColumnName},
{"keepAliveIdle", &tsKeepAliveIdle},
{"logKeepDays", &tsLogKeepDays},
{"maxInsertBatchRows", &tsMaxInsertBatchRows},
{"maxRetryWaitTime", &tsMaxRetryWaitTime},

2
source/dnode/vnode/inc/vnode.h
View File

@ -172,7 +172,7 @@ void tsdbReleaseDataBlock2(STsdbReader *pReader);
int32_t tsdbRetrieveDataBlock2(STsdbReader *pReader, SSDataBlock **pBlock, SArray *pIdList);
int32_t tsdbReaderReset2(STsdbReader *pReader, SQueryTableDataCond *pCond);
int32_t tsdbGetFileBlocksDistInfo2(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader *pHandle);
int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader *pHandle, uint32_t *rows);
void *tsdbGetIdx2(SMeta *pMeta);
void *tsdbGetIvtIdx2(SMeta *pMeta);
uint64_t tsdbGetReaderMaxVersion2(STsdbReader *pReader);

326
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -25,82 +25,109 @@
#define HASTYPE(_type, _t) (((_type) & (_t)) == (_t))
static int32_t setFirstLastResColToNull(SColumnInfoData* pCol, int32_t row) {
char* buf = taosMemoryCalloc(1, pCol->info.bytes);
if (buf == NULL) {
return terrno;
}
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
char* buf = NULL;
SFirstLastRes* pRes = NULL;
SFirstLastRes* pRes = (SFirstLastRes*)((char*)buf + VARSTR_HEADER_SIZE);
TSDB_CHECK_NULL(pCol, code, lino, _end, TSDB_CODE_INVALID_PARA);
buf = taosMemoryCalloc(1, pCol->info.bytes);
TSDB_CHECK_NULL(buf, code, lino, _end, terrno);
pRes = (SFirstLastRes*)((char*)buf + VARSTR_HEADER_SIZE);
pRes->bytes = 0;
pRes->hasResult = true;
pRes->isNull = true;
varDataSetLen(buf, pCol->info.bytes - VARSTR_HEADER_SIZE);
int32_t code = colDataSetVal(pCol, row, buf, false);
taosMemoryFree(buf);
code = colDataSetVal(pCol, row, buf, false);
TSDB_CHECK_CODE(code, lino, _end);
_end:
if (code != TSDB_CODE_SUCCESS) {
tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (buf != NULL) {
taosMemoryFreeClear(buf);
}
return code;
}
static int32_t saveOneRowForLastRaw(SLastCol* pColVal, SCacheRowsReader* pReader, const int32_t slotId,
SColumnInfoData* pColInfoData, int32_t numOfRows) {
SColVal* pVal = &pColVal->colVal;
int32_t code = 0;
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SColVal* pVal = NULL;
TSDB_CHECK_NULL(pColVal, code, lino, _end, TSDB_CODE_INVALID_PARA);
TSDB_CHECK_NULL(pColInfoData, code, lino, _end, TSDB_CODE_INVALID_PARA);
pVal = &pColVal->colVal;
// allNullRow = false;
if (IS_VAR_DATA_TYPE(pColVal->colVal.value.type)) {
if (!COL_VAL_IS_VALUE(&pColVal->colVal)) {
colDataSetNULL(pColInfoData, numOfRows);
} else {
TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
varDataSetLen(pReader->transferBuf[slotId], pVal->value.nData);
memcpy(varDataVal(pReader->transferBuf[slotId]), pVal->value.pData, pVal->value.nData);
code = colDataSetVal(pColInfoData, numOfRows, pReader->transferBuf[slotId], false);
TSDB_CHECK_CODE(code, lino, _end);
}
} else {
code = colDataSetVal(pColInfoData, numOfRows, (const char*)&pVal->value.val, !COL_VAL_IS_VALUE(pVal));
TSDB_CHECK_CODE(code, lino, _end);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* pReader, const int32_t* slotIds,
const int32_t* dstSlotIds, void** pRes, const char* idStr) {
int32_t numOfRows = pBlock->info.rows;
int32_t code = 0;
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t numOfRows = 0;
SArray* funcTypeBlockArray = NULL;
TSDB_CHECK_NULL(pBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
if (pReader->numOfCols > 0) {
TSDB_CHECK_NULL(slotIds, code, lino, _end, TSDB_CODE_INVALID_PARA);
TSDB_CHECK_NULL(dstSlotIds, code, lino, _end, TSDB_CODE_INVALID_PARA);
TSDB_CHECK_NULL(pRes, code, lino, _end, TSDB_CODE_INVALID_PARA);
}
numOfRows = pBlock->info.rows;
if (HASTYPE(pReader->type, CACHESCAN_RETRIEVE_LAST)) {
uint64_t ts = TSKEY_MIN;
SFirstLastRes* p = NULL;
col_id_t colId = -1;
SArray* funcTypeBlockArray = taosArrayInit(pReader->numOfCols, sizeof(int32_t));
if (funcTypeBlockArray == NULL) {
return terrno;
}
funcTypeBlockArray = taosArrayInit(pReader->numOfCols, sizeof(int32_t));
TSDB_CHECK_NULL(funcTypeBlockArray, code, lino, _end, terrno);
for (int32_t i = 0; i < pReader->numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]);
if (pColInfoData == NULL) {
return TSDB_CODE_INVALID_PARA;
}
TSDB_CHECK_NULL(pColInfoData, code, lino, _end, TSDB_CODE_INVALID_PARA);
int32_t funcType = FUNCTION_TYPE_CACHE_LAST;
if (pReader->pFuncTypeList != NULL && taosArrayGetSize(pReader->pFuncTypeList) > i) {
void* pVal = taosArrayGet(pReader->pFuncTypeList, i);
if (pVal == NULL) {
return TSDB_CODE_INVALID_PARA;
}
TSDB_CHECK_NULL(pVal, code, lino, _end, TSDB_CODE_INVALID_PARA);
funcType = *(int32_t*) pVal;
funcType = *(int32_t*)pVal;
pVal = taosArrayGet(pReader->pFuncTypeList, i);
if (pVal == NULL) {
return TSDB_CODE_INVALID_PARA;
}
TSDB_CHECK_NULL(pVal, code, lino, _end, TSDB_CODE_INVALID_PARA);
void* px = taosArrayInsert(funcTypeBlockArray, dstSlotIds[i], pVal);
if (px == NULL) {
return terrno;
}
TSDB_CHECK_NULL(px, code, lino, _end, terrno);
}
if (slotIds[i] == -1) {
@ -110,24 +137,18 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
}
code = setFirstLastResColToNull(pColInfoData, numOfRows);
if (code) {
return code;
}
TSDB_CHECK_CODE(code, lino, _end);
continue;
}
int32_t slotId = slotIds[i];
SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i);
if (pColVal == NULL) {
return TSDB_CODE_INVALID_PARA;
}
TSDB_CHECK_NULL(pColVal, code, lino, _end, TSDB_CODE_INVALID_PARA);
colId = pColVal->colVal.cid;
if (FUNCTION_TYPE_CACHE_LAST_ROW == funcType) {
code = saveOneRowForLastRaw(pColVal, pReader, slotId, pColInfoData, numOfRows);
if (code) {
return code;
}
TSDB_CHECK_CODE(code, lino, _end);
continue;
}
@ -154,22 +175,16 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
p->hasResult = true;
varDataSetLen(pRes[i], pColInfoData->info.bytes - VARSTR_HEADER_SIZE);
code = colDataSetVal(pColInfoData, numOfRows, (const char*)pRes[i], false);
if (code) {
return code;
}
TSDB_CHECK_CODE(code, lino, _end);
}
for (int32_t idx = 0; idx < taosArrayGetSize(pBlock->pDataBlock); ++idx) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, idx);
if (pCol == NULL) {
return TSDB_CODE_INVALID_PARA;
}
TSDB_CHECK_NULL(pCol, code, lino, _end, TSDB_CODE_INVALID_PARA);
if (idx < funcTypeBlockArray->size) {
void* pVal = taosArrayGet(funcTypeBlockArray, idx);
if (pVal == NULL) {
return TSDB_CODE_INVALID_PARA;
}
TSDB_CHECK_NULL(pVal, code, lino, _end, TSDB_CODE_INVALID_PARA);
int32_t funcType = *(int32_t*)pVal;
if (FUNCTION_TYPE_CACHE_LAST_ROW == funcType) {
@ -182,17 +197,13 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
colDataSetNULL(pCol, numOfRows);
} else {
code = colDataSetVal(pCol, numOfRows, (const char*)&ts, false);
if (code) {
return code;
}
TSDB_CHECK_CODE(code, lino, _end);
}
continue;
} else if (pReader->numOfCols == 1 && idx != dstSlotIds[0] && (pCol->info.colId == colId || colId == -1)) {
if (p && !p->isNull) {
code = colDataSetVal(pCol, numOfRows, p->buf, false);
if (code) {
return code;
}
TSDB_CHECK_CODE(code, lino, _end);
} else {
colDataSetNULL(pCol, numOfRows);
}
@ -201,13 +212,10 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
// pBlock->info.rows += allNullRow ? 0 : 1;
++pBlock->info.rows;
taosArrayDestroy(funcTypeBlockArray);
} else if (HASTYPE(pReader->type, CACHESCAN_RETRIEVE_LAST_ROW)) {
for (int32_t i = 0; i < pReader->numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]);
if (pColInfoData == NULL) {
return TSDB_CODE_INVALID_PARA;
}
TSDB_CHECK_NULL(pColInfoData, code, lino, _end, TSDB_CODE_INVALID_PARA);
int32_t slotId = slotIds[i];
if (slotId == -1) {
@ -216,47 +224,53 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
}
SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i);
if (pColVal == NULL) {
return TSDB_CODE_INVALID_PARA;
}
TSDB_CHECK_NULL(pColVal, code, lino, _end, TSDB_CODE_INVALID_PARA);
code = saveOneRowForLastRaw(pColVal, pReader, slotId, pColInfoData, numOfRows);
if (code) {
return code;
}
TSDB_CHECK_CODE(code, lino, _end);
}
// pBlock->info.rows += allNullRow ? 0 : 1;
++pBlock->info.rows;
} else {
tsdbError("invalid retrieve type:%d, %s", pReader->type, idStr);
return TSDB_CODE_INVALID_PARA;
code = TSDB_CODE_INVALID_PARA;
TSDB_CHECK_CODE(code, lino, _end);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (funcTypeBlockArray != NULL) {
taosArrayDestroy(funcTypeBlockArray);
}
return code;
}
static int32_t setTableSchema(SCacheRowsReader* p, uint64_t suid, const char* idstr) {
int32_t numOfTables = p->numOfTables;
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t numOfTables = 0;
TSDB_CHECK_NULL(p, code, lino, _end, TSDB_CODE_INVALID_PARA);
numOfTables = p->numOfTables;
if (suid != 0) {
code = metaGetTbTSchemaNotNull(p->pVnode->pMeta, suid, -1, 1, &p->pSchema);
if (TSDB_CODE_SUCCESS != code) {
tsdbWarn("stable:%" PRIu64 " has been dropped, failed to retrieve cached rows, %s", suid, idstr);
if(code == TSDB_CODE_NOT_FOUND) {
return TSDB_CODE_PAR_TABLE_NOT_EXIST;
} else {
return code;
if (code == TSDB_CODE_NOT_FOUND) {
code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
}
TSDB_CHECK_CODE(code, lino, _end);
}
} else {
for (int32_t i = 0; i < numOfTables; ++i) {
uint64_t uid = p->pTableList[i].uid;
code = metaGetTbTSchemaMaybeNull(p->pVnode->pMeta, uid, -1, 1, &p->pSchema);
if(code != TSDB_CODE_SUCCESS) {
return code;
}
TSDB_CHECK_CODE(code, lino, _end);
if (p->pSchema != NULL) {
break;
}
@ -267,33 +281,52 @@ static int32_t setTableSchema(SCacheRowsReader* p, uint64_t suid, const char* id
// all queried tables have been dropped already, return immediately.
if (p->pSchema == NULL) {
tsdbWarn("all queried tables has been dropped, try next group, %s", idstr);
return TSDB_CODE_PAR_TABLE_NOT_EXIST;
code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
TSDB_CHECK_CODE(code, lino, _end);
}
}
return TSDB_CODE_SUCCESS;
_end:
if (code != TSDB_CODE_SUCCESS) {
tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t tsdbReuseCacherowsReader(void* reader, void* pTableIdList, int32_t numOfTables) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SCacheRowsReader* pReader = (SCacheRowsReader*)reader;
TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
pReader->pTableList = pTableIdList;
pReader->numOfTables = numOfTables;
pReader->lastTs = INT64_MIN;
destroySttBlockReader(pReader->pLDataIterArray, NULL);
pReader->pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
TSDB_CHECK_NULL(pReader->pLDataIterArray, code, lino, _end, terrno);
return (pReader->pLDataIterArray != NULL) ? TSDB_CODE_SUCCESS : terrno;
_end:
if (code != TSDB_CODE_SUCCESS) {
tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, int32_t numOfTables, int32_t numOfCols,
SArray* pCidList, int32_t* pSlotIds, uint64_t suid, void** pReader, const char* idstr,
SArray* pFuncTypeList, SColumnInfo* pPkCol, int32_t numOfPks) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SCacheRowsReader* p = NULL;
TSDB_CHECK_NULL(pVnode, code, lino, _end, TSDB_CODE_INVALID_PARA);
TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
*pReader = NULL;
SCacheRowsReader* p = taosMemoryCalloc(1, sizeof(SCacheRowsReader));
if (p == NULL) {
return terrno;
}
p = taosMemoryCalloc(1, sizeof(SCacheRowsReader));
TSDB_CHECK_NULL(p, code, lino, _end, terrno);
p->type = type;
p->pVnode = pVnode;
@ -307,12 +340,13 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList,
p->rowKey.numOfPKs = numOfPks;
if (numOfPks > 0) {
TSDB_CHECK_NULL(pPkCol, code, lino, _end, TSDB_CODE_INVALID_PARA);
p->rowKey.pks[0].type = pPkCol->type;
if (IS_VAR_DATA_TYPE(pPkCol->type)) {
p->rowKey.pks[0].pData = taosMemoryCalloc(1, pPkCol->bytes);
if (p->rowKey.pks[0].pData == NULL) {
taosMemoryFree(p);
return terrno;
taosMemoryFreeClear(p);
TSDB_CHECK_NULL(p->rowKey.pks[0].pData, code, lino, _end, terrno);
}
}
@ -321,48 +355,46 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList,
if (numOfTables == 0) {
*pReader = p;
return TSDB_CODE_SUCCESS;
p = NULL;
goto _end;
}
p->pTableList = pTableIdList;
p->numOfTables = numOfTables;
int32_t code = setTableSchema(p, suid, idstr);
if (code != TSDB_CODE_SUCCESS) {
tsdbCacherowsReaderClose(p);
return code;
}
code = setTableSchema(p, suid, idstr);
TSDB_CHECK_CODE(code, lino, _end);
p->transferBuf = taosMemoryCalloc(p->pSchema->numOfCols, POINTER_BYTES);
if (p->transferBuf == NULL) {
tsdbCacherowsReaderClose(p);
return terrno;
}
TSDB_CHECK_NULL(p->transferBuf, code, lino, _end, terrno);
for (int32_t i = 0; i < p->pSchema->numOfCols; ++i) {
if (IS_VAR_DATA_TYPE(p->pSchema->columns[i].type)) {
p->transferBuf[i] = taosMemoryMalloc(p->pSchema->columns[i].bytes);
if (p->transferBuf[i] == NULL) {
tsdbCacherowsReaderClose(p);
return terrno;
}
TSDB_CHECK_NULL(p->transferBuf[i], code, lino, _end, terrno);
}
}
if (idstr != NULL) {
p->idstr = taosStrdup(idstr);
if (idstr != NULL && p->idstr == NULL) {
tsdbCacherowsReaderClose(p);
return terrno;
TSDB_CHECK_NULL(p->idstr, code, lino, _end, terrno);
}
code = taosThreadMutexInit(&p->readerMutex, NULL);
if (code) {
tsdbCacherowsReaderClose(p);
return code;
}
TSDB_CHECK_CODE(code, lino, _end);
p->lastTs = INT64_MIN;
*pReader = p;
p = NULL;
_end:
if (code != TSDB_CODE_SUCCESS) {
tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
*pReader = NULL;
}
if (p != NULL) {
tsdbCacherowsReaderClose(p);
}
return code;
}
@ -393,6 +425,7 @@ void tsdbCacherowsReaderClose(void* pReader) {
if (p->pLDataIterArray) {
destroySttBlockReader(p->pLDataIterArray, NULL);
p->pLDataIterArray = NULL;
}
if (p->pFileReader) {
@ -401,7 +434,7 @@ void tsdbCacherowsReaderClose(void* pReader) {
}
taosMemoryFree((void*)p->idstr);
(void) taosThreadMutexDestroy(&p->readerMutex);
(void)taosThreadMutexDestroy(&p->readerMutex);
if (p->pTableMap) {
void* pe = NULL;
@ -443,39 +476,32 @@ static int32_t tsdbCacheQueryReseek(void* pQHandle) {
int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32_t* slotIds, const int32_t* dstSlotIds,
SArray* pTableUidList, bool* pGotAll) {
if (pReader == NULL || pResBlock == NULL) {
return TSDB_CODE_INVALID_PARA;
}
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
bool hasRes = false;
SArray* pRow = NULL;
void** pRes = NULL;
SCacheRowsReader* pr = pReader;
SCacheRowsReader* pr = NULL;
int32_t pkBufLen = 0;
TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
TSDB_CHECK_NULL(pResBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
pr = pReader;
pr->pReadSnap = NULL;
pRow = taosArrayInit(TARRAY_SIZE(pr->pCidList), sizeof(SLastCol));
if (pRow == NULL) {
code = terrno;
goto _end;
}
TSDB_CHECK_NULL(pRow, code, lino, _end, terrno);
pRes = taosMemoryCalloc(pr->numOfCols, POINTER_BYTES);
if (pRes == NULL) {
code = terrno;
goto _end;
}
TSDB_CHECK_NULL(pRes, code, lino, _end, terrno);
pkBufLen = (pr->rowKey.numOfPKs > 0) ? pr->pkColumn.bytes : 0;
for (int32_t j = 0; j < pr->numOfCols; ++j) {
int32_t bytes = (slotIds[j] == -1) ? 1 : pr->pSchema->columns[slotIds[j]].bytes;
pRes[j] = taosMemoryCalloc(1, sizeof(SFirstLastRes) + bytes + pkBufLen + VARSTR_HEADER_SIZE);
if (pRes[j] == NULL) {
code = terrno;
goto _end;
}
TSDB_CHECK_NULL(pRes[j], code, lino, _end, terrno);
SFirstLastRes* p = (SFirstLastRes*)varDataVal(pRes[j]);
p->ts = INT64_MIN;
@ -483,9 +509,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
(void)taosThreadMutexLock(&pr->readerMutex);
code = tsdbTakeReadSnap2((STsdbReader*)pr, tsdbCacheQueryReseek, &pr->pReadSnap, pr->idstr);
if (code != TSDB_CODE_SUCCESS) {
goto _end;
}
TSDB_CHECK_CODE(code, lino, _end);
int8_t ltype = (pr->type & CACHESCAN_RETRIEVE_LAST) >> 3;
@ -494,20 +518,14 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
// retrieve the only one last row of all tables in the uid list.
if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_SINGLE)) {
SArray* pLastCols = taosArrayInit(pr->numOfCols, sizeof(SLastCol));
if (pLastCols == NULL) {
code = terrno;
goto _end;
}
TSDB_CHECK_NULL(pLastCols, code, lino, _end, terrno);
for (int32_t i = 0; i < pr->numOfCols; ++i) {
int32_t slotId = slotIds[i];
if (slotId == -1) {
SLastCol p = {.rowKey.ts = INT64_MIN, .colVal.value.type = TSDB_DATA_TYPE_BOOL, .colVal.flag = CV_FLAG_NULL};
void* px = taosArrayPush(pLastCols, &p);
if (px == NULL) {
code = terrno;
goto _end;
}
TSDB_CHECK_NULL(px, code, lino, _end, terrno);
continue;
}
struct STColumn* pCol = &pr->pSchema->columns[slotId];
@ -518,29 +536,19 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
for (int32_t j = 0; j < pr->rowKey.numOfPKs; j++) {
p.rowKey.pks[j].type = pr->pkColumn.type;
if (IS_VAR_DATA_TYPE(pr->pkColumn.type)) {
p.rowKey.pks[j].pData = taosMemoryCalloc(1, pr->pkColumn.bytes);
if (p.rowKey.pks[j].pData == NULL) {
code = terrno;
goto _end;
}
TSDB_CHECK_NULL(p.rowKey.pks[j].pData, code, lino, _end, terrno);
}
}
}
if (IS_VAR_DATA_TYPE(pCol->type)) {
p.colVal.value.pData = taosMemoryCalloc(pCol->bytes, sizeof(char));
if (p.colVal.value.pData == NULL) {
code = terrno;
goto _end;
}
TSDB_CHECK_NULL(p.colVal.value.pData, code, lino, _end, terrno);
}
void* px = taosArrayPush(pLastCols, &p);
if (px == NULL) {
code = terrno;
goto _end;
}
TSDB_CHECK_NULL(px, code, lino, _end, terrno);
}
int64_t st = taosGetTimestampUs();
@ -549,11 +557,10 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
tb_uid_t uid = pTableList[i].uid;
code = tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype);
if (code == -1) {// fix the invalid return code
if (code == -1) { // fix the invalid return code
code = 0;
} else if (code != 0) {
goto _end;
}
TSDB_CHECK_CODE(code, lino, _end);
if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) {
taosArrayClearEx(pRow, tsdbCacheFreeSLastColItem);
@ -600,10 +607,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
if (k == 0) {
if (TARRAY_SIZE(pTableUidList) == 0) {
void* px = taosArrayPush(pTableUidList, &uid);
if (px == NULL) {
code = terrno;
goto _end;
}
TSDB_CHECK_NULL(px, code, lino, _end, terrno);
} else {
taosArraySet(pTableUidList, 0, &uid);
}
@ -654,9 +658,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
if (hasRes) {
code = saveOneRow(pLastCols, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr);
if (code) {
goto _end;
}
TSDB_CHECK_CODE(code, lino, _end);
}
taosArrayDestroyEx(pLastCols, tsdbCacheFreeSLastColItem);
@ -666,11 +668,10 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
tb_uid_t uid = pTableList[i].uid;
if ((code = tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype)) != 0) {
if (code == -1) {// fix the invalid return code
if (code == -1) { // fix the invalid return code
code = 0;
} else if (code != 0) {
goto _end;
}
TSDB_CHECK_CODE(code, lino, _end);
}
if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) {
@ -679,17 +680,12 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
}
code = saveOneRow(pRow, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr);
if (code) {
goto _end;
}
TSDB_CHECK_CODE(code, lino, _end);
taosArrayClearEx(pRow, tsdbCacheFreeSLastColItem);
void* px = taosArrayPush(pTableUidList, &uid);
if (px == NULL) {
code = terrno;
goto _end;
}
TSDB_CHECK_NULL(px, code, lino, _end, terrno);
++pr->tableIndex;
if (pResBlock->info.rows >= pResBlock->info.capacity) {
@ -702,6 +698,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
}
} else {
code = TSDB_CODE_INVALID_PARA;
TSDB_CHECK_CODE(code, lino, _end);
}
_end:
@ -723,5 +720,8 @@ _end:
taosMemoryFree(pRes);
taosArrayDestroy(pRow);
if (code != TSDB_CODE_SUCCESS) {
tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}

3805
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

File diff suppressed because it is too large Load Diff

1006
source/dnode/vnode/src/tsdb/tsdbReadUtil.c
View File

File diff suppressed because it is too large Load Diff

10
source/dnode/vnode/src/tsdb/tsdbReadUtil.h
View File

@ -30,13 +30,13 @@ extern "C" {
do { \
(_w)->skey = INT64_MAX; \
(_w)->ekey = INT64_MIN; \
} while (0);
} while (0)
#define INIT_KEYRANGE(_k) \
do { \
(_k)->skey.ts = INT64_MAX; \
(_k)->ekey.ts = INT64_MIN; \
} while (0);
} while (0)
#define tRowGetKeyEx(_pRow, _pKey) \
{ \
@ -72,7 +72,6 @@ typedef struct STsdbReaderInfo {
} STsdbReaderInfo;
typedef struct SBlockInfoBuf {
int32_t currentIndex;
SArray* pData;
int32_t numPerBucket;
int32_t numOfTables;
@ -241,7 +240,6 @@ typedef struct SDataBlockIter {
int32_t index;
SArray* blockList; // SArray<SFileDataBlockInfo>
int32_t order;
SDataBlk block; // current SDataBlk data
} SDataBlockIter;
typedef struct SFileBlockDumpInfo {
@ -321,7 +319,7 @@ int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, c
int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap,
STsdbReader* pReader);
void clearBlockScanInfo(STableBlockScanInfo* p);
void destroyAllBlockScanInfo(SSHashObj* pTableMap);
void destroyAllBlockScanInfo(SSHashObj** pTableMap);
void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step);
void cleanupInfoForNextFileset(SSHashObj* pTableMap);
int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables);
@ -335,7 +333,7 @@ void clearBrinBlockIter(SBrinRecordIter* pIter);
// initialize block iterator API
int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList);
bool blockIteratorNext(SDataBlockIter* pBlockIter, const char* idStr);
bool blockIteratorNext(SDataBlockIter* pBlockIter);
// load tomb data API (stt/mem only for one table each, tomb data from data files are load for all tables at one time)
int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver);

3
source/libs/command/src/command.c
View File

@ -56,6 +56,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe
int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, dataEncodeBufSize, numOfCols);
if(len < 0) {
taosMemoryFree(*pRsp);
*pRsp = NULL;
return terrno;
}
SET_PAYLOAD_LEN((*pRsp)->data, len, len);
@ -957,8 +958,8 @@ static int32_t buildLocalVariablesResultDataBlock(SSDataBlock** pOutput) {
_exit:
if (terrno != TSDB_CODE_SUCCESS) {
taosMemoryFree(pBlock);
taosArrayDestroy(pBlock->pDataBlock);
taosMemoryFree(pBlock);
}
return terrno;
}

4
source/libs/command/src/explain.c
View File

@ -2112,6 +2112,7 @@ static int32_t qExplainGenerateRsp(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp)
}
int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t groupId, SRetrieveTableRsp **pRsp) {
if(!pCtx || !pRspMsg || !pRsp) return TSDB_CODE_INVALID_PARA;
SExplainResNode *node = NULL;
int32_t code = 0;
bool groupDone = false;
@ -2176,6 +2177,7 @@ _exit:
}
int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) {
if (!pDag || !pRsp) return TSDB_CODE_INVALID_PARA;
int32_t code = 0;
SExplainCtx *pCtx = NULL;
@ -2188,6 +2190,7 @@ _return:
}
int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int64_t startTs) {
if(!pDag || !pCtx) return TSDB_CODE_INVALID_PARA;
QRY_ERR_RET(qExplainPrepareCtx(pDag, pCtx));
(*pCtx)->reqStartTs = startTs;
@ -2197,6 +2200,7 @@ int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int64_t startTs)
}
int32_t qExecExplainEnd(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) {
if(!pCtx || !pRsp) return TSDB_CODE_INVALID_PARA;
int32_t code = 0;
pCtx->jobDoneTs = taosGetTimestampUs();

14
source/libs/executor/src/sysscanoperator.c
View File

@ -685,11 +685,11 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
pAPI->metaFn.pauseTableMetaCursor(pInfo->pCur);
break;
}
} else {
}
// if pInfo->pRes->info.rows == 0, also need to add the meta to pDataBlock
code = sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, pDataBlock, tableName, schemaRow, typeName);
QUERY_CHECK_CODE(code, lino, _end);
}
}
if (numOfRows > 0) {
pAPI->metaFn.pauseTableMetaCursor(pInfo->pCur);
@ -761,7 +761,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
SMetaReader smrChildTable = {0};
pAPI->metaReaderFn.initReader(&smrChildTable, pInfo->readHandle.vnode, META_READER_LOCK, &pAPI->metaFn);
int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrChildTable, condTableName);
code = pAPI->metaReaderFn.getTableEntryByName(&smrChildTable, condTableName);
if (code != TSDB_CODE_SUCCESS) {
// terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly
pAPI->metaReaderFn.clearReader(&smrChildTable);
@ -847,7 +847,8 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
pAPI->metaReaderFn.clearReader(&smrSuperTable);
break;
}
} else {
}
// if pInfo->pRes->info.rows == 0, also need to add this meta into datablock.
code = sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows,
dataBlock);
if (code != TSDB_CODE_SUCCESS) {
@ -859,7 +860,6 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
dataBlock = NULL;
T_LONG_JMP(pTaskInfo->env, terrno);
}
}
pAPI->metaReaderFn.clearReader(&smrSuperTable);
}
@ -2792,7 +2792,9 @@ static int32_t doBlockInfoScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes
code = pAPI->tsdReader.tsdReaderGetDataBlockDistInfo(pBlockScanInfo->pHandle, &blockDistInfo);
QUERY_CHECK_CODE(code, lino, _end);
blockDistInfo.numOfInmemRows = (int32_t)pAPI->tsdReader.tsdReaderGetNumOfInMemRows(pBlockScanInfo->pHandle);
blockDistInfo.numOfInmemRows = 0;
code = pAPI->tsdReader.tsdReaderGetNumOfInMemRows(pBlockScanInfo->pHandle, &blockDistInfo.numOfInmemRows);
QUERY_CHECK_CODE(code, lino, _end);
SSDataBlock* pBlock = pBlockScanInfo->pResBlock;

2
source/libs/function/inc/tpercentile.h
View File

@ -26,7 +26,7 @@ extern "C" {
struct tMemBucket;
int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval, bool hasWindowOrGroup,
struct tMemBucket **pBucket);
struct tMemBucket **pBucket, int32_t numOfElements);
void tMemBucketDestroy(struct tMemBucket **pBucket);

2
source/libs/function/src/builtinsimpl.c
View File

@ -1805,7 +1805,7 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) {
pResInfo->complete = true;
return TSDB_CODE_SUCCESS;
} else {
code = tMemBucketCreate(pCol->info.bytes, type, pInfo->minval, pInfo->maxval, pCtx->hasWindowOrGroup, &pInfo->pMemBucket);
code = tMemBucketCreate(pCol->info.bytes, type, pInfo->minval, pInfo->maxval, pCtx->hasWindowOrGroup, &pInfo->pMemBucket, pInfo->numOfElems);
if (TSDB_CODE_SUCCESS != code) {
return code;
}

10
source/libs/function/src/tpercentile.c
View File

@ -269,18 +269,16 @@ static void resetSlotInfo(tMemBucket *pBucket) {
}
int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval, bool hasWindowOrGroup,
tMemBucket **pBucket) {
tMemBucket **pBucket, int32_t numOfElements) {
*pBucket = (tMemBucket *)taosMemoryCalloc(1, sizeof(tMemBucket));
if (*pBucket == NULL) {
return terrno;
}
if (hasWindowOrGroup) {
// With window or group by, we need to shrink page size and reduce page num to save memory.
(*pBucket)->numOfSlots = DEFAULT_NUM_OF_SLOT / 8 ; // 128 bucket
// With window or group by, we need to shrink page size to save memory.
(*pBucket)->bufPageSize = 4096; // 4k per page
} else {
(*pBucket)->numOfSlots = DEFAULT_NUM_OF_SLOT;
(*pBucket)->bufPageSize = 16384 * 4; // 16k per page
}
@ -302,6 +300,8 @@ int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, dou
}
(*pBucket)->elemPerPage = ((*pBucket)->bufPageSize - sizeof(SFilePage)) / (*pBucket)->bytes;
(*pBucket)->numOfSlots = TMIN((int16_t)(numOfElements / ((*pBucket)->elemPerPage * 6)) + 1, DEFAULT_NUM_OF_SLOT);
(*pBucket)->comparFn = getKeyComparFunc((*pBucket)->type, TSDB_ORDER_ASC);
(*pBucket)->hashFunc = getHashFunc((*pBucket)->type);
@ -587,7 +587,7 @@ int32_t getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction
// try next round
tMemBucket *tmpBucket = NULL;
int32_t code = tMemBucketCreate(pMemBucket->bytes, pMemBucket->type, pSlot->range.dMinVal, pSlot->range.dMaxVal,
false, &tmpBucket);
false, &tmpBucket, pSlot->info.size);
if (TSDB_CODE_SUCCESS != code) {
tMemBucketDestroy(&tmpBucket);
return code;

37
source/libs/function/src/tudf.c
View File

@ -62,6 +62,7 @@ static void udfUdfdStopAsyncCb(uv_async_t *async);
static void udfWatchUdfd(void *args);
void udfUdfdExit(uv_process_t *process, int64_t exitStatus, int32_t termSignal) {
TAOS_UDF_CHECK_PTR_RVOID(process);
fnInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal);
SUdfdData *pData = process->data;
if(pData == NULL) {
@ -81,6 +82,7 @@ void udfUdfdExit(uv_process_t *process, int64_t exitStatus, int32_t termSignal)
static int32_t udfSpawnUdfd(SUdfdData *pData) {
fnInfo("start to init udfd");
TAOS_UDF_CHECK_PTR_RCODE(pData);
int32_t err = 0;
uv_process_options_t options = {0};
@ -271,17 +273,20 @@ _OVER:
}
static void udfUdfdCloseWalkCb(uv_handle_t *handle, void *arg) {
TAOS_UDF_CHECK_PTR_RVOID(handle);
if (!uv_is_closing(handle)) {
uv_close(handle, NULL);
}
}
static void udfUdfdStopAsyncCb(uv_async_t *async) {
TAOS_UDF_CHECK_PTR_RVOID(async);
SUdfdData *pData = async->data;
uv_stop(&pData->loop);
}
static void udfWatchUdfd(void *args) {
TAOS_UDF_CHECK_PTR_RVOID(args);
SUdfdData *pData = args;
TAOS_UV_CHECK_ERRNO(uv_loop_init(&pData->loop));
TAOS_UV_CHECK_ERRNO(uv_async_init(&pData->loop, &pData->stopAsync, udfUdfdStopAsyncCb));
@ -877,6 +882,7 @@ void *decodeUdfResponse(const void *buf, SUdfResponse *rsp) {
}
void freeUdfColumnData(SUdfColumnData *data, SUdfColumnMeta *meta) {
TAOS_UDF_CHECK_PTR_RVOID(data, meta);
if (IS_VAR_DATA_TYPE(meta->type)) {
taosMemoryFree(data->varLenCol.varOffsets);
data->varLenCol.varOffsets = NULL;
@ -890,9 +896,13 @@ void freeUdfColumnData(SUdfColumnData *data, SUdfColumnMeta *meta) {
}
}
void freeUdfColumn(SUdfColumn *col) { freeUdfColumnData(&col->colData, &col->colMeta); }
void freeUdfColumn(SUdfColumn *col) {
TAOS_UDF_CHECK_PTR_RVOID(col);
freeUdfColumnData(&col->colData, &col->colMeta);
}
void freeUdfDataDataBlock(SUdfDataBlock *block) {
TAOS_UDF_CHECK_PTR_RVOID(block);
for (int32_t i = 0; i < block->numOfCols; ++i) {
freeUdfColumn(block->udfCols[i]);
taosMemoryFree(block->udfCols[i]);
@ -903,11 +913,17 @@ void freeUdfDataDataBlock(SUdfDataBlock *block) {
}
void freeUdfInterBuf(SUdfInterBuf *buf) {
TAOS_UDF_CHECK_PTR_RVOID(buf);
taosMemoryFree(buf->buf);
buf->buf = NULL;
}
int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock) {
TAOS_UDF_CHECK_PTR_RCODE(block, udfBlock);
int32_t code = blockDataCheck(block);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
udfBlock->numOfRows = block->info.rows;
udfBlock->numOfCols = taosArrayGetSize(block->pDataBlock);
udfBlock->udfCols = taosMemoryCalloc(taosArrayGetSize(block->pDataBlock), sizeof(SUdfColumn *));
@ -977,6 +993,7 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo
}
int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) {
TAOS_UDF_CHECK_PTR_RCODE(udfCol, block);
int32_t code = 0, lino = 0;
SUdfColumnMeta *meta = &udfCol->colMeta;
@ -1002,6 +1019,8 @@ int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) {
}
block->info.rows = udfCol->colData.numOfRows;
code = blockDataCheck(block);
TAOS_CHECK_GOTO(code, &lino, _exit);
_exit:
if (code != 0) {
fnError("failed to convert udf column to data block, code:%d, line:%d", code, lino);
@ -1010,6 +1029,7 @@ _exit:
}
int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SSDataBlock *output) {
TAOS_UDF_CHECK_PTR_RCODE(input, output);
int32_t code = 0, lino = 0;
int32_t numOfRows = 0;
for (int32_t i = 0; i < numOfCols; ++i) {
@ -1057,6 +1077,7 @@ _exit:
}
int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output) {
TAOS_UDF_CHECK_PTR_RCODE(input, output);
if (taosArrayGetSize(input->pDataBlock) != 1) {
fnError("scalar function only support one column");
return 0;
@ -1135,6 +1156,7 @@ int32_t compareUdfcFuncSub(const void *elem1, const void *elem2) {
}
int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) {
TAOS_UDF_CHECK_PTR_RCODE(udfName, pHandle);
int32_t code = 0, line = 0;
uv_mutex_lock(&gUdfcProxy.udfStubsMutex);
SUdfcFuncStub key = {0};
@ -1193,6 +1215,7 @@ _exit:
}
void releaseUdfFuncHandle(char *udfName, UdfcFuncHandle handle) {
TAOS_UDF_CHECK_PTR_RVOID(udfName);
uv_mutex_lock(&gUdfcProxy.udfStubsMutex);
SUdfcFuncStub key = {0};
tstrncpy(key.udfName, udfName, TSDB_FUNC_NAME_LEN);
@ -1295,6 +1318,7 @@ int32_t cleanUpUdfs() {
}
int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output) {
TAOS_UDF_CHECK_PTR_RCODE(udfName, input, output);
UdfcFuncHandle handle = NULL;
int32_t code = acquireUdfFuncHandle(udfName, &handle);
if (code != 0) {
@ -1324,6 +1348,10 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols,
}
bool udfAggGetEnv(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv) {
if (pFunc == NULL || pEnv == NULL) {
fnError("udfAggGetEnv: invalid input lint: %d", __LINE__);
return false;
}
if (fmIsScalarFunc(pFunc->funcId)) {
return false;
}
@ -1332,6 +1360,7 @@ bool udfAggGetEnv(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv) {
}
int32_t udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo) {
TAOS_UDF_CHECK_PTR_RCODE(pCtx, pResultCellInfo);
if (pResultCellInfo->initialized) {
return TSDB_CODE_SUCCESS;
}
@ -1373,6 +1402,7 @@ int32_t udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pRes
}
int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
TAOS_UDF_CHECK_PTR_RCODE(pCtx);
int32_t udfCode = 0;
UdfcFuncHandle handle = 0;
if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) {
@ -1444,6 +1474,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
}
int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock) {
TAOS_UDF_CHECK_PTR_RCODE(pCtx, pBlock);
int32_t udfCode = 0;
UdfcFuncHandle handle = 0;
if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) {
@ -1538,7 +1569,7 @@ int32_t udfcGetUdfTaskResultFromUvTask(SClientUdfTask *task, SClientUvTaskNode *
}
// TODO: the call buffer is setup and freed by udf invocation
taosMemoryFree(uvTask->rspBuf.base);
taosMemoryFreeClear(uvTask->rspBuf.base);
} else {
code = uvTask->errCode;
if (code != 0) {
@ -1763,7 +1794,7 @@ int32_t udfcInitializeUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvT
}
if (uv_sem_init(&uvTask->taskSem, 0) != 0) {
if (uvTaskType == UV_TASK_REQ_RSP) {
taosMemoryFree(uvTask->reqBuf.base);
taosMemoryFreeClear(uvTask->reqBuf.base);
}
fnError("udfc create uv task, init semaphore failed.");
return TSDB_CODE_UDF_UV_EXEC_FAILURE;

104
source/libs/function/src/udfd.c
View File

@ -55,6 +55,7 @@ int32_t udfdCPluginOpen(SScriptUdfEnvItem *items, int numItems) { return 0; }
int32_t udfdCPluginClose() { return 0; }
int32_t udfdCPluginUdfInitLoadInitDestoryFuncs(SUdfCPluginCtx *udfCtx, const char *udfName) {
TAOS_UDF_CHECK_PTR_RCODE(udfCtx, udfName);
char initFuncName[TSDB_FUNC_NAME_LEN + 6] = {0};
char *initSuffix = "_init";
snprintf(initFuncName, sizeof(initFuncName), "%s%s", udfName, initSuffix);
@ -68,6 +69,7 @@ int32_t udfdCPluginUdfInitLoadInitDestoryFuncs(SUdfCPluginCtx *udfCtx, const cha
}
int32_t udfdCPluginUdfInitLoadAggFuncs(SUdfCPluginCtx *udfCtx, const char *udfName) {
TAOS_UDF_CHECK_PTR_RCODE(udfCtx, udfName);
char processFuncName[TSDB_FUNC_NAME_LEN] = {0};
snprintf(processFuncName, sizeof(processFuncName), "%s", udfName);
TAOS_CHECK_RETURN(uv_dlsym(&udfCtx->lib, processFuncName, (void **)(&udfCtx->aggProcFunc)));
@ -93,6 +95,7 @@ int32_t udfdCPluginUdfInitLoadAggFuncs(SUdfCPluginCtx *udfCtx, const char *udfNa
}
int32_t udfdCPluginUdfInit(SScriptUdfInfo *udf, void **pUdfCtx) {
TAOS_UDF_CHECK_PTR_RCODE(udf, pUdfCtx);
int32_t err = 0;
SUdfCPluginCtx *udfCtx = taosMemoryCalloc(1, sizeof(SUdfCPluginCtx));
if (NULL == udfCtx) {
@ -146,6 +149,7 @@ _exit:
}
int32_t udfdCPluginUdfDestroy(void *udfCtx) {
TAOS_UDF_CHECK_PTR_RCODE(udfCtx);
SUdfCPluginCtx *ctx = udfCtx;
int32_t code = 0;
if (ctx->destroyFunc) {
@ -157,6 +161,7 @@ int32_t udfdCPluginUdfDestroy(void *udfCtx) {
}
int32_t udfdCPluginUdfScalarProc(SUdfDataBlock *block, SUdfColumn *resultCol, void *udfCtx) {
TAOS_UDF_CHECK_PTR_RCODE(block, resultCol, udfCtx);
SUdfCPluginCtx *ctx = udfCtx;
if (ctx->scalarProcFunc) {
return ctx->scalarProcFunc(block, resultCol);
@ -167,6 +172,7 @@ int32_t udfdCPluginUdfScalarProc(SUdfDataBlock *block, SUdfColumn *resultCol, vo
}
int32_t udfdCPluginUdfAggStart(SUdfInterBuf *buf, void *udfCtx) {
TAOS_UDF_CHECK_PTR_RCODE(buf, udfCtx);
SUdfCPluginCtx *ctx = udfCtx;
if (ctx->aggStartFunc) {
return ctx->aggStartFunc(buf);
@ -178,6 +184,7 @@ int32_t udfdCPluginUdfAggStart(SUdfInterBuf *buf, void *udfCtx) {
}
int32_t udfdCPluginUdfAggProc(SUdfDataBlock *block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf, void *udfCtx) {
TAOS_UDF_CHECK_PTR_RCODE(block, interBuf, newInterBuf, udfCtx);
SUdfCPluginCtx *ctx = udfCtx;
if (ctx->aggProcFunc) {
return ctx->aggProcFunc(block, interBuf, newInterBuf);
@ -189,6 +196,7 @@ int32_t udfdCPluginUdfAggProc(SUdfDataBlock *block, SUdfInterBuf *interBuf, SUdf
int32_t udfdCPluginUdfAggMerge(SUdfInterBuf *inputBuf1, SUdfInterBuf *inputBuf2, SUdfInterBuf *outputBuf,
void *udfCtx) {
TAOS_UDF_CHECK_PTR_RCODE(inputBuf1, inputBuf2, outputBuf, udfCtx);
SUdfCPluginCtx *ctx = udfCtx;
if (ctx->aggMergeFunc) {
return ctx->aggMergeFunc(inputBuf1, inputBuf2, outputBuf);
@ -199,6 +207,7 @@ int32_t udfdCPluginUdfAggMerge(SUdfInterBuf *inputBuf1, SUdfInterBuf *inputBuf2,
}
int32_t udfdCPluginUdfAggFinish(SUdfInterBuf *buf, SUdfInterBuf *resultData, void *udfCtx) {
TAOS_UDF_CHECK_PTR_RCODE(buf, resultData, udfCtx);
SUdfCPluginCtx *ctx = udfCtx;
if (ctx->aggFinishFunc) {
return ctx->aggFinishFunc(buf, resultData);
@ -360,6 +369,7 @@ int32_t udfdNewUdf(SUdf **pUdf, const char *udfName);
void udfdGetFuncBodyPath(const SUdf *udf, char *path);
int32_t udfdInitializeCPlugin(SUdfScriptPlugin *plugin) {
TAOS_UDF_CHECK_PTR_RCODE(plugin);
plugin->scriptType = TSDB_FUNC_SCRIPT_BIN_LIB;
plugin->openFunc = udfdCPluginOpen;
plugin->closeFunc = udfdCPluginClose;
@ -378,6 +388,7 @@ int32_t udfdInitializeCPlugin(SUdfScriptPlugin *plugin) {
}
int32_t udfdLoadSharedLib(char *libPath, uv_lib_t *pLib, const char *funcName[], void **func[], int numOfFuncs) {
TAOS_UDF_CHECK_PTR_RCODE(libPath, pLib, funcName, func);
int err = uv_dlopen(libPath, pLib);
if (err != 0) {
fnError("can not load library %s. error: %s", libPath, uv_strerror(err));
@ -394,6 +405,7 @@ int32_t udfdLoadSharedLib(char *libPath, uv_lib_t *pLib, const char *funcName[],
}
int32_t udfdInitializePythonPlugin(SUdfScriptPlugin *plugin) {
TAOS_UDF_CHECK_PTR_RCODE(plugin);
plugin->scriptType = TSDB_FUNC_SCRIPT_PYTHON;
// todo: windows support
snprintf(plugin->libPath, PATH_MAX, "%s", "libtaospyudf.so");
@ -439,6 +451,7 @@ int32_t udfdInitializePythonPlugin(SUdfScriptPlugin *plugin) {
}
void udfdDeinitCPlugin(SUdfScriptPlugin *plugin) {
TAOS_UDF_CHECK_PTR_RVOID(plugin);
if (plugin->closeFunc) {
if (plugin->closeFunc() != 0) {
fnError("udf script c plugin close func failed.line:%d", __LINE__);
@ -457,8 +470,9 @@ void udfdDeinitCPlugin(SUdfScriptPlugin *plugin) {
}
void udfdDeinitPythonPlugin(SUdfScriptPlugin *plugin) {
TAOS_UDF_CHECK_PTR_RVOID(plugin);
if (plugin->closeFunc) {
if(plugin->closeFunc() != 0) {
if (plugin->closeFunc() != 0) {
fnError("udf script python plugin close func failed.line:%d", __LINE__);
}
}
@ -517,22 +531,29 @@ void udfdDeinitScriptPlugins() {
if (plugin != NULL) {
udfdDeinitPythonPlugin(plugin);
taosMemoryFree(plugin);
global.scriptPlugins[TSDB_FUNC_SCRIPT_PYTHON] = NULL;
}
plugin = global.scriptPlugins[TSDB_FUNC_SCRIPT_BIN_LIB];
if (plugin != NULL) {
udfdDeinitCPlugin(plugin);
taosMemoryFree(plugin);
global.scriptPlugins[TSDB_FUNC_SCRIPT_BIN_LIB] = NULL;
}
return;
}
void udfdProcessRequest(uv_work_t *req) {
TAOS_UDF_CHECK_PTR_RVOID(req);
SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data);
if (uvUdf == NULL) {
fnError("udf work is NULL");
return;
}
SUdfRequest request = {0};
if(decodeUdfRequest(uvUdf->input.base, &request) == NULL)
{
taosMemoryFree(uvUdf->input.base);
taosMemoryFreeClear(uvUdf->input.base);
fnError("udf request decode failed");
return;
}
@ -557,7 +578,7 @@ void udfdProcessRequest(uv_work_t *req) {
}
}
void convertUdf2UdfInfo(SUdf *udf, SScriptUdfInfo *udfInfo) {
static void convertUdf2UdfInfo(SUdf *udf, SScriptUdfInfo *udfInfo) {
udfInfo->bufSize = udf->bufSize;
if (udf->funcType == TSDB_FUNC_TYPE_AGGREGATE) {
udfInfo->funcType = UDF_FUNC_TYPE_AGG;
@ -573,7 +594,8 @@ void convertUdf2UdfInfo(SUdf *udf, SScriptUdfInfo *udfInfo) {
udfInfo->scriptType = udf->scriptType;
}
int32_t udfdInitUdf(char *udfName, SUdf *udf) {
static int32_t udfdInitUdf(char *udfName, SUdf *udf) {
TAOS_UDF_CHECK_PTR_RCODE(udfName, udf);
int32_t err = 0;
err = udfdFillUdfInfoFromMNode(global.clientRpc, udfName, udf);
if (err != 0) {
@ -611,6 +633,7 @@ int32_t udfdInitUdf(char *udfName, SUdf *udf) {
}
int32_t udfdNewUdf(SUdf **pUdf, const char *udfName) {
TAOS_UDF_CHECK_PTR_RCODE(pUdf, udfName);
SUdf *udfNew = taosMemoryCalloc(1, sizeof(SUdf));
if (NULL == udfNew) {
return terrno;
@ -654,6 +677,7 @@ void udfdFreeUdf(void *pData) {
}
int32_t udfdGetOrCreateUdf(SUdf **ppUdf, const char *udfName) {
TAOS_UDF_CHECK_PTR_RCODE(ppUdf, udfName);
uv_mutex_lock(&global.udfsMutex);
SUdf **pUdfHash = taosHashGet(global.udfsHash, udfName, strlen(udfName));
int64_t currTime = taosGetTimestampMs();
@ -693,6 +717,7 @@ int32_t udfdGetOrCreateUdf(SUdf **ppUdf, const char *udfName) {
}
void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
TAOS_UDF_CHECK_PTR_RVOID(uvUdf, request);
// TODO: tracable id from client. connect, setup, call, teardown
fnInfo("setup request. seq num: %" PRId64 ", udf name: %s", request->seqNum, request->setup.udfName);
@ -760,11 +785,55 @@ _send:
uvUdf->output = uv_buf_init(bufBegin, len);
taosMemoryFree(uvUdf->input.base);
taosMemoryFreeClear(uvUdf->input.base);
return;
}
static int32_t checkUDFScalaResult(SSDataBlock *block, SUdfColumn *output) {
if (tsSafetyCheckLevel == TSDB_SAFETY_CHECK_LEVELL_NEVER) {
return TSDB_CODE_SUCCESS;
}
if (output->colData.numOfRows != block->info.rows) {
fnError("udf scala result num of rows %d not equal to input rows %" PRId64, output->colData.numOfRows, block->info.rows);
return TSDB_CODE_UDF_FUNC_EXEC_FAILURE;
}
if (tsSafetyCheckLevel == TSDB_SAFETY_CHECK_LEVELL_BYROW) {
for (int32_t i = 0; i < output->colData.numOfRows; ++i) {
if (!udfColDataIsNull(output, i)) {
if (IS_VAR_DATA_TYPE(output->colMeta.type)) {
TAOS_UDF_CHECK_CONDITION(output->colData.varLenCol.payload != NULL, TSDB_CODE_UDF_FUNC_EXEC_FAILURE);
TAOS_UDF_CHECK_CONDITION(output->colData.varLenCol.varOffsets[i] >= 0 &&
output->colData.varLenCol.varOffsets[i] < output->colData.varLenCol.payloadLen,
TSDB_CODE_UDF_FUNC_EXEC_FAILURE);
} else {
TAOS_UDF_CHECK_CONDITION(
output->colMeta.bytes * output->colData.numOfRows <= output->colData.fixLenCol.dataLen,
TSDB_CODE_UDF_FUNC_EXEC_FAILURE);
break;
}
}
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t checkUDFAggResult(SSDataBlock *block, SUdfInterBuf *output) {
if (tsSafetyCheckLevel == TSDB_SAFETY_CHECK_LEVELL_NEVER) {
return TSDB_CODE_SUCCESS;
}
if (output->numOfResult != 1 && output->numOfResult != 0) {
fnError("udf agg result num of rows %d not equal to 1", output->numOfResult);
return TSDB_CODE_UDF_FUNC_EXEC_FAILURE;
}
TAOS_UDF_CHECK_CONDITION(output->buf != NULL, TSDB_CODE_UDF_FUNC_EXEC_FAILURE);
TAOS_UDF_CHECK_CONDITION(output->bufLen > 0, TSDB_CODE_UDF_FUNC_EXEC_FAILURE);
return TSDB_CODE_SUCCESS;
}
void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
TAOS_UDF_CHECK_PTR_RVOID(uvUdf, request);
SUdfCallRequest *call = &request->call;
fnDebug("call request. call type %d, handle: %" PRIx64 ", seq num %" PRId64, call->callType, call->udfHandle,
request->seqNum);
@ -787,6 +856,7 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
code = convertDataBlockToUdfDataBlock(&call->block, &input);
if (code == TSDB_CODE_SUCCESS) code = udf->scriptPlugin->udfScalarProcFunc(&input, &output, udf->scriptUdfCtx);
freeUdfDataDataBlock(&input);
if (code == TSDB_CODE_SUCCESS) code = checkUDFScalaResult(&call->block, &output);
if (code == TSDB_CODE_SUCCESS) code = convertUdfColumnToDataBlock(&output, &response.callRsp.resultData);
}
freeUdfColumn(&output);
@ -809,6 +879,7 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
if (outBuf.buf != NULL) {
code = udf->scriptPlugin->udfAggProcFunc(&input, &call->interBuf, &outBuf, udf->scriptUdfCtx);
freeUdfInterBuf(&call->interBuf);
if (code == TSDB_CODE_SUCCESS) code = checkUDFAggResult(&call->block, &outBuf);
subRsp->resultBuf = outBuf;
} else {
code = terrno;
@ -900,11 +971,12 @@ _exit:
break;
}
taosMemoryFree(uvUdf->input.base);
taosMemoryFreeClear(uvUdf->input.base);
return;
}
void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
TAOS_UDF_CHECK_PTR_RVOID(uvUdf, request);
SUdfTeardownRequest *teardown = &request->teardown;
fnInfo("teardown. seq number: %" PRId64 ", handle:%" PRIx64, request->seqNum, teardown->udfHandle);
SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(teardown->udfHandle);
@ -964,6 +1036,7 @@ _send:
}
void udfdGetFuncBodyPath(const SUdf *udf, char *path) {
TAOS_UDF_CHECK_PTR_RVOID(udf, path);
if (udf->scriptType == TSDB_FUNC_SCRIPT_BIN_LIB) {
#ifdef WINDOWS
snprintf(path, PATH_MAX, "%s%s_%d_%" PRIx64 ".dll", global.udfDataDir, udf->name, udf->version, udf->createdTime);
@ -987,6 +1060,7 @@ void udfdGetFuncBodyPath(const SUdf *udf, char *path) {
}
int32_t udfdSaveFuncBodyToFile(SFuncInfo *pFuncInfo, SUdf *udf) {
TAOS_UDF_CHECK_PTR_RCODE(pFuncInfo, udf);
if (!osDataSpaceAvailable()) {
terrno = TSDB_CODE_NO_DISKSPACE;
fnError("udfd create shared library failed since %s", terrstr());
@ -1022,6 +1096,7 @@ int32_t udfdSaveFuncBodyToFile(SFuncInfo *pFuncInfo, SUdf *udf) {
}
void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
TAOS_UDF_CHECK_PTR_RVOID(parent, pMsg);
SUdfdRpcSendRecvInfo *msgInfo = (SUdfdRpcSendRecvInfo *)pMsg->info.ahandle;
if (pEpSet) {
@ -1093,6 +1168,7 @@ _return:
}
int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf) {
TAOS_UDF_CHECK_PTR_RCODE(clientRpc, udfName, udf);
SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 1;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
@ -1233,6 +1309,7 @@ void udfdCloseClientRpc() {
}
void udfdOnWrite(uv_write_t *req, int status) {
TAOS_UDF_CHECK_PTR_RVOID(req);
SUvUdfWork *work = (SUvUdfWork *)req->data;
if (status < 0) {
fnError("udfd send response error, length: %zu code: %s", work->output.len, uv_err_name(status));
@ -1254,6 +1331,7 @@ void udfdOnWrite(uv_write_t *req, int status) {
}
void udfdSendResponse(uv_work_t *work, int status) {
TAOS_UDF_CHECK_PTR_RVOID(work);
SUvUdfWork *udfWork = (SUvUdfWork *)(work->data);
if (udfWork->conn != NULL) {
@ -1274,6 +1352,7 @@ void udfdSendResponse(uv_work_t *work, int status) {
}
void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) {
TAOS_UDF_CHECK_PTR_RVOID(handle, buf);
SUdfdUvConn *ctx = handle->data;
int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t);
if (ctx->inputCap == 0) {
@ -1307,6 +1386,10 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) {
}
bool isUdfdUvMsgComplete(SUdfdUvConn *pipe) {
if (pipe == NULL) {
fnError("udfd pipe is NULL, LINE:%d", __LINE__);
return false;
}
if (pipe->inputTotal == -1 && pipe->inputLen >= sizeof(int32_t)) {
pipe->inputTotal = *(int32_t *)(pipe->inputBuf);
}
@ -1318,6 +1401,7 @@ bool isUdfdUvMsgComplete(SUdfdUvConn *pipe) {
}
void udfdHandleRequest(SUdfdUvConn *conn) {
TAOS_UDF_CHECK_PTR_RVOID(conn);
char *inputBuf = conn->inputBuf;
int32_t inputLen = conn->inputLen;
@ -1350,6 +1434,7 @@ void udfdHandleRequest(SUdfdUvConn *conn) {
}
void udfdPipeCloseCb(uv_handle_t *pipe) {
TAOS_UDF_CHECK_PTR_RVOID(pipe);
SUdfdUvConn *conn = pipe->data;
SUvUdfWork *pWork = conn->pWorkList;
while (pWork != NULL) {
@ -1363,6 +1448,7 @@ void udfdPipeCloseCb(uv_handle_t *pipe) {
}
void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
TAOS_UDF_CHECK_PTR_RVOID(client, buf);
fnDebug("udfd read %zd bytes from client", nread);
if (nread == 0) return;
@ -1389,6 +1475,7 @@ void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
}
void udfdOnNewConnection(uv_stream_t *server, int status) {
TAOS_UDF_CHECK_PTR_RVOID(server);
if (status < 0) {
fnError("udfd new connection error. code: %s", uv_strerror(status));
return;
@ -1434,6 +1521,7 @@ _exit:
}
void udfdIntrSignalHandler(uv_signal_t *handle, int signum) {
TAOS_UDF_CHECK_PTR_RVOID(handle);
fnInfo("udfd signal received: %d\n", signum);
uv_fs_t req;
int32_t code = uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL);
@ -1482,6 +1570,7 @@ static int32_t udfdInitLog() {
}
void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf) {
TAOS_UDF_CHECK_PTR_RVOID(buf);
buf->base = taosMemoryMalloc(suggested_size);
if (buf->base == NULL) {
fnError("udfd ctrl pipe alloc buffer failed");
@ -1491,6 +1580,7 @@ void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *bu
}
void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf) {
TAOS_UDF_CHECK_PTR_RVOID(q, buf);
if (nread < 0) {
fnError("udfd ctrl pipe read error. %s", uv_err_name(nread));
taosMemoryFree(buf->base);
@ -1580,7 +1670,7 @@ static void udfdGlobalDataDeinit() {
taosHashCleanup(global.udfsHash);
uv_mutex_destroy(&global.udfsMutex);
uv_mutex_destroy(&global.scriptPluginsMutex);
taosMemoryFree(global.loop);
taosMemoryFreeClear(global.loop);
fnInfo("udfd global data deinit");
}

8
source/libs/qcom/inc/queryInt.h
View File

@ -23,7 +23,13 @@ extern "C" {
#define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS)
#define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS)
#define QUERY_PARAM_CHECK(_p) \
do { \
if ((_p) == NULL) { \
qError("function:%s, param invalid, line:%d", __FUNCTION__, __LINE__); \
return TSDB_CODE_TSC_INVALID_INPUT; \
} \
} while (0)
#ifdef __cplusplus
}

22
source/libs/qcom/src/queryUtil.c
View File

@ -59,6 +59,9 @@ const SSchema* tGetTbnameColumnSchema() {
}
static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen) {
if (!pSchema) {
return false;
}
int32_t rowLen = 0;
for (int32_t i = 0; i < numOfCols; ++i) {
@ -100,7 +103,7 @@ static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen
}
bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags) {
if (!VALIDNUMOFCOLS(numOfCols)) {
if (!pSchema || !VALIDNUMOFCOLS(numOfCols)) {
return false;
}
@ -127,6 +130,7 @@ bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTag
static STaskQueue taskQueue = {0};
static void processTaskQueue(SQueueInfo *pInfo, SSchedMsg *pSchedMsg) {
if(!pSchedMsg || !pSchedMsg->ahandle) return;
__async_exec_fn_t execFn = (__async_exec_fn_t)pSchedMsg->ahandle;
(void)execFn(pSchedMsg->thandle);
taosFreeQitem(pSchedMsg);
@ -206,6 +210,10 @@ void destroyAhandle(void *ahandle) {
int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo,
bool persistHandle, void* rpcCtx) {
QUERY_PARAM_CHECK(pTransporter);
QUERY_PARAM_CHECK(epSet);
QUERY_PARAM_CHECK(pInfo);
char* pMsg = rpcMallocCont(pInfo->msgInfo.len);
if (NULL == pMsg) {
qError("0x%" PRIx64 " msg:%s malloc failed", pInfo->requestId, TMSG_INFO(pInfo->msgType));
@ -236,6 +244,7 @@ int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransp
return asyncSendMsgToServerExt(pTransporter, epSet, pTransporterId, pInfo, false, NULL);
}
int32_t asyncFreeConnById(void* pTransporter, int64_t pid) {
QUERY_PARAM_CHECK(pTransporter);
return rpcFreeConnById(pTransporter, pid);
}
@ -314,6 +323,8 @@ void destroyQueryExecRes(SExecResult* pRes) {
}
// clang-format on
int32_t dataConverToStr(char* str, int64_t capacity, int type, void* buf, int32_t bufSize, int32_t* len) {
QUERY_PARAM_CHECK(str);
QUERY_PARAM_CHECK(buf);
int32_t n = 0;
switch (type) {
@ -420,6 +431,10 @@ int32_t dataConverToStr(char* str, int64_t capacity, int type, void* buf, int32_
}
void parseTagDatatoJson(void* p, char** jsonStr) {
if (!p || !jsonStr) {
qError("parseTagDatatoJson invalid input, line:%d", __LINE__);
return;
}
char* string = NULL;
SArray* pTagVals = NULL;
cJSON* json = NULL;
@ -520,6 +535,7 @@ end:
}
int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) {
QUERY_PARAM_CHECK(pDst);
if (NULL == pSrc) {
*pDst = NULL;
return TSDB_CODE_SUCCESS;
@ -553,6 +569,7 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) {
}
void getColumnTypeFromMeta(STableMeta* pMeta, char* pName, ETableColumnType* pType) {
if(!pMeta || !pName || !pType) return;
int32_t nums = pMeta->tableInfo.numOfTags + pMeta->tableInfo.numOfColumns;
for (int32_t i = 0; i < nums; ++i) {
if (0 == strcmp(pName, pMeta->schema[i].name)) {
@ -576,6 +593,7 @@ void freeVgInfo(SDBVgInfo* vgInfo) {
}
int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) {
QUERY_PARAM_CHECK(pDst);
if (NULL == pSrc) {
*pDst = NULL;
return TSDB_CODE_SUCCESS;
@ -617,6 +635,7 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) {
}
int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst) {
QUERY_PARAM_CHECK(pDst);
if (NULL == pSrc) {
*pDst = NULL;
return TSDB_CODE_SUCCESS;
@ -674,6 +693,7 @@ int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst) {
_exit:
tdDestroySVCreateTbReq(*pDst);
taosMemoryFree(*pDst);
*pDst = NULL;
return terrno;
}

96
source/libs/qcom/src/querymsg.c
View File

@ -29,6 +29,8 @@ int32_t (*queryBuildMsg[TDMT_MAX])(void *input, char **msg, int32_t msgSize, int
int32_t (*queryProcessMsgRsp[TDMT_MAX])(void *output, char *msg, int32_t msgSize) = {0};
int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) {
QUERY_PARAM_CHECK(pOut);
QUERY_PARAM_CHECK(usedbRsp);
memcpy(pOut->db, usedbRsp->db, TSDB_DB_FNAME_LEN);
pOut->dbId = usedbRsp->uid;
@ -71,10 +73,10 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) {
int32_t queryBuildTableMetaReqMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen,
void *(*mallcFp)(int64_t)) {
QUERY_PARAM_CHECK(input);
QUERY_PARAM_CHECK(msg);
QUERY_PARAM_CHECK(msgLen);
SBuildTableInput *pInput = input;
if (NULL == input || NULL == msg || NULL == msgLen) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
STableInfoReq infoReq = {0};
infoReq.option = pInput->option;
@ -101,10 +103,10 @@ int32_t queryBuildTableMetaReqMsg(void *input, char **msg, int32_t msgSize, int3
}
int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) {
QUERY_PARAM_CHECK(input);
QUERY_PARAM_CHECK(msg);
QUERY_PARAM_CHECK(msgLen);
SBuildUseDBInput *pInput = input;
if (NULL == pInput || NULL == msg || NULL == msgLen) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
SUseDbReq usedbReq = {0};
tstrncpy(usedbReq.db, pInput->db, TSDB_DB_FNAME_LEN);
@ -131,9 +133,8 @@ int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *ms
}
int32_t queryBuildQnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) {
if (NULL == msg || NULL == msgLen) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
QUERY_PARAM_CHECK(msg);
QUERY_PARAM_CHECK(msgLen);
SQnodeListReq qnodeListReq = {0};
qnodeListReq.rowNum = -1;
@ -155,9 +156,8 @@ int32_t queryBuildQnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t
}
int32_t queryBuildDnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) {
if (NULL == msg || NULL == msgLen) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
QUERY_PARAM_CHECK(msg);
QUERY_PARAM_CHECK(msgLen);
SDnodeListReq dnodeListReq = {0};
dnodeListReq.rowNum = -1;
@ -179,9 +179,8 @@ int32_t queryBuildDnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t
}
int32_t queryBuildGetSerVerMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) {
if (NULL == msg || NULL == msgLen) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
QUERY_PARAM_CHECK(msg);
QUERY_PARAM_CHECK(msgLen);
SServerVerReq req = {0};
@ -202,9 +201,9 @@ int32_t queryBuildGetSerVerMsg(void *input, char **msg, int32_t msgSize, int32_t
}
int32_t queryBuildGetDBCfgMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) {
if (NULL == msg || NULL == msgLen) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
QUERY_PARAM_CHECK(input);
QUERY_PARAM_CHECK(msg);
QUERY_PARAM_CHECK(msgLen);
SDbCfgReq dbCfgReq = {0};
tstrncpy(dbCfgReq.db, input, TSDB_DB_FNAME_LEN);
@ -226,9 +225,9 @@ int32_t queryBuildGetDBCfgMsg(void *input, char **msg, int32_t msgSize, int32_t
}
int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) {
if (NULL == msg || NULL == msgLen) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
QUERY_PARAM_CHECK(input);
QUERY_PARAM_CHECK(msg);
QUERY_PARAM_CHECK(msgLen);
SUserIndexReq indexReq = {0};
tstrncpy(indexReq.indexFName, input, TSDB_INDEX_FNAME_LEN);
@ -251,9 +250,9 @@ int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t
int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen,
void *(*mallcFp)(int64_t)) {
if (NULL == msg || NULL == msgLen) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
QUERY_PARAM_CHECK(input);
QUERY_PARAM_CHECK(msg);
QUERY_PARAM_CHECK(msgLen);
SRetrieveFuncReq funcReq = {0};
funcReq.numOfFuncs = 1;
@ -288,9 +287,9 @@ int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int3
}
int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) {
if (NULL == msg || NULL == msgLen) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
QUERY_PARAM_CHECK(input);
QUERY_PARAM_CHECK(msg);
QUERY_PARAM_CHECK(msgLen);
SGetUserAuthReq req = {0};
tstrncpy(req.user, input, TSDB_USER_LEN);
@ -311,9 +310,9 @@ int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32
}
int32_t queryBuildGetTbIndexMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) {
if (NULL == msg || NULL == msgLen) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
QUERY_PARAM_CHECK(input);
QUERY_PARAM_CHECK(msg);
QUERY_PARAM_CHECK(msgLen);
STableIndexReq indexReq = {0};
tstrncpy(indexReq.tbFName, input, TSDB_TABLE_FNAME_LEN);
@ -335,9 +334,9 @@ int32_t queryBuildGetTbIndexMsg(void *input, char **msg, int32_t msgSize, int32_
}
int32_t queryBuildGetTbCfgMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) {
if (NULL == msg || NULL == msgLen) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
QUERY_PARAM_CHECK(input);
QUERY_PARAM_CHECK(msg);
QUERY_PARAM_CHECK(msgLen);
SBuildTableInput *pInput = input;
STableCfgReq cfgReq = {0};
@ -362,9 +361,9 @@ int32_t queryBuildGetTbCfgMsg(void *input, char **msg, int32_t msgSize, int32_t
}
int32_t queryBuildGetViewMetaMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) {
if (NULL == msg || NULL == msgLen) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
QUERY_PARAM_CHECK(input);
QUERY_PARAM_CHECK(msg);
QUERY_PARAM_CHECK(msgLen);
SViewMetaReq req = {0};
tstrncpy(req.fullname, input, TSDB_VIEW_FNAME_LEN);
@ -387,9 +386,9 @@ int32_t queryBuildGetViewMetaMsg(void *input, char **msg, int32_t msgSize, int32
int32_t queryBuildGetTableTSMAMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen,
void *(*mallcFp)(int64_t)) {
if (NULL == msg || NULL == msgLen) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
QUERY_PARAM_CHECK(input);
QUERY_PARAM_CHECK(msg);
QUERY_PARAM_CHECK(msgLen);
STableTSMAInfoReq req = {0};
tstrncpy(req.name, input, TSDB_TABLE_FNAME_LEN);
@ -411,9 +410,9 @@ int32_t queryBuildGetTableTSMAMsg(void *input, char **msg, int32_t msgSize, int3
int32_t queryBuildGetTSMAMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen,
void *(*mallcFp)(int64_t)) {
if (NULL == msg || NULL == msgLen) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
QUERY_PARAM_CHECK(input);
QUERY_PARAM_CHECK(msg);
QUERY_PARAM_CHECK(msgLen);
STableTSMAInfoReq req = {0};
req.fetchingWithTsmaName = true;
@ -436,9 +435,9 @@ int32_t queryBuildGetTSMAMsg(void *input, char **msg, int32_t msgSize, int32_t *
}
int32_t queryBuildGetStreamProgressMsg(void* input, char** msg, int32_t msgSize, int32_t *msgLen, void*(*mallcFp)(int64_t)) {
if (!msg || !msgLen) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
QUERY_PARAM_CHECK(input);
QUERY_PARAM_CHECK(msg);
QUERY_PARAM_CHECK(msgLen);
int32_t len = tSerializeStreamProgressReq(NULL, 0, input);
void* pBuf = (*mallcFp)(len);
@ -504,6 +503,7 @@ PROCESS_USEDB_OVER:
}
static int32_t queryConvertTableMetaMsg(STableMetaRsp *pMetaMsg) {
QUERY_PARAM_CHECK(pMetaMsg);
if (pMetaMsg->numOfTags < 0 || pMetaMsg->numOfTags > TSDB_MAX_TAGS) {
qError("invalid numOfTags[%d] in table meta rsp msg", pMetaMsg->numOfTags);
return TSDB_CODE_TSC_INVALID_VALUE;
@ -539,6 +539,8 @@ static int32_t queryConvertTableMetaMsg(STableMetaRsp *pMetaMsg) {
}
int32_t queryCreateCTableMetaFromMsg(STableMetaRsp *msg, SCTableMeta *pMeta) {
QUERY_PARAM_CHECK(msg);
QUERY_PARAM_CHECK(pMeta);
pMeta->vgId = msg->vgId;
pMeta->tableType = msg->tableType;
pMeta->uid = msg->tuid;
@ -551,6 +553,8 @@ int32_t queryCreateCTableMetaFromMsg(STableMetaRsp *msg, SCTableMeta *pMeta) {
}
int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isStb, STableMeta **pMeta) {
QUERY_PARAM_CHECK(msg);
QUERY_PARAM_CHECK(pMeta);
int32_t total = msg->numOfColumns + msg->numOfTags;
int32_t metaSize = sizeof(STableMeta) + sizeof(SSchema) * total;
int32_t schemaExtSize = (useCompress(msg->tableType) && msg->pSchemaExt) ? sizeof(SSchemaExt) * msg->numOfColumns : 0;
@ -605,6 +609,8 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isStb, STableMeta *
}
int32_t queryCreateTableMetaExFromMsg(STableMetaRsp *msg, bool isStb, STableMeta **pMeta) {
QUERY_PARAM_CHECK(msg);
QUERY_PARAM_CHECK(pMeta);
int32_t total = msg->numOfColumns + msg->numOfTags;
int32_t metaSize = sizeof(STableMeta) + sizeof(SSchema) * total;
int32_t schemaExtSize = (useCompress(msg->tableType) && msg->pSchemaExt) ? sizeof(SSchemaExt) * msg->numOfColumns : 0;

1
source/libs/sync/inc/syncSnapshot.h
View File

@ -91,6 +91,7 @@ typedef struct SSyncSnapshotReceiver {
// update when begin
void *pWriter;
TdThreadMutex writerMutex;
SSnapshotParam snapshotParam;
SSnapshot snapshot;

25
source/libs/sync/src/syncSnapshot.c
View File

@ -429,6 +429,12 @@ int32_t snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId, SSyncSnapsh
pReceiver->startTime = 0;
pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN;
pReceiver->pWriter = NULL;
code = taosThreadMutexInit(&pReceiver->writerMutex, NULL);
if (code != 0) {
taosMemoryFree(pReceiver);
pReceiver = NULL;
TAOS_RETURN(code);
}
pReceiver->pSyncNode = pSyncNode;
pReceiver->fromId = fromId;
pReceiver->term = raftStoreGetTerm(pSyncNode);
@ -440,6 +446,10 @@ int32_t snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId, SSyncSnapsh
SSyncSnapBuffer *pRcvBuf = NULL;
code = syncSnapBufferCreate(&pRcvBuf);
if (pRcvBuf == NULL) {
int32_t ret = taosThreadMutexDestroy(&pReceiver->writerMutex);
if (ret != 0) {
sError("failed to destroy mutex since %s", tstrerror(ret));
}
taosMemoryFree(pReceiver);
pReceiver = NULL;
TAOS_RETURN(code);
@ -468,6 +478,7 @@ static int32_t snapshotReceiverClearInfoData(SSyncSnapshotReceiver *pReceiver) {
void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) {
if (pReceiver == NULL) return;
(void)taosThreadMutexLock(&pReceiver->writerMutex);
// close writer
if (pReceiver->pWriter != NULL) {
int32_t code = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter,
@ -478,6 +489,9 @@ void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) {
}
pReceiver->pWriter = NULL;
}
(void)taosThreadMutexUnlock(&pReceiver->writerMutex);
(void)taosThreadMutexDestroy(&pReceiver->writerMutex);
// free snap buf
if (pReceiver->pRcvBuf) {
@ -556,7 +570,8 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) {
int8_t stopped = !atomic_val_compare_exchange_8(&pReceiver->start, true, false);
if (stopped) return;
(void)taosThreadMutexLock(&pReceiver->pRcvBuf->mutex);
(void)taosThreadMutexLock(&pReceiver->writerMutex);
{
if (pReceiver->pWriter != NULL) {
int32_t code = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter,
@ -568,7 +583,11 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) {
} else {
sRInfo(pReceiver, "snapshot receiver stop, writer is null");
}
}
(void)taosThreadMutexUnlock(&pReceiver->writerMutex);
(void)taosThreadMutexLock(&pReceiver->pRcvBuf->mutex);
{
syncSnapBufferReset(pReceiver->pRcvBuf);
(void)snapshotReceiverClearInfoData(pReceiver);
@ -600,6 +619,8 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap
raftStoreSetTerm(pReceiver->pSyncNode, pReceiver->snapshot.lastApplyTerm);
}
(void)taosThreadMutexLock(&pReceiver->writerMutex);
if (pReceiver->pWriter != NULL) {
// stop writer, apply data
code = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, true,
&pReceiver->snapshot);
@ -609,6 +630,8 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap
}
pReceiver->pWriter = NULL;
sRInfo(pReceiver, "snapshot receiver write stopped");
}
(void)taosThreadMutexUnlock(&pReceiver->writerMutex);
// update progress
pReceiver->ack = SYNC_SNAPSHOT_SEQ_END;

122
source/libs/transport/src/transCli.c
View File

@ -254,6 +254,7 @@ static FORCE_INLINE void cliMayUpdateFqdnCache(SHashObj* cache, char* dst);
// process data read from server, add decompress etc later
// handle except about conn
#define REQS_ON_CONN(conn) (conn ? (transQueueSize(&conn->reqsToSend) + transQueueSize(&conn->reqsSentOut)) : 0)
static void doNotifyCb(SCliReq* pReq, SCliThrd* pThrd, int32_t code);
// handle req from app
static void cliHandleReq(SCliThrd* pThrd, SCliReq* pReq);
@ -289,7 +290,7 @@ int32_t cliMayGetStateByQid(SCliThrd* pThrd, SCliReq* pReq, SCliConn** pConn);
static SCliConn* getConnFromHeapCache(SHashObj* pConnHeapCache, char* key);
static int32_t addConnToHeapCache(SHashObj* pConnHeapCacahe, SCliConn* pConn);
static int32_t delConnFromHeapCache(SHashObj* pConnHeapCache, SCliConn* pConn);
static int32_t balanceConnHeapCache(SHashObj* pConnHeapCache, SCliConn* pConn);
static int8_t balanceConnHeapCache(SHashObj* pConnHeapCache, SCliConn* pConn, SCliConn** pNewConn);
// thread obj
static int32_t createThrdObj(void* trans, SCliThrd** pThrd);
@ -327,14 +328,18 @@ typedef struct {
int64_t lastConnFailTs;
} SHeap;
int32_t compareHeapNode(const HeapNode* a, const HeapNode* b);
int32_t transHeapInit(SHeap* heap, int32_t (*cmpFunc)(const HeapNode* a, const HeapNode* b));
void transHeapDestroy(SHeap* heap);
int32_t transHeapGet(SHeap* heap, SCliConn** p);
int32_t transHeapInsert(SHeap* heap, SCliConn* p);
int32_t transHeapDelete(SHeap* heap, SCliConn* p);
int32_t transHeapBalance(SHeap* heap, SCliConn* p);
int32_t transHeapUpdateFailTs(SHeap* heap, SCliConn* p);
static int32_t compareHeapNode(const HeapNode* a, const HeapNode* b);
static int32_t transHeapInit(SHeap* heap, int32_t (*cmpFunc)(const HeapNode* a, const HeapNode* b));
static void transHeapDestroy(SHeap* heap);
static int32_t transHeapGet(SHeap* heap, SCliConn** p);
static int32_t transHeapInsert(SHeap* heap, SCliConn* p);
static int32_t transHeapDelete(SHeap* heap, SCliConn* p);
static int32_t transHeapBalance(SHeap* heap, SCliConn* p);
static int32_t transHeapUpdateFailTs(SHeap* heap, SCliConn* p);
static int32_t transHeapMayBalance(SHeap* heap, SCliConn* p);
static FORCE_INLINE void logConnMissHit(SCliConn* pConn);
#define CLI_RELEASE_UV(loop) \
do { \
@ -494,15 +499,19 @@ int8_t cliMayRecycleConn(SCliConn* conn) {
if (transQueueSize(&conn->reqsToSend) == 0 && transQueueSize(&conn->reqsSentOut) == 0 &&
taosHashGetSize(conn->pQTable) == 0) {
cliResetConnTimer(conn);
conn->forceDelFromHeap = 1;
code = delConnFromHeapCache(pThrd->connHeapCache, conn);
if (code == TSDB_CODE_RPC_ASYNC_IN_PROCESS) {
tDebug("%s conn %p failed to remove conn from heap cache since %s", CONN_GET_INST_LABEL(conn), conn,
tstrerror(code));
TAOS_UNUSED(transHeapMayBalance(conn->heap, conn));
return 1;
} else {
if (code != 0) {
tDebug("%s conn %p failed to remove conn from heap cache since %s", CONN_GET_INST_LABEL(conn), conn,
tstrerror(code));
return 0;
}
}
addConnToPool(pThrd->pool, conn);
@ -510,31 +519,10 @@ int8_t cliMayRecycleConn(SCliConn* conn) {
} else if ((transQueueSize(&conn->reqsToSend) == 0) && (transQueueSize(&conn->reqsSentOut) == 0) &&
(taosHashGetSize(conn->pQTable) != 0)) {
tDebug("%s conn %p do balance directly", CONN_GET_INST_LABEL(conn), conn);
TAOS_UNUSED(transHeapBalance(conn->heap, conn));
TAOS_UNUSED(transHeapMayBalance(conn->heap, conn));
} else {
SCliConn* topConn = NULL;
if (conn->heap != NULL) {
code = transHeapGet(conn->heap, &topConn);
if (code != 0) {
tDebug("%s conn %p failed to get top conn since %s", CONN_GET_INST_LABEL(conn), conn, tstrerror(code));
return 0;
}
if (topConn == conn) {
return 0;
}
int32_t topReqs = transQueueSize(&topConn->reqsSentOut) + transQueueSize(&topConn->reqsToSend);
int32_t currReqs = transQueueSize(&conn->reqsSentOut) + transQueueSize(&conn->reqsToSend);
if (topReqs <= currReqs) {
tTrace("%s conn %p not balance conn heap since top conn has less req, topConnReqs:%d, currConnReqs:%d",
CONN_GET_INST_LABEL(conn), conn, topReqs, currReqs);
return 0;
} else {
tDebug("%s conn %p do balance conn heap since top conn has more reqs, topConnReqs:%d, currConnReqs:%d",
CONN_GET_INST_LABEL(conn), conn, topReqs, currReqs);
TAOS_UNUSED(transHeapBalance(conn->heap, conn));
}
}
tTrace("%s conn %p may do balance", CONN_GET_INST_LABEL(conn), conn);
TAOS_UNUSED(transHeapMayBalance(conn->heap, conn));
}
return 0;
}
@ -785,15 +773,8 @@ void cliConnCheckTimoutMsg(SCliConn* conn) {
if (transQueueSize(&conn->reqsSentOut) == 0) {
return;
}
code = cliConnRemoveTimeoutMsg(conn);
if (code != 0) {
tDebug("%s conn %p do remove timeout msg", CONN_GET_INST_LABEL(conn), conn);
if (!cliMayRecycleConn(conn)) {
TAOS_UNUSED(transHeapBalance(conn->heap, conn));
}
} else {
TAOS_UNUSED(cliConnRemoveTimeoutMsg(conn));
TAOS_UNUSED(cliMayRecycleConn(conn));
}
}
void cliConnTimeout__checkReq(uv_timer_t* handle) {
SCliConn* conn = handle->data;
@ -3804,6 +3785,8 @@ static FORCE_INLINE int8_t shouldSWitchToOtherConn(SCliConn* pConn, char* key) {
int32_t totalReqs = reqsNum + reqsSentOut;
if (totalReqs >= pInst->shareConnLimit) {
logConnMissHit(pConn);
if (pConn->list == NULL && pConn->dstAddr != NULL) {
pConn->list = taosHashGet((SHashObj*)pThrd->pool, pConn->dstAddr, strlen(pConn->dstAddr));
if (pConn->list != NULL) {
@ -3860,11 +3843,12 @@ static SCliConn* getConnFromHeapCache(SHashObj* pConnHeapCache, char* key) {
} else {
tTrace("conn %p get conn from heap cache for key:%s", pConn, key);
if (shouldSWitchToOtherConn(pConn, key)) {
code = balanceConnHeapCache(pConnHeapCache, pConn);
if (code != 0) {
tTrace("failed to balance conn heap cache for key:%s", key);
SCliConn* pNewConn = NULL;
code = balanceConnHeapCache(pConnHeapCache, pConn, &pNewConn);
if (code == 1) {
tTrace("conn %p start to handle reqs", pNewConn);
return pNewConn;
}
logConnMissHit(pConn);
return NULL;
}
}
@ -3916,15 +3900,19 @@ static int32_t delConnFromHeapCache(SHashObj* pConnHeapCache, SCliConn* pConn) {
return code;
}
static int32_t balanceConnHeapCache(SHashObj* pConnHeapCache, SCliConn* pConn) {
static int8_t balanceConnHeapCache(SHashObj* pConnHeapCache, SCliConn* pConn, SCliConn** pNewConn) {
SCliThrd* pThrd = pConn->hostThrd;
STrans* pInst = pThrd->pInst;
SCliConn* pTopConn = NULL;
if (pConn->heap != NULL && pConn->inHeap != 0) {
SHeap* heap = pConn->heap;
tTrace("conn %p'heap may should do balance, numOfConn:%d", pConn, (int)(heap->heap->nelts));
int64_t now = taosGetTimestampMs();
if (((now - heap->lastUpdateTs) / 1000) > 30) {
heap->lastUpdateTs = now;
tTrace("conn %p'heap do balance, numOfConn:%d", pConn, (int)(heap->heap->nelts));
return transHeapBalance(pConn->heap, pConn);
TAOS_UNUSED(transHeapBalance(pConn->heap, pConn));
if (transHeapGet(pConn->heap, &pTopConn) == 0 && pConn != pTopConn) {
int32_t curReqs = REQS_ON_CONN(pConn);
int32_t topReqs = REQS_ON_CONN(pTopConn);
if (curReqs > topReqs && topReqs < pInst->shareConnLimit) {
*pNewConn = pTopConn;
return 1;
}
}
}
return 0;
@ -3934,8 +3922,8 @@ int32_t compareHeapNode(const HeapNode* a, const HeapNode* b) {
SCliConn* args1 = container_of(a, SCliConn, node);
SCliConn* args2 = container_of(b, SCliConn, node);
int32_t totalReq1 = transQueueSize(&args1->reqsToSend) + transQueueSize(&args1->reqsSentOut);
int32_t totalReq2 = transQueueSize(&args2->reqsToSend) + transQueueSize(&args2->reqsSentOut);
int32_t totalReq1 = REQS_ON_CONN(args1);
int32_t totalReq2 = REQS_ON_CONN(args2);
if (totalReq1 > totalReq2) {
return 0;
}
@ -4016,6 +4004,30 @@ int32_t transHeapUpdateFailTs(SHeap* heap, SCliConn* p) {
heap->lastConnFailTs = taosGetTimestampMs();
return 0;
}
int32_t transHeapMayBalance(SHeap* heap, SCliConn* p) {
if (p->inHeap == 0 || heap == NULL || heap->heap == NULL) {
return 0;
}
SCliThrd* pThrd = p->hostThrd;
STrans* pInst = pThrd->pInst;
int32_t balanceLimit = pInst->shareConnLimit >= 4 ? pInst->shareConnLimit / 2 : 2;
SCliConn* topConn = NULL;
int32_t code = transHeapGet(heap, &topConn);
if (code != 0) {
return code;
}
if (topConn == p) return code;
int32_t reqsOnTop = REQS_ON_CONN(topConn);
int32_t reqsOnCur = REQS_ON_CONN(p);
if (reqsOnTop >= balanceLimit && reqsOnCur < balanceLimit) {
TAOS_UNUSED(transHeapBalance(heap, p));
}
return code;
}
int32_t transHeapBalance(SHeap* heap, SCliConn* p) {
if (p->inHeap == 0 || heap == NULL || heap->heap == NULL) {

3
source/libs/wal/src/walMeta.c
View File

@ -424,9 +424,6 @@ static void printFileSet(int32_t vgId, SArray* fileSet, const char* str) {
int32_t walCheckAndRepairMeta(SWal* pWal) {
// load log files, get first/snapshot/last version info
if (pWal->cfg.level == TAOS_WAL_SKIP) {
return TSDB_CODE_SUCCESS;
}
int32_t code = 0;
const char* logPattern = "^[0-9]+.log$";
const char* idxPattern = "^[0-9]+.idx$";

57
source/libs/wal/src/walMgmt.c
View File

@ -90,6 +90,45 @@ static int32_t walInitLock(SWal *pWal) {
return 0;
}
int32_t walInitWriteFileForSkip(SWal *pWal) {
TdFilePtr pIdxTFile = NULL, pLogTFile = NULL;
int64_t fileFirstVer = 0;
int32_t code = 0;
char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, fileFirstVer, fnameStr);
pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxTFile == NULL) {
wError("vgId:%d, failed to open file since %s", pWal->cfg.vgId, tstrerror(terrno));
code = terrno;
goto _exit;
}
walBuildLogName(pWal, fileFirstVer, fnameStr);
pLogTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pLogTFile == NULL) {
wError("vgId:%d, failed to open file since %s", pWal->cfg.vgId, tstrerror(terrno));
code = terrno;
goto _exit;
}
// switch file
pWal->pIdxFile = pIdxTFile;
pWal->pLogFile = pLogTFile;
SWalFileInfo fileInfo;
(void)memset(&fileInfo, -1, sizeof(SWalFileInfo));
if (!taosArrayPush(pWal->fileInfoSet, &fileInfo)) {
wError("vgId:%d, failed to push fileInfo into array since %s", pWal->cfg.vgId, tstrerror(terrno));
code = terrno;
goto _exit;
}
pWal->writeCur = 0;
_exit:
if (code != TSDB_CODE_SUCCESS) {
(void)taosCloseFile(&pIdxTFile);
(void)taosCloseFile(&pLogTFile);
}
TAOS_RETURN(code);
}
SWal *walOpen(const char *path, SWalCfg *pCfg) {
int32_t code = 0;
SWal *pWal = taosMemoryCalloc(1, sizeof(SWal));
@ -165,7 +204,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
if (code < 0) {
wWarn("vgId:%d, failed to load meta since %s", pWal->cfg.vgId, tstrerror(code));
}
if (pWal->cfg.level != TAOS_WAL_SKIP) {
code = walCheckAndRepairMeta(pWal);
if (code < 0) {
wError("vgId:%d, cannot open wal since repair meta file failed since %s", pWal->cfg.vgId, tstrerror(code));
@ -177,6 +216,14 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
wError("vgId:%d, cannot open wal since repair idx file failed since %s", pWal->cfg.vgId, tstrerror(code));
goto _err;
}
} else {
code = walInitWriteFileForSkip(pWal);
if (code < 0) {
wError("vgId:%d, cannot open wal since init write file for wal_level = 0 failed since %s", pWal->cfg.vgId,
tstrerror(code));
goto _err;
}
}
// add ref
pWal->refId = taosAddRef(tsWal.refSetId, pWal);
@ -217,6 +264,14 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) {
pWal->cfg.vgId, pWal->cfg.level, pWal->cfg.fsyncPeriod, pWal->cfg.retentionPeriod, pWal->cfg.retentionSize,
pCfg->level, pCfg->fsyncPeriod, pCfg->retentionPeriod, pCfg->retentionSize);
if (pWal->cfg.level == TAOS_WAL_SKIP && pCfg->level != TAOS_WAL_SKIP) {
wInfo("vgId:%d, remove all wals, path:%s", pWal->cfg.vgId, pWal->path);
taosRemoveDir(pWal->path);
if (taosMkDir(pWal->path) != 0) {
wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, tstrerror(terrno));
}
}
pWal->cfg.level = pCfg->level;
pWal->cfg.fsyncPeriod = pCfg->fsyncPeriod;
pWal->cfg.retentionPeriod = pCfg->retentionPeriod;

25
source/os/src/osDir.c
View File

@ -116,7 +116,10 @@ void taosRemoveDir(const char *dirname) {
return;
}
bool taosDirExist(const char *dirname) { return taosCheckExistFile(dirname); }
bool taosDirExist(const char *dirname) {
if (dirname == NULL || strlen(dirname) >= TDDIRMAXLEN) return false;
return taosCheckExistFile(dirname);
}
int32_t taosMkDir(const char *dirname) {
if (taosDirExist(dirname)) return 0;
@ -333,6 +336,8 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) {
}
int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) {
OS_PARAM_CHECK(dirname);
OS_PARAM_CHECK(outname);
wordexp_t full_path;
int32_t code = wordexp(dirname, &full_path, 0);
switch (code) {
@ -355,6 +360,8 @@ int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) {
}
int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen) {
OS_PARAM_CHECK(dirname);
OS_PARAM_CHECK(realPath);
char tmp[PATH_MAX] = {0};
#ifdef WINDOWS
if (_fullpath(tmp, dirname, maxlen) != NULL) {
@ -386,6 +393,10 @@ bool taosIsDir(const char *dirname) {
}
char *taosDirName(char *name) {
if(name == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
#ifdef WINDOWS
char Drive1[MAX_PATH], Dir1[MAX_PATH];
_splitpath(name, Drive1, Dir1, NULL, NULL);
@ -412,12 +423,16 @@ char *taosDirName(char *name) {
}
char *taosDirEntryBaseName(char *name) {
if(name == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
#ifdef WINDOWS
char Filename1[MAX_PATH], Ext1[MAX_PATH];
_splitpath(name, NULL, NULL, Filename1, Ext1);
return name + (strlen(name) - strlen(Filename1) - strlen(Ext1));
#else
if (name == NULL || (name[0] == '/' && name[1] == '\0')) return name;
if ((name[0] == '/' && name[1] == '\0')) return name;
char *pPoint = strrchr(name, '/');
if (pPoint != NULL) {
if (*(pPoint + 1) == '\0') {
@ -515,9 +530,9 @@ bool taosDirEntryIsDir(TdDirEntryPtr pDirEntry) {
}
char *taosGetDirEntryName(TdDirEntryPtr pDirEntry) {
/*if (pDirEntry == NULL) {*/
/*return NULL;*/
/*}*/
if (pDirEntry == NULL) {
return NULL;
}
#ifdef WINDOWS
return pDirEntry->findFileData.cFileName;
#else

11
source/os/src/osEnv.c
View File

@ -127,8 +127,13 @@ bool osTempSpaceSufficient() { return tsTempSpace.size.avail > tsTempSpace.reser
int32_t osSetTimezone(const char *tz) { return taosSetSystemTimezone(tz, tsTimezoneStr, &tsDaylight, &tsTimezone); }
void osSetSystemLocale(const char *inLocale, const char *inCharSet) {
(void)memcpy(tsLocale, inLocale, strlen(inLocale) + 1);
(void)memcpy(tsCharset, inCharSet, strlen(inCharSet) + 1);
if (inLocale) (void)memcpy(tsLocale, inLocale, strlen(inLocale) + 1);
if (inCharSet) (void)memcpy(tsCharset, inCharSet, strlen(inCharSet) + 1);
}
void osSetProcPath(int32_t argc, char **argv) { tsProcPath = argv[0]; }
void osSetProcPath(int32_t argc, char **argv) {
if (argv == NULL || argc < 1) {
return; // no command line arguments
}
tsProcPath = argv[0];
}

71
source/os/src/osFile.c
View File

@ -80,6 +80,7 @@ typedef struct TdFile {
#endif
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) {
if (inputTmpDir == NULL || fileNamePrefix == NULL) return;
#ifdef WINDOWS
char tmpPath[PATH_MAX];
@ -120,6 +121,10 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, cha
}
int64_t taosCopyFile(const char *from, const char *to) {
if (from == NULL || to == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return -1;
}
#ifdef WINDOWS
if (CopyFile(from, to, 0)) {
return 1;
@ -133,15 +138,15 @@ int64_t taosCopyFile(const char *from, const char *to) {
int64_t bytes;
int32_t code = TSDB_CODE_SUCCESS;
// fidfrom = open(from, O_RDONLY);
TdFilePtr pFileFrom = taosOpenFile(from, TD_FILE_READ);
TdFilePtr pFileFrom = NULL;
TdFilePtr pFileTo = NULL;
pFileFrom = taosOpenFile(from, TD_FILE_READ);
if (pFileFrom == NULL) {
code = terrno;
goto _err;
}
// fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755);
TdFilePtr pFileTo = taosOpenFile(to, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_EXCL);
pFileTo = taosOpenFile(to, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_EXCL);
if (pFileTo == NULL) {
code = terrno;
goto _err;
@ -193,6 +198,10 @@ _err:
}
TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions) {
if(path == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
TdFilePtr fp = taosOpenFile(path, tdFileOptions);
if (!fp) {
if (terrno == TAOS_SYSTEM_ERROR(ENOENT)) {
@ -213,6 +222,7 @@ TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions) {
}
int32_t taosRemoveFile(const char *path) {
OS_PARAM_CHECK(path);
int32_t code = remove(path);
if (-1 == code) {
terrno = TAOS_SYSTEM_ERROR(errno);
@ -222,6 +232,8 @@ int32_t taosRemoveFile(const char *path) {
}
int32_t taosRenameFile(const char *oldName, const char *newName) {
OS_PARAM_CHECK(oldName);
OS_PARAM_CHECK(newName);
#ifdef WINDOWS
bool finished = false;
@ -262,6 +274,7 @@ int32_t taosRenameFile(const char *oldName, const char *newName) {
}
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *atime) {
OS_PARAM_CHECK(path);
#ifdef WINDOWS
struct _stati64 fileStat;
int32_t code = _stati64(path, &fileStat);
@ -335,6 +348,10 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) {
}
FILE *taosOpenFileForStream(const char *path, int32_t tdFileOptions) {
if (path == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
char *mode = NULL;
if (tdFileOptions & TD_FILE_APPEND) {
mode = (tdFileOptions & TD_FILE_TEXT) ? "at+" : "ab+";
@ -358,6 +375,10 @@ FILE *taosOpenFileForStream(const char *path, int32_t tdFileOptions) {
#ifdef WINDOWS
HANDLE taosOpenFileNotStream(const char *path, int32_t tdFileOptions) {
if (path == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return INVALID_HANDLE_VALUE;
}
DWORD openMode = 0;
DWORD access = 0;
DWORD fileFlag = FILE_ATTRIBUTE_NORMAL;
@ -408,6 +429,10 @@ HANDLE taosOpenFileNotStream(const char *path, int32_t tdFileOptions) {
}
int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) {
if (pFile == NULL || buf == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
#if FILE_WITH_LOCK
(void)taosThreadRwlockRdlock(&(pFile->rwlock));
#endif
@ -435,7 +460,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) {
}
int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
if (pFile == NULL || pFile->hFile == NULL) {
if (pFile == NULL || pFile->hFile == NULL || buf == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return 0;
}
@ -457,7 +482,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
}
int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t offset) {
if (pFile == NULL) {
if (pFile == NULL || buf == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return 0;
}
@ -719,6 +744,10 @@ bool lastErrorIsFileNotExist() {
#else
int taosOpenFileNotStream(const char *path, int32_t tdFileOptions) {
if (path == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return -1;
}
int access = O_BINARY;
access |= (tdFileOptions & TD_FILE_CREATE) ? O_CREAT : 0;
if ((tdFileOptions & TD_FILE_WRITE) && (tdFileOptions & TD_FILE_READ)) {
@ -742,6 +771,10 @@ int taosOpenFileNotStream(const char *path, int32_t tdFileOptions) {
}
int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) {
if (pFile == NULL || buf == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return -1;
}
STUB_RAND_IO_ERR(terrno)
#if FILE_WITH_LOCK
(void)taosThreadRwlockRdlock(&(pFile->rwlock));
@ -797,7 +830,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) {
int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
STUB_RAND_IO_ERR(terrno)
if (pFile == NULL) {
if (pFile == NULL || buf == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return 0;
}
@ -843,7 +876,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t offset) {
STUB_RAND_IO_ERR(terrno)
if (pFile == NULL) {
if (pFile == NULL || buf == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return 0;
}
@ -1050,6 +1083,10 @@ bool lastErrorIsFileNotExist() { return terrno == TAOS_SYSTEM_ERROR(ENOENT); }
#endif // WINDOWS
TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) {
if (path == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
STUB_RAND_IO_ERR(NULL)
FILE *fp = NULL;
#ifdef WINDOWS
@ -1399,6 +1436,10 @@ int32_t taosEOFFile(TdFilePtr pFile) {
}
bool taosCheckAccessFile(const char *pathname, int32_t tdFileAccessOptions) {
if (pathname == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return false; // invalid parameter
}
int flags = 0;
if (tdFileAccessOptions & TD_FILE_ACCESS_EXIST_OK) {
@ -1422,6 +1463,8 @@ bool taosCheckAccessFile(const char *pathname, int32_t tdFileAccessOptions) {
bool taosCheckExistFile(const char *pathname) { return taosCheckAccessFile(pathname, TD_FILE_ACCESS_EXIST_OK); };
int32_t taosCompressFile(char *srcFileName, char *destFileName) {
OS_PARAM_CHECK(srcFileName);
OS_PARAM_CHECK(destFileName);
int32_t compressSize = 163840;
int32_t ret = 0;
int32_t len = 0;
@ -1516,6 +1559,10 @@ int32_t taosLinkFile(char *src, char *dst) {
}
FILE *taosOpenCFile(const char *filename, const char *mode) {
if (filename == NULL || mode == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
STUB_RAND_IO_ERR(NULL)
FILE *f = fopen(filename, mode);
if (NULL == f) {
@ -1525,6 +1572,10 @@ FILE *taosOpenCFile(const char *filename, const char *mode) {
}
int taosSeekCFile(FILE *file, int64_t offset, int whence) {
if(NULL == file) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
#ifdef WINDOWS
return _fseeki64(file, offset, whence);
#else
@ -1538,6 +1589,10 @@ int taosSeekCFile(FILE *file, int64_t offset, int whence) {
}
size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream) {
if (buffer == NULL || stream == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return 0;
}
STUB_RAND_IO_ERR(terrno)
return fread(buffer, size, count, stream);
}

6
source/os/src/osLocale.c
View File

@ -52,6 +52,9 @@ typedef struct CharsetPair {
} CharsetPair;
char *taosCharsetReplace(char *charsetstr) {
if (charsetstr == NULL) {
return NULL;
}
CharsetPair charsetRep[] = {
{"utf8", "UTF-8"},
{"936", "CP936"},
@ -76,6 +79,8 @@ char *taosCharsetReplace(char *charsetstr) {
* In case that the setLocale failed to be executed, the right charset needs to be set.
*/
int32_t taosSetSystemLocale(const char *inLocale, const char *inCharSet) {
OS_PARAM_CHECK(inLocale);
OS_PARAM_CHECK(inCharSet);
if (!taosValidateEncodec(inCharSet)) {
return terrno;
}
@ -90,6 +95,7 @@ int32_t taosSetSystemLocale(const char *inLocale, const char *inCharSet) {
}
void taosGetSystemLocale(char *outLocale, char *outCharset) {
if (outLocale == NULL || outCharset == NULL) return;
#ifdef WINDOWS
char *locale = setlocale(LC_CTYPE, "en_US.UTF-8");
if (locale != NULL) {

105
source/os/src/osSemaphore.c
View File

@ -29,7 +29,11 @@
bool taosCheckPthreadValid(TdThread thread) { return thread.p != NULL; }
void taosResetPthread(TdThread* thread) { thread->p = 0; }
void taosResetPthread(TdThread* thread) {
if (thread != NULL) {
thread->p = NULL;
}
}
int64_t taosGetPthreadId(TdThread thread) {
#ifdef PTW32_VERSION
@ -46,6 +50,7 @@ bool taosComparePthread(TdThread first, TdThread second) { return first.p == sec
int32_t taosGetPId() { return GetCurrentProcessId(); }
int32_t taosGetAppName(char* name, int32_t* len) {
OS_PARAM_CHECK(name);
char filepath[1024] = {0};
if (GetModuleFileName(NULL, filepath, MAX_PATH) == 0) {
@ -72,7 +77,11 @@ int32_t taosGetAppName(char* name, int32_t* len) {
return 0;
}
int32_t taosGetPIdByName(const char* name, int32_t* pPId) { return -1;}
int32_t tsem_wait(tsem_t* sem) {
OS_PARAM_CHECK(sem);
OS_PARAM_CHECK(*sem);
DWORD ret = WaitForSingleObject(*sem, INFINITE);
if (ret == WAIT_OBJECT_0) {
return 0;
@ -82,6 +91,8 @@ int32_t tsem_wait(tsem_t* sem) {
}
int32_t tsem_timewait(tsem_t* sem, int64_t timeout_ms) {
OS_PARAM_CHECK(sem);
OS_PARAM_CHECK(*sem);
DWORD result = WaitForSingleObject(*sem, timeout_ms);
if (result == WAIT_OBJECT_0) {
return 0; // Semaphore acquired
@ -94,16 +105,21 @@ int32_t tsem_timewait(tsem_t* sem, int64_t timeout_ms) {
// Inter-process sharing is not currently supported. The pshared parameter is invalid.
int32_t tsem_init(tsem_t* sem, int pshared, unsigned int value) {
OS_PARAM_CHECK(sem);
*sem = CreateSemaphore(NULL, value, LONG_MAX, NULL);
return (*sem != NULL) ? 0 : TAOS_SYSTEM_WINAPI_ERROR(GetLastError());
}
int32_t tsem_post(tsem_t* sem) {
OS_PARAM_CHECK(sem);
OS_PARAM_CHECK(*sem);
if (ReleaseSemaphore(*sem, 1, NULL)) return 0;
return TAOS_SYSTEM_WINAPI_ERROR(GetLastError());
}
int32_t tsem_destroy(tsem_t* sem) {
OS_PARAM_CHECK(sem);
OS_PARAM_CHECK(*sem);
if (CloseHandle(*sem)) return 0;
return TAOS_SYSTEM_WINAPI_ERROR(GetLastError());
}
@ -113,6 +129,7 @@ int32_t tsem_destroy(tsem_t* sem) {
#include <libproc.h>
int32_t tsem_init(tsem_t *psem, int flags, unsigned int count) {
OS_PARAM_CHECK(psem);
*psem = dispatch_semaphore_create(count);
if (*psem == NULL) return TAOS_SYSTEM_ERROR(errno);
return 0;
@ -126,19 +143,19 @@ int32_t tsem_destroy(tsem_t *psem) {
}
int32_t tsem_post(tsem_t *psem) {
if (psem == NULL || *psem == NULL) return -1;
if (psem == NULL || *psem == NULL) return TSDB_CODE_INVALID_PARA;
(void)dispatch_semaphore_signal(*psem);
return 0;
}
int32_t tsem_wait(tsem_t *psem) {
if (psem == NULL || *psem == NULL) return -1;
if (psem == NULL || *psem == NULL) return TSDB_CODE_INVALID_PARA;
dispatch_semaphore_wait(*psem, DISPATCH_TIME_FOREVER);
return 0;
}
int32_t tsem_timewait(tsem_t *psem, int64_t milis) {
if (psem == NULL || *psem == NULL) return -1;
if (psem == NULL || *psem == NULL) return TSDB_CODE_INVALID_PARA;
dispatch_time_t time = dispatch_time(DISPATCH_TIME_NOW, (int64_t)(milis * USEC_PER_SEC));
if (dispatch_semaphore_wait(*psem, time) == 0) {
return 0;
@ -156,13 +173,18 @@ int64_t taosGetSelfPthreadId() {
int64_t taosGetPthreadId(TdThread thread) { return (int64_t)thread; }
void taosResetPthread(TdThread *thread) { *thread = NULL; }
void taosResetPthread(TdThread *thread) {
if (thread) {
*thread = NULL;
}
}
bool taosComparePthread(TdThread first, TdThread second) { return taosThreadEqual(first, second) ? true : false; }
int32_t taosGetPId() { return (int32_t)getpid(); }
int32_t taosGetAppName(char *name, int32_t *len) {
OS_PARAM_CHECK(name);
char buf[PATH_MAX + 1];
buf[0] = '\0';
proc_name(getpid(), buf, sizeof(buf) - 1);
@ -173,6 +195,8 @@ int32_t taosGetAppName(char *name, int32_t *len) {
return 0;
}
int32_t taosGetPIdByName(const char* name, int32_t* pPId) {return -1;}
#else
/*
@ -192,7 +216,11 @@ int64_t taosGetSelfPthreadId() {
}
int64_t taosGetPthreadId(TdThread thread) { return (int64_t)thread; }
void taosResetPthread(TdThread* thread) { *thread = 0; }
void taosResetPthread(TdThread* thread) {
if (thread) {
*thread = 0;
}
}
bool taosComparePthread(TdThread first, TdThread second) { return first == second; }
int32_t taosGetPId() {
@ -203,6 +231,7 @@ int32_t taosGetPId() {
}
int32_t taosGetAppName(char* name, int32_t* len) {
OS_PARAM_CHECK(name);
const char* self = "/proc/self/exe";
char path[PATH_MAX] = {0};
@ -228,6 +257,61 @@ int32_t taosGetAppName(char* name, int32_t* len) {
return 0;
}
int32_t taosGetPIdByName(const char* name, int32_t* pPId) {
OS_PARAM_CHECK(name);
OS_PARAM_CHECK(pPId);
DIR* dir = NULL;
struct dirent* ptr = NULL;
FILE* fp = NULL;
char filepath[512];
char bufx[50];
char buf[1024] = {0};
*pPId = -1;
dir = opendir("/proc");
if (dir == NULL) {
return TAOS_SYSTEM_ERROR(errno);
}
while ((ptr = readdir(dir)) != NULL) {
if ((strcmp(ptr->d_name, ".") == 0) || (strcmp(ptr->d_name, "..") == 0)) {
continue;
}
if (DT_DIR != ptr->d_type) {
continue;
}
int32_t ret = tsnprintf(filepath, tListLen(filepath), "/proc/%s/status", ptr->d_name);
if (ret == -1) {
continue;
}
fp = fopen(filepath, "r");
if (NULL != fp) {
if (fgets(buf, tListLen(buf) - 1, fp) == NULL) {
TAOS_UNUSED(fclose(fp));
continue;
}
ret = sscanf(buf, "%*s %s", bufx);
if (!strcmp(bufx, name)) {
char* end = NULL;
*pPId = taosStr2Int32(ptr->d_name, &end, 10);
}
TAOS_UNUSED(fclose(fp));
}
}
TAOS_UNUSED(closedir(dir));
if ((*pPId) == -1) {
return TAOS_SYSTEM_ERROR(ESRCH);
} else {
return TSDB_CODE_SUCCESS;
}
}
int32_t tsem_init(tsem_t* psem, int flags, unsigned int count) {
if (sem_init(psem, flags, count) == 0) {
return 0;
@ -237,6 +321,7 @@ int32_t tsem_init(tsem_t* psem, int flags, unsigned int count) {
}
int32_t tsem_timewait(tsem_t* sem, int64_t ms) {
OS_PARAM_CHECK(sem);
int ret = 0;
struct timespec ts = {0};
@ -265,6 +350,7 @@ int32_t tsem_timewait(tsem_t* sem, int64_t ms) {
}
int32_t tsem_wait(tsem_t* sem) {
OS_PARAM_CHECK(sem);
int ret = 0;
do {
ret = sem_wait(sem);
@ -279,6 +365,7 @@ int32_t tsem_wait(tsem_t* sem) {
}
int tsem2_init(tsem2_t* sem, int pshared, unsigned int value) {
OS_PARAM_CHECK(sem);
int ret = taosThreadMutexInit(&sem->mutex, NULL);
if (ret != 0) return ret;
@ -308,6 +395,7 @@ int tsem2_init(tsem2_t* sem, int pshared, unsigned int value) {
}
int32_t tsem_post(tsem_t* psem) {
OS_PARAM_CHECK(psem);
if (sem_post(psem) == 0) {
return 0;
} else {
@ -316,6 +404,7 @@ int32_t tsem_post(tsem_t* psem) {
}
int32_t tsem_destroy(tsem_t* sem) {
OS_PARAM_CHECK(sem);
if (sem_destroy(sem) == 0) {
return 0;
} else {
@ -324,6 +413,7 @@ int32_t tsem_destroy(tsem_t* sem) {
}
int tsem2_post(tsem2_t* sem) {
OS_PARAM_CHECK(sem);
int32_t code = taosThreadMutexLock(&sem->mutex);
if (code) {
return code;
@ -344,6 +434,7 @@ int tsem2_post(tsem2_t* sem) {
}
int tsem2_destroy(tsem2_t* sem) {
OS_PARAM_CHECK(sem);
(void)taosThreadMutexDestroy(&sem->mutex);
(void)taosThreadCondDestroy(&sem->cond);
(void)taosThreadCondAttrDestroy(&sem->attr);
@ -352,6 +443,7 @@ int tsem2_destroy(tsem2_t* sem) {
}
int32_t tsem2_wait(tsem2_t* sem) {
OS_PARAM_CHECK(sem);
int32_t code = taosThreadMutexLock(&sem->mutex);
if (code) {
return code;
@ -377,6 +469,7 @@ int32_t tsem2_wait(tsem2_t* sem) {
}
int32_t tsem2_timewait(tsem2_t* sem, int64_t ms) {
OS_PARAM_CHECK(sem);
int32_t code = 0;
code = taosThreadMutexLock(&sem->mutex);

6
source/os/src/osSocket.c
View File

@ -233,6 +233,8 @@ int32_t taosBlockSIGPIPE() {
}
int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip) {
OS_PARAM_CHECK(fqdn);
OS_PARAM_CHECK(ip);
#ifdef WINDOWS
// Initialize Winsock
WSADATA wsaData;
@ -309,6 +311,7 @@ int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip) {
}
int32_t taosGetFqdn(char *fqdn) {
OS_PARAM_CHECK(fqdn);
#ifdef WINDOWS
// Initialize Winsock
WSADATA wsaData;
@ -384,6 +387,9 @@ int32_t taosGetFqdn(char *fqdn) {
}
void tinet_ntoa(char *ipstr, uint32_t ip) {
if (ipstr == NULL) {
return;
}
(void)snprintf(ipstr, TD_IP_LEN, "%d.%d.%d.%d", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, ip >> 24);
}

153
source/os/src/osString.c
View File

@ -25,11 +25,15 @@ extern int wcwidth(wchar_t c);
extern int wcswidth(const wchar_t *s, size_t n);
char *tstrdup(const char *str) {
if (str == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
#ifdef WINDOWS
return _strdup(str);
#else
char* p = strdup(str);
if (str != NULL && NULL == p) {
char *p = strdup(str);
if (NULL == p) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
}
return p;
@ -41,11 +45,19 @@ char *tstrdup(const char *str) {
// No errors are expected to occur
char *strsep(char **stringp, const char *delim) {
if (stringp == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
char *s;
const char *spanp;
int32_t c, sc;
char *tok;
if ((s = *stringp) == NULL) return (NULL);
if (delim == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
for (tok = s;;) {
c = *s++;
spanp = delim;
@ -81,6 +93,10 @@ char *taosStrndup(const char *s, int size) {
/* Copy no more than N characters of SRC to DEST, returning the address of
the terminating '\0' in DEST, if any, or else DEST + N. */
char *stpncpy(char *dest, const char *src, int n) {
if (dest == NULL || src == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
size_t size = strnlen(src, n);
memcpy(dest, src, size);
dest += size;
@ -117,6 +133,8 @@ int32_t taosStr2int64(const char *str, int64_t *val) {
}
int32_t taosStr2int16(const char *str, int16_t *val) {
OS_PARAM_CHECK(str);
OS_PARAM_CHECK(val);
int64_t tmp = 0;
int32_t code = taosStr2int64(str, &tmp);
if (code) {
@ -130,6 +148,8 @@ int32_t taosStr2int16(const char *str, int16_t *val) {
}
int32_t taosStr2int32(const char *str, int32_t *val) {
OS_PARAM_CHECK(str);
OS_PARAM_CHECK(val);
int64_t tmp = 0;
int32_t code = taosStr2int64(str, &tmp);
if (code) {
@ -143,6 +163,8 @@ int32_t taosStr2int32(const char *str, int32_t *val) {
}
int32_t taosStr2int8(const char *str, int8_t *val) {
OS_PARAM_CHECK(str);
OS_PARAM_CHECK(val);
int64_t tmp = 0;
int32_t code = taosStr2int64(str, &tmp);
if (code) {
@ -156,6 +178,9 @@ int32_t taosStr2int8(const char *str, int8_t *val) {
}
int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) {
if ((f1_ucs4 == NULL || f2_ucs4 == NULL)) {
return TSDB_CODE_INVALID_PARA;
}
for (int32_t i = 0; i < bytes; i += sizeof(TdUcs4)) {
int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i);
int32_t f2 = *(int32_t *)((char *)f2_ucs4 + i);
@ -191,6 +216,9 @@ int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) {
}
int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) {
if (target_ucs4 == NULL || source_ucs4 == NULL || len_ucs4 <= 0) {
return TSDB_CODE_INVALID_PARA;
}
if (taosMemorySize(target_ucs4) < len_ucs4 * sizeof(TdUcs4)) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
@ -261,6 +289,10 @@ void taosConvDestroy() {
}
iconv_t taosAcquireConv(int32_t *idx, ConvType type) {
if(idx == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return (iconv_t)-1;
}
if (gConvMaxNum[type] <= 0) {
*idx = -1;
if (type == M2C) {
@ -321,6 +353,13 @@ void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type) {
}
bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len) {
if (ucs4_max_len == 0) {
return true;
}
if(ucs4_max_len < 0 || mbs == NULL || ucs4 == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return false;
}
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV
printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n");
terrno = TSDB_CODE_APP_ERROR;
@ -359,6 +398,13 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4
// if success, return the number of bytes written to mbs ( >= 0)
// otherwise return error code ( < 0)
int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) {
if (ucs4_max_len == 0) {
return 0;
}
if(ucs4_max_len < 0 || ucs4 == NULL || mbs == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV
printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n");
terrno = TSDB_CODE_APP_ERROR;
@ -390,6 +436,13 @@ int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) {
// if success, return the number of bytes written to mbs ( >= 0)
// otherwise return error code ( < 0)
int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t conv) {
if (ucs4_max_len == 0) {
return 0;
}
if(ucs4_max_len < 0 || ucs4 == NULL || mbs == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV
printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n");
terrno = TSDB_CODE_APP_ERROR;
@ -408,6 +461,10 @@ int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t c
}
bool taosValidateEncodec(const char *encodec) {
if(encodec == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return false;
}
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV
printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n");
terrno = TSDB_CODE_APP_ERROR;
@ -443,7 +500,7 @@ int32_t taosUcs4len(TdUcs4 *ucs4) {
// dst buffer size should be at least 2*len + 1
int32_t taosHexEncode(const unsigned char *src, char *dst, int32_t len, int32_t bufSize) {
if (!dst) {
if (!dst || !src || bufSize <= 0) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
@ -456,7 +513,7 @@ int32_t taosHexEncode(const unsigned char *src, char *dst, int32_t len, int32_t
}
int32_t taosHexDecode(const char *src, char *dst, int32_t len) {
if (!dst) {
if(!src || !dst || len <= 0) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
@ -475,19 +532,42 @@ int32_t taosHexDecode(const char *src, char *dst, int32_t len) {
int32_t taosWcharWidth(TdWchar wchar) { return wcwidth(wchar); }
int32_t taosWcharsWidth(TdWchar *pWchar, int32_t size) { return wcswidth(pWchar, size); }
int32_t taosWcharsWidth(TdWchar *pWchar, int32_t size) {
if (pWchar == NULL || size <= 0) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
return wcswidth(pWchar, size);
}
int32_t taosMbToWchar(TdWchar *pWchar, const char *pStr, int32_t size) { return mbtowc(pWchar, pStr, size); }
int32_t taosMbToWchar(TdWchar *pWchar, const char *pStr, int32_t size) {
if (pWchar == NULL || pStr == NULL || size <= 0) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
return mbtowc(pWchar, pStr, size);
}
int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size) { return mbstowcs(pWchars, pStrs, size); }
int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size) {
if (pWchars == NULL || pStrs == NULL || size <= 0) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
return mbstowcs(pWchars, pStrs, size);
}
int32_t taosWcharToMb(char *pStr, TdWchar wchar) { return wctomb(pStr, wchar); }
int32_t taosWcharToMb(char *pStr, TdWchar wchar) {
OS_PARAM_CHECK(pStr);
return wctomb(pStr, wchar); }
char *taosStrCaseStr(const char *str, const char *pattern) {
if (str == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
if (!pattern || !*pattern) return (char *)str;
size_t i;
if (!*pattern) return (char *)str;
for (; *str; str++) {
if (toupper(*str) == toupper(*pattern)) {
for (i = 1;; i++) {
@ -500,6 +580,10 @@ char *taosStrCaseStr(const char *str, const char *pattern) {
}
int64_t taosStr2Int64(const char *str, char **pEnd, int32_t radix) {
if(str == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return 0;
}
int64_t tmp = strtoll(str, pEnd, radix);
#if defined(DARWIN) || defined(_ALPINE)
if (errno == EINVAL) errno = 0;
@ -508,6 +592,10 @@ int64_t taosStr2Int64(const char *str, char **pEnd, int32_t radix) {
}
uint64_t taosStr2UInt64(const char *str, char **pEnd, int32_t radix) {
if(str == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return 0;
}
uint64_t tmp = strtoull(str, pEnd, radix);
#if defined(DARWIN) || defined(_ALPINE)
if (errno == EINVAL) errno = 0;
@ -516,6 +604,10 @@ uint64_t taosStr2UInt64(const char *str, char **pEnd, int32_t radix) {
}
int32_t taosStr2Int32(const char *str, char **pEnd, int32_t radix) {
if(str == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return 0;
}
int32_t tmp = strtol(str, pEnd, radix);
#if defined(DARWIN) || defined(_ALPINE)
if (errno == EINVAL) errno = 0;
@ -524,6 +616,10 @@ int32_t taosStr2Int32(const char *str, char **pEnd, int32_t radix) {
}
uint32_t taosStr2UInt32(const char *str, char **pEnd, int32_t radix) {
if(str == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return 0;
}
uint32_t tmp = strtol(str, pEnd, radix);
#if defined(DARWIN) || defined(_ALPINE)
if (errno == EINVAL) errno = 0;
@ -532,6 +628,10 @@ uint32_t taosStr2UInt32(const char *str, char **pEnd, int32_t radix) {
}
int16_t taosStr2Int16(const char *str, char **pEnd, int32_t radix) {
if(str == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return 0;
}
int32_t tmp = strtol(str, pEnd, radix);
#if defined(DARWIN) || defined(_ALPINE)
if (errno == EINVAL) errno = 0;
@ -540,6 +640,10 @@ int16_t taosStr2Int16(const char *str, char **pEnd, int32_t radix) {
}
uint16_t taosStr2UInt16(const char *str, char **pEnd, int32_t radix) {
if(str == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return 0;
}
uint32_t tmp = strtoul(str, pEnd, radix);
#if defined(DARWIN) || defined(_ALPINE)
if (errno == EINVAL) errno = 0;
@ -548,11 +652,19 @@ uint16_t taosStr2UInt16(const char *str, char **pEnd, int32_t radix) {
}
int8_t taosStr2Int8(const char *str, char **pEnd, int32_t radix) {
if(str == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return 0;
}
int32_t tmp = strtol(str, pEnd, radix);
return tmp;
}
uint8_t taosStr2UInt8(const char *str, char **pEnd, int32_t radix) {
if(str == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return 0;
}
uint32_t tmp = strtoul(str, pEnd, radix);
#if defined(DARWIN) || defined(_ALPINE)
if (errno == EINVAL) errno = 0;
@ -561,11 +673,19 @@ uint8_t taosStr2UInt8(const char *str, char **pEnd, int32_t radix) {
}
double taosStr2Double(const char *str, char **pEnd) {
if(str == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return 0;
}
double tmp = strtod(str, pEnd);
return tmp;
}
float taosStr2Float(const char *str, char **pEnd) {
if(str == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return 0;
}
float tmp = strtof(str, pEnd);
return tmp;
}
@ -578,6 +698,10 @@ bool isHex(const char *z, uint32_t n) {
}
bool isValidateHex(const char *z, uint32_t n) {
if(!z) {
terrno = TSDB_CODE_INVALID_PARA;
return false;
}
if ((n & 1) != 0) return false;
for (size_t i = HEX_PREFIX_LEN; i < n; i++) {
if (isxdigit(z[i]) == 0) {
@ -588,6 +712,9 @@ bool isValidateHex(const char *z, uint32_t n) {
}
int32_t taosHex2Ascii(const char *z, uint32_t n, void **data, uint32_t *size) {
OS_PARAM_CHECK(z);
OS_PARAM_CHECK(data);
OS_PARAM_CHECK(size);
n -= HEX_PREFIX_LEN; // remove 0x
z += HEX_PREFIX_LEN;
*size = n / HEX_PREFIX_LEN;
@ -712,6 +839,10 @@ int32_t taosAscii2Hex(const char *z, uint32_t n, void **data, uint32_t *size) {
}
int64_t tsnprintf(char *dst, int64_t size, const char *format, ...) {
if (dst == NULL || format == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return 0;
}
if (size <= 0) return 0;
if (size == 1) {
dst[0] = '\0';

55
source/os/src/osSysinfo.c
View File

@ -154,6 +154,7 @@ static void taosGetProcIOnfos() {
#endif
static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) {
OS_PARAM_CHECK(cpuInfo);
int32_t code = 0;
#ifdef WINDOWS
FILETIME pre_idleTime = {0};
@ -206,6 +207,7 @@ static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) {
}
static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) {
OS_PARAM_CHECK(cpuInfo);
int32_t code = 0;
#ifdef WINDOWS
@ -287,6 +289,7 @@ void taosGetSystemInfo() {
}
int32_t taosGetEmail(char *email, int32_t maxLen) {
OS_PARAM_CHECK(email);
#ifdef WINDOWS
return 0;
#elif defined(_TD_DARWIN_64)
@ -330,6 +333,7 @@ int32_t taosGetEmail(char *email, int32_t maxLen) {
#ifdef WINDOWS
bool getWinVersionReleaseName(char *releaseName, int32_t maxLen) {
if(releaseName == NULL) return false;
TCHAR szFileName[MAX_PATH];
DWORD dwHandle;
DWORD dwLen;
@ -367,6 +371,7 @@ bool getWinVersionReleaseName(char *releaseName, int32_t maxLen) {
#endif
int32_t taosGetOsReleaseName(char *releaseName, char* sName, char* ver, int32_t maxLen) {
OS_PARAM_CHECK(releaseName);
#ifdef WINDOWS
if (!getWinVersionReleaseName(releaseName, maxLen)) {
snprintf(releaseName, maxLen, "Windows");
@ -437,6 +442,8 @@ int32_t taosGetOsReleaseName(char *releaseName, char* sName, char* ver, int32_t
}
int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) {
OS_PARAM_CHECK(cpuModel);
OS_PARAM_CHECK(numOfCores);
#ifdef WINDOWS
char value[100];
DWORD bufferSize = sizeof(value);
@ -541,6 +548,7 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) {
// Returns the container's CPU quota if successful, otherwise returns the physical CPU cores
static int32_t taosCntrGetCpuCores(float *numOfCores) {
OS_PARAM_CHECK(numOfCores);
#ifdef WINDOWS
return TSDB_CODE_UNSUPPORT_OS;
#elif defined(_TD_DARWIN_64)
@ -600,6 +608,7 @@ _end:
}
int32_t taosGetCpuCores(float *numOfCores, bool physical) {
OS_PARAM_CHECK(numOfCores);
#ifdef WINDOWS
SYSTEM_INFO info;
GetSystemInfo(&info);
@ -702,6 +711,7 @@ int32_t taosGetCpuInstructions(char* sse42, char* avx, char* avx2, char* fma, ch
}
int32_t taosGetTotalMemory(int64_t *totalKB) {
OS_PARAM_CHECK(totalKB);
#ifdef WINDOWS
MEMORYSTATUSEX memsStat;
memsStat.dwLength = sizeof(memsStat);
@ -723,6 +733,7 @@ int32_t taosGetTotalMemory(int64_t *totalKB) {
}
int32_t taosGetProcMemory(int64_t *usedKB) {
OS_PARAM_CHECK(usedKB);
#ifdef WINDOWS
unsigned bytes_used = 0;
@ -769,6 +780,7 @@ int32_t taosGetProcMemory(int64_t *usedKB) {
}
int32_t taosGetSysMemory(int64_t *usedKB) {
OS_PARAM_CHECK(usedKB);
#ifdef WINDOWS
MEMORYSTATUSEX memsStat;
memsStat.dwLength = sizeof(memsStat);
@ -794,6 +806,8 @@ int32_t taosGetSysMemory(int64_t *usedKB) {
}
int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) {
OS_PARAM_CHECK(dataDir);
OS_PARAM_CHECK(diskSize);
#if defined(WINDOWS)
unsigned _int64 i64FreeBytesToCaller;
unsigned _int64 i64TotalBytes;
@ -839,21 +853,25 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) {
}
int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes) {
OS_PARAM_CHECK(rchars);
OS_PARAM_CHECK(wchars);
OS_PARAM_CHECK(read_bytes);
OS_PARAM_CHECK(write_bytes);
#ifdef WINDOWS
IO_COUNTERS io_counter;
if (GetProcessIoCounters(GetCurrentProcess(), &io_counter)) {
if (rchars) *rchars = io_counter.ReadTransferCount;
if (wchars) *wchars = io_counter.WriteTransferCount;
if (read_bytes) *read_bytes = 0;
if (write_bytes) *write_bytes = 0;
*rchars = io_counter.ReadTransferCount;
*wchars = io_counter.WriteTransferCount;
*read_bytes = 0;
*write_bytes = 0;
return 0;
}
return TAOS_SYSTEM_WINAPI_ERROR(GetLastError());
#elif defined(_TD_DARWIN_64)
if (rchars) *rchars = 0;
if (wchars) *wchars = 0;
if (read_bytes) *read_bytes = 0;
if (write_bytes) *write_bytes = 0;
*rchars = 0;
*wchars = 0;
*read_bytes = 0;
*write_bytes = 0;
return 0;
#else
TdFilePtr pFile = taosOpenFile(tsProcIOFile, TD_FILE_READ | TD_FILE_STREAM);
@ -900,6 +918,9 @@ int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int
}
int32_t taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes) {
if (rchars == NULL || wchars == NULL || read_bytes == NULL || write_bytes == NULL) {
return TSDB_CODE_INVALID_PARA;
}
static int64_t last_rchars = -1;
static int64_t last_wchars = -1;
static int64_t last_read_bytes = -1;
@ -932,13 +953,15 @@ int32_t taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes
return 0;
}
void taosSetDefaultProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes) {
*rchars = 0;
*wchars = 0;
*read_bytes = 0;
*write_bytes = 0;
if(rchars) *rchars = 0;
if(wchars) *wchars = 0;
if(read_bytes) *read_bytes = 0;
if(write_bytes) *write_bytes = 0;
}
int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) {
OS_PARAM_CHECK(receive_bytes);
OS_PARAM_CHECK(transmit_bytes);
*receive_bytes = 0;
*transmit_bytes = 0;
@ -994,6 +1017,8 @@ int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) {
}
int32_t taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes) {
OS_PARAM_CHECK(receive_bytes);
OS_PARAM_CHECK(transmit_bytes);
static int64_t last_receive_bytes = -1;
static int64_t last_transmit_bytes = -1;
int64_t cur_receive_bytes = 0;
@ -1017,8 +1042,8 @@ int32_t taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes) {
return 0;
}
void taosSetDefaultCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes) {
*receive_bytes = 0;
*transmit_bytes = 0;
if (receive_bytes) *receive_bytes = 0;
if (transmit_bytes) *transmit_bytes = 0;
}
void taosKillSystem() {
@ -1037,6 +1062,7 @@ void taosKillSystem() {
#define UUIDLEN (36)
int32_t taosGetSystemUUIDLimit36(char *uid, int32_t uidlen) {
OS_PARAM_CHECK(uid);
#ifdef WINDOWS
GUID guid;
HRESULT h = CoCreateGuid(&guid);
@ -1334,6 +1360,7 @@ int32_t getMacLocalHostNameBySCD(char *hostname, size_t maxLen) {
#endif
int32_t taosGetlocalhostname(char *hostname, size_t maxLen) {
OS_PARAM_CHECK(hostname);
#ifdef _TD_DARWIN_64
int res = getMacLocalHostNameBySCD(hostname, maxLen);
if (res != 0) {

20
source/os/src/osSystem.c
View File

@ -107,26 +107,6 @@ void* taosLoadDll(const char* filename) {
#endif
}
void* taosLoadSym(void* handle, char* name) {
#if defined(WINDOWS)
return NULL;
#elif defined(_TD_DARWIN_64)
return NULL;
#else
void* sym = dlsym(handle, name);
char* error = NULL;
if ((error = dlerror()) != NULL) {
// printf("load sym:%s failed, error:%s", name, dlerror());
return NULL;
}
// printf("sym %s loaded", name);
return sym;
#endif
}
void taosCloseDll(void* handle) {
#if defined(WINDOWS)
return;

75
source/os/src/osThread.c
View File

@ -18,6 +18,8 @@
#include "os.h"
int32_t taosThreadCreate(TdThread *tid, const TdThreadAttr *attr, void *(*start)(void *), void *arg) {
OS_PARAM_CHECK(tid);
OS_PARAM_CHECK(start);
int32_t code = pthread_create(tid, attr, start, arg);
if (code) {
taosThreadClear(tid);
@ -28,6 +30,7 @@ int32_t taosThreadCreate(TdThread *tid, const TdThreadAttr *attr, void *(*start)
}
int32_t taosThreadAttrDestroy(TdThreadAttr *attr) {
OS_PARAM_CHECK(attr);
int32_t code = pthread_attr_destroy(attr);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -37,6 +40,8 @@ int32_t taosThreadAttrDestroy(TdThreadAttr *attr) {
}
int32_t taosThreadAttrGetDetachState(const TdThreadAttr *attr, int32_t *detachstate) {
OS_PARAM_CHECK(attr);
OS_PARAM_CHECK(detachstate);
int32_t code = pthread_attr_getdetachstate(attr, detachstate);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -46,6 +51,8 @@ int32_t taosThreadAttrGetDetachState(const TdThreadAttr *attr, int32_t *detachst
}
int32_t taosThreadAttrGetInheritSched(const TdThreadAttr *attr, int32_t *inheritsched) {
OS_PARAM_CHECK(attr);
OS_PARAM_CHECK(inheritsched);
int32_t code = pthread_attr_getinheritsched(attr, inheritsched);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -55,6 +62,8 @@ int32_t taosThreadAttrGetInheritSched(const TdThreadAttr *attr, int32_t *inherit
}
int32_t taosThreadAttrGetSchedParam(const TdThreadAttr *attr, struct sched_param *param) {
OS_PARAM_CHECK(attr);
OS_PARAM_CHECK(param);
int32_t code = pthread_attr_getschedparam(attr, param);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -64,6 +73,8 @@ int32_t taosThreadAttrGetSchedParam(const TdThreadAttr *attr, struct sched_param
}
int32_t taosThreadAttrGetSchedPolicy(const TdThreadAttr *attr, int32_t *policy) {
OS_PARAM_CHECK(attr);
OS_PARAM_CHECK(policy);
int32_t code = pthread_attr_getschedpolicy(attr, policy);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -73,6 +84,8 @@ int32_t taosThreadAttrGetSchedPolicy(const TdThreadAttr *attr, int32_t *policy)
}
int32_t taosThreadAttrGetScope(const TdThreadAttr *attr, int32_t *contentionscope) {
OS_PARAM_CHECK(attr);
OS_PARAM_CHECK(contentionscope);
int32_t code = pthread_attr_getscope(attr, contentionscope);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -82,6 +95,8 @@ int32_t taosThreadAttrGetScope(const TdThreadAttr *attr, int32_t *contentionscop
}
int32_t taosThreadAttrGetStackSize(const TdThreadAttr *attr, size_t *stacksize) {
OS_PARAM_CHECK(attr);
OS_PARAM_CHECK(stacksize);
int32_t code = pthread_attr_getstacksize(attr, stacksize);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -91,6 +106,7 @@ int32_t taosThreadAttrGetStackSize(const TdThreadAttr *attr, size_t *stacksize)
}
int32_t taosThreadAttrInit(TdThreadAttr *attr) {
OS_PARAM_CHECK(attr);
int32_t code = pthread_attr_init(attr);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -100,6 +116,7 @@ int32_t taosThreadAttrInit(TdThreadAttr *attr) {
}
int32_t taosThreadAttrSetDetachState(TdThreadAttr *attr, int32_t detachstate) {
OS_PARAM_CHECK(attr);
int32_t code = pthread_attr_setdetachstate(attr, detachstate);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -109,6 +126,7 @@ int32_t taosThreadAttrSetDetachState(TdThreadAttr *attr, int32_t detachstate) {
}
int32_t taosThreadAttrSetInheritSched(TdThreadAttr *attr, int32_t inheritsched) {
OS_PARAM_CHECK(attr);
int32_t code = pthread_attr_setinheritsched(attr, inheritsched);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -118,6 +136,7 @@ int32_t taosThreadAttrSetInheritSched(TdThreadAttr *attr, int32_t inheritsched)
}
int32_t taosThreadAttrSetSchedParam(TdThreadAttr *attr, const struct sched_param *param) {
OS_PARAM_CHECK(attr);
int32_t code = pthread_attr_setschedparam(attr, param);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -127,6 +146,7 @@ int32_t taosThreadAttrSetSchedParam(TdThreadAttr *attr, const struct sched_param
}
int32_t taosThreadAttrSetSchedPolicy(TdThreadAttr *attr, int32_t policy) {
OS_PARAM_CHECK(attr);
int32_t code = pthread_attr_setschedpolicy(attr, policy);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -136,6 +156,7 @@ int32_t taosThreadAttrSetSchedPolicy(TdThreadAttr *attr, int32_t policy) {
}
int32_t taosThreadAttrSetScope(TdThreadAttr *attr, int32_t contentionscope) {
OS_PARAM_CHECK(attr);
int32_t code = pthread_attr_setscope(attr, contentionscope);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -145,6 +166,7 @@ int32_t taosThreadAttrSetScope(TdThreadAttr *attr, int32_t contentionscope) {
}
int32_t taosThreadAttrSetStackSize(TdThreadAttr *attr, size_t stacksize) {
OS_PARAM_CHECK(attr);
int32_t code = pthread_attr_setstacksize(attr, stacksize);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -163,6 +185,7 @@ int32_t taosThreadCancel(TdThread thread) {
}
int32_t taosThreadCondDestroy(TdThreadCond *cond) {
OS_PARAM_CHECK(cond);
#ifdef __USE_WIN_THREAD
return 0;
#else
@ -176,6 +199,7 @@ int32_t taosThreadCondDestroy(TdThreadCond *cond) {
}
int32_t taosThreadCondInit(TdThreadCond *cond, const TdThreadCondAttr *attr) {
OS_PARAM_CHECK(cond);
#ifdef __USE_WIN_THREAD
InitializeConditionVariable(cond);
return 0;
@ -190,6 +214,7 @@ int32_t taosThreadCondInit(TdThreadCond *cond, const TdThreadCondAttr *attr) {
}
int32_t taosThreadCondSignal(TdThreadCond *cond) {
OS_PARAM_CHECK(cond);
#ifdef __USE_WIN_THREAD
WakeConditionVariable(cond);
return 0;
@ -204,6 +229,7 @@ int32_t taosThreadCondSignal(TdThreadCond *cond) {
}
int32_t taosThreadCondBroadcast(TdThreadCond *cond) {
OS_PARAM_CHECK(cond);
#ifdef __USE_WIN_THREAD
WakeAllConditionVariable(cond);
return 0;
@ -218,6 +244,8 @@ int32_t taosThreadCondBroadcast(TdThreadCond *cond) {
}
int32_t taosThreadCondWait(TdThreadCond *cond, TdThreadMutex *mutex) {
OS_PARAM_CHECK(cond);
OS_PARAM_CHECK(mutex);
#ifdef __USE_WIN_THREAD
if (!SleepConditionVariableCS(cond, mutex, INFINITE)) {
return EINVAL;
@ -234,8 +262,10 @@ int32_t taosThreadCondWait(TdThreadCond *cond, TdThreadMutex *mutex) {
}
int32_t taosThreadCondTimedWait(TdThreadCond *cond, TdThreadMutex *mutex, const struct timespec *abstime) {
#ifdef __USE_WIN_THREAD
if (!abstime) return 0;
OS_PARAM_CHECK(cond);
OS_PARAM_CHECK(mutex);
#ifdef __USE_WIN_THREAD
if (SleepConditionVariableCS(cond, mutex, (DWORD)(abstime->tv_sec * 1e3 + abstime->tv_nsec / 1e6))) return 0;
DWORD error = GetLastError();
if (error == ERROR_TIMEOUT) {
@ -258,6 +288,7 @@ int32_t taosThreadCondAttrDestroy(TdThreadCondAttr *attr) {
#ifdef __USE_WIN_THREAD
return 0;
#else
OS_PARAM_CHECK(attr);
int32_t code = pthread_condattr_destroy(attr);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -268,10 +299,12 @@ int32_t taosThreadCondAttrDestroy(TdThreadCondAttr *attr) {
}
int32_t taosThreadCondAttrGetPshared(const TdThreadCondAttr *attr, int32_t *pshared) {
OS_PARAM_CHECK(pshared);
#ifdef __USE_WIN_THREAD
if (pshared) *pshared = PTHREAD_PROCESS_PRIVATE;
return 0;
#else
OS_PARAM_CHECK(attr);
int32_t code = pthread_condattr_getpshared(attr, pshared);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -285,6 +318,7 @@ int32_t taosThreadCondAttrInit(TdThreadCondAttr *attr) {
#ifdef __USE_WIN_THREAD
return 0;
#else
OS_PARAM_CHECK(attr);
int32_t code = pthread_condattr_init(attr);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -300,6 +334,7 @@ int32_t taosThreadCondAttrSetclock(TdThreadCondAttr *attr, int clockId) {
#elif defined(__APPLE__)
return 0;
#else
OS_PARAM_CHECK(attr);
int32_t code = pthread_condattr_setclock(attr, clockId);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -310,6 +345,7 @@ int32_t taosThreadCondAttrSetclock(TdThreadCondAttr *attr, int clockId) {
}
int32_t taosThreadCondAttrSetPshared(TdThreadCondAttr *attr, int32_t pshared) {
OS_PARAM_CHECK(attr);
#ifdef __USE_WIN_THREAD
return 0;
#else
@ -336,10 +372,12 @@ int32_t taosThreadEqual(TdThread t1, TdThread t2) {
}
void taosThreadExit(void *valuePtr) {
return pthread_exit(valuePtr);
if(valuePtr) return pthread_exit(valuePtr);
}
int32_t taosThreadGetSchedParam(TdThread thread, int32_t *policy, struct sched_param *param) {
OS_PARAM_CHECK(policy);
OS_PARAM_CHECK(param);
int32_t code = pthread_getschedparam(thread, policy, param);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -362,6 +400,7 @@ int32_t taosThreadJoin(TdThread thread, void **valuePtr) {
}
int32_t taosThreadKeyCreate(TdThreadKey *key, void (*destructor)(void *)) {
OS_PARAM_CHECK(key);
int32_t code = pthread_key_create(key, destructor);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -393,6 +432,7 @@ int32_t taosThreadKill(TdThread thread, int32_t sig) {
// }
int32_t taosThreadMutexDestroy(TdThreadMutex *mutex) {
OS_PARAM_CHECK(mutex);
#ifdef __USE_WIN_THREAD
DeleteCriticalSection(mutex);
return 0;
@ -407,6 +447,7 @@ int32_t taosThreadMutexDestroy(TdThreadMutex *mutex) {
}
int32_t taosThreadMutexInit(TdThreadMutex *mutex, const TdThreadMutexAttr *attr) {
OS_PARAM_CHECK(mutex);
#ifdef __USE_WIN_THREAD
/**
* Windows Server 2003 and Windows XP: In low memory situations, InitializeCriticalSection can raise a
@ -426,6 +467,7 @@ int32_t taosThreadMutexInit(TdThreadMutex *mutex, const TdThreadMutexAttr *attr)
}
int32_t taosThreadMutexLock(TdThreadMutex *mutex) {
OS_PARAM_CHECK(mutex);
#ifdef __USE_WIN_THREAD
EnterCriticalSection(mutex);
return 0;
@ -444,6 +486,7 @@ int32_t taosThreadMutexLock(TdThreadMutex *mutex) {
// }
int32_t taosThreadMutexTryLock(TdThreadMutex *mutex) {
OS_PARAM_CHECK(mutex);
#ifdef __USE_WIN_THREAD
if (TryEnterCriticalSection(mutex)) return 0;
return EBUSY;
@ -457,6 +500,7 @@ int32_t taosThreadMutexTryLock(TdThreadMutex *mutex) {
}
int32_t taosThreadMutexUnlock(TdThreadMutex *mutex) {
OS_PARAM_CHECK(mutex);
#ifdef __USE_WIN_THREAD
LeaveCriticalSection(mutex);
return 0;
@ -474,6 +518,7 @@ int32_t taosThreadMutexAttrDestroy(TdThreadMutexAttr *attr) {
#ifdef __USE_WIN_THREAD
return 0;
#else
OS_PARAM_CHECK(attr);
int32_t code = pthread_mutexattr_destroy(attr);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -484,10 +529,12 @@ int32_t taosThreadMutexAttrDestroy(TdThreadMutexAttr *attr) {
}
int32_t taosThreadMutexAttrGetPshared(const TdThreadMutexAttr *attr, int32_t *pshared) {
OS_PARAM_CHECK(pshared);
#ifdef __USE_WIN_THREAD
if (pshared) *pshared = PTHREAD_PROCESS_PRIVATE;
return 0;
#else
OS_PARAM_CHECK(attr);
int32_t code = pthread_mutexattr_getpshared(attr, pshared);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -502,10 +549,12 @@ int32_t taosThreadMutexAttrGetPshared(const TdThreadMutexAttr *attr, int32_t *ps
// }
int32_t taosThreadMutexAttrGetType(const TdThreadMutexAttr *attr, int32_t *kind) {
OS_PARAM_CHECK(kind);
#ifdef __USE_WIN_THREAD
if (kind) *kind = PTHREAD_MUTEX_NORMAL;
return 0;
#else
OS_PARAM_CHECK(attr);
int32_t code = pthread_mutexattr_gettype(attr, kind);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -519,6 +568,7 @@ int32_t taosThreadMutexAttrInit(TdThreadMutexAttr *attr) {
#ifdef __USE_WIN_THREAD
return 0;
#else
OS_PARAM_CHECK(attr);
int32_t code = pthread_mutexattr_init(attr);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -532,6 +582,7 @@ int32_t taosThreadMutexAttrSetPshared(TdThreadMutexAttr *attr, int32_t pshared)
#ifdef __USE_WIN_THREAD
return 0;
#else
OS_PARAM_CHECK(attr);
int32_t code = pthread_mutexattr_setpshared(attr, pshared);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -549,6 +600,7 @@ int32_t taosThreadMutexAttrSetType(TdThreadMutexAttr *attr, int32_t kind) {
#ifdef __USE_WIN_THREAD
return 0;
#else
OS_PARAM_CHECK(attr);
int32_t code = pthread_mutexattr_settype(attr, kind);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -574,6 +626,7 @@ int32_t taosThreadRwlockDestroy(TdThreadRwlock *rwlock) {
*/
return 0;
#else
OS_PARAM_CHECK(rwlock);
int32_t code = pthread_rwlock_destroy(rwlock);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -584,6 +637,7 @@ int32_t taosThreadRwlockDestroy(TdThreadRwlock *rwlock) {
}
int32_t taosThreadRwlockInit(TdThreadRwlock *rwlock, const TdThreadRwlockAttr *attr) {
OS_PARAM_CHECK(rwlock);
#ifdef __USE_WIN_THREAD
memset(rwlock, 0, sizeof(*rwlock));
InitializeSRWLock(&rwlock->lock);
@ -599,6 +653,7 @@ int32_t taosThreadRwlockInit(TdThreadRwlock *rwlock, const TdThreadRwlockAttr *a
}
int32_t taosThreadRwlockRdlock(TdThreadRwlock *rwlock) {
OS_PARAM_CHECK(rwlock);
#ifdef __USE_WIN_THREAD
AcquireSRWLockShared(&rwlock->lock);
return 0;
@ -621,6 +676,7 @@ int32_t taosThreadRwlockRdlock(TdThreadRwlock *rwlock) {
// }
int32_t taosThreadRwlockTryRdlock(TdThreadRwlock *rwlock) {
OS_PARAM_CHECK(rwlock);
#ifdef __USE_WIN_THREAD
if (!TryAcquireSRWLockShared(&rwlock->lock)) return EBUSY;
return 0;
@ -635,6 +691,7 @@ int32_t taosThreadRwlockTryRdlock(TdThreadRwlock *rwlock) {
}
int32_t taosThreadRwlockTryWrlock(TdThreadRwlock *rwlock) {
OS_PARAM_CHECK(rwlock);
#ifdef __USE_WIN_THREAD
if (!TryAcquireSRWLockExclusive(&rwlock->lock)) return EBUSY;
atomic_store_8(&rwlock->excl, 1);
@ -650,6 +707,7 @@ int32_t taosThreadRwlockTryWrlock(TdThreadRwlock *rwlock) {
}
int32_t taosThreadRwlockUnlock(TdThreadRwlock *rwlock) {
OS_PARAM_CHECK(rwlock);
#ifdef __USE_WIN_THREAD
if (1 == atomic_val_compare_exchange_8(&rwlock->excl, 1, 0)) {
ReleaseSRWLockExclusive(&rwlock->lock);
@ -668,6 +726,7 @@ int32_t taosThreadRwlockUnlock(TdThreadRwlock *rwlock) {
}
int32_t taosThreadRwlockWrlock(TdThreadRwlock *rwlock) {
OS_PARAM_CHECK(rwlock);
#ifdef __USE_WIN_THREAD
AcquireSRWLockExclusive(&rwlock->lock);
atomic_store_8(&rwlock->excl, 1);
@ -686,6 +745,7 @@ int32_t taosThreadRwlockAttrDestroy(TdThreadRwlockAttr *attr) {
#ifdef __USE_WIN_THREAD
return 0;
#else
OS_PARAM_CHECK(attr);
int32_t code = pthread_rwlockattr_destroy(attr);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -696,6 +756,7 @@ int32_t taosThreadRwlockAttrDestroy(TdThreadRwlockAttr *attr) {
}
int32_t taosThreadRwlockAttrGetPshared(const TdThreadRwlockAttr *attr, int32_t *pshared) {
OS_PARAM_CHECK(pshared);
#ifdef __USE_WIN_THREAD
if (pshared) *pshared = PTHREAD_PROCESS_PRIVATE;
return 0;
@ -713,6 +774,7 @@ int32_t taosThreadRwlockAttrInit(TdThreadRwlockAttr *attr) {
#ifdef __USE_WIN_THREAD
return 0;
#else
OS_PARAM_CHECK(attr);
int32_t code = pthread_rwlockattr_init(attr);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -726,6 +788,7 @@ int32_t taosThreadRwlockAttrSetPshared(TdThreadRwlockAttr *attr, int32_t pshared
#ifdef __USE_WIN_THREAD
return 0;
#else
OS_PARAM_CHECK(attr);
int32_t code = pthread_rwlockattr_setpshared(attr, pshared);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -756,6 +819,7 @@ int32_t taosThreadSetCancelType(int32_t type, int32_t *oldtype) {
}
int32_t taosThreadSetSchedParam(TdThread thread, int32_t policy, const struct sched_param *param) {
OS_PARAM_CHECK(param);
int32_t code = pthread_setschedparam(thread, policy, param);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -765,6 +829,7 @@ int32_t taosThreadSetSchedParam(TdThread thread, int32_t policy, const struct sc
}
int32_t taosThreadSetSpecific(TdThreadKey key, const void *value) {
OS_PARAM_CHECK(value);
int32_t code = pthread_setspecific(key, value);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
@ -774,6 +839,7 @@ int32_t taosThreadSetSpecific(TdThreadKey key, const void *value) {
}
int32_t taosThreadSpinDestroy(TdThreadSpinlock *lock) {
OS_PARAM_CHECK(lock);
#ifdef TD_USE_SPINLOCK_AS_MUTEX
return pthread_mutex_destroy((pthread_mutex_t *)lock);
#else
@ -787,6 +853,7 @@ int32_t taosThreadSpinDestroy(TdThreadSpinlock *lock) {
}
int32_t taosThreadSpinInit(TdThreadSpinlock *lock, int32_t pshared) {
OS_PARAM_CHECK(lock);
#ifdef TD_USE_SPINLOCK_AS_MUTEX
if (pshared != 0) return TSDB_CODE_INVALID_PARA;
return pthread_mutex_init((pthread_mutex_t *)lock, NULL);
@ -801,6 +868,7 @@ int32_t taosThreadSpinInit(TdThreadSpinlock *lock, int32_t pshared) {
}
int32_t taosThreadSpinLock(TdThreadSpinlock *lock) {
OS_PARAM_CHECK(lock);
#ifdef TD_USE_SPINLOCK_AS_MUTEX
return pthread_mutex_lock((pthread_mutex_t *)lock);
#else
@ -814,6 +882,7 @@ int32_t taosThreadSpinLock(TdThreadSpinlock *lock) {
}
int32_t taosThreadSpinTrylock(TdThreadSpinlock *lock) {
OS_PARAM_CHECK(lock);
#ifdef TD_USE_SPINLOCK_AS_MUTEX
return pthread_mutex_trylock((pthread_mutex_t *)lock);
#else
@ -826,6 +895,7 @@ int32_t taosThreadSpinTrylock(TdThreadSpinlock *lock) {
}
int32_t taosThreadSpinUnlock(TdThreadSpinlock *lock) {
OS_PARAM_CHECK(lock);
#ifdef TD_USE_SPINLOCK_AS_MUTEX
return pthread_mutex_unlock((pthread_mutex_t *)lock);
#else
@ -843,6 +913,7 @@ void taosThreadTestCancel(void) {
}
void taosThreadClear(TdThread *thread) {
if (!thread) return;
(void)memset(thread, 0, sizeof(TdThread));
}

9
source/os/test/osSemaphoreTests.cpp
View File

@ -245,3 +245,12 @@ TEST(osSemaphoreTests, Performance4_2) {
(void)tsem2_destroy(&sem);
}
}
TEST(osSemaphoreTests, GetPID) {
#ifdef LINUX
pid_t pid = 0;
int32_t ret = taosGetPIdByName("osSemaphoreTest", &pid);
EXPECT_EQ(ret, 0);
EXPECT_EQ(pid, taosGetPId());
#endif
}

2
source/util/src/tconfig.c
View File

@ -25,7 +25,7 @@
#include "tunit.h"
#include "tutil.h"
#define CFG_NAME_PRINT_LEN 24
#define CFG_NAME_PRINT_LEN 32
#define CFG_SRC_PRINT_LEN 12
struct SConfig {

44
tests/army/query/function/generate_answer.sh Executable file
View File

@ -0,0 +1,44 @@
#!/bin/bash
set -e
set -x
# 检查是否传入了两个参数
# echo "参数个数: $#"
# if [ "$#" -ne 4 ]; then
# echo "使用方法: $0 -i <sqlfile> -o <query_result_file>"
# exit 1
# fi
# 读取传入的参数
while getopts "i:o:" opt; do
case $opt in
i)
sqlfile="$OPTARG"
;;
o)
query_result_file="$OPTARG"
;;
\?)
echo "无效选项: -$OPTARG" >&2
exit 1
;;
:)
echo "选项 -$OPTARG 需要一个参数." >&2
exit 1
;;
esac
done
# 删除sqlfile文件中每一行末尾的分号和空格
sed -i 's/;\s*$//' "$sqlfile"
# 执行SQL文件并生成query_result_file文件
taos -f "$sqlfile" | grep -v 'Query OK' | grep -v 'Copyright' | grep -v 'Welcome to the TDengine Command' > "$query_result_file"
# echo $(cat "$query_result_file")
# echo "1"
# sed -i 's/ ([^()]*)$//' "$query_result_file"
# echo "1"
# 打印输入的文件名
echo "输入的文件: $sqlfile"
# 打印输出的文件名
echo "输出的文件: $query_result_file"

2
tests/ci/Dockerfile
View File

@ -7,7 +7,7 @@ RUN apt-get install -y locales psmisc sudo tree libgeos-dev libgflags2.2 libgfl
RUN sed -i 's/# en_US.UTF-8/en_US.UTF-8/' /etc/locale.gen && locale-gen
RUN pip3 config set global.index-url http://admin:123456@192.168.0.212:3141/admin/dev/+simple/
RUN pip3 config set global.trusted-host 192.168.0.212
RUN pip3 install taospy==2.7.15 taos-ws-py==0.3.1 pandas psutil fabric2 requests faker simplejson toml pexpect tzlocal distro decorator loguru hyperloglog
RUN pip3 install taospy==2.7.16 taos-ws-py==0.3.3 pandas psutil fabric2 requests faker simplejson toml pexpect tzlocal distro decorator loguru hyperloglog
ENV LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LC_ALL=en_US.UTF-8
RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9
RUN add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/'

2
tests/parallel_test/cases.task
View File

@ -329,6 +329,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3311.py
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3821.py
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-5130.py
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-5580.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/balance_vgroups_r1.py -N 6
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShell.py
@ -362,6 +363,7 @@
,,n,system-test,python3 ./test.py -f 0-others/tag_index_basic.py
,,n,system-test,python3 ./test.py -f 0-others/udfpy_main.py
,,n,system-test,python3 ./test.py -N 3 -f 0-others/walRetention.py
,,n,system-test,python3 ./test.py -f 0-others/wal_level_skip.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroup.py -N 3 -n 1
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroupWal.py -N 3 -n 1
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroup.py -N 3 -n 3

4
tests/parallel_test/container_build.sh
View File

@ -83,7 +83,7 @@ docker run \
-v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \
-v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
-v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \
--rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0 -DCMAKE_EXPORT_COMPILE_COMMANDS=1 ;make -j|| exit 1"
--rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0 -DCMAKE_EXPORT_COMPILE_COMMANDS=1 ;make -j|| exit 1"
# -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \
if [[ -d ${WORKDIR}/debugNoSan ]] ;then
@ -137,7 +137,7 @@ docker run \
-v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \
-v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
-v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \
--rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=false -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true -DCMAKE_BUILD_TYPE=Debug -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j|| exit 1 "
--rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=false -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true -DCMAKE_BUILD_TYPE=Debug -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j|| exit 1 "
mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan

13
tests/parallel_test/run_case.sh
View File

@ -76,16 +76,9 @@ ulimit -c unlimited
md5sum /usr/lib/libtaos.so.1
md5sum /home/TDinternal/debug/build/lib/libtaos.so
#define taospy 2.7.16
pip3 list|grep taospy
pip3 uninstall taospy -y
pip3 install --default-timeout=120 taospy==2.7.16
#define taos-ws-py 0.3.1
pip3 list|grep taos-ws-py
pip3 uninstall taos-ws-py -y
pip3 install --default-timeout=600 taos-ws-py==0.3.3
#get python connector and update: taospy 2.7.16 taos-ws-py 0.3.3
pip3 install taospy==2.7.16
pip3 install taos-ws-py==0.3.3
$TIMEOUT_CMD $cmd
RET=$?
echo "cmd exit code: $RET"

3
tests/pytest/util/dnodes.py
View File

@ -594,8 +594,7 @@ class TDDnode:
def forcestop(self):
if self.asan:
stopCmd = "%s -s stop -n dnode%d -x SIGKILL" + \
(self.execPath, self.index)
stopCmd = "%s -s stop -n dnode%d -x SIGKILL" % (self.execPath, self.index)
tdLog.info("execute script: " + stopCmd)
os.system(stopCmd)
return

170
tests/system-test/0-others/wal_level_skip.py Normal file
View File

@ -0,0 +1,170 @@
import sys
import taos
import os
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql,replicaVar=1):
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor())
def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__))
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
if ("taosd" in files or "taosd.exe" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root) - len("/build/bin")]
break
return buildPath
def preData(self):
tdSql.execute("drop database if exists db0;")
tdSql.execute("create database db0 KEEP 30 vgroups 1 buffer 3 wal_level 0;")
tdSql.execute("create table if not exists db0.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned);")
tdSql.execute("create table db0.ct1 using db0.stb tags(1000);")
tdSql.execute("create table db0.ct2 using db0.stb tags(2000);")
tdSql.execute("create table if not exists db0.ntb (ts timestamp, c1 int, c2 float, c3 double) ;")
tdSql.query("show db0.stables;")
tdSql.execute("insert into db0.ct1 values(now+0s, 10, 2.0, 3.0);")
tdSql.execute("insert into db0.ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 13, 2.3, 3.3);")
tdSql.execute("insert into db0.ntb values(now+2s, 10, 2.0, 3.0);")
def insertData(self):
tdSql.execute("insert into db0.ct1 values(now+0s, 10, 2.0, 3.0);")
tdSql.execute("insert into db0.ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 13, 2.3, 3.3);")
tdSql.execute("insert into db0.ntb values(now+2s, 10, 2.0, 3.0);")
def createSubTableAndInsertData(self):
tdSql.execute("create table db0.ct1 using db0.stb tags(1000);")
tdSql.execute("create table db0.ct2 using db0.stb tags(2000);")
tdSql.execute("create table if not exists db0.ntb (ts timestamp, c1 int, c2 float, c3 double) ;")
tdSql.execute("insert into db0.ct1 values(now+0s, 10, 2.0, 3.0);")
tdSql.execute("insert into db0.ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 13, 2.3, 3.3);")
tdSql.execute("insert into db0.ntb values(now+2s, 10, 2.0, 3.0);")
def alterWalLevel(self,level):
tdSql.execute("alter database db0 wal_level %d;"%level)
def run(self):
tdSql.prepare()
tdLog.info("-----------test for stop taosd before alter wal level-----------")
tdLog.info("create database wal_level = 0 and insert data")
self.preData()
tdDnodes.stop(1)
time.sleep(2)
tdLog.info("restart taosd")
tdDnodes.start(1)
tdLog.info(" alter wal level from 0 to 1")
self.alterWalLevel(1)
self.insertData()
tdDnodes.stop(1)
tdDnodes.start(1)
tdLog.info("create database wal_level = 0 and insert data")
self.preData()
tdDnodes.stop(1)
time.sleep(2)
tdLog.info("restart taosd")
tdDnodes.start(1)
tdLog.info(" alter wal level from 0 to 2")
self.alterWalLevel(2)
self.insertData()
tdDnodes.forcestop(1)
tdDnodes.start(1)
tdLog.info("-----------test for kill taosd before alter wal level-----------")
tdLog.info("create database wal_level = 0 and insert data")
self.preData()
tdDnodes.forcestop(1)
time.sleep(2)
tdLog.info("restart taosd")
tdDnodes.start(1)
tdLog.info(" alter wal level from 0 to 1")
self.alterWalLevel(1)
tdDnodes.forcestop(1)
tdDnodes.start(1)
tdLog.info("create database wal_level = 0 and insert data")
self.preData()
tdDnodes.forcestop(1)
time.sleep(2)
tdLog.info("restart taosd")
tdDnodes.start(1)
tdLog.info(" alter wal level from 0 to 2")
self.alterWalLevel(2)
tdDnodes.forcestop(1)
tdDnodes.start(1)
tdLog.info("-----------test for stop taosd after alter wal level-----------")
tdLog.info("create database wal_level = 0 and insert data")
self.preData()
tdLog.info(" alter wal level from 0 to 1")
self.alterWalLevel(1)
time.sleep(1)
self.insertData()
tdDnodes.stop(1)
time.sleep(2)
tdLog.info("restart taosd")
tdDnodes.start(1)
tdLog.info("create database wal_level = 0 and insert data")
self.preData()
tdLog.info(" alter wal level from 0 to 2")
self.alterWalLevel(2)
time.sleep(1)
self.insertData()
tdDnodes.stop(1)
time.sleep(2)
tdLog.info("restart taosd")
tdDnodes.start(1)
tdLog.info("-----------test for kill taosd after alter wal level-----------")
tdLog.info("create database wal_level = 0 and insert data")
self.preData()
tdLog.info(" alter wal level from 0 to 1")
self.alterWalLevel(1)
time.sleep(1)
self.insertData()
tdDnodes.forcestop(1)
time.sleep(2)
tdLog.info("restart taosd")
tdDnodes.start(1)
tdLog.info("create database wal_level = 0 and insert data")
self.preData()
tdLog.info(" alter wal level from 0 to 2")
self.alterWalLevel(2)
time.sleep(1)
self.insertData()
tdDnodes.forcestop(1)
time.sleep(2)
tdLog.info("restart taosd")
tdDnodes.start(1)
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

2
tests/system-test/2-query/agg_group_NotReturnValue.py
View File

@ -1590,12 +1590,10 @@ class TDTestCase(TDTestCase):
self.modify_tables()
tdSql.execute('alter local "countAlwaysReturnValue" "0"')
for i in range(2):
self.tag_count_all()
self.tbname_count_all()
self.tbname_agg_all()
endTime = time.time()
print("total time %ds" % (endTime - startTime))

118
tests/system-test/99-TDcase/TS-5580.py Normal file
View File

@ -0,0 +1,118 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import time
from util.log import *
from util.cases import *
from util.sql import *
from util.common import *
from util.sqlset import *
class TDTestCase:
updatecfgDict = {'qDebugFlag':135 , 'mDebugFlag':135}
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), True)
self.setsql = TDSetSql()
self.dbname = 'db'
self.stbname = 'stb'
self.binary_length = 20 # the length of binary for column_dict
self.nchar_length = 20 # the length of nchar for column_dict
self.ts = 1537146000000
self.column_dict = {
'ts' : 'timestamp',
'col1': 'tinyint',
'col2': 'smallint',
'col3': 'int',
'col4': 'bigint',
'col5': 'float',
'col6': 'double',
'col7': 'double',
'col8': 'double',
'col9': 'double',
'col10': 'double',
'col11': 'double',
'col12': 'double',
'col13': 'double',
'col14': 'double',
'col15': 'double',
'col16': 'double',
'col17': 'double',
'col18': 'double',
'col19': 'double'
}
self.tbnum = 500
self.rowNum = 10
self.tag_dict = {
't0':'int',
't1':'bigint',
't2':'float',
't3':'double',
't4':'bool',
't5':'bool',
't6':'bool',
't7':'bool',
't8':'bool',
't9':'bool',
't10':'bool',
't11':'bool',
't12':'bool',
't13':'bool',
't14':'bool',
't15':'bool',
't16':'bool',
't17':'bool',
't18':'bool',
't19':'bool',
}
self.tag_values = [
f'1','1','1','1','true','true','true','true','true','true','true','true','true','true','true','true','true',
'true','true','true'
]
def prepare_data(self):
tdSql.execute(f"create database if not exists {self.dbname} vgroups 2")
tdSql.execute(f'use {self.dbname}')
tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict))
for i in range(self.tbnum):
tdSql.execute(f"create table {self.stbname}_{i} using {self.stbname} tags({self.tag_values[0]}, {self.tag_values[1]}, "
f"{self.tag_values[2]}, {self.tag_values[3]}, {self.tag_values[4]}, {self.tag_values[5]}, "
f"{self.tag_values[6]}, {self.tag_values[7]}, {self.tag_values[8]}, {self.tag_values[9]}, "
f"{self.tag_values[10]}, {self.tag_values[11]}, {self.tag_values[12]}, {self.tag_values[13]}, "
f"{self.tag_values[14]}, {self.tag_values[15]}, {self.tag_values[16]}, {self.tag_values[17]}, "
f"{self.tag_values[18]}, {self.tag_values[19]})")
def test_query_ins_tags(self):
for i in range(self.tbnum):
sql = f'select tag_name, tag_value from information_schema.ins_tags where table_name = "{self.stbname}_{i}"'
tdSql.query(sql)
tdSql.checkRows(20)
def test_query_ins_columns(self):
for i in range(self.tbnum):
sql = f'select col_name from information_schema.ins_columns where table_name = "{self.stbname}_{i}"'
tdSql.query(sql)
tdSql.checkRows(20)
def run(self):
self.prepare_data()
self.test_query_ins_tags()
self.test_query_ins_columns()
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())