xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #22020 from taosdata/fix/liaohj 

other: merge main
This commit is contained in:
Haojun Liao 2023-07-13 17:55:28 +08:00 committed by GitHub
parent f4249234ef 0cc7dca3e3
commit e0ea0f2df4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
109 changed files with 3533 additions and 893 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmake/taostools_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taos-tools # taos-tools
ExternalProject_Add(taos-tools ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG 3.0 GIT_TAG main
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR "" BINARY_DIR ""
#BUILD_IN_SOURCE TRUE #BUILD_IN_SOURCE TRUE

2
docs/en/12-taos-sql/02-database.md
View File

@ -41,7 +41,7 @@ database_option: {
## Parameters ## Parameters
- BUFFER: specifies the size (in MB) of the write buffer for each vnode. Enter a value between 3 and 16384. The default value is 96. - BUFFER: specifies the size (in MB) of the write buffer for each vnode. Enter a value between 3 and 16384. The default value is 256.
- CACHEMODEL: specifies how the latest data in subtables is stored in the cache. The default value is none. - CACHEMODEL: specifies how the latest data in subtables is stored in the cache. The default value is none.
- none: The latest data is not cached. - none: The latest data is not cached.
- last_row: The last row of each subtable is cached. This option significantly improves the performance of the LAST_ROW function. - last_row: The last row of each subtable is cached. This option significantly improves the performance of the LAST_ROW function.

2
docs/en/12-taos-sql/19-limit.md
View File

@ -36,7 +36,7 @@ The following characters cannot occur in a password: single quotation marks ('),
- Maximum numbers of databases, STables, tables are dependent only on the system resources. - Maximum numbers of databases, STables, tables are dependent only on the system resources.
- The number of replicas can only be 1 or 3. - The number of replicas can only be 1 or 3.
- The maximum length of a username is 23 bytes. - The maximum length of a username is 23 bytes.
- The maximum length of a password is 128 bytes. - The maximum length of a password is 31 bytes.
- The maximum number of rows depends on system resources. - The maximum number of rows depends on system resources.
- The maximum number of vnodes in a database is 1024. - The maximum number of vnodes in a database is 1024.

2
docs/en/12-taos-sql/25-grant.md
View File

@ -16,7 +16,7 @@ This statement creates a user account.
The maximum length of user_name is 23 bytes. The maximum length of user_name is 23 bytes.
The maximum length of password is 128 bytes. The password can include leters, digits, and special characters excluding single quotation marks, double quotation marks, backticks, backslashes, and spaces. The password cannot be empty. The maximum length of password is 31 bytes. The password can include leters, digits, and special characters excluding single quotation marks, double quotation marks, backticks, backslashes, and spaces. The password cannot be empty.
`SYSINFO` indicates whether the user is allowed to view system information. `1` means allowed, `0` means not allowed. System information includes server configuration, dnode, vnode, storage. The default value is `1`. `SYSINFO` indicates whether the user is allowed to view system information. `1` means allowed, `0` means not allowed. System information includes server configuration, dnode, vnode, storage. The default value is `1`.

19
docs/en/12-taos-sql/27-index.md
View File

@ -28,6 +28,24 @@ Performs pre-aggregation on the specified column over the time window defined by
- WATERMARK: Enter a value between 0ms and 900000ms. The most precise unit supported is milliseconds. The default value is 5 seconds. This option can be used only on supertables. - WATERMARK: Enter a value between 0ms and 900000ms. The most precise unit supported is milliseconds. The default value is 5 seconds. This option can be used only on supertables.
- MAX_DELAY: Enter a value between 1ms and 900000ms. The most precise unit supported is milliseconds. The default value is the value of interval provided that it does not exceed 900000ms. This option can be used only on supertables. Note: Retain the default value if possible. Configuring a small MAX_DELAY may cause results to be frequently pushed, affecting storage and query performance. - MAX_DELAY: Enter a value between 1ms and 900000ms. The most precise unit supported is milliseconds. The default value is the value of interval provided that it does not exceed 900000ms. This option can be used only on supertables. Note: Retain the default value if possible. Configuring a small MAX_DELAY may cause results to be frequently pushed, affecting storage and query performance.
```sql
DROP DATABASE IF EXISTS d0;
CREATE DATABASE d0;
USE d0;
CREATE TABLE IF NOT EXISTS st1 (ts timestamp, c1 int, c2 float, c3 double) TAGS (t1 int unsigned);
CREATE TABLE ct1 USING st1 TAGS(1000);
CREATE TABLE ct2 USING st1 TAGS(2000);
INSERT INTO ct1 VALUES(now+0s, 10, 2.0, 3.0);
INSERT INTO ct1 VALUES(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 13, 2.3, 3.3);
CREATE SMA INDEX sma_index_name1 ON st1 FUNCTION(max(c1),max(c2),min(c1)) INTERVAL(5m,10s) SLIDING(5m) WATERMARK 5s MAX_DELAY 1m;
-- query from SMA Index
ALTER LOCAL 'querySmaOptimize' '1';
SELECT max(c2),min(c1) FROM st1 INTERVAL(5m,10s) SLIDING(5m);
SELECT _wstart,_wend,_wduration,max(c2),min(c1) FROM st1 INTERVAL(5m,10s) SLIDING(5m);
-- query from raw data
ALTER LOCAL 'querySmaOptimize' '0';
```
### FULLTEXT Indexing ### FULLTEXT Indexing
Creates a text index for the specified column. FULLTEXT indexing improves performance for queries with text filtering. The index_option syntax is not supported for FULLTEXT indexing. FULLTEXT indexing is supported for JSON tag columns only. Multiple columns cannot be indexed together. However, separate indices can be created for each column. Creates a text index for the specified column. FULLTEXT indexing improves performance for queries with text filtering. The index_option syntax is not supported for FULLTEXT indexing. FULLTEXT indexing is supported for JSON tag columns only. Multiple columns cannot be indexed together. However, separate indices can be created for each column.
@ -41,7 +59,6 @@ DROP INDEX index_name;
## View Indices ## View Indices
````sql ````sql
```sql
SHOW INDEXES FROM tbl_name [FROM db_name]; SHOW INDEXES FROM tbl_name [FROM db_name];
SHOW INDEXES FROM [db_name.]tbl_name ; SHOW INDEXES FROM [db_name.]tbl_name ;
```` ````

23
docs/en/14-reference/05-taosbenchmark.md
View File

@ -470,3 +470,26 @@ The configuration parameters for subscribing to a super table are set in `super_
- **sql**: The SQL command to be executed. For the query SQL of super table, keep "xxxx" in the SQL command. The program will automatically replace it with all the sub-table names of the super table. - **sql**: The SQL command to be executed. For the query SQL of super table, keep "xxxx" in the SQL command. The program will automatically replace it with all the sub-table names of the super table.
Replace it with all the sub-table names in the super table. Replace it with all the sub-table names in the super table.
- **result**: The file to save the query result. If not specified, taosBenchmark will not save result. - **result**: The file to save the query result. If not specified, taosBenchmark will not save result.
#### data type on taosBenchmark
| # | **TDengine** | **taosBenchmark**
| --- | :----------------: | :---------------:
| 1 | TIMESTAMP | timestamp
| 2 | INT | int
| 3 | INT UNSIGNED | uint
| 4 | BIGINT | bigint
| 5 | BIGINT UNSIGNED | ubigint
| 6 | FLOAT | float
| 7 | DOUBLE | double
| 8 | BINARY | binary
| 9 | SMALLINT | smallint
| 10 | SMALLINT UNSIGNED | usmallint
| 11 | TINYINT | tinyint
| 12 | TINYINT UNSIGNED | utinyint
| 13 | BOOL | bool
| 14 | NCHAR | nchar
| 15 | VARCHAR | varchar
| 15 | JSON | json
note：Lowercase characters must be used on taosBenchmark datatype

37
docs/en/20-third-party/14-dbeaver.md vendored
View File

@ -12,50 +12,25 @@ To use DBeaver to manage TDengine, you need to prepare the following:
- Install DBeaver. DBeaver supports mainstream operating systems including Windows, macOS, and Linux. Please make sure you download and install the correct version (23.1.1+) and platform package. Please refer to the [official DBeaver documentation](https://github.com/dbeaver/dbeaver/wiki/Installation) for detailed installation steps. - Install DBeaver. DBeaver supports mainstream operating systems including Windows, macOS, and Linux. Please make sure you download and install the correct version (23.1.1+) and platform package. Please refer to the [official DBeaver documentation](https://github.com/dbeaver/dbeaver/wiki/Installation) for detailed installation steps.
- If you use an on-premises TDengine cluster, please make sure that TDengine and taosAdapter are deployed and running properly. For detailed information, please refer to the taosAdapter User Manual. - If you use an on-premises TDengine cluster, please make sure that TDengine and taosAdapter are deployed and running properly. For detailed information, please refer to the taosAdapter User Manual.
- If you use TDengine Cloud, please [register](https://cloud.tdengine.com/) for an account.
## Usage ## Use DBeaver to access on-premises TDengine cluster
### Use DBeaver to access on-premises TDengine cluster
1. Start the DBeaver application, click the button or menu item to choose **New Database Connection**, and then select **TDengine** in the **Timeseries** category. 1. Start the DBeaver application, click the button or menu item to choose **New Database Connection**, and then select **TDengine** in the **Timeseries** category.
![Connect TDengine with DBeaver](./dbeaver/dbeaver-connect-tdengine-en.webp) ![Connect TDengine with DBeaver](./dbeaver/dbeaver-connect-tdengine-en.webp)
2. Configure the TDengine connection by filling in the host address, port number, username, and password. If TDengine is deployed on the local machine, you are only required to fill in the username and password. The default username is root and the default password is taosdata. Click **Test Connection** to check whether the connection is workable. If you do not have the TDengine Java connector installed on the local machine, DBeaver will prompt you to download and install it. 2. Configure the TDengine connection by filling in the host address, port number, username, and password. If TDengine is deployed on the local machine, you are only required to fill in the username and password. The default username is root and the default password is taosdata. Click **Test Connection** to check whether the connection is workable. If you do not have the TDengine Java connector installed on the local machine, DBeaver will prompt you to download and install it.
![Configure the TDengine connection](./dbeaver/dbeaver-config-tdengine-en.webp)) ![Configure the TDengine connection](./dbeaver/dbeaver-config-tdengine-en.webp))
3. If the connection is successful, it will be displayed as shown in the following figure. If the connection fails, please check whether the TDengine service and taosAdapter are running correctly and whether the host address, port number, username, and password are correct. 3. If the connection is successful, it will be displayed as shown in the following figure. If the connection fails, please check whether the TDengine service and taosAdapter are running correctly and whether the host address, port number, username, and password are correct.
![Connection successful](./dbeaver/dbeaver-connect-tdengine-test-en.webp) ![Connection successful](./dbeaver/dbeaver-connect-tdengine-test-en.webp)
4. Use DBeaver to select databases and tables and browse your data stored in TDengine. 4. Use DBeaver to select databases and tables and browse your data stored in TDengine.
![Browse TDengine data with DBeaver](./dbeaver/dbeaver-browse-data-en.webp) ![Browse TDengine data with DBeaver](./dbeaver/dbeaver-browse-data-en.webp)
5. You can also manipulate TDengine data by executing SQL commands. 5. You can also manipulate TDengine data by executing SQL commands.
![Use SQL commands to manipulate TDengine data in DBeaver](./dbeaver/dbeaver-sql-execution-en.webp) ![Use SQL commands to manipulate TDengine data in DBeaver](./dbeaver/dbeaver-sql-execution-en.webp)
### Use DBeaver to access TDengine Cloud
1. Log in to the TDengine Cloud service, select **Programming** > **Java** in the management console, and then copy the string value of `TDENGINE_JDBC_URL` displayed in the **Config** section.
![Copy JDBC URL from TDengine Cloud](./dbeaver/tdengine-cloud-jdbc-dsn-en.webp)
2. Start the DBeaver application, click the button or menu item to choose **New Database Connection**, and then select **TDengine Cloud** in the **Timeseries** category.
![Connect TDengine Cloud with DBeaver](./dbeaver/dbeaver-connect-tdengine-cloud-en.webp)
3. Configure the TDengine Cloud connection by filling in the JDBC URL value. Click **Test Connection**. If you do not have the TDengine Java connector installed on the local machine, DBeaver will prompt you to download and install it. If the connection is successful, it will be displayed as shown in the following figure. If the connection fails, please check whether the TDengine Cloud service is running properly and whether the JDBC URL is correct.
![Configure the TDengine Cloud connection](./dbeaver/dbeaver-connect-tdengine-cloud-test-en.webp)
4. Use DBeaver to select databases and tables and browse your data stored in TDengine Cloud.
![Browse TDengine Cloud data with DBeaver](./dbeaver/dbeaver-browse-data-cloud-en.webp)
5. You can also manipulate TDengine Cloud data by executing SQL commands.
![Use SQL commands to manipulate TDengine Cloud data in DBeaver](./dbeaver/dbeaver-sql-execution-cloud-en.webp)

1
docs/zh/08-connector/30-python.mdx
View File

@ -1,4 +1,5 @@
--- ---
toc_max_heading_level: 4
sidebar_label: Python sidebar_label: Python
title: TDengine Python Connector title: TDengine Python Connector
description: "taospy 是 TDengine 的官方 Python 连接器。taospy 提供了丰富的 API， 使得 Python 应用可以很方便地使用 TDengine。tasopy 对 TDengine 的原生接口和 REST 接口都进行了封装， 分别对应 tasopy 的两个子模块：taos 和 taosrest。除了对原生接口和 REST 接口的封装，taospy 还提供了符合 Python 数据访问规范(PEP 249)的编程接口。这使得 taospy 和很多第三方工具集成变得简单，比如 SQLAlchemy 和 pandas" description: "taospy 是 TDengine 的官方 Python 连接器。taospy 提供了丰富的 API， 使得 Python 应用可以很方便地使用 TDengine。tasopy 对 TDengine 的原生接口和 REST 接口都进行了封装， 分别对应 tasopy 的两个子模块：taos 和 taosrest。除了对原生接口和 REST 接口的封装，taospy 还提供了符合 Python 数据访问规范(PEP 249)的编程接口。这使得 taospy 和很多第三方工具集成变得简单，比如 SQLAlchemy 和 pandas"

2
docs/zh/12-taos-sql/02-database.md
View File

@ -41,7 +41,7 @@ database_option: {
### 参数说明 ### 参数说明
- BUFFER: 一个 VNODE 写入内存池大小，单位为 MB，默认为 96，最小为 3，最大为 16384。 - BUFFER: 一个 VNODE 写入内存池大小，单位为 MB，默认为 256，最小为 3，最大为 16384。
- CACHEMODEL：表示是否在内存中缓存子表的最近数据。默认为 none。 - CACHEMODEL：表示是否在内存中缓存子表的最近数据。默认为 none。
- none：表示不缓存。 - none：表示不缓存。
- last_row：表示缓存子表最近一行数据。这将显著改善 LAST_ROW 函数的性能表现。 - last_row：表示缓存子表最近一行数据。这将显著改善 LAST_ROW 函数的性能表现。

2
docs/zh/12-taos-sql/19-limit.md
View File

@ -36,7 +36,7 @@ description: 合法字符集和命名中的限制规则
- 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制 - 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制
- 数据库的副本数只能设置为 1 或 3 - 数据库的副本数只能设置为 1 或 3
- 用户名的最大长度是 23 字节 - 用户名的最大长度是 23 字节
- 用户密码的最大长度是 128 字节 - 用户密码的最大长度是 31 字节
- 总数据行数取决于可用资源 - 总数据行数取决于可用资源
- 单个数据库的虚拟结点数上限为 1024 - 单个数据库的虚拟结点数上限为 1024

2
docs/zh/12-taos-sql/25-grant.md
View File

@ -16,7 +16,7 @@ CREATE USER use_name PASS 'password' [SYSINFO {1|0}];
use_name 最长为 23 字节。 use_name 最长为 23 字节。
password 最长为 128 字节，合法字符包括"a-zA-Z0-9!?$%^&*()_–+={[}]:;@~#|<,>.?/"，不可以出现单双引号、撇号、反斜杠和空格，且不可以为空。 password 最长为 31 字节，合法字符包括"a-zA-Z0-9!?$%^&*()_–+={[}]:;@~#|<,>.?/"，不可以出现单双引号、撇号、反斜杠和空格，且不可以为空。
SYSINFO 表示用户是否可以查看系统信息。1 表示可以查看，0 表示不可以查看。系统信息包括服务端配置信息、服务端各种节点信息（如 DNODE、QNODE等）、存储相关的信息等。默认为可以查看系统信息。 SYSINFO 表示用户是否可以查看系统信息。1 表示可以查看，0 表示不可以查看。系统信息包括服务端配置信息、服务端各种节点信息（如 DNODE、QNODE等）、存储相关的信息等。默认为可以查看系统信息。

19
docs/zh/12-taos-sql/27-index.md
View File

@ -28,6 +28,24 @@ functions:
- WATERMARK: 最小单位毫秒，取值范围 [0ms, 900000ms]，默认值为 5 秒，只可用于超级表。 - WATERMARK: 最小单位毫秒，取值范围 [0ms, 900000ms]，默认值为 5 秒，只可用于超级表。
- MAX_DELAY: 最小单位毫秒，取值范围 [1ms, 900000ms]，默认值为 interval 的值(但不能超过最大值)，只可用于超级表。注：不建议 MAX_DELAY 设置太小，否则会过于频繁的推送结果，影响存储和查询性能，如无特殊需求，取默认值即可。 - MAX_DELAY: 最小单位毫秒，取值范围 [1ms, 900000ms]，默认值为 interval 的值(但不能超过最大值)，只可用于超级表。注：不建议 MAX_DELAY 设置太小，否则会过于频繁的推送结果，影响存储和查询性能，如无特殊需求，取默认值即可。
```sql
DROP DATABASE IF EXISTS d0;
CREATE DATABASE d0;
USE d0;
CREATE TABLE IF NOT EXISTS st1 (ts timestamp, c1 int, c2 float, c3 double) TAGS (t1 int unsigned);
CREATE TABLE ct1 USING st1 TAGS(1000);
CREATE TABLE ct2 USING st1 TAGS(2000);
INSERT INTO ct1 VALUES(now+0s, 10, 2.0, 3.0);
INSERT INTO ct1 VALUES(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 13, 2.3, 3.3);
CREATE SMA INDEX sma_index_name1 ON st1 FUNCTION(max(c1),max(c2),min(c1)) INTERVAL(5m,10s) SLIDING(5m) WATERMARK 5s MAX_DELAY 1m;
-- 从 SMA 索引查询
ALTER LOCAL 'querySmaOptimize' '1';
SELECT max(c2),min(c1) FROM st1 INTERVAL(5m,10s) SLIDING(5m);
SELECT _wstart,_wend,_wduration,max(c2),min(c1) FROM st1 INTERVAL(5m,10s) SLIDING(5m);
-- 从原始数据查询
ALTER LOCAL 'querySmaOptimize' '0';
```
### FULLTEXT 索引 ### FULLTEXT 索引
对指定列建立文本索引，可以提升含有文本过滤的查询的性能。FULLTEXT 索引不支持 index_option 语法。现阶段只支持对 JSON 类型的标签列创建 FULLTEXT 索引。不支持多列联合索引，但可以为每个列分布创建 FULLTEXT 索引。 对指定列建立文本索引，可以提升含有文本过滤的查询的性能。FULLTEXT 索引不支持 index_option 语法。现阶段只支持对 JSON 类型的标签列创建 FULLTEXT 索引。不支持多列联合索引，但可以为每个列分布创建 FULLTEXT 索引。
@ -41,7 +59,6 @@ DROP INDEX index_name;
## 查看索引 ## 查看索引
````sql ````sql
```sql
SHOW INDEXES FROM tbl_name [FROM db_name]; SHOW INDEXES FROM tbl_name [FROM db_name];
SHOW INDEXES FROM [db_name.]tbl_name; SHOW INDEXES FROM [db_name.]tbl_name;
```` ````

26
docs/zh/14-reference/05-taosbenchmark.md
View File

@ -437,3 +437,29 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\)
- **sqls** ： - **sqls** ：
- **sql** : 执行的 SQL 命令，必填。 - **sql** : 执行的 SQL 命令，必填。
#### 配置文件中数据类型书写对照表
| # | **引擎** | **taosBenchmark**
| --- | :----------------: | :---------------:
| 1 | TIMESTAMP | timestamp
| 2 | INT | int
| 3 | INT UNSIGNED | uint
| 4 | BIGINT | bigint
| 5 | BIGINT UNSIGNED | ubigint
| 6 | FLOAT | float
| 7 | DOUBLE | double
| 8 | BINARY | binary
| 9 | SMALLINT | smallint
| 10 | SMALLINT UNSIGNED | usmallint
| 11 | TINYINT | tinyint
| 12 | TINYINT UNSIGNED | utinyint
| 13 | BOOL | bool
| 14 | NCHAR | nchar
| 15 | VARCHAR | varchar
| 15 | JSON | json
注意：taosBenchmark 配置文件中数据类型必须小写方可识别

40
docs/zh/20-third-party/13-dbeaver.md vendored
View File

@ -8,21 +8,16 @@ DBeaver 是一款流行的跨平台数据库管理工具，方便开发者、数
## 前置条件 ## 前置条件
### 安装 DBeaver
使用 DBeaver 管理 TDengine 需要以下几方面的准备工作。 使用 DBeaver 管理 TDengine 需要以下几方面的准备工作。
- 安装 DBeaver。DBeaver 支持主流操作系统包括 Windows、macOS 和 Linux。请注意[下载](https://dbeaver.io/download/)正确平台和版本（23.1.1+）的安装包。详细安装步骤请参考 [DBeaver 官方文档](https://github.com/dbeaver/dbeaver/wiki/Installation)。 - 安装 DBeaver。DBeaver 支持主流操作系统包括 Windows、macOS 和 Linux。请注意[下载](https://dbeaver.io/download/)正确平台和版本（23.1.1+）的安装包。详细安装步骤请参考 [DBeaver 官方文档](https://github.com/dbeaver/dbeaver/wiki/Installation)。
- 如果使用独立部署的 TDengine 集群，请确认 TDengine 正常运行，并且 taosAdapter 已经安装并正常运行，具体细节请参考 [taosAdapter 的使用手册](/reference/taosadapter)。 - 如果使用独立部署的 TDengine 集群，请确认 TDengine 正常运行，并且 taosAdapter 已经安装并正常运行，具体细节请参考 [taosAdapter 的使用手册](/reference/taosadapter)。
- 如果使用 TDengine Cloud，请[注册](https://cloud.taosdata.com/)相应账号。
## 使用步骤 ## 使用 DBeaver 访问内部部署的 TDengine
### 使用 DBeaver 访问内部部署的 TDengine
1. 启动 DBeaver 应用，点击按钮或菜单项选择“连接到数据库”，然后在时间序列分类栏中选择 TDengine。 1. 启动 DBeaver 应用，点击按钮或菜单项选择“连接到数据库”，然后在时间序列分类栏中选择 TDengine。
![DBeaver 连接 TDengine](./dbeaver/dbeaver-connect-tdengine-zh.webp) ![DBeaver 连接 TDengine](./dbeaver/dbeaver-connect-tdengine-zh.webp)
2. 配置 TDengine 连接，填入主机地址、端口号、用户名和密码。如果 TDengine 部署在本机，可以只填用户名和密码，默认用户名为 root，默认密码为 taosdata。点击“测试连接”可以对连接是否可用进行测试。如果本机没有安装 TDengine Java 2. 配置 TDengine 连接，填入主机地址、端口号、用户名和密码。如果 TDengine 部署在本机，可以只填用户名和密码，默认用户名为 root，默认密码为 taosdata。点击“测试连接”可以对连接是否可用进行测试。如果本机没有安装 TDengine Java
连接器，DBeaver 会提示下载安装。 连接器，DBeaver 会提示下载安装。
@ -31,37 +26,12 @@ DBeaver 是一款流行的跨平台数据库管理工具，方便开发者、数
3. 连接成功将显示如下图所示。如果显示连接失败，请检查 TDengine 服务和 taosAdapter 是否正确运行，主机地址、端口号、用户名和密码是否正确。 3. 连接成功将显示如下图所示。如果显示连接失败，请检查 TDengine 服务和 taosAdapter 是否正确运行，主机地址、端口号、用户名和密码是否正确。
![连接成功](./dbeaver/dbeaver-connect-tdengine-test-zh.webp) ![连接成功](./dbeaver/dbeaver-connect-tdengine-test-zh.webp)
4. 使用 DBeaver 选择数据库和表可以浏览 TDengine 服务的数据。 4. 使用 DBeaver 选择数据库和表可以浏览 TDengine 服务的数据。
![DBeaver 浏览 TDengine 数据](./dbeaver/dbeaver-browse-data-zh.webp) ![DBeaver 浏览 TDengine 数据](./dbeaver/dbeaver-browse-data-zh.webp)
5. 也可以通过执行 SQL 命令的方式对 TDengine 数据进行操作。 5. 也可以通过执行 SQL 命令的方式对 TDengine 数据进行操作。
![DBeaver SQL 命令](./dbeaver/dbeaver-sql-execution-zh.webp) ![DBeaver SQL 命令](./dbeaver/dbeaver-sql-execution-zh.webp)
### 使用 DBeaver 访问 TDengine Cloud
1. 登录 TDengine Cloud 服务，在管理界面中选择“编程”和“Java”，然后复制 TDENGINE_JDBC_URL 的字符串值。
![复制 TDengine Cloud DSN](./dbeaver/tdengine-cloud-jdbc-dsn-zh.webp)
2. 启动 DBeaver 应用，点击按钮或菜单项选择“连接到数据库”，然后在时间序列分类栏中选择 TDengine Cloud。
![DBeaver 连接 TDengine Cloud](./dbeaver/dbeaver-connect-tdengine-cloud-zh.webp)
3. 配置 TDengine Cloud 连接，填入 JDBC_URL 值。点击“测试连接”，如果本机没有安装 TDengine Java
连接器，DBeaver 会提示下载安装。连接成功将显示如下图所示。如果显示连接失败，请检查 TDengine Cloud 服务是否启动，JDBC_URL 是否正确。
![配置 TDengine Cloud 连接](./dbeaver/dbeaver-connect-tdengine-cloud-test-zh.webp)
4. 使用 DBeaver 选择数据库和表可以浏览 TDengine Cloud 服务的数据。
![DBeaver 浏览 TDengine Cloud 数据](./dbeaver/dbeaver-browse-cloud-data-zh.webp)
5. 也可以通过执行 SQL 命令的方式对 TDengine Cloud 数据进行操作。
![DBeaver SQL 命令 操作 TDengine Cloud](./dbeaver/dbeaver-sql-execution-cloud-zh.webp)

6
examples/C#/taosdemo/README.md
View File

@ -36,7 +36,11 @@ dotnet build -c Release
## Usage ## Usage
``` ```
Usage: mono taosdemo.exe [OPTION...] Usage with mono:
$ mono taosdemo.exe [OPTION...]
Usage with dotnet:
Usage: .\bin\Release\net5.0\taosdemo.exe [OPTION...]
--help Show usage. --help Show usage.

4
examples/C#/taosdemo/taosdemo.cs
View File

@ -72,7 +72,7 @@ namespace TDengineDriver
{ {
if ("--help" == argv[i]) if ("--help" == argv[i])
{ {
Console.WriteLine("Usage: mono taosdemo.exe [OPTION...]"); Console.WriteLine("Usage: taosdemo.exe [OPTION...]");
Console.WriteLine(""); Console.WriteLine("");
HelpPrint("--help", "Show usage."); HelpPrint("--help", "Show usage.");
Console.WriteLine(""); Console.WriteLine("");
@ -305,7 +305,7 @@ namespace TDengineDriver
this.conn = TDengine.Connect(this.host, this.user, this.password, db, this.port); this.conn = TDengine.Connect(this.host, this.user, this.password, db, this.port);
if (this.conn == IntPtr.Zero) if (this.conn == IntPtr.Zero)
{ {
Console.WriteLine("Connect to TDengine failed"); Console.WriteLine("Connect to TDengine failed. Reason: {0}\n", TDengine.Error(0));
CleanAndExitProgram(1); CleanAndExitProgram(1);
} }
else else

18
include/common/tmsg.h
View File

@ -106,7 +106,6 @@ enum {
HEARTBEAT_KEY_DBINFO, HEARTBEAT_KEY_DBINFO,
HEARTBEAT_KEY_STBINFO, HEARTBEAT_KEY_STBINFO,
HEARTBEAT_KEY_TMQ, HEARTBEAT_KEY_TMQ,
HEARTBEAT_KEY_USER_PASSINFO,
}; };
typedef enum _mgmt_table { typedef enum _mgmt_table {
@ -636,6 +635,7 @@ typedef struct {
SEpSet epSet; SEpSet epSet;
int32_t svrTimestamp; int32_t svrTimestamp;
int32_t passVer; int32_t passVer;
int32_t authVer;
char sVer[TSDB_VERSION_LEN]; char sVer[TSDB_VERSION_LEN];
char sDetailVer[128]; char sDetailVer[128];
} SConnectRsp; } SConnectRsp;
@ -720,14 +720,6 @@ int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pR
int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp);
void tFreeSGetUserAuthRsp(SGetUserAuthRsp* pRsp); void tFreeSGetUserAuthRsp(SGetUserAuthRsp* pRsp);
typedef struct SUserPassVersion {
char user[TSDB_USER_LEN];
int32_t version;
} SUserPassVersion;
typedef SGetUserAuthReq SGetUserPassReq;
typedef SUserPassVersion SGetUserPassRsp;
/* /*
* for client side struct, only column id, type, bytes are necessary * for client side struct, only column id, type, bytes are necessary
* But for data in vnode side, we need all the following information. * But for data in vnode side, we need all the following information.
@ -1071,14 +1063,6 @@ int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp
int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp); int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp);
void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp); void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp);
typedef struct {
SArray* pArray; // Array of SGetUserPassRsp
} SUserPassBatchRsp;
int32_t tSerializeSUserPassBatchRsp(void* buf, int32_t bufLen, SUserPassBatchRsp* pRsp);
int32_t tDeserializeSUserPassBatchRsp(void* buf, int32_t bufLen, SUserPassBatchRsp* pRsp);
void tFreeSUserPassBatchRsp(SUserPassBatchRsp* pRsp);
typedef struct { typedef struct {
char db[TSDB_DB_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN];
STimeWindow timeRange; STimeWindow timeRange;

1
include/libs/parser/parser.h
View File

@ -58,6 +58,7 @@ typedef struct SParseContext {
bool isSuperUser; bool isSuperUser;
bool enableSysInfo; bool enableSysInfo;
bool async; bool async;
bool hasInvisibleCol;
const char* svrVer; const char* svrVer;
bool nodeOffline; bool nodeOffline;
SArray* pTableMetaPos; // sql table pos => catalog data pos SArray* pTableMetaPos; // sql table pos => catalog data pos

2
include/libs/transport/trpc.h
View File

@ -46,6 +46,7 @@ typedef struct SRpcHandleInfo {
int8_t noResp; // has response or not(default 0, 0: resp, 1: no resp) int8_t noResp; // has response or not(default 0, 0: resp, 1: no resp)
int8_t persistHandle; // persist handle or not int8_t persistHandle; // persist handle or not
int8_t hasEpSet; int8_t hasEpSet;
int32_t cliVer;
// app info // app info
void *ahandle; // app handle set by client void *ahandle; // app handle set by client
@ -83,6 +84,7 @@ typedef struct SRpcInit {
int32_t sessions; // number of sessions allowed int32_t sessions; // number of sessions allowed
int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS
int32_t idleTime; // milliseconds, 0 means idle timer is disabled int32_t idleTime; // milliseconds, 0 means idle timer is disabled
int32_t compatibilityVer;
int32_t retryMinInterval; // retry init interval int32_t retryMinInterval; // retry init interval
int32_t retryStepFactor; // retry interval factor int32_t retryStepFactor; // retry interval factor

1
include/util/taoserror.h
View File

@ -775,6 +775,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_TMQ_CONSUMER_ERROR TAOS_DEF_ERROR_CODE(0, 0x4003) #define TSDB_CODE_TMQ_CONSUMER_ERROR TAOS_DEF_ERROR_CODE(0, 0x4003)
#define TSDB_CODE_TMQ_TOPIC_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x4004) #define TSDB_CODE_TMQ_TOPIC_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x4004)
#define TSDB_CODE_TMQ_GROUP_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x4005) #define TSDB_CODE_TMQ_GROUP_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x4005)
#define TSDB_CODE_TMQ_SNAPSHOT_ERROR TAOS_DEF_ERROR_CODE(0, 0x4006)
// stream // stream
#define TSDB_CODE_STREAM_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x4100) #define TSDB_CODE_STREAM_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x4100)

4
packaging/checkPackageRuning.py
View File

@ -87,7 +87,7 @@ os.system("rm -rf /tmp/dumpdata/*")
# dump data out # dump data out
print("taosdump dump out data") print("taosdump dump out data")
os.system("taosdump -o /tmp/dumpdata -D test -y -h %s "%serverHost) os.system("taosdump -o /tmp/dumpdata -D test -h %s "%serverHost)
# drop database of test # drop database of test
print("drop database test") print("drop database test")
@ -95,7 +95,7 @@ os.system(" taos -s ' drop database test ;' -h %s "%serverHost)
# dump data in # dump data in
print("taosdump dump data in") print("taosdump dump data in")
os.system("taosdump -i /tmp/dumpdata -y -h %s "%serverHost) os.system("taosdump -i /tmp/dumpdata -h %s "%serverHost)
result = conn.query("SELECT count(*) from test.meters") result = conn.query("SELECT count(*) from test.meters")

6
source/client/inc/clientInt.h
View File

@ -63,7 +63,7 @@ typedef struct {
// statistics // statistics
int32_t reportCnt; int32_t reportCnt;
int32_t connKeyCnt; int32_t connKeyCnt;
int32_t passKeyCnt; // with passVer call back int8_t connHbFlag; // 0 init, 1 send req, 2 get resp
int64_t reportBytes; // not implemented int64_t reportBytes; // not implemented
int64_t startTime; int64_t startTime;
// ctl // ctl
@ -83,8 +83,9 @@ typedef struct {
int8_t threadStop; int8_t threadStop;
int8_t quitByKill; int8_t quitByKill;
TdThread thread; TdThread thread;
TdThreadMutex lock; // used when app init and cleanup TdThreadMutex lock; // used when app init and cleanup
SHashObj* appSummary; SHashObj* appSummary;
SHashObj* appHbHash; // key: clusterId
SArray* appHbMgrs; // SArray<SAppHbMgr*> one for each cluster SArray* appHbMgrs; // SArray<SAppHbMgr*> one for each cluster
FHbReqHandle reqHandle[CONN_TYPE__MAX]; FHbReqHandle reqHandle[CONN_TYPE__MAX];
FHbRspHandle rspHandle[CONN_TYPE__MAX]; FHbRspHandle rspHandle[CONN_TYPE__MAX];
@ -146,6 +147,7 @@ typedef struct STscObj {
int64_t id; // ref ID returned by taosAddRef int64_t id; // ref ID returned by taosAddRef
TdThreadMutex mutex; // used to protect the operation on db TdThreadMutex mutex; // used to protect the operation on db
int32_t numOfReqs; // number of sqlObj bound to this connection int32_t numOfReqs; // number of sqlObj bound to this connection
int32_t authVer;
SAppInstInfo* pAppInfo; SAppInstInfo* pAppInfo;
SHashObj* pRequests; SHashObj* pRequests;
SPassInfo passInfo; SPassInfo passInfo;

45
source/client/src/clientEnv.c
View File

@ -29,6 +29,7 @@
#include "trpc.h" #include "trpc.h"
#include "tsched.h" #include "tsched.h"
#include "ttime.h" #include "ttime.h"
#include "tversion.h"
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
#include "cus_name.h" #include "cus_name.h"
@ -111,7 +112,8 @@ static void deregisterRequest(SRequestObj *pRequest) {
atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1); atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1);
if (tsSlowLogScope & reqType) { if (tsSlowLogScope & reqType) {
taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 ", Duration:%" PRId64 "us, SQL:%s", taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 ", Duration:%" PRId64 "us, SQL:%s",
taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration, pRequest->sqlstr); taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration,
pRequest->sqlstr);
} }
} }
@ -175,6 +177,8 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) {
rpcInit.connLimitNum = connLimitNum; rpcInit.connLimitNum = connLimitNum;
rpcInit.timeToGetConn = tsTimeToGetAvailableConn; rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
void *pDnodeConn = rpcOpen(&rpcInit); void *pDnodeConn = rpcOpen(&rpcInit);
if (pDnodeConn == NULL) { if (pDnodeConn == NULL) {
tscError("failed to init connection to server"); tscError("failed to init connection to server");
@ -358,17 +362,16 @@ int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, ri
int32_t removeRequest(int64_t rid) { return taosRemoveRef(clientReqRefPool, rid); } int32_t removeRequest(int64_t rid) { return taosRemoveRef(clientReqRefPool, rid); }
void destroySubRequests(SRequestObj *pRequest) { void destroySubRequests(SRequestObj *pRequest) {
int32_t reqIdx = -1; int32_t reqIdx = -1;
SRequestObj *pReqList[16] = {NULL}; SRequestObj *pReqList[16] = {NULL};
uint64_t tmpRefId = 0; uint64_t tmpRefId = 0;
if (pRequest->relation.userRefId && pRequest->relation.userRefId != pRequest->self) { if (pRequest->relation.userRefId && pRequest->relation.userRefId != pRequest->self) {
return; return;
} }
SRequestObj* pTmp = pRequest; SRequestObj *pTmp = pRequest;
while (pTmp->relation.prevRefId) { while (pTmp->relation.prevRefId) {
tmpRefId = pTmp->relation.prevRefId; tmpRefId = pTmp->relation.prevRefId;
pTmp = acquireRequest(tmpRefId); pTmp = acquireRequest(tmpRefId);
@ -376,9 +379,9 @@ void destroySubRequests(SRequestObj *pRequest) {
pReqList[++reqIdx] = pTmp; pReqList[++reqIdx] = pTmp;
releaseRequest(tmpRefId); releaseRequest(tmpRefId);
} else { } else {
tscError("0x%" PRIx64 ", prev req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pTmp->self, tscError("0x%" PRIx64 ", prev req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pTmp->self, tmpRefId,
tmpRefId, pTmp->requestId); pTmp->requestId);
break; break;
} }
} }
@ -391,16 +394,15 @@ void destroySubRequests(SRequestObj *pRequest) {
pTmp = acquireRequest(tmpRefId); pTmp = acquireRequest(tmpRefId);
if (pTmp) { if (pTmp) {
tmpRefId = pTmp->relation.nextRefId; tmpRefId = pTmp->relation.nextRefId;
removeRequest(pTmp->self); removeRequest(pTmp->self);
releaseRequest(pTmp->self); releaseRequest(pTmp->self);
} else { } else {
tscError("0x%" PRIx64 " is not there", tmpRefId); tscError("0x%" PRIx64 " is not there", tmpRefId);
break; break;
} }
} }
} }
void doDestroyRequest(void *p) { void doDestroyRequest(void *p) {
if (NULL == p) { if (NULL == p) {
return; return;
@ -412,7 +414,7 @@ void doDestroyRequest(void *p) {
tscTrace("begin to destroy request %" PRIx64 " p:%p", reqId, pRequest); tscTrace("begin to destroy request %" PRIx64 " p:%p", reqId, pRequest);
destroySubRequests(pRequest); destroySubRequests(pRequest);
taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self)); taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self));
schedulerFreeJob(&pRequest->body.queryJob, 0); schedulerFreeJob(&pRequest->body.queryJob, 0);
@ -473,15 +475,15 @@ void taosStopQueryImpl(SRequestObj *pRequest) {
} }
void stopAllQueries(SRequestObj *pRequest) { void stopAllQueries(SRequestObj *pRequest) {
int32_t reqIdx = -1; int32_t reqIdx = -1;
SRequestObj *pReqList[16] = {NULL}; SRequestObj *pReqList[16] = {NULL};
uint64_t tmpRefId = 0; uint64_t tmpRefId = 0;
if (pRequest->relation.userRefId && pRequest->relation.userRefId != pRequest->self) { if (pRequest->relation.userRefId && pRequest->relation.userRefId != pRequest->self) {
return; return;
} }
SRequestObj* pTmp = pRequest; SRequestObj *pTmp = pRequest;
while (pTmp->relation.prevRefId) { while (pTmp->relation.prevRefId) {
tmpRefId = pTmp->relation.prevRefId; tmpRefId = pTmp->relation.prevRefId;
pTmp = acquireRequest(tmpRefId); pTmp = acquireRequest(tmpRefId);
@ -489,9 +491,9 @@ void stopAllQueries(SRequestObj *pRequest) {
pReqList[++reqIdx] = pTmp; pReqList[++reqIdx] = pTmp;
releaseRequest(tmpRefId); releaseRequest(tmpRefId);
} else { } else {
tscError("0x%" PRIx64 ", prev req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pTmp->self, tscError("0x%" PRIx64 ", prev req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pTmp->self, tmpRefId,
tmpRefId, pTmp->requestId); pTmp->requestId);
break; break;
} }
} }
@ -510,12 +512,11 @@ void stopAllQueries(SRequestObj *pRequest) {
releaseRequest(pTmp->self); releaseRequest(pTmp->self);
} else { } else {
tscError("0x%" PRIx64 " is not there", tmpRefId); tscError("0x%" PRIx64 " is not there", tmpRefId);
break; break;
} }
} }
} }
void crashReportThreadFuncUnexpectedStopped(void) { atomic_store_32(&clientStop, -1); } void crashReportThreadFuncUnexpectedStopped(void) { atomic_store_32(&clientStop, -1); }
static void *tscCrashReportThreadFp(void *param) { static void *tscCrashReportThreadFp(void *param) {

209
source/client/src/clientHb.c
View File

@ -22,10 +22,10 @@
typedef struct { typedef struct {
union { union {
struct { struct {
int64_t clusterId; SAppHbMgr *pAppHbMgr;
int32_t passKeyCnt; int64_t clusterId;
int32_t passVer; int32_t reqCnt;
int32_t reqCnt; int8_t connHbFlag;
}; };
}; };
} SHbParam; } SHbParam;
@ -34,12 +34,14 @@ static SClientHbMgr clientHbMgr = {0};
static int32_t hbCreateThread(); static int32_t hbCreateThread();
static void hbStopThread(); static void hbStopThread();
static int32_t hbUpdateUserAuthInfo(SAppHbMgr *pAppHbMgr, SUserAuthBatchRsp *batchRsp);
static int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { return 0; } static int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { return 0; }
static int32_t hbMqHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; } static int32_t hbMqHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; }
static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog,
SAppHbMgr *pAppHbMgr) {
int32_t code = 0; int32_t code = 0;
SUserAuthBatchRsp batchRsp = {0}; SUserAuthBatchRsp batchRsp = {0};
@ -56,54 +58,68 @@ static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SC
catalogUpdateUserAuthInfo(pCatalog, rsp); catalogUpdateUserAuthInfo(pCatalog, rsp);
} }
if (numOfBatchs > 0) hbUpdateUserAuthInfo(pAppHbMgr, &batchRsp);
atomic_val_compare_exchange_8(&pAppHbMgr->connHbFlag, 1, 2);
taosArrayDestroy(batchRsp.pArray); taosArrayDestroy(batchRsp.pArray);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t hbProcessUserPassInfoRsp(void *value, int32_t valueLen, SClientHbKey *connKey, SAppHbMgr *pAppHbMgr) { static int32_t hbUpdateUserAuthInfo(SAppHbMgr *pAppHbMgr, SUserAuthBatchRsp *batchRsp) {
int32_t code = 0; uint64_t clusterId = pAppHbMgr->pAppInstInfo->clusterId;
int32_t numOfBatchs = 0; for (int i = 0; i < TARRAY_SIZE(clientHbMgr.appHbMgrs); ++i) {
SUserPassBatchRsp batchRsp = {0}; SAppHbMgr *hbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i);
if (tDeserializeSUserPassBatchRsp(value, valueLen, &batchRsp) != 0) { if (!hbMgr || hbMgr->pAppInstInfo->clusterId != clusterId) {
code = TSDB_CODE_INVALID_MSG;
return code;
}
numOfBatchs = taosArrayGetSize(batchRsp.pArray);
SClientHbReq *pReq = NULL;
while ((pReq = taosHashIterate(pAppHbMgr->activeInfo, pReq))) {
STscObj *pTscObj = (STscObj *)acquireTscObj(pReq->connKey.tscRid);
if (!pTscObj) {
continue;
}
SPassInfo *passInfo = &pTscObj->passInfo;
if (!passInfo->fp) {
releaseTscObj(pReq->connKey.tscRid);
continue; continue;
} }
for (int32_t i = 0; i < numOfBatchs; ++i) { SClientHbReq *pReq = NULL;
SGetUserPassRsp *rsp = taosArrayGet(batchRsp.pArray, i); SGetUserAuthRsp *pRsp = NULL;
if (0 == strncmp(rsp->user, pTscObj->user, TSDB_USER_LEN)) { while ((pReq = taosHashIterate(hbMgr->activeInfo, pReq))) {
int32_t oldVer = atomic_load_32(&passInfo->ver); STscObj *pTscObj = (STscObj *)acquireTscObj(pReq->connKey.tscRid);
if (oldVer < rsp->version) { if (!pTscObj) {
atomic_store_32(&passInfo->ver, rsp->version); continue;
if (passInfo->fp) { }
(*passInfo->fp)(passInfo->param, &passInfo->ver, TAOS_NOTIFY_PASSVER);
if (!pRsp) {
for (int32_t j = 0; j < TARRAY_SIZE(batchRsp->pArray); ++j) {
SGetUserAuthRsp *rsp = TARRAY_GET_ELEM(batchRsp->pArray, j);
if (0 == strncmp(rsp->user, pTscObj->user, TSDB_USER_LEN)) {
pRsp = rsp;
break;
} }
tscDebug("update passVer of user %s from %d to %d, tscRid:%" PRIi64, rsp->user, oldVer, }
if (!pRsp) {
releaseTscObj(pReq->connKey.tscRid);
break;
}
}
pTscObj->authVer = pRsp->version;
#if 0
if (pTscObj->sysInfo != pRsp->sysInfo) {
tscDebug("update sysInfo of user %s from %" PRIi8 " to %" PRIi8 ", tscRid:%" PRIi64, pRsp->user,
pTscObj->sysInfo, pRsp->sysInfo, pTscObj->id);
pTscObj->sysInfo = pRsp->sysInfo;
}
#endif
if (pTscObj->passInfo.fp) {
SPassInfo *passInfo = &pTscObj->passInfo;
int32_t oldVer = atomic_load_32(&passInfo->ver);
if (oldVer < pRsp->passVer) {
atomic_store_32(&passInfo->ver, pRsp->passVer);
if (passInfo->fp) {
(*passInfo->fp)(passInfo->param, &pRsp->passVer, TAOS_NOTIFY_PASSVER);
}
tscDebug("update passVer of user %s from %d to %d, tscRid:%" PRIi64, pRsp->user, oldVer,
atomic_load_32(&passInfo->ver), pTscObj->id); atomic_load_32(&passInfo->ver), pTscObj->id);
} }
break;
} }
releaseTscObj(pReq->connKey.tscRid);
} }
releaseTscObj(pReq->connKey.tscRid);
} }
return 0;
taosArrayDestroy(batchRsp.pArray);
return code;
} }
static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) { static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) {
@ -121,7 +137,6 @@ static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) {
vgInfo->hashSuffix = rsp->hashSuffix; vgInfo->hashSuffix = rsp->hashSuffix;
vgInfo->vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); vgInfo->vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
if (NULL == vgInfo->vgHash) { if (NULL == vgInfo->vgHash) {
taosMemoryFree(vgInfo);
tscError("hash init[%d] failed", rsp->vgNum); tscError("hash init[%d] failed", rsp->vgNum);
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _return; goto _return;
@ -131,8 +146,6 @@ static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) {
SVgroupInfo *pInfo = taosArrayGet(rsp->pVgroupInfos, j); SVgroupInfo *pInfo = taosArrayGet(rsp->pVgroupInfos, j);
if (taosHashPut(vgInfo->vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) { if (taosHashPut(vgInfo->vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) {
tscError("hash push failed, errno:%d", errno); tscError("hash push failed, errno:%d", errno);
taosHashCleanup(vgInfo->vgHash);
taosMemoryFree(vgInfo);
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _return; goto _return;
} }
@ -316,7 +329,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
break; break;
} }
hbProcessUserAuthInfoRsp(kv->value, kv->valueLen, pCatalog); hbProcessUserAuthInfoRsp(kv->value, kv->valueLen, pCatalog, pAppHbMgr);
break; break;
} }
case HEARTBEAT_KEY_DBINFO: { case HEARTBEAT_KEY_DBINFO: {
@ -353,15 +366,6 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
hbProcessStbInfoRsp(kv->value, kv->valueLen, pCatalog); hbProcessStbInfoRsp(kv->value, kv->valueLen, pCatalog);
break; break;
} }
case HEARTBEAT_KEY_USER_PASSINFO: {
if (kv->valueLen <= 0 || NULL == kv->value) {
tscError("invalid hb user pass info, len:%d, value:%p", kv->valueLen, kv->value);
break;
}
hbProcessUserPassInfoRsp(kv->value, kv->valueLen, &pRsp->connKey, pAppHbMgr);
break;
}
default: default:
tscError("invalid hb key type:%d", kv->key); tscError("invalid hb key type:%d", kv->key);
break; break;
@ -479,7 +483,6 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) {
if (code) { if (code) {
taosArrayDestroy(desc.subDesc); taosArrayDestroy(desc.subDesc);
desc.subDesc = NULL; desc.subDesc = NULL;
desc.subPlanNum = 0;
} }
desc.subPlanNum = taosArrayGetSize(desc.subDesc); desc.subPlanNum = taosArrayGetSize(desc.subDesc);
} else { } else {
@ -543,7 +546,7 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t hbGetUserBasicInfo(SClientHbKey *connKey, SHbParam *param, SClientHbReq *req) { static int32_t hbGetUserAuthInfo(SClientHbKey *connKey, SHbParam *param, SClientHbReq *req) {
STscObj *pTscObj = (STscObj *)acquireTscObj(connKey->tscRid); STscObj *pTscObj = (STscObj *)acquireTscObj(connKey->tscRid);
if (!pTscObj) { if (!pTscObj) {
tscWarn("tscObj rid %" PRIx64 " not exist", connKey->tscRid); tscWarn("tscObj rid %" PRIx64 " not exist", connKey->tscRid);
@ -552,46 +555,61 @@ static int32_t hbGetUserBasicInfo(SClientHbKey *connKey, SHbParam *param, SClien
int32_t code = 0; int32_t code = 0;
if (param && (param->passVer != INT32_MIN) && (param->passVer <= pTscObj->passInfo.ver)) { SKv kv = {.key = HEARTBEAT_KEY_USER_AUTHINFO};
tscDebug("hb got user basic info, no need since passVer %d <= %d", param->passVer, pTscObj->passInfo.ver); SKv *pKv = NULL;
if ((pKv = taosHashGet(req->info, &kv.key, sizeof(kv.key)))) {
int32_t userNum = pKv->valueLen / sizeof(SUserAuthVersion);
SUserAuthVersion *userAuths = (SUserAuthVersion *)pKv->value;
for (int32_t i = 0; i < userNum; ++i) {
SUserAuthVersion *pUserAuth = userAuths + i;
// both key and user exist, update version
if (strncmp(pUserAuth->user, pTscObj->user, TSDB_USER_LEN) == 0) {
pUserAuth->version = htonl(-1); // force get userAuthInfo
goto _return;
}
}
// key exists, user not exist, append user
SUserAuthVersion *qUserAuth =
(SUserAuthVersion *)taosMemoryRealloc(pKv->value, (userNum + 1) * sizeof(SUserAuthVersion));
if (qUserAuth) {
strncpy((qUserAuth + userNum)->user, pTscObj->user, TSDB_USER_LEN);
(qUserAuth + userNum)->version = htonl(-1); // force get userAuthInfo
pKv->value = qUserAuth;
pKv->valueLen += sizeof(SUserAuthVersion);
} else {
code = TSDB_CODE_OUT_OF_MEMORY;
}
goto _return; goto _return;
} }
SUserPassVersion *user = taosMemoryMalloc(sizeof(SUserPassVersion)); // key/user not exist, add user
SUserAuthVersion *user = taosMemoryMalloc(sizeof(SUserAuthVersion));
if (!user) { if (!user) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _return; goto _return;
} }
strncpy(user->user, pTscObj->user, TSDB_USER_LEN); tstrncpy(user->user, pTscObj->user, TSDB_USER_LEN);
user->version = htonl(pTscObj->passInfo.ver); user->version = htonl(-1); // force get userAuthInfo
kv.valueLen = sizeof(SUserAuthVersion);
kv.value = user;
SKv kv = { tscDebug("hb got user auth info, valueLen:%d, user:%s, authVer:%d, tscRid:%" PRIi64, kv.valueLen, user->user,
.key = HEARTBEAT_KEY_USER_PASSINFO, pTscObj->authVer, connKey->tscRid);
.valueLen = sizeof(SUserPassVersion),
.value = user,
};
tscDebug("hb got user basic info, valueLen:%d, user:%s, passVer:%d, tscRid:%" PRIi64, kv.valueLen, user->user,
pTscObj->passInfo.ver, connKey->tscRid);
if (!req->info) { if (!req->info) {
req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
} }
if (taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)) < 0) { if (taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)) < 0) {
taosMemoryFree(user);
code = terrno ? terrno : TSDB_CODE_APP_ERROR; code = terrno ? terrno : TSDB_CODE_APP_ERROR;
goto _return; goto _return;
} }
// assign the passVer
if (param) {
param->passVer = pTscObj->passInfo.ver;
}
_return: _return:
releaseTscObj(connKey->tscRid); releaseTscObj(connKey->tscRid);
if (code) { if (code) {
tscError("hb got user basic info failed since %s", terrstr(code)); tscError("hb got user auth info failed since %s", terrstr(code));
} }
return code; return code;
@ -749,14 +767,21 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req
hbGetQueryBasicInfo(connKey, req); hbGetQueryBasicInfo(connKey, req);
if (hbParam->passKeyCnt > 0) {
hbGetUserBasicInfo(connKey, hbParam, req);
}
if (hbParam->reqCnt == 0) { if (hbParam->reqCnt == 0) {
code = hbGetExpiredUserInfo(connKey, pCatalog, req); if (!taosHashGet(clientHbMgr.appHbHash, &hbParam->clusterId, sizeof(hbParam->clusterId))) {
if (TSDB_CODE_SUCCESS != code) { code = hbGetExpiredUserInfo(connKey, pCatalog, req);
return code; if (TSDB_CODE_SUCCESS != code) {
return code;
}
}
// invoke after hbGetExpiredUserInfo
if (2 != atomic_load_8(&hbParam->pAppHbMgr->connHbFlag)) {
code = hbGetUserAuthInfo(connKey, hbParam, req);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
atomic_store_8(&hbParam->pAppHbMgr->connHbFlag, 1);
} }
code = hbGetExpiredDBInfo(connKey, pCatalog, req); code = hbGetExpiredDBInfo(connKey, pCatalog, req);
@ -770,7 +795,7 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req
} }
} }
++hbParam->reqCnt; // success to get catalog info ++hbParam->reqCnt; // success to get catalog info
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -815,9 +840,9 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) {
if (param.clusterId == 0) { if (param.clusterId == 0) {
// init // init
param.clusterId = pOneReq->clusterId; param.clusterId = pOneReq->clusterId;
param.passVer = INT32_MIN; param.pAppHbMgr = pAppHbMgr;
param.connHbFlag = atomic_load_8(&pAppHbMgr->connHbFlag);
} }
param.passKeyCnt = atomic_load_32(&pAppHbMgr->passKeyCnt);
break; break;
} }
default: default:
@ -901,6 +926,10 @@ static void *hbThreadFunc(void *param) {
int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); int sz = taosArrayGetSize(clientHbMgr.appHbMgrs);
if (sz > 0) { if (sz > 0) {
hbGatherAppInfo(); hbGatherAppInfo();
if (sz > 1 && !clientHbMgr.appHbHash) {
clientHbMgr.appHbHash = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
}
taosHashClear(clientHbMgr.appHbHash);
} }
for (int i = 0; i < sz; i++) { for (int i = 0; i < sz; i++) {
@ -953,7 +982,7 @@ static void *hbThreadFunc(void *param) {
asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, pInfo); asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, pInfo);
tFreeClientHbBatchReq(pReq); tFreeClientHbBatchReq(pReq);
// hbClearReqInfo(pAppHbMgr); // hbClearReqInfo(pAppHbMgr);
taosHashPut(clientHbMgr.appHbHash, &pAppHbMgr->pAppInstInfo->clusterId, sizeof(uint64_t), NULL, 0);
atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1);
} }
@ -961,6 +990,7 @@ static void *hbThreadFunc(void *param) {
taosMsleep(HEARTBEAT_INTERVAL); taosMsleep(HEARTBEAT_INTERVAL);
} }
taosHashCleanup(clientHbMgr.appHbHash);
return NULL; return NULL;
} }
@ -1009,7 +1039,7 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) {
// init stat // init stat
pAppHbMgr->startTime = taosGetTimestampMs(); pAppHbMgr->startTime = taosGetTimestampMs();
pAppHbMgr->connKeyCnt = 0; pAppHbMgr->connKeyCnt = 0;
pAppHbMgr->passKeyCnt = 0; pAppHbMgr->connHbFlag = 0;
pAppHbMgr->reportCnt = 0; pAppHbMgr->reportCnt = 0;
pAppHbMgr->reportBytes = 0; pAppHbMgr->reportBytes = 0;
pAppHbMgr->key = taosStrdup(key); pAppHbMgr->key = taosStrdup(key);
@ -1127,7 +1157,6 @@ void hbMgrCleanUp() {
appHbMgrCleanup(); appHbMgrCleanup();
taosArrayDestroy(clientHbMgr.appHbMgrs); taosArrayDestroy(clientHbMgr.appHbMgrs);
taosThreadMutexUnlock(&clientHbMgr.lock); taosThreadMutexUnlock(&clientHbMgr.lock);
clientHbMgr.appHbMgrs = NULL; clientHbMgr.appHbMgrs = NULL;
} }
@ -1180,12 +1209,6 @@ void hbDeregisterConn(STscObj *pTscObj, SClientHbKey connKey) {
} }
atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1);
taosThreadMutexLock(&pTscObj->mutex);
if (pTscObj->passInfo.fp) {
atomic_sub_fetch_32(&pAppHbMgr->passKeyCnt, 1);
}
taosThreadMutexUnlock(&pTscObj->mutex);
} }
// set heart beat thread quit mode , if quicByKill 1 then kill thread else quit from inner // set heart beat thread quit mode , if quicByKill 1 then kill thread else quit from inner

36
source/client/src/clientImpl.c
View File

@ -26,7 +26,7 @@
#include "tpagedbuf.h" #include "tpagedbuf.h"
#include "tref.h" #include "tref.h"
#include "tsched.h" #include "tsched.h"
#include "tversion.h"
static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet); static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet);
static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest); static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest);
@ -237,8 +237,9 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t buildPreviousRequest(SRequestObj *pRequest, const char* sql, SRequestObj** pNewRequest) { int32_t buildPreviousRequest(SRequestObj* pRequest, const char* sql, SRequestObj** pNewRequest) {
int32_t code = buildRequest(pRequest->pTscObj->id, sql, strlen(sql), pRequest, pRequest->validateOnly, pNewRequest, 0); int32_t code =
buildRequest(pRequest->pTscObj->id, sql, strlen(sql), pRequest, pRequest->validateOnly, pNewRequest, 0);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
pRequest->relation.prevRefId = (*pNewRequest)->self; pRequest->relation.prevRefId = (*pNewRequest)->self;
(*pNewRequest)->relation.nextRefId = pRequest->self; (*pNewRequest)->relation.nextRefId = pRequest->self;
@ -502,8 +503,7 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t
pResInfo->userFields[i].bytes = pSchema[i].bytes; pResInfo->userFields[i].bytes = pSchema[i].bytes;
pResInfo->userFields[i].type = pSchema[i].type; pResInfo->userFields[i].type = pSchema[i].type;
if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR || if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR || pSchema[i].type == TSDB_DATA_TYPE_GEOMETRY) {
pSchema[i].type == TSDB_DATA_TYPE_GEOMETRY) {
pResInfo->userFields[i].bytes -= VARSTR_HEADER_SIZE; pResInfo->userFields[i].bytes -= VARSTR_HEADER_SIZE;
} else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) { } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) {
pResInfo->userFields[i].bytes = (pResInfo->userFields[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; pResInfo->userFields[i].bytes = (pResInfo->userFields[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
@ -891,7 +891,7 @@ static bool incompletaFileParsing(SNode* pStmt) {
void continuePostSubQuery(SRequestObj* pRequest, TAOS_ROW row) { void continuePostSubQuery(SRequestObj* pRequest, TAOS_ROW row) {
SSqlCallbackWrapper* pWrapper = pRequest->pWrapper; SSqlCallbackWrapper* pWrapper = pRequest->pWrapper;
int32_t code = nodesAcquireAllocator(pWrapper->pParseCtx->allocatorId); int32_t code = nodesAcquireAllocator(pWrapper->pParseCtx->allocatorId);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
int64_t analyseStart = taosGetTimestampUs(); int64_t analyseStart = taosGetTimestampUs();
code = qContinueParsePostQuery(pWrapper->pParseCtx, pRequest->pQuery, (void**)row); code = qContinueParsePostQuery(pWrapper->pParseCtx, pRequest->pQuery, (void**)row);
@ -934,7 +934,7 @@ void postSubQueryFetchCb(void* param, TAOS_RES* res, int32_t rowNum) {
TAOS_ROW row = NULL; TAOS_ROW row = NULL;
if (rowNum > 0) { if (rowNum > 0) {
row = taos_fetch_row(res); // for single row only now row = taos_fetch_row(res); // for single row only now
} }
SRequestObj* pNextReq = acquireRequest(pRequest->relation.nextRefId); SRequestObj* pNextReq = acquireRequest(pRequest->relation.nextRefId);
@ -2135,6 +2135,7 @@ TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* de
connLimitNum = TMIN(connLimitNum, 500); connLimitNum = TMIN(connLimitNum, 500);
rpcInit.connLimitNum = connLimitNum; rpcInit.connLimitNum = connLimitNum;
rpcInit.timeToGetConn = tsTimeToGetAvailableConn; rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
clientRpc = rpcOpen(&rpcInit); clientRpc = rpcOpen(&rpcInit);
if (clientRpc == NULL) { if (clientRpc == NULL) {
@ -2494,11 +2495,10 @@ TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly,
return pRequest; return pRequest;
} }
static void fetchCallback(void* pResult, void* param, int32_t code) {
SRequestObj* pRequest = (SRequestObj*)param;
static void fetchCallback(void *pResult, void *param, int32_t code) { SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
SRequestObj *pRequest = (SRequestObj *)param;
SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code,
tstrerror(code), pRequest->requestId); tstrerror(code), pRequest->requestId);
@ -2520,7 +2520,7 @@ static void fetchCallback(void *pResult, void *param, int32_t code) {
} }
pRequest->code = pRequest->code =
setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp *)pResultInfo->pData, pResultInfo->convertUcs4, true); setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4, true);
if (pRequest->code != TSDB_CODE_SUCCESS) { if (pRequest->code != TSDB_CODE_SUCCESS) {
pResultInfo->numOfRows = 0; pResultInfo->numOfRows = 0;
pRequest->code = code; pRequest->code = code;
@ -2531,19 +2531,19 @@ static void fetchCallback(void *pResult, void *param, int32_t code) {
pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed,
pRequest->requestId); pRequest->requestId);
STscObj *pTscObj = pRequest->pTscObj; STscObj* pTscObj = pRequest->pTscObj;
SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; SAppClusterSummary* pActivity = &pTscObj->pAppInfo->summary;
atomic_add_fetch_64((int64_t *)&pActivity->fetchBytes, pRequest->body.resInfo.payloadLen); atomic_add_fetch_64((int64_t*)&pActivity->fetchBytes, pRequest->body.resInfo.payloadLen);
} }
pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows); pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows);
} }
void taosAsyncFetchImpl(SRequestObj *pRequest, __taos_async_fn_t fp, void *param) { void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param) {
pRequest->body.fetchFp = fp; pRequest->body.fetchFp = fp;
pRequest->body.param = param; pRequest->body.param = param;
SReqResultInfo *pResultInfo = &pRequest->body.resInfo; SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
// this query has no results or error exists, return directly // this query has no results or error exists, return directly
if (taos_num_fields(pRequest) == 0 || pRequest->code != TSDB_CODE_SUCCESS) { if (taos_num_fields(pRequest) == 0 || pRequest->code != TSDB_CODE_SUCCESS) {
@ -2578,5 +2578,3 @@ void taosAsyncFetchImpl(SRequestObj *pRequest, __taos_async_fn_t fp, void *param
schedulerFetchRows(pRequest->body.queryJob, &req); schedulerFetchRows(pRequest->body.queryJob, &req);
} }

59
source/client/src/clientMain.c
View File

@ -135,11 +135,6 @@ int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t fp, void *param, int type)
switch (type) { switch (type) {
case TAOS_NOTIFY_PASSVER: { case TAOS_NOTIFY_PASSVER: {
taosThreadMutexLock(&pObj->mutex); taosThreadMutexLock(&pObj->mutex);
if (fp && !pObj->passInfo.fp) {
atomic_add_fetch_32(&pObj->pAppInfo->pAppHbMgr->passKeyCnt, 1);
} else if (!fp && pObj->passInfo.fp) {
atomic_sub_fetch_32(&pObj->pAppInfo->pAppHbMgr->passKeyCnt, 1);
}
pObj->passInfo.fp = fp; pObj->passInfo.fp = fp;
pObj->passInfo.param = param; pObj->passInfo.param = param;
taosThreadMutexUnlock(&pObj->mutex); taosThreadMutexUnlock(&pObj->mutex);
@ -563,13 +558,12 @@ int taos_select_db(TAOS *taos, const char *db) {
return code; return code;
} }
void taos_stop_query(TAOS_RES *res) { void taos_stop_query(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) { if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) {
return; return;
} }
stopAllQueries((SRequestObj*)res); stopAllQueries((SRequestObj *)res);
} }
bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) { bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) {
@ -790,7 +784,7 @@ void destorySqlCallbackWrapper(SSqlCallbackWrapper *pWrapper) {
taosMemoryFree(pWrapper); taosMemoryFree(pWrapper);
} }
void destroyCtxInRequest(SRequestObj* pRequest) { void destroyCtxInRequest(SRequestObj *pRequest) {
schedulerFreeJob(&pRequest->body.queryJob, 0); schedulerFreeJob(&pRequest->body.queryJob, 0);
qDestroyQuery(pRequest->pQuery); qDestroyQuery(pRequest->pQuery);
pRequest->pQuery = NULL; pRequest->pQuery = NULL;
@ -798,7 +792,6 @@ void destroyCtxInRequest(SRequestObj* pRequest) {
pRequest->pWrapper = NULL; pRequest->pWrapper = NULL;
} }
static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t code) { static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t code) {
SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)param; SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)param;
SRequestObj *pRequest = pWrapper->pRequest; SRequestObj *pRequest = pWrapper->pRequest;
@ -812,15 +805,15 @@ static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = qAnalyseSqlSemantic(pWrapper->pParseCtx, pWrapper->pCatalogReq, pResultMeta, pQuery); code = qAnalyseSqlSemantic(pWrapper->pParseCtx, pWrapper->pCatalogReq, pResultMeta, pQuery);
} }
pRequest->metric.analyseCostUs += taosGetTimestampUs() - analyseStart; pRequest->metric.analyseCostUs += taosGetTimestampUs() - analyseStart;
handleQueryAnslyseRes(pWrapper, pResultMeta, code); handleQueryAnslyseRes(pWrapper, pResultMeta, code);
} }
int32_t cloneCatalogReq(SCatalogReq* * ppTarget, SCatalogReq* pSrc) { int32_t cloneCatalogReq(SCatalogReq **ppTarget, SCatalogReq *pSrc) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SCatalogReq* pTarget = taosMemoryCalloc(1, sizeof(SCatalogReq)); SCatalogReq *pTarget = taosMemoryCalloc(1, sizeof(SCatalogReq));
if (pTarget == NULL) { if (pTarget == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
} else { } else {
@ -847,17 +840,16 @@ int32_t cloneCatalogReq(SCatalogReq* * ppTarget, SCatalogReq* pSrc) {
return code; return code;
} }
void handleSubQueryFromAnalyse(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta, SNode *pRoot) {
void handleSubQueryFromAnalyse(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta, SNode* pRoot) { SRequestObj *pNewRequest = NULL;
SRequestObj* pNewRequest = NULL; SSqlCallbackWrapper *pNewWrapper = NULL;
SSqlCallbackWrapper* pNewWrapper = NULL; int32_t code = buildPreviousRequest(pWrapper->pRequest, pWrapper->pRequest->sqlstr, &pNewRequest);
int32_t code = buildPreviousRequest(pWrapper->pRequest, pWrapper->pRequest->sqlstr, &pNewRequest);
if (code) { if (code) {
handleQueryAnslyseRes(pWrapper, pResultMeta, code); handleQueryAnslyseRes(pWrapper, pResultMeta, code);
return; return;
} }
pNewRequest->pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); pNewRequest->pQuery = (SQuery *)nodesMakeNode(QUERY_NODE_QUERY);
if (NULL == pNewRequest->pQuery) { if (NULL == pNewRequest->pQuery) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
} else { } else {
@ -876,16 +868,16 @@ void handleSubQueryFromAnalyse(SSqlCallbackWrapper *pWrapper, SMetaData *pResult
} }
void handleQueryAnslyseRes(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta, int32_t code) { void handleQueryAnslyseRes(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta, int32_t code) {
SRequestObj *pRequest = pWrapper->pRequest; SRequestObj *pRequest = pWrapper->pRequest;
SQuery *pQuery = pRequest->pQuery; SQuery *pQuery = pRequest->pQuery;
if (code == TSDB_CODE_SUCCESS && pQuery->pPrevRoot) { if (code == TSDB_CODE_SUCCESS && pQuery->pPrevRoot) {
SNode* prevRoot = pQuery->pPrevRoot; SNode *prevRoot = pQuery->pPrevRoot;
pQuery->pPrevRoot = NULL; pQuery->pPrevRoot = NULL;
handleSubQueryFromAnalyse(pWrapper, pResultMeta, prevRoot); handleSubQueryFromAnalyse(pWrapper, pResultMeta, prevRoot);
return; return;
} }
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
pRequest->stableQuery = pQuery->stableQuery; pRequest->stableQuery = pQuery->stableQuery;
if (pQuery->pRoot) { if (pQuery->pRoot) {
@ -1048,7 +1040,7 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) {
} }
int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *pRequest, bool updateMetaForce) { int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *pRequest, bool updateMetaForce) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
STscObj *pTscObj = pRequest->pTscObj; STscObj *pTscObj = pRequest->pTscObj;
SSqlCallbackWrapper *pWrapper = taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper)); SSqlCallbackWrapper *pWrapper = taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper));
if (pWrapper == NULL) { if (pWrapper == NULL) {
@ -1086,7 +1078,6 @@ int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *p
return code; return code;
} }
void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
SSqlCallbackWrapper *pWrapper = NULL; SSqlCallbackWrapper *pWrapper = NULL;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -1133,12 +1124,12 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
} }
void restartAsyncQuery(SRequestObj *pRequest, int32_t code) { void restartAsyncQuery(SRequestObj *pRequest, int32_t code) {
int32_t reqIdx = 0; int32_t reqIdx = 0;
SRequestObj *pReqList[16] = {NULL}; SRequestObj *pReqList[16] = {NULL};
SRequestObj *pUserReq = NULL; SRequestObj *pUserReq = NULL;
pReqList[0] = pRequest; pReqList[0] = pRequest;
uint64_t tmpRefId = 0; uint64_t tmpRefId = 0;
SRequestObj* pTmp = pRequest; SRequestObj *pTmp = pRequest;
while (pTmp->relation.prevRefId) { while (pTmp->relation.prevRefId) {
tmpRefId = pTmp->relation.prevRefId; tmpRefId = pTmp->relation.prevRefId;
pTmp = acquireRequest(tmpRefId); pTmp = acquireRequest(tmpRefId);
@ -1146,9 +1137,9 @@ void restartAsyncQuery(SRequestObj *pRequest, int32_t code) {
pReqList[++reqIdx] = pTmp; pReqList[++reqIdx] = pTmp;
releaseRequest(tmpRefId); releaseRequest(tmpRefId);
} else { } else {
tscError("0x%" PRIx64 ", prev req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pTmp->self, tscError("0x%" PRIx64 ", prev req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pTmp->self, tmpRefId,
tmpRefId, pTmp->requestId); pTmp->requestId);
break; break;
} }
} }
@ -1157,11 +1148,11 @@ void restartAsyncQuery(SRequestObj *pRequest, int32_t code) {
pTmp = acquireRequest(tmpRefId); pTmp = acquireRequest(tmpRefId);
if (pTmp) { if (pTmp) {
tmpRefId = pTmp->relation.nextRefId; tmpRefId = pTmp->relation.nextRefId;
removeRequest(pTmp->self); removeRequest(pTmp->self);
releaseRequest(pTmp->self); releaseRequest(pTmp->self);
} else { } else {
tscError("0x%" PRIx64 " is not there", tmpRefId); tscError("0x%" PRIx64 " is not there", tmpRefId);
break; break;
} }
} }

1
source/client/src/clientMsgHandler.c
View File

@ -131,6 +131,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
pTscObj->connType = connectRsp.connType; pTscObj->connType = connectRsp.connType;
pTscObj->passInfo.ver = connectRsp.passVer; pTscObj->passInfo.ver = connectRsp.passVer;
pTscObj->authVer = connectRsp.authVer;
hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pTscObj->id, connectRsp.clusterId, connectRsp.connType); hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pTscObj->id, connectRsp.clusterId, connectRsp.connType);

4
source/client/src/clientRawBlockWrite.c
View File

@ -1286,6 +1286,10 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
taosArrayPush(pArray, &pVgData); taosArrayPush(pArray, &pVgData);
pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY);
if (NULL == pQuery) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
pQuery->msgType = TDMT_VND_ALTER_TABLE; pQuery->msgType = TDMT_VND_ALTER_TABLE;
pQuery->stableQuery = false; pQuery->stableQuery = false;

303
source/client/src/clientTmq.c
View File

@ -151,7 +151,7 @@ typedef struct {
int32_t vgId; int32_t vgId;
int32_t vgStatus; int32_t vgStatus;
int32_t vgSkipCnt; // here used to mark the slow vgroups int32_t vgSkipCnt; // here used to mark the slow vgroups
bool receivedInfoFromVnode; // has already received info from vnode // bool receivedInfoFromVnode; // has already received info from vnode
int64_t emptyBlockReceiveTs; // once empty block is received, idle for ignoreCnt then start to poll data int64_t emptyBlockReceiveTs; // once empty block is received, idle for ignoreCnt then start to poll data
bool seekUpdated; // offset is updated by seek operator, therefore, not update by vnode rsp. bool seekUpdated; // offset is updated by seek operator, therefore, not update by vnode rsp.
SEpSet epSet; SEpSet epSet;
@ -636,6 +636,7 @@ static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, int32_t type, tm
pParamSet->callbackFn = pCommitFp; pParamSet->callbackFn = pCommitFp;
pParamSet->userParam = userParam; pParamSet->userParam = userParam;
taosRLockLatch(&tmq->lock);
int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics);
tscDebug("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId); tscDebug("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId);
@ -646,6 +647,7 @@ static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, int32_t type, tm
pTopicName, numOfTopics); pTopicName, numOfTopics);
taosMemoryFree(pParamSet); taosMemoryFree(pParamSet);
pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam); pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam);
taosRUnLockLatch(&tmq->lock);
return; return;
} }
@ -663,6 +665,7 @@ static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, int32_t type, tm
vgId, numOfVgroups, pTopicName); vgId, numOfVgroups, pTopicName);
taosMemoryFree(pParamSet); taosMemoryFree(pParamSet);
pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam); pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam);
taosRUnLockLatch(&tmq->lock);
return; return;
} }
@ -675,10 +678,13 @@ static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, int32_t type, tm
taosMemoryFree(pParamSet); taosMemoryFree(pParamSet);
pCommitFp(tmq, code, userParam); pCommitFp(tmq, code, userParam);
} }
// update the offset value.
pVg->offsetInfo.committedOffset = pVg->offsetInfo.currentOffset;
} else { // do not perform commit, callback user function directly. } else { // do not perform commit, callback user function directly.
taosMemoryFree(pParamSet); taosMemoryFree(pParamSet);
pCommitFp(tmq, code, userParam); pCommitFp(tmq, code, userParam);
} }
taosRUnLockLatch(&tmq->lock);
} }
static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam) { static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam) {
@ -696,6 +702,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us
// init as 1 to prevent concurrency issue // init as 1 to prevent concurrency issue
pParamSet->waitingRspNum = 1; pParamSet->waitingRspNum = 1;
taosRLockLatch(&tmq->lock);
int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics);
tscDebug("consumer:0x%" PRIx64 " start to commit offset for %d topics", tmq->consumerId, numOfTopics); tscDebug("consumer:0x%" PRIx64 " start to commit offset for %d topics", tmq->consumerId, numOfTopics);
@ -725,6 +732,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us
} }
} }
} }
taosRUnLockLatch(&tmq->lock);
tscDebug("consumer:0x%" PRIx64 " total commit:%d for %d topics", tmq->consumerId, pParamSet->waitingRspNum - 1, tscDebug("consumer:0x%" PRIx64 " total commit:%d for %d topics", tmq->consumerId, pParamSet->waitingRspNum - 1,
numOfTopics); numOfTopics);
@ -799,6 +807,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
SMqHbReq req = {0}; SMqHbReq req = {0};
req.consumerId = tmq->consumerId; req.consumerId = tmq->consumerId;
req.epoch = tmq->epoch; req.epoch = tmq->epoch;
taosRLockLatch(&tmq->lock);
// if(tmq->needReportOffsetRows){ // if(tmq->needReportOffsetRows){
req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows)); req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows));
for(int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++){ for(int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++){
@ -820,6 +829,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
} }
// tmq->needReportOffsetRows = false; // tmq->needReportOffsetRows = false;
// } // }
taosRUnLockLatch(&tmq->lock);
int32_t tlen = tSerializeSMqHbReq(NULL, 0, &req); int32_t tlen = tSerializeSMqHbReq(NULL, 0, &req);
if (tlen < 0) { if (tlen < 0) {
@ -986,10 +996,12 @@ int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) {
if (*topics == NULL) { if (*topics == NULL) {
*topics = tmq_list_new(); *topics = tmq_list_new();
} }
taosRLockLatch(&tmq->lock);
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
SMqClientTopic* topic = taosArrayGet(tmq->clientTopics, i); SMqClientTopic* topic = taosArrayGet(tmq->clientTopics, i);
tmq_list_append(*topics, strchr(topic->topicName, '.') + 1); tmq_list_append(*topics, strchr(topic->topicName, '.') + 1);
} }
taosRUnLockLatch(&tmq->lock);
return 0; return 0;
} }
@ -1414,7 +1426,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
tDecoderClear(&decoder); tDecoderClear(&decoder);
memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead));
char buf[TSDB_OFFSET_LEN]; char buf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf, TSDB_OFFSET_LEN, &pRspWrapper->dataRsp.rspOffset); tFormatOffset(buf, TSDB_OFFSET_LEN, &pRspWrapper->dataRsp.rspOffset);
tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d, reqId:0x%" PRIx64, tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d, reqId:0x%" PRIx64,
tmq->consumerId, vgId, pRspWrapper->dataRsp.reqOffset.version, buf, rspType, requestId); tmq->consumerId, vgId, pRspWrapper->dataRsp.reqOffset.version, buf, rspType, requestId);
@ -1509,7 +1521,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic
clientVg.offsetInfo.walVerBegin = -1; clientVg.offsetInfo.walVerBegin = -1;
clientVg.offsetInfo.walVerEnd = -1; clientVg.offsetInfo.walVerEnd = -1;
clientVg.seekUpdated = false; clientVg.seekUpdated = false;
clientVg.receivedInfoFromVnode = false; // clientVg.receivedInfoFromVnode = false;
taosArrayPush(pTopic->vgs, &clientVg); taosArrayPush(pTopic->vgs, &clientVg);
} }
@ -1527,12 +1539,7 @@ static void freeClientVgInfo(void* param) {
static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) {
bool set = false; bool set = false;
int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics);
int32_t topicNumGet = taosArrayGetSize(pRsp->topics); int32_t topicNumGet = taosArrayGetSize(pRsp->topics);
char vgKey[TSDB_TOPIC_FNAME_LEN + 22];
tscInfo("consumer:0x%" PRIx64 " update ep epoch from %d to epoch %d, incoming topics:%d, existed topics:%d",
tmq->consumerId, tmq->epoch, epoch, topicNumGet, topicNumCur);
if (epoch <= tmq->epoch) { if (epoch <= tmq->epoch) {
return false; return false;
} }
@ -1548,6 +1555,12 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
return false; return false;
} }
taosWLockLatch(&tmq->lock);
int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics);
char vgKey[TSDB_TOPIC_FNAME_LEN + 22];
tscInfo("consumer:0x%" PRIx64 " update ep epoch from %d to epoch %d, incoming topics:%d, existed topics:%d",
tmq->consumerId, tmq->epoch, epoch, topicNumGet, topicNumCur);
// todo extract method // todo extract method
for (int32_t i = 0; i < topicNumCur; i++) { for (int32_t i = 0; i < topicNumCur; i++) {
// find old topic // find old topic
@ -1559,7 +1572,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j);
makeTopicVgroupKey(vgKey, pTopicCur->topicName, pVgCur->vgId); makeTopicVgroupKey(vgKey, pTopicCur->topicName, pVgCur->vgId);
char buf[TSDB_OFFSET_LEN]; char buf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf, TSDB_OFFSET_LEN, &pVgCur->offsetInfo.currentOffset); tFormatOffset(buf, TSDB_OFFSET_LEN, &pVgCur->offsetInfo.currentOffset);
tscInfo("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch, pVgCur->vgId, tscInfo("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch, pVgCur->vgId,
vgKey, buf); vgKey, buf);
@ -1579,7 +1592,6 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
taosHashCleanup(pVgOffsetHashMap); taosHashCleanup(pVgOffsetHashMap);
taosWLockLatch(&tmq->lock);
// destroy current buffered existed topics info // destroy current buffered existed topics info
if (tmq->clientTopics) { if (tmq->clientTopics) {
taosArrayDestroyEx(tmq->clientTopics, freeClientVgInfo); taosArrayDestroyEx(tmq->clientTopics, freeClientVgInfo);
@ -1788,7 +1800,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p
sendInfo->msgType = TDMT_VND_TMQ_CONSUME; sendInfo->msgType = TDMT_VND_TMQ_CONSUME;
int64_t transporterId = 0; int64_t transporterId = 0;
char offsetFormatBuf[TSDB_OFFSET_LEN]; char offsetFormatBuf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->offsetInfo.currentOffset); tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->offsetInfo.currentOffset);
tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, pTmq->consumerId, tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, pTmq->consumerId,
@ -1807,6 +1819,9 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
if(atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER){ if(atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER){
return 0; return 0;
} }
int32_t code = 0;
taosWLockLatch(&tmq->lock);
int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics);
tscDebug("consumer:0x%" PRIx64 " start to poll data, numOfTopics:%d", tmq->consumerId, numOfTopics); tscDebug("consumer:0x%" PRIx64 " start to poll data, numOfTopics:%d", tmq->consumerId, numOfTopics);
@ -1816,7 +1831,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
for (int j = 0; j < numOfVg; j++) { for (int j = 0; j < numOfVg; j++) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < EMPTY_BLOCK_POLL_IDLE_DURATION) { // less than 100ms if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < EMPTY_BLOCK_POLL_IDLE_DURATION) { // less than 10ms
tscTrace("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId, tscTrace("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId,
tmq->epoch, pVg->vgId); tmq->epoch, pVg->vgId);
continue; continue;
@ -1831,15 +1846,17 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
} }
atomic_store_32(&pVg->vgSkipCnt, 0); atomic_store_32(&pVg->vgSkipCnt, 0);
int32_t code = doTmqPollImpl(tmq, pTopic, pVg, timeout); code = doTmqPollImpl(tmq, pTopic, pVg, timeout);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; goto end;
} }
} }
} }
tscDebug("consumer:0x%" PRIx64 " end to poll data", tmq->consumerId); end:
return 0; taosWUnLockLatch(&tmq->lock);
tscDebug("consumer:0x%" PRIx64 " end to poll data, code:%d", tmq->consumerId, code);
return code;
} }
static int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) { static int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) {
@ -1862,6 +1879,23 @@ static int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* p
return 0; return 0;
} }
static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* offset, int64_t sver, int64_t ever, int64_t consumerId){
if (!pVg->seekUpdated) {
tscDebug("consumer:0x%" PRIx64" local offset is update, since seekupdate not set", consumerId);
pVg->offsetInfo.currentOffset = *offset;
} else {
tscDebug("consumer:0x%" PRIx64" local offset is NOT update, since seekupdate is set", consumerId);
}
// update the status
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
// update the valid wal version range
pVg->offsetInfo.walVerBegin = sver;
pVg->offsetInfo.walVerEnd = ever;
// pVg->receivedInfoFromVnode = true;
}
static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
tscDebug("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, tmq->qall->numOfItems); tscDebug("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, tmq->qall->numOfItems);
@ -1891,12 +1925,14 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
SMqDataRsp* pDataRsp = &pollRspWrapper->dataRsp; SMqDataRsp* pDataRsp = &pollRspWrapper->dataRsp;
if (pDataRsp->head.epoch == consumerEpoch) { if (pDataRsp->head.epoch == consumerEpoch) {
taosWLockLatch(&tmq->lock);
SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId);
pollRspWrapper->vgHandle = pVg; pollRspWrapper->vgHandle = pVg;
pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName);
if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){ if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){
tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId,
pollRspWrapper->topicName, pollRspWrapper->vgId); pollRspWrapper->topicName, pollRspWrapper->vgId);
taosWUnLockLatch(&tmq->lock);
return NULL; return NULL;
} }
// update the epset // update the epset
@ -1908,24 +1944,9 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
pVg->epSet = *pollRspWrapper->pEpset; pVg->epSet = *pollRspWrapper->pEpset;
} }
// update the local offset value only for the returned values, only when the local offset is NOT updated updateVgInfo(pVg, &pDataRsp->rspOffset, pDataRsp->head.walsver, pDataRsp->head.walever, tmq->consumerId);
// by tmq_offset_seek function
if (!pVg->seekUpdated) {
tscDebug("consumer:0x%" PRIx64" local offset is update, since seekupdate not set", tmq->consumerId);
pVg->offsetInfo.currentOffset = pDataRsp->rspOffset;
} else {
tscDebug("consumer:0x%" PRIx64" local offset is NOT update, since seekupdate is set", tmq->consumerId);
}
// update the status char buf[TSDB_OFFSET_LEN] = {0};
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
// update the valid wal version range
pVg->offsetInfo.walVerBegin = pDataRsp->head.walsver;
pVg->offsetInfo.walVerEnd = pDataRsp->head.walever;
pVg->receivedInfoFromVnode = true;
char buf[TSDB_OFFSET_LEN];
tFormatOffset(buf, TSDB_OFFSET_LEN, &pDataRsp->rspOffset); tFormatOffset(buf, TSDB_OFFSET_LEN, &pDataRsp->rspOffset);
if (pDataRsp->blockNum == 0) { if (pDataRsp->blockNum == 0) {
tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, vg total:%" PRId64 tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, vg total:%" PRId64
@ -1944,8 +1965,10 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows,
pollRspWrapper->reqId); pollRspWrapper->reqId);
taosFreeQitem(pollRspWrapper); taosFreeQitem(pollRspWrapper);
taosWUnLockLatch(&tmq->lock);
return pRsp; return pRsp;
} }
taosWUnLockLatch(&tmq->lock);
} else { } else {
tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
tmq->consumerId, pollRspWrapper->vgId, pDataRsp->head.epoch, consumerEpoch); tmq->consumerId, pollRspWrapper->vgId, pDataRsp->head.epoch, consumerEpoch);
@ -1960,23 +1983,22 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
tscDebug("consumer:0x%" PRIx64 " process meta rsp", tmq->consumerId); tscDebug("consumer:0x%" PRIx64 " process meta rsp", tmq->consumerId);
if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) { if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) {
taosWLockLatch(&tmq->lock);
SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId);
pollRspWrapper->vgHandle = pVg; pollRspWrapper->vgHandle = pVg;
pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName);
if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){ if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){
tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId,
pollRspWrapper->topicName, pollRspWrapper->vgId); pollRspWrapper->topicName, pollRspWrapper->vgId);
taosWUnLockLatch(&tmq->lock);
return NULL; return NULL;
} }
if(pollRspWrapper->metaRsp.rspOffset.type != 0){ // if offset is validate updateVgInfo(pVg, &pollRspWrapper->metaRsp.rspOffset, pollRspWrapper->metaRsp.head.walsver, pollRspWrapper->metaRsp.head.walever, tmq->consumerId);
pVg->offsetInfo.currentOffset = pollRspWrapper->metaRsp.rspOffset;
}
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
// build rsp // build rsp
SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper); SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper);
taosFreeQitem(pollRspWrapper); taosFreeQitem(pollRspWrapper);
taosWUnLockLatch(&tmq->lock);
return pRsp; return pRsp;
} else { } else {
tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
@ -1989,27 +2011,18 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
int32_t consumerEpoch = atomic_load_32(&tmq->epoch); int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
if (pollRspWrapper->taosxRsp.head.epoch == consumerEpoch) { if (pollRspWrapper->taosxRsp.head.epoch == consumerEpoch) {
taosWLockLatch(&tmq->lock);
SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId);
pollRspWrapper->vgHandle = pVg; pollRspWrapper->vgHandle = pVg;
pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName);
if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){ if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){
tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId,
pollRspWrapper->topicName, pollRspWrapper->vgId); pollRspWrapper->topicName, pollRspWrapper->vgId);
taosWUnLockLatch(&tmq->lock);
return NULL; return NULL;
} }
// update the local offset value only for the returned values, only when the local offset is NOT updated updateVgInfo(pVg, &pollRspWrapper->taosxRsp.rspOffset, pollRspWrapper->taosxRsp.head.walsver, pollRspWrapper->taosxRsp.head.walever, tmq->consumerId);
// by tmq_offset_seek function
if (!pVg->seekUpdated) {
if(pollRspWrapper->taosxRsp.rspOffset.type != 0) { // if offset is validate
tscDebug("consumer:0x%" PRIx64" local offset is update, since seekupdate not set", tmq->consumerId);
pVg->offsetInfo.currentOffset = pollRspWrapper->taosxRsp.rspOffset;
}
} else {
tscDebug("consumer:0x%" PRIx64" local offset is NOT update, since seekupdate is set", tmq->consumerId);
}
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
if (pollRspWrapper->taosxRsp.blockNum == 0) { if (pollRspWrapper->taosxRsp.blockNum == 0) {
tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", reqId:0x%" PRIx64, tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", reqId:0x%" PRIx64,
@ -2017,6 +2030,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
pVg->emptyBlockReceiveTs = taosGetTimestampMs(); pVg->emptyBlockReceiveTs = taosGetTimestampMs();
pRspWrapper = tmqFreeRspWrapper(pRspWrapper); pRspWrapper = tmqFreeRspWrapper(pRspWrapper);
taosFreeQitem(pollRspWrapper); taosFreeQitem(pollRspWrapper);
taosWUnLockLatch(&tmq->lock);
continue; continue;
} else { } else {
pVg->emptyBlockReceiveTs = 0; // reset the ts pVg->emptyBlockReceiveTs = 0; // reset the ts
@ -2033,16 +2047,16 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
tmq->totalRows += numOfRows; tmq->totalRows += numOfRows;
char buf[TSDB_OFFSET_LEN]; char buf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.currentOffset); tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.currentOffset);
tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64
", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64, ", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64,
tmq->consumerId, pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum, numOfRows, pVg->numOfRows, tmq->consumerId, pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum, numOfRows, pVg->numOfRows,
tmq->totalRows, pollRspWrapper->reqId); tmq->totalRows, pollRspWrapper->reqId);
taosFreeQitem(pollRspWrapper); taosFreeQitem(pollRspWrapper);
taosWUnLockLatch(&tmq->lock);
return pRsp; return pRsp;
} else { } else {
tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->taosxRsp.head.epoch, consumerEpoch); tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->taosxRsp.head.epoch, consumerEpoch);
@ -2121,7 +2135,8 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
} }
} }
static void displayConsumeStatistics(const tmq_t* pTmq) { static void displayConsumeStatistics(tmq_t* pTmq) {
taosRLockLatch(&pTmq->lock);
int32_t numOfTopics = taosArrayGetSize(pTmq->clientTopics); int32_t numOfTopics = taosArrayGetSize(pTmq->clientTopics);
tscDebug("consumer:0x%" PRIx64 " closing poll:%" PRId64 " rows:%" PRId64 " topics:%d, final epoch:%d", tscDebug("consumer:0x%" PRIx64 " closing poll:%" PRId64 " rows:%" PRId64 " topics:%d, final epoch:%d",
pTmq->consumerId, pTmq->pollCnt, pTmq->totalRows, numOfTopics, pTmq->epoch); pTmq->consumerId, pTmq->pollCnt, pTmq->totalRows, numOfTopics, pTmq->epoch);
@ -2137,7 +2152,7 @@ static void displayConsumeStatistics(const tmq_t* pTmq) {
tscDebug("topic:%s, %d. vgId:%d rows:%" PRId64, pTopics->topicName, j, pVg->vgId, pVg->numOfRows); tscDebug("topic:%s, %d. vgId:%d rows:%" PRId64, pTopics->topicName, j, pVg->vgId, pVg->numOfRows);
} }
} }
taosRUnLockLatch(&pTmq->lock);
tscDebug("consumer:0x%" PRIx64 " rows dist end", pTmq->consumerId); tscDebug("consumer:0x%" PRIx64 " rows dist end", pTmq->consumerId);
} }
@ -2533,6 +2548,9 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) {
} }
static void destroyCommonInfo(SMqVgCommon* pCommon) { static void destroyCommonInfo(SMqVgCommon* pCommon) {
if(pCommon == NULL){
return;
}
taosArrayDestroy(pCommon->pList); taosArrayDestroy(pCommon->pList);
tsem_destroy(&pCommon->rsp); tsem_destroy(&pCommon->rsp);
taosThreadMutexDestroy(&pCommon->mutex); taosThreadMutexDestroy(&pCommon->mutex);
@ -2540,56 +2558,75 @@ static void destroyCommonInfo(SMqVgCommon* pCommon) {
taosMemoryFree(pCommon); taosMemoryFree(pCommon);
} }
static bool isInSnapshotMode(int8_t type, bool useSnapshot){
if ((type < TMQ_OFFSET__LOG && useSnapshot) || type > TMQ_OFFSET__LOG) {
return true;
}
return false;
}
int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_assignment** assignment, int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_assignment** assignment,
int32_t* numOfAssignment) { int32_t* numOfAssignment) {
*numOfAssignment = 0; *numOfAssignment = 0;
*assignment = NULL; *assignment = NULL;
SMqVgCommon* pCommon = NULL;
int32_t accId = tmq->pTscObj->acctId; int32_t accId = tmq->pTscObj->acctId;
char tname[128] = {0}; char tname[128] = {0};
sprintf(tname, "%d.%s", accId, pTopicName); sprintf(tname, "%d.%s", accId, pTopicName);
int32_t code = TSDB_CODE_SUCCESS;
taosWLockLatch(&tmq->lock);
SMqClientTopic* pTopic = getTopicByName(tmq, tname); SMqClientTopic* pTopic = getTopicByName(tmq, tname);
if (pTopic == NULL) { if (pTopic == NULL) {
return TSDB_CODE_INVALID_PARA; code = TSDB_CODE_INVALID_PARA;
goto end;
} }
// in case of snapshot is opened, no valid offset will return // in case of snapshot is opened, no valid offset will return
*numOfAssignment = taosArrayGetSize(pTopic->vgs); *numOfAssignment = taosArrayGetSize(pTopic->vgs);
for (int32_t j = 0; j < (*numOfAssignment); ++j) {
SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j);
int32_t type = pClientVg->offsetInfo.currentOffset.type;
if (isInSnapshotMode(type, tmq->useSnapshot)) {
tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, assignment not allowed", tmq->consumerId, type);
code = TSDB_CODE_TMQ_SNAPSHOT_ERROR;
goto end;
}
}
*assignment = taosMemoryCalloc(*numOfAssignment, sizeof(tmq_topic_assignment)); *assignment = taosMemoryCalloc(*numOfAssignment, sizeof(tmq_topic_assignment));
if (*assignment == NULL) { if (*assignment == NULL) {
tscError("consumer:0x%" PRIx64 " failed to malloc buffer, size:%" PRIzu, tmq->consumerId, tscError("consumer:0x%" PRIx64 " failed to malloc buffer, size:%" PRIzu, tmq->consumerId,
(*numOfAssignment) * sizeof(tmq_topic_assignment)); (*numOfAssignment) * sizeof(tmq_topic_assignment));
return TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto end;
} }
bool needFetch = false; bool needFetch = false;
for (int32_t j = 0; j < (*numOfAssignment); ++j) { for (int32_t j = 0; j < (*numOfAssignment); ++j) {
SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j); SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j);
if (!pClientVg->receivedInfoFromVnode) { if (pClientVg->offsetInfo.currentOffset.type != TMQ_OFFSET__LOG) {
needFetch = true; needFetch = true;
break; break;
} }
tmq_topic_assignment* pAssignment = &(*assignment)[j]; tmq_topic_assignment* pAssignment = &(*assignment)[j];
if (pClientVg->offsetInfo.currentOffset.type == TMQ_OFFSET__LOG) { pAssignment->currentOffset = pClientVg->offsetInfo.currentOffset.version;
pAssignment->currentOffset = pClientVg->offsetInfo.currentOffset.version;
} else {
pAssignment->currentOffset = 0;
}
pAssignment->begin = pClientVg->offsetInfo.walVerBegin; pAssignment->begin = pClientVg->offsetInfo.walVerBegin;
pAssignment->end = pClientVg->offsetInfo.walVerEnd; pAssignment->end = pClientVg->offsetInfo.walVerEnd;
pAssignment->vgId = pClientVg->vgId; pAssignment->vgId = pClientVg->vgId;
tscInfo("consumer:0x%" PRIx64 " get assignment from local:%d->%" PRId64, tmq->consumerId,
pAssignment->vgId, pAssignment->currentOffset);
} }
if (needFetch) { if (needFetch) {
SMqVgCommon* pCommon = taosMemoryCalloc(1, sizeof(SMqVgCommon)); pCommon = taosMemoryCalloc(1, sizeof(SMqVgCommon));
if (pCommon == NULL) { if (pCommon == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno; code = terrno;
goto end;
} }
pCommon->pList= taosArrayInit(4, sizeof(tmq_topic_assignment)); pCommon->pList= taosArrayInit(4, sizeof(tmq_topic_assignment));
@ -2604,8 +2641,8 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
SMqVgWalInfoParam* pParam = taosMemoryMalloc(sizeof(SMqVgWalInfoParam)); SMqVgWalInfoParam* pParam = taosMemoryMalloc(sizeof(SMqVgWalInfoParam));
if (pParam == NULL) { if (pParam == NULL) {
destroyCommonInfo(pCommon); code = terrno;
return terrno; goto end;
} }
pParam->epoch = tmq->epoch; pParam->epoch = tmq->epoch;
@ -2619,30 +2656,30 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
int32_t msgSize = tSerializeSMqPollReq(NULL, 0, &req); int32_t msgSize = tSerializeSMqPollReq(NULL, 0, &req);
if (msgSize < 0) { if (msgSize < 0) {
taosMemoryFree(pParam); taosMemoryFree(pParam);
destroyCommonInfo(pCommon); code = terrno;
return terrno; goto end;
} }
char* msg = taosMemoryCalloc(1, msgSize); char* msg = taosMemoryCalloc(1, msgSize);
if (NULL == msg) { if (NULL == msg) {
taosMemoryFree(pParam); taosMemoryFree(pParam);
destroyCommonInfo(pCommon); code = terrno;
return terrno; goto end;
} }
if (tSerializeSMqPollReq(msg, msgSize, &req) < 0) { if (tSerializeSMqPollReq(msg, msgSize, &req) < 0) {
taosMemoryFree(msg); taosMemoryFree(msg);
taosMemoryFree(pParam); taosMemoryFree(pParam);
destroyCommonInfo(pCommon); code = terrno;
return terrno; goto end;
} }
SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
if (sendInfo == NULL) { if (sendInfo == NULL) {
taosMemoryFree(pParam); taosMemoryFree(pParam);
taosMemoryFree(msg); taosMemoryFree(msg);
destroyCommonInfo(pCommon); code = terrno;
return terrno; goto end;
} }
sendInfo->msgInfo = (SDataBuf){.pData = msg, .len = msgSize, .handle = NULL}; sendInfo->msgInfo = (SDataBuf){.pData = msg, .len = msgSize, .handle = NULL};
@ -2653,29 +2690,26 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
sendInfo->msgType = TDMT_VND_TMQ_VG_WALINFO; sendInfo->msgType = TDMT_VND_TMQ_VG_WALINFO;
int64_t transporterId = 0; int64_t transporterId = 0;
char offsetFormatBuf[TSDB_OFFSET_LEN]; char offsetFormatBuf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.currentOffset); tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.currentOffset);
tscInfo("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, tscInfo("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64,
tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId); tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId);
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pClientVg->epSet, &transporterId, sendInfo); asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pClientVg->epSet, &transporterId, sendInfo);
} }
tsem_wait(&pCommon->rsp); tsem_wait(&pCommon->rsp);
int32_t code = pCommon->code; code = pCommon->code;
terrno = code; terrno = code;
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(*assignment); goto end;
*assignment = NULL;
*numOfAssignment = 0;
} else {
int32_t num = taosArrayGetSize(pCommon->pList);
for(int32_t i = 0; i < num; ++i) {
(*assignment)[i] = *(tmq_topic_assignment*)taosArrayGet(pCommon->pList, i);
}
*numOfAssignment = num;
} }
int32_t num = taosArrayGetSize(pCommon->pList);
for(int32_t i = 0; i < num; ++i) {
(*assignment)[i] = *(tmq_topic_assignment*)taosArrayGet(pCommon->pList, i);
}
*numOfAssignment = num;
for (int32_t j = 0; j < (*numOfAssignment); ++j) { for (int32_t j = 0; j < (*numOfAssignment); ++j) {
tmq_topic_assignment* p = &(*assignment)[j]; tmq_topic_assignment* p = &(*assignment)[j];
@ -2687,26 +2721,23 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
} }
SVgOffsetInfo* pOffsetInfo = &pClientVg->offsetInfo; SVgOffsetInfo* pOffsetInfo = &pClientVg->offsetInfo;
tscInfo("vgId:%d offset is update to:%"PRId64, p->vgId, p->currentOffset);
pOffsetInfo->currentOffset.type = TMQ_OFFSET__LOG;
char offsetBuf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(offsetBuf, tListLen(offsetBuf), &pOffsetInfo->currentOffset);
tscInfo("vgId:%d offset is update to:%s", p->vgId, offsetBuf);
pOffsetInfo->walVerBegin = p->begin; pOffsetInfo->walVerBegin = p->begin;
pOffsetInfo->walVerEnd = p->end; pOffsetInfo->walVerEnd = p->end;
pOffsetInfo->currentOffset.version = p->currentOffset;
pOffsetInfo->committedOffset.version = p->currentOffset;
} }
} }
destroyCommonInfo(pCommon);
return code;
} else {
return TSDB_CODE_SUCCESS;
} }
end:
if(code != TSDB_CODE_SUCCESS){
taosMemoryFree(*assignment);
*assignment = NULL;
*numOfAssignment = 0;
}
destroyCommonInfo(pCommon);
taosWUnLockLatch(&tmq->lock);
return code;
} }
void tmq_free_assignment(tmq_topic_assignment* pAssignment) { void tmq_free_assignment(tmq_topic_assignment* pAssignment) {
@ -2727,9 +2758,11 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_
char tname[128] = {0}; char tname[128] = {0};
sprintf(tname, "%d.%s", accId, pTopicName); sprintf(tname, "%d.%s", accId, pTopicName);
taosWLockLatch(&tmq->lock);
SMqClientTopic* pTopic = getTopicByName(tmq, tname); SMqClientTopic* pTopic = getTopicByName(tmq, tname);
if (pTopic == NULL) { if (pTopic == NULL) {
tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName);
taosWUnLockLatch(&tmq->lock);
return TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
} }
@ -2745,56 +2778,58 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_
if (pVg == NULL) { if (pVg == NULL) {
tscError("consumer:0x%" PRIx64 " invalid vgroup id:%d", tmq->consumerId, vgId); tscError("consumer:0x%" PRIx64 " invalid vgroup id:%d", tmq->consumerId, vgId);
taosWUnLockLatch(&tmq->lock);
return TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
} }
SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo;
int32_t type = pOffsetInfo->currentOffset.type; int32_t type = pOffsetInfo->currentOffset.type;
if (type != TMQ_OFFSET__LOG && !OFFSET_IS_RESET_OFFSET(type)) { if (isInSnapshotMode(type, tmq->useSnapshot)) {
tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, seek not allowed", tmq->consumerId, type); tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, seek not allowed", tmq->consumerId, type);
return TSDB_CODE_INVALID_PARA; taosWUnLockLatch(&tmq->lock);
return TSDB_CODE_TMQ_SNAPSHOT_ERROR;
} }
if (type == TMQ_OFFSET__LOG && (offset < pOffsetInfo->walVerBegin || offset > pOffsetInfo->walVerEnd)) { if (type == TMQ_OFFSET__LOG && (offset < pOffsetInfo->walVerBegin || offset > pOffsetInfo->walVerEnd)) {
tscError("consumer:0x%" PRIx64 " invalid seek params, offset:%" PRId64 ", valid range:[%" PRId64 ", %" PRId64 "]", tscError("consumer:0x%" PRIx64 " invalid seek params, offset:%" PRId64 ", valid range:[%" PRId64 ", %" PRId64 "]",
tmq->consumerId, offset, pOffsetInfo->walVerBegin, pOffsetInfo->walVerEnd); tmq->consumerId, offset, pOffsetInfo->walVerBegin, pOffsetInfo->walVerEnd);
taosWUnLockLatch(&tmq->lock);
return TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
} }
// update the offset, and then commit to vnode // update the offset, and then commit to vnode
if (pOffsetInfo->currentOffset.type == TMQ_OFFSET__LOG) { pOffsetInfo->currentOffset.type = TMQ_OFFSET__LOG;
pOffsetInfo->currentOffset.version = offset; pOffsetInfo->currentOffset.version = offset >= 1 ? offset - 1 : 0;
pOffsetInfo->committedOffset.version = INT64_MIN; // pOffsetInfo->committedOffset.version = INT64_MIN;
pVg->seekUpdated = true; pVg->seekUpdated = true;
}
SMqRspObj rspObj = {.resType = RES_TYPE__TMQ, .vgId = pVg->vgId};
tstrncpy(rspObj.topic, tname, tListLen(rspObj.topic));
tscInfo("consumer:0x%" PRIx64 " seek to %" PRId64 " on vgId:%d", tmq->consumerId, offset, pVg->vgId); tscInfo("consumer:0x%" PRIx64 " seek to %" PRId64 " on vgId:%d", tmq->consumerId, offset, pVg->vgId);
taosWUnLockLatch(&tmq->lock);
SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); // SMqRspObj rspObj = {.resType = RES_TYPE__TMQ, .vgId = pVg->vgId};
if (pInfo == NULL) { // tstrncpy(rspObj.topic, tname, tListLen(rspObj.topic));
tscError("consumer:0x%"PRIx64" failed to prepare seek operation", tmq->consumerId); //
return TSDB_CODE_OUT_OF_MEMORY; // SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo));
} // if (pInfo == NULL) {
// tscError("consumer:0x%"PRIx64" failed to prepare seek operation", tmq->consumerId);
// return TSDB_CODE_OUT_OF_MEMORY;
// }
//
// tsem_init(&pInfo->sem, 0, 0);
// pInfo->code = 0;
//
// asyncCommitOffset(tmq, &rspObj, TDMT_VND_TMQ_SEEK_TO_OFFSET, commitCallBackFn, pInfo);
//
// tsem_wait(&pInfo->sem);
// int32_t code = pInfo->code;
//
// tsem_destroy(&pInfo->sem);
// taosMemoryFree(pInfo);
//
// if (code != TSDB_CODE_SUCCESS) {
// tscError("consumer:0x%" PRIx64 " failed to send seek to vgId:%d, code:%s", tmq->consumerId, pVg->vgId, tstrerror(code));
// }
tsem_init(&pInfo->sem, 0, 0); return 0;
pInfo->code = 0;
asyncCommitOffset(tmq, &rspObj, TDMT_VND_TMQ_SEEK_TO_OFFSET, commitCallBackFn, pInfo);
tsem_wait(&pInfo->sem);
int32_t code = pInfo->code;
tsem_destroy(&pInfo->sem);
taosMemoryFree(pInfo);
if (code != TSDB_CODE_SUCCESS) {
tscError("consumer:0x%" PRIx64 " failed to send seek to vgId:%d, code:%s", tmq->consumerId, pVg->vgId,
tstrerror(code));
}
return code;
} }

140
source/client/test/clientTests.cpp
View File

@ -1073,6 +1073,146 @@ TEST(clientCase, sub_db_test) {
fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows);
} }
TEST(clientCase, td_25129) {
// taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg");
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
tmq_conf_t* conf = tmq_conf_new();
tmq_conf_set(conf, "enable.auto.commit", "false");
tmq_conf_set(conf, "auto.commit.interval.ms", "2000");
tmq_conf_set(conf, "group.id", "group_id_2");
tmq_conf_set(conf, "td.connect.user", "root");
tmq_conf_set(conf, "td.connect.pass", "taosdata");
tmq_conf_set(conf, "auto.offset.reset", "earliest");
tmq_conf_set(conf, "msg.with.table.name", "true");
tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
tmq_conf_destroy(conf);
// 创建订阅 topics 列表
tmq_list_t* topicList = tmq_list_new();
tmq_list_append(topicList, "tp");
// 启动订阅
tmq_subscribe(tmq, topicList);
tmq_list_destroy(topicList);
TAOS_FIELD* fields = NULL;
int32_t numOfFields = 0;
int32_t precision = 0;
int32_t totalRows = 0;
int32_t msgCnt = 0;
int32_t timeout = 2000;
int32_t count = 0;
tmq_topic_assignment* pAssign = NULL;
int32_t numOfAssign = 0;
int32_t code = tmq_get_topic_assignment(tmq, "tp", &pAssign, &numOfAssign);
if (code != 0) {
printf("error occurs:%s\n", tmq_err2str(code));
tmq_free_assignment(pAssign);
tmq_consumer_close(tmq);
taos_close(pConn);
fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows);
return;
}
for(int i = 0; i < numOfAssign; i++){
printf("assign i:%d, vgId:%d, offset:%lld, start:%lld, end:%lld\n", i, pAssign[i].vgId, pAssign[i].currentOffset, pAssign[i].begin, pAssign[i].end);
}
// tmq_offset_seek(tmq, "tp", pAssign[0].vgId, 4);
tmq_free_assignment(pAssign);
code = tmq_get_topic_assignment(tmq, "tp", &pAssign, &numOfAssign);
if (code != 0) {
printf("error occurs:%s\n", tmq_err2str(code));
tmq_free_assignment(pAssign);
tmq_consumer_close(tmq);
taos_close(pConn);
fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows);
return;
}
for(int i = 0; i < numOfAssign; i++){
printf("assign i:%d, vgId:%d, offset:%lld, start:%lld, end:%lld\n", i, pAssign[i].vgId, pAssign[i].currentOffset, pAssign[i].begin, pAssign[i].end);
}
tmq_free_assignment(pAssign);
code = tmq_get_topic_assignment(tmq, "tp", &pAssign, &numOfAssign);
if (code != 0) {
printf("error occurs:%s\n", tmq_err2str(code));
tmq_free_assignment(pAssign);
tmq_consumer_close(tmq);
taos_close(pConn);
fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows);
return;
}
for(int i = 0; i < numOfAssign; i++){
printf("assign i:%d, vgId:%d, offset:%lld, start:%lld, end:%lld\n", i, pAssign[i].vgId, pAssign[i].currentOffset, pAssign[i].begin, pAssign[i].end);
}
while (1) {
TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout);
if (pRes) {
char buf[128];
const char* topicName = tmq_get_topic_name(pRes);
// const char* dbName = tmq_get_db_name(pRes);
// int32_t vgroupId = tmq_get_vgroup_id(pRes);
//
// printf("topic: %s\n", topicName);
// printf("db: %s\n", dbName);
// printf("vgroup id: %d\n", vgroupId);
printSubResults(pRes, &totalRows);
} else {
tmq_offset_seek(tmq, "tp", pAssign[0].vgId, pAssign[0].currentOffset);
tmq_offset_seek(tmq, "tp", pAssign[1].vgId, pAssign[1].currentOffset);
continue;
}
// tmq_commit_sync(tmq, pRes);
if (pRes != NULL) {
taos_free_result(pRes);
// if ((++count) > 1) {
// break;
// }
} else {
break;
}
// tmq_offset_seek(tmq, "tp", pAssign[0].vgId, pAssign[0].begin);
}
tmq_free_assignment(pAssign);
code = tmq_get_topic_assignment(tmq, "tp", &pAssign, &numOfAssign);
if (code != 0) {
printf("error occurs:%s\n", tmq_err2str(code));
tmq_free_assignment(pAssign);
tmq_consumer_close(tmq);
taos_close(pConn);
fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows);
return;
}
for(int i = 0; i < numOfAssign; i++){
printf("assign i:%d, vgId:%d, offset:%lld, start:%lld, end:%lld\n", i, pAssign[i].vgId, pAssign[i].currentOffset, pAssign[i].begin, pAssign[i].end);
}
tmq_consumer_close(tmq);
taos_close(pConn);
fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows);
}
TEST(clientCase, sub_tb_test) { TEST(clientCase, sub_tb_test) {
taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg"); taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg");

2
source/common/src/systable.c
View File

@ -315,7 +315,7 @@ static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema), true}, {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema), true},
{TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema), true}, {TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema), true},
{TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema), true}, {TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema), true},
{TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)}, {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema), true},
{TSDB_INS_TABLE_CLUSTER, clusterSchema, tListLen(clusterSchema), true}, {TSDB_INS_TABLE_CLUSTER, clusterSchema, tListLen(clusterSchema), true},
{TSDB_INS_TABLE_DATABASES, userDBSchema, tListLen(userDBSchema), false}, {TSDB_INS_TABLE_DATABASES, userDBSchema, tListLen(userDBSchema), false},
{TSDB_INS_TABLE_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema), false}, {TSDB_INS_TABLE_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema), false},

69
source/common/src/tmsg.c
View File

@ -1525,6 +1525,9 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
useDb = taosHashIterate(pRsp->useDbs, useDb); useDb = taosHashIterate(pRsp->useDbs, useDb);
} }
// since 3.0.7.0
if (tEncodeI32(pEncoder, pRsp->passVer) < 0) return -1;
return 0; return 0;
} }
@ -1646,6 +1649,12 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs
taosHashPut(pRsp->useDbs, key, strlen(key), &ref, sizeof(ref)); taosHashPut(pRsp->useDbs, key, strlen(key), &ref, sizeof(ref));
taosMemoryFree(key); taosMemoryFree(key);
} }
// since 3.0.7.0
if (!tDecodeIsEnd(pDecoder)) {
if (tDecodeI32(pDecoder, &pRsp->passVer) < 0) return -1;
} else {
pRsp->passVer = 0;
}
} }
return 0; return 0;
@ -3031,59 +3040,6 @@ void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp *pRsp) {
taosArrayDestroy(pRsp->pArray); taosArrayDestroy(pRsp->pArray);
} }
int32_t tSerializeSUserPassBatchRsp(void *buf, int32_t bufLen, SUserPassBatchRsp *pRsp) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
int32_t numOfBatch = taosArrayGetSize(pRsp->pArray);
if (tEncodeI32(&encoder, numOfBatch) < 0) return -1;
for (int32_t i = 0; i < numOfBatch; ++i) {
SGetUserPassRsp *pUserPassRsp = taosArrayGet(pRsp->pArray, i);
if (tEncodeCStr(&encoder, pUserPassRsp->user) < 0) return -1;
if (tEncodeI32(&encoder, pUserPassRsp->version) < 0) return -1;
}
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSUserPassBatchRsp(void *buf, int32_t bufLen, SUserPassBatchRsp *pRsp) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
int32_t numOfBatch = taosArrayGetSize(pRsp->pArray);
if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1;
pRsp->pArray = taosArrayInit(numOfBatch, sizeof(SGetUserPassRsp));
if (pRsp->pArray == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
for (int32_t i = 0; i < numOfBatch; ++i) {
SGetUserPassRsp rsp = {0};
if (tDecodeCStrTo(&decoder, rsp.user) < 0) return -1;
if (tDecodeI32(&decoder, &rsp.version) < 0) return -1;
taosArrayPush(pRsp->pArray, &rsp);
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
void tFreeSUserPassBatchRsp(SUserPassBatchRsp *pRsp) {
if(pRsp) {
taosArrayDestroy(pRsp->pArray);
}
}
int32_t tSerializeSDbCfgReq(void *buf, int32_t bufLen, SDbCfgReq *pReq) { int32_t tSerializeSDbCfgReq(void *buf, int32_t bufLen, SDbCfgReq *pReq) {
SEncoder encoder = {0}; SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen); tEncoderInit(&encoder, buf, bufLen);
@ -4161,6 +4117,7 @@ int32_t tSerializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) {
if (tEncodeCStr(&encoder, pRsp->sVer) < 0) return -1; if (tEncodeCStr(&encoder, pRsp->sVer) < 0) return -1;
if (tEncodeCStr(&encoder, pRsp->sDetailVer) < 0) return -1; if (tEncodeCStr(&encoder, pRsp->sDetailVer) < 0) return -1;
if (tEncodeI32(&encoder, pRsp->passVer) < 0) return -1; if (tEncodeI32(&encoder, pRsp->passVer) < 0) return -1;
if (tEncodeI32(&encoder, pRsp->authVer) < 0) return -1;
tEndEncode(&encoder); tEndEncode(&encoder);
int32_t tlen = encoder.pos; int32_t tlen = encoder.pos;
@ -4190,6 +4147,12 @@ int32_t tDeserializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) {
} else { } else {
pRsp->passVer = 0; pRsp->passVer = 0;
} }
// since 3.0.7.0
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pRsp->authVer) < 0) return -1;
} else {
pRsp->authVer = 0;
}
tEndDecode(&decoder); tEndDecode(&decoder);

2
source/common/src/trow.c
View File

@ -423,7 +423,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
val = (const void *)&pColVal->value.val; val = (const void *)&pColVal->value.val;
} }
} else { } else {
pColVal = NULL; // pColVal = NULL;
valType = TD_VTYPE_NONE; valType = TD_VTYPE_NONE;
} }

7
source/dnode/mgmt/exe/dmMain.c
View File

@ -19,6 +19,9 @@
#include "tconfig.h" #include "tconfig.h"
#include "tglobal.h" #include "tglobal.h"
#include "version.h" #include "version.h"
#ifdef TD_JEMALLOC_ENABLED
#include "jemalloc/jemalloc.h"
#endif
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
#include "cus_name.h" #include "cus_name.h"
@ -255,6 +258,10 @@ static void taosCleanupArgs() {
} }
int main(int argc, char const *argv[]) { int main(int argc, char const *argv[]) {
#ifdef TD_JEMALLOC_ENABLED
bool jeBackgroundThread = true;
mallctl("background_thread", NULL, NULL, &jeBackgroundThread, sizeof(bool));
#endif
if (!taosCheckSystemIsLittleEnd()) { if (!taosCheckSystemIsLittleEnd()) {
printf("failed to start since on non-little-end machines\n"); printf("failed to start since on non-little-end machines\n");
return -1; return -1;

11
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -16,6 +16,7 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "dmMgmt.h" #include "dmMgmt.h"
#include "qworker.h" #include "qworker.h"
#include "tversion.h"
static inline void dmSendRsp(SRpcMsg *pMsg) { rpcSendResponse(pMsg); } static inline void dmSendRsp(SRpcMsg *pMsg) { rpcSendResponse(pMsg); }
@ -73,6 +74,13 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
dGTrace("msg:%s is received, handle:%p len:%d code:0x%x app:%p refId:%" PRId64, TMSG_INFO(pRpc->msgType), dGTrace("msg:%s is received, handle:%p len:%d code:0x%x app:%p refId:%" PRId64, TMSG_INFO(pRpc->msgType),
pRpc->info.handle, pRpc->contLen, pRpc->code, pRpc->info.ahandle, pRpc->info.refId); pRpc->info.handle, pRpc->contLen, pRpc->code, pRpc->info.ahandle, pRpc->info.refId);
int32_t svrVer = 0;
taosVersionStrToInt(version, &svrVer);
if (0 != taosCheckVersionCompatible(pRpc->info.cliVer, svrVer, 3)) {
dError("Version not compatible, cli ver: %d, svr ver: %d", pRpc->info.cliVer, svrVer);
goto _OVER;
}
switch (pRpc->msgType) { switch (pRpc->msgType) {
case TDMT_DND_NET_TEST: case TDMT_DND_NET_TEST:
dmProcessNetTestReq(pDnode, pRpc); dmProcessNetTestReq(pDnode, pRpc);
@ -305,6 +313,7 @@ int32_t dmInitClient(SDnode *pDnode) {
rpcInit.supportBatch = 1; rpcInit.supportBatch = 1;
rpcInit.batchSize = 8 * 1024; rpcInit.batchSize = 8 * 1024;
rpcInit.timeToGetConn = tsTimeToGetAvailableConn; rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
pTrans->clientRpc = rpcOpen(&rpcInit); pTrans->clientRpc = rpcOpen(&rpcInit);
if (pTrans->clientRpc == NULL) { if (pTrans->clientRpc == NULL) {
@ -339,7 +348,7 @@ int32_t dmInitServer(SDnode *pDnode) {
rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.parent = pDnode; rpcInit.parent = pDnode;
rpcInit.compressSize = tsCompressMsgSize; rpcInit.compressSize = tsCompressMsgSize;
taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
pTrans->serverRpc = rpcOpen(&rpcInit); pTrans->serverRpc = rpcOpen(&rpcInit);
if (pTrans->serverRpc == NULL) { if (pTrans->serverRpc == NULL) {
dError("failed to init dnode rpc server"); dError("failed to init dnode rpc server");

2
source/dnode/mgmt/test/sut/src/client.cpp
View File

@ -16,6 +16,7 @@
#include "sut.h" #include "sut.h"
#include "tdatablock.h" #include "tdatablock.h"
#include "tmisce.h" #include "tmisce.h"
#include "tversion.h"
static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) { static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) {
TestClient* client = (TestClient*)parent; TestClient* client = (TestClient*)parent;
@ -53,6 +54,7 @@ void TestClient::DoInit() {
rpcInit.parent = this; rpcInit.parent = this;
// rpcInit.secret = (char*)secretEncrypt; // rpcInit.secret = (char*)secretEncrypt;
// rpcInit.spi = 1; // rpcInit.spi = 1;
taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
clientRpc = rpcOpen(&rpcInit); clientRpc = rpcOpen(&rpcInit);
ASSERT(clientRpc); ASSERT(clientRpc);

2
source/dnode/mnode/impl/inc/mndUser.h
View File

@ -35,8 +35,6 @@ SHashObj *mndDupTableHash(SHashObj *pOld);
SHashObj *mndDupTopicHash(SHashObj *pOld); SHashObj *mndDupTopicHash(SHashObj *pOld);
int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp,
int32_t *pRspLen); int32_t *pRspLen);
int32_t mndValidateUserPassInfo(SMnode *pMnode, SUserPassVersion *pUsers, int32_t numOfUses, void **ppRsp,
int32_t *pRspLen);
int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db); int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db);
int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic); int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic);

3
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -419,6 +419,9 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
mDebug("heartbeat report offset rows.%s:%s", pConsumer->cgroup, data->topicName); mDebug("heartbeat report offset rows.%s:%s", pConsumer->cgroup, data->topicName);
SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, data->topicName); SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, data->topicName);
if(pSub == NULL){
continue;
}
taosWLockLatch(&pSub->lock); taosWLockLatch(&pSub->lock);
SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &consumerId, sizeof(int64_t)); SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &consumerId, sizeof(int64_t));
if(pConsumerEp){ if(pConsumerEp){

36
source/dnode/mnode/impl/src/mndDb.c
View File

@ -382,6 +382,40 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
return terrno; return terrno;
} }
static int32_t mndCheckInChangeDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
if (pCfg->buffer < TSDB_MIN_BUFFER_PER_VNODE || pCfg->buffer > TSDB_MAX_BUFFER_PER_VNODE) return -1;
if (pCfg->pages < TSDB_MIN_PAGES_PER_VNODE || pCfg->pages > TSDB_MAX_PAGES_PER_VNODE) return -1;
if (pCfg->pageSize < TSDB_MIN_PAGESIZE_PER_VNODE || pCfg->pageSize > TSDB_MAX_PAGESIZE_PER_VNODE) return -1;
if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1;
if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP) return -1;
if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP) return -1;
if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > TSDB_MAX_KEEP) return -1;
if (pCfg->daysToKeep0 < pCfg->daysPerFile) return -1;
if (pCfg->daysToKeep0 > pCfg->daysToKeep1) return -1;
if (pCfg->daysToKeep1 > pCfg->daysToKeep2) return -1;
if (pCfg->walFsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->walFsyncPeriod > TSDB_MAX_FSYNC_PERIOD) return -1;
if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) return -1;
if (pCfg->cacheLast < TSDB_CACHE_MODEL_NONE || pCfg->cacheLast > TSDB_CACHE_MODEL_BOTH) return -1;
if (pCfg->cacheLastSize < TSDB_MIN_DB_CACHE_SIZE || pCfg->cacheLastSize > TSDB_MAX_DB_CACHE_SIZE) return -1;
if (pCfg->replications < TSDB_MIN_DB_REPLICA || pCfg->replications > TSDB_MAX_DB_REPLICA) return -1;
if (pCfg->replications != 1 && pCfg->replications != 3) return -1;
if (pCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1;
if (pCfg->minRows < TSDB_MIN_MINROWS_FBLOCK || pCfg->minRows > TSDB_MAX_MINROWS_FBLOCK) return -1;
if (pCfg->maxRows < TSDB_MIN_MAXROWS_FBLOCK || pCfg->maxRows > TSDB_MAX_MAXROWS_FBLOCK) return -1;
if (pCfg->minRows > pCfg->maxRows) return -1;
if (pCfg->walRetentionPeriod < TSDB_DB_MIN_WAL_RETENTION_PERIOD) return -1;
if (pCfg->walRetentionSize < TSDB_DB_MIN_WAL_RETENTION_SIZE) return -1;
if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1;
if (pCfg->replications > mndGetDnodeSize(pMnode)) {
terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
return -1;
}
terrno = 0;
return terrno;
}
static void mndSetDefaultDbCfg(SDbCfg *pCfg) { static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->numOfVgroups < 0) pCfg->numOfVgroups = TSDB_DEFAULT_VN_PER_DB; if (pCfg->numOfVgroups < 0) pCfg->numOfVgroups = TSDB_DEFAULT_VN_PER_DB;
if (pCfg->numOfStables < 0) pCfg->numOfStables = TSDB_DEFAULT_DB_SINGLE_STABLE; if (pCfg->numOfStables < 0) pCfg->numOfStables = TSDB_DEFAULT_DB_SINGLE_STABLE;
@ -897,7 +931,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) {
code = mndSetDbCfgFromAlterDbReq(&dbObj, &alterReq); code = mndSetDbCfgFromAlterDbReq(&dbObj, &alterReq);
if (code != 0) goto _OVER; if (code != 0) goto _OVER;
code = mndCheckDbCfg(pMnode, &dbObj.cfg); code = mndCheckInChangeDbCfg(pMnode, &dbObj.cfg);
if (code != 0) goto _OVER; if (code != 0) goto _OVER;
dbObj.cfgVersion++; dbObj.cfgVersion++;

2
source/dnode/mnode/impl/src/mndPrivilege.c
View File

@ -36,7 +36,9 @@ int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp
memcpy(pRsp->user, pUser->user, TSDB_USER_LEN); memcpy(pRsp->user, pUser->user, TSDB_USER_LEN);
pRsp->superAuth = 1; pRsp->superAuth = 1;
pRsp->enable = pUser->enable; pRsp->enable = pUser->enable;
pRsp->sysInfo = pUser->sysInfo;
pRsp->version = pUser->authVersion; pRsp->version = pUser->authVersion;
pRsp->passVer = pUser->passVersion;
return 0; return 0;
} }
#endif #endif

14
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -288,6 +288,7 @@ _CONNECT:
connectRsp.dnodeNum = mndGetDnodeSize(pMnode); connectRsp.dnodeNum = mndGetDnodeSize(pMnode);
connectRsp.svrTimestamp = taosGetTimestampSec(); connectRsp.svrTimestamp = taosGetTimestampSec();
connectRsp.passVer = pUser->passVersion; connectRsp.passVer = pUser->passVersion;
connectRsp.authVer = pUser->authVersion;
strcpy(connectRsp.sVer, version); strcpy(connectRsp.sVer, version);
snprintf(connectRsp.sDetailVer, sizeof(connectRsp.sDetailVer), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo, snprintf(connectRsp.sDetailVer, sizeof(connectRsp.sDetailVer), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo,
@ -552,16 +553,6 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
} }
break; break;
} }
case HEARTBEAT_KEY_USER_PASSINFO: {
void *rspMsg = NULL;
int32_t rspLen = 0;
mndValidateUserPassInfo(pMnode, kv->value, kv->valueLen / sizeof(SUserPassVersion), &rspMsg, &rspLen);
if (rspMsg && rspLen > 0) {
SKv kv1 = {.key = HEARTBEAT_KEY_USER_PASSINFO, .valueLen = rspLen, .value = rspMsg};
taosArrayPush(hbRsp.info, &kv1);
}
break;
}
default: default:
mError("invalid kv key:%d", kv->key); mError("invalid kv key:%d", kv->key);
hbRsp.status = TSDB_CODE_APP_ERROR; hbRsp.status = TSDB_CODE_APP_ERROR;
@ -827,6 +818,9 @@ static int32_t packQueriesIntoBlock(SShowObj* pShow, SConnObj* pConn, SSDataBloc
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, curRowIndex, (const char *)&pQuery->stableQuery, false); colDataSetVal(pColInfo, curRowIndex, (const char *)&pQuery->stableQuery, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, curRowIndex, (const char *)&pQuery->isSubQuery, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, curRowIndex, (const char *)&pQuery->subPlanNum, false); colDataSetVal(pColInfo, curRowIndex, (const char *)&pQuery->subPlanNum, false);

2
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -275,7 +275,7 @@ static void doAddNewConsumers(SMqRebOutputObj *pOutput, const SMqRebInputObj *pI
taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp)); taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp));
taosArrayPush(pOutput->newConsumers, &consumerId); taosArrayPush(pOutput->newConsumers, &consumerId);
mInfo("sub:%s mq rebalance add new consumer:%" PRIx64, pSubKey, consumerId); mInfo("sub:%s mq rebalance add new consumer:0x%" PRIx64, pSubKey, consumerId);
} }
} }

67
source/dnode/mnode/impl/src/mndUser.c
View File

@ -801,7 +801,8 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
goto _OVER; goto _OVER;
} }
if (TSDB_ALTER_USER_PASSWD == alterReq.alterType && alterReq.pass[0] == 0) { if (TSDB_ALTER_USER_PASSWD == alterReq.alterType &&
(alterReq.pass[0] == 0 || strlen(alterReq.pass) >= TSDB_PASSWORD_LEN)) {
terrno = TSDB_CODE_MND_INVALID_PASS_FORMAT; terrno = TSDB_CODE_MND_INVALID_PASS_FORMAT;
goto _OVER; goto _OVER;
} }
@ -824,7 +825,6 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
if (mndUserDupObj(pUser, &newUser) != 0) goto _OVER; if (mndUserDupObj(pUser, &newUser) != 0) goto _OVER;
newUser.passVersion = pUser->passVersion;
if (alterReq.alterType == TSDB_ALTER_USER_PASSWD) { if (alterReq.alterType == TSDB_ALTER_USER_PASSWD) {
char pass[TSDB_PASSWORD_LEN + 1] = {0}; char pass[TSDB_PASSWORD_LEN + 1] = {0};
taosEncryptPass_c((uint8_t *)alterReq.pass, strlen(alterReq.pass), pass); taosEncryptPass_c((uint8_t *)alterReq.pass, strlen(alterReq.pass), pass);
@ -1431,69 +1431,6 @@ _OVER:
return code; return code;
} }
int32_t mndValidateUserPassInfo(SMnode *pMnode, SUserPassVersion *pUsers, int32_t numOfUses, void **ppRsp,
int32_t *pRspLen) {
int32_t code = 0;
SUserPassBatchRsp batchRsp = {0};
for (int32_t i = 0; i < numOfUses; ++i) {
SUserObj *pUser = mndAcquireUser(pMnode, pUsers[i].user);
if (pUser == NULL) {
mError("user:%s, failed to validate user pass since %s", pUsers[i].user, terrstr());
continue;
}
pUsers[i].version = ntohl(pUsers[i].version);
if (pUser->passVersion <= pUsers[i].version) {
mTrace("user:%s, not update since mnd passVer %d <= client passVer %d", pUsers[i].user, pUser->passVersion,
pUsers[i].version);
mndReleaseUser(pMnode, pUser);
continue;
}
SGetUserPassRsp rsp = {0};
memcpy(rsp.user, pUser->user, TSDB_USER_LEN);
rsp.version = pUser->passVersion;
if (!batchRsp.pArray && !(batchRsp.pArray = taosArrayInit(numOfUses, sizeof(SGetUserPassRsp)))) {
code = TSDB_CODE_OUT_OF_MEMORY;
mndReleaseUser(pMnode, pUser);
goto _OVER;
}
taosArrayPush(batchRsp.pArray, &rsp);
mndReleaseUser(pMnode, pUser);
}
if (taosArrayGetSize(batchRsp.pArray) <= 0) {
goto _OVER;
}
int32_t rspLen = tSerializeSUserPassBatchRsp(NULL, 0, &batchRsp);
if (rspLen < 0) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
void *pRsp = taosMemoryMalloc(rspLen);
if (pRsp == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
tSerializeSUserPassBatchRsp(pRsp, rspLen, &batchRsp);
*ppRsp = pRsp;
*pRspLen = rspLen;
_OVER:
if (code) {
*ppRsp = NULL;
*pRspLen = 0;
}
tFreeSUserPassBatchRsp(&batchRsp);
return code;
}
int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db) { int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db) {
int32_t code = 0; int32_t code = 0;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;

3
source/dnode/vnode/src/inc/tq.h
View File

@ -134,7 +134,8 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision); int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision);
int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp,
int32_t type, int32_t vgId); int32_t type, int32_t vgId);
int32_t tqPushDataRsp(STqHandle* pHandle, int32_t vgId); //int32_t tqPushDataRsp(STqHandle* pHandle, int32_t vgId);
int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId);
// tqMeta // tqMeta
int32_t tqMetaOpen(STQ* pTq); int32_t tqMetaOpen(STQ* pTq);

7
source/dnode/vnode/src/meta/metaTable.c
View File

@ -207,7 +207,10 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
tb_uid_t uid = *(tb_uid_t *)pData; tb_uid_t uid = *(tb_uid_t *)pData;
tdbFree(pData); tdbFree(pData);
SMetaInfo info; SMetaInfo info;
metaGetInfo(pMeta, uid, &info, NULL); if (metaGetInfo(pMeta, uid, &info, NULL) == TSDB_CODE_NOT_FOUND) {
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1;
}
if (info.uid == info.suid) { if (info.uid == info.suid) {
return 0; return 0;
} else { } else {
@ -939,7 +942,7 @@ int metaTtlDropTable(SMeta *pMeta, int64_t timePointMs, SArray *tbUids) {
return 0; return 0;
} }
metaInfo("ttl find expired table count: %zu" , TARRAY_SIZE(tbUids)); metaInfo("ttl find expired table count: %zu", TARRAY_SIZE(tbUids));
metaDropTables(pMeta, tbUids); metaDropTables(pMeta, tbUids);
return 0; return 0;

128
source/dnode/vnode/src/tq/tq.c
View File

@ -266,26 +266,43 @@ void tqNotifyClose(STQ* pTq) {
// return 0; // return 0;
//} //}
int32_t tqPushDataRsp(STqHandle* pHandle, int32_t vgId) { int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) {
SMqPollReq req = {0};
if (tDeserializeSMqPollReq(pHandle->msg->pCont, pHandle->msg->contLen, &req) < 0) {
tqError("tDeserializeSMqPollReq %d failed", pHandle->msg->contLen);
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
SMqDataRsp dataRsp = {0}; SMqDataRsp dataRsp = {0};
dataRsp.head.consumerId = pHandle->consumerId; tqInitDataRsp(&dataRsp, &req);
dataRsp.head.epoch = pHandle->epoch; dataRsp.blockNum = 0;
dataRsp.head.mqMsgType = TMQ_MSG_TYPE__POLL_DATA_RSP; dataRsp.rspOffset = dataRsp.reqOffset;
tqSendDataRsp(pHandle, pHandle->msg, &req, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
int64_t sver = 0, ever = 0; tDeleteMqDataRsp(&dataRsp);
walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever);
tqDoSendDataRsp(&pHandle->msg->info, &dataRsp, pHandle->epoch, pHandle->consumerId, TMQ_MSG_TYPE__POLL_DATA_RSP, sver,
ever);
char buf1[TSDB_OFFSET_LEN] = {0};
char buf2[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf1, tListLen(buf1), &dataRsp.reqOffset);
tFormatOffset(buf2, tListLen(buf2), &dataRsp.rspOffset);
tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) push rsp, block num: %d, req:%s, rsp:%s", vgId,
dataRsp.head.consumerId, dataRsp.head.epoch, dataRsp.blockNum, buf1, buf2);
return 0; return 0;
} }
//int32_t tqPushDataRsp(STqHandle* pHandle, int32_t vgId) {
// SMqDataRsp dataRsp = {0};
// dataRsp.head.consumerId = pHandle->consumerId;
// dataRsp.head.epoch = pHandle->epoch;
// dataRsp.head.mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
//
// int64_t sver = 0, ever = 0;
// walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever);
// tqDoSendDataRsp(&pHandle->msg->info, &dataRsp, pHandle->epoch, pHandle->consumerId, TMQ_MSG_TYPE__POLL_RSP, sver,
// ever);
//
// char buf1[TSDB_OFFSET_LEN] = {0};
// char buf2[TSDB_OFFSET_LEN] = {0};
// tFormatOffset(buf1, tListLen(buf1), &dataRsp.reqOffset);
// tFormatOffset(buf2, tListLen(buf2), &dataRsp.rspOffset);
// tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) push rsp, block num: %d, req:%s, rsp:%s", vgId,
// dataRsp.head.consumerId, dataRsp.head.epoch, dataRsp.blockNum, buf1, buf2);
// return 0;
//}
int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp,
int32_t type, int32_t vgId) { int32_t type, int32_t vgId) {
int64_t sver = 0, ever = 0; int64_t sver = 0, ever = 0;
@ -524,7 +541,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
if (!exec) { if (!exec) {
tqSetHandleExec(pHandle); tqSetHandleExec(pHandle);
// qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); // qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS);
tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, tqDebug("tmq poll: consumer:0x%" PRIx64 " vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId,
req.subKey, pHandle); req.subKey, pHandle);
taosWUnLockLatch(&pTq->lock); taosWUnLockLatch(&pTq->lock);
break; break;
@ -544,7 +561,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
pHandle->epoch = reqEpoch; pHandle->epoch = reqEpoch;
} }
char buf[TSDB_OFFSET_LEN]; char buf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf, TSDB_OFFSET_LEN, &reqOffset); tFormatOffset(buf, TSDB_OFFSET_LEN, &reqOffset);
tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64, tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64,
consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId); consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId);
@ -552,7 +569,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); code = tqExtractDataForMq(pTq, pHandle, &req, pMsg);
tqSetHandleIdle(pHandle); tqSetHandleIdle(pHandle);
tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, , set handle idle, pHandle:%p", consumerId, vgId, tqDebug("tmq poll: consumer:0x%" PRIx64 " vgId:%d, topic:%s, set handle idle, pHandle:%p", consumerId, vgId,
req.subKey, pHandle); req.subKey, pHandle);
return code; return code;
} }
@ -594,48 +611,47 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) {
SMqDataRsp dataRsp = {0}; SMqDataRsp dataRsp = {0};
tqInitDataRsp(&dataRsp, &req); tqInitDataRsp(&dataRsp, &req);
STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, req.subKey); if (req.useSnapshot == true) {
if (pOffset != NULL) { tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s snapshot not support wal info", consumerId, vgId, req.subKey);
if (pOffset->val.type != TMQ_OFFSET__LOG) { terrno = TSDB_CODE_INVALID_PARA;
tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s use snapshot, no valid wal info", consumerId, vgId, req.subKey); tDeleteMqDataRsp(&dataRsp);
terrno = TSDB_CODE_INVALID_PARA; return -1;
tDeleteMqDataRsp(&dataRsp); }
return -1;
}
dataRsp.rspOffset.type = TMQ_OFFSET__LOG; dataRsp.rspOffset.type = TMQ_OFFSET__LOG;
dataRsp.rspOffset.version = pOffset->val.version;
} else {
if (req.useSnapshot == true) {
tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s snapshot not support wal info", consumerId, vgId, req.subKey);
terrno = TSDB_CODE_INVALID_PARA;
tDeleteMqDataRsp(&dataRsp);
return -1;
}
dataRsp.rspOffset.type = TMQ_OFFSET__LOG; if (reqOffset.type == TMQ_OFFSET__LOG) {
dataRsp.rspOffset.version = reqOffset.version;
if (reqOffset.type == TMQ_OFFSET__LOG) { } else if(reqOffset.type < 0){
int64_t currentVer = walReaderGetCurrentVer(pHandle->execHandle.pTqReader->pWalReader); STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, req.subKey);
if (currentVer == -1) { // not start to read data from wal yet, return req offset directly if (pOffset != NULL) {
dataRsp.rspOffset.version = reqOffset.version; if (pOffset->val.type != TMQ_OFFSET__LOG) {
} else { tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s, no valid wal info", consumerId, vgId, req.subKey);
dataRsp.rspOffset.version = currentVer; // return current consume offset value terrno = TSDB_CODE_INVALID_PARA;
tDeleteMqDataRsp(&dataRsp);
return -1;
} }
} else if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEST) {
dataRsp.rspOffset.version = sver; // not consume yet, set the earliest position dataRsp.rspOffset.version = pOffset->val.version;
} else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { tqInfo("consumer:0x%" PRIx64 " vgId:%d subkey:%s get assignment from store:%"PRId64, consumerId, vgId, req.subKey, dataRsp.rspOffset.version);
dataRsp.rspOffset.version = ever; }else{
} else { if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEST) {
tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s invalid offset type:%d", consumerId, vgId, req.subKey, dataRsp.rspOffset.version = sver; // not consume yet, set the earliest position
reqOffset.type); } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
terrno = TSDB_CODE_INVALID_PARA; dataRsp.rspOffset.version = ever;
tDeleteMqDataRsp(&dataRsp); }
return -1; tqInfo("consumer:0x%" PRIx64 " vgId:%d subkey:%s get assignment from init:%"PRId64, consumerId, vgId, req.subKey, dataRsp.rspOffset.version);
} }
} else {
tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s invalid offset type:%d", consumerId, vgId, req.subKey,
reqOffset.type);
terrno = TSDB_CODE_INVALID_PARA;
tDeleteMqDataRsp(&dataRsp);
return -1;
} }
tqDoSendDataRsp(&pMsg->info, &dataRsp, req.epoch, req.consumerId, TMQ_MSG_TYPE__WALINFO_RSP, sver, ever); tqDoSendDataRsp(&pMsg->info, &dataRsp, req.epoch, req.consumerId, TMQ_MSG_TYPE__WALINFO_RSP, sver, ever);
tDeleteMqDataRsp(&dataRsp);
return 0; return 0;
} }
@ -1300,7 +1316,7 @@ int32_t extractDelDataBlock(const void* pData, int32_t len, int64_t ver, SStream
SDecoder* pCoder = &(SDecoder){0}; SDecoder* pCoder = &(SDecoder){0};
SDeleteRes* pRes = &(SDeleteRes){0}; SDeleteRes* pRes = &(SDeleteRes){0};
*pRefBlock = NULL; (*pRefBlock) = NULL;
pRes->uidList = taosArrayInit(0, sizeof(tb_uid_t)); pRes->uidList = taosArrayInit(0, sizeof(tb_uid_t));
if (pRes->uidList == NULL) { if (pRes->uidList == NULL) {
@ -1340,7 +1356,7 @@ int32_t extractDelDataBlock(const void* pData, int32_t len, int64_t ver, SStream
taosArrayDestroy(pRes->uidList); taosArrayDestroy(pRes->uidList);
*pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0); *pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0);
if (pRefBlock == NULL) { if ((*pRefBlock) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }

2
source/dnode/vnode/src/tq/tqOffset.c
View File

@ -104,7 +104,7 @@ STqOffsetStore* tqOffsetOpen(STQ* pTq) {
pStore->needCommit = 0; pStore->needCommit = 0;
pTq->pOffsetStore = pStore; pTq->pOffsetStore = pStore;
pStore->pHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); pStore->pHash = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
if (pStore->pHash == NULL) { if (pStore->pHash == NULL) {
taosMemoryFree(pStore); taosMemoryFree(pStore);
return NULL; return NULL;

7
source/dnode/vnode/src/tq/tqPush.c
View File

@ -64,7 +64,9 @@ int32_t tqRegisterPushHandle(STQ* pTq, void* handle, SRpcMsg* pMsg) {
memcpy(pHandle->msg, pMsg, sizeof(SRpcMsg)); memcpy(pHandle->msg, pMsg, sizeof(SRpcMsg));
pHandle->msg->pCont = rpcMallocCont(pMsg->contLen); pHandle->msg->pCont = rpcMallocCont(pMsg->contLen);
} else { } else {
tqPushDataRsp(pHandle, vgId); // tqPushDataRsp(pHandle, vgId);
tqPushEmptyDataRsp(pHandle, vgId);
void* tmp = pHandle->msg->pCont; void* tmp = pHandle->msg->pCont;
memcpy(pHandle->msg, pMsg, sizeof(SRpcMsg)); memcpy(pHandle->msg, pMsg, sizeof(SRpcMsg));
pHandle->msg->pCont = tmp; pHandle->msg->pCont = tmp;
@ -89,7 +91,8 @@ int32_t tqUnregisterPushHandle(STQ* pTq, void *handle) {
tqDebug("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId); tqDebug("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId);
if(pHandle->msg != NULL) { if(pHandle->msg != NULL) {
tqPushDataRsp(pHandle, vgId); // tqPushDataRsp(pHandle, vgId);
tqPushEmptyDataRsp(pHandle, vgId);
rpcFreeCont(pHandle->msg->pCont); rpcFreeCont(pHandle->msg->pCont);
taosMemoryFree(pHandle->msg); taosMemoryFree(pHandle->msg);

3
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -93,7 +93,7 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
if (pOffset != NULL) { if (pOffset != NULL) {
*pOffsetVal = pOffset->val; *pOffsetVal = pOffset->val;
char formatBuf[TSDB_OFFSET_LEN]; char formatBuf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(formatBuf, TSDB_OFFSET_LEN, pOffsetVal); tFormatOffset(formatBuf, TSDB_OFFSET_LEN, pOffsetVal);
tqDebug("tmq poll: consumer:0x%" PRIx64 tqDebug("tmq poll: consumer:0x%" PRIx64
", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. reqId:0x%" PRIx64, ", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. reqId:0x%" PRIx64,
@ -145,6 +145,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
uint64_t consumerId = pRequest->consumerId; uint64_t consumerId = pRequest->consumerId;
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
int code = 0; int code = 0;
terrno = 0;
SMqDataRsp dataRsp = {0}; SMqDataRsp dataRsp = {0};
tqInitDataRsp(&dataRsp, pRequest); tqInitDataRsp(&dataRsp, pRequest);

115
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -1026,7 +1026,7 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache
return code; return code;
} }
/*
int32_t tsdbCacheGet(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype) { int32_t tsdbCacheGet(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype) {
int32_t code = 0; int32_t code = 0;
SLRUCache *pCache = pTsdb->lruCache; SLRUCache *pCache = pTsdb->lruCache;
@ -1074,7 +1074,7 @@ int32_t tsdbCacheGet(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsR
return code; return code;
} }
*/
int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey) { int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey) {
int32_t code = 0; int32_t code = 0;
// fetch schema // fetch schema
@ -1824,10 +1824,11 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
} }
*pIgnoreEarlierTs = false; *pIgnoreEarlierTs = false;
/*
if (!hasVal) { if (!hasVal) {
state->state = SFSLASTNEXTROW_FILESET; state->state = SFSLASTNEXTROW_FILESET;
} }
*/
if (!state->checkRemainingRow) { if (!state->checkRemainingRow) {
state->checkRemainingRow = true; state->checkRemainingRow = true;
} }
@ -2015,10 +2016,9 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetDataBlk); tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetDataBlk);
if (block.maxKey.ts <= state->lastTs) { if (block.maxKey.ts <= state->lastTs) {
*pIgnoreEarlierTs = true; *pIgnoreEarlierTs = true;
if (state->pBlockData) {
tBlockDataDestroy(state->pBlockData); tBlockDataDestroy(state->pBlockData);
state->pBlockData = NULL; state->pBlockData = NULL;
}
*ppRow = NULL; *ppRow = NULL;
return code; return code;
@ -3171,97 +3171,46 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray,
TSKEY rowTs = TSDBROW_TS(pRow); TSKEY rowTs = TSDBROW_TS(pRow);
if (lastRowTs == TSKEY_MAX) { lastRowTs = rowTs;
lastRowTs = rowTs;
for (int16_t iCol = noneCol; iCol < nCols; ++iCol) {
if (iCol >= nLastCol) {
break;
}
SLastCol *pCol = taosArrayGet(pColArray, iCol);
if (pCol->colVal.cid != pTSchema->columns[slotIds[iCol]].colId) {
continue;
}
if (slotIds[iCol] == 0) {
STColumn *pTColumn = &pTSchema->columns[0];
*pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = rowTs});
taosArraySet(pColArray, 0, &(SLastCol){.ts = rowTs, .colVal = *pColVal});
continue;
}
tsdbRowGetColVal(pRow, pTSchema, slotIds[iCol], pColVal);
*pCol = (SLastCol){.ts = rowTs, .colVal = *pColVal};
if (IS_VAR_DATA_TYPE(pColVal->type) /*&& pColVal->value.nData > 0*/) {
pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData);
if (pCol->colVal.value.pData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
if (pColVal->value.nData > 0) {
memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
}
}
/*if (COL_VAL_IS_NONE(pColVal)) {
if (!setNoneCol) {
noneCol = iCol;
setNoneCol = true;
}
} else {*/
int32_t aColIndex = taosArraySearchIdx(aColArray, &pColVal->cid, compareInt16Val, TD_EQ);
if (aColIndex >= 0) {
taosArrayRemove(aColArray, aColIndex);
}
//}
}
if (!setNoneCol) {
// done, goto return pColArray
break;
} else {
continue;
}
}
// merge into pColArray
setNoneCol = false;
for (int16_t iCol = noneCol; iCol < nCols; ++iCol) { for (int16_t iCol = noneCol; iCol < nCols; ++iCol) {
if (iCol >= nLastCol) { if (iCol >= nLastCol) {
break; break;
} }
// high version's column value SLastCol *pCol = taosArrayGet(pColArray, iCol);
SLastCol *lastColVal = (SLastCol *)taosArrayGet(pColArray, iCol); if (pCol->colVal.cid != pTSchema->columns[slotIds[iCol]].colId) {
if (lastColVal->colVal.cid != pTSchema->columns[slotIds[iCol]].colId) {
continue; continue;
} }
SColVal *tColVal = &lastColVal->colVal; if (slotIds[iCol] == 0) {
STColumn *pTColumn = &pTSchema->columns[0];
*pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = rowTs});
taosArraySet(pColArray, 0, &(SLastCol){.ts = rowTs, .colVal = *pColVal});
continue;
}
tsdbRowGetColVal(pRow, pTSchema, slotIds[iCol], pColVal); tsdbRowGetColVal(pRow, pTSchema, slotIds[iCol], pColVal);
if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) {
SLastCol lastCol = {.ts = rowTs, .colVal = *pColVal};
if (IS_VAR_DATA_TYPE(pColVal->type) && pColVal->value.nData > 0) {
SLastCol *pLastCol = (SLastCol *)taosArrayGet(pColArray, iCol);
taosMemoryFree(pLastCol->colVal.value.pData);
lastCol.colVal.value.pData = taosMemoryMalloc(lastCol.colVal.value.nData); *pCol = (SLastCol){.ts = rowTs, .colVal = *pColVal};
if (lastCol.colVal.value.pData == NULL) { if (IS_VAR_DATA_TYPE(pColVal->type) /*&& pColVal->value.nData > 0*/) {
terrno = TSDB_CODE_OUT_OF_MEMORY; pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData);
code = TSDB_CODE_OUT_OF_MEMORY; if (pCol->colVal.value.pData == NULL) {
goto _err; terrno = TSDB_CODE_OUT_OF_MEMORY;
} code = TSDB_CODE_OUT_OF_MEMORY;
memcpy(lastCol.colVal.value.pData, pColVal->value.pData, pColVal->value.nData); goto _err;
} }
if (pColVal->value.nData > 0) {
memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
}
}
taosArraySet(pColArray, iCol, &lastCol); int32_t aColIndex = taosArraySearchIdx(aColArray, &pColVal->cid, compareInt16Val, TD_EQ);
int32_t aColIndex = taosArraySearchIdx(aColArray, &lastCol.colVal.cid, compareInt16Val, TD_EQ); if (aColIndex >= 0) {
taosArrayRemove(aColArray, aColIndex); taosArrayRemove(aColArray, aColIndex);
} else if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal) && !setNoneCol) {
noneCol = iCol;
setNoneCol = true;
} }
} }
} while (setNoneCol);
break;
} while (1);
if (!hasRow) { if (!hasRow) {
if (ignoreEarlierTs) { if (ignoreEarlierTs) {

6
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -289,6 +289,10 @@ static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pC
static int32_t updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) { static int32_t updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) {
int32_t i = 0, j = 0; int32_t i = 0, j = 0;
if (j < pSupInfo->numOfCols && PRIMARYKEY_TIMESTAMP_COL_ID == pSupInfo->colId[j]) {
j += 1;
}
while (i < pSchema->numOfCols && j < pSupInfo->numOfCols) { while (i < pSchema->numOfCols && j < pSupInfo->numOfCols) {
STColumn* pTCol = &pSchema->columns[i]; STColumn* pTCol = &pSchema->columns[i];
if (pTCol->colId == pSupInfo->colId[j]) { if (pTCol->colId == pSupInfo->colId[j]) {
@ -3064,6 +3068,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr
// only check here, since the iterate data in memory is very fast. // only check here, since the iterate data in memory is very fast.
if (pReader->code != TSDB_CODE_SUCCESS) { if (pReader->code != TSDB_CODE_SUCCESS) {
tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr);
taosArrayDestroy(pIndexList);
return pReader->code; return pReader->code;
} }
@ -5586,4 +5591,3 @@ void tsdbReaderSetId(STsdbReader* pReader, const char* idstr) {
} }
void tsdbReaderSetCloseFlag(STsdbReader* pReader) { pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED; } void tsdbReaderSetCloseFlag(STsdbReader* pReader) { pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED; }

7
source/dnode/vnode/src/vnd/vnodeCommit.c
View File

@ -356,7 +356,12 @@ static int32_t vnodeCommitTask(void *arg) {
// commit // commit
code = vnodeCommitImpl(pInfo); code = vnodeCommitImpl(pInfo);
if (code) goto _exit; if (code) {
vFatal("vgId:%d, failed to commit vnode since %s", TD_VID(pVnode), terrstr());
taosMsleep(100);
exit(EXIT_FAILURE);
goto _exit;
}
vnodeReturnBufPool(pVnode); vnodeReturnBufPool(pVnode);

2
source/dnode/vnode/src/vnd/vnodeRetention.c
View File

@ -113,7 +113,7 @@ int32_t vnodeAsyncRentention(SVnode *pVnode, int64_t now) {
_exit: _exit:
if (code) { if (code) {
vError("vgId:%d %s failed at line %d since %s", TD_VID(pInfo->pVnode), __func__, lino, tstrerror(code)); vError("vgId:%d %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code));
if (pInfo) taosMemoryFree(pInfo); if (pInfo) taosMemoryFree(pInfo);
} else { } else {
vInfo("vgId:%d %s done", TD_VID(pInfo->pVnode), __func__); vInfo("vgId:%d %s done", TD_VID(pInfo->pVnode), __func__);

11
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -567,7 +567,9 @@ int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
vTrace("message in vnode query queue is processing"); vTrace("message in vnode query queue is processing");
if ((pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_VND_TMQ_CONSUME || pMsg->msgType == TDMT_VND_TMQ_CONSUME_PUSH) && !syncIsReadyForRead(pVnode->sync)) { if ((pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_VND_TMQ_CONSUME ||
pMsg->msgType == TDMT_VND_TMQ_CONSUME_PUSH) &&
!syncIsReadyForRead(pVnode->sync)) {
vnodeRedirectRpcMsg(pVnode, pMsg, terrno); vnodeRedirectRpcMsg(pVnode, pMsg, terrno);
return 0; return 0;
} }
@ -623,8 +625,8 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
return vnodeGetTableCfg(pVnode, pMsg, true); return vnodeGetTableCfg(pVnode, pMsg, true);
case TDMT_VND_BATCH_META: case TDMT_VND_BATCH_META:
return vnodeGetBatchMeta(pVnode, pMsg); return vnodeGetBatchMeta(pVnode, pMsg);
// case TDMT_VND_TMQ_CONSUME: // case TDMT_VND_TMQ_CONSUME:
// return tqProcessPollReq(pVnode->pTq, pMsg); // return tqProcessPollReq(pVnode->pTq, pMsg);
case TDMT_VND_TMQ_VG_WALINFO: case TDMT_VND_TMQ_VG_WALINFO:
return tqProcessVgWalInfoReq(pVnode->pTq, pMsg); return tqProcessVgWalInfoReq(pVnode->pTq, pMsg);
default: default:
@ -1376,7 +1378,8 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in
} }
if (info.suid) { if (info.suid) {
metaGetInfo(pVnode->pMeta, info.suid, &info, NULL); code = metaGetInfo(pVnode->pMeta, info.suid, &info, NULL);
ASSERT(code == 0);
} }
if (pSubmitTbData->sver != info.skmVer) { if (pSubmitTbData->sver != info.skmVer) {

5
source/libs/catalog/src/catalog.c
View File

@ -341,13 +341,10 @@ int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pReq,
SCtgAuthReq req = {0}; SCtgAuthReq req = {0};
req.pRawReq = pReq; req.pRawReq = pReq;
req.pConn = pConn; req.pConn = pConn;
req.onlyCache = exists ? true : false; req.onlyCache = false;
CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, pReq->user, &req.authInfo, NULL)); CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, pReq->user, &req.authInfo, NULL));
CTG_ERR_JRET(ctgChkSetAuthRes(pCtg, &req, &rsp)); CTG_ERR_JRET(ctgChkSetAuthRes(pCtg, &req, &rsp));
if (rsp.metaNotExists && exists) {
*exists = false;
}
_return: _return:

4
source/libs/catalog/src/ctgCache.c
View File

@ -1721,9 +1721,7 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam
ctgDebug("stb 0x%" PRIx64 " updated to cache, dbFName:%s, tbName:%s, tbType:%d", meta->suid, dbFName, tbName, ctgDebug("stb 0x%" PRIx64 " updated to cache, dbFName:%s, tbName:%s, tbType:%d", meta->suid, dbFName, tbName,
meta->tableType); meta->tableType);
if (pCache) { CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbId, meta->suid, pCache));
CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbId, meta->suid, pCache));
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

1
source/libs/catalog/src/ctgUtil.c
View File

@ -926,7 +926,6 @@ int32_t ctgGenerateVgList(SCatalog* pCtg, SHashObj* vgHash, SArray** pList) {
} }
pIter = taosHashIterate(vgHash, pIter); pIter = taosHashIterate(vgHash, pIter);
vgInfo = NULL;
} }
*pList = vgList; *pList = vgList;

9
source/libs/command/src/command.c
View File

@ -87,7 +87,7 @@ static int32_t buildDescResultDataBlock(SSDataBlock** pOutput) {
return code; return code;
} }
static void setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, int32_t numOfRows, STableMeta* pMeta) { static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, int32_t numOfRows, STableMeta* pMeta) {
blockDataEnsureCapacity(pBlock, numOfRows); blockDataEnsureCapacity(pBlock, numOfRows);
pBlock->info.rows = 0; pBlock->info.rows = 0;
@ -114,6 +114,11 @@ static void setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, in
colDataSetVal(pCol4, pBlock->info.rows, buf, false); colDataSetVal(pCol4, pBlock->info.rows, buf, false);
++(pBlock->info.rows); ++(pBlock->info.rows);
} }
if (pBlock->info.rows <= 0) {
qError("no permission to view any columns");
return TSDB_CODE_PAR_PERMISSION_DENIED;
}
return TSDB_CODE_SUCCESS;
} }
static int32_t execDescribe(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp) { static int32_t execDescribe(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp) {
@ -123,7 +128,7 @@ static int32_t execDescribe(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp**
SSDataBlock* pBlock = NULL; SSDataBlock* pBlock = NULL;
int32_t code = buildDescResultDataBlock(&pBlock); int32_t code = buildDescResultDataBlock(&pBlock);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
setDescResultIntoDataBlock(sysInfoUser, pBlock, numOfRows, pDesc->pMeta); code = setDescResultIntoDataBlock(sysInfoUser, pBlock, numOfRows, pDesc->pMeta);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = buildRetrieveTableRsp(pBlock, DESCRIBE_RESULT_COLS, pRsp); code = buildRetrieveTableRsp(pBlock, DESCRIBE_RESULT_COLS, pRsp);

1
source/libs/executor/src/executil.c
View File

@ -1109,7 +1109,6 @@ int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, S
code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status, &pStorageAPI->metaFilter); code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status, &pStorageAPI->metaFilter);
if (code != 0 || status == SFLT_NOT_INDEX) { // temporarily disable it for performance sake if (code != 0 || status == SFLT_NOT_INDEX) { // temporarily disable it for performance sake
qDebug("failed to get tableIds from index, suid:%" PRIu64, pScanNode->uid); qDebug("failed to get tableIds from index, suid:%" PRIu64, pScanNode->uid);
code = TSDB_CODE_SUCCESS;
} else { } else {
qInfo("succ to get filter result, table num: %d", (int)taosArrayGetSize(pUidList)); qInfo("succ to get filter result, table num: %d", (int)taosArrayGetSize(pUidList));
} }

9
source/libs/executor/src/joinoperator.c
View File

@ -150,9 +150,12 @@ static int32_t initTagColskeyBuf(int32_t* keyLen, char** keyBuf, const SArray* p
int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols; int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols;
(*keyLen) += nullFlagSize; (*keyLen) += nullFlagSize;
(*keyBuf) = taosMemoryCalloc(1, (*keyLen)); if (*keyLen >= 0) {
if ((*keyBuf) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; (*keyBuf) = taosMemoryCalloc(1, (*keyLen));
if ((*keyBuf) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;

14
source/libs/executor/src/timesliceoperator.c
View File

@ -315,7 +315,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) { if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) {
float v = 0; float v = 0;
if (!IS_VAR_DATA_TYPE(pVar->nType)) { if (!IS_VAR_DATA_TYPE(pVar->nType)) {
GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); GET_TYPED_DATA(v, float, pVar->nType, &pVar->f);
} else { } else {
v = taosStr2Float(varDataVal(pVar->pz), NULL); v = taosStr2Float(varDataVal(pVar->pz), NULL);
} }
@ -323,7 +323,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
} else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) { } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) {
double v = 0; double v = 0;
if (!IS_VAR_DATA_TYPE(pVar->nType)) { if (!IS_VAR_DATA_TYPE(pVar->nType)) {
GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); GET_TYPED_DATA(v, double, pVar->nType, &pVar->d);
} else { } else {
v = taosStr2Double(varDataVal(pVar->pz), NULL); v = taosStr2Double(varDataVal(pVar->pz), NULL);
} }
@ -333,7 +333,15 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
if (!IS_VAR_DATA_TYPE(pVar->nType)) { if (!IS_VAR_DATA_TYPE(pVar->nType)) {
GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i);
} else { } else {
v = taosStr2int64(varDataVal(pVar->pz)); v = taosStr2Int64(varDataVal(pVar->pz), NULL, 10);
}
colDataSetVal(pDst, rows, (char*)&v, false);
} else if (IS_UNSIGNED_NUMERIC_TYPE(pDst->info.type)) {
uint64_t v = 0;
if (!IS_VAR_DATA_TYPE(pVar->nType)) {
GET_TYPED_DATA(v, uint64_t, pVar->nType, &pVar->u);
} else {
v = taosStr2UInt64(varDataVal(pVar->pz), NULL, 10);
} }
colDataSetVal(pDst, rows, (char*)&v, false); colDataSetVal(pDst, rows, (char*)&v, false);
} else if (IS_BOOLEAN_TYPE(pDst->info.type)) { } else if (IS_BOOLEAN_TYPE(pDst->info.type)) {

109
source/libs/function/src/builtinsimpl.c
View File

@ -2702,13 +2702,12 @@ static int32_t doSetPrevVal(SDiffInfo* pDiffInfo, int32_t type, const char* pv,
} }
static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, SColumnInfoData* pOutput, int32_t pos, static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, SColumnInfoData* pOutput, int32_t pos,
int32_t order, int64_t ts) { int64_t ts) {
int32_t factor = (order == TSDB_ORDER_ASC) ? 1 : -1;
pDiffInfo->prevTs = ts; pDiffInfo->prevTs = ts;
switch (type) { switch (type) {
case TSDB_DATA_TYPE_INT: { case TSDB_DATA_TYPE_INT: {
int32_t v = *(int32_t*)pv; int32_t v = *(int32_t*)pv;
int64_t delta = factor * (v - pDiffInfo->prev.i64); // direct previous may be null int64_t delta = v - pDiffInfo->prev.i64; // direct previous may be null
if (delta < 0 && pDiffInfo->ignoreNegative) { if (delta < 0 && pDiffInfo->ignoreNegative) {
colDataSetNull_f_s(pOutput, pos); colDataSetNull_f_s(pOutput, pos);
} else { } else {
@ -2721,7 +2720,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv,
case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT: { case TSDB_DATA_TYPE_TINYINT: {
int8_t v = *(int8_t*)pv; int8_t v = *(int8_t*)pv;
int64_t delta = factor * (v - pDiffInfo->prev.i64); // direct previous may be null int64_t delta = v - pDiffInfo->prev.i64; // direct previous may be null
if (delta < 0 && pDiffInfo->ignoreNegative) { if (delta < 0 && pDiffInfo->ignoreNegative) {
colDataSetNull_f_s(pOutput, pos); colDataSetNull_f_s(pOutput, pos);
} else { } else {
@ -2732,7 +2731,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv,
} }
case TSDB_DATA_TYPE_SMALLINT: { case TSDB_DATA_TYPE_SMALLINT: {
int16_t v = *(int16_t*)pv; int16_t v = *(int16_t*)pv;
int64_t delta = factor * (v - pDiffInfo->prev.i64); // direct previous may be null int64_t delta = v - pDiffInfo->prev.i64; // direct previous may be null
if (delta < 0 && pDiffInfo->ignoreNegative) { if (delta < 0 && pDiffInfo->ignoreNegative) {
colDataSetNull_f_s(pOutput, pos); colDataSetNull_f_s(pOutput, pos);
} else { } else {
@ -2744,7 +2743,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv,
case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_BIGINT: { case TSDB_DATA_TYPE_BIGINT: {
int64_t v = *(int64_t*)pv; int64_t v = *(int64_t*)pv;
int64_t delta = factor * (v - pDiffInfo->prev.i64); // direct previous may be null int64_t delta = v - pDiffInfo->prev.i64; // direct previous may be null
if (delta < 0 && pDiffInfo->ignoreNegative) { if (delta < 0 && pDiffInfo->ignoreNegative) {
colDataSetNull_f_s(pOutput, pos); colDataSetNull_f_s(pOutput, pos);
} else { } else {
@ -2755,7 +2754,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv,
} }
case TSDB_DATA_TYPE_FLOAT: { case TSDB_DATA_TYPE_FLOAT: {
float v = *(float*)pv; float v = *(float*)pv;
double delta = factor * (v - pDiffInfo->prev.d64); // direct previous may be null double delta = v - pDiffInfo->prev.d64; // direct previous may be null
if ((delta < 0 && pDiffInfo->ignoreNegative) || isinf(delta) || isnan(delta)) { // check for overflow if ((delta < 0 && pDiffInfo->ignoreNegative) || isinf(delta) || isnan(delta)) { // check for overflow
colDataSetNull_f_s(pOutput, pos); colDataSetNull_f_s(pOutput, pos);
} else { } else {
@ -2766,7 +2765,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv,
} }
case TSDB_DATA_TYPE_DOUBLE: { case TSDB_DATA_TYPE_DOUBLE: {
double v = *(double*)pv; double v = *(double*)pv;
double delta = factor * (v - pDiffInfo->prev.d64); // direct previous may be null double delta = v - pDiffInfo->prev.d64; // direct previous may be null
if ((delta < 0 && pDiffInfo->ignoreNegative) || isinf(delta) || isnan(delta)) { // check for overflow if ((delta < 0 && pDiffInfo->ignoreNegative) || isinf(delta) || isnan(delta)) { // check for overflow
colDataSetNull_f_s(pOutput, pos); colDataSetNull_f_s(pOutput, pos);
} else { } else {
@ -2797,82 +2796,42 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) {
SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput;
if (pCtx->order == TSDB_ORDER_ASC) { for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) {
for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { int32_t pos = startOffset + numOfElems;
int32_t pos = startOffset + numOfElems;
if (colDataIsNull_f(pInputCol->nullbitmap, i)) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) {
if (pDiffInfo->includeNull) { if (pDiffInfo->includeNull) {
colDataSetNull_f_s(pOutput, pos); colDataSetNull_f_s(pOutput, pos);
numOfElems += 1; numOfElems += 1;
}
continue;
} }
continue;
char* pv = colDataGetData(pInputCol, i);
if (pDiffInfo->hasPrev) {
if (tsList[i] == pDiffInfo->prevTs) {
return TSDB_CODE_FUNC_DUP_TIMESTAMP;
}
int32_t code = doHandleDiff(pDiffInfo, pInputCol->info.type, pv, pOutput, pos, pCtx->order, tsList[i]);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
// handle selectivity
if (pCtx->subsidiaries.num > 0) {
appendSelectivityValue(pCtx, i, pos);
}
numOfElems++;
} else {
int32_t code = doSetPrevVal(pDiffInfo, pInputCol->info.type, pv, tsList[i]);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
}
pDiffInfo->hasPrev = true;
} }
} else {
for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) {
int32_t pos = startOffset + numOfElems;
if (colDataIsNull_f(pInputCol->nullbitmap, i)) { char* pv = colDataGetData(pInputCol, i);
if (pDiffInfo->includeNull) {
colDataSetNull_f_s(pOutput, pos); if (pDiffInfo->hasPrev) {
numOfElems += 1; if (tsList[i] == pDiffInfo->prevTs) {
} return TSDB_CODE_FUNC_DUP_TIMESTAMP;
continue; }
int32_t code = doHandleDiff(pDiffInfo, pInputCol->info.type, pv, pOutput, pos, tsList[i]);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
// handle selectivity
if (pCtx->subsidiaries.num > 0) {
appendSelectivityValue(pCtx, i, pos);
} }
char* pv = colDataGetData(pInputCol, i); numOfElems++;
} else {
// there is a row of previous data block to be handled in the first place. int32_t code = doSetPrevVal(pDiffInfo, pInputCol->info.type, pv, tsList[i]);
if (pDiffInfo->hasPrev) { if (code != TSDB_CODE_SUCCESS) {
if (tsList[i] == pDiffInfo->prevTs) { return code;
return TSDB_CODE_FUNC_DUP_TIMESTAMP;
}
int32_t code = doHandleDiff(pDiffInfo, pInputCol->info.type, pv, pOutput, pos, pCtx->order, tsList[i]);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
// handle selectivity
if (pCtx->subsidiaries.num > 0) {
appendSelectivityValue(pCtx, i, pos);
}
numOfElems++;
} else {
int32_t code = doSetPrevVal(pDiffInfo, pInputCol->info.type, pv, tsList[i]);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
} }
pDiffInfo->hasPrev = true;
} }
pDiffInfo->hasPrev = true;
} }
pResInfo->numOfRes = numOfElems; pResInfo->numOfRes = numOfElems;

9
source/libs/function/src/udfd.c
View File

@ -29,6 +29,7 @@
#include "tmsg.h" #include "tmsg.h"
#include "trpc.h" #include "trpc.h"
#include "tmisce.h" #include "tmisce.h"
#include "tversion.h"
// clang-format on // clang-format on
#define UDFD_MAX_SCRIPT_PLUGINS 64 #define UDFD_MAX_SCRIPT_PLUGINS 64
@ -61,7 +62,6 @@ const char *udfdCPluginUdfInitLoadInitDestoryFuncs(SUdfCPluginCtx *udfCtx, const
char destroyFuncName[TSDB_FUNC_NAME_LEN + 9] = {0}; char destroyFuncName[TSDB_FUNC_NAME_LEN + 9] = {0};
char *destroySuffix = "_destroy"; char *destroySuffix = "_destroy";
strcpy(destroyFuncName, udfName);
snprintf(destroyFuncName, sizeof(destroyFuncName), "%s%s", udfName, destroySuffix); snprintf(destroyFuncName, sizeof(destroyFuncName), "%s%s", udfName, destroySuffix);
uv_dlsym(&udfCtx->lib, destroyFuncName, (void **)(&udfCtx->destroyFunc)); uv_dlsym(&udfCtx->lib, destroyFuncName, (void **)(&udfCtx->destroyFunc));
return udfName; return udfName;
@ -69,7 +69,7 @@ const char *udfdCPluginUdfInitLoadInitDestoryFuncs(SUdfCPluginCtx *udfCtx, const
void udfdCPluginUdfInitLoadAggFuncs(SUdfCPluginCtx *udfCtx, const char *udfName) { void udfdCPluginUdfInitLoadAggFuncs(SUdfCPluginCtx *udfCtx, const char *udfName) {
char processFuncName[TSDB_FUNC_NAME_LEN] = {0}; char processFuncName[TSDB_FUNC_NAME_LEN] = {0};
strcpy(processFuncName, udfName); strncpy(processFuncName, udfName, sizeof(processFuncName));
uv_dlsym(&udfCtx->lib, processFuncName, (void **)(&udfCtx->aggProcFunc)); uv_dlsym(&udfCtx->lib, processFuncName, (void **)(&udfCtx->aggProcFunc));
char startFuncName[TSDB_FUNC_NAME_LEN + 7] = {0}; char startFuncName[TSDB_FUNC_NAME_LEN + 7] = {0};
@ -94,6 +94,7 @@ int32_t udfdCPluginUdfInit(SScriptUdfInfo *udf, void **pUdfCtx) {
err = uv_dlopen(udf->path, &udfCtx->lib); err = uv_dlopen(udf->path, &udfCtx->lib);
if (err != 0) { if (err != 0) {
fnError("can not load library %s. error: %s", udf->path, uv_strerror(err)); fnError("can not load library %s. error: %s", udf->path, uv_strerror(err));
taosMemoryFree(udfCtx);
return TSDB_CODE_UDF_LOAD_UDF_FAILURE; return TSDB_CODE_UDF_LOAD_UDF_FAILURE;
} }
const char *udfName = udf->name; const char *udfName = udf->name;
@ -102,7 +103,7 @@ int32_t udfdCPluginUdfInit(SScriptUdfInfo *udf, void **pUdfCtx) {
if (udf->funcType == UDF_FUNC_TYPE_SCALAR) { if (udf->funcType == UDF_FUNC_TYPE_SCALAR) {
char processFuncName[TSDB_FUNC_NAME_LEN] = {0}; char processFuncName[TSDB_FUNC_NAME_LEN] = {0};
strcpy(processFuncName, udfName); strncpy(processFuncName, udfName, sizeof(processFuncName));
uv_dlsym(&udfCtx->lib, processFuncName, (void **)(&udfCtx->scalarProcFunc)); uv_dlsym(&udfCtx->lib, processFuncName, (void **)(&udfCtx->scalarProcFunc));
} else if (udf->funcType == UDF_FUNC_TYPE_AGG) { } else if (udf->funcType == UDF_FUNC_TYPE_AGG) {
udfdCPluginUdfInitLoadAggFuncs(udfCtx, udfName); udfdCPluginUdfInitLoadAggFuncs(udfCtx, udfName);
@ -1038,7 +1039,7 @@ int32_t udfdOpenClientRpc() {
connLimitNum = TMIN(connLimitNum, 500); connLimitNum = TMIN(connLimitNum, 500);
rpcInit.connLimitNum = connLimitNum; rpcInit.connLimitNum = connLimitNum;
rpcInit.timeToGetConn = tsTimeToGetAvailableConn; rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
global.clientRpc = rpcOpen(&rpcInit); global.clientRpc = rpcOpen(&rpcInit);
if (global.clientRpc == NULL) { if (global.clientRpc == NULL) {
fnError("failed to init dnode rpc client"); fnError("failed to init dnode rpc client");

4
source/libs/parser/src/parInsertSml.c
View File

@ -127,7 +127,7 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
if(kv->keyLen != strlen(pTagSchema->name) || memcmp(kv->key, pTagSchema->name, kv->keyLen) != 0 || kv->type != pTagSchema->type){ if(kv->keyLen != strlen(pTagSchema->name) || memcmp(kv->key, pTagSchema->name, kv->keyLen) != 0 || kv->type != pTagSchema->type){
code = TSDB_CODE_SML_INVALID_DATA; code = TSDB_CODE_SML_INVALID_DATA;
uError("SML smlBuildCol error col not same %s", pTagSchema->name); uError("SML smlBuildTagRow error col not same %s", pTagSchema->name);
goto end; goto end;
} }
@ -210,7 +210,7 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32
SSmlKv* kv = (SSmlKv*)data; SSmlKv* kv = (SSmlKv*)data;
if(kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || kv->type != pColSchema->type){ if(kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || kv->type != pColSchema->type){
ret = TSDB_CODE_SML_INVALID_DATA; ret = TSDB_CODE_SML_INVALID_DATA;
uError("SML smlBuildCol error col not same %s", pColSchema->name); uInfo("SML smlBuildCol error col not same %s", pColSchema->name);
goto end; goto end;
} }
if (kv->type == TSDB_DATA_TYPE_NCHAR) { if (kv->type == TSDB_DATA_TYPE_NCHAR) {

12
source/libs/parser/src/parTranslater.c
View File

@ -882,6 +882,7 @@ static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* p
(igTags ? 0 : ((TSDB_SUPER_TABLE == pMeta->tableType) ? pMeta->tableInfo.numOfTags : 0)); (igTags ? 0 : ((TSDB_SUPER_TABLE == pMeta->tableType) ? pMeta->tableInfo.numOfTags : 0));
for (int32_t i = 0; i < nums; ++i) { for (int32_t i = 0; i < nums; ++i) {
if (invisibleColumn(pCxt->pParseCxt->enableSysInfo, pMeta->tableType, pMeta->schema[i].flags)) { if (invisibleColumn(pCxt->pParseCxt->enableSysInfo, pMeta->tableType, pMeta->schema[i].flags)) {
pCxt->pParseCxt->hasInvisibleCol = true;
continue; continue;
} }
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
@ -3207,7 +3208,11 @@ static int32_t translateSelectList(STranslateContext* pCxt, SSelectStmt* pSelect
code = translateFillValues(pCxt, pSelect); code = translateFillValues(pCxt, pSelect);
} }
if (NULL == pSelect->pProjectionList || 0 >= pSelect->pProjectionList->length) { if (NULL == pSelect->pProjectionList || 0 >= pSelect->pProjectionList->length) {
code = TSDB_CODE_PAR_INVALID_SELECTED_EXPR; if (pCxt->pParseCxt->hasInvisibleCol) {
code = TSDB_CODE_PAR_PERMISSION_DENIED;
} else {
code = TSDB_CODE_PAR_INVALID_SELECTED_EXPR;
}
} }
return code; return code;
} }
@ -6106,6 +6111,9 @@ static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt*
// for (int32_t i = 0; i < pMeta->tableInfo.numOfColumns; ++i) { // for (int32_t i = 0; i < pMeta->tableInfo.numOfColumns; ++i) {
SSchema* column = &pMeta->schema[0]; SSchema* column = &pMeta->schema[0];
SColumnNode* col = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); SColumnNode* col = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == col) {
return TSDB_CODE_OUT_OF_MEMORY;
}
strcpy(col->colName, column->name); strcpy(col->colName, column->name);
strcpy(col->node.aliasName, col->colName); strcpy(col->node.aliasName, col->colName);
strcpy(col->node.userAlias, col->colName); strcpy(col->node.userAlias, col->colName);
@ -6216,7 +6224,7 @@ static int32_t translateAlterLocal(STranslateContext* pCxt, SAlterLocalStmt* pSt
char* p = strchr(pStmt->config, ' '); char* p = strchr(pStmt->config, ' ');
if (NULL != p) { if (NULL != p) {
*p = 0; *p = 0;
strcpy(pStmt->value, p + 1); tstrncpy(pStmt->value, p + 1, sizeof(pStmt->value));
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

1
source/libs/scheduler/src/schJob.c
View File

@ -135,6 +135,7 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) {
break; break;
case JOB_TASK_STATUS_DROP: case JOB_TASK_STATUS_DROP:
SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED);
break;
default: default:
SCH_JOB_ELOG("invalid job status:%s", jobTaskStatusStr(oriStatus)); SCH_JOB_ELOG("invalid job status:%s", jobTaskStatusStr(oriStatus));

1
source/libs/scheduler/src/schRemote.c
View File

@ -392,6 +392,7 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD
// NEVER REACH HERE // NEVER REACH HERE
SCH_TASK_ELOG("invalid status to handle drop task rsp, refId:0x%" PRIx64, pJob->refId); SCH_TASK_ELOG("invalid status to handle drop task rsp, refId:0x%" PRIx64, pJob->refId);
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
break;
} }
case TDMT_SCH_LINK_BROKEN: case TDMT_SCH_LINK_BROKEN:
SCH_TASK_ELOG("link broken received, error:%x - %s", rspCode, tstrerror(rspCode)); SCH_TASK_ELOG("link broken received, error:%x - %s", rspCode, tstrerror(rspCode));

1
source/libs/scheduler/src/schTask.c
View File

@ -962,7 +962,6 @@ int32_t schHandleExplainRes(SArray *pExplainRes) {
localRsp->rsp.numOfPlans = 0; localRsp->rsp.numOfPlans = 0;
localRsp->rsp.subplanInfo = NULL; localRsp->rsp.subplanInfo = NULL;
pTask = NULL; pTask = NULL;
pJob = NULL;
} }
_return: _return:

1
source/libs/stream/src/streamData.c
View File

@ -27,6 +27,7 @@ SStreamDataBlock* createStreamDataFromDispatchMsg(const SStreamDispatchReq* pReq
int32_t blockNum = pReq->blockNum; int32_t blockNum = pReq->blockNum;
SArray* pArray = taosArrayInit_s(sizeof(SSDataBlock), blockNum); SArray* pArray = taosArrayInit_s(sizeof(SSDataBlock), blockNum);
if (pArray == NULL) { if (pArray == NULL) {
taosFreeQitem(pData);
return NULL; return NULL;
} }

3
source/libs/stream/src/streamExec.c
View File

@ -232,8 +232,9 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) {
} }
if (taosArrayGetSize(pRes) == 0) { if (taosArrayGetSize(pRes) == 0) {
taosArrayDestroy(pRes);
if (finished) { if (finished) {
taosArrayDestroy(pRes);
qDebug("s-task:%s finish recover exec task ", pTask->id.idStr); qDebug("s-task:%s finish recover exec task ", pTask->id.idStr);
break; break;
} else { } else {

26
source/libs/sync/test/sync_test_lib/src/syncIO.c
View File

@ -21,6 +21,7 @@
#include "tglobal.h" #include "tglobal.h"
#include "ttimer.h" #include "ttimer.h"
#include "tutil.h" #include "tutil.h"
#include "tversion.h"
bool gRaftDetailLog = false; bool gRaftDetailLog = false;
SSyncIO *gSyncIO = NULL; SSyncIO *gSyncIO = NULL;
@ -188,7 +189,7 @@ static int32_t syncIOStartInternal(SSyncIO *io) {
rpcInit.idleTime = 100; rpcInit.idleTime = 100;
rpcInit.user = "sync-io"; rpcInit.user = "sync-io";
rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.connType = TAOS_CONN_CLIENT;
taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
io->clientRpc = rpcOpen(&rpcInit); io->clientRpc = rpcOpen(&rpcInit);
if (io->clientRpc == NULL) { if (io->clientRpc == NULL) {
sError("failed to initialize RPC"); sError("failed to initialize RPC");
@ -209,7 +210,7 @@ static int32_t syncIOStartInternal(SSyncIO *io) {
rpcInit.idleTime = 2 * 1500; rpcInit.idleTime = 2 * 1500;
rpcInit.parent = io; rpcInit.parent = io;
rpcInit.connType = TAOS_CONN_SERVER; rpcInit.connType = TAOS_CONN_SERVER;
taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
void *pRpc = rpcOpen(&rpcInit); void *pRpc = rpcOpen(&rpcInit);
if (pRpc == NULL) { if (pRpc == NULL) {
sError("failed to start RPC server"); sError("failed to start RPC server");
@ -470,11 +471,10 @@ static void syncIOTickPing(void *param, void *tmrId) {
taosTmrReset(syncIOTickPing, io->pingTimerMS, io, io->timerMgr, &io->pingTimer); taosTmrReset(syncIOTickPing, io->pingTimerMS, io, io->timerMgr, &io->pingTimer);
} }
void syncEntryDestory(SSyncRaftEntry* pEntry) {} void syncEntryDestory(SSyncRaftEntry *pEntry) {}
void syncUtilMsgNtoH(void *msg) {
void syncUtilMsgNtoH(void* msg) { SMsgHead *pHead = msg;
SMsgHead* pHead = msg;
pHead->contLen = ntohl(pHead->contLen); pHead->contLen = ntohl(pHead->contLen);
pHead->vgId = ntohl(pHead->vgId); pHead->vgId = ntohl(pHead->vgId);
} }
@ -487,9 +487,9 @@ static inline bool syncUtilCanPrint(char c) {
} }
} }
char* syncUtilPrintBin(char* ptr, uint32_t len) { char *syncUtilPrintBin(char *ptr, uint32_t len) {
int64_t memLen = (int64_t)(len + 1); int64_t memLen = (int64_t)(len + 1);
char* s = taosMemoryMalloc(memLen); char *s = taosMemoryMalloc(memLen);
ASSERT(s != NULL); ASSERT(s != NULL);
memset(s, 0, len + 1); memset(s, 0, len + 1);
memcpy(s, ptr, len); memcpy(s, ptr, len);
@ -502,13 +502,13 @@ char* syncUtilPrintBin(char* ptr, uint32_t len) {
return s; return s;
} }
char* syncUtilPrintBin2(char* ptr, uint32_t len) { char *syncUtilPrintBin2(char *ptr, uint32_t len) {
uint32_t len2 = len * 4 + 1; uint32_t len2 = len * 4 + 1;
char* s = taosMemoryMalloc(len2); char *s = taosMemoryMalloc(len2);
ASSERT(s != NULL); ASSERT(s != NULL);
memset(s, 0, len2); memset(s, 0, len2);
char* p = s; char *p = s;
for (int32_t i = 0; i < len; ++i) { for (int32_t i = 0; i < len; ++i) {
int32_t n = sprintf(p, "%d,", ptr[i]); int32_t n = sprintf(p, "%d,", ptr[i]);
p += n; p += n;
@ -516,7 +516,7 @@ char* syncUtilPrintBin2(char* ptr, uint32_t len) {
return s; return s;
} }
void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port) { void syncUtilU642Addr(uint64_t u64, char *host, int64_t len, uint16_t *port) {
uint32_t hostU32 = (uint32_t)((u64 >> 32) & 0x00000000FFFFFFFF); uint32_t hostU32 = (uint32_t)((u64 >> 32) & 0x00000000FFFFFFFF);
struct in_addr addr = {.s_addr = hostU32}; struct in_addr addr = {.s_addr = hostU32};
@ -524,7 +524,7 @@ void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port) {
*port = (uint16_t)((u64 & 0x00000000FFFF0000) >> 16); *port = (uint16_t)((u64 & 0x00000000FFFF0000) >> 16);
} }
uint64_t syncUtilAddr2U64(const char* host, uint16_t port) { uint64_t syncUtilAddr2U64(const char *host, uint16_t port) {
uint32_t hostU32 = taosGetIpv4FromFqdn(host); uint32_t hostU32 = taosGetIpv4FromFqdn(host);
if (hostU32 == (uint32_t)-1) { if (hostU32 == (uint32_t)-1) {
sError("failed to resolve ipv4 addr, host:%s", host); sError("failed to resolve ipv4 addr, host:%s", host);

10
source/libs/tdb/src/db/tdbBtree.c
View File

@ -360,7 +360,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL
} }
*ppKey = pTKey; *ppKey = pTKey;
*pkLen = cd.kLen; *pkLen = cd.kLen;
memcpy(*ppKey, cd.pKey, cd.kLen); memcpy(*ppKey, cd.pKey, (size_t)cd.kLen);
} }
if (ppVal) { if (ppVal) {
@ -372,7 +372,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL
} }
*ppVal = pTVal; *ppVal = pTVal;
*vLen = cd.vLen; *vLen = cd.vLen;
memcpy(*ppVal, cd.pVal, cd.vLen); memcpy(*ppVal, cd.pVal, (size_t)cd.vLen);
} }
if (TDB_CELLDECODER_FREE_KEY(&cd)) { if (TDB_CELLDECODER_FREE_KEY(&cd)) {
@ -1866,7 +1866,7 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
*ppKey = pKey; *ppKey = pKey;
*kLen = cd.kLen; *kLen = cd.kLen;
memcpy(pKey, cd.pKey, cd.kLen); memcpy(pKey, cd.pKey, (size_t)cd.kLen);
if (ppVal) { if (ppVal) {
if (cd.vLen > 0) { if (cd.vLen > 0) {
@ -1925,7 +1925,7 @@ int tdbBtreePrev(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
*ppKey = pKey; *ppKey = pKey;
*kLen = cd.kLen; *kLen = cd.kLen;
memcpy(pKey, cd.pKey, cd.kLen); memcpy(pKey, cd.pKey, (size_t)cd.kLen);
if (ppVal) { if (ppVal) {
// TODO: vLen may be zero // TODO: vLen may be zero
@ -1937,7 +1937,7 @@ int tdbBtreePrev(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
*ppVal = pVal; *ppVal = pVal;
*vLen = cd.vLen; *vLen = cd.vLen;
memcpy(pVal, cd.pVal, cd.vLen); memcpy(pVal, cd.pVal, (size_t)cd.vLen);
} }
ret = tdbBtcMoveToPrev(pBtc); ret = tdbBtcMoveToPrev(pBtc);

5
source/libs/tdb/src/db/tdbDb.c
View File

@ -62,7 +62,10 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i
} }
memset(pDb->pgrHash, 0, tsize); memset(pDb->pgrHash, 0, tsize);
taosMulModeMkDir(dbname, 0755); ret = taosMulModeMkDir(dbname, 0755);
if (ret < 0) {
return -1;
}
#ifdef USE_MAINDB #ifdef USE_MAINDB
// open main db // open main db

1
source/libs/tdb/src/db/tdbPager.c
View File

@ -1092,6 +1092,7 @@ int tdbPagerRestoreJournals(SPager *pPager) {
jname[dirLen] = '/'; jname[dirLen] = '/';
sprintf(jname + dirLen + 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId); sprintf(jname + dirLen + 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId);
if (tdbPagerRestore(pPager, jname) < 0) { if (tdbPagerRestore(pPager, jname) < 0) {
taosArrayDestroy(pTxnList);
tdbCloseDir(&pDir); tdbCloseDir(&pDir);
tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), jname); tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), jname);

2
source/libs/transport/inc/transComm.h
View File

@ -154,6 +154,7 @@ typedef struct {
#pragma pack(push, 1) #pragma pack(push, 1)
#define TRANS_VER 2
typedef struct { typedef struct {
char version : 4; // RPC version char version : 4; // RPC version
char comp : 2; // compression algorithm, 0:no compression 1:lz4 char comp : 2; // compression algorithm, 0:no compression 1:lz4
@ -166,6 +167,7 @@ typedef struct {
uint64_t timestamp; uint64_t timestamp;
char user[TSDB_UNI_LEN]; char user[TSDB_UNI_LEN];
int32_t compatibilityVer;
uint32_t magicNum; uint32_t magicNum;
STraceId traceId; STraceId traceId;
uint64_t ahandle; // ahandle assigned by client uint64_t ahandle; // ahandle assigned by client

8
source/libs/transport/inc/transportInt.h
View File

@ -46,10 +46,10 @@ typedef struct {
int8_t connType; int8_t connType;
char label[TSDB_LABEL_LEN]; char label[TSDB_LABEL_LEN];
char user[TSDB_UNI_LEN]; // meter ID char user[TSDB_UNI_LEN]; // meter ID
int32_t compatibilityVer;
int32_t compressSize; // -1: no compress, 0 : all data compressed, size: compress data if larger than size int32_t compressSize; // -1: no compress, 0 : all data compressed, size: compress data if larger than size
int8_t encryption; // encrypt or not int8_t encryption; // encrypt or not
int32_t retryMinInterval; // retry init interval int32_t retryMinInterval; // retry init interval
int32_t retryStepFactor; // retry interval factor int32_t retryStepFactor; // retry interval factor
int32_t retryMaxInterval; // retry max interval int32_t retryMaxInterval; // retry max interval

1
source/libs/transport/src/trans.c
View File

@ -50,6 +50,7 @@ void* rpcOpen(const SRpcInit* pInit) {
} }
pRpc->encryption = pInit->encryption; pRpc->encryption = pInit->encryption;
pRpc->compatibilityVer = pInit->compatibilityVer;
pRpc->retryMinInterval = pInit->retryMinInterval; // retry init interval pRpc->retryMinInterval = pInit->retryMinInterval; // retry init interval
pRpc->retryStepFactor = pInit->retryStepFactor; pRpc->retryStepFactor = pInit->retryStepFactor;

18
source/libs/transport/src/transCli.c
View File

@ -391,6 +391,7 @@ void cliHandleResp(SCliConn* conn) {
transMsg.info.ahandle = NULL; transMsg.info.ahandle = NULL;
transMsg.info.traceId = pHead->traceId; transMsg.info.traceId = pHead->traceId;
transMsg.info.hasEpSet = pHead->hasEpSet; transMsg.info.hasEpSet = pHead->hasEpSet;
transMsg.info.cliVer = htonl(pHead->compatibilityVer);
SCliMsg* pMsg = NULL; SCliMsg* pMsg = NULL;
STransConnCtx* pCtx = NULL; STransConnCtx* pCtx = NULL;
@ -488,6 +489,7 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) {
transMsg.code = code == -1 ? (pConn->broken ? TSDB_CODE_RPC_BROKEN_LINK : TSDB_CODE_RPC_NETWORK_UNAVAIL) : code; transMsg.code = code == -1 ? (pConn->broken ? TSDB_CODE_RPC_BROKEN_LINK : TSDB_CODE_RPC_NETWORK_UNAVAIL) : code;
transMsg.msgType = pMsg ? pMsg->msg.msgType + 1 : 0; transMsg.msgType = pMsg ? pMsg->msg.msgType + 1 : 0;
transMsg.info.ahandle = NULL; transMsg.info.ahandle = NULL;
transMsg.info.cliVer = pTransInst->compatibilityVer;
if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) { if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) {
transMsg.info.ahandle = transCtxDumpVal(&pConn->ctx, transMsg.msgType); transMsg.info.ahandle = transCtxDumpVal(&pConn->ctx, transMsg.msgType);
@ -984,11 +986,10 @@ void cliSendBatch(SCliConn* pConn) {
SCliThrd* pThrd = pConn->hostThrd; SCliThrd* pThrd = pConn->hostThrd;
STrans* pTransInst = pThrd->pTransInst; STrans* pTransInst = pThrd->pTransInst;
SCliBatch* pBatch = pConn->pBatch; SCliBatch* pBatch = pConn->pBatch;
SCliBatchList* pList = pBatch->pList; int32_t wLen = pBatch->wLen;
pList->connCnt += 1;
int32_t wLen = pBatch->wLen; pBatch->pList->connCnt += 1;
uv_buf_t* wb = taosMemoryCalloc(wLen, sizeof(uv_buf_t)); uv_buf_t* wb = taosMemoryCalloc(wLen, sizeof(uv_buf_t));
int i = 0; int i = 0;
@ -1018,6 +1019,8 @@ void cliSendBatch(SCliConn* pConn) {
memcpy(pHead->user, pTransInst->user, strlen(pTransInst->user)); memcpy(pHead->user, pTransInst->user, strlen(pTransInst->user));
pHead->traceId = pMsg->info.traceId; pHead->traceId = pMsg->info.traceId;
pHead->magicNum = htonl(TRANS_MAGIC_NUM); pHead->magicNum = htonl(TRANS_MAGIC_NUM);
pHead->version = TRANS_VER;
pHead->compatibilityVer = htonl(pTransInst->compatibilityVer);
} }
pHead->timestamp = taosHton64(taosGetTimestampUs()); pHead->timestamp = taosHton64(taosGetTimestampUs());
@ -1074,6 +1077,8 @@ void cliSend(SCliConn* pConn) {
memcpy(pHead->user, pTransInst->user, strlen(pTransInst->user)); memcpy(pHead->user, pTransInst->user, strlen(pTransInst->user));
pHead->traceId = pMsg->info.traceId; pHead->traceId = pMsg->info.traceId;
pHead->magicNum = htonl(TRANS_MAGIC_NUM); pHead->magicNum = htonl(TRANS_MAGIC_NUM);
pHead->version = TRANS_VER;
pHead->compatibilityVer = htonl(pTransInst->compatibilityVer);
} }
pHead->timestamp = taosHton64(taosGetTimestampUs()); pHead->timestamp = taosHton64(taosGetTimestampUs());
@ -1346,6 +1351,7 @@ static void doNotifyApp(SCliMsg* pMsg, SCliThrd* pThrd) {
transMsg.info.ahandle = pMsg->ctx->ahandle; transMsg.info.ahandle = pMsg->ctx->ahandle;
transMsg.info.traceId = pMsg->msg.info.traceId; transMsg.info.traceId = pMsg->msg.info.traceId;
transMsg.info.hasEpSet = false; transMsg.info.hasEpSet = false;
transMsg.info.cliVer = pTransInst->compatibilityVer;
if (pCtx->pSem != NULL) { if (pCtx->pSem != NULL) {
if (pCtx->pRsp == NULL) { if (pCtx->pRsp == NULL) {
} else { } else {
@ -1527,6 +1533,9 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) {
// persist conn already release by server // persist conn already release by server
STransMsg resp; STransMsg resp;
cliBuildExceptResp(pMsg, &resp); cliBuildExceptResp(pMsg, &resp);
// refactorr later
resp.info.cliVer = pTransInst->compatibilityVer;
if (pMsg->type != Release) { if (pMsg->type != Release) {
pTransInst->cfp(pTransInst->parent, &resp, NULL); pTransInst->cfp(pTransInst->parent, &resp, NULL);
} }
@ -1836,6 +1845,7 @@ void cliIteraConnMsgs(SCliConn* conn) {
if (-1 == cliBuildExceptResp(cmsg, &resp)) { if (-1 == cliBuildExceptResp(cmsg, &resp)) {
continue; continue;
} }
resp.info.cliVer = pTransInst->compatibilityVer;
pTransInst->cfp(pTransInst->parent, &resp, NULL); pTransInst->cfp(pTransInst->parent, &resp, NULL);
cmsg->ctx->ahandle = NULL; cmsg->ctx->ahandle = NULL;

2
source/libs/transport/src/transComm.c
View File

@ -192,7 +192,7 @@ bool transReadComplete(SConnBuffer* connBuf) {
memcpy((char*)&head, connBuf->buf, sizeof(head)); memcpy((char*)&head, connBuf->buf, sizeof(head));
int32_t msgLen = (int32_t)htonl(head.msgLen); int32_t msgLen = (int32_t)htonl(head.msgLen);
p->total = msgLen; p->total = msgLen;
p->invalid = TRANS_NOVALID_PACKET(htonl(head.magicNum)); p->invalid = TRANS_NOVALID_PACKET(htonl(head.magicNum)) || head.version != TRANS_VER;
} }
if (p->total >= p->len) { if (p->total >= p->len) {
p->left = p->total - p->len; p->left = p->total - p->len;

12
source/libs/transport/src/transSvr.c
View File

@ -196,6 +196,7 @@ static bool uvHandleReq(SSvrConn* pConn) {
tError("%s conn %p recv invalid packet, failed to decompress", transLabel(pTransInst), pConn); tError("%s conn %p recv invalid packet, failed to decompress", transLabel(pTransInst), pConn);
return false; return false;
} }
tDebug("head version: %d 2", pHead->version);
pHead->code = htonl(pHead->code); pHead->code = htonl(pHead->code);
pHead->msgLen = htonl(pHead->msgLen); pHead->msgLen = htonl(pHead->msgLen);
@ -236,8 +237,8 @@ static bool uvHandleReq(SSvrConn* pConn) {
if (pConn->status == ConnNormal && pHead->noResp == 0) { if (pConn->status == ConnNormal && pHead->noResp == 0) {
transRefSrvHandle(pConn); transRefSrvHandle(pConn);
if (cost >= EXCEPTION_LIMIT_US) { if (cost >= EXCEPTION_LIMIT_US) {
tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, cost:%dus, recv exception", transLabel(pTransInst), tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, cost:%dus, recv exception",
pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, (int)cost); transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, (int)cost);
} else { } else {
tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, cost:%dus", transLabel(pTransInst), pConn, tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, cost:%dus", transLabel(pTransInst), pConn,
TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, (int)cost); TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, (int)cost);
@ -245,8 +246,8 @@ static bool uvHandleReq(SSvrConn* pConn) {
} else { } else {
if (cost >= EXCEPTION_LIMIT_US) { if (cost >= EXCEPTION_LIMIT_US) {
tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, noResp:%d, code:%d, cost:%dus, recv exception", tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, noResp:%d, code:%d, cost:%dus, recv exception",
transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp, transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp,
transMsg.code, (int)(cost)); transMsg.code, (int)(cost));
} else { } else {
tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, noResp:%d, code:%d, cost:%dus", tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, noResp:%d, code:%d, cost:%dus",
transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp, transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp,
@ -262,6 +263,7 @@ static bool uvHandleReq(SSvrConn* pConn) {
transMsg.info.handle = (void*)transAcquireExHandle(transGetRefMgt(), pConn->refId); transMsg.info.handle = (void*)transAcquireExHandle(transGetRefMgt(), pConn->refId);
transMsg.info.refId = pConn->refId; transMsg.info.refId = pConn->refId;
transMsg.info.traceId = pHead->traceId; transMsg.info.traceId = pHead->traceId;
transMsg.info.cliVer = htonl(pHead->compatibilityVer);
tGTrace("%s handle %p conn:%p translated to app, refId:%" PRIu64, transLabel(pTransInst), transMsg.info.handle, pConn, tGTrace("%s handle %p conn:%p translated to app, refId:%" PRIu64, transLabel(pTransInst), transMsg.info.handle, pConn,
pConn->refId); pConn->refId);
@ -410,6 +412,8 @@ static int uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
pHead->traceId = pMsg->info.traceId; pHead->traceId = pMsg->info.traceId;
pHead->hasEpSet = pMsg->info.hasEpSet; pHead->hasEpSet = pMsg->info.hasEpSet;
pHead->magicNum = htonl(TRANS_MAGIC_NUM); pHead->magicNum = htonl(TRANS_MAGIC_NUM);
pHead->compatibilityVer = htonl(((STrans*)pConn->pTransInst)->compatibilityVer);
pHead->version = TRANS_VER;
// handle invalid drop_task resp, TD-20098 // handle invalid drop_task resp, TD-20098
if (pConn->inType == TDMT_SCH_DROP_TASK && pMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { if (pConn->inType == TDMT_SCH_DROP_TASK && pMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) {

3
source/libs/transport/test/cliBench.c
View File

@ -19,6 +19,7 @@
#include "transLog.h" #include "transLog.h"
#include "trpc.h" #include "trpc.h"
#include "tutil.h" #include "tutil.h"
#include "tversion.h"
typedef struct { typedef struct {
int index; int index;
@ -155,7 +156,7 @@ int main(int argc, char *argv[]) {
} }
initLogEnv(); initLogEnv();
taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
void *pRpc = rpcOpen(&rpcInit); void *pRpc = rpcOpen(&rpcInit);
if (pRpc == NULL) { if (pRpc == NULL) {
tError("failed to initialize RPC"); tError("failed to initialize RPC");

7
source/libs/transport/test/svrBench.c
View File

@ -13,12 +13,13 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
//#define _DEFAULT_SOURCE // #define _DEFAULT_SOURCE
#include "os.h" #include "os.h"
#include "tglobal.h" #include "tglobal.h"
#include "tqueue.h" #include "tqueue.h"
#include "transLog.h" #include "transLog.h"
#include "trpc.h" #include "trpc.h"
#include "tversion.h"
int msgSize = 128; int msgSize = 128;
int commit = 0; int commit = 0;
@ -151,6 +152,8 @@ int main(int argc, char *argv[]) {
rpcInit.numOfThreads = 1; rpcInit.numOfThreads = 1;
rpcInit.cfp = processRequestMsg; rpcInit.cfp = processRequestMsg;
rpcInit.idleTime = 2 * 1500; rpcInit.idleTime = 2 * 1500;
taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
rpcDebugFlag = 131; rpcDebugFlag = 131;
for (int i = 1; i < argc; ++i) { for (int i = 1; i < argc; ++i) {
@ -187,7 +190,7 @@ int main(int argc, char *argv[]) {
rpcInit.connType = TAOS_CONN_SERVER; rpcInit.connType = TAOS_CONN_SERVER;
initLogEnv(); initLogEnv();
taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
void *pRpc = rpcOpen(&rpcInit); void *pRpc = rpcOpen(&rpcInit);
if (pRpc == NULL) { if (pRpc == NULL) {
tError("failed to start RPC server"); tError("failed to start RPC server");

8
source/libs/transport/test/transUT.cpp
View File

@ -18,10 +18,10 @@
#include "tdatablock.h" #include "tdatablock.h"
#include "tglobal.h" #include "tglobal.h"
#include "tlog.h" #include "tlog.h"
#include "tmisce.h"
#include "transLog.h" #include "transLog.h"
#include "trpc.h" #include "trpc.h"
#include "tmisce.h" #include "tversion.h"
using namespace std; using namespace std;
const char *label = "APP"; const char *label = "APP";
@ -54,6 +54,8 @@ class Client {
rpcInit_.user = (char *)user; rpcInit_.user = (char *)user;
rpcInit_.parent = this; rpcInit_.parent = this;
rpcInit_.connType = TAOS_CONN_CLIENT; rpcInit_.connType = TAOS_CONN_CLIENT;
taosVersionStrToInt(version, &(rpcInit_.compatibilityVer));
this->transCli = rpcOpen(&rpcInit_); this->transCli = rpcOpen(&rpcInit_);
tsem_init(&this->sem, 0, 0); tsem_init(&this->sem, 0, 0);
} }
@ -66,6 +68,7 @@ class Client {
void Restart(CB cb) { void Restart(CB cb) {
rpcClose(this->transCli); rpcClose(this->transCli);
rpcInit_.cfp = cb; rpcInit_.cfp = cb;
taosVersionStrToInt(version, &(rpcInit_.compatibilityVer));
this->transCli = rpcOpen(&rpcInit_); this->transCli = rpcOpen(&rpcInit_);
} }
void Stop() { void Stop() {
@ -117,6 +120,7 @@ class Server {
rpcInit_.cfp = processReq; rpcInit_.cfp = processReq;
rpcInit_.user = (char *)user; rpcInit_.user = (char *)user;
rpcInit_.connType = TAOS_CONN_SERVER; rpcInit_.connType = TAOS_CONN_SERVER;
taosVersionStrToInt(version, &(rpcInit_.compatibilityVer));
} }
void Start() { void Start() {
this->transSrv = rpcOpen(&this->rpcInit_); this->transSrv = rpcOpen(&this->rpcInit_);

14
source/libs/wal/src/walMeta.c
View File

@ -596,18 +596,18 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
// ftruncate idx file // ftruncate idx file
if (offset < fileSize) { if (offset < fileSize) {
if (taosFtruncateFile(pIdxFile, offset) < 0) { if (taosFtruncateFile(pIdxFile, offset) < 0) {
wError("vgId:%d, failed to ftruncate file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId,
strerror(errno), offset, fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, failed to ftruncate file since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(),
offset, fnameStr);
goto _err; goto _err;
} }
} }
// rebuild idx file // rebuild idx file
if (taosLSeekFile(pIdxFile, 0, SEEK_END) < 0) { if (taosLSeekFile(pIdxFile, 0, SEEK_END) < 0) {
wError("vgId:%d, failed to seek file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno),
offset, fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, failed to seek file since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(), offset,
fnameStr);
goto _err; goto _err;
} }
@ -619,11 +619,12 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
idxEntry.offset += sizeof(SWalCkHead) + ckHead.head.bodyLen; idxEntry.offset += sizeof(SWalCkHead) + ckHead.head.bodyLen;
if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) { if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) {
wError("vgId:%d, failed to read wal log head since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(), wError("vgId:%d, failed to read wal log head since %s. index:%" PRId64 ", offset:%" PRId64 ", file:%s",
idxEntry.offset, fLogNameStr); pWal->cfg.vgId, terrstr(), idxEntry.ver, idxEntry.offset, fLogNameStr);
goto _err; goto _err;
} }
if (taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) < 0) { if (taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, failed to append file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr); wError("vgId:%d, failed to append file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr);
goto _err; goto _err;
} }
@ -631,6 +632,7 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
} }
if (taosFsyncFile(pIdxFile) < 0) { if (taosFsyncFile(pIdxFile) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, faild to fsync file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr); wError("vgId:%d, faild to fsync file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr);
goto _err; goto _err;
} }

5
source/libs/wal/src/walRead.c
View File

@ -82,6 +82,11 @@ int32_t walNextValidMsg(SWalReader *pReader) {
", applied index:%" PRId64", end index:%" PRId64, ", applied index:%" PRId64", end index:%" PRId64,
pReader->pWal->cfg.vgId, fetchVer, lastVer, committedVer, appliedVer, endVer); pReader->pWal->cfg.vgId, fetchVer, lastVer, committedVer, appliedVer, endVer);
if (fetchVer > endVer){
terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
return -1;
}
while (fetchVer <= endVer) { while (fetchVer <= endVer) {
if (walFetchHeadNew(pReader, fetchVer) < 0) { if (walFetchHeadNew(pReader, fetchVer) < 0) {
return -1; return -1;

17
source/libs/wal/src/walWrite.c
View File

@ -473,7 +473,10 @@ static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
// check alignment of idx entries // check alignment of idx entries
int64_t endOffset = taosLSeekFile(pWal->pIdxFile, 0, SEEK_END); int64_t endOffset = taosLSeekFile(pWal->pIdxFile, 0, SEEK_END);
if (endOffset < 0) { if (endOffset < 0) {
wFatal("vgId:%d, failed to seek end of idxfile due to %s. ver:%" PRId64 "", pWal->cfg.vgId, strerror(errno), ver); wFatal("vgId:%d, failed to seek end of WAL idxfile due to %s. ver:%" PRId64 "", pWal->cfg.vgId, strerror(errno),
ver);
taosMsleep(100);
exit(EXIT_FAILURE);
} }
return 0; return 0;
} }
@ -533,16 +536,20 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
END: END:
// recover in a reverse order // recover in a reverse order
if (taosFtruncateFile(pWal->pLogFile, offset) < 0) { if (taosFtruncateFile(pWal->pLogFile, offset) < 0) {
wFatal("vgId:%d, failed to ftruncate logfile to offset:%" PRId64 " during recovery due to %s", pWal->cfg.vgId,
offset, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
wFatal("vgId:%d, failed to recover WAL logfile from write error since %s, offset:%" PRId64, pWal->cfg.vgId,
terrstr(), offset);
taosMsleep(100);
exit(EXIT_FAILURE);
} }
int64_t idxOffset = (index - pFileInfo->firstVer) * sizeof(SWalIdxEntry); int64_t idxOffset = (index - pFileInfo->firstVer) * sizeof(SWalIdxEntry);
if (taosFtruncateFile(pWal->pIdxFile, idxOffset) < 0) { if (taosFtruncateFile(pWal->pIdxFile, idxOffset) < 0) {
wFatal("vgId:%d, failed to ftruncate idxfile to offset:%" PRId64 "during recovery due to %s", pWal->cfg.vgId,
idxOffset, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
wFatal("vgId:%d, failed to recover WAL idxfile from write error since %s, offset:%" PRId64, pWal->cfg.vgId,
terrstr(), idxOffset);
taosMsleep(100);
exit(EXIT_FAILURE);
} }
return -1; return -1;
} }

1
source/util/src/terror.c
View File

@ -632,6 +632,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SCALAR_CONVERT_ERROR, "Cannot convert to s
//tmq //tmq
TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG, "Invalid message") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG, "Invalid message")
TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_SNAPSHOT_ERROR, "Can not operate in snapshot mode")
TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_MISMATCH, "Consumer mismatch") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_MISMATCH, "Consumer mismatch")
TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_CLOSED, "Consumer closed") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_CLOSED, "Consumer closed")
TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_ERROR, "Consumer error, to see log") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_ERROR, "Consumer error, to see log")

13
source/util/src/tlog.c
View File

@ -486,24 +486,11 @@ static inline int32_t taosBuildLogHead(char *buffer, const char *flags) {
static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *buffer, int32_t len) { static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *buffer, int32_t len) {
if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile != NULL && osLogSpaceAvailable()) { if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile != NULL && osLogSpaceAvailable()) {
taosUpdateLogNums(level); taosUpdateLogNums(level);
#if 0
// DEBUG_FATAL and DEBUG_ERROR are duplicated
// fsync will cause thread blocking and may also generate log misalignment in case of asyncLog
if (tsAsyncLog && level != DEBUG_FATAL) {
taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
} else {
taosWriteFile(tsLogObj.logHandle->pFile, buffer, len);
if (level == DEBUG_FATAL) {
taosFsyncFile(tsLogObj.logHandle->pFile);
}
}
#else
if (tsAsyncLog) { if (tsAsyncLog) {
taosPushLogBuffer(tsLogObj.logHandle, buffer, len); taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
} else { } else {
taosWriteFile(tsLogObj.logHandle->pFile, buffer, len); taosWriteFile(tsLogObj.logHandle->pFile, buffer, len);
} }
#endif
if (tsLogObj.maxLines > 0) { if (tsLogObj.maxLines > 0) {
atomic_add_fetch_32(&tsLogObj.lines, 1); atomic_add_fetch_32(&tsLogObj.lines, 1);

7
tests/parallel_test/cases.task
View File

@ -33,8 +33,12 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb3.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb3.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb0.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb0.py -N 3 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/ins_topics_test.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/ins_topics_test.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxTopic.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqClientConsLog.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqClientConsLog.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxGroupIds.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumeDiscontinuousData.py
,,n,system-test,python3 ./test.py -f 7-tmq/tmqDropConsumer.py
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_stable.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_stable.py
@ -320,6 +324,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py -R
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -R
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/orderBy.py -N 5
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -R
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py
@ -751,6 +756,8 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/odbc.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/odbc.py
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py
,,n,system-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insertMix.py -N 3
,,n,system-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/stt.py -N 3
#tsim test #tsim test
,,y,script,./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim ,,y,script,./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim

197
tests/script/api/passwdTest.c
View File

@ -32,9 +32,21 @@
#define nRoot 10 #define nRoot 10
#define nUser 10 #define nUser 10
#define USER_LEN 24 #define USER_LEN 24
#define BUF_LEN 1024
typedef uint16_t VarDataLenT;
#define TSDB_NCHAR_SIZE sizeof(int32_t)
#define VARSTR_HEADER_SIZE sizeof(VarDataLenT)
#define GET_FLOAT_VAL(x) (*(float *)(x))
#define GET_DOUBLE_VAL(x) (*(double *)(x))
#define varDataLen(v) ((VarDataLenT *)(v))[0]
void createUsers(TAOS *taos, const char *host, char *qstr); void createUsers(TAOS *taos, const char *host, char *qstr);
void passVerTestMulti(const char *host, char *qstr); void passVerTestMulti(const char *host, char *qstr);
void sysInfoTest(TAOS *taos, const char *host, char *qstr);
int nPassVerNotified = 0; int nPassVerNotified = 0;
TAOS *taosu[nRoot] = {0}; TAOS *taosu[nRoot] = {0};
@ -83,6 +95,95 @@ static void queryDB(TAOS *taos, char *command) {
taos_free_result(pSql); taos_free_result(pSql);
} }
int printRow(char *str, TAOS_ROW row, TAOS_FIELD *fields, int numFields) {
int len = 0;
char split = ' ';
for (int i = 0; i < numFields; ++i) {
if (i > 0) {
str[len++] = split;
}
if (row[i] == NULL) {
len += sprintf(str + len, "%s", "NULL");
continue;
}
switch (fields[i].type) {
case TSDB_DATA_TYPE_TINYINT:
len += sprintf(str + len, "%d", *((int8_t *)row[i]));
break;
case TSDB_DATA_TYPE_UTINYINT:
len += sprintf(str + len, "%u", *((uint8_t *)row[i]));
break;
case TSDB_DATA_TYPE_SMALLINT:
len += sprintf(str + len, "%d", *((int16_t *)row[i]));
break;
case TSDB_DATA_TYPE_USMALLINT:
len += sprintf(str + len, "%u", *((uint16_t *)row[i]));
break;
case TSDB_DATA_TYPE_INT:
len += sprintf(str + len, "%d", *((int32_t *)row[i]));
break;
case TSDB_DATA_TYPE_UINT:
len += sprintf(str + len, "%u", *((uint32_t *)row[i]));
break;
case TSDB_DATA_TYPE_BIGINT:
len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i]));
break;
case TSDB_DATA_TYPE_UBIGINT:
len += sprintf(str + len, "%" PRIu64, *((uint64_t *)row[i]));
break;
case TSDB_DATA_TYPE_FLOAT: {
float fv = 0;
fv = GET_FLOAT_VAL(row[i]);
len += sprintf(str + len, "%f", fv);
} break;
case TSDB_DATA_TYPE_DOUBLE: {
double dv = 0;
dv = GET_DOUBLE_VAL(row[i]);
len += sprintf(str + len, "%lf", dv);
} break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_GEOMETRY: {
int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE);
memcpy(str + len, row[i], charLen);
len += charLen;
} break;
case TSDB_DATA_TYPE_TIMESTAMP:
len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i]));
break;
case TSDB_DATA_TYPE_BOOL:
len += sprintf(str + len, "%d", *((int8_t *)row[i]));
default:
break;
}
}
return len;
}
static int printResult(TAOS_RES *res, char *output) {
int numFields = taos_num_fields(res);
TAOS_FIELD *fields = taos_fetch_fields(res);
char header[BUF_LEN] = {0};
int len = 0;
for (int i = 0; i < numFields; ++i) {
len += sprintf(header + len, "%s ", fields[i].name);
}
puts(header);
if (output) {
strncpy(output, header, BUF_LEN);
}
TAOS_ROW row = NULL;
while ((row = taos_fetch_row(res))) {
char temp[BUF_LEN] = {0};
printRow(temp, row, fields, numFields);
puts(temp);
}
}
int main(int argc, char *argv[]) { int main(int argc, char *argv[]) {
char qstr[1024]; char qstr[1024];
@ -99,6 +200,7 @@ int main(int argc, char *argv[]) {
} }
createUsers(taos, argv[1], qstr); createUsers(taos, argv[1], qstr);
passVerTestMulti(argv[1], qstr); passVerTestMulti(argv[1], qstr);
sysInfoTest(taos, argv[1], qstr);
taos_close(taos); taos_close(taos);
taos_cleanup(); taos_cleanup();
@ -167,6 +269,8 @@ void passVerTestMulti(const char *host, char *qstr) {
int nConn = nRoot + nUser; int nConn = nRoot + nUser;
for (int i = 0; i < 15; ++i) { for (int i = 0; i < 15; ++i) {
printf("%s:%d [%d] second(s) elasped, passVer notification received:%d, total:%d\n", __func__, __LINE__, i,
nPassVerNotified, nConn);
if (nPassVerNotified >= nConn) break; if (nPassVerNotified >= nConn) break;
sleep(1); sleep(1);
} }
@ -175,19 +279,100 @@ void passVerTestMulti(const char *host, char *qstr) {
for (int i = 0; i < nRoot; ++i) { for (int i = 0; i < nRoot; ++i) {
taos_close(taos[i]); taos_close(taos[i]);
printf("%s:%d close taos[%d]\n", __func__, __LINE__, i); printf("%s:%d close taos[%d]\n", __func__, __LINE__, i);
sleep(1); // sleep(1);
} }
for (int i = 0; i < nUser; ++i) { for (int i = 0; i < nUser; ++i) {
taos_close(taosu[i]); taos_close(taosu[i]);
printf("%s:%d close taosu[%d]\n", __func__, __LINE__, i); printf("%s:%d close taosu[%d]\n", __func__, __LINE__, i);
// sleep(1);
}
fprintf(stderr, "######## %s #########\n", __func__);
if (nPassVerNotified >= nConn) {
fprintf(stderr, ">>> succeed to get passVer notification since nNotify %d >= nConn %d\n", nPassVerNotified,
nConn);
} else {
fprintf(stderr, ">>> failed to get passVer notification since nNotify %d < nConn %d\n", nPassVerNotified, nConn);
}
fprintf(stderr, "######## %s #########\n", __func__);
// sleep(300);
}
void sysInfoTest(TAOS *taosRoot, const char *host, char *qstr) {
TAOS *taos[nRoot] = {0};
char userName[USER_LEN] = "user0";
for (int i = 0; i < nRoot; ++i) {
taos[i] = taos_connect(host, "user0", "taos", NULL, 0);
if (taos[i] == NULL) {
fprintf(stderr, "failed to connect to server, reason:%s\n", "null taos" /*taos_errstr(taos)*/);
exit(1);
}
}
queryDB(taosRoot, "create database if not exists demo11 vgroups 1 minrows 10");
queryDB(taosRoot, "create database if not exists demo12 vgroups 1 minrows 10");
queryDB(taosRoot, "create database if not exists demo13 vgroups 1 minrows 10");
queryDB(taosRoot, "create table demo11.stb (ts timestamp, c1 int) tags(t1 int)");
queryDB(taosRoot, "create table demo12.stb (ts timestamp, c1 int) tags(t1 int)");
queryDB(taosRoot, "create table demo13.stb (ts timestamp, c1 int) tags(t1 int)");
sprintf(qstr, "show grants");
char output[BUF_LEN];
TAOS_RES *res = NULL;
int32_t nRep = 0;
_REP:
fprintf(stderr, "######## %s loop:%d #########\n", __func__, nRep);
res = taos_query(taos[0], qstr);
if (taos_errno(res) != 0) {
fprintf(stderr, "%s:%d failed to execute: %s since %s\n", __func__, __LINE__, qstr, taos_errstr(res));
taos_free_result(res);
exit(EXIT_FAILURE);
}
printResult(res, output);
taos_free_result(res);
if (!strstr(output, "timeseries")) {
fprintf(stderr, "%s:%d expected output: 'timeseries' not occur\n", __func__, __LINE__);
exit(EXIT_FAILURE);
}
queryDB(taosRoot, "alter user user0 sysinfo 0");
fprintf(stderr, "%s:%d sleep 2 seconds to wait HB take effect\n", __func__, __LINE__);
for (int i = 1; i <= 2; ++i) {
sleep(1); sleep(1);
} }
if (nPassVerNotified >= nConn) { res = taos_query(taos[0], qstr);
fprintf(stderr, "succeed to get passVer notification since nNotify %d >= nConn %d\n", nPassVerNotified, nConn); if (taos_errno(res) != 0) {
} else { if (!strstr(taos_errstr(res), "Permission denied")) {
fprintf(stderr, "failed to get passVer notification since nNotify %d < nConn %d\n", nPassVerNotified, nConn); fprintf(stderr, "%s:%d expected error: 'Permission denied' not occur\n", __func__, __LINE__);
taos_free_result(res);
exit(EXIT_FAILURE);
}
} }
// sleep(300); taos_free_result(res);
queryDB(taosRoot, "alter user user0 sysinfo 1");
fprintf(stderr, "%s:%d sleep 2 seconds to wait HB take effect\n", __func__, __LINE__);
for (int i = 1; i <= 2; ++i) {
sleep(1);
}
if(++nRep < 5) {
goto _REP;
}
// close the taos_conn
for (int i = 0; i < nRoot; ++i) {
taos_close(taos[i]);
fprintf(stderr, "%s:%d close taos[%d]\n", __func__, __LINE__, i);
}
fprintf(stderr, "######## %s #########\n", __func__);
fprintf(stderr, ">>> succeed to run sysInfoTest\n");
fprintf(stderr, "######## %s #########\n", __func__);
} }

1
tests/system-test/0-others/compatibility.py
View File

@ -152,6 +152,7 @@ class TDTestCase:
os.system("LD_LIBRARY_PATH=/usr/lib taos -f 0-others/TS-3131.tsql") os.system("LD_LIBRARY_PATH=/usr/lib taos -f 0-others/TS-3131.tsql")
cmd = f" LD_LIBRARY_PATH={bPath}/build/lib {bPath}/build/bin/taos -h localhost ;" cmd = f" LD_LIBRARY_PATH={bPath}/build/lib {bPath}/build/bin/taos -h localhost ;"
tdLog.info(f"new client version connect to old version taosd, commad return value:{cmd}")
if os.system(cmd) == 0: if os.system(cmd) == 0:
raise Exception("failed to execute system command. cmd: %s" % cmd) raise Exception("failed to execute system command. cmd: %s" % cmd)

409
tests/system-test/0-others/user_privilege_all.py Normal file
View File

@ -0,0 +1,409 @@
from itertools import product
import taos
import time
from taos.tmq import *
from util.cases import *
from util.common import *
from util.log import *
from util.sql import *
from util.sqlset import *
class TDTestCase:
"""This test case is used to veirfy the user privilege for insert and select operation on
stable、child table and table
"""
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__)
# init the tdsql
tdSql.init(conn.cursor())
self.setsql = TDSetSql()
# user info
self.username = 'test'
self.password = 'test'
# db info
self.dbname = "user_privilege_all_db"
self.stbname = 'stb'
self.common_tbname = "tb"
self.ctbname_list = ["ct1", "ct2"]
self.common_table_dict = {
'ts':'timestamp',
'col1':'float',
'col2':'int'
}
self.stable_column_dict = {
'ts': 'timestamp',
'col1': 'float',
'col2': 'int',
}
self.tag_dict = {
'ctbname': 'binary(10)'
}
# case list
self.cases = {
"test_db_table_both_no_permission": {
"db_privilege": "none",
"stable_priviege": "none",
"child_table_ct1_privilege": "none",
"child_table_ct2_privilege": "none",
"table_tb_privilege": "none",
"sql": ["insert into ct1 using stb tags('ct1') values(now, 1.1, 1)",
"select * from stb;",
"select * from ct1;",
"select * from ct2;",
"insert into tb values(now, 3.3, 3);",
"select * from tb;"],
"res": [False, False, False, False, False, False]
},
"test_db_no_permission_table_read": {
"db_privilege": "none",
"stable_priviege": "none",
"child_table_ct1_privilege": "none",
"child_table_ct2_privilege": "none",
"table_tb_privilege": "read",
"sql": ["insert into ct1 using stb tags('ct1') values(now, 1.1, 1)",
"select * from stb;",
"select * from ct1;",
"select * from ct2;",
"insert into tb values(now, 3.3, 3);",
"select * from tb;"],
"res": [False, False, False, False, False, True]
},
"test_db_no_permission_childtable_read": {
"db_privilege": "none",
"stable_priviege": "none",
"child_table_ct1_privilege": "read",
"child_table_ct2_privilege": "none",
"table_tb_privilege": "none",
"sql": ["insert into ct1 using stb tags('ct1') values(now, 1.1, 1)",
"select * from stb;",
"select * from ct1;",
"select * from ct2;",
"insert into tb values(now, 3.3, 3);",
"select * from tb;"],
"res": [False, True, True, False, False, False]
},
"test_db_no_permission_table_write": {
"db_privilege": "none",
"stable_priviege": "none",
"child_table_ct1_privilege": "none",
"child_table_ct2_privilege": "none",
"table_tb_privilege": "write",
"sql": ["insert into ct1 using stb tags('ct1') values(now, 1.1, 1)",
"select * from stb;",
"select * from ct1;",
"select * from ct2;",
"insert into tb values(now, 3.3, 3);",
"select * from tb;"],
"res": [False, False, False, False, True, False]
},
"test_db_no_permission_childtable_write": {
"db_privilege": "none",
"stable_priviege": "none",
"child_table_ct1_privilege": "none",
"child_table_ct2_privilege": "write",
"table_tb_privilege": "none",
"sql": ["insert into ct2 using stb tags('ct2') values(now, 1.1, 1)",
"select * from stb;",
"select * from ct1;",
"select * from ct2;",
"insert into tb values(now, 3.3, 3);",
"select * from tb;"],
"res": [True, False, False, False, False, False]
},
"test_db_read_table_no_permission": {
"db_privilege": "read",
"stable_priviege": "none",
"child_table_ct1_privilege": "none",
"child_table_ct2_privilege": "none",
"table_tb_privilege": "none",
"sql": ["insert into ct2 using stb tags('ct2') values(now, 1.1, 1)",
"select * from stb;",
"select * from ct1;",
"select * from ct2;",
"insert into tb values(now, 3.3, 3);",
"select * from tb;"],
"res": [False, True, True, True, False, True]
},
"test_db_read_table_read": {
"db_privilege": "read",
"stable_priviege": "none",
"child_table_ct1_privilege": "none",
"child_table_ct2_privilege": "none",
"table_tb_privilege": "read",
"sql": ["insert into ct2 using stb tags('ct2') values(now, 1.1, 1)",
"select * from stb;",
"select * from ct1;",
"select * from ct2;",
"insert into tb values(now, 3.3, 3);",
"select * from tb;"],
"res": [False, True, True, True, False, True]
},
"test_db_read_childtable_read": {
"db_privilege": "read",
"stable_priviege": "none",
"child_table_ct1_privilege": "read",
"child_table_ct2_privilege": "read",
"table_tb_privilege": "none",
"sql": ["insert into ct2 using stb tags('ct2') values(now, 1.1, 1)",
"select * from stb;",
"select * from ct1;",
"select * from ct2;",
"insert into tb values(now, 3.3, 3);",
"select * from tb;"],
"res": [False, True, True, True, False, True]
},
"test_db_read_table_write": {
"db_privilege": "read",
"stable_priviege": "none",
"child_table_ct1_privilege": "none",
"child_table_ct2_privilege": "none",
"table_tb_privilege": "write",
"sql": ["insert into ct2 using stb tags('ct2') values(now, 1.1, 1)",
"select * from stb;",
"select * from ct1;",
"select * from ct2;",
"insert into tb values(now, 4.4, 4);",
"select * from tb;"],
"res": [False, True, True, True, True, True]
},
"test_db_read_childtable_write": {
"db_privilege": "read",
"stable_priviege": "none",
"child_table_ct1_privilege": "write",
"child_table_ct2_privilege": "none",
"table_tb_privilege": "none",
"sql": ["insert into ct2 using stb tags('ct2') values(now, 1.1, 1)",
"insert into ct1 using stb tags('ct1') values(now, 5.5, 5)",
"select * from stb;",
"select * from ct1;",
"select * from ct2;",
"insert into tb values(now, 4.4, 4);",
"select * from tb;"],
"res": [False, True, True, True, True, False, True]
},
"test_db_write_table_no_permission": {
"db_privilege": "write",
"stable_priviege": "none",
"child_table_ct1_privilege": "none",
"child_table_ct2_privilege": "none",
"table_tb_privilege": "none",
"sql": ["insert into ct2 using stb tags('ct2') values(now, 6.6, 6)",
"insert into ct1 using stb tags('ct1') values(now, 7.7, 7)",
"select * from stb;",
"select * from ct1;",
"select * from ct2;",
"insert into tb values(now, 8.8, 8);",
"select * from tb;"],
"res": [True, True, False, False, False, True, False]
},
"test_db_write_table_write": {
"db_privilege": "write",
"stable_priviege": "none",
"child_table_ct1_privilege": "none",
"child_table_ct2_privilege": "none",
"table_tb_privilege": "none",
"sql": ["insert into ct2 using stb tags('ct2') values(now, 9.9, 9)",
"insert into ct1 using stb tags('ct1') values(now, 10.0, 10)",
"select * from stb;",
"select * from ct1;",
"select * from ct2;",
"insert into tb values(now, 11.1, 11);",
"select * from tb;"],
"res": [True, True, False, False, False, True, False]
},
"test_db_write_childtable_write": {
"db_privilege": "write",
"stable_priviege": "none",
"child_table_ct1_privilege": "none",
"child_table_ct2_privilege": "none",
"table_tb_privilege": "none",
"sql": ["insert into ct2 using stb tags('ct2') values(now, 12.2, 12)",
"insert into ct1 using stb tags('ct1') values(now, 13.3, 13)",
"select * from stb;",
"select * from ct1;",
"select * from ct2;",
"insert into tb values(now, 14.4, 14);",
"select * from tb;"],
"res": [True, True, False, False, False, True, False]
},
"test_db_write_table_read": {
"db_privilege": "write",
"stable_priviege": "none",
"child_table_ct1_privilege": "none",
"child_table_ct2_privilege": "none",
"table_tb_privilege": "read",
"sql": ["insert into ct2 using stb tags('ct2') values(now, 15.5, 15)",
"insert into ct1 using stb tags('ct1') values(now, 16.6, 16)",
"select * from stb;",
"select * from ct1;",
"select * from ct2;",
"insert into tb values(now, 17.7, 17);",
"select * from tb;"],
"res": [True, True, False, False, False, True, True]
},
"test_db_write_childtable_read": {
"db_privilege": "write",
"stable_priviege": "none",
"child_table_ct1_privilege": "read",
"child_table_ct2_privilege": "none",
"table_tb_privilege": "none",
"sql": ["insert into ct2 using stb tags('ct2') values(now, 18.8, 18)",
"insert into ct1 using stb tags('ct1') values(now, 19.9, 19)",
"select * from stb;",
"select * from ct1;",
"select * from ct2;",
"insert into tb values(now, 20.0, 20);",
"select * from tb;"],
"res": [True, True, True, True, False, True, False]
}
}
def prepare_data(self):
"""Create the db and data for test
"""
tdLog.debug("Start to prepare the data for test")
# create datebase
tdSql.execute(f"create database {self.dbname}")
tdSql.execute(f"use {self.dbname}")
# create stable
tdSql.execute(self.setsql.set_create_stable_sql(self.stbname, self.stable_column_dict, self.tag_dict))
tdLog.debug("Create stable {} successfully".format(self.stbname))
# insert data into child table
for ctname in self.ctbname_list:
tdSql.execute(f"insert into {ctname} using {self.stbname} tags('{ctname}') values(now, 1.1, 1)")
tdSql.execute(f"insert into {ctname} using {self.stbname} tags('{ctname}') values(now, 2.1, 2)")
# create common table
tdSql.execute(self.setsql.set_create_normaltable_sql(self.common_tbname, self.common_table_dict))
tdLog.debug("Create common table {} successfully".format(self.common_tbname))
# insert data into common table
tdSql.execute(f"insert into {self.common_tbname} values(now, 1.1, 1)")
tdSql.execute(f"insert into {self.common_tbname} values(now, 2.2, 2)")
tdLog.debug("Finish to prepare the data")
def create_user(self):
"""Create the user for test
"""
tdSql.execute(f'create user {self.username} pass "{self.password}"')
tdLog.debug("sql:" + f'create user {self.username} pass "{self.password}" successfully')
def grant_privilege(self, username, privilege, table, tag_condition=None):
"""Add the privilege for the user
"""
try:
if tag_condition:
tdSql.execute(f'grant {privilege} on {self.dbname}.{table} with {tag_condition} to {username}')
else:
tdSql.execute(f'grant {privilege} on {self.dbname}.{table} to {username}')
time.sleep(2)
tdLog.debug("Grant {} privilege on {}.{} with condition {} to {} successfully".format(privilege, self.dbname, table, tag_condition, username))
except Exception as ex:
tdLog.exit(ex)
def remove_privilege(self, username, privilege, table, tag_condition=None):
"""Remove the privilege for the user
"""
try:
if tag_condition:
tdSql.execute(f'revoke {privilege} on {self.dbname}.{table} with {tag_condition} from {username}')
else:
tdSql.execute(f'revoke {privilege} on {self.dbname}.{table} from {username}')
tdLog.debug("Revoke {} privilege on {}.{} with condition {} from {} successfully".format(privilege, self.dbname, table, tag_condition, username))
except Exception as ex:
tdLog.exit(ex)
def run(self):
self.create_user()
# prepare the test data
self.prepare_data()
for case_name in self.cases.keys():
tdLog.debug("Execute the case {} with params {}".format(case_name, str(self.cases[case_name])))
# grant privilege for user test if case need
if self.cases[case_name]["db_privilege"] != "none":
self.grant_privilege(self.username, self.cases[case_name]["db_privilege"], "*")
if self.cases[case_name]["stable_priviege"] != "none":
self.grant_privilege(self.username, self.cases[case_name]["stable_priviege"], self.stbname)
if self.cases[case_name]["child_table_ct1_privilege"] != "none" and self.cases[case_name]["child_table_ct2_privilege"] != "none":
self.grant_privilege(self.username, self.cases[case_name]["child_table_ct1_privilege"], self.stbname, "ctbname='ct1' or ctbname='ct2'")
elif self.cases[case_name]["child_table_ct1_privilege"] != "none":
self.grant_privilege(self.username, self.cases[case_name]["child_table_ct1_privilege"], self.stbname, "ctbname='ct1'")
elif self.cases[case_name]["child_table_ct2_privilege"] != "none":
self.grant_privilege(self.username, self.cases[case_name]["child_table_ct2_privilege"], self.stbname, "ctbname='ct2'")
if self.cases[case_name]["table_tb_privilege"] != "none":
self.grant_privilege(self.username, self.cases[case_name]["table_tb_privilege"], self.common_tbname)
# connect db with user test
testconn = taos.connect(user=self.username, password=self.password)
if case_name != "test_db_table_both_no_permission":
testconn.execute("use %s;" % self.dbname)
# check privilege of user test from ins_user_privileges table
res = testconn.query("select * from information_schema.ins_user_privileges;")
tdLog.debug("Current information_schema.ins_user_privileges values: {}".format(res.fetch_all()))
# check privilege of user test by executing sql query
for index in range(len(self.cases[case_name]["sql"])):
tdLog.debug("Execute sql: {}".format(self.cases[case_name]["sql"][index]))
try:
# for write privilege
if "insert " in self.cases[case_name]["sql"][index]:
testconn.execute(self.cases[case_name]["sql"][index])
# check the expected result
if self.cases[case_name]["res"][index]:
tdLog.debug("Write data with sql {} successfully".format(self.cases[case_name]["sql"][index]))
# for read privilege
elif "select " in self.cases[case_name]["sql"][index]:
res = testconn.query(self.cases[case_name]["sql"][index])
data = res.fetch_all()
tdLog.debug("query result: {}".format(data))
# check query results by cases
if case_name in ["test_db_no_permission_childtable_read", "test_db_write_childtable_read"] and self.cases[case_name]["sql"][index] == "select * from ct2;":
if not self.cases[case_name]["res"][index]:
if 0 == len(data):
tdLog.debug("Query with sql {} successfully as expected with empty result".format(self.cases[case_name]["sql"][index]))
continue
else:
tdLog.exit("Query with sql {} failed with result {}".format(self.cases[case_name]["sql"][index], data))
# check the expected result
if self.cases[case_name]["res"][index]:
if len(data) > 0:
tdLog.debug("Query with sql {} successfully".format(self.cases[case_name]["sql"][index]))
else:
tdLog.exit("Query with sql {} failed with result {}".format(self.cases[case_name]["sql"][index], data))
else:
tdLog.exit("Execute query sql {} successfully, but expected failed".format(self.cases[case_name]["sql"][index]))
except BaseException as ex:
# check the expect false result
if not self.cases[case_name]["res"][index]:
tdLog.debug("Execute sql {} failed with {} as expected".format(self.cases[case_name]["sql"][index], str(ex)))
continue
# unexpected exception
else:
tdLog.exit(ex)
# remove the privilege
if self.cases[case_name]["db_privilege"] != "none":
self.remove_privilege(self.username, self.cases[case_name]["db_privilege"], "*")
if self.cases[case_name]["stable_priviege"] != "none":
self.remove_privilege(self.username, self.cases[case_name]["stable_priviege"], self.stbname)
if self.cases[case_name]["child_table_ct1_privilege"] != "none":
self.remove_privilege(self.username, self.cases[case_name]["child_table_ct1_privilege"], self.stbname, "ctbname='ct1'")
if self.cases[case_name]["child_table_ct2_privilege"] != "none":
self.remove_privilege(self.username, self.cases[case_name]["child_table_ct2_privilege"], self.stbname, "ctbname='ct2'")
if self.cases[case_name]["table_tb_privilege"] != "none":
self.remove_privilege(self.username, self.cases[case_name]["table_tb_privilege"], self.common_tbname)
# close the connection of user test
testconn.close()
def stop(self):
# remove the user
tdSql.execute(f'drop user {self.username}')
# close the connection
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

36
tests/system-test/2-query/diff.py
View File

@ -23,7 +23,7 @@ class TDTestCase:
tdSql.execute( tdSql.execute(
f"create table {dbname}.ntb(ts timestamp,c1 int,c2 double,c3 float)") f"create table {dbname}.ntb(ts timestamp,c1 int,c2 double,c3 float)")
tdSql.execute( tdSql.execute(
f"insert into {dbname}.ntb values(now,1,1.0,10.5)(now+1s,10,-100.0,5.1)(now+10s,-1,15.1,5.0)") f"insert into {dbname}.ntb values('2023-01-01 00:00:01',1,1.0,10.5)('2023-01-01 00:00:02',10,-100.0,5.1)('2023-01-01 00:00:03',-1,15.1,5.0)")
tdSql.query(f"select diff(c1,0) from {dbname}.ntb") tdSql.query(f"select diff(c1,0) from {dbname}.ntb")
tdSql.checkRows(2) tdSql.checkRows(2)
@ -233,6 +233,40 @@ class TDTestCase:
tdSql.checkRows(19) tdSql.checkRows(19)
tdSql.checkData(0,0,None) tdSql.checkData(0,0,None)
# TD-25098
tdSql.query(f"select ts, diff(c1) from {dbname}.ntb order by ts")
tdSql.checkRows(2)
tdSql.checkData(0, 0, '2023-01-01 00:00:02.000')
tdSql.checkData(1, 0, '2023-01-01 00:00:03.000')
tdSql.checkData(0, 1, 9)
tdSql.checkData(1, 1, -11)
tdSql.query(f"select ts, diff(c1) from {dbname}.ntb order by ts desc")
tdSql.checkRows(2)
tdSql.checkData(0, 0, '2023-01-01 00:00:03.000')
tdSql.checkData(1, 0, '2023-01-01 00:00:02.000')
tdSql.checkData(0, 1, -11)
tdSql.checkData(1, 1, 9)
tdSql.query(f"select ts, diff(c1) from (select * from {dbname}.ntb order by ts)")
tdSql.checkRows(2)
tdSql.checkData(0, 0, '2023-01-01 00:00:02.000')
tdSql.checkData(1, 0, '2023-01-01 00:00:03.000')
tdSql.checkData(0, 1, 9)
tdSql.checkData(1, 1, -11)
tdSql.query(f"select ts, diff(c1) from (select * from {dbname}.ntb order by ts desc)")
tdSql.checkRows(2)
tdSql.checkData(0, 0, '2023-01-01 00:00:02.000')
tdSql.checkData(1, 0, '2023-01-01 00:00:01.000')
tdSql.checkData(0, 1, 11)
tdSql.checkData(1, 1, -9)
def stop(self): def stop(self):
tdSql.close() tdSql.close()
tdLog.success("%s successfully executed" % __file__) tdLog.success("%s successfully executed" % __file__)

86
tests/system-test/2-query/interp.py
View File

@ -44,7 +44,7 @@ class TDTestCase:
tdSql.execute( tdSql.execute(
f'''create table if not exists {dbname}.{tbname} f'''create table if not exists {dbname}.{tbname}
(ts timestamp, c0 tinyint, c1 smallint, c2 int, c3 bigint, c4 double, c5 float, c6 bool, c7 varchar(10), c8 nchar(10)) (ts timestamp, c0 tinyint, c1 smallint, c2 int, c3 bigint, c4 double, c5 float, c6 bool, c7 varchar(10), c8 nchar(10), c9 tinyint unsigned, c10 smallint unsigned, c11 int unsigned, c12 bigint unsigned)
''' '''
) )
@ -52,9 +52,9 @@ class TDTestCase:
tdSql.execute(f"use db") tdSql.execute(f"use db")
tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:05', 5, 5, 5, 5, 5.0, 5.0, true, 'varchar', 'nchar')") tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:05', 5, 5, 5, 5, 5.0, 5.0, true, 'varchar', 'nchar', 5, 5, 5, 5)")
tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:10', 10, 10, 10, 10, 10.0, 10.0, true, 'varchar', 'nchar')") tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:10', 10, 10, 10, 10, 10.0, 10.0, true, 'varchar', 'nchar', 10, 10, 10, 10)")
tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:15', 15, 15, 15, 15, 15.0, 15.0, true, 'varchar', 'nchar')") tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:15', 15, 15, 15, 15, 15.0, 15.0, true, 'varchar', 'nchar', 15, 15, 15, 15)")
tdLog.printNoPrefix("==========step3:fill null") tdLog.printNoPrefix("==========step3:fill null")
@ -129,21 +129,71 @@ class TDTestCase:
tdLog.printNoPrefix("==========step4:fill value") tdLog.printNoPrefix("==========step4:fill value")
## {. . .} ## {. . .}
tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)") col_list = {'c0', 'c1', 'c2', 'c3', 'c9', 'c10', 'c11', 'c12'}
for col in col_list:
tdSql.query(f"select interp({col}) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)")
tdSql.checkRows(13)
tdSql.checkData(0, 0, 1)
tdSql.checkData(1, 0, 5)
tdSql.checkData(2, 0, 1)
tdSql.checkData(3, 0, 1)
tdSql.checkData(4, 0, 1)
tdSql.checkData(5, 0, 1)
tdSql.checkData(6, 0, 10)
tdSql.checkData(7, 0, 1)
tdSql.checkData(8, 0, 1)
tdSql.checkData(9, 0, 1)
tdSql.checkData(10, 0, 1)
tdSql.checkData(11, 0, 15)
tdSql.checkData(12, 0, 1)
tdSql.query(f"select interp(c4) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)")
tdSql.checkRows(13) tdSql.checkRows(13)
tdSql.checkData(0, 0, 1) tdSql.checkData(0, 0, 1.0)
tdSql.checkData(1, 0, 5) tdSql.checkData(1, 0, 5.0)
tdSql.checkData(2, 0, 1) tdSql.checkData(2, 0, 1.0)
tdSql.checkData(3, 0, 1) tdSql.checkData(3, 0, 1.0)
tdSql.checkData(4, 0, 1) tdSql.checkData(4, 0, 1.0)
tdSql.checkData(5, 0, 1) tdSql.checkData(5, 0, 1.0)
tdSql.checkData(6, 0, 10) tdSql.checkData(6, 0, 10.0)
tdSql.checkData(7, 0, 1) tdSql.checkData(7, 0, 1.0)
tdSql.checkData(8, 0, 1) tdSql.checkData(8, 0, 1.0)
tdSql.checkData(9, 0, 1) tdSql.checkData(9, 0, 1.0)
tdSql.checkData(10, 0, 1) tdSql.checkData(10, 0, 1.0)
tdSql.checkData(11, 0, 15) tdSql.checkData(11, 0, 15.0)
tdSql.checkData(12, 0, 1) tdSql.checkData(12, 0, 1.0)
tdSql.query(f"select interp(c5) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)")
tdSql.checkRows(13)
tdSql.checkData(0, 0, 1.0)
tdSql.checkData(1, 0, 5.0)
tdSql.checkData(2, 0, 1.0)
tdSql.checkData(3, 0, 1.0)
tdSql.checkData(4, 0, 1.0)
tdSql.checkData(5, 0, 1.0)
tdSql.checkData(6, 0, 10.0)
tdSql.checkData(7, 0, 1.0)
tdSql.checkData(8, 0, 1.0)
tdSql.checkData(9, 0, 1.0)
tdSql.checkData(10, 0, 1.0)
tdSql.checkData(11, 0, 15.0)
tdSql.checkData(12, 0, 1.0)
tdSql.query(f"select interp(c6) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)")
tdSql.checkRows(13)
tdSql.checkData(0, 0, True)
tdSql.checkData(1, 0, True)
tdSql.checkData(2, 0, True)
tdSql.checkData(3, 0, True)
tdSql.checkData(4, 0, True)
tdSql.checkData(5, 0, True)
tdSql.checkData(6, 0, True)
tdSql.checkData(7, 0, True)
tdSql.checkData(8, 0, True)
tdSql.checkData(9, 0, True)
tdSql.checkData(10, 0, True)
tdSql.checkData(11, 0, True)
tdSql.checkData(12, 0, True)
## {} ... ## {} ...
tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:01', '2020-02-01 00:00:04') every(1s) fill(value, 1)") tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:01', '2020-02-01 00:00:04') every(1s) fill(value, 1)")

298
tests/system-test/2-query/orderBy.py Normal file
View File

@ -0,0 +1,298 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import random
import time
import copy
import taos
from util.log import *
from util.cases import *
from util.sql import *
class TDTestCase:
# get col value and total max min ...
def getColsValue(self, i, j):
# c1 value
if random.randint(1, 10) == 5:
c1 = None
else:
c1 = 1
# c2 value
if j % 3200 == 0:
c2 = 8764231
elif random.randint(1, 10) == 5:
c2 = None
else:
c2 = random.randint(-87654297, 98765321)
# c3 is order
c3 = i * self.childRow + j
value = f"({self.ts}, "
# c1
if c1 is None:
value += "null,"
else:
self.c1Cnt += 1
value += f"{c1},"
# c2
if c2 is None:
value += "null,"
else:
value += f"{c2},"
# total count
self.c2Cnt += 1
# max
if self.c2Max is None:
self.c2Max = c2
else:
if c2 > self.c2Max:
self.c2Max = c2
# min
if self.c2Min is None:
self.c2Min = c2
else:
if c2 < self.c2Min:
self.c2Min = c2
# sum
if self.c2Sum is None:
self.c2Sum = c2
else:
self.c2Sum += c2
# c3
value += f"{c3},"
# ts1 same with ts
value += f"{self.ts})"
# move next
self.ts += 1
return value
# insert data
def insertData(self):
tdLog.info("insert data ....")
sqls = ""
for i in range(self.childCnt):
# insert child table
values = ""
pre_insert = f"insert into t{i} values "
for j in range(self.childRow):
if values == "":
values = self.getColsValue(i, j)
else:
values += "," + self.getColsValue(i, j)
# batch insert
if j % self.batchSize == 0 and values != "":
sql = pre_insert + values
tdSql.execute(sql)
values = ""
# append last
if values != "":
sql = pre_insert + values
tdSql.execute(sql)
values = ""
sql = "flush database db;"
tdLog.info(sql)
tdSql.execute(sql)
# insert finished
tdLog.info(f"insert data successfully.\n"
f" inserted child table = {self.childCnt}\n"
f" inserted child rows = {self.childRow}\n"
f" total inserted rows = {self.childCnt*self.childRow}\n")
return
# prepareEnv
def prepareEnv(self):
# init
self.ts = 1680000000000*1000
self.childCnt = 10
self.childRow = 100000
self.batchSize = 5000
# total
self.c1Cnt = 0
self.c2Cnt = 0
self.c2Max = None
self.c2Min = None
self.c2Sum = None
# create database db
sql = f"create database db vgroups 2 precision 'us' "
tdLog.info(sql)
tdSql.execute(sql)
sql = f"use db"
tdSql.execute(sql)
# alter config
sql = "alter local 'querySmaOptimize 1';"
tdLog.info(sql)
tdSql.execute(sql)
# create super talbe st
sql = f"create table st(ts timestamp, c1 int, c2 bigint, c3 bigint, ts1 timestamp) tags(area int)"
tdLog.info(sql)
tdSql.execute(sql)
# create child table
for i in range(self.childCnt):
sql = f"create table t{i} using st tags({i}) "
tdSql.execute(sql)
# insert data
self.insertData()
# check data correct
def checkExpect(self, sql, expectVal):
tdSql.query(sql)
rowCnt = tdSql.getRows()
for i in range(rowCnt):
val = tdSql.getData(i,0)
if val != expectVal:
tdLog.exit(f"Not expect . query={val} expect={expectVal} i={i} sql={sql}")
return False
tdLog.info(f"check expect ok. sql={sql} expect ={expectVal} rowCnt={rowCnt}")
return True
# check query
def queryResultSame(self, sql1, sql2):
# sql
tdLog.info(sql1)
start1 = time.time()
rows1 = tdSql.query(sql1)
spend1 = time.time() - start1
res1 = copy.copy(tdSql.queryResult)
tdLog.info(sql2)
start2 = time.time()
tdSql.query(sql2)
spend2 = time.time() - start2
res2 = tdSql.queryResult
rowlen1 = len(res1)
rowlen2 = len(res2)
if rowlen1 != rowlen2:
tdLog.exit(f"rowlen1={rowlen1} rowlen2={rowlen2} both not equal.")
return False
for i in range(rowlen1):
row1 = res1[i]
row2 = res2[i]
collen1 = len(row1)
collen2 = len(row2)
if collen1 != collen2:
tdLog.exit(f"collen1={collen1} collen2={collen2} both not equal.")
return False
for j in range(collen1):
if row1[j] != row2[j]:
tdLog.exit(f"col={j} col1={row1[j]} col2={row2[j]} both col not equal.")
return False
# warning performance
diff = (spend2 - spend1)*100/spend1
tdLog.info("spend1=%.6fs spend2=%.6fs diff=%.1f%%"%(spend1, spend2, diff))
if spend2 > spend1 and diff > 50:
tdLog.info("warning: the diff for performance after spliting is over 20%")
return True
# init
def init(self, conn, logSql, replicaVar=1):
seed = time.clock_gettime(time.CLOCK_REALTIME)
random.seed(seed)
self.replicaVar = int(replicaVar)
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), True)
# check time macro
def queryBasic(self):
# check count
expectVal = self.childCnt * self.childRow
sql = f"select count(ts) from st "
self.checkExpect(sql, expectVal)
# check diff
sql = f"select count(*) from (select diff(ts) as dif from st order by ts)"
self.checkExpect(sql, expectVal - 1)
# check ts order count
sql = f"select count(*) from (select diff(ts) as dif from st order by ts) where dif!=1"
self.checkExpect(sql, 0)
# check ts1 order count
sql = f"select count(*) from (select diff(ts1) as dif from st order by ts1) where dif!=1"
self.checkExpect(sql, 0)
# check c3 order asc
sql = f"select count(*) from (select diff(c3) as dif from st order by c3) where dif!=1"
self.checkExpect(sql, 0)
# check c3 order desc todo FIX
#sql = f"select count(*) from (select diff(c3) as dif from st order by c3 desc) where dif!=-1"
#self.checkExpect(sql, 0)
# advance
def queryAdvance(self):
# interval order todo FIX
#sql = f"select _wstart,count(ts),max(c2),min(c2) from st interval(100u) sliding(50u) order by _wstart limit 10"
#tdSql.query(sql)
#tdSql.checkRows(10)
# simulate crash sql
sql = f"select _wstart,count(ts),max(c2),min(c2) from st interval(100a) sliding(10a) order by _wstart limit 10"
tdSql.query(sql)
tdSql.checkRows(10)
# extent
sql = f"select _wstart,count(ts),max(c2),min(c2) from st interval(100a) sliding(10a) order by _wstart desc limit 5"
tdSql.query(sql)
tdSql.checkRows(5)
# data correct checked
sql1 = "select sum(a),sum(b), max(c), min(d),sum(e) from (select _wstart,count(ts) as a,count(c2) as b ,max(c2) as c, min(c2) as d, sum(c2) as e from st interval(100a) sliding(100a) order by _wstart desc);"
sql2 = "select count(*) as a, count(c2) as b, max(c2) as c, min(c2) as d, sum(c2) as e from st;"
self.queryResultSame(sql1, sql2)
# run
def run(self):
# prepare env
self.prepareEnv()
# basic
self.queryBasic()
# advance
self.queryAdvance()
# stop
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

102
tests/system-test/5-taos-tools/taosbenchmark/insertMix.py Normal file
View File

@ -0,0 +1,102 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import os
import subprocess
import time
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def caseDescription(self):
"""
[TD-13823] taosBenchmark test cases
"""
return
def init(self, conn, logSql, replicaVar=1):
# comment off by Shuduo for CI self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def getPath(self, tool="taosBenchmark"):
if (platform.system().lower() == 'windows'):
tool = tool + ".exe"
selfPath = os.path.dirname(os.path.realpath(__file__))
if "community" in selfPath:
projPath = selfPath[: selfPath.find("community")]
else:
projPath = selfPath[: selfPath.find("tests")]
paths = []
for root, dirs, files in os.walk(projPath):
if (tool) in files:
rootRealPath = os.path.dirname(os.path.realpath(root))
if "packaging" not in rootRealPath:
paths.append(os.path.join(root, tool))
break
if len(paths) == 0:
tdLog.exit("taosBenchmark not found!")
return
else:
tdLog.info("taosBenchmark found in %s" % paths[0])
return paths[0]
def checkDataCorrect(self):
sql = "select count(*) from meters"
tdSql.query(sql)
allCnt = tdSql.getData(0, 0)
if allCnt < 2000000:
tdLog.exit(f"taosbenchmark insert row small. row count={allCnt} sql={sql}")
return
# group by 10 child table
rowCnt = tdSql.query("select count(*),tbname from meters group by tbname")
tdSql.checkRows(10)
# interval
sql = "select count(*),max(ic),min(dc),last(*) from meters interval(1s)"
rowCnt = tdSql.query(sql)
if rowCnt < 10:
tdLog.exit(f"taosbenchmark interval(1s) count small. row cout={rowCnt} sql={sql}")
return
# nest query
tdSql.query("select count(*) from (select * from meters order by ts desc)")
tdSql.checkData(0, 0, allCnt)
def run(self):
binPath = self.getPath()
cmd = "%s -f ./5-taos-tools/taosbenchmark/json/insertMix.json" % binPath
tdLog.info("%s" % cmd)
errcode = os.system("%s" % cmd)
if errcode != 0:
tdLog.exit(f"execute taosBenchmark ret error code={errcode}")
return
tdSql.execute("use mixdb")
self.checkDataCorrect()
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

81
tests/system-test/5-taos-tools/taosbenchmark/json/insertMix.json Normal file
View File

@ -0,0 +1,81 @@
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"connection_pool_size": 8,
"num_of_records_per_req": 3000,
"thread_count": 10,
"create_table_thread_count": 2,
"result_file": "./insert_res_mix.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"check_sql": "yes",
"continue_if_fail": "no",
"databases": [
{
"dbinfo": {
"name": "mixdb",
"drop": "yes",
"vgroups": 6,
"replica": 3,
"precision": "ms",
"keep": 3650,
"minRows": 100,
"maxRows": 4096
},
"super_tables": [
{
"name": "meters",
"child_table_exists": "no",
"childtable_count": 10,
"insert_rows": 300000,
"childtable_prefix": "d",
"insert_mode": "taosc",
"insert_interval": 0,
"timestamp_step": 100,
"start_timestamp":1500000000000,
"disorder_ratio": 10,
"update_ratio": 5,
"delete_ratio": 1,
"disorder_fill_interval": 300,
"update_fill_interval": 25,
"generate_row_rule": 2,
"columns": [
{ "type": "bool", "name": "bc"},
{ "type": "float", "name": "fc", "max": 1, "min": 0 },
{ "type": "double", "name": "dc", "max": 1, "min": 0 },
{ "type": "tinyint", "name": "ti", "max": 100, "min": 0 },
{ "type": "smallint", "name": "si", "max": 100, "min": 0 },
{ "type": "int", "name": "ic", "max": 100, "min": 0 },
{ "type": "bigint", "name": "bi", "max": 100, "min": 0 },
{ "type": "utinyint", "name": "uti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "usi", "max": 100, "min": 0 },
{ "type": "uint", "name": "ui", "max": 100, "min": 0 },
{ "type": "ubigint", "name": "ubi", "max": 100, "min": 0 },
{ "type": "binary", "name": "bin", "len": 32},
{ "type": "nchar", "name": "nch", "len": 64}
],
"tags": [
{
"type": "tinyint",
"name": "groupid",
"max": 10,
"min": 1
},
{
"name": "location",
"type": "binary",
"len": 16,
"values": ["San Francisco", "Los Angles", "San Diego",
"San Jose", "Palo Alto", "Campbell", "Mountain View",
"Sunnyvale", "Santa Clara", "Cupertino"]
}
]
}
]
}
]
}

81
tests/system-test/5-taos-tools/taosbenchmark/json/stt.json Normal file
View File

@ -0,0 +1,81 @@
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"connection_pool_size": 8,
"num_of_records_per_req": 3000,
"thread_count": 20,
"create_table_thread_count": 5,
"result_file": "./insert_res_wal.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"databases": [
{
"dbinfo": {
"name": "db",
"drop": "yes",
"flush_each_batch": "yes",
"vgroups": 2,
"replica": 1,
"precision": "ms",
"keep": 3650,
"minRows": 100,
"maxRows": 4096
},
"super_tables": [
{
"name": "meters",
"child_table_exists": "no",
"childtable_count": 1000,
"insert_rows": 2850,
"childtable_prefix": "d",
"insert_mode": "taosc",
"insert_interval": 0,
"timestamp_step": 10,
"disorder_ratio": 10,
"update_ratio": 5,
"delete_ratio": 1,
"disorder_fill_interval": 30,
"update_fill_interval": 25,
"generate_row_rule": 2,
"start_timestamp":"2022-01-01 10:00:00",
"columns": [
{ "type": "bool", "name": "bc"},
{ "type": "float", "name": "fc", "max": 1, "min": 0 },
{ "type": "double", "name": "dc", "max": 1, "min": 0 },
{ "type": "tinyint", "name": "ti", "max": 100, "min": 0 },
{ "type": "smallint", "name": "si", "max": 100, "min": 0 },
{ "type": "int", "name": "ic", "max": 100, "min": 0 },
{ "type": "bigint", "name": "bi", "max": 100, "min": 0 },
{ "type": "utinyint", "name": "uti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "usi", "max": 100, "min": 0 },
{ "type": "uint", "name": "ui", "max": 100, "min": 0 },
{ "type": "ubigint", "name": "ubi", "max": 100, "min": 0 },
{ "type": "binary", "name": "bin", "len": 32},
{ "type": "nchar", "name": "nch", "len": 64}
],
"tags": [
{
"type": "tinyint",
"name": "groupid",
"max": 10,
"min": 1
},
{
"name": "location",
"type": "binary",
"len": 16,
"values": ["San Francisco", "Los Angles", "San Diego",
"San Jose", "Palo Alto", "Campbell", "Mountain View",
"Sunnyvale", "Santa Clara", "Cupertino"]
}
]
}
]
}
]
}

102
tests/system-test/5-taos-tools/taosbenchmark/stt.py Normal file
View File

@ -0,0 +1,102 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import os
import subprocess
import time
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def caseDescription(self):
"""
[TD-13823] taosBenchmark test cases
"""
return
def init(self, conn, logSql, replicaVar=1):
# comment off by Shuduo for CI self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def getPath(self, tool="taosBenchmark"):
if (platform.system().lower() == 'windows'):
tool = tool + ".exe"
selfPath = os.path.dirname(os.path.realpath(__file__))
if "community" in selfPath:
projPath = selfPath[: selfPath.find("community")]
else:
projPath = selfPath[: selfPath.find("tests")]
paths = []
for root, dirs, files in os.walk(projPath):
if (tool) in files:
rootRealPath = os.path.dirname(os.path.realpath(root))
if "packaging" not in rootRealPath:
paths.append(os.path.join(root, tool))
break
if len(paths) == 0:
tdLog.exit("taosBenchmark not found!")
return
else:
tdLog.info("taosBenchmark found in %s" % paths[0])
return paths[0]
def checkDataCorrect(self):
sql = "select count(*) from meters"
tdSql.query(sql)
allCnt = tdSql.getData(0, 0)
if allCnt < 2000000:
tdLog.exit(f"taosbenchmark insert row small. row count={allCnt} sql={sql}")
return
# group by 10 child table
rowCnt = tdSql.query("select count(*),tbname from meters group by tbname")
tdSql.checkRows(1000)
# interval
sql = "select count(*),max(ic),min(dc),last(*) from meters interval(1s)"
rowCnt = tdSql.query(sql)
if rowCnt < 10:
tdLog.exit(f"taosbenchmark interval(1s) count small. row cout={rowCnt} sql={sql}")
return
# nest query
tdSql.query("select count(*) from (select * from meters order by ts desc)")
tdSql.checkData(0, 0, allCnt)
def run(self):
binPath = self.getPath()
cmd = "%s -f ./5-taos-tools/taosbenchmark/json/stt.json" % binPath
tdLog.info("%s" % cmd)
errcode = os.system("%s" % cmd)
if errcode != 0:
tdLog.exit(f"execute taosBenchmark ret error code={errcode}")
return
tdSql.execute("use db")
self.checkDataCorrect()
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

4
tests/system-test/7-tmq/subscribeStb3.py
View File

@ -546,7 +546,7 @@ class TDTestCase:
keyList = 'group.id:cgrp1,\ keyList = 'group.id:cgrp1,\
enable.auto.commit:false,\ enable.auto.commit:false,\
auto.commit.interval.ms:6000,\ auto.commit.interval.ms:6000,\
auto.offset.reset:none' auto.offset.reset:earliest'
self.insertConsumerInfo(consumerId, expectrowcnt/2,topicList,keyList,ifcheckdata,ifManualCommit) self.insertConsumerInfo(consumerId, expectrowcnt/2,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("again start consume processor") tdLog.info("again start consume processor")
@ -569,7 +569,7 @@ class TDTestCase:
keyList = 'group.id:cgrp1,\ keyList = 'group.id:cgrp1,\
enable.auto.commit:false,\ enable.auto.commit:false,\
auto.commit.interval.ms:6000,\ auto.commit.interval.ms:6000,\
auto.offset.reset:none' auto.offset.reset:earliest'
self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("again start consume processor") tdLog.info("again start consume processor")

Some files were not shown because too many files have changed in this diff Show More