xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #28654 from taosdata/3.0 

3.0
This commit is contained in:
dongming chen 2024-11-05 18:09:31 +08:00 committed by GitHub
parent 9d04ddf727 cca24b7381
commit 3671d3c51f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
197 changed files with 13531 additions and 7525 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/taoskeeper-ci.yml vendored
View File

@ -28,7 +28,7 @@ jobs:
run: | run: |
mkdir debug mkdir debug
cd debug cd debug
cmake .. -DBUILD_HTTP=false -DBUILD_JDBC=false -DBUILD_TOOLS=false -DBUILD_TEST=off -DBUILD_KEEPER=true cmake .. -DBUILD_HTTP=false -DBUILD_JDBC=false -DBUILD_TOOLS=false -DBUILD_TEST=off -DBUILD_KEEPER=true -DBUILD_DEPENDENCY_TESTS=false
make -j 4 make -j 4
sudo make install sudo make install
which taosd which taosd
@ -48,7 +48,7 @@ jobs:
working-directory: tools/keeper working-directory: tools/keeper
run: | run: |
go mod tidy go mod tidy
go test -v -coverpkg=./... -coverprofile=coverage.out ./... sudo go test -v -ldflags="-X 'github.com/taosdata/taoskeeper/version.IsEnterprise=true'" -coverpkg=./... -coverprofile=coverage.out ./...
go tool cover -func=coverage.out go tool cover -func=coverage.out
- name: Clean up - name: Clean up

41
cmake/cmake.define
View File

@ -191,48 +191,11 @@ ELSE()
SET(COMPILER_SUPPORT_AVX512VL false) SET(COMPILER_SUPPORT_AVX512VL false)
ELSE() ELSE()
CHECK_C_COMPILER_FLAG("-mfma" COMPILER_SUPPORT_FMA) CHECK_C_COMPILER_FLAG("-mfma" COMPILER_SUPPORT_FMA)
CHECK_C_COMPILER_FLAG("-mavx" COMPILER_SUPPORT_AVX)
CHECK_C_COMPILER_FLAG("-mavx2" COMPILER_SUPPORT_AVX2)
CHECK_C_COMPILER_FLAG("-mavx512f" COMPILER_SUPPORT_AVX512F) CHECK_C_COMPILER_FLAG("-mavx512f" COMPILER_SUPPORT_AVX512F)
CHECK_C_COMPILER_FLAG("-mavx512vbmi" COMPILER_SUPPORT_AVX512BMI) CHECK_C_COMPILER_FLAG("-mavx512vbmi" COMPILER_SUPPORT_AVX512BMI)
CHECK_C_COMPILER_FLAG("-mavx512vl" COMPILER_SUPPORT_AVX512VL) CHECK_C_COMPILER_FLAG("-mavx512vl" COMPILER_SUPPORT_AVX512VL)
INCLUDE(CheckCSourceRuns)
SET(CMAKE_REQUIRED_FLAGS "-mavx")
check_c_source_runs("
#include <immintrin.h>
int main() {
__m256d a, b, c;
double buf[4] = {0};
a = _mm256_loadu_pd(buf);
b = _mm256_loadu_pd(buf);
c = _mm256_add_pd(a, b);
_mm256_storeu_pd(buf, c);
for (int i = 0; i < sizeof(buf) / sizeof(buf[0]); ++i) {
IF (buf[i] != 0) {
return 1;
}
}
return 0;
}
" COMPILER_SUPPORT_AVX)
SET(CMAKE_REQUIRED_FLAGS "-mavx2")
check_c_source_runs("
#include <immintrin.h>
int main() {
__m256i a, b, c;
int buf[8] = {0};
a = _mm256_loadu_si256((__m256i *)buf);
b = _mm256_loadu_si256((__m256i *)buf);
c = _mm256_and_si256(a, b);
_mm256_storeu_si256((__m256i *)buf, c);
for (int i = 0; i < sizeof(buf) / sizeof(buf[0]); ++i) {
IF (buf[i] != 0) {
return 1;
}
}
return 0;
}
" COMPILER_SUPPORT_AVX2)
ENDIF() ENDIF()
IF(COMPILER_SUPPORT_SSE42) IF(COMPILER_SUPPORT_SSE42)

8
contrib/CMakeLists.txt
View File

@ -261,11 +261,19 @@ if(${TD_DARWIN})
endif(${TD_DARWIN}) endif(${TD_DARWIN})
add_subdirectory(zlib EXCLUDE_FROM_ALL) add_subdirectory(zlib EXCLUDE_FROM_ALL)
if(${TD_DARWIN})
target_compile_options(zlibstatic PRIVATE -Wno-error=deprecated-non-prototype)
endif()
target_include_directories( target_include_directories(
zlibstatic zlibstatic
PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib
PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/zlib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/zlib
) )
if(${TD_DARWIN})
target_compile_options(zlib PRIVATE -Wno-error=deprecated-non-prototype)
endif()
target_include_directories( target_include_directories(
zlib zlib
PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib

2
docs/en/03-intro.md
View File

@ -46,7 +46,7 @@ For more details on features, please read through the entire documentation.
By making full use of [characteristics of time series data](https://tdengine.com/characteristics-of-time-series-data/), TDengine differentiates itself from other time series databases with the following advantages. By making full use of [characteristics of time series data](https://tdengine.com/characteristics-of-time-series-data/), TDengine differentiates itself from other time series databases with the following advantages.
- **[High-Performance](https://tdengine.com/high-performance/)**: TDengine is the only time-series database to solve the high cardinality issue to support billions of data collection points while out performing other time-series databases for data ingestion, querying and data compression. - **[High-Performance](https://tdengine.com/high-performance/)**: TDengine is the only time-series database to solve the high cardinality issue to support billions of data collection points while outperforming other time-series databases for data ingestion, querying and data compression.
- **[Simplified Solution](https://tdengine.com/comprehensive-industrial-data-solution/)**: Through built-in caching, stream processing and data subscription features, TDengine provides a simplified solution for time-series data processing. It reduces system design complexity and operation costs significantly. - **[Simplified Solution](https://tdengine.com/comprehensive-industrial-data-solution/)**: Through built-in caching, stream processing and data subscription features, TDengine provides a simplified solution for time-series data processing. It reduces system design complexity and operation costs significantly.

2
docs/en/14-reference/03-taos-sql/10-function.md
View File

@ -1187,7 +1187,7 @@ CSUM(expr)
### DERIVATIVE ### DERIVATIVE
```sql ```sql
DERIVATIVE(expr, time_inerval, ignore_negative) DERIVATIVE(expr, time_interval, ignore_negative)
ignore_negative: { ignore_negative: {
0 0

4
docs/en/14-reference/03-taos-sql/31-compress.md
View File

@ -41,7 +41,7 @@ In this article, it specifically refers to the level within the secondary compre
### Create Table with Compression ### Create Table with Compression
```sql ```sql
CREATE [dbname.]tabname (colName colType [ENCODE 'encode_type'] [COMPRESS 'compress_type' [LEVEL 'level'], [, other cerate_definition]...]) CREATE [dbname.]tabname (colName colType [ENCODE 'encode_type'] [COMPRESS 'compress_type' [LEVEL 'level'], [, other create_definition]...])
``` ```
**Parameter Description** **Parameter Description**
@ -58,7 +58,7 @@ CREATE [dbname.]tabname (colName colType [ENCODE 'encode_type'] [COMPRESS 'compr
### Change Compression Method ### Change Compression Method
```sql ```sql
ALTER TABLE [db_name.]tabName MODIFY COLUMN colName [ENCODE 'ecode_type'] [COMPRESS 'compress_type'] [LEVEL "high"] ALTER TABLE [db_name.]tabName MODIFY COLUMN colName [ENCODE 'encode_type'] [COMPRESS 'compress_type'] [LEVEL "high"]
``` ```
**Parameter Description** **Parameter Description**

2
docs/en/14-reference/05-connectors/60-rest-api.mdx
View File

@ -125,7 +125,7 @@ where `TOKEN` is the string after Base64 encoding of `{username}:{password}`, e.
Starting from `TDengine 3.0.3.0`, `taosAdapter` provides a configuration parameter `httpCodeServerError` to set whether to return a non-200 http status code when the C interface returns an error Starting from `TDengine 3.0.3.0`, `taosAdapter` provides a configuration parameter `httpCodeServerError` to set whether to return a non-200 http status code when the C interface returns an error
| **Description** | **httpCodeServerError false** | **httpCodeServerError true** | | **Description** | **httpCodeServerError false** | **httpCodeServerError true** |
|--------------------|---------------------------- ------|---------------------------------------| |--------------------|----------------------------------|---------------------------------------|
| taos_errno() returns 0 | 200 | 200 | | taos_errno() returns 0 | 200 | 200 |
| taos_errno() returns non-0 | 200 (except authentication error) | 500 (except authentication error and 400/502 error) | | taos_errno() returns non-0 | 200 (except authentication error) | 500 (except authentication error and 400/502 error) |
| Parameter error | 400 (only handle HTTP request URL parameter error) | 400 (handle HTTP request URL parameter error and taosd return error) | | Parameter error | 400 (only handle HTTP request URL parameter error) | 400 (handle HTTP request URL parameter error and taosd return error) |

19
docs/en/14-reference/12-config/index.md
View File

@ -701,15 +701,6 @@ The charset that takes effect is UTF-8.
| Type | String | | Type | String |
| Default Value | _tag_null | | Default Value | _tag_null |
### smlDataFormat
| Attribute | Description |
| ----------- | ----------------------------------------------------------------------------------- |
| Applicable | Client only |
| Meaning | Whether schemaless columns are consistently ordered, depat, discarded since 3.0.3.0 |
| Value Range | 0: not consistent; 1: consistent. |
| Default | 0 |
### smlTsDefaultName ### smlTsDefaultName
| Attribute | Description | | Attribute | Description |
@ -719,6 +710,16 @@ The charset that takes effect is UTF-8.
| Type | String | | Type | String |
| Default Value | _ts | | Default Value | _ts |
### smlDot2Underline
| Attribute | Description |
| -------- | -------------------------------------------------------- |
| Applicable | Client only |
| Meaning | Convert the dot in the supertable name to an underscore |
| Type | Bool |
| Default Value | true |
## Compress Parameters ## Compress Parameters
### compressMsgSize ### compressMsgSize

8
docs/en/26-tdinternal/04-load-balance.md
View File

@ -4,7 +4,7 @@ sidebar_label: Load Balance
description: This document describes how TDengine implements load balancing. description: This document describes how TDengine implements load balancing.
--- ---
The load balance in TDengine is mainly about processing data series data. TDengine employes builtin hash algorithm to distribute all the tables, sub-tables and their data of a database across all the vgroups that belongs to the database. Each table or sub-table can only be handled by a single vgroup, while each vgroup can process multiple table or sub-table. The load balance in TDengine is mainly about processing data series data. TDengine employs builtin hash algorithm to distribute all the tables, sub-tables and their data of a database across all the vgroups that belongs to the database. Each table or sub-table can only be handled by a single vgroup, while each vgroup can process multiple table or sub-table.
The number of vgroup can be specified when creating a database, using the parameter `vgroups`. The number of vgroup can be specified when creating a database, using the parameter `vgroups`.
@ -12,10 +12,10 @@ The number of vgroup can be specified when creating a database, using the parame
create database db0 vgroups 100; create database db0 vgroups 100;
``` ```
The proper value of `vgroups` depends on available system resources. Assuming there is only one database to be created in the system, then the number of `vgroups` is determined by the available resources from all dnodes. In principle more vgroups can be created if you have more CPU and memory. Disk I/O is another important factor to consider. Once the bottleneck shows on disk I/O, more vgroups may downgrad the system performance significantly. If multiple databases are to be created in the system, then the total number of `vroups` of all the databases are dependent on the available system resources. It needs to be careful to distribute vgroups among these databases, you need to consider the number of tables, data writing frequency, size of each data row for all these databases. A recommended practice is to firstly choose a starting number for `vgroups`, for example double of the number of CPU cores, then try to adjust and optimize system configurations to find the best setting for `vgroups`, then distribute these vgroups among databases. The proper value of `vgroups` depends on available system resources. Assuming there is only one database to be created in the system, then the number of `vgroups` is determined by the available resources from all dnodes. In principle more vgroups can be created if you have more CPU and memory. Disk I/O is another important factor to consider. Once the bottleneck shows on disk I/O, more vgroups may degrade the system performance significantly. If multiple databases are to be created in the system, then the total number of `vgroups` of all the databases are dependent on the available system resources. It needs to be careful to distribute vgroups among these databases, you need to consider the number of tables, data writing frequency, size of each data row for all these databases. A recommended practice is to firstly choose a starting number for `vgroups`, for example double of the number of CPU cores, then try to adjust and optimize system configurations to find the best setting for `vgroups`, then distribute these vgroups among databases.
Furthermode, TDengine distributes the vgroups of each database equally among all dnodes. In case of replica 3, the distribution is even more complex, TDengine tries its best to prevent any dnode from becoming a bottleneck. Furthermore, TDengine distributes the vgroups of each database equally among all dnodes. In case of replica 3, the distribution is even more complex, TDengine tries its best to prevent any dnode from becoming a bottleneck.
TDegnine utilizes the above ways to achieve load balance in a cluster, and finally achieve higher throughput. TDengine utilizes the above ways to achieve load balance in a cluster, and finally achieve higher throughput.
Once the load balance is achieved, after some operations like deleting tables or dropping databases, the load across all dnodes may become imbalanced, the method of rebalance will be provided in later versions. However, even without explicit rebalancing, TDengine will try its best to achieve new balance without manual interfering when a new database is created. Once the load balance is achieved, after some operations like deleting tables or dropping databases, the load across all dnodes may become imbalanced, the method of rebalance will be provided in later versions. However, even without explicit rebalancing, TDengine will try its best to achieve new balance without manual interfering when a new database is created.

4
docs/examples/JDBC/taosdemo/readme.md
View File

@ -7,8 +7,4 @@ java -jar target/taosdemo-2.0.1-jar-with-dependencies.jar -host <hostname> -data
java -jar target/taosdemo-2.0.1-jar-with-dependencies.jar -host <hostname> -database <db name> -doCreateTable false -superTableSQL "create table weather(ts timestamp, f1 int) tags(t1 nchar(4))" -numOfTables 1000 -numOfRowsPerTable 100000000 -numOfThreadsForInsert 10 -numOfTablesPerSQL 10 -numOfValuesPerSQL 100 java -jar target/taosdemo-2.0.1-jar-with-dependencies.jar -host <hostname> -database <db name> -doCreateTable false -superTableSQL "create table weather(ts timestamp, f1 int) tags(t1 nchar(4))" -numOfTables 1000 -numOfRowsPerTable 100000000 -numOfThreadsForInsert 10 -numOfTablesPerSQL 10 -numOfValuesPerSQL 100
``` ```
如果发生错误 Exception in thread "main" java.lang.UnsatisfiedLinkError: no taos in java.library.path
请检查是否安装 TDengine 客户端安装包或编译 TDengine 安装。如果确定已经安装过还出现这个错误，可以在命令行 java 后加 -Djava.library.path=/usr/lib 来指定寻找共享库的路径。
If you encounter the error Exception in thread "main" `java.lang.UnsatisfiedLinkError: no taos in java.library.path`, please check whether the TDengine client package is installed or TDengine is compiled and installed. If you are sure it is installed and still encounter this error, you can add `-Djava.library.path=/usr/lib` after the `java` command to specify the path to the shared library. If you encounter the error Exception in thread "main" `java.lang.UnsatisfiedLinkError: no taos in java.library.path`, please check whether the TDengine client package is installed or TDengine is compiled and installed. If you are sure it is installed and still encounter this error, you can add `-Djava.library.path=/usr/lib` after the `java` command to specify the path to the shared library.

2
docs/examples/node/package.json
View File

@ -4,6 +4,6 @@
"main": "index.js", "main": "index.js",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"@tdengine/websocket": "^3.1.0" "@tdengine/websocket": "^3.1.1"
} }
} }

1
docs/examples/node/websocketexample/sql_example.js
View File

@ -3,7 +3,6 @@ const taos = require("@tdengine/websocket");
let dsn = 'ws://localhost:6041'; let dsn = 'ws://localhost:6041';
async function createConnect() { async function createConnect() {
try { try {
let conf = new taos.WSConfig(dsn); let conf = new taos.WSConfig(dsn);
conf.setUser('root'); conf.setUser('root');

1
docs/examples/node/websocketexample/tmq_seek_example.js
View File

@ -10,7 +10,6 @@ const groupId = "group1";
const clientId = "client1"; const clientId = "client1";
async function createConsumer() { async function createConsumer() {
let groupId = "group1"; let groupId = "group1";
let clientId = "client1"; let clientId = "client1";
let configMap = new Map([ let configMap = new Map([

311
docs/zh/05-basic/03-query.md
View File

@ -8,7 +8,7 @@ toc_max_heading_level: 4
## 基本查询 ## 基本查询
为了更好的介绍 TDengine 数据查询，使用 如下 taosBenchmark 命令，生成本章内容需要的时序数据。 为了更好的介绍 TDengine 数据查询，使用如下 taosBenchmark 命令，生成本章内容需要的时序数据。
```shell ```shell
taosBenchmark --start-timestamp=1600000000000 --tables=100 --records=10000000 --time-step=10000 taosBenchmark --start-timestamp=1600000000000 --tables=100 --records=10000000 --time-step=10000
@ -20,21 +20,22 @@ taosBenchmark --start-timestamp=1600000000000 --tables=100 --records=10000000 --
```sql ```sql
SELECT * FROM meters SELECT * FROM meters
WHERE voltage > 10 WHERE voltage > 230
ORDER BY ts DESC ORDER BY ts DESC
LIMIT 5 LIMIT 5;
``` ```
上面的 SQL，从超级表 `meters` 中查询出电压 `voltage` 大于 10 的记录，按时间降序排列，且仅输出前 5 行。查询结果如下： 上面的 SQL，从超级表 `meters` 中查询出电压 `voltage` 大于 230V 的记录，按时间降序排列，且仅输出前 5 行。查询结果如下：
```text ```text
ts | current | voltage | phase | groupid | location | ts | current | voltage | phase | groupid | location |
========================================================================================================== ===================================================================================================
2023-11-14 22:13:10.000 | 1.1294620 | 18 | 0.3531540 | 8 | California.MountainView | 2023-11-15 06:13:10.000 | 14.0601978 | 232 | 146.5000000 | 10 | California.Sunnyvale |
2023-11-14 22:13:10.000 | 1.0294620 | 12 | 0.3631540 | 2 | California.Campbell | 2023-11-15 06:13:10.000 | 14.0601978 | 232 | 146.5000000 | 1 | California.LosAngles |
2023-11-14 22:13:10.000 | 1.0294620 | 16 | 0.3531540 | 1 | California.Campbell | 2023-11-15 06:13:10.000 | 14.0601978 | 232 | 146.5000000 | 10 | California.Sunnyvale |
2023-11-14 22:13:10.000 | 1.1294620 | 18 | 0.3531540 | 2 | California.Campbell | 2023-11-15 06:13:10.000 | 14.0601978 | 232 | 146.5000000 | 5 | California.Cupertino |
2023-11-14 22:13:10.000 | 1.1294620 | 16 | 0.3431540 | 7 | California.PaloAlto | 2023-11-15 06:13:10.000 | 14.0601978 | 232 | 146.5000000 | 4 | California.SanFrancisco |
Query OK, 5 row(s) in set (0.145403s)
``` ```
## 聚合查询 ## 聚合查询
@ -48,28 +49,28 @@ TDengine 支持通过 GROUP BY 子句，对数据进行聚合查询。SQL 语句
group by 子句用于对数据进行分组，并为每个分组返回一行汇总信息。在 group by 子句中，可以使用表或视图中的任何列作为分组依据，这些列不需要出现在 select 列表中。此外，用户可以直接在超级表上执行聚合查询，无须预先创建子表。以智能电表的数据模型为例，使用 group by 子句的 SQL 如下： group by 子句用于对数据进行分组，并为每个分组返回一行汇总信息。在 group by 子句中，可以使用表或视图中的任何列作为分组依据，这些列不需要出现在 select 列表中。此外，用户可以直接在超级表上执行聚合查询，无须预先创建子表。以智能电表的数据模型为例，使用 group by 子句的 SQL 如下：
```sql ```sql
SELECT groupid,avg(voltage) SELECT groupid, avg(voltage)
FROM meters FROM meters
WHERE ts >= "2022-01-01T00:00:00+08:00" WHERE ts >= "2022-01-01T00:00:00+08:00"
AND ts < "2023-01-01T00:00:00+08:00" AND ts < "2023-01-01T00:00:00+08:00"
GROUP BY groupid GROUP BY groupid;
``` ```
上面的 SQL，查询超级表 `meters` 中，时间戳大于等于 `2022-01-01T00:00:00+08:00`，且时间戳小于 `2023-01-01T00:00:00+08:00` 的数据，按照 `groupid` 进行分组，求每组的平均电压。查询结果如下： 上面的 SQL，查询超级表 `meters` 中，时间戳大于等于 `2022-01-01T00:00:00+08:00`，且时间戳小于 `2023-01-01T00:00:00+08:00` 的数据，按照 `groupid` 进行分组，求每组的平均电压。查询结果如下：
```text ```text
groupid | avg(voltage) | groupid | avg(voltage) |
========================================== ======================================
8 | 9.104040404040404 | 8 | 243.961981544901079 |
5 | 9.078333333333333 | 5 | 243.961981544901079 |
1 | 9.087037037037037 | 1 | 243.961981544901079 |
7 | 8.991414141414142 | 7 | 243.961981544901079 |
9 | 8.789814814814815 | 9 | 243.961981544901079 |
6 | 9.051010101010101 | 6 | 243.961981544901079 |
4 | 9.135353535353536 | 4 | 243.961981544901079 |
10 | 9.213131313131314 | 10 | 243.961981544901079 |
2 | 9.008888888888889 | 2 | 243.961981544901079 |
3 | 8.783888888888889 | 3 | 243.961981544901079 |
Query OK, 10 row(s) in set (0.042446s) Query OK, 10 row(s) in set (0.042446s)
``` ```
@ -110,24 +111,24 @@ TDengine 按如下方式处理数据切分子句。
```sql ```sql
SELECT location, avg(voltage) SELECT location, avg(voltage)
FROM meters FROM meters
PARTITION BY location PARTITION BY location;
``` ```
上面的示例 SQL 查询超级表 `meters`，将数据按标签 `location` 进行分组，每个分组计算电压的平均值。查询结果如下： 上面的示例 SQL 查询超级表 `meters`，将数据按标签 `location` 进行分组，每个分组计算电压的平均值。查询结果如下：
```text ```text
location | avg(voltage) | location | avg(voltage) |
========================================================= ======================================================
California.SantaClara | 8.793334320000000 | California.SantaClara | 243.962050000000005 |
California.SanFrancisco | 9.017645882352941 | California.SanFrancisco | 243.962050000000005 |
California.SanJose | 9.156112940000000 | California.SanJose | 243.962050000000005 |
California.LosAngles | 9.036753507692307 | California.LosAngles | 243.962050000000005 |
California.SanDiego | 8.967037053333334 | California.SanDiego | 243.962050000000005 |
California.Sunnyvale | 8.978572085714285 | California.Sunnyvale | 243.962050000000005 |
California.PaloAlto | 8.936665800000000 | California.PaloAlto | 243.962050000000005 |
California.Cupertino | 8.987654066666666 | California.Cupertino | 243.962050000000005 |
California.MountainView | 9.046297266666667 | California.MountainView | 243.962050000000005 |
California.Campbell | 9.149999028571429 | California.Campbell | 243.962050000000005 |
Query OK, 10 row(s) in set (2.415961s) Query OK, 10 row(s) in set (2.415961s)
``` ```
@ -200,20 +201,20 @@ SLIMIT 2;
上面的 SQL，查询超级表 `meters` 中，时间戳大于等于 `2022-01-01T00:00:00+08:00`，且时间戳小于 `2022-01-01T00:05:00+08:00` 的数据；数据首先按照子表名 `tbname` 进行数据切分，再按照每 1 分钟的时间窗口进行切分，且每个时间窗口向后偏移 5 秒；最后，仅取前 2 个分片的数据作为结果。查询结果如下： 上面的 SQL，查询超级表 `meters` 中，时间戳大于等于 `2022-01-01T00:00:00+08:00`，且时间戳小于 `2022-01-01T00:05:00+08:00` 的数据；数据首先按照子表名 `tbname` 进行数据切分，再按照每 1 分钟的时间窗口进行切分，且每个时间窗口向后偏移 5 秒；最后，仅取前 2 个分片的数据作为结果。查询结果如下：
```text ```text
tbname | _wstart | _wend | avg(voltage) | tbname | _wstart | _wend | avg(voltage) |
========================================================================================== ======================================================================================
d40 | 2021-12-31 15:59:05.000 | 2021-12-31 16:00:05.000 | 4.000000000000000 | d2 | 2021-12-31 23:59:05.000 | 2022-01-01 00:00:05.000 | 253.000000000000000 |
d40 | 2021-12-31 16:00:05.000 | 2021-12-31 16:01:05.000 | 5.000000000000000 | d2 | 2022-01-01 00:00:05.000 | 2022-01-01 00:01:05.000 | 244.166666666666657 |
d40 | 2021-12-31 16:01:05.000 | 2021-12-31 16:02:05.000 | 8.000000000000000 | d2 | 2022-01-01 00:01:05.000 | 2022-01-01 00:02:05.000 | 241.833333333333343 |
d40 | 2021-12-31 16:02:05.000 | 2021-12-31 16:03:05.000 | 7.666666666666667 | d2 | 2022-01-01 00:02:05.000 | 2022-01-01 00:03:05.000 | 243.166666666666657 |
d40 | 2021-12-31 16:03:05.000 | 2021-12-31 16:04:05.000 | 9.666666666666666 | d2 | 2022-01-01 00:03:05.000 | 2022-01-01 00:04:05.000 | 240.833333333333343 |
d40 | 2021-12-31 16:04:05.000 | 2021-12-31 16:05:05.000 | 15.199999999999999 | d2 | 2022-01-01 00:04:05.000 | 2022-01-01 00:05:05.000 | 244.800000000000011 |
d41 | 2021-12-31 15:59:05.000 | 2021-12-31 16:00:05.000 | 4.000000000000000 | d26 | 2021-12-31 23:59:05.000 | 2022-01-01 00:00:05.000 | 253.000000000000000 |
d41 | 2021-12-31 16:00:05.000 | 2021-12-31 16:01:05.000 | 7.000000000000000 | d26 | 2022-01-01 00:00:05.000 | 2022-01-01 00:01:05.000 | 244.166666666666657 |
d41 | 2021-12-31 16:01:05.000 | 2021-12-31 16:02:05.000 | 9.000000000000000 | d26 | 2022-01-01 00:01:05.000 | 2022-01-01 00:02:05.000 | 241.833333333333343 |
d41 | 2021-12-31 16:02:05.000 | 2021-12-31 16:03:05.000 | 10.666666666666666 | d26 | 2022-01-01 00:02:05.000 | 2022-01-01 00:03:05.000 | 243.166666666666657 |
d41 | 2021-12-31 16:03:05.000 | 2021-12-31 16:04:05.000 | 8.333333333333334 | d26 | 2022-01-01 00:03:05.000 | 2022-01-01 00:04:05.000 | 240.833333333333343 |
d41 | 2021-12-31 16:04:05.000 | 2021-12-31 16:05:05.000 | 9.600000000000000 | d26 | 2022-01-01 00:04:05.000 | 2022-01-01 00:05:05.000 | 244.800000000000011 |
Query OK, 12 row(s) in set (0.021265s) Query OK, 12 row(s) in set (0.021265s)
``` ```
@ -255,19 +256,19 @@ SLIMIT 1;
上面的 SQL，查询超级表 `meters` 中，时间戳大于等于 `2022-01-01T00:00:00+08:00`，且时间戳小于 `2022-01-01T00:05:00+08:00` 的数据，数据首先按照子表名 `tbname` 进行数据切分，再按照每 1 分钟的时间窗口进行切分，且时间窗口按照 30 秒进行滑动；最后，仅取前 1 个分片的数据作为结果。查询结果如下： 上面的 SQL，查询超级表 `meters` 中，时间戳大于等于 `2022-01-01T00:00:00+08:00`，且时间戳小于 `2022-01-01T00:05:00+08:00` 的数据，数据首先按照子表名 `tbname` 进行数据切分，再按照每 1 分钟的时间窗口进行切分，且时间窗口按照 30 秒进行滑动；最后，仅取前 1 个分片的数据作为结果。查询结果如下：
```text ```text
tbname | _wstart | avg(voltage) | tbname | _wstart | avg(voltage) |
================================================================ =============================================================
d40 | 2021-12-31 15:59:30.000 | 4.000000000000000 | d2 | 2021-12-31 23:59:30.000 | 248.333333333333343 |
d40 | 2021-12-31 16:00:00.000 | 5.666666666666667 | d2 | 2022-01-01 00:00:00.000 | 246.000000000000000 |
d40 | 2021-12-31 16:00:30.000 | 4.333333333333333 | d2 | 2022-01-01 00:00:30.000 | 244.666666666666657 |
d40 | 2021-12-31 16:01:00.000 | 5.000000000000000 | d2 | 2022-01-01 00:01:00.000 | 240.833333333333343 |
d40 | 2021-12-31 16:01:30.000 | 9.333333333333334 | d2 | 2022-01-01 00:01:30.000 | 239.500000000000000 |
d40 | 2021-12-31 16:02:00.000 | 9.666666666666666 | d2 | 2022-01-01 00:02:00.000 | 243.833333333333343 |
d40 | 2021-12-31 16:02:30.000 | 10.000000000000000 | d2 | 2022-01-01 00:02:30.000 | 243.833333333333343 |
d40 | 2021-12-31 16:03:00.000 | 10.333333333333334 | d2 | 2022-01-01 00:03:00.000 | 241.333333333333343 |
d40 | 2021-12-31 16:03:30.000 | 10.333333333333334 | d2 | 2022-01-01 00:03:30.000 | 241.666666666666657 |
d40 | 2021-12-31 16:04:00.000 | 13.000000000000000 | d2 | 2022-01-01 00:04:00.000 | 244.166666666666657 |
d40 | 2021-12-31 16:04:30.000 | 15.333333333333334 | d2 | 2022-01-01 00:04:30.000 | 244.666666666666657 |
Query OK, 11 row(s) in set (0.013153s) Query OK, 11 row(s) in set (0.013153s)
``` ```
@ -290,13 +291,13 @@ SLIMIT 1;
上面的 SQL，查询超级表 `meters` 中，时间戳大于等于 `2022-01-01T00:00:00+08:00`，且时间戳小于 `2022-01-01T00:05:00+08:00` 的数据，数据首先按照子表名 `tbname` 进行数据切分，再按照每 1 分钟的时间窗口进行切分，且时间窗口按照 1 分钟进行切分；最后，仅取前 1 个分片的数据作为结果。查询结果如下： 上面的 SQL，查询超级表 `meters` 中，时间戳大于等于 `2022-01-01T00:00:00+08:00`，且时间戳小于 `2022-01-01T00:05:00+08:00` 的数据，数据首先按照子表名 `tbname` 进行数据切分，再按照每 1 分钟的时间窗口进行切分，且时间窗口按照 1 分钟进行切分；最后，仅取前 1 个分片的数据作为结果。查询结果如下：
```text ```text
tbname | _wstart | _wend | avg(voltage) | tbname | _wstart | _wend | avg(voltage) |
================================================================================================================= ======================================================================================
d28 | 2021-12-31 16:00:00.000 | 2021-12-31 16:01:00.000 | 7.333333333333333 | d2 | 2022-01-01 00:00:00.000 | 2022-01-01 00:01:00.000 | 246.000000000000000 |
d28 | 2021-12-31 16:01:00.000 | 2021-12-31 16:02:00.000 | 8.000000000000000 | d2 | 2022-01-01 00:01:00.000 | 2022-01-01 00:02:00.000 | 240.833333333333343 |
d28 | 2021-12-31 16:02:00.000 | 2021-12-31 16:03:00.000 | 11.000000000000000 | d2 | 2022-01-01 00:02:00.000 | 2022-01-01 00:03:00.000 | 243.833333333333343 |
d28 | 2021-12-31 16:03:00.000 | 2021-12-31 16:04:00.000 | 6.666666666666667 | d2 | 2022-01-01 00:03:00.000 | 2022-01-01 00:04:00.000 | 241.333333333333343 |
d28 | 2021-12-31 16:04:00.000 | 2021-12-31 16:05:00.000 | 10.000000000000000 | d2 | 2022-01-01 00:04:00.000 | 2022-01-01 00:05:00.000 | 244.166666666666657 |
Query OK, 5 row(s) in set (0.016812s) Query OK, 5 row(s) in set (0.016812s)
``` ```
@ -342,53 +343,65 @@ SLIMIT 2;
上面的 SQL，查询超级表 `meters` 中，时间戳大于等于 `2022-01-01T00:00:00+08:00`，且时间戳小于 `2022-01-01T00:05:00+08:00` 的数据；数据首先按照子表名 `tbname` 进行数据切分，再按照每 1 分钟的时间窗口进行切分，如果窗口内的数据出现缺失，则使用使用前一个非 NULL 值填充数据；最后，仅取前 2 个分片的数据作为结果。查询结果如下： 上面的 SQL，查询超级表 `meters` 中，时间戳大于等于 `2022-01-01T00:00:00+08:00`，且时间戳小于 `2022-01-01T00:05:00+08:00` 的数据；数据首先按照子表名 `tbname` 进行数据切分，再按照每 1 分钟的时间窗口进行切分，如果窗口内的数据出现缺失，则使用使用前一个非 NULL 值填充数据；最后，仅取前 2 个分片的数据作为结果。查询结果如下：
```text ```text
tbname | _wstart | _wend | avg(voltage) | tbname | _wstart | _wend | avg(voltage) |
================================================================================================================= =======================================================================================
d40 | 2021-12-31 16:00:00.000 | 2021-12-31 16:01:00.000 | 5.666666666666667 | d2 | 2022-01-01 00:00:00.000 | 2022-01-01 00:01:00.000 | 246.000000000000000 |
d40 | 2021-12-31 16:01:00.000 | 2021-12-31 16:02:00.000 | 5.000000000000000 | d2 | 2022-01-01 00:01:00.000 | 2022-01-01 00:02:00.000 | 240.833333333333343 |
d40 | 2021-12-31 16:02:00.000 | 2021-12-31 16:03:00.000 | 9.666666666666666 | d2 | 2022-01-01 00:02:00.000 | 2022-01-01 00:03:00.000 | 243.833333333333343 |
d40 | 2021-12-31 16:03:00.000 | 2021-12-31 16:04:00.000 | 10.333333333333334 | d2 | 2022-01-01 00:03:00.000 | 2022-01-01 00:04:00.000 | 241.333333333333343 |
d40 | 2021-12-31 16:04:00.000 | 2021-12-31 16:05:00.000 | 13.000000000000000 | d2 | 2022-01-01 00:04:00.000 | 2022-01-01 00:05:00.000 | 244.166666666666657 |
d41 | 2021-12-31 16:00:00.000 | 2021-12-31 16:01:00.000 | 5.666666666666667 | d26 | 2022-01-01 00:00:00.000 | 2022-01-01 00:01:00.000 | 246.000000000000000 |
d41 | 2021-12-31 16:01:00.000 | 2021-12-31 16:02:00.000 | 9.333333333333334 | d26 | 2022-01-01 00:01:00.000 | 2022-01-01 00:02:00.000 | 240.833333333333343 |
d41 | 2021-12-31 16:02:00.000 | 2021-12-31 16:03:00.000 | 11.000000000000000 | d26 | 2022-01-01 00:02:00.000 | 2022-01-01 00:03:00.000 | 243.833333333333343 |
d41 | 2021-12-31 16:03:00.000 | 2021-12-31 16:04:00.000 | 7.666666666666667 | d26 | 2022-01-01 00:03:00.000 | 2022-01-01 00:04:00.000 | 241.333333333333343 |
d41 | 2021-12-31 16:04:00.000 | 2021-12-31 16:05:00.000 | 10.000000000000000 | d26 | 2022-01-01 00:04:00.000 | 2022-01-01 00:05:00.000 | 244.166666666666657 |
Query OK, 10 row(s) in set (0.022866s) Query OK, 10 row(s) in set (0.022866s)
``` ```
### 状态窗口 ### 状态窗口
使用整数（布尔值）或字符串来标识产生记录时候设备的状态量。产生的记录如果具有相同的状态量数值则归属于同一个状态窗口，数值改变后该窗口关闭。TDengine 还支持将 CASE 表达式用在状态量，可以表达某个状态的开始是由满足某个条件而触发，这个状态的结束是由另外一个条件满足而触发的语义。以智能电表为例，电压正常范围是 205V 到 235V，那么可以通过监控电压来判断电路是否正常。 使用整数（布尔值）或字符串来标识产生记录时候设备的状态量。产生的记录如果具有相同的状态量数值则归属于同一个状态窗口，数值改变后该窗口关闭。TDengine 还支持将 CASE 表达式用在状态量，可以表达某个状态的开始是由满足某个条件而触发，这个状态的结束是由另外一个条件满足而触发的语义。以智能电表为例，电压正常范围是 225V 到 235V，那么可以通过监控电压来判断电路是否正常。
```sql ```sql
SELECT tbname, _wstart, _wend,_wduration, CASE WHEN voltage >= 205 and voltage <= 235 THEN 1 ELSE 0 END status SELECT tbname, _wstart, _wend,_wduration, CASE WHEN voltage >= 225 and voltage <= 235 THEN 1 ELSE 0 END status
FROM meters FROM meters
WHERE ts >= "2022-01-01T00:00:00+08:00" WHERE ts >= "2022-01-01T00:00:00+08:00"
AND ts < "2022-01-01T00:05:00+08:00" AND ts < "2022-01-01T00:05:00+08:00"
PARTITION BY tbname PARTITION BY tbname
STATE_WINDOW( STATE_WINDOW(
CASE WHEN voltage >= 205 and voltage <= 235 THEN 1 ELSE 0 END CASE WHEN voltage >= 225 and voltage <= 235 THEN 1 ELSE 0 END
) )
SLIMIT 10; SLIMIT 2;
``` ```
以上 SQL，查询超级表 meters 中，时间戳大于等于 2022-01-01T00:00:00+08:00，且时间戳小于 2022-01-01T00:05:00+08:00的数据；数据首先按照子表名 tbname 进行数据切分；根据电压是否在正常范围内进行状态窗口的划分；最后，取前 10 个分片的数据作为结果。查询结果如下： 以上 SQL，查询超级表 meters 中，时间戳大于等于 2022-01-01T00:00:00+08:00，且时间戳小于 2022-01-01T00:05:00+08:00的数据；数据首先按照子表名 tbname 进行数据切分；根据电压是否在正常范围内进行状态窗口的划分；最后，取前 2 个分片的数据作为结果。查询结果如下：（由于数据是随机生成，结果集包含的数据条数会有不同）
```text ```text
tbname | _wstart | _wend | _wduration | status | tbname | _wstart | _wend | _wduration | status |
===================================================================================================================================== ===============================================================================================
d76 | 2021-12-31 16:00:00.000 | 2021-12-31 16:04:50.000 | 290000 | 0 | d2 | 2022-01-01 00:00:00.000 | 2022-01-01 00:01:20.000 | 80000 | 0 |
d47 | 2021-12-31 16:00:00.000 | 2021-12-31 16:04:50.000 | 290000 | 0 | d2 | 2022-01-01 00:01:30.000 | 2022-01-01 00:01:30.000 | 0 | 1 |
d37 | 2021-12-31 16:00:00.000 | 2021-12-31 16:04:50.000 | 290000 | 0 | d2 | 2022-01-01 00:01:40.000 | 2022-01-01 00:01:40.000 | 0 | 0 |
d87 | 2021-12-31 16:00:00.000 | 2021-12-31 16:04:50.000 | 290000 | 0 | d2 | 2022-01-01 00:01:50.000 | 2022-01-01 00:01:50.000 | 0 | 1 |
d64 | 2021-12-31 16:00:00.000 | 2021-12-31 16:04:50.000 | 290000 | 0 | d2 | 2022-01-01 00:02:00.000 | 2022-01-01 00:02:20.000 | 20000 | 0 |
d35 | 2021-12-31 16:00:00.000 | 2021-12-31 16:04:50.000 | 290000 | 0 | d2 | 2022-01-01 00:02:30.000 | 2022-01-01 00:02:30.000 | 0 | 1 |
d83 | 2021-12-31 16:00:00.000 | 2021-12-31 16:04:50.000 | 290000 | 0 | d2 | 2022-01-01 00:02:40.000 | 2022-01-01 00:03:00.000 | 20000 | 0 |
d51 | 2021-12-31 16:00:00.000 | 2021-12-31 16:04:50.000 | 290000 | 0 | d2 | 2022-01-01 00:03:10.000 | 2022-01-01 00:03:10.000 | 0 | 1 |
d63 | 2021-12-31 16:00:00.000 | 2021-12-31 16:04:50.000 | 290000 | 0 | d2 | 2022-01-01 00:03:20.000 | 2022-01-01 00:03:40.000 | 20000 | 0 |
d0 | 2021-12-31 16:00:00.000 | 2021-12-31 16:04:50.000 | 290000 | 0 | d2 | 2022-01-01 00:03:50.000 | 2022-01-01 00:03:50.000 | 0 | 1 |
Query OK, 10 row(s) in set (0.040495s) d2 | 2022-01-01 00:04:00.000 | 2022-01-01 00:04:50.000 | 50000 | 0 |
d26 | 2022-01-01 00:00:00.000 | 2022-01-01 00:01:20.000 | 80000 | 0 |
d26 | 2022-01-01 00:01:30.000 | 2022-01-01 00:01:30.000 | 0 | 1 |
d26 | 2022-01-01 00:01:40.000 | 2022-01-01 00:01:40.000 | 0 | 0 |
d26 | 2022-01-01 00:01:50.000 | 2022-01-01 00:01:50.000 | 0 | 1 |
d26 | 2022-01-01 00:02:00.000 | 2022-01-01 00:02:20.000 | 20000 | 0 |
d26 | 2022-01-01 00:02:30.000 | 2022-01-01 00:02:30.000 | 0 | 1 |
d26 | 2022-01-01 00:02:40.000 | 2022-01-01 00:03:00.000 | 20000 | 0 |
d26 | 2022-01-01 00:03:10.000 | 2022-01-01 00:03:10.000 | 0 | 1 |
d26 | 2022-01-01 00:03:20.000 | 2022-01-01 00:03:40.000 | 20000 | 0 |
d26 | 2022-01-01 00:03:50.000 | 2022-01-01 00:03:50.000 | 0 | 1 |
d26 | 2022-01-01 00:04:00.000 | 2022-01-01 00:04:50.000 | 50000 | 0 |
Query OK, 22 row(s) in set (0.153403s)
``` ```
### 会话窗口 ### 会话窗口
@ -417,18 +430,18 @@ SLIMIT 10;
上面的 SQL，查询超级表 meters 中，时间戳大于等于 2022-01-01T00:00:00+08:00，且时间戳小于 2022-01-01T00:10:00+08:00的数据；数据先按照子表名 tbname 进行数据切分，再根据 10 分钟的会话窗口进行切分；最后，取前 10 个分片的数据作为结果，返回子表名、窗口开始时间、窗口结束时间、窗口宽度、窗口内数据条数。查询结果如下： 上面的 SQL，查询超级表 meters 中，时间戳大于等于 2022-01-01T00:00:00+08:00，且时间戳小于 2022-01-01T00:10:00+08:00的数据；数据先按照子表名 tbname 进行数据切分，再根据 10 分钟的会话窗口进行切分；最后，取前 10 个分片的数据作为结果，返回子表名、窗口开始时间、窗口结束时间、窗口宽度、窗口内数据条数。查询结果如下：
```text ```text
tbname | _wstart | _wend | _wduration | count(*) | tbname | _wstart | _wend | _wduration | count(*) |
===================================================================================================================================== ===============================================================================================
d76 | 2021-12-31 16:00:00.000 | 2021-12-31 16:09:50.000 | 590000 | 60 | d2 | 2022-01-01 00:00:00.000 | 2022-01-01 00:09:50.000 | 590000 | 60 |
d47 | 2021-12-31 16:00:00.000 | 2021-12-31 16:09:50.000 | 590000 | 60 | d26 | 2022-01-01 00:00:00.000 | 2022-01-01 00:09:50.000 | 590000 | 60 |
d37 | 2021-12-31 16:00:00.000 | 2021-12-31 16:09:50.000 | 590000 | 60 | d52 | 2022-01-01 00:00:00.000 | 2022-01-01 00:09:50.000 | 590000 | 60 |
d87 | 2021-12-31 16:00:00.000 | 2021-12-31 16:09:50.000 | 590000 | 60 | d64 | 2022-01-01 00:00:00.000 | 2022-01-01 00:09:50.000 | 590000 | 60 |
d64 | 2021-12-31 16:00:00.000 | 2021-12-31 16:09:50.000 | 590000 | 60 | d76 | 2022-01-01 00:00:00.000 | 2022-01-01 00:09:50.000 | 590000 | 60 |
d35 | 2021-12-31 16:00:00.000 | 2021-12-31 16:09:50.000 | 590000 | 60 | d28 | 2022-01-01 00:00:00.000 | 2022-01-01 00:09:50.000 | 590000 | 60 |
d83 | 2021-12-31 16:00:00.000 | 2021-12-31 16:09:50.000 | 590000 | 60 | d4 | 2022-01-01 00:00:00.000 | 2022-01-01 00:09:50.000 | 590000 | 60 |
d51 | 2021-12-31 16:00:00.000 | 2021-12-31 16:09:50.000 | 590000 | 60 | d88 | 2022-01-01 00:00:00.000 | 2022-01-01 00:09:50.000 | 590000 | 60 |
d63 | 2021-12-31 16:00:00.000 | 2021-12-31 16:09:50.000 | 590000 | 60 | d77 | 2022-01-01 00:00:00.000 | 2022-01-01 00:09:50.000 | 590000 | 60 |
d0 | 2021-12-31 16:00:00.000 | 2021-12-31 16:09:50.000 | 590000 | 60 | d54 | 2022-01-01 00:00:00.000 | 2022-01-01 00:09:50.000 | 590000 | 60 |
Query OK, 10 row(s) in set (0.043489s) Query OK, 10 row(s) in set (0.043489s)
``` ```
@ -458,26 +471,26 @@ FROM meters
WHERE ts >= "2022-01-01T00:00:00+08:00" WHERE ts >= "2022-01-01T00:00:00+08:00"
AND ts < "2022-01-01T00:10:00+08:00" AND ts < "2022-01-01T00:10:00+08:00"
PARTITION BY tbname PARTITION BY tbname
EVENT_WINDOW START WITH voltage >= 10 END WITH voltage < 20 EVENT_WINDOW START WITH voltage >= 225 END WITH voltage < 235
LIMIT 10; LIMIT 5;
``` ```
上面的 SQL，查询超级表meters中，时间戳大于等于2022-01-01T00:00:00+08:00，且时间戳小于2022-01-01T00:10:00+08:00的数据；数据先按照子表名tbname进行数据切分，再根据事件窗口条件：电压大于等于 10V，且小于 20V 进行切分；最后，取前 10 行的数据作为结果，返回子表名、窗口开始时间、窗口结束时间、窗口宽度、窗口内数据条数。查询结果如下： 上面的 SQL，查询超级表meters中，时间戳大于等于2022-01-01T00:00:00+08:00，且时间戳小于2022-01-01T00:10:00+08:00的数据；数据先按照子表名tbname进行数据切分，再根据事件窗口条件：电压大于等于 225V，且小于 235V 进行切分；最后，取每个分片的前 5 行的数据作为结果，返回子表名、窗口开始时间、窗口结束时间、窗口宽度、窗口内数据条数。查询结果如下：
```text ```text
tbname | _wstart | _wend | _wduration | count(*) | tbname | _wstart | _wend | _wduration | count(*) |
===================================================================================================================================== ==============================================================================================
d0 | 2021-12-31 16:00:00.000 | 2021-12-31 16:00:00.000 | 0 | 1 | d0 | 2022-01-01 00:00:00.000 | 2022-01-01 00:01:30.000 | 90000 | 10 |
d0 | 2021-12-31 16:00:30.000 | 2021-12-31 16:00:30.000 | 0 | 1 | d0 | 2022-01-01 00:01:40.000 | 2022-01-01 00:02:30.000 | 50000 | 6 |
d0 | 2021-12-31 16:00:40.000 | 2021-12-31 16:00:40.000 | 0 | 1 | d0 | 2022-01-01 00:02:40.000 | 2022-01-01 00:03:10.000 | 30000 | 4 |
d0 | 2021-12-31 16:01:20.000 | 2021-12-31 16:01:20.000 | 0 | 1 | d0 | 2022-01-01 00:03:20.000 | 2022-01-01 00:07:10.000 | 230000 | 24 |
d0 | 2021-12-31 16:02:20.000 | 2021-12-31 16:02:20.000 | 0 | 1 | d0 | 2022-01-01 00:07:20.000 | 2022-01-01 00:07:50.000 | 30000 | 4 |
d0 | 2021-12-31 16:02:30.000 | 2021-12-31 16:02:30.000 | 0 | 1 | d1 | 2022-01-01 00:00:00.000 | 2022-01-01 00:01:30.000 | 90000 | 10 |
d0 | 2021-12-31 16:03:10.000 | 2021-12-31 16:03:10.000 | 0 | 1 | d1 | 2022-01-01 00:01:40.000 | 2022-01-01 00:02:30.000 | 50000 | 6 |
d0 | 2021-12-31 16:03:30.000 | 2021-12-31 16:03:30.000 | 0 | 1 | d1 | 2022-01-01 00:02:40.000 | 2022-01-01 00:03:10.000 | 30000 | 4 |
d0 | 2021-12-31 16:03:40.000 | 2021-12-31 16:03:40.000 | 0 | 1 | d1 | 2022-01-01 00:03:20.000 | 2022-01-01 00:07:10.000 | 230000 | 24 |
d0 | 2021-12-31 16:03:50.000 | 2021-12-31 16:03:50.000 | 0 | 1 | ……
Query OK, 10 row(s) in set (0.034127s) Query OK, 500 row(s) in set (0.328557s)
``` ```
### 计数窗口 ### 计数窗口
@ -492,17 +505,25 @@ sliding_val 是一个常量，表示窗口滑动的数量，类似于 interval
select _wstart, _wend, count(*) select _wstart, _wend, count(*)
from meters from meters
where ts >= "2022-01-01T00:00:00+08:00" and ts < "2022-01-01T00:30:00+08:00" where ts >= "2022-01-01T00:00:00+08:00" and ts < "2022-01-01T00:30:00+08:00"
count_window(10); count_window(1000);
``` ```
上面的 SQL 查询超级表 meters 中时间戳大于等于 2022-01-01T00:00:00+08:00 且时间戳小于 2022-01-01T00:10:00+08:00 的数据。以每 10 条数据为一组，返回每组的开始时间、结束时间和分组条数。查询结果如下。 上面的 SQL 查询超级表 meters 中时间戳大于等于 2022-01-01T00:00:00+08:00 且时间戳小于 2022-01-01T00:10:00+08:00 的数据。以每 1000 条数据为一组，返回每组的开始时间、结束时间和分组条数。查询结果如下：
```text ```text
_wstart | _wend |count(*)| _wstart | _wend | count(*) |
=========================================================== =====================================================================
2021-12-31 16:00:00.000 | 2021-12-31 16:10:00.000 | 10 | 2022-01-01 00:00:00.000 | 2022-01-01 00:01:30.000 | 1000 |
2021-12-31 16:10:00.000 | 2021-12-31 16:20:00.000 | 10 | 2022-01-01 00:01:40.000 | 2022-01-01 00:03:10.000 | 1000 |
2021-12-31 16:20:00.000 | 2021-12-31 16:30:00.000 | 10 | 2022-01-01 00:03:20.000 | 2022-01-01 00:04:50.000 | 1000 |
2022-01-01 00:05:00.000 | 2022-01-01 00:06:30.000 | 1000 |
2022-01-01 00:06:40.000 | 2022-01-01 00:08:10.000 | 1000 |
2022-01-01 00:08:20.000 | 2022-01-01 00:09:50.000 | 1000 |
2022-01-01 00:10:00.000 | 2022-01-01 00:11:30.000 | 1000 |
2022-01-01 00:11:40.000 | 2022-01-01 00:13:10.000 | 1000 |
2022-01-01 00:13:20.000 | 2022-01-01 00:14:50.000 | 1000 |
2022-01-01 00:15:00.000 | 2022-01-01 00:16:30.000 | 1000 |
Query OK, 10 row(s) in set (0.062794s)
``` ```
## 时序数据特有函数 ## 时序数据特有函数
@ -563,14 +584,14 @@ UNION ALL
上面的 SQL，分别查询：子表 d1 的 1 条数据，子表 d11 的 2 条数据，子表 d21 的 3 条数据，并将结果合并。返回的结果如下： 上面的 SQL，分别查询：子表 d1 的 1 条数据，子表 d11 的 2 条数据，子表 d21 的 3 条数据，并将结果合并。返回的结果如下：
```text ```text
tbname | ts | current | voltage | phase | tbname | ts | current | voltage | phase |
================================================================================================= ====================================================================================
d11 | 2020-09-13 12:26:40.000 | 1.0260611 | 6 | 0.3620200 | d11 | 2020-09-13 20:26:40.000 | 11.5680809 | 247 | 146.5000000 |
d11 | 2020-09-13 12:26:50.000 | 2.9544230 | 8 | 1.0048079 | d11 | 2020-09-13 20:26:50.000 | 14.2392311 | 234 | 148.0000000 |
d21 | 2020-09-13 12:26:40.000 | 1.0260611 | 2 | 0.3520200 | d1 | 2020-09-13 20:26:40.000 | 11.5680809 | 247 | 146.5000000 |
d21 | 2020-09-13 12:26:50.000 | 2.9544230 | 2 | 0.9948080 | d21 | 2020-09-13 20:26:40.000 | 11.5680809 | 247 | 146.5000000 |
d21 | 2020-09-13 12:27:00.000 | -0.0000430 | 12 | 0.0099860 | d21 | 2020-09-13 20:26:50.000 | 14.2392311 | 234 | 148.0000000 |
d1 | 2020-09-13 12:26:40.000 | 1.0260611 | 10 | 0.3520200 | d21 | 2020-09-13 20:27:00.000 | 10.0999422 | 251 | 146.0000000 |
Query OK, 6 row(s) in set (0.006438s) Query OK, 6 row(s) in set (0.006438s)
``` ```

4
docs/zh/06-advanced/02-cache.md
View File

@ -54,10 +54,10 @@ TDengine 利用这些日志文件实现故障前的状态恢复。在写入 WAL
数据库参数 wal_level 和 wal_fsync_period 共同决定了 WAL 的保存行为。。 数据库参数 wal_level 和 wal_fsync_period 共同决定了 WAL 的保存行为。。
- wal_level：此参数控制 WAL 的保存级别。级别 1 表示仅将数据写入 WAL，但不立即执行 fsync 函数；级别 2 则表示在写入 WAL 的同时执行 fsync 函数。默认情况下，wal_level 设为 1。虽然执行 fsync 函数可以提高数据的持久性，但相应地也会降低写入性能。 - wal_level：此参数控制 WAL 的保存级别。级别 1 表示仅将数据写入 WAL，但不立即执行 fsync 函数；级别 2 则表示在写入 WAL 的同时执行 fsync 函数。默认情况下，wal_level 设为 1。虽然执行 fsync 函数可以提高数据的持久性，但相应地也会降低写入性能。
- wal_fsync_period：当 wal_level 设置为 1 时，这个参数控制执行 fsync 的频率。设置为 0 表示每次写入后立即执行 fsync，这可以确保数据的安全性，但可能会牺牲一些性能。当设置为大于 0 的数值时，表示 fsync 周期，默认为 3000，范围是[1， 180000]，单位毫秒。 - wal_fsync_period：当 wal_level 设置为 2 时，这个参数控制执行 fsync 的频率。设置为 0 表示每次写入后立即执行 fsync，这可以确保数据的安全性，但可能会牺牲一些性能。当设置为大于 0 的数值时，表示 fsync 周期，默认为 3000，范围是[1， 180000]，单位毫秒。
```sql ```sql
CREATE DATABASE POWER WAL_LEVEL 1 WAL_FSYNC_PERIOD 3000; CREATE DATABASE POWER WAL_LEVEL 2 WAL_FSYNC_PERIOD 3000;
``` ```
在创建数据库时可以选择不同的参数类型，来选择性能优先或者可靠性优先。 在创建数据库时可以选择不同的参数类型，来选择性能优先或者可靠性优先。

48
docs/zh/06-advanced/06-data-analysis/01-arima.md
View File

@ -3,8 +3,52 @@ title: "ARIMA"
sidebar_label: "ARIMA" sidebar_label: "ARIMA"
--- ---
本节讲述如何 ARIMA 算法的使用方法。 本节讲述 ARIMA 算法模型的使用方法。
## 功能概述 ## 功能概述
…… ARIMA 即自回归移动平均模型(Autoregressive Integrated Moving Average, ARIMA)，也记作 ARIMA(p,d,q)，是统计模型中最常见的一种用来进行时间序列预测的模型。
ARIMA模型是一种自回归模型，只需要自变量即可预测后续的值。ARIMA模型要求时序数据是**平稳**，或经过差分处理后平稳，如果是不平稳的数据，**无法**获得正确的结果。
>平稳的时间序列：其性质不随观测时间的变化而变化。具有趋势或季节性的时间序列不是平稳时间序列——趋势和季节性使得时间序列在不同时段呈现不同性质。
以下参数可以动态输入控制预测过程中生成 合适的 ARIMA 的模型。
- p= 自回归模型阶数
- d= 差分阶数
- q= 移动平均模型阶数
### 参数
分析平台中使用自动化的 ARIMA 模型进行计算，因此每次计算的时候会根据输入的数据自动拟合最合适的模型，然后根据该模型进行预测输出结果。
|参数名称|说明|必填项|
|---|---|---|
|period|输入时间序列数据每个周期包含的数据点个数。如果不设置该参数或则该参数设置为 0， 将使用非季节性/周期性的 ARIMA 模型预测。|选填|
|start_p| 自回归模型阶数的 起始值，0 开始的整数，不推荐大于 10 |选填|
|max_p| 自回归模型阶数的 结束值，0 开始的整数，不推荐大于 10 |选填|
|start_q| 移动平均模型阶数的起始值， 0 开始的整数，不推荐大于 10 |选填|
|max_q| 移动平均模型阶数的结束值， 0 开始的整数，不推荐大于 10 |选填|
|d| 差分阶数|选填|
`start_p`、`max_p` `start_q` `max_q` 四个参数约束了模型在多大的范围内去搜寻合适的最优解。相同输入数据的条件下，参数范围越大，消耗的资源越多，系统响应的时间越长。
### 示例及结果
针对 i32 列进行数据预测，输入列 i32 每 10 个点是一个周期，start_p 起始是 1， 最大拟合是 5，start_q是1，最大值是5，预测结果中返回 95% 置信区间范围边界。
```
FORECAST(i32, "algo=arima,alpha=95,period=10, start_p=1, max_p=5, start_q=1, max_q=5")
```
```json5
{
"rows": fc_rows, // 预测结果的行数
"period": period, // 返回结果的周期性，同输入
"alpha": alpha, // 返回结果的置信区间，同输入
"algo": "arima", // 返回结果使用的算法
"mse":mse, // 拟合输入时序数据时候生成模型的最小均方误差(MSE)
"res": res // 列模式的结果
}
```
### 参考文献
- https://en.wikipedia.org/wiki/Autoregressive_moving-average_model
- https://baike.baidu.com/item/%E8%87%AA%E5%9B%9E%E5%BD%92%E6%BB%91%E5%8A%A8%E5%B9%B3%E5%9D%87%E6%A8%A1%E5%9E%8B/5023931?fromtitle=ARMA%E6%A8%A1%E5%9E%8B&fromid=8048415

43
docs/zh/06-advanced/06-data-analysis/02-holtwinters.md Normal file
View File

@ -0,0 +1,43 @@
---
title: "HoltWinters"
sidebar_label: "HoltWinters"
---
本节讲述 HoltWinters 算法模型的使用方法。
## 功能概述
HoltWinters模型又称为多次指数平滑模型（EMA）。对含有线性趋势和周期波动的非平稳序列适用，利用指数平滑法让模型参数不断适应非平稳序列的变化，并对未来趋势进行**短期**预测。
HoltWinters有两种不同的季节性组成部分，当季节变化在该时间序列中大致保持不变时，通常选择**加法模型**；而当季节变化与时间序列的水平成比例变化时，通常选择**乘法模型**。
该模型对于返回数据也不提供计算的置信区间范围结果。在 95% 置信区间的上下界结果与预测结果相同。
### 参数
分析平台中使用自动化的 ARIMA 模型进行计算，因此每次计算的时候会根据输入的数据自动拟合最合适的模型，然后根据该模型进行预测输出结果。
|参数名称|说明|必填项|
|---|---|---|
|period| 输入时间序列数据每个周期包含的数据点个数。如果不设置该参数或则该参数设置为 0， 将使用一次（简单）指数平滑方式进行数据拟合，并据此进行未来数据的预测|选填|
|trend| 趋势模型使用加法模型还是乘法模型|选填|
|seasonal| 季节性采用加法模型还是乘法模型|选填|
参数 `trend` 和 `seasonal`的均可以选择 `add` （加法模型）或 `mul`（乘法模型）。
### 示例及结果
针对 i32 列进行数据预测，输入列 i32 每 10 个点是一个周期，趋势采用乘法模型，季节采用乘法模型
```
FORECAST(i32, "algo=holtwinters,period=10,trend=mul,seasonal=mul")
```
```json5
{
"rows": rows, // 结果的行数
"period": period, // 返回结果的周期性， 该结果与输入的周期性相同，如果没有周期性，该值为 0
"algo": 'holtwinters' // 返回结果使用的计算模型
"mse":mse, // 最小均方误差（minmum square error）
"res": res // 具体的结果，按照列形式返回的结果。一般意义上包含了 两列[timestamp][fc_results]。
}
```
### 参考文献
- https://en.wikipedia.org/wiki/Exponential_smoothing
- https://orangematter.solarwinds.com/2019/12/15/holt-winters-forecasting-simplified/

46
docs/zh/06-advanced/06-data-analysis/03-anomaly-detection.md Normal file
View File

@ -0,0 +1,46 @@
---
title: "Anomaly-detection"
sidebar_label: "Anomaly-detection"
---
本节讲述 异常检测 算法模型的使用方法。
## 概述
分析平台提供了 6 种异常检查模型，6 种异常检查模型分为 3 个类别，分别属于基于统计的异常检测模型、基于数据密度的检测模型、基于深度学习的异常检测模型。在不指定异常检测使用的方法的情况下，默认调用 iqr 的方法进行计算。
### 统计学异常检测方法
- k-sigma<sup>[1]</sup>: 即 ***68–95–99.7 rule*** 。***k***值默认为3， 即序列均值的 3 倍标准差范围为边界，超过边界的是异常值。KSigma 要求数据整体上服从正态分布，如果一个点偏离均值K倍标准差，则该点被视为异常点.
|参数名称|说明|是否必选|默认值|
|---|---|---|---|
|k|标准差倍数|选填|3|
- IQR<sup>[2]</sup>：四分位距 (Interquartile range, IQR) 是一种衡量变异性的方法. 四分位数将一个按等级排序的数据集划分为四个相等的部分。即 Q1（第 1 个四分位数）、Q2（第 2 个四分位数）和 Q3（第 3 个四分位数）。IQR 定义为 Q3–Q1，位于 Q3+1.5 。无输入参数。
- Grubbs<sup>[3]</sup>: 又称为 Grubbs' test，即最大标准残差测试。Grubbs 通常用作检验最大值、最小值偏离均值的程度是否为异常，该单变量数据集遵循近似标准正态分布。非正态分布数据集不能使用该方法。无输入参数。
- SHESD<sup>[4]</sup>： 带有季节性的 ESD 检测算法。ESD 可以检测时间序列数据的多异常点。需要指定异常点比例的上界***k***，最差的情况是至多49.9%。数据集的异常比例一般不超过5%
|参数名称|说明|是否必选|默认值|
|---|---|---|---|
|k|异常点在输入数据集中占比，范围是$`1\le K \le 49.9`$ |选填|5|
### 基于数据密度的检测方法
LOF<sup>[5]</sup>: 局部离群因子(LOF，又叫局部异常因子)算法是Breunig于2000年提出的一种基于密度的局部离群点检测算法，该方法适用于不同类簇密度分散情况迥异的数据。根据数据点周围的数据密集情况，首先计算每个数据点的一个局部可达密度，然后通过局部可达密度进一步计算得到每个数据点的一个离群因子，该离群因子即标识了一个数据点的离群程度，因子值越大，表示离群程度越高，因子值越小，表示离群程度越低。最后，输出离群程度最大的top(n)个点。
### 基于深度学习的检测方法
使用自动编码器的异常检测模型。可以对具有周期性的数据具有较好的检测结果。但是使用该模型需要针对输入的时序数据进行训练，同时将训练完成的模型部署到服务目录中，才能够运行与使用。
### 参考文献
1. https://en.wikipedia.org/wiki/68%E2%80%9395%E2%80%9399.7_rule
2. https://en.wikipedia.org/wiki/Interquartile_range
3. Adikaram, K. K. L. B.; Hussein, M. A.; Effenberger, M.; Becker, T. (2015-01-14). "Data Transformation Technique to Improve the Outlier Detection Power of Grubbs's Test for Data Expected to Follow Linear Relation". Journal of Applied Mathematics. 2015: 1–9. doi:10.1155/2015/708948.
4. Hochenbaum, O. S. Vallis, and A. Kejariwal. 2017. Automatic Anomaly Detection in the Cloud Via Statistical Learning. arXiv preprint arXiv:1704.07706 (2017).
5. Breunig, M. M.; Kriegel, H.-P.; Ng, R. T.; Sander, J. (2000). LOF: Identifying Density-based Local Outliers (PDF). Proceedings of the 2000 ACM SIGMOD International Conference on Management of Data. SIGMOD. pp. 93–104. doi:10.1145/335191.335388. ISBN 1-58113-217-4.

167
docs/zh/06-advanced/06-data-analysis/addins.md Normal file
View File

@ -0,0 +1,167 @@
---
title: "addins"
sidebar_label: "addins"
---
本节说明如何将自己开发的新预测算法和异常检测算法整合到 TDengine 分析平台， 并能够通过 SQL 语句进行调用。
## 目录结构
![数据分析功能架构图](./pic/dir.png)
|目录|说明|
|---|---|
|taos|Python 源代码目录，其下包含了算法具体保存目录 algo，放置杂项目录 misc， 单元测试和集成测试目录 test。 algo目录下 ad 放置异常检测算法代码， fc 放置预测算法代码|
|script|是安装脚本和发布脚本放置目录|
|model|放置针对数据集完成的训练模型|
|cfg| 配置文件目录|
## 约定与限制
定义异常检测算法的 Python 代码文件 需放在 /taos/algo/ad 目录中，预测算法 Python 代码文件需要放在 /taos/algo/fc 目录中，以确保系统启动的时候能够正常加载对应目录下的 Python 文件。
### 类命名规范
算法类的名称需要 以下划线开始，以 Service 结尾。例如：_KsigmaService 是 KSigma 异常检测算法的实现类。
### 类继承约定
异常检测算法需要从 `AbstractAnomalyDetectionService` 继承，并实现其核心抽象方法 `execute`.
预测算法需要从 `AbstractForecastService` 继承，同样需要实现其核心抽象方法 `execute`。
### 类属性初始化
每个算法实现的类需要静态初始化两个类属性，分别是
`name`: 的触发调用关键词，全小写英文字母。
`desc`：该算法的描述信息。
### 核心方法输入与输出约定
`execute` 是算法处理的核心方法。调用该方法的时候， `self.list` 已经设置好输入数组。
异常检测输出结果
`execute` 的返回值是长度与 `self.list` 相同的数组，数组位置为 -1 的即为异常值点。例如：输入数组是 [2, 2, 2, 2, 100]， 如果 100 是异常点，那么返回值是 [1, 1, 1, 1, -1]。
预测输出结果
对于预测算法， `AbstractForecastService` 的对象属性说明如下：
|属性名称|说明|默认值|
|---|---|---|
|period|输入时序数据的周期性，多少个数据点表示一个完整的周期。如果没有周期性，那么设置为 0 即可。| 0|
|start_ts|预测数据的开始时间| 0|
|time_step|预测结果的两个数据点之间时间间隔|0 |
|fc_rows|预测结果数量| 0 |
|return_conf|返回结果中是否包含执行区间范围，如果算法计算结果不包含置信区间，那么上界和下界与自身相同| 1|
|conf|执行区间分位数 0.05|
预测返回结果如下：
```python
return {
"rows": self.fc_rows, # 预测数据行数
"period": self.period, # 数据周期性，同输入
"algo": "holtwinters", # 预测使用的算法
"mse": mse, # 预测算法的 mse
"res": res # 结果数组 [时间戳数组, 预测结果数组, 预测结果执行区间下界数组，预测结果执行区间上界数组]
}
```
## 示例代码
```python
import numpy as np
from service import AbstractAnomalyDetectionService
# 算法实现类名称 需要以下划线 "_" 开始，并以 Service 结束，如下 _IqrService 是 IQR 异常检测算法的实现类。
class _IqrService(AbstractAnomalyDetectionService):
""" IQR algorithm 定义类，从 AbstractAnomalyDetectionService 继承，并实现 AbstractAnomalyDetectionService类的抽象函数 """
# 定义算法调用关键词，全小写ASCII码(必须添加)
name = 'iqr'
# 该算法的描述信息(建议添加)
desc = """found the anomaly data according to the inter-quartile range"""
def __init__(self):
super().__init__()
def execute(self):
""" execute 是算法实现逻辑的核心实现，直接修改该实现即可 """
# self.list 是输入数值列，list 类型，例如：[1,2,3,4,5]。设置 self.list 的方法在父类中已经进行了定义。实现自己的算法，修改该文件即可，以下代码使用自己的实现替换即可。
#lower = np.quantile(self.list, 0.25)
#upper = np.quantile(self.list, 0.75)
#min_val = lower - 1.5 * (upper - lower)
#max_val = upper + 1.5 * (upper - lower)
#threshold = [min_val, max_val]
# 返回值是与输入数值列长度相同的数据列，异常值对应位置是 -1。例如上述输入数据列，返回数值列是 [1, 1, 1, 1, -1],表示 [5] 是异常值。
return [-1 if k < threshold[0] or k > threshold[1] else 1 for k in self.list]
def set_params(self, params):
"""该算法无需任何输入参数，直接重载父类该函数，不处理算法参数设置逻辑"""
pass
```
## 单元测试
在测试文件目录中的 anomaly_test.py 中增加单元测试用例。
```python
def test_iqr(self):
""" 测试 _IqrService 类 """
s = loader.get_service("iqr")
# 设置需要进行检测的输入数据
s.set_input_list(AnomalyDetectionTest.input_list)
# 测试 set_params 的处理逻辑
try:
s.set_params({"k": 2})
except ValueError as e:
self.assertEqual(1, 0)
r = s.execute()
# 绘制异常检测结果
draw_ad_results(AnomalyDetectionTest.input_list, r, "iqr")
# 检查结果
self.assertEqual(r[-1], -1)
self.assertEqual(len(r), len(AnomalyDetectionTest.input_list))
```
## 需要模型的算法
针对特定数据集，进行模型训练的算法，在训练完成后。需要将训练得到的模型保存在 model 目录中。需要注意的是，针对每个算法，需要建立独立的文件夹。例如 auto_encoder 的训练算法在 model 目录下建立了， autoencoder的目录，使用该算法针对不同数据集训练得到的模型，均需要放置在该目录下。
训练完成后的模型，使用 joblib 进行保存。
并在 model 目录下建立对应的文件夹存放该模型。
保存模型的调用，可参考 encoder.py 的方式，用户通过调用 set_params 方法，并指定参数 `{"model": "ad_encoder_keras"}` 的方式，可以调用该模型进行计算。
具体的调用方式如下：
```python
def test_autoencoder_ad(self):
# 获取特定的算法服务
s = loader.get_service("ac")
data = self.__load_remote_data_for_ad()
# 设置异常检查的输入数据
s.set_input_list(data)
# 指定调用的模型，该模型是之前针对该数据集进行训练获得
s.set_params({"model": "ad_encoder_keras"})
# 执行检查动作，并返回结果
r = s.execute()
num_of_error = -(sum(filter(lambda x: x == -1, r)))
self.assertEqual(num_of_error, 109)
```

311
docs/zh/06-advanced/06-data-analysis/index.md
View File

@ -5,9 +5,314 @@ title: 数据分析功能
## 概述 ## 概述
TDengine 提供数据分析功能的扩展组件，通过引入 ANode，TDengine 能够支持时间序列的机器学习分析 TDengine 通过 ANode(AnalysisNode) 是提供数据分析功能的扩展组件，通过 Restful 接口提供分析服务，从而拓展 TDengine 的功能，支持时间序列高级分析功能。
ANode 是无状态的数据分析节点，集群中可以存在多个 ANode节点，相互之间没有关联。将 ANode 注册到 TDengine 集群以后，通过 SQL 语句即可调用并完成时序分析任务。
下图展示了数据分析的技术架构。 下图是数据分析的技术架构示意图。
![数据分析功能架构图](./pic/data-analysis.png) ![数据分析功能架构图](./pic/data-analysis.png)
## 安装部署
### 环境准备
ANode 的要求节点上准备有 Python 3.10 及以上版本以及相应的Python包自动安装组件 Pip ，同时请确保能够正常连接互联网。
### 安装及卸载
使用专门的 ANode 安装包 TDengine-enterprise-anode-1.x.x.tar.gz 进行 ANode 的安装部署工作，安装过程与 TDengien 的安装流程一致。
```bash
tar -xzvf TDengine-enterprise-anode-1.0.0.tar.gz
cd TDengine-enterprise-anode-1.0.0
sudo ./install.sh
```
卸载 ANode，执行命令 `rmtaosanode` 即可。
### 其他
为了避免 ANode 安装后影响目标节点现有的 Python 库。 ANode 使用 Python 虚拟环境运行，安装后的默认 Python 目录处于 `/var/lib/taos/taosanode/venv/`。为了避免反复安装虚拟环境带来的开销，卸载 ANode 并不会自动删除该虚拟环境，如果您确认不需要 Python 的虚拟环境，可以手动删除。
## 启动及停止服务
安装 ANode 以后，可以使用`systemctl`来管理 ANode 的服务。使用如下命令可以启动/停止/检查状态。
```bash
systemctl start taosanoded
systemctl stop taosanoded
systemctl status taosanoded
```
## 目录及配置说明
|目录/文件|说明|
|---------------|------|
|/usr/local/taos/taosanode/bin|可执行文件目录|
|/usr/local/taos/taosanode/resource|资源文件目录，链接到文件夹 /var/lib/taos/taosanode/resource/|
|/usr/local/taos/taosanode/lib|库文件目录|
|/var/lib/taos/taosanode/model/|模型文件目录，链接到文件夹 /var/lib/taos/taosanode/model|
|/var/log/taos/taosanode/|日志文件目录|
|/etc/taos/taosanode.ini|配置文件|
### 配置说明
Anode 提供的 RestFul 服务使用 uWSGI 驱动，因此 ANode 的配置和 uWSGI 的配置在同一个配置文件中，具体如下：
```ini
[uwsgi]
# charset
env=LC_ALL=en_US.UTF-8
# ip:port
http = 127.0.0.1:6050
# the local unix socket file than communicate to Nginx
#socket = 127.0.0.1:8001
#socket-timeout=10
# base directory
chdir = /usr/local/taos/taosanode/lib
# initialize python file
wsgi-file = /usr/local/taos/taosanode/lib/app.py
# invoke app model
callable = app
# auto remove unix Socket and pid file when stopping
vacuum = true
# socket exec model
#chmod-socket = 664
# uWSGI pid
uid=root
# uWSGI gid
gid=root
# main process
master = true
# the number of worker processes
processes = 2
# pid file
pidfile = /usr/local/taos/taosanode/uwsgi.pid
# enable threads
enable-threads=true
# the number of threads for each process
threads=2
# memory useage report
memory-report = true
reload-mercy = 10
# conflict with systemctl, so do NOT uncomment this
# daemonize = /var/log/taos/taosanode/taosanode.log
# set log
logto = /var/log/taos/taosanode/taosanode.log
# monitor server
stats = 127.0.0.1:8387
# python virtual environment directory
virtualenv = /usr/local/taos/taosanode/venv/
[taosanode]
# default app log file
app-log = /var/log/taos/taosanode/taosanode.app.log
# model storage directory
model-dir=/usr/local/taos/taosanode/model/
# default log level
log-level = DEBUG
```
**提示**
请勿设置 `daemonize` 参数，该参数会导致 uWSGI 与 systemctl 冲突，从而无法正常启动。
## ANode 基本操作
### 管理 ANode
#### 创建 ANode
```sql
CREATE ANODE {node_url}
```
node_url 是提供服务的 ANode 的 IP 和 PORT, 例如：`create anode 'http://localhost:6050'`。启动 ANode 以后如果不注册到 TDengine 集群中，无法提供正常的服务。不建议 ANode 注册到两个或多个集群中。
#### 查看 ANode
列出集群中所有的数据分析节点，包括其 `FQDN`, `PORT`, `STATUS`。
```sql
SHOW ANODES;
```
#### 查看提供的时序数据分析服务
```SQL
SHOW ANODES FULL;
```
#### 强制刷新 TDengine 集群中分析算法缓存
```SQL
UPDATE ANODE {node_id}
UPDATE ALL ANODES
```
#### 删除 ANode
```sql
DROP ANODE {anode_id}
```
删除 ANode 只是将 ANode 从 TDengine 集群中删除，管理 ANode 的启停仍然需要使用`systemctl`命令。
### 时序数据分析功能
#### 白噪声检查
平台提供 Restful的服务检测输入时间序列是否是白噪声时间序列（White Noise Data, WND），白噪声时间序列及随机数序列。
此外，分析平台要求输入的数据不能是 , 因此针对的所有数据均默认进行 白噪声检查。当前白噪声检查采用通行的 `Ljung-Box`检验，`Ljung-Box` 统计量检查过程需要遍历整个输入序列并进行计算。
如果用户能够明确输入序列一定不是白噪声序列，那么可以通过输入参数，指定预测之前忽略该检查，从而节省分析过程的 CPU 计算资源。
同时支持独立地针对输入序列进行白噪声检测（该检测功能暂不独立对外开放）。
#### 数据重采样和时间戳对齐
数据分析平台支持将输入的数据进行重采样的预处理，从而确保输出结果按照用户指定的等间隔进行处理。处理过程分为两种类别：
- 数据时间戳对齐。由于真实数据时间可能并非严格按照查询指定的时间戳输入。此时数据平台将自动将数据的时间间隔按照指定的时间间隔进行对齐。例如有输入时间序列：[11, 22, 29, 41]，用户指定时间间隔为 10，那么该序列将被对齐重整为以下序列 [10, 20, 30, 40]。
- 数据时间重采样。用户输入的时间序列其采样频率超过了指定的查询需要获得结果的时间间隔，例如输入原始数据是 5， 但是输出结果的频率是 10. [0， 5， 10， 15， 20， 25， 30]，那么该输入数据列将重采用为间隔 为 10 的输入序列，其结果如下 [0, 10, 20，30]。[5, 15, 25] 处的数据将被丢弃。
需要注意的是，数据输入平台不支持缺失数据补齐后进行的预测分析，如果输入时间序列数据[11, 22, 29, 49]，并且用户要求的时间间隔为 10， 重整对齐后的序列是 [10, 20, 30, 50] 那么该序列进行预测分析将返回错误。
#### 时序数据异常检测
异常检测是针对输入的时序数据，使用预设或用户指定的算法确定时间序列中**可能**出现异常时间序列点，对于时间序列中若干个连续的异常点，将自动合并成为一个连续的（闭区间）异常窗口。对于只有单个点的场景，异常窗口窗口退化成为一个起始时间和结束时间相同的点。
异常检测生成的异常窗口受检测算法和算法参数的共同影响，对于异常窗口范围内的数据，可以应用 TDengine 提供的聚合和标量函数进行查询或变换处理。
对于输入时间序列 (1, 20), (2, 22), (3, 91), (4, 120), (5, 18), (6, 19)。系统检测到 (3, 91), (4, 120) 为异常点，那么返回的异常窗口是闭区间 [3, 4]。
##### 语法
```SQL
ANOMALY_WINDOW(column_name, option_expr)
option_expr: {"
algo=expr1
[,wncheck=1|0]
[,expr2]
"}
```
1. `column`：进行时序数据异常检测的输入数据列，当前只支持单列输入，且只能是数值类型，不能是字符类型（例如：`NCHAR` `VARCHAR` `VARBINARY`等类型），**不支持函数表达式**。
2. `options`：字符串。其中使用 K/V 调用异常检测的算法，及与算法相关的参数。采用 逗号分隔的K/V字符串表示，其中的字符串不需要使用单引号、双引号、或转意号等符号，不能使用中文及其他宽字符。例如：`algo=ksigma, k=2` 表示进行异常检测的算法是 ksigma，该算法接受的输入参数是 2。
3. 异常检测的结果可以作为外层查询的子查询输入，在 `SELECT` 子句中使用的聚合函数或标量函数与其他类型的窗口查询相同。
4. 输入数据默认进行白噪声检查，如果检查结果是输入数据是白噪声，将不会有任何（异常）窗口信息返回。
**参数说明**
|参数|含义|默认值|
|---|---|---|
|algo|异常检测调用的算法|iqr|
|wncheck|对输入数据列是否进行白噪声检查|取值为0或者1，默认值为 1，表示进行白噪声检查|
异常检测的返回结果以窗口的形式呈现，因此窗口查询相关的伪列在这种场景下仍然可用。可以使用的伪列如下：
1. `_WSTART`： 异常窗口开始时间戳
2. `_WEND`：异常窗口结束时间戳
3. `_WDURATION`：异常窗口持续时间
**示例**
```SQL
--- 使用 iqr 算法进行异常检测，检测列 i32 列。
SELECT _wstart, _wend, SUM(i32)
FROM ai.atb
ANOMALY_WINDOW(i32, "algo=iqr");
--- 使用 ksigma 算法进行异常检测，输入参数 k 值为 2，检测列 i32 列
SELECT _wstart, _wend, SUM(i32)
FROM ai.atb
ANOMALY_WINDOW(i32, "algo=ksigma,k=2");
```
```
taos> SELECT _wstart, _wend, count(*) FROM ai.atb ANOMAYL_WINDOW(i32);
_wstart | _wend | count(*) |
============================================================================
2020-01-01 00:00:16.000 | 2020-01-01 00:00:16.001 | 1 |
Query OK, 1 row(s) in set (0.028946s)
```
**可用异常检测算法**
- iqr
- ksigma
- grubbs
- lof
- shesd
- tac
#### 时序数据预测
数据预测以一段训练数据作为输入，预测接下来一个连续时间区间内，时序数据的趋势。
##### 语法
```SQL
FORECAST(column_expr, option_expr)
option_expr: {"
algo=expr1
[,wncheck=1|0]
[,conf=conf_val]
[,every=every_val]
[,rows=rows_val]
[,start=start_ts_val]
[,expr2]
"}
```
1. `column_expr`：预测的时序数据列。与异常检测相同，只支持数值类型输入。
2. `options`：异常检测函数的参数，使用规则与 anomaly_window 相同。预测还支持`conf`, `every`, `rows`, `start`, `rows` 几个参数，其含义如下：
**参数说明**
|参数|含义|默认值|
|---|---|---|
|algo|预测分析使用的算法|holtwinters|
|wncheck|白噪声（white noise data）检查|默认值为 1，0 表示不进行检查|
|conf|预测数据的置信区间范围 ，取值范围[0, 100]|95|
|every|预测数据的采样间隔|输入数据的采样间隔|
|start|预测结果的开始时间戳|输入数据最后一个时间戳加上一个采样时间段|
|rows|预测结果的记录数|10|
1. 预测查询结果新增了三个伪列，具体如下： `_FROWTS`：预测结果的时间戳、`_FLOW`：置信区间下界、`_FHIGH`：置信区间上界, 对于没有置信区间的预测算法，其置信区间同预测结果
2. 更改参数 `START`：返回预测结果的起始时间，改变这个起始时间不会影响返回的预测数值，只影响起始时间。
3. `EVERY`：可以与输入数据的采样频率不同。采样频率只能低于或等于输入数据采样频率，不能**高于**输入数据的采样频率。
4. 对于某些不需要计算置信区间的算法，即使指定了置信区间，返回的结果中其上下界退化成为一个点。
**示例**
```SQL
--- 使用 arima 算法进行预测，预测结果是 10 条记录（默认值），数据进行白噪声检查，默认置信区间 95%.
SELECT _flow, _fhigh, _frowts, FORECAST(i32, "algo=arima")
FROM ai.ftb;
--- 使用 arima 算法进行预测，输入数据的是周期数据，每10个采样点是一个周期。返回置信区间是 95%.
SELECT _flow, _fhigh, _frowts, FORECAST(i32, "algo=arima,alpha=95,period=10")
FROM ai.ftb;
```
```
taos> select _flow, _fhigh, _frowts, forecast(i32) from ai.ftb;
_flow | _fhigh | _frowts | forecast(i32) |
========================================================================================
10.5286684 | 41.8038254 | 2020-01-01 00:01:35.001 | 26 |
-21.9861946 | 83.3938904 | 2020-01-01 00:01:36.001 | 30 |
-78.5686035 | 144.6729126 | 2020-01-01 00:01:37.001 | 33 |
-154.9797363 | 230.3057709 | 2020-01-01 00:01:38.001 | 37 |
-253.9852905 | 337.6083984 | 2020-01-01 00:01:39.001 | 41 |
-375.7857971 | 466.4594727 | 2020-01-01 00:01:40.001 | 45 |
-514.8043823 | 622.4426270 | 2020-01-01 00:01:41.001 | 53 |
-680.6343994 | 796.2861328 | 2020-01-01 00:01:42.001 | 57 |
-868.4956665 | 992.8603516 | 2020-01-01 00:01:43.001 | 62 |
-1076.1566162 | 1214.4498291 | 2020-01-01 00:01:44.001 | 69 |
```
**可用预测算法**
- arima
- holtwinters

BIN
docs/zh/06-advanced/06-data-analysis/pic/data-analysis.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 62 KiB

After

Width:  |  Height:  |  Size: 49 KiB

BIN
docs/zh/06-advanced/06-data-analysis/pic/dir.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 7.1 KiB

5
docs/zh/14-reference/01-components/01-taosd.md
View File

@ -156,8 +156,8 @@ charset 的有效值是 UTF-8。
### 内存相关 ### 内存相关
| 参数名称 | 参数说明 | | 参数名称 | 参数说明 |
| :----------------: | :---------------------------------------------: | | :----------------: | :---------------------------------------------: |
| rpcQueueMemoryAllowed | 一个 dnode 允许的 rpc 消息占用的内存最大值，单位 bytes，取值范围：10485760-INT64_MAX，缺省值：服务器内存的 1/10 | | rpcQueueMemoryAllowed | 一个 dnode 允许的 rpc 消息占用的内存最大值，单位 bytes，取值范围：104857600-INT64_MAX，缺省值：服务器内存的 1/10 |
| syncLogBufferMemoryAllowed | 一个 dnode 允许的 sync 日志缓存消息占用的内存最大值，单位 bytes，取值范围：10485760-INT64_MAX，缺省值：服务器内存的 1/10，3.1.3.2/3.3.2.13 版本开始生效 | | syncLogBufferMemoryAllowed | 一个 dnode 允许的 sync 日志缓存消息占用的内存最大值，单位 bytes，取值范围：104857600-INT64_MAX，缺省值：服务器内存的 1/10，3.1.3.2/3.3.2.13 版本开始生效 |
### 性能调优 ### 性能调优
@ -180,6 +180,7 @@ charset 的有效值是 UTF-8。
| tmrDebugFlag | 定时器模块的日志开关，取值范围同上 | | tmrDebugFlag | 定时器模块的日志开关，取值范围同上 |
| uDebugFlag | 共用功能模块的日志开关，取值范围同上 | | uDebugFlag | 共用功能模块的日志开关，取值范围同上 |
| rpcDebugFlag | rpc 模块的日志开关，取值范围同上 | | rpcDebugFlag | rpc 模块的日志开关，取值范围同上 |
| cDebugFlag | 客户端模块的日志开关，取值范围同上 |
| jniDebugFlag | jni 模块的日志开关，取值范围同上 | | jniDebugFlag | jni 模块的日志开关，取值范围同上 |
| qDebugFlag | query 模块的日志开关，取值范围同上 | | qDebugFlag | query 模块的日志开关，取值范围同上 |
| dDebugFlag | dnode 模块的日志开关，取值范围同上，缺省值 135 | | dDebugFlag | dnode 模块的日志开关，取值范围同上，缺省值 135 |

1
docs/zh/14-reference/01-components/02-taosc.md
View File

@ -35,6 +35,7 @@ TDengine 客户端驱动提供了应用编程所需要的全部 API，并且在
|smlAutoChildTableNameDelimiter | schemaless tag之间的连接符，连起来作为子表名，无缺省值 | |smlAutoChildTableNameDelimiter | schemaless tag之间的连接符，连起来作为子表名，无缺省值 |
|smlTagName | schemaless tag 为空时默认的 tag 名字, 缺省值 "_tag_null" | |smlTagName | schemaless tag 为空时默认的 tag 名字, 缺省值 "_tag_null" |
|smlTsDefaultName | schemaless自动建表的时间列名字通过该配置设置, 缺省值 "_ts" | |smlTsDefaultName | schemaless自动建表的时间列名字通过该配置设置, 缺省值 "_ts" |
|smlDot2Underline | schemaless 把超级表名中的 dot 转成下划线 |
|enableCoreFile | crash 时是否生成 core 文件，0: 不生成， 1： 生成；缺省值：1 | |enableCoreFile | crash 时是否生成 core 文件，0: 不生成， 1： 生成；缺省值：1 |
|enableScience | 是否开启科学计数法显示浮点数; 0: 不开始, 1: 开启；缺省值：1 | |enableScience | 是否开启科学计数法显示浮点数; 0: 不开始, 1: 开启；缺省值：1 |
|compressMsgSize | 是否对 RPC 消息进行压缩; -1: 所有消息都不压缩; 0: 所有消息都压缩; N (N>0): 只有大于 N 个字节的消息才压缩; 缺省值 -1| |compressMsgSize | 是否对 RPC 消息进行压缩; -1: 所有消息都不压缩; 0: 所有消息都压缩; N (N>0): 只有大于 N 个字节的消息才压缩; 缺省值 -1|

2
docs/zh/14-reference/02-tools/10-taosbenchmark.md
View File

@ -364,7 +364,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\)
- **max** : 数据类型的 列/标签 的最大值。生成的值将小于最小值。 - **max** : 数据类型的 列/标签 的最大值。生成的值将小于最小值。
- **scalingFactor** : 浮点数精度增强因子，仅当数据类型是float/double时生效，有效值范围为1至1000000的正整数。用于增强生成浮点数的精度，特别是在min或max值较小的情况下。此属性按10的幂次增强小数点后的精度：scalingFactor为10表示增强1位小数精度，100表示增强2位，依此类推。 - **scalingFactor** : 浮点数精度增强因子，仅当数据类型是 float/double 时生效，有效值范围为 1 至 1000000 的正整数。用于增强生成浮点数的精度，特别是在 min 或 max 值较小的情况下。此属性按 10 的幂次增强小数点后的精度：scalingFactor 为 10 表示增强 1 位小数精度，100 表示增强 2 位，依此类推。
- **fun** : 此列数据以函数填充，目前只支持 sin 和 cos 两函数，输入参数为时间戳换算成角度值，换算公式： 角度 x = 输入的时间列ts值 % 360。同时支持系数调节，随机波动因子调节，以固定格式的表达式展现，如 fun=“10\*sin(x)+100\*random(5)” , x 表示角度，取值 0 ~ 360度，增长步长与时间列步长一致。10 表示乘的系数，100 表示加或减的系数，5 表示波动幅度在 5% 的随机范围内。目前支持的数据类型为 int, bigint, float, double 四种数据类型。注意：表达式为固定模式，不可前后颠倒。 - **fun** : 此列数据以函数填充，目前只支持 sin 和 cos 两函数，输入参数为时间戳换算成角度值，换算公式： 角度 x = 输入的时间列ts值 % 360。同时支持系数调节，随机波动因子调节，以固定格式的表达式展现，如 fun=“10\*sin(x)+100\*random(5)” , x 表示角度，取值 0 ~ 360度，增长步长与时间列步长一致。10 表示乘的系数，100 表示加或减的系数，5 表示波动幅度在 5% 的随机范围内。目前支持的数据类型为 int, bigint, float, double 四种数据类型。注意：表达式为固定模式，不可前后颠倒。

8
docs/zh/14-reference/03-taos-sql/02-database.md
View File

@ -30,6 +30,7 @@ database_option: {
| SINGLE_STABLE {0 | 1} | SINGLE_STABLE {0 | 1}
| TABLE_PREFIX value | TABLE_PREFIX value
| TABLE_SUFFIX value | TABLE_SUFFIX value
| DNODES value
| TSDB_PAGESIZE value | TSDB_PAGESIZE value
| WAL_LEVEL {1 | 2} | WAL_LEVEL {1 | 2}
| WAL_FSYNC_PERIOD value | WAL_FSYNC_PERIOD value
@ -63,19 +64,20 @@ database_option: {
- MAXROWS：文件块中记录的最大条数，默认为 4096 条。 - MAXROWS：文件块中记录的最大条数，默认为 4096 条。
- MINROWS：文件块中记录的最小条数，默认为 100 条。 - MINROWS：文件块中记录的最小条数，默认为 100 条。
- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于3倍的 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 \<= keep 1 \<= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。 - KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于3倍的 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 \<= keep 1 \<= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。
- STT_TRIGGER：表示落盘文件触发文件合并的个数。默认为 1，范围 1 到 16。对于少表高频场景，此参数建议使用默认配置，或较小的值；而对于多表低频场景，此参数建议配置较大的值。 - STT_TRIGGER：表示落盘文件触发文件合并的个数。开源版本固定为 1，企业版本可设置范围为 1 到 16。对于少表高频写入场景，此参数建议使用默认配置；而对于多表低频写入场景，此参数建议配置较大的值。
- SINGLE_STABLE：表示此数据库中是否只可以创建一个超级表，用于超级表列非常多的情况。 - SINGLE_STABLE：表示此数据库中是否只可以创建一个超级表，用于超级表列非常多的情况。
- 0：表示可以创建多张超级表。 - 0：表示可以创建多张超级表。
- 1：表示只可以创建一张超级表。 - 1：表示只可以创建一张超级表。
- TABLE_PREFIX：当其为正值时，在决定把一个表分配到哪个 vgroup 时要忽略表名中指定长度的前缀；当其为负值时，在决定把一个表分配到哪个 vgroup 时只使用表名中指定长度的前缀；例如，假定表名为 "v30001"，当 TSDB_PREFIX = 2 时 使用 "0001" 来决定分配到哪个 vgroup ，当 TSDB_PREFIX = -2 时使用 "v3" 来决定分配到哪个 vgroup - TABLE_PREFIX：当其为正值时，在决定把一个表分配到哪个 vgroup 时要忽略表名中指定长度的前缀；当其为负值时，在决定把一个表分配到哪个 vgroup 时只使用表名中指定长度的前缀；例如，假定表名为 "v30001"，当 TSDB_PREFIX = 2 时 使用 "0001" 来决定分配到哪个 vgroup ，当 TSDB_PREFIX = -2 时使用 "v3" 来决定分配到哪个 vgroup
- TABLE_SUFFIX：当其为正值时，在决定把一个表分配到哪个 vgroup 时要忽略表名中指定长度的后缀；当其为负值时，在决定把一个表分配到哪个 vgroup 时只使用表名中指定长度的后缀；例如，假定表名为 "v30001"，当 TSDB_SUFFIX = 2 时 使用 "v300" 来决定分配到哪个 vgroup ，当 TSDB_SUFFIX = -2 时使用 "01" 来决定分配到哪个 vgroup。 - TABLE_SUFFIX：当其为正值时，在决定把一个表分配到哪个 vgroup 时要忽略表名中指定长度的后缀；当其为负值时，在决定把一个表分配到哪个 vgroup 时只使用表名中指定长度的后缀；例如，假定表名为 "v30001"，当 TSDB_SUFFIX = 2 时 使用 "v300" 来决定分配到哪个 vgroup ，当 TSDB_SUFFIX = -2 时使用 "01" 来决定分配到哪个 vgroup。
- TSDB_PAGESIZE：一个 VNODE 中时序数据存储引擎的页大小，单位为 KB，默认为 4 KB。范围为 1 到 16384，即 1 KB到 16 MB。 - TSDB_PAGESIZE：一个 VNODE 中时序数据存储引擎的页大小，单位为 KB，默认为 4 KB。范围为 1 到 16384，即 1 KB到 16 MB。
- DNODES：指定 VNODE 所在的 DNODE 列表，如 '1,2,3'，以逗号区分且字符间不能有空格，仅企业版支持。
- WAL_LEVEL：WAL 级别，默认为 1。 - WAL_LEVEL：WAL 级别，默认为 1。
- 1：写 WAL，但不执行 fsync。 - 1：写 WAL，但不执行 fsync。
- 2：写 WAL，而且执行 fsync。 - 2：写 WAL，而且执行 fsync。
- WAL_FSYNC_PERIOD：当 WAL_LEVEL 参数设置为 2 时，用于设置落盘的周期。默认为 3000，单位毫秒。最小为 0，表示每次写入立即落盘；最大为 180000，即三分钟。 - WAL_FSYNC_PERIOD：当 WAL_LEVEL 参数设置为 2 时，用于设置落盘的周期。默认为 3000，单位毫秒。最小为 0，表示每次写入立即落盘；最大为 180000，即三分钟。
- WAL_RETENTION_PERIOD: 为了数据订阅消费，需要WAL日志文件额外保留的最大时长策略。WAL日志清理，不受订阅客户端消费状态影响。单位为 s。默认为 3600，表示在 WAL 保留最近 3600 秒的数据，请根据数据订阅的需要修改这个参数为适当值。 - WAL_RETENTION_PERIOD: 为了数据订阅消费，需要 WAL 日志文件额外保留的最大时长策略。WAL 日志清理，不受订阅客户端消费状态影响。单位为 s。默认为 3600，表示在 WAL 保留最近 3600 秒的数据，请根据数据订阅的需要修改这个参数为适当值。
- WAL_RETENTION_SIZE：为了数据订阅消费，需要WAL日志文件额外保留的最大累计大小策略。单位为 KB。默认为 0，表示累计大小无上限。 - WAL_RETENTION_SIZE：为了数据订阅消费，需要 WAL 日志文件额外保留的最大累计大小策略。单位为 KB。默认为 0，表示累计大小无上限。
### 创建数据库示例 ### 创建数据库示例
```sql ```sql

12
docs/zh/14-reference/03-taos-sql/03-table.md
View File

@ -42,12 +42,12 @@ table_option: {
**使用说明** **使用说明**
1. 表（列）名命名规则参见[名称命名规则](./19-limit.md#名称命名规则)。 1. 表（列）名命名规则参见[名称命名规则](./19-limit.md#名称命名规则)。
1. 表名最大长度为 192。 2. 表名最大长度为 192。
1. 表的第一个字段必须是 TIMESTAMP，并且系统自动将其设为主键。 3. 表的第一个字段必须是 TIMESTAMP，并且系统自动将其设为主键。
1. 除时间戳主键列之外，还可以通过 PRIMARY KEY 关键字指定第二列为额外的主键列。被指定为主键列的第二列必须为整型或字符串类型（varchar）。 4. 除时间戳主键列之外，还可以通过 PRIMARY KEY 关键字指定第二列为额外的主键列。被指定为主键列的第二列必须为整型或字符串类型（VARCHAR）。
1. 表的每行长度不能超过 48KB（从 3.0.5.0 版本开始为 64KB）;（注意：每个 BINARY/NCHAR/GEOMETRY 类型的列还会额外占用 2 个字节的存储位置）。 5. 表的每行长度不能超过 48KB（从 3.0.5.0 版本开始为 64KB）;（注意：每个 VARCHAR/NCHAR/GEOMETRY 类型的列还会额外占用 2 个字节的存储位置）。
1. 使用数据类型 BINARY/NCHAR/GEOMETRY，需指定其最长的字节数，如 BINARY(20)，表示 20 字节。 6. 使用数据类型 VARCHAR/NCHAR/GEOMETRY，需指定其最长的字节数，如 VARCHAR(20)，表示 20 字节。
1. 关于 `ENCODE` 和 `COMPRESS` 的使用，请参考[按列压缩](../compress) 7. 关于 `ENCODE` 和 `COMPRESS` 的使用，请参考[按列压缩](../compress)
**参数说明** **参数说明**

2
docs/zh/14-reference/03-taos-sql/07-tag-index.md
View File

@ -11,7 +11,7 @@ description: 使用标签索引提升查询性能
创建索引的语法如下 创建索引的语法如下
```sql ```sql
CREATE INDEX index_name ON tbl_name (tagColName） CREATE INDEX index_name ON tbl_name (tagColName)
``` ```
其中 `index_name` 为索引名称， `tbl_name` 为超级表名称，`tagColName` 为要在其上建立索引的 tag 列的名称。`tagColName` 的类型不受限制，即任何类型的 tag 列都可以建立索引。 其中 `index_name` 为索引名称， `tbl_name` 为超级表名称，`tagColName` 为要在其上建立索引的 tag 列的名称。`tagColName` 的类型不受限制，即任何类型的 tag 列都可以建立索引。

1
docs/zh/14-reference/03-taos-sql/14-stream.md
View File

@ -291,3 +291,4 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name;
CREATE SNODE ON DNODE [id] CREATE SNODE ON DNODE [id]
``` ```
其中的 id 是集群中的 dnode 的序号。请注意选择的dnode，流计算的中间状态将自动在其上进行备份。 其中的 id 是集群中的 dnode 的序号。请注意选择的dnode，流计算的中间状态将自动在其上进行备份。
从 3.3.4.0 版本开始，在多副本环境中创建流会进行 snode 的**存在性检查**，要求首先创建 snode。如果 snode 不存在，无法创建流。

1
docs/zh/14-reference/05-connector/35-node.mdx
View File

@ -26,6 +26,7 @@ Node.js 连接器目前仅支持 WebSocket 连接器， 其通过 taosAdapter
| Node.js 连接器 版本 | 主要变化 | TDengine 版本 | | Node.js 连接器 版本 | 主要变化 | TDengine 版本 |
| :------------------: | :----------------------: | :----------------: | | :------------------: | :----------------------: | :----------------: |
| 3.1.1 | 优化了数据传输性能 | 3.3.2.0 及更高版本 |
| 3.1.0 | 新版本发布，支持 WebSocket 连接 | 3.2.0.0 及更高版本 | | 3.1.0 | 新版本发布，支持 WebSocket 连接 | 3.2.0.0 及更高版本 |
## 处理异常 ## 处理异常

4
docs/zh/14-reference/05-connector/index.md
View File

@ -34,8 +34,8 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器
| **3.0.0.0 及以上** | 3.0.2以上 | 当前版本 | 3.0 分支 | 3.0.0 | 3.1.0 | 当前版本 | 与 TDengine 相同版本 | | **3.0.0.0 及以上** | 3.0.2以上 | 当前版本 | 3.0 分支 | 3.0.0 | 3.1.0 | 当前版本 | 与 TDengine 相同版本 |
| **2.4.0.14 及以上** | 2.0.38 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | 与 TDengine 相同版本 | | **2.4.0.14 及以上** | 2.0.38 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | 与 TDengine 相同版本 |
| **2.4.0.4 - 2.4.0.13** | 2.0.37 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | 与 TDengine 相同版本 | | **2.4.0.4 - 2.4.0.13** | 2.0.37 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | 与 TDengine 相同版本 |
| **2.2.x.x ** | 2.0.36 | 当前版本 | master 分支 | n/a | 2.0.7 - 2.0.9 | 当前版本 | 与 TDengine 相同版本 | | **2.2.x.x** | 2.0.36 | 当前版本 | master 分支 | n/a | 2.0.7 - 2.0.9 | 当前版本 | 与 TDengine 相同版本 |
| **2.0.x.x ** | 2.0.34 | 当前版本 | master 分支 | n/a | 2.0.1 - 2.0.6 | 当前版本 | 与 TDengine 相同版本 | | **2.0.x.x** | 2.0.34 | 当前版本 | master 分支 | n/a | 2.0.1 - 2.0.6 | 当前版本 | 与 TDengine 相同版本 |
## 功能特性 ## 功能特性

47
include/client/taos.h
View File

@ -81,6 +81,13 @@ typedef enum {
TSDB_SML_TIMESTAMP_NANO_SECONDS, TSDB_SML_TIMESTAMP_NANO_SECONDS,
} TSDB_SML_TIMESTAMP_TYPE; } TSDB_SML_TIMESTAMP_TYPE;
typedef enum TAOS_FIELD_T {
TAOS_FIELD_COL = 1,
TAOS_FIELD_TAG,
TAOS_FIELD_QUERY,
TAOS_FIELD_TBNAME,
} TAOS_FIELD_T;
typedef struct taosField { typedef struct taosField {
char name[65]; char name[65];
int8_t type; int8_t type;
@ -95,6 +102,15 @@ typedef struct TAOS_FIELD_E {
int32_t bytes; int32_t bytes;
} TAOS_FIELD_E; } TAOS_FIELD_E;
typedef struct TAOS_FIELD_STB {
char name[65];
int8_t type;
uint8_t precision;
uint8_t scale;
int32_t bytes;
TAOS_FIELD_T field_type;
} TAOS_FIELD_STB;
#ifdef WINDOWS #ifdef WINDOWS
#define DLL_EXPORT __declspec(dllexport) #define DLL_EXPORT __declspec(dllexport)
#else #else
@ -195,13 +211,6 @@ DLL_EXPORT int taos_stmt_affected_rows_once(TAOS_STMT *stmt);
typedef void TAOS_STMT2; typedef void TAOS_STMT2;
typedef enum TAOS_FIELD_T {
TAOS_FIELD_COL = 1,
TAOS_FIELD_TAG,
TAOS_FIELD_QUERY,
TAOS_FIELD_TBNAME,
} TAOS_FIELD_T;
typedef struct TAOS_STMT2_OPTION { typedef struct TAOS_STMT2_OPTION {
int64_t reqid; int64_t reqid;
bool singleStbInsert; bool singleStbInsert;
@ -232,7 +241,9 @@ DLL_EXPORT int taos_stmt2_exec(TAOS_STMT2 *stmt, int *affected_rows);
DLL_EXPORT int taos_stmt2_close(TAOS_STMT2 *stmt); DLL_EXPORT int taos_stmt2_close(TAOS_STMT2 *stmt);
DLL_EXPORT int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert); DLL_EXPORT int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert);
DLL_EXPORT int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, TAOS_FIELD_E **fields); DLL_EXPORT int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, TAOS_FIELD_E **fields);
DLL_EXPORT int taos_stmt2_get_stb_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_STB **fields);
DLL_EXPORT void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_E *fields); DLL_EXPORT void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_E *fields);
DLL_EXPORT void taos_stmt2_free_stb_fields(TAOS_STMT2 *stmt, TAOS_FIELD_STB *fields);
DLL_EXPORT TAOS_RES *taos_stmt2_result(TAOS_STMT2 *stmt); DLL_EXPORT TAOS_RES *taos_stmt2_result(TAOS_STMT2 *stmt);
DLL_EXPORT char *taos_stmt2_error(TAOS_STMT2 *stmt); DLL_EXPORT char *taos_stmt2_error(TAOS_STMT2 *stmt);
@ -251,17 +262,17 @@ DLL_EXPORT int64_t taos_affected_rows64(TAOS_RES *res);
DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res);
DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); DLL_EXPORT int taos_select_db(TAOS *taos, const char *db);
DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields);
DLL_EXPORT int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); DLL_EXPORT int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields);
DLL_EXPORT void taos_stop_query(TAOS_RES *res); DLL_EXPORT void taos_stop_query(TAOS_RES *res);
DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col); DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col);
DLL_EXPORT int taos_is_null_by_column(TAOS_RES *res, int columnIndex, bool result[], int *rows); DLL_EXPORT int taos_is_null_by_column(TAOS_RES *res, int columnIndex, bool result[], int *rows);
DLL_EXPORT bool taos_is_update_query(TAOS_RES *res); DLL_EXPORT bool taos_is_update_query(TAOS_RES *res);
DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows);
DLL_EXPORT int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows); DLL_EXPORT int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows);
DLL_EXPORT int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData); DLL_EXPORT int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData);
DLL_EXPORT int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex); DLL_EXPORT int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex);
DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql);
DLL_EXPORT void taos_reset_current_db(TAOS *taos); DLL_EXPORT void taos_reset_current_db(TAOS *taos);
DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res); DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res);
DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res); DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res);

8
include/common/tglobal.h
View File

@ -194,10 +194,10 @@ extern int32_t tsMinIntervalTime;
extern int32_t tsMaxInsertBatchRows; extern int32_t tsMaxInsertBatchRows;
// build info // build info
extern char version[]; extern char td_version[];
extern char compatible_version[]; extern char td_compatible_version[];
extern char gitinfo[]; extern char td_gitinfo[];
extern char buildinfo[]; extern char td_buildinfo[];
// lossy // lossy
extern char tsLossyColumns[]; extern char tsLossyColumns[];

5
include/common/tmsg.h
View File

@ -421,7 +421,7 @@ typedef enum ENodeType {
// physical plan node // physical plan node
QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN = 1100, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN = 1100,
QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN,
QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, // INACTIVE
QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN,
QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN,
QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN,
@ -435,7 +435,7 @@ typedef enum ENodeType {
QUERY_NODE_PHYSICAL_PLAN_SORT, QUERY_NODE_PHYSICAL_PLAN_SORT,
QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT,
QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, // INACTIVE
QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL,
@ -1337,6 +1337,7 @@ typedef struct {
char* sql; char* sql;
int8_t withArbitrator; int8_t withArbitrator;
int8_t encryptAlgorithm; int8_t encryptAlgorithm;
char dnodeListStr[TSDB_DNODE_LIST_LEN];
} SCreateDbReq; } SCreateDbReq;
int32_t tSerializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq); int32_t tSerializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq);

22
include/common/ttypes.h
View File

@ -238,12 +238,26 @@ typedef struct {
case TSDB_DATA_TYPE_UBIGINT: \ case TSDB_DATA_TYPE_UBIGINT: \
snprintf(_output, (int32_t)(_outputBytes), "%" PRIu64, *(uint64_t *)(_input)); \ snprintf(_output, (int32_t)(_outputBytes), "%" PRIu64, *(uint64_t *)(_input)); \
break; \ break; \
case TSDB_DATA_TYPE_FLOAT: \ case TSDB_DATA_TYPE_FLOAT: { \
snprintf(_output, (int32_t)(_outputBytes), "%f", *(float *)(_input)); \ int32_t n = snprintf(_output, (int32_t)(_outputBytes), "%f", *(float *)(_input)); \
if (n >= (_outputBytes)) { \
n = snprintf(_output, (int32_t)(_outputBytes), "%.7e", *(float *)(_input)); \
if (n >= (_outputBytes)) { \
snprintf(_output, (int32_t)(_outputBytes), "%f", *(float *)(_input)); \
} \
} \
break; \ break; \
case TSDB_DATA_TYPE_DOUBLE: \ } \
snprintf(_output, (int32_t)(_outputBytes), "%f", *(double *)(_input)); \ case TSDB_DATA_TYPE_DOUBLE: { \
int32_t n = snprintf(_output, (int32_t)(_outputBytes), "%f", *(double *)(_input)); \
if (n >= (_outputBytes)) { \
snprintf(_output, (int32_t)(_outputBytes), "%.15e", *(double *)(_input)); \
if (n >= (_outputBytes)) { \
snprintf(_output, (int32_t)(_outputBytes), "%f", *(double *)(_input)); \
} \
} \
break; \ break; \
} \
case TSDB_DATA_TYPE_UINT: \ case TSDB_DATA_TYPE_UINT: \
snprintf(_output, (int32_t)(_outputBytes), "%u", *(uint32_t *)(_input)); \ snprintf(_output, (int32_t)(_outputBytes), "%u", *(uint32_t *)(_input)); \
break; \ break; \

2
include/libs/function/functionMgt.h
View File

@ -26,6 +26,8 @@ extern "C" {
#define FUNC_AGGREGATE_UDF_ID 5001 #define FUNC_AGGREGATE_UDF_ID 5001
#define FUNC_SCALAR_UDF_ID 5002 #define FUNC_SCALAR_UDF_ID 5002
extern const int32_t funcMgtBuiltinsNum;
typedef enum EFunctionType { typedef enum EFunctionType {
// aggregate function // aggregate function
FUNCTION_TYPE_APERCENTILE = 1, FUNCTION_TYPE_APERCENTILE = 1,

1
include/libs/nodes/cmdnodes.h
View File

@ -72,6 +72,7 @@ typedef struct SDatabaseOptions {
int8_t compressionLevel; int8_t compressionLevel;
int8_t encryptAlgorithm; int8_t encryptAlgorithm;
int32_t daysPerFile; int32_t daysPerFile;
char dnodeListStr[TSDB_DNODE_LIST_LEN];
char encryptAlgorithmStr[TSDB_ENCRYPT_ALGO_STR_LEN]; char encryptAlgorithmStr[TSDB_ENCRYPT_ALGO_STR_LEN];
SValueNode* pDaysPerFile; SValueNode* pDaysPerFile;
int32_t fsyncPeriod; int32_t fsyncPeriod;

6
include/libs/nodes/plannodes.h
View File

@ -788,9 +788,9 @@ typedef struct SDataDeleterNode {
char tableFName[TSDB_TABLE_NAME_LEN]; char tableFName[TSDB_TABLE_NAME_LEN];
char tsColName[TSDB_COL_NAME_LEN]; char tsColName[TSDB_COL_NAME_LEN];
STimeWindow deleteTimeRange; STimeWindow deleteTimeRange;
SNode* pAffectedRows; SNode* pAffectedRows; // usless
SNode* pStartTs; SNode* pStartTs; // usless
SNode* pEndTs; SNode* pEndTs; // usless
} SDataDeleterNode; } SDataDeleterNode;
typedef struct SSubplan { typedef struct SSubplan {

3
include/libs/parser/parser.h
View File

@ -65,7 +65,7 @@ typedef struct SParseCsvCxt {
const char* pLastSqlPos; // the location of the last parsed sql const char* pLastSqlPos; // the location of the last parsed sql
} SParseCsvCxt; } SParseCsvCxt;
typedef void(*setQueryFn)(int64_t); typedef void (*setQueryFn)(int64_t);
typedef struct SParseContext { typedef struct SParseContext {
uint64_t requestId; uint64_t requestId;
@ -147,6 +147,7 @@ int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, c
int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen,
int32_t colIdx, int32_t rowNum); int32_t colIdx, int32_t rowNum);
int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD_E** fields); int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD_E** fields);
int32_t qBuildStmtStbColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_STB** fields);
int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields); int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields);
int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName,
TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen);

2
include/libs/qcom/query.h
View File

@ -364,7 +364,7 @@ void* getTaskPoolWorkerCb();
#define NEED_CLIENT_REFRESH_VG_ERROR(_code) \ #define NEED_CLIENT_REFRESH_VG_ERROR(_code) \
((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID) ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
#define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) \ #define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) \
((_code) == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER || (_code) == TSDB_CODE_MND_INVALID_SCHEMA_VER) ((_code) == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER || (_code) == TSDB_CODE_MND_INVALID_SCHEMA_VER || (_code) == TSDB_CODE_SCH_DATA_SRC_EP_MISS)
#define NEED_CLIENT_HANDLE_ERROR(_code) \ #define NEED_CLIENT_HANDLE_ERROR(_code) \
(NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \ (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \
NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))

4
include/libs/scheduler/scheduler.h
View File

@ -76,8 +76,6 @@ int32_t schedulerExecJob(SSchedulerReq* pReq, int64_t* pJob);
int32_t schedulerFetchRows(int64_t jobId, SSchedulerReq* pReq); int32_t schedulerFetchRows(int64_t jobId, SSchedulerReq* pReq);
void schedulerFetchRowsA(int64_t job, schedulerFetchFp fp, void* param);
int32_t schedulerGetTasksStatus(int64_t job, SArray* pSub); int32_t schedulerGetTasksStatus(int64_t job, SArray* pSub);
void schedulerStopQueryHb(void* pTrans); void schedulerStopQueryHb(void* pTrans);
@ -100,6 +98,8 @@ void schedulerFreeJob(int64_t* job, int32_t errCode);
void schedulerDestroy(void); void schedulerDestroy(void);
int32_t schedulerValidatePlan(SQueryPlan* pPlan);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

13
include/libs/stream/tstream.h
View File

@ -70,7 +70,8 @@ typedef struct SActiveCheckpointInfo SActiveCheckpointInfo;
#define SSTREAM_TASK_NEED_CONVERT_VER 2 #define SSTREAM_TASK_NEED_CONVERT_VER 2
#define SSTREAM_TASK_SUBTABLE_CHANGED_VER 3 #define SSTREAM_TASK_SUBTABLE_CHANGED_VER 3
extern int32_t streamMetaId; extern int32_t streamMetaRefPool;
extern int32_t streamTaskRefPool;
enum { enum {
STREAM_STATUS__NORMAL = 0, STREAM_STATUS__NORMAL = 0,
@ -258,6 +259,7 @@ typedef struct STaskId {
typedef struct SStreamTaskId { typedef struct SStreamTaskId {
int64_t streamId; int64_t streamId;
int32_t taskId; int32_t taskId;
int64_t refId;
const char* idStr; const char* idStr;
} SStreamTaskId; } SStreamTaskId;
@ -291,7 +293,6 @@ typedef struct SStreamStatus {
int8_t schedStatus; int8_t schedStatus;
int8_t statusBackup; int8_t statusBackup;
int32_t schedIdleTime; // idle time before invoke again int32_t schedIdleTime; // idle time before invoke again
int32_t timerActive; // timer is active
int64_t lastExecTs; // last exec time stamp int64_t lastExecTs; // last exec time stamp
int32_t inScanHistorySentinel; int32_t inScanHistorySentinel;
bool appendTranstateBlock; // has append the transfer state data block already bool appendTranstateBlock; // has append the transfer state data block already
@ -454,7 +455,6 @@ struct SStreamTask {
// the followings attributes don't be serialized // the followings attributes don't be serialized
SScanhistorySchedInfo schedHistoryInfo; SScanhistorySchedInfo schedHistoryInfo;
int32_t refCnt;
int32_t transferStateAlignCnt; int32_t transferStateAlignCnt;
struct SStreamMeta* pMeta; struct SStreamMeta* pMeta;
SSHashObj* pNameMap; SSHashObj* pNameMap;
@ -546,7 +546,7 @@ typedef int32_t (*__state_trans_user_fn)(SStreamTask*, void* param);
int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam, int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam,
SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5, SStreamTask** pTask); SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5, SStreamTask** pTask);
void tFreeStreamTask(SStreamTask* pTask); void tFreeStreamTask(void* pTask);
int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask); int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, int64_t ver); int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, int64_t ver);
@ -664,6 +664,8 @@ void streamTaskResetStatus(SStreamTask* pTask);
void streamTaskSetStatusReady(SStreamTask* pTask); void streamTaskSetStatusReady(SStreamTask* pTask);
ETaskStatus streamTaskGetPrevStatus(const SStreamTask* pTask); ETaskStatus streamTaskGetPrevStatus(const SStreamTask* pTask);
const char* streamTaskGetExecType(int32_t type); const char* streamTaskGetExecType(int32_t type);
int32_t streamTaskAllocRefId(SStreamTask* pTask, int64_t** pRefId);
void streamTaskFreeRefId(int64_t* pRefId);
bool streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList); bool streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList);
void streamTaskResetUpstreamStageInfo(SStreamTask* pTask); void streamTaskResetUpstreamStageInfo(SStreamTask* pTask);
@ -752,16 +754,15 @@ int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTa
int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta); int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta);
int32_t streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask); int32_t streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask);
int32_t streamMetaAcquireTaskUnsafe(SStreamMeta* pMeta, STaskId* pId, SStreamTask** pTask);
int32_t streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask); int32_t streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask);
void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
int32_t streamMetaAcquireOneTask(SStreamTask* pTask);
void streamMetaClear(SStreamMeta* pMeta); void streamMetaClear(SStreamMeta* pMeta);
void streamMetaInitBackend(SStreamMeta* pMeta); void streamMetaInitBackend(SStreamMeta* pMeta);
int32_t streamMetaCommit(SStreamMeta* pMeta); int32_t streamMetaCommit(SStreamMeta* pMeta);
int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta); int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta);
void streamMetaNotifyClose(SStreamMeta* pMeta); void streamMetaNotifyClose(SStreamMeta* pMeta);
void streamMetaStartHb(SStreamMeta* pMeta); void streamMetaStartHb(SStreamMeta* pMeta);
bool streamMetaTaskInTimer(SStreamMeta* pMeta);
int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, int64_t startTs, int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, int64_t startTs,
int64_t endTs, bool ready); int64_t endTs, bool ready);
int32_t streamMetaInitStartInfo(STaskStartInfo* pStartInfo); int32_t streamMetaInitStartInfo(STaskStartInfo* pStartInfo);

3
include/util/taoserror.h
View File

@ -352,6 +352,8 @@ int32_t taosGetErrSize();
#define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x039A) #define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x039A)
#define TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE TAOS_DEF_ERROR_CODE(0, 0x039B) #define TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE TAOS_DEF_ERROR_CODE(0, 0x039B)
#define TSDB_CODE_MND_INVALID_WAL_LEVEL TAOS_DEF_ERROR_CODE(0, 0x039C) #define TSDB_CODE_MND_INVALID_WAL_LEVEL TAOS_DEF_ERROR_CODE(0, 0x039C)
#define TSDB_CODE_MND_INVALID_DNODE_LIST_FMT TAOS_DEF_ERROR_CODE(0, 0x039D)
#define TSDB_CODE_MND_DNODE_LIST_REPEAT TAOS_DEF_ERROR_CODE(0, 0x039E)
// mnode-node // mnode-node
#define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0) #define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0)
@ -771,6 +773,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_SCH_TIMEOUT_ERROR TAOS_DEF_ERROR_CODE(0, 0x2504) #define TSDB_CODE_SCH_TIMEOUT_ERROR TAOS_DEF_ERROR_CODE(0, 0x2504)
#define TSDB_CODE_SCH_JOB_IS_DROPPING TAOS_DEF_ERROR_CODE(0, 0x2505) #define TSDB_CODE_SCH_JOB_IS_DROPPING TAOS_DEF_ERROR_CODE(0, 0x2505)
#define TSDB_CODE_SCH_JOB_NOT_EXISTS TAOS_DEF_ERROR_CODE(0, 0x2506) #define TSDB_CODE_SCH_JOB_NOT_EXISTS TAOS_DEF_ERROR_CODE(0, 0x2506)
#define TSDB_CODE_SCH_DATA_SRC_EP_MISS TAOS_DEF_ERROR_CODE(0, 0x2507)
//parser //parser
#define TSDB_CODE_PAR_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x2600) #define TSDB_CODE_PAR_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x2600)

9
include/util/tcompression.h
View File

@ -152,15 +152,12 @@ int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int
// for internal usage // for internal usage
int32_t getWordLength(char type); int32_t getWordLength(char type);
#ifdef __AVX2__
int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, char *const output, const char type); int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, char *const output, const char type);
int32_t tsDecompressFloatImpAvx2(const char *input, int32_t nelements, char *output); int32_t tsDecompressFloatImpAvx2(const char *input, int32_t nelements, char *output);
int32_t tsDecompressDoubleImpAvx2(const char *input, int32_t nelements, char *output); int32_t tsDecompressDoubleImpAvx2(const char *input, int32_t nelements, char *output);
#endif int32_t tsDecompressTimestampAvx2(const char *input, int32_t nelements, char *output, bool bigEndian);
#ifdef __AVX512VL__ int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output,
void tsDecompressTimestampAvx2(const char *input, int32_t nelements, char *output, bool bigEndian); bool bigEndian);
void tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output, bool bigEndian);
#endif
/************************************************************************* /*************************************************************************
* REGULAR COMPRESSION 2 * REGULAR COMPRESSION 2

43
include/util/tdef.h
View File

@ -41,6 +41,7 @@ extern const int32_t TYPE_BYTES[21];
#define FLOAT_BYTES sizeof(float) #define FLOAT_BYTES sizeof(float)
#define DOUBLE_BYTES sizeof(double) #define DOUBLE_BYTES sizeof(double)
#define POINTER_BYTES sizeof(void *) #define POINTER_BYTES sizeof(void *)
#define M256_BYTES 32
#define TSDB_KEYSIZE sizeof(TSKEY) #define TSDB_KEYSIZE sizeof(TSKEY)
#define TSDB_NCHAR_SIZE sizeof(TdUcs4) #define TSDB_NCHAR_SIZE sizeof(TdUcs4)
@ -188,6 +189,47 @@ typedef enum EOperatorType {
OP_TYPE_ASSIGN = 200 OP_TYPE_ASSIGN = 200
} EOperatorType; } EOperatorType;
static const EOperatorType OPERATOR_ARRAY[] = {
OP_TYPE_ADD,
OP_TYPE_SUB,
OP_TYPE_MULTI,
OP_TYPE_DIV,
OP_TYPE_REM,
OP_TYPE_MINUS,
OP_TYPE_BIT_AND,
OP_TYPE_BIT_OR,
OP_TYPE_GREATER_THAN,
OP_TYPE_GREATER_EQUAL,
OP_TYPE_LOWER_THAN,
OP_TYPE_LOWER_EQUAL,
OP_TYPE_EQUAL,
OP_TYPE_NOT_EQUAL,
OP_TYPE_IN,
OP_TYPE_NOT_IN,
OP_TYPE_LIKE,
OP_TYPE_NOT_LIKE,
OP_TYPE_MATCH,
OP_TYPE_NMATCH,
OP_TYPE_IS_NULL,
OP_TYPE_IS_NOT_NULL,
OP_TYPE_IS_TRUE,
OP_TYPE_IS_FALSE,
OP_TYPE_IS_UNKNOWN,
OP_TYPE_IS_NOT_TRUE,
OP_TYPE_IS_NOT_FALSE,
OP_TYPE_IS_NOT_UNKNOWN,
//OP_TYPE_COMPARE_MAX_VALUE,
OP_TYPE_JSON_GET_VALUE,
OP_TYPE_JSON_CONTAINS,
OP_TYPE_ASSIGN
};
#define OP_TYPE_CALC_MAX OP_TYPE_BIT_OR #define OP_TYPE_CALC_MAX OP_TYPE_BIT_OR
typedef enum ELogicConditionType { typedef enum ELogicConditionType {
@ -411,6 +453,7 @@ typedef enum ELogicConditionType {
#define TSDB_CACHE_MODEL_LAST_ROW 1 #define TSDB_CACHE_MODEL_LAST_ROW 1
#define TSDB_CACHE_MODEL_LAST_VALUE 2 #define TSDB_CACHE_MODEL_LAST_VALUE 2
#define TSDB_CACHE_MODEL_BOTH 3 #define TSDB_CACHE_MODEL_BOTH 3
#define TSDB_DNODE_LIST_LEN 256
#define TSDB_ENCRYPT_ALGO_STR_LEN 16 #define TSDB_ENCRYPT_ALGO_STR_LEN 16
#define TSDB_ENCRYPT_ALGO_NONE_STR "none" #define TSDB_ENCRYPT_ALGO_NONE_STR "none"
#define TSDB_ENCRYPT_ALGO_SM4_STR "sm4" #define TSDB_ENCRYPT_ALGO_SM4_STR "sm4"

10
include/util/version.h
View File

@ -20,11 +20,11 @@
extern "C" { extern "C" {
#endif #endif
extern char version[]; extern char td_version[];
extern char compatible_version[]; extern char td_compatible_version[];
extern char gitinfo[]; extern char td_gitinfo[];
extern char gitinfoOfInternal[]; extern char td_gitinfoOfInternal[];
extern char buildinfo[]; extern char td_buildinfo[];
#ifdef __cplusplus #ifdef __cplusplus
} }

9
source/client/CMakeLists.txt
View File

@ -10,6 +10,10 @@ else()
add_library(taos SHARED ${CLIENT_SRC}) add_library(taos SHARED ${CLIENT_SRC})
endif() endif()
if(${TD_DARWIN})
target_compile_options(taos PRIVATE -Wno-error=deprecated-non-prototype)
endif()
INCLUDE_DIRECTORIES(jni) INCLUDE_DIRECTORIES(jni)
target_include_directories( target_include_directories(
@ -46,6 +50,11 @@ set_target_properties(
) )
add_library(taos_static STATIC ${CLIENT_SRC}) add_library(taos_static STATIC ${CLIENT_SRC})
if(${TD_DARWIN})
target_compile_options(taos_static PRIVATE -Wno-error=deprecated-non-prototype)
endif()
target_include_directories( target_include_directories(
taos_static taos_static
PUBLIC "${TD_SOURCE_DIR}/include/client" PUBLIC "${TD_SOURCE_DIR}/include/client"

2
source/client/inc/clientInt.h
View File

@ -58,6 +58,8 @@ enum {
#define TD_RES_TMQ_METADATA(res) (*(int8_t*)(res) == RES_TYPE__TMQ_METADATA) #define TD_RES_TMQ_METADATA(res) (*(int8_t*)(res) == RES_TYPE__TMQ_METADATA)
#define TD_RES_TMQ_BATCH_META(res) (*(int8_t*)(res) == RES_TYPE__TMQ_BATCH_META) #define TD_RES_TMQ_BATCH_META(res) (*(int8_t*)(res) == RES_TYPE__TMQ_BATCH_META)
#define TSC_MAX_SUBPLAN_CAPACITY_NUM 1000
typedef struct SAppInstInfo SAppInstInfo; typedef struct SAppInstInfo SAppInstInfo;
typedef struct { typedef struct {

1
source/client/inc/clientStmt2.h
View File

@ -222,6 +222,7 @@ int stmtSetTbTags2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *tags);
int stmtBindBatch2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *bind, int32_t colIdx); int stmtBindBatch2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *bind, int32_t colIdx);
int stmtGetTagFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_E **fields); int stmtGetTagFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_E **fields);
int stmtGetColFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_E **fields); int stmtGetColFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_E **fields);
int stmtGetStbColFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_STB **fields);
int stmtGetParamNum2(TAOS_STMT2 *stmt, int *nums); int stmtGetParamNum2(TAOS_STMT2 *stmt, int *nums);
int stmtGetParamTbName(TAOS_STMT2 *stmt, int *nums); int stmtGetParamTbName(TAOS_STMT2 *stmt, int *nums);
int stmtIsInsert2(TAOS_STMT2 *stmt, int *insert); int stmtIsInsert2(TAOS_STMT2 *stmt, int *insert);

2
source/client/src/clientEnv.c
View File

@ -375,7 +375,7 @@ int32_t openTransporter(const char *user, const char *auth, int32_t numOfThread,
rpcInit.startReadTimer = 1; rpcInit.startReadTimer = 1;
rpcInit.readTimeout = tsReadTimeout; rpcInit.readTimeout = tsReadTimeout;
int32_t code = taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); int32_t code = taosVersionStrToInt(td_version, &rpcInit.compatibilityVer);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
tscError("invalid version string."); tscError("invalid version string.");
return code; return code;

59
source/client/src/clientImpl.c
View File

@ -1250,6 +1250,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
void launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res) { void launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res) {
int32_t code = 0; int32_t code = 0;
int32_t subplanNum = 0;
if (pQuery->pRoot) { if (pQuery->pRoot) {
pRequest->stmtType = pQuery->pRoot->type; pRequest->stmtType = pQuery->pRoot->type;
@ -1405,6 +1406,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = schedulerExecJob(&req, &pRequest->body.queryJob); code = schedulerExecJob(&req, &pRequest->body.queryJob);
} }
taosArrayDestroy(pNodeList); taosArrayDestroy(pNodeList);
} else { } else {
qDestroyQueryPlan(pDag); qDestroyQueryPlan(pDag);
@ -1698,7 +1700,7 @@ static int32_t buildConnectMsg(SRequestObj* pRequest, SMsgSendInfo** pMsgSendInf
tstrncpy(connectReq.app, appInfo.appName, sizeof(connectReq.app)); tstrncpy(connectReq.app, appInfo.appName, sizeof(connectReq.app));
tstrncpy(connectReq.user, pObj->user, sizeof(connectReq.user)); tstrncpy(connectReq.user, pObj->user, sizeof(connectReq.user));
tstrncpy(connectReq.passwd, pObj->pass, sizeof(connectReq.passwd)); tstrncpy(connectReq.passwd, pObj->pass, sizeof(connectReq.passwd));
tstrncpy(connectReq.sVer, version, sizeof(connectReq.sVer)); tstrncpy(connectReq.sVer, td_version, sizeof(connectReq.sVer));
int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq);
void* pReq = taosMemoryMalloc(contLen); void* pReq = taosMemoryMalloc(contLen);
@ -1768,19 +1770,15 @@ void updateTargetEpSet(SMsgSendInfo* pSendInfo, STscObj* pTscObj, SRpcMsg* pMsg,
} }
} }
int32_t doProcessMsgFromServer(void* param) { int32_t doProcessMsgFromServerImpl(SRpcMsg* pMsg, SEpSet* pEpSet) {
AsyncArg* arg = (AsyncArg*)param;
SRpcMsg* pMsg = &arg->msg;
SEpSet* pEpSet = arg->pEpset;
SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle; SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle;
if (pMsg->info.ahandle == NULL) { if (pMsg->info.ahandle == NULL) {
tscError("doProcessMsgFromServer pMsg->info.ahandle == NULL"); tscError("doProcessMsgFromServer pMsg->info.ahandle == NULL");
taosMemoryFree(arg->pEpset);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosMemoryFree(arg); taosMemoryFree(pEpSet);
return TSDB_CODE_TSC_INTERNAL_ERROR; return TSDB_CODE_TSC_INTERNAL_ERROR;
} }
STscObj* pTscObj = NULL; STscObj* pTscObj = NULL;
STraceId* trace = &pMsg->info.traceId; STraceId* trace = &pMsg->info.traceId;
@ -1800,10 +1798,9 @@ int32_t doProcessMsgFromServer(void* param) {
if (TSDB_CODE_SUCCESS != taosReleaseRef(clientReqRefPool, pSendInfo->requestObjRefId)) { if (TSDB_CODE_SUCCESS != taosReleaseRef(clientReqRefPool, pSendInfo->requestObjRefId)) {
tscError("doProcessMsgFromServer taosReleaseRef failed"); tscError("doProcessMsgFromServer taosReleaseRef failed");
} }
taosMemoryFree(arg->pEpset);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosMemoryFree(pEpSet);
destroySendMsgInfo(pSendInfo); destroySendMsgInfo(pSendInfo);
taosMemoryFree(arg);
return TSDB_CODE_TSC_INTERNAL_ERROR; return TSDB_CODE_TSC_INTERNAL_ERROR;
} }
pTscObj = pRequest->pTscObj; pTscObj = pRequest->pTscObj;
@ -1842,20 +1839,24 @@ int32_t doProcessMsgFromServer(void* param) {
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
destroySendMsgInfo(pSendInfo); destroySendMsgInfo(pSendInfo);
taosMemoryFree(arg);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t doProcessMsgFromServer(void* param) {
AsyncArg* arg = (AsyncArg*)param;
int32_t code = doProcessMsgFromServerImpl(&arg->msg, arg->pEpset);
taosMemoryFree(arg);
return code;
}
void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
int32_t code = 0;
SEpSet* tEpSet = NULL; SEpSet* tEpSet = NULL;
if (pEpSet != NULL) { if (pEpSet != NULL) {
tEpSet = taosMemoryCalloc(1, sizeof(SEpSet)); tEpSet = taosMemoryCalloc(1, sizeof(SEpSet));
if (NULL == tEpSet) { if (NULL == tEpSet) {
pMsg->code = TSDB_CODE_OUT_OF_MEMORY; code = terrno;
rpcFreeCont(pMsg->pCont); pMsg->code = terrno;
destroySendMsgInfo(pMsg->info.ahandle); goto _exit;
return;
} }
(void)memcpy((void*)tEpSet, (void*)pEpSet, sizeof(SEpSet)); (void)memcpy((void*)tEpSet, (void*)pEpSet, sizeof(SEpSet));
} }
@ -1877,21 +1878,25 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
AsyncArg* arg = taosMemoryCalloc(1, sizeof(AsyncArg)); AsyncArg* arg = taosMemoryCalloc(1, sizeof(AsyncArg));
if (NULL == arg) { if (NULL == arg) {
pMsg->code = TSDB_CODE_OUT_OF_MEMORY; code = terrno;
taosMemoryFree(tEpSet); pMsg->code = code;
rpcFreeCont(pMsg->pCont); goto _exit;
destroySendMsgInfo(pMsg->info.ahandle);
return;
} }
arg->msg = *pMsg; arg->msg = *pMsg;
arg->pEpset = tEpSet; arg->pEpset = tEpSet;
if (0 != taosAsyncExec(doProcessMsgFromServer, arg, NULL)) { if ((code = taosAsyncExec(doProcessMsgFromServer, arg, NULL)) != 0) {
tscError("failed to sched msg to tsc, tsc ready to quit"); pMsg->code = code;
rpcFreeCont(pMsg->pCont);
taosMemoryFree(arg->pEpset);
destroySendMsgInfo(pMsg->info.ahandle);
taosMemoryFree(arg); taosMemoryFree(arg);
goto _exit;
}
return;
_exit:
tscError("failed to sched msg to tsc since %s", tstrerror(code));
code = doProcessMsgFromServerImpl(pMsg, tEpSet);
if (code != 0) {
tscError("failed to sched msg to tsc, tsc ready quit");
} }
} }
@ -2571,7 +2576,7 @@ TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* de
rpcInit.connLimitNum = connLimitNum; rpcInit.connLimitNum = connLimitNum;
rpcInit.timeToGetConn = tsTimeToGetAvailableConn; rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
rpcInit.readTimeout = tsReadTimeout; rpcInit.readTimeout = tsReadTimeout;
if (TSDB_CODE_SUCCESS != taosVersionStrToInt(version, &(rpcInit.compatibilityVer))) { if (TSDB_CODE_SUCCESS != taosVersionStrToInt(td_version, &rpcInit.compatibilityVer)) {
tscError("faild to convert taos version from str to int, errcode:%s", terrstr()); tscError("faild to convert taos version from str to int, errcode:%s", terrstr());
goto _OVER; goto _OVER;
} }

29
source/client/src/clientMain.c
View File

@ -84,7 +84,7 @@ void taos_cleanup(void) {
taosCloseRef(id); taosCloseRef(id);
nodesDestroyAllocatorSet(); nodesDestroyAllocatorSet();
// cleanupAppInfo(); // cleanupAppInfo();
rpcCleanup(); rpcCleanup();
tscDebug("rpc cleanup"); tscDebug("rpc cleanup");
@ -388,7 +388,6 @@ void taos_free_result(TAOS_RES *res) {
tDeleteMqBatchMetaRsp(&pRsp->batchMetaRsp); tDeleteMqBatchMetaRsp(&pRsp->batchMetaRsp);
} }
taosMemoryFree(pRsp); taosMemoryFree(pRsp);
} }
void taos_kill_query(TAOS *taos) { void taos_kill_query(TAOS *taos) {
@ -484,7 +483,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) {
return taos_print_row_with_size(str, INT32_MAX, row, fields, num_fields); return taos_print_row_with_size(str, INT32_MAX, row, fields, num_fields);
} }
int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields){ int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) {
int32_t len = 0; int32_t len = 0;
for (int i = 0; i < num_fields; ++i) { for (int i = 0; i < num_fields; ++i) {
if (i > 0 && len < size - 1) { if (i > 0 && len < size - 1) {
@ -589,7 +588,7 @@ int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD
break; break;
} }
} }
if (len < size){ if (len < size) {
str[len] = 0; str[len] = 0;
} }
@ -670,7 +669,7 @@ const char *taos_data_type(int type) {
} }
} }
const char *taos_get_client_info() { return version; } const char *taos_get_client_info() { return td_version; }
// return int32_t // return int32_t
int taos_affected_rows(TAOS_RES *res) { int taos_affected_rows(TAOS_RES *res) {
@ -2082,7 +2081,7 @@ int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert) {
} }
int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, TAOS_FIELD_E **fields) { int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, TAOS_FIELD_E **fields) {
if (stmt == NULL || NULL == count) { if (stmt == NULL || count == NULL) {
tscError("NULL parameter for %s", __FUNCTION__); tscError("NULL parameter for %s", __FUNCTION__);
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return terrno; return terrno;
@ -2103,12 +2102,28 @@ int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count,
} }
} }
int taos_stmt2_get_stb_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_STB **fields) {
if (stmt == NULL || count == NULL) {
tscError("NULL parameter for %s", __FUNCTION__);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
return stmtGetStbColFields2(stmt, count, fields);
}
void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_E *fields) { void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_E *fields) {
(void)stmt; (void)stmt;
if (!fields) return; if (!fields) return;
taosMemoryFree(fields); taosMemoryFree(fields);
} }
DLL_EXPORT void taos_stmt2_free_stb_fields(TAOS_STMT2 *stmt, TAOS_FIELD_STB *fields) {
(void)stmt;
if (!fields) return;
taosMemoryFree(fields);
}
TAOS_RES *taos_stmt2_result(TAOS_STMT2 *stmt) { TAOS_RES *taos_stmt2_result(TAOS_STMT2 *stmt) {
if (stmt == NULL) { if (stmt == NULL) {
tscError("NULL parameter for %s", __FUNCTION__); tscError("NULL parameter for %s", __FUNCTION__);
@ -2144,4 +2159,4 @@ int taos_set_conn_mode(TAOS *taos, int mode, int value) {
return 0; return 0;
} }
char *getBuildInfo() { return buildinfo; } char *getBuildInfo() { return td_buildinfo; }

4
source/client/src/clientMsgHandler.c
View File

@ -80,8 +80,8 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
goto End; goto End;
} }
if ((code = taosCheckVersionCompatibleFromStr(version, connectRsp.sVer, 3)) != 0) { if ((code = taosCheckVersionCompatibleFromStr(td_version, connectRsp.sVer, 3)) != 0) {
tscError("version not compatible. client version: %s, server version: %s", version, connectRsp.sVer); tscError("version not compatible. client version: %s, server version: %s", td_version, connectRsp.sVer);
goto End; goto End;
} }

50
source/client/src/clientStmt2.c
View File

@ -1068,6 +1068,34 @@ static int stmtFetchColFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_E
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int stmtFetchStbColFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_STB** fields) {
if (pStmt->errCode != TSDB_CODE_SUCCESS) {
return pStmt->errCode;
}
if (STMT_TYPE_QUERY == pStmt->sql.type) {
tscError("invalid operation to get query column fileds");
STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
}
STableDataCxt** pDataBlock = NULL;
if (pStmt->sql.stbInterlaceMode) {
pDataBlock = &pStmt->sql.siInfo.pDataCtx;
} else {
pDataBlock =
(STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
if (NULL == pDataBlock) {
tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName);
STMT_ERR_RET(TSDB_CODE_APP_ERROR);
}
}
STMT_ERR_RET(qBuildStmtStbColFields(*pDataBlock, fieldNum, fields));
return TSDB_CODE_SUCCESS;
}
/* /*
SArray* stmtGetFreeCol(STscStmt2* pStmt, int32_t* idx) { SArray* stmtGetFreeCol(STscStmt2* pStmt, int32_t* idx) {
while (true) { while (true) {
@ -1808,7 +1836,7 @@ _return:
return code; return code;
} }
int stmtGetColFields2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_E** fields) { int stmtParseColFields2(TAOS_STMT2* stmt) {
int32_t code = 0; int32_t code = 0;
STscStmt2* pStmt = (STscStmt2*)stmt; STscStmt2* pStmt = (STscStmt2*)stmt;
int32_t preCode = pStmt->errCode; int32_t preCode = pStmt->errCode;
@ -1842,8 +1870,6 @@ int stmtGetColFields2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_E** fields) {
STMT_ERRI_JRET(stmtParseSql(pStmt)); STMT_ERRI_JRET(stmtParseSql(pStmt));
} }
STMT_ERRI_JRET(stmtFetchColFields2(stmt, nums, fields));
_return: _return:
pStmt->errCode = preCode; pStmt->errCode = preCode;
@ -1851,6 +1877,24 @@ _return:
return code; return code;
} }
int stmtGetColFields2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_E** fields) {
int32_t code = stmtParseColFields2(stmt);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
return stmtFetchColFields2(stmt, nums, fields);
}
int stmtGetStbColFields2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_STB** fields) {
int32_t code = stmtParseColFields2(stmt);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
return stmtFetchStbColFields2(stmt, nums, fields);
}
int stmtGetParamNum2(TAOS_STMT2* stmt, int* nums) { int stmtGetParamNum2(TAOS_STMT2* stmt, int* nums) {
STscStmt2* pStmt = (STscStmt2*)stmt; STscStmt2* pStmt = (STscStmt2*)stmt;

3
source/common/src/tcol.c
View File

@ -363,6 +363,9 @@ int8_t validColEncode(uint8_t type, uint8_t l1) {
if (l1 == TSDB_COLVAL_ENCODE_NOCHANGE) { if (l1 == TSDB_COLVAL_ENCODE_NOCHANGE) {
return 1; return 1;
} }
if (l1 == TSDB_COLVAL_ENCODE_DISABLED) {
return 1;
}
if (type == TSDB_DATA_TYPE_BOOL) { if (type == TSDB_DATA_TYPE_BOOL) {
return TSDB_COLVAL_ENCODE_RLE == l1 ? 1 : 0; return TSDB_COLVAL_ENCODE_RLE == l1 ? 1 : 0;
} else if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_INT) { } else if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_INT) {

20
source/common/src/tglobal.c
View File

@ -675,10 +675,10 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddString(pCfg, "os version", info.version, CFG_SCOPE_BOTH, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "os version", info.version, CFG_SCOPE_BOTH, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "os machine", info.machine, CFG_SCOPE_BOTH, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "os machine", info.machine, CFG_SCOPE_BOTH, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "version", version, CFG_SCOPE_BOTH, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "version", td_version, CFG_SCOPE_BOTH, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "compatible_version", compatible_version, CFG_SCOPE_BOTH, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "compatible_version", td_compatible_version, CFG_SCOPE_BOTH, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "gitinfo", gitinfo, CFG_SCOPE_BOTH, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "gitinfo", td_gitinfo, CFG_SCOPE_BOTH, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "buildinfo", buildinfo, CFG_SCOPE_BOTH, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "buildinfo", td_buildinfo, CFG_SCOPE_BOTH, CFG_DYN_NONE));
TAOS_RETURN(TSDB_CODE_SUCCESS); TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
@ -1646,6 +1646,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "checkpointInterval"); TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "checkpointInterval");
tsStreamCheckpointInterval = pItem->i32; tsStreamCheckpointInterval = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "concurrentCheckpoint");
tsMaxConcurrentCheckpoint = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "streamSinkDataRate"); TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "streamSinkDataRate");
tsSinkDataRate = pItem->fval; tsSinkDataRate = pItem->fval;
@ -2346,8 +2349,13 @@ static void taosCheckAndSetDebugFlag(int32_t *pFlagPtr, char *name, int32_t flag
if (noNeedToSetVars != NULL && taosArraySearch(noNeedToSetVars, name, taosLogVarComp, TD_EQ) != NULL) { if (noNeedToSetVars != NULL && taosArraySearch(noNeedToSetVars, name, taosLogVarComp, TD_EQ) != NULL) {
return; return;
} }
if (taosSetDebugFlag(pFlagPtr, name, flag) != 0) { int32_t code = 0;
uError("failed to set flag %s to %d", name, flag); if ((code = taosSetDebugFlag(pFlagPtr, name, flag)) != 0) {
if (code != TSDB_CODE_CFG_NOT_FOUND) {
uError("failed to set flag %s to %d, since:%s", name, flag, tstrerror(code));
} else {
uDebug("failed to set flag %s to %d, since:%s", name, flag, tstrerror(code));
}
} }
return; return;
} }

7
source/common/src/tmisce.c
View File

@ -221,10 +221,9 @@ int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t
} }
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "memory", tmp), NULL, _exit); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "memory", tmp), NULL, _exit);
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "version", version), NULL, _exit); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "version", td_version), NULL, _exit);
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "buildInfo", buildinfo), NULL, _exit); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "buildInfo", td_buildinfo), NULL, _exit);
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "gitInfo", td_gitinfo), NULL, _exit);
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "gitInfo", gitinfo), NULL, _exit);
TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "crashSig", signum), NULL, _exit); TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "crashSig", signum), NULL, _exit);
TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "crashTs", taosGetTimestampUs()), NULL, _exit); TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "crashTs", taosGetTimestampUs()), NULL, _exit);

5
source/common/src/tmsg.c
View File

@ -3874,6 +3874,7 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) {
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->s3ChunkSize)); TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->s3ChunkSize));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->s3KeepLocal)); TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->s3KeepLocal));
TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->s3Compact)); TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->s3Compact));
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->dnodeListStr));
tEndEncode(&encoder); tEndEncode(&encoder);
@ -3962,6 +3963,10 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq)
TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->s3Compact)); TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->s3Compact));
} }
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->dnodeListStr));
}
tEndDecode(&decoder); tEndDecode(&decoder);
_exit: _exit:

9
source/dnode/mgmt/exe/dmMain.c
View File

@ -297,12 +297,13 @@ static void dmPrintArgs(int32_t argc, char const *argv[]) {
static void dmGenerateGrant() { mndGenerateMachineCode(); } static void dmGenerateGrant() { mndGenerateMachineCode(); }
static void dmPrintVersion() { static void dmPrintVersion() {
printf("%s\n%sd version: %s compatible_version: %s\n", TD_PRODUCT_NAME, CUS_PROMPT, version, compatible_version); printf("%s\n%sd version: %s compatible_version: %s\n", TD_PRODUCT_NAME, CUS_PROMPT, td_version,
printf("git: %s\n", gitinfo); td_compatible_version);
printf("git: %s\n", td_gitinfo);
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE
printf("gitOfInternal: %s\n", gitinfoOfInternal); printf("gitOfInternal: %s\n", td_gitinfoOfInternal);
#endif #endif
printf("build: %s\n", buildinfo); printf("build: %s\n", td_buildinfo);
} }
static void dmPrintHelp() { static void dmPrintHelp() {

5
source/dnode/mgmt/mgmt_snode/CMakeLists.txt
View File

@ -1,5 +1,10 @@
aux_source_directory(src MGMT_SNODE) aux_source_directory(src MGMT_SNODE)
add_library(mgmt_snode STATIC ${MGMT_SNODE}) add_library(mgmt_snode STATIC ${MGMT_SNODE})
if(${TD_DARWIN})
target_compile_options(mgmt_snode PRIVATE -Wno-error=deprecated-non-prototype)
endif()
target_include_directories( target_include_directories(
mgmt_snode mgmt_snode
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"

5
source/dnode/mgmt/mgmt_vnode/CMakeLists.txt
View File

@ -1,5 +1,10 @@
aux_source_directory(src MGMT_VNODE) aux_source_directory(src MGMT_VNODE)
add_library(mgmt_vnode STATIC ${MGMT_VNODE}) add_library(mgmt_vnode STATIC ${MGMT_VNODE})
if(${TD_DARWIN})
target_compile_options(mgmt_vnode PRIVATE -Wno-error=deprecated-non-prototype)
endif()
target_include_directories( target_include_directories(
mgmt_vnode mgmt_vnode
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"

2
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -741,7 +741,7 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t vgId = alterReq.vgId; int32_t vgId = alterReq.vgId;
dInfo( dInfo(
"vgId:%d,vnode management handle msgType:%s, start to alter vnode replica:%d selfIndex:%d leanerReplica:%d " "vgId:%d, vnode management handle msgType:%s, start to alter vnode replica:%d selfIndex:%d leanerReplica:%d "
"learnerSelfIndex:%d strict:%d changeVersion:%d", "learnerSelfIndex:%d strict:%d changeVersion:%d",
vgId, TMSG_INFO(pMsg->msgType), alterReq.replica, alterReq.selfIndex, alterReq.learnerReplica, vgId, TMSG_INFO(pMsg->msgType), alterReq.replica, alterReq.selfIndex, alterReq.learnerReplica,
alterReq.learnerSelfIndex, alterReq.strict, alterReq.changeVersion); alterReq.learnerSelfIndex, alterReq.strict, alterReq.changeVersion);

20
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -138,9 +138,9 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
pRpc->info.handle, pRpc->contLen, pRpc->code, pRpc->info.ahandle, pRpc->info.refId); pRpc->info.handle, pRpc->contLen, pRpc->code, pRpc->info.ahandle, pRpc->info.refId);
int32_t svrVer = 0; int32_t svrVer = 0;
code = taosVersionStrToInt(version, &svrVer); code = taosVersionStrToInt(td_version, &svrVer);
if (code != 0) { if (code != 0) {
dError("failed to convert version string:%s to int, code:%d", version, code); dError("failed to convert version string:%s to int, code:%d", td_version, code);
goto _OVER; goto _OVER;
} }
if ((code = taosCheckVersionCompatible(pRpc->info.cliVer, svrVer, 3)) != 0) { if ((code = taosCheckVersionCompatible(pRpc->info.cliVer, svrVer, 3)) != 0) {
@ -434,8 +434,8 @@ int32_t dmInitClient(SDnode *pDnode) {
rpcInit.startReadTimer = 1; rpcInit.startReadTimer = 1;
rpcInit.readTimeout = tsReadTimeout; rpcInit.readTimeout = tsReadTimeout;
if (taosVersionStrToInt(version, &(rpcInit.compatibilityVer)) != 0) { if (taosVersionStrToInt(td_version, &rpcInit.compatibilityVer) != 0) {
dError("failed to convert version string:%s to int", version); dError("failed to convert version string:%s to int", td_version);
} }
pTrans->clientRpc = rpcOpen(&rpcInit); pTrans->clientRpc = rpcOpen(&rpcInit);
@ -483,8 +483,8 @@ int32_t dmInitStatusClient(SDnode *pDnode) {
rpcInit.startReadTimer = 0; rpcInit.startReadTimer = 0;
rpcInit.readTimeout = 0; rpcInit.readTimeout = 0;
if (taosVersionStrToInt(version, &(rpcInit.compatibilityVer)) != 0) { if (taosVersionStrToInt(td_version, &rpcInit.compatibilityVer) != 0) {
dError("failed to convert version string:%s to int", version); dError("failed to convert version string:%s to int", td_version);
} }
pTrans->statusRpc = rpcOpen(&rpcInit); pTrans->statusRpc = rpcOpen(&rpcInit);
@ -533,8 +533,8 @@ int32_t dmInitSyncClient(SDnode *pDnode) {
rpcInit.startReadTimer = 1; rpcInit.startReadTimer = 1;
rpcInit.readTimeout = tsReadTimeout; rpcInit.readTimeout = tsReadTimeout;
if (taosVersionStrToInt(version, &(rpcInit.compatibilityVer)) != 0) { if (taosVersionStrToInt(td_version, &rpcInit.compatibilityVer) != 0) {
dError("failed to convert version string:%s to int", version); dError("failed to convert version string:%s to int", td_version);
} }
pTrans->syncRpc = rpcOpen(&rpcInit); pTrans->syncRpc = rpcOpen(&rpcInit);
@ -588,8 +588,8 @@ int32_t dmInitServer(SDnode *pDnode) {
rpcInit.compressSize = tsCompressMsgSize; rpcInit.compressSize = tsCompressMsgSize;
rpcInit.shareConnLimit = tsShareConnLimit * 16; rpcInit.shareConnLimit = tsShareConnLimit * 16;
if (taosVersionStrToInt(version, &(rpcInit.compatibilityVer)) != 0) { if (taosVersionStrToInt(td_version, &rpcInit.compatibilityVer) != 0) {
dError("failed to convert version string:%s to int", version); dError("failed to convert version string:%s to int", td_version);
} }
pTrans->serverRpc = rpcOpen(&rpcInit); pTrans->serverRpc = rpcOpen(&rpcInit);

2
source/dnode/mgmt/test/sut/src/client.cpp
View File

@ -54,7 +54,7 @@ void TestClient::DoInit() {
rpcInit.parent = this; rpcInit.parent = this;
// rpcInit.secret = (char*)secretEncrypt; // rpcInit.secret = (char*)secretEncrypt;
// rpcInit.spi = 1; // rpcInit.spi = 1;
taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); taosVersionStrToInt(td_version, &rpcInit.compatibilityVer);
clientRpc = rpcOpen(&rpcInit); clientRpc = rpcOpen(&rpcInit);
ASSERT(clientRpc); ASSERT(clientRpc);

9
source/dnode/mnode/impl/inc/mndArbGroup.h
View File

@ -47,6 +47,15 @@ bool mndUpdateArbGroupBySetAssignedLeader(SArbGroup *pGroup, int32_t vgId, char
int32_t mndGetArbGroupSize(SMnode *pMnode); int32_t mndGetArbGroupSize(SMnode *pMnode);
typedef enum {
CHECK_SYNC_NONE = 0,
CHECK_SYNC_SET_ASSIGNED_LEADER = 1,
CHECK_SYNC_CHECK_SYNC = 2,
CHECK_SYNC_UPDATE = 3
} ECheckSyncOp;
void mndArbCheckSync(SArbGroup *pArbGroup, int64_t nowMs, ECheckSyncOp *pOp, SArbGroup *pNewGroup);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

1
source/dnode/mnode/impl/inc/mndDb.h
View File

@ -37,6 +37,7 @@ const char *mndGetDbStr(const char *src);
const char *mndGetStableStr(const char *src); const char *mndGetStableStr(const char *src);
int32_t mndProcessCompactDbReq(SRpcMsg *pReq); int32_t mndProcessCompactDbReq(SRpcMsg *pReq);
int32_t mndCheckDbDnodeList(SMnode *pMnode, char *db, char *dnodeListStr, SArray *dnodeList);
#ifdef __cplusplus #ifdef __cplusplus
} }

2
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -70,7 +70,7 @@ typedef enum {
MND_OPER_WRITE_DB, MND_OPER_WRITE_DB,
MND_OPER_READ_DB, MND_OPER_READ_DB,
MND_OPER_READ_OR_WRITE_DB, MND_OPER_READ_OR_WRITE_DB,
MND_OPER_SHOW_VARIBALES, MND_OPER_SHOW_VARIABLES,
MND_OPER_SUBSCRIBE, MND_OPER_SUBSCRIBE,
MND_OPER_CREATE_TOPIC, MND_OPER_CREATE_TOPIC,
MND_OPER_DROP_TOPIC, MND_OPER_DROP_TOPIC,

1
source/dnode/mnode/impl/inc/mndStream.h
View File

@ -133,6 +133,7 @@ int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgr
int32_t mndGetStreamObj(SMnode *pMnode, int64_t streamId, SStreamObj** pStream); int32_t mndGetStreamObj(SMnode *pMnode, int64_t streamId, SStreamObj** pStream);
bool mndStreamNodeIsUpdated(SMnode *pMnode); bool mndStreamNodeIsUpdated(SMnode *pMnode);
int32_t mndCheckForSnode(SMnode *pMnode, SDbObj *pSrcDb);
int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t taskId, int32_t nodeId); int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t taskId, int32_t nodeId);
int32_t mndProcessStreamHb(SRpcMsg *pReq); int32_t mndProcessStreamHb(SRpcMsg *pReq);

4
source/dnode/mnode/impl/inc/mndVgroup.h
View File

@ -35,9 +35,9 @@ void mndSortVnodeGid(SVgObj *pVgroup);
int64_t mndGetVnodesMemory(SMnode *pMnode, int32_t dnodeId); int64_t mndGetVnodesMemory(SMnode *pMnode, int32_t dnodeId);
int64_t mndGetVgroupMemory(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup); int64_t mndGetVgroupMemory(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup);
SArray *mndBuildDnodesArray(SMnode *, int32_t exceptDnodeId); SArray *mndBuildDnodesArray(SMnode *, int32_t exceptDnodeId, SArray *dnodeList);
int32_t mndAllocSmaVgroup(SMnode *, SDbObj *pDb, SVgObj *pVgroup); int32_t mndAllocSmaVgroup(SMnode *, SDbObj *pDb, SVgObj *pVgroup);
int32_t mndAllocVgroup(SMnode *, SDbObj *pDb, SVgObj **ppVgroups); int32_t mndAllocVgroup(SMnode *, SDbObj *pDb, SVgObj **ppVgroups, SArray *dnodeList);
int32_t mndAddNewVgPrepareAction(SMnode *, STrans *pTrans, SVgObj *pVg); int32_t mndAddNewVgPrepareAction(SMnode *, STrans *pTrans, SVgObj *pVg);
int32_t mndAddCreateVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid); int32_t mndAddCreateVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid);
int32_t mndAddAlterVnodeConfirmAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup); int32_t mndAddAlterVnodeConfirmAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup);

192
source/dnode/mnode/impl/src/mndArbGroup.c
View File

@ -250,6 +250,12 @@ static int32_t mndArbGroupActionUpdate(SSdb *pSdb, SArbGroup *pOld, SArbGroup *p
pOld->assignedLeader.acked = pNew->assignedLeader.acked; pOld->assignedLeader.acked = pNew->assignedLeader.acked;
pOld->version++; pOld->version++;
mInfo(
"arbgroup:%d, perform update action. members[0].token:%s, members[1].token:%s, isSync:%d, as-dnodeid:%d, "
"as-token:%s, as-acked:%d, version:%" PRId64,
pOld->vgId, pOld->members[0].state.token, pOld->members[1].state.token, pOld->isSync,
pOld->assignedLeader.dnodeId, pOld->assignedLeader.token, pOld->assignedLeader.acked, pOld->version);
_OVER: _OVER:
(void)taosThreadMutexUnlock(&pOld->mutex); (void)taosThreadMutexUnlock(&pOld->mutex);
@ -577,19 +583,77 @@ static int32_t mndSendArbSetAssignedLeaderReq(SMnode *pMnode, int32_t dnodeId, i
return code; return code;
} }
void mndArbCheckSync(SArbGroup *pArbGroup, int64_t nowMs, ECheckSyncOp *pOp, SArbGroup *pNewGroup) {
*pOp = CHECK_SYNC_NONE;
int32_t code = 0;
int32_t vgId = pArbGroup->vgId;
bool member0IsTimeout = mndCheckArbMemberHbTimeout(pArbGroup, 0, nowMs);
bool member1IsTimeout = mndCheckArbMemberHbTimeout(pArbGroup, 1, nowMs);
SArbAssignedLeader *pAssignedLeader = &pArbGroup->assignedLeader;
int32_t currentAssignedDnodeId = pAssignedLeader->dnodeId;
// 1. has assigned && no response => send req
if (currentAssignedDnodeId != 0 && pAssignedLeader->acked == false) {
*pOp = CHECK_SYNC_SET_ASSIGNED_LEADER;
return;
}
// 2. both of the two members are timeout => skip
if (member0IsTimeout && member1IsTimeout) {
return;
}
// 3. no member is timeout => check sync
if (member0IsTimeout == false && member1IsTimeout == false) {
// no assigned leader and not sync
if (currentAssignedDnodeId == 0 && !pArbGroup->isSync) {
*pOp = CHECK_SYNC_CHECK_SYNC;
}
return;
}
// 4. one of the members is timeout => set assigned leader
int32_t candidateIndex = member0IsTimeout ? 1 : 0;
SArbGroupMember *pMember = &pArbGroup->members[candidateIndex];
// has assigned leader and dnodeId not match => skip
if (currentAssignedDnodeId != 0 && currentAssignedDnodeId != pMember->info.dnodeId) {
mInfo("arb skip to set assigned leader to vgId:%d dnodeId:%d, assigned leader has been set to dnodeId:%d", vgId,
pMember->info.dnodeId, currentAssignedDnodeId);
return;
}
// not sync => skip
if (pArbGroup->isSync == false) {
if (currentAssignedDnodeId == pMember->info.dnodeId) {
mDebug("arb skip to set assigned leader to vgId:%d dnodeId:%d, arb group is not sync", vgId,
pMember->info.dnodeId);
} else {
mInfo("arb skip to set assigned leader to vgId:%d dnodeId:%d, arb group is not sync", vgId,
pMember->info.dnodeId);
}
return;
}
// is sync && no assigned leader => write to sdb
mndArbGroupDupObj(pArbGroup, pNewGroup);
mndArbGroupSetAssignedLeader(pNewGroup, candidateIndex);
*pOp = CHECK_SYNC_UPDATE;
}
static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) { static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) {
int32_t code = 0; int32_t code = 0, lino = 0;
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SArbGroup *pArbGroup = NULL; SArbGroup *pArbGroup = NULL;
SArbGroup arbGroupDup = {0};
void *pIter = NULL; void *pIter = NULL;
SArray *pUpdateArray = NULL;
char arbToken[TSDB_ARB_TOKEN_SIZE]; char arbToken[TSDB_ARB_TOKEN_SIZE];
if ((code = mndGetArbToken(pMnode, arbToken)) != 0) { TAOS_CHECK_EXIT(mndGetArbToken(pMnode, arbToken));
mError("failed to get arb token for arb-check-sync timer");
TAOS_RETURN(code);
}
int64_t term = mndGetTerm(pMnode); int64_t term = mndGetTerm(pMnode);
if (term < 0) { if (term < 0) {
mError("arb failed to get term since %s", terrstr()); mError("arb failed to get term since %s", terrstr());
@ -606,87 +670,63 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) {
return 0; return 0;
} }
SArray *pUpdateArray = taosArrayInit(16, sizeof(SArbGroup));
while (1) { while (1) {
pIter = sdbFetch(pSdb, SDB_ARBGROUP, pIter, (void **)&pArbGroup); pIter = sdbFetch(pSdb, SDB_ARBGROUP, pIter, (void **)&pArbGroup);
if (pIter == NULL) break; if (pIter == NULL) break;
SArbGroup arbGroupDup = {0};
(void)taosThreadMutexLock(&pArbGroup->mutex); (void)taosThreadMutexLock(&pArbGroup->mutex);
mndArbGroupDupObj(pArbGroup, &arbGroupDup); mndArbGroupDupObj(pArbGroup, &arbGroupDup);
(void)taosThreadMutexUnlock(&pArbGroup->mutex); (void)taosThreadMutexUnlock(&pArbGroup->mutex);
int32_t vgId = arbGroupDup.vgId;
bool member0IsTimeout = mndCheckArbMemberHbTimeout(&arbGroupDup, 0, nowMs);
bool member1IsTimeout = mndCheckArbMemberHbTimeout(&arbGroupDup, 1, nowMs);
SArbAssignedLeader *pAssignedLeader = &arbGroupDup.assignedLeader;
int32_t currentAssignedDnodeId = pAssignedLeader->dnodeId;
// 1. has assigned && is sync && no response => send req
if (currentAssignedDnodeId != 0 && arbGroupDup.isSync == true && pAssignedLeader->acked == false) {
(void)mndSendArbSetAssignedLeaderReq(pMnode, currentAssignedDnodeId, vgId, arbToken, term,
pAssignedLeader->token);
mInfo("vgId:%d, arb send set assigned leader to dnodeId:%d", vgId, currentAssignedDnodeId);
sdbRelease(pSdb, pArbGroup);
continue;
}
// 2. both of the two members are timeout => skip
if (member0IsTimeout && member1IsTimeout) {
sdbRelease(pSdb, pArbGroup);
continue;
}
// 3. no member is timeout => check sync
if (member0IsTimeout == false && member1IsTimeout == false) {
// no assigned leader and not sync
if (currentAssignedDnodeId == 0 && !arbGroupDup.isSync) {
(void)mndSendArbCheckSyncReq(pMnode, arbGroupDup.vgId, arbToken, term, arbGroupDup.members[0].state.token,
arbGroupDup.members[1].state.token);
}
sdbRelease(pSdb, pArbGroup);
continue;
}
// 4. one of the members is timeout => set assigned leader
int32_t candidateIndex = member0IsTimeout ? 1 : 0;
SArbGroupMember *pMember = &arbGroupDup.members[candidateIndex];
// has assigned leader and dnodeId not match => skip
if (currentAssignedDnodeId != 0 && currentAssignedDnodeId != pMember->info.dnodeId) {
mInfo("arb skip to set assigned leader to vgId:%d dnodeId:%d, assigned leader has been set to dnodeId:%d", vgId,
pMember->info.dnodeId, currentAssignedDnodeId);
sdbRelease(pSdb, pArbGroup);
continue;
}
// not sync => skip
if (arbGroupDup.isSync == false) {
if (currentAssignedDnodeId == pMember->info.dnodeId) {
mDebug("arb skip to set assigned leader to vgId:%d dnodeId:%d, arb group is not sync", vgId,
pMember->info.dnodeId);
} else {
mInfo("arb skip to set assigned leader to vgId:%d dnodeId:%d, arb group is not sync", vgId,
pMember->info.dnodeId);
}
sdbRelease(pSdb, pArbGroup);
continue;
}
// is sync && no assigned leader => write to sdb
SArbGroup newGroup = {0};
mndArbGroupDupObj(&arbGroupDup, &newGroup);
mndArbGroupSetAssignedLeader(&newGroup, candidateIndex);
if (taosArrayPush(pUpdateArray, &newGroup) == NULL) {
taosArrayDestroy(pUpdateArray);
return terrno;
}
sdbRelease(pSdb, pArbGroup); sdbRelease(pSdb, pArbGroup);
ECheckSyncOp op = CHECK_SYNC_NONE;
SArbGroup newGroup = {0};
mndArbCheckSync(&arbGroupDup, nowMs, &op, &newGroup);
int32_t vgId = arbGroupDup.vgId;
SArbAssignedLeader *pAssgndLeader = &arbGroupDup.assignedLeader;
int32_t assgndDnodeId = pAssgndLeader->dnodeId;
switch (op) {
case CHECK_SYNC_NONE:
mTrace("vgId:%d, arb skip to send msg by check sync", vgId);
break;
case CHECK_SYNC_SET_ASSIGNED_LEADER:
(void)mndSendArbSetAssignedLeaderReq(pMnode, assgndDnodeId, vgId, arbToken, term, pAssgndLeader->token);
mInfo("vgId:%d, arb send set assigned leader to dnodeId:%d", vgId, assgndDnodeId);
break;
case CHECK_SYNC_CHECK_SYNC:
(void)mndSendArbCheckSyncReq(pMnode, vgId, arbToken, term, arbGroupDup.members[0].state.token,
arbGroupDup.members[1].state.token);
mInfo("vgId:%d, arb send check sync request", vgId);
break;
case CHECK_SYNC_UPDATE:
if (!pUpdateArray) {
pUpdateArray = taosArrayInit(16, sizeof(SArbGroup));
if (!pUpdateArray) {
TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY);
}
}
if (taosArrayPush(pUpdateArray, &newGroup) == NULL) {
TAOS_CHECK_EXIT(terrno);
}
break;
default:
mError("vgId:%d, arb unknown check sync op:%d", vgId, op);
break;
}
} }
TAOS_CHECK_RETURN(mndPullupArbUpdateGroupBatch(pMnode, pUpdateArray)); TAOS_CHECK_EXIT(mndPullupArbUpdateGroupBatch(pMnode, pUpdateArray));
_exit:
if (code != 0) {
mError("failed to check sync at line %d since %s", lino, terrstr());
}
taosArrayDestroy(pUpdateArray); taosArrayDestroy(pUpdateArray);
return 0; return 0;

32
source/dnode/mnode/impl/src/mndDb.c
View File

@ -462,8 +462,8 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
if (pCfg->cacheLast < TSDB_CACHE_MODEL_NONE || pCfg->cacheLast > TSDB_CACHE_MODEL_BOTH) return code; if (pCfg->cacheLast < TSDB_CACHE_MODEL_NONE || pCfg->cacheLast > TSDB_CACHE_MODEL_BOTH) return code;
if (pCfg->hashMethod != 1) return code; if (pCfg->hashMethod != 1) return code;
if (pCfg->replications > mndGetDnodeSize(pMnode)) { if (pCfg->replications > mndGetDnodeSize(pMnode)) {
terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; code = TSDB_CODE_MND_NO_ENOUGH_DNODES;
return code; TAOS_RETURN(code);
} }
if (pCfg->walRetentionPeriod < TSDB_DB_MIN_WAL_RETENTION_PERIOD) return code; if (pCfg->walRetentionPeriod < TSDB_DB_MIN_WAL_RETENTION_PERIOD) return code;
if (pCfg->walRetentionSize < TSDB_DB_MIN_WAL_RETENTION_SIZE) return code; if (pCfg->walRetentionSize < TSDB_DB_MIN_WAL_RETENTION_SIZE) return code;
@ -746,7 +746,7 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj
TAOS_RETURN(code); TAOS_RETURN(code);
} }
static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, SUserObj *pUser) { static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, SUserObj *pUser, SArray *dnodeList) {
int32_t code = 0; int32_t code = 0;
SUserObj newUserObj = {0}; SUserObj newUserObj = {0};
SDbObj dbObj = {0}; SDbObj dbObj = {0};
@ -823,7 +823,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
} }
SVgObj *pVgroups = NULL; SVgObj *pVgroups = NULL;
if ((code = mndAllocVgroup(pMnode, &dbObj, &pVgroups)) != 0) { if ((code = mndAllocVgroup(pMnode, &dbObj, &pVgroups, dnodeList)) != 0) {
mError("db:%s, failed to create, alloc vgroup failed, since %s", pCreate->db, terrstr()); mError("db:%s, failed to create, alloc vgroup failed, since %s", pCreate->db, terrstr());
TAOS_RETURN(code); TAOS_RETURN(code);
} }
@ -925,6 +925,17 @@ _exit:
TAOS_RETURN(code); TAOS_RETURN(code);
} }
#ifndef TD_ENTERPRISE
int32_t mndCheckDbDnodeList(SMnode *pMnode, char *db, char *dnodeListStr, SArray *dnodeList) {
if (dnodeListStr[0] != 0) {
terrno = TSDB_CODE_OPS_NOT_SUPPORT;
return terrno;
} else {
return 0;
}
}
#endif
static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
int32_t code = -1; int32_t code = -1;
@ -932,6 +943,10 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
SDbObj *pDb = NULL; SDbObj *pDb = NULL;
SUserObj *pUser = NULL; SUserObj *pUser = NULL;
SCreateDbReq createReq = {0}; SCreateDbReq createReq = {0};
SArray *dnodeList = NULL;
dnodeList = taosArrayInit(mndGetDnodeSize(pMnode), sizeof(int32_t));
TSDB_CHECK_NULL(dnodeList, code, lino, _OVER, TSDB_CODE_OUT_OF_MEMORY);
TAOS_CHECK_GOTO(tDeserializeSCreateDbReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER); TAOS_CHECK_GOTO(tDeserializeSCreateDbReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
#ifdef WINDOWS #ifdef WINDOWS
@ -975,9 +990,11 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
TAOS_CHECK_GOTO(mndCheckDbEncryptKey(pMnode, &createReq), &lino, _OVER); TAOS_CHECK_GOTO(mndCheckDbEncryptKey(pMnode, &createReq), &lino, _OVER);
TAOS_CHECK_GOTO(mndCheckDbDnodeList(pMnode, createReq.db, createReq.dnodeListStr, dnodeList), &lino, _OVER);
TAOS_CHECK_GOTO(mndAcquireUser(pMnode, pReq->info.conn.user, &pUser), &lino, _OVER); TAOS_CHECK_GOTO(mndAcquireUser(pMnode, pReq->info.conn.user, &pUser), &lino, _OVER);
TAOS_CHECK_GOTO(mndCreateDb(pMnode, pReq, &createReq, pUser), &lino, _OVER); TAOS_CHECK_GOTO(mndCreateDb(pMnode, pReq, &createReq, pUser, dnodeList), &lino, _OVER);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
SName name = {0}; SName name = {0};
@ -994,6 +1011,7 @@ _OVER:
mndReleaseDb(pMnode, pDb); mndReleaseDb(pMnode, pDb);
mndReleaseUser(pMnode, pUser); mndReleaseUser(pMnode, pUser);
tFreeSCreateDbReq(&createReq); tFreeSCreateDbReq(&createReq);
taosArrayDestroy(dnodeList);
TAOS_RETURN(code); TAOS_RETURN(code);
} }
@ -1168,7 +1186,9 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL; void *pIter = NULL;
SVgObj *pVgroup = NULL; SVgObj *pVgroup = NULL;
SArray *pArray = mndBuildDnodesArray(pMnode, 0); SArray *pArray = mndBuildDnodesArray(pMnode, 0, NULL);
TSDB_CHECK_NULL(pArray, code, lino, _err, TSDB_CODE_OUT_OF_MEMORY);
while (1) { while (1) {
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);

2
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -1068,7 +1068,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) {
SShowVariablesRsp rsp = {0}; SShowVariablesRsp rsp = {0};
int32_t code = -1; int32_t code = -1;
if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_SHOW_VARIBALES) != 0) { if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_SHOW_VARIABLES) != 0) {
goto _OVER; goto _OVER;
} }

2
source/dnode/mnode/impl/src/mndMain.c
View File

@ -1021,7 +1021,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
} }
// cluster info // cluster info
tstrncpy(pClusterInfo->version, version, sizeof(pClusterInfo->version)); tstrncpy(pClusterInfo->version, td_version, sizeof(pClusterInfo->version));
pClusterInfo->monitor_interval = tsMonitorInterval; pClusterInfo->monitor_interval = tsMonitorInterval;
pClusterInfo->connections_total = mndGetNumOfConnections(pMnode); pClusterInfo->connections_total = mndGetNumOfConnections(pMnode);
pClusterInfo->dbs_total = sdbGetSize(pSdb, SDB_DB); pClusterInfo->dbs_total = sdbGetSize(pSdb, SDB_DB);

12
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -239,8 +239,8 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
goto _OVER; goto _OVER;
} }
if ((code = taosCheckVersionCompatibleFromStr(connReq.sVer, version, 3)) != 0) { if ((code = taosCheckVersionCompatibleFromStr(connReq.sVer, td_version, 3)) != 0) {
mGError("version not compatible. client version: %s, server version: %s", connReq.sVer, version); mGError("version not compatible. client version: %s, server version: %s", connReq.sVer, td_version);
goto _OVER; goto _OVER;
} }
@ -308,9 +308,9 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
tstrncpy(connectRsp.monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb, TSDB_DB_NAME_LEN); tstrncpy(connectRsp.monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb, TSDB_DB_NAME_LEN);
connectRsp.whiteListVer = pUser->ipWhiteListVer; connectRsp.whiteListVer = pUser->ipWhiteListVer;
(void)strcpy(connectRsp.sVer, version); tstrncpy(connectRsp.sVer, td_version, sizeof(connectRsp.sVer));
(void)snprintf(connectRsp.sDetailVer, sizeof(connectRsp.sDetailVer), "ver:%s\nbuild:%s\ngitinfo:%s", version, (void)snprintf(connectRsp.sDetailVer, sizeof(connectRsp.sDetailVer), "ver:%s\nbuild:%s\ngitinfo:%s", td_version,
buildinfo, gitinfo); td_buildinfo, td_gitinfo);
mndGetMnodeEpSet(pMnode, &connectRsp.epSet); mndGetMnodeEpSet(pMnode, &connectRsp.epSet);
int32_t contLen = tSerializeSConnectRsp(NULL, 0, &connectRsp); int32_t contLen = tSerializeSConnectRsp(NULL, 0, &connectRsp);
@ -813,7 +813,7 @@ static int32_t mndProcessSvrVerReq(SRpcMsg *pReq) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
SServerVerRsp rsp = {0}; SServerVerRsp rsp = {0};
tstrncpy(rsp.ver, version, sizeof(rsp.ver)); tstrncpy(rsp.ver, td_version, sizeof(rsp.ver));
int32_t contLen = tSerializeSServerVerRsp(NULL, 0, &rsp); int32_t contLen = tSerializeSServerVerRsp(NULL, 0, &rsp);
if (contLen < 0) { if (contLen < 0) {

19
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -248,7 +248,7 @@ static int32_t doAddSinkTask(SStreamObj* pStream, SMnode* pMnode, SVgObj* pVgrou
return code; return code;
} }
mDebug("doAddSinkTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory); mDebug("doAddSinkTask taskId:%s, %p vgId:%d, isFillHistory:%d", pTask->id.idStr, pTask, pVgroup->vgId, isFillhistory);
pTask->info.nodeId = pVgroup->vgId; pTask->info.nodeId = pVgroup->vgId;
pTask->info.epSet = mndGetVgroupEpset(pMnode, pVgroup); pTask->info.epSet = mndGetVgroupEpset(pMnode, pVgroup);
@ -364,12 +364,13 @@ static int32_t buildSourceTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillh
static void addNewTaskList(SStreamObj* pStream) { static void addNewTaskList(SStreamObj* pStream) {
SArray* pTaskList = taosArrayInit(0, POINTER_BYTES); SArray* pTaskList = taosArrayInit(0, POINTER_BYTES);
if (taosArrayPush(pStream->tasks, &pTaskList) == NULL) { if (taosArrayPush(pStream->tasks, &pTaskList) == NULL) {
mError("failed to put array"); mError("failed to put into array");
} }
if (pStream->conf.fillHistory) { if (pStream->conf.fillHistory) {
pTaskList = taosArrayInit(0, POINTER_BYTES); pTaskList = taosArrayInit(0, POINTER_BYTES);
if (taosArrayPush(pStream->pHTasksList, &pTaskList) == NULL) { if (taosArrayPush(pStream->pHTasksList, &pTaskList) == NULL) {
mError("failed to put array"); mError("failed to put into array");
} }
} }
} }
@ -402,7 +403,8 @@ static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStre
return code; return code;
} }
mDebug("doAddSourceTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory); mDebug("doAddSourceTask taskId:%s, %p vgId:%d, isFillHistory:%d", pTask->id.idStr, pTask, pVgroup->vgId,
isFillhistory);
if (pStream->conf.fillHistory) { if (pStream->conf.fillHistory) {
haltInitialTaskStatus(pTask, plan, isFillhistory); haltInitialTaskStatus(pTask, plan, isFillhistory);
@ -512,19 +514,20 @@ static int32_t doAddAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan,
SSnodeObj* pSnode, bool isFillhistory, bool useTriggerParam) { SSnodeObj* pSnode, bool isFillhistory, bool useTriggerParam) {
int32_t code = 0; int32_t code = 0;
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
const char* id = NULL;
code = buildAggTask(pStream, pEpset, isFillhistory, useTriggerParam, &pTask); code = buildAggTask(pStream, pEpset, isFillhistory, useTriggerParam, &pTask);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
id = pTask->id.idStr;
if (pSnode != NULL) { if (pSnode != NULL) {
code = mndAssignStreamTaskToSnode(pMnode, pTask, plan, pSnode); code = mndAssignStreamTaskToSnode(pMnode, pTask, plan, pSnode);
mDebug("doAddAggTask taskId:%s, snode id:%d, isFillHistory:%d", pTask->id.idStr, pSnode->id, isFillhistory); mDebug("doAddAggTask taskId:%s, %p snode id:%d, isFillHistory:%d", id, pTask, pSnode->id, isFillhistory);
} else { } else {
code = mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup); code = mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup);
mDebug("doAddAggTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory); mDebug("doAddAggTask taskId:%s, %p vgId:%d, isFillHistory:%d", id, pTask, pVgroup->vgId, isFillhistory);
} }
return code; return code;
} }
@ -678,7 +681,7 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
if (numOfPlanLevel > 1 || externalTargetDB || multiTarget || pStream->fixedSinkVgId) { if (numOfPlanLevel > 1 || externalTargetDB || multiTarget || pStream->fixedSinkVgId) {
// add extra sink // add extra sink
hasExtraSink = true; hasExtraSink = true;
int32_t code = addSinkTask(pMnode, pStream, pEpset); code = addSinkTask(pMnode, pStream, pEpset);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }

29
source/dnode/mnode/impl/src/mndStream.c
View File

@ -795,12 +795,22 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
} }
if (createReq.sql != NULL) { if (createReq.sql != NULL) {
sqlLen = strlen(createReq.sql); sql = taosStrdup(createReq.sql);
sql = taosMemoryMalloc(sqlLen + 1);
TSDB_CHECK_NULL(sql, code, lino, _OVER, terrno); TSDB_CHECK_NULL(sql, code, lino, _OVER, terrno);
}
memset(sql, 0, sqlLen + 1); SDbObj *pSourceDb = mndAcquireDb(pMnode, createReq.sourceDB);
memcpy(sql, createReq.sql, sqlLen); if (pSourceDb == NULL) {
code = terrno;
mInfo("stream:%s failed to create, acquire source db %s failed, code:%s", createReq.name, createReq.sourceDB,
tstrerror(code));
goto _OVER;
}
code = mndCheckForSnode(pMnode, pSourceDb);
mndReleaseDb(pMnode, pSourceDb);
if (code != 0) {
goto _OVER;
} }
// build stream obj from request // build stream obj from request
@ -1284,9 +1294,10 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
void* p = taosArrayPush(pList, &in); void* p = taosArrayPush(pList, &in);
if (p) { if (p) {
int32_t currentSize = taosArrayGetSize(pList); int32_t currentSize = taosArrayGetSize(pList);
mDebug("stream:%s (uid:0x%" PRIx64 ") checkpoint interval beyond threshold: %ds(%" PRId64 mDebug("stream:%s (uid:0x%" PRIx64 ") total %d stream(s) beyond chpt interval threshold: %ds(%" PRId64
"s) beyond concurrently launch threshold:%d", "s), concurrently launch threshold:%d",
pStream->name, pStream->uid, tsStreamCheckpointInterval, duration / 1000, currentSize); pStream->name, pStream->uid, currentSize, tsStreamCheckpointInterval, duration / 1000,
tsMaxConcurrentCheckpoint);
} else { } else {
mError("failed to record the checkpoint interval info, stream:0x%" PRIx64, pStream->uid); mError("failed to record the checkpoint interval info, stream:0x%" PRIx64, pStream->uid);
} }
@ -1338,7 +1349,7 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
code = mndProcessStreamCheckpointTrans(pMnode, p, checkpointId, 1, true); code = mndProcessStreamCheckpointTrans(pMnode, p, checkpointId, 1, true);
sdbRelease(pSdb, p); sdbRelease(pSdb, p);
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code == 0 || code == TSDB_CODE_ACTION_IN_PROGRESS) {
started += 1; started += 1;
if (started >= capacity) { if (started >= capacity) {
@ -1346,6 +1357,8 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
(started + numOfCheckpointTrans)); (started + numOfCheckpointTrans));
break; break;
} }
} else {
mError("failed to start checkpoint trans, code:%s", tstrerror(code));
} }
} }
} }

24
source/dnode/mnode/impl/src/mndStreamUtil.c
View File

@ -1497,6 +1497,30 @@ bool mndStreamNodeIsUpdated(SMnode *pMnode) {
return updated; return updated;
} }
int32_t mndCheckForSnode(SMnode *pMnode, SDbObj *pSrcDb) {
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
SSnodeObj *pObj = NULL;
if (pSrcDb->cfg.replications == 1) {
return TSDB_CODE_SUCCESS;
} else {
while (1) {
pIter = sdbFetch(pSdb, SDB_SNODE, pIter, (void **)&pObj);
if (pIter == NULL) {
break;
}
sdbRelease(pSdb, pObj);
sdbCancelFetch(pSdb, pIter);
return TSDB_CODE_SUCCESS;
}
mError("snode not existed when trying to create stream in db with multiple replica");
return TSDB_CODE_SNODE_NOT_DEPLOYED;
}
}
uint32_t seed = 0; uint32_t seed = 0;
static SRpcMsg createRpcMsg(STransAction* pAction, int64_t traceId, int64_t signature) { static SRpcMsg createRpcMsg(STransAction* pAction, int64_t traceId, int64_t signature) {
SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen, .info.ahandle = (void *)signature}; SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen, .info.ahandle = (void *)signature};

6
source/dnode/mnode/impl/src/mndTelem.c
View File

@ -115,9 +115,9 @@ static char* mndBuildTelemetryReport(SMnode* pMnode) {
snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB); snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB);
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "memory", tmp), &lino, _OVER); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "memory", tmp), &lino, _OVER);
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "version", version), &lino, _OVER); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "version", td_version), &lino, _OVER);
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "buildInfo", buildinfo), &lino, _OVER); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "buildInfo", td_buildinfo), &lino, _OVER);
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "gitInfo", gitinfo), &lino, _OVER); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "gitInfo", td_gitinfo), &lino, _OVER);
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "email", pMgmt->email), &lino, _OVER); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "email", pMgmt->email), &lino, _OVER);
mndBuildRuntimeInfo(pMnode, pJson); mndBuildRuntimeInfo(pMnode, pJson);

33
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -717,11 +717,28 @@ static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2
SDnodeObj *pDnode = pObj; SDnodeObj *pDnode = pObj;
SArray *pArray = p1; SArray *pArray = p1;
int32_t exceptDnodeId = *(int32_t *)p2; int32_t exceptDnodeId = *(int32_t *)p2;
SArray *dnodeList = p3;
if (exceptDnodeId == pDnode->id) { if (exceptDnodeId == pDnode->id) {
return true; return true;
} }
if (dnodeList != NULL) {
int32_t dnodeListSize = taosArrayGetSize(dnodeList);
if (dnodeListSize > 0) {
bool inDnodeList = false;
for (int32_t index = 0; index < dnodeListSize; ++index) {
int32_t dnodeId = *(int32_t *)taosArrayGet(dnodeList, index);
if (pDnode->id == dnodeId) {
inDnodeList = true;
}
}
if (!inDnodeList) {
return true;
}
}
}
int64_t curMs = taosGetTimestampMs(); int64_t curMs = taosGetTimestampMs();
bool online = mndIsDnodeOnline(pDnode, curMs); bool online = mndIsDnodeOnline(pDnode, curMs);
bool isMnode = mndIsMnode(pMnode, pDnode->id); bool isMnode = mndIsMnode(pMnode, pDnode->id);
@ -741,7 +758,7 @@ static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2
return true; return true;
} }
SArray *mndBuildDnodesArray(SMnode *pMnode, int32_t exceptDnodeId) { SArray *mndBuildDnodesArray(SMnode *pMnode, int32_t exceptDnodeId, SArray *dnodeList) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
int32_t numOfDnodes = mndGetDnodeSize(pMnode); int32_t numOfDnodes = mndGetDnodeSize(pMnode);
@ -752,7 +769,7 @@ SArray *mndBuildDnodesArray(SMnode *pMnode, int32_t exceptDnodeId) {
} }
sdbTraverse(pSdb, SDB_DNODE, mndResetDnodesArrayFp, NULL, NULL, NULL); sdbTraverse(pSdb, SDB_DNODE, mndResetDnodesArrayFp, NULL, NULL, NULL);
sdbTraverse(pSdb, SDB_DNODE, mndBuildDnodesArrayFp, pArray, &exceptDnodeId, NULL); sdbTraverse(pSdb, SDB_DNODE, mndBuildDnodesArrayFp, pArray, &exceptDnodeId, dnodeList);
mDebug("build %d dnodes array", (int32_t)taosArrayGetSize(pArray)); mDebug("build %d dnodes array", (int32_t)taosArrayGetSize(pArray));
for (int32_t i = 0; i < (int32_t)taosArrayGetSize(pArray); ++i) { for (int32_t i = 0; i < (int32_t)taosArrayGetSize(pArray); ++i) {
@ -845,7 +862,7 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup
int32_t mndAllocSmaVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup) { int32_t mndAllocSmaVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup) {
int32_t code = 0; int32_t code = 0;
SArray *pArray = mndBuildDnodesArray(pMnode, 0); SArray *pArray = mndBuildDnodesArray(pMnode, 0, NULL);
if (pArray == NULL) { if (pArray == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno; if (terrno != 0) code = terrno;
@ -868,7 +885,7 @@ int32_t mndAllocSmaVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup) {
return 0; return 0;
} }
int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups, SArray *dnodeList) {
int32_t code = -1; int32_t code = -1;
SArray *pArray = NULL; SArray *pArray = NULL;
SVgObj *pVgroups = NULL; SVgObj *pVgroups = NULL;
@ -879,7 +896,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
goto _OVER; goto _OVER;
} }
pArray = mndBuildDnodesArray(pMnode, 0); pArray = mndBuildDnodesArray(pMnode, 0, dnodeList);
if (pArray == NULL) { if (pArray == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno; if (terrno != 0) code = terrno;
@ -2062,7 +2079,7 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb,
int32_t mndSetMoveVgroupsInfoToTrans(SMnode *pMnode, STrans *pTrans, int32_t delDnodeId, bool force, bool unsafe) { int32_t mndSetMoveVgroupsInfoToTrans(SMnode *pMnode, STrans *pTrans, int32_t delDnodeId, bool force, bool unsafe) {
int32_t code = 0; int32_t code = 0;
SArray *pArray = mndBuildDnodesArray(pMnode, delDnodeId); SArray *pArray = mndBuildDnodesArray(pMnode, delDnodeId, NULL);
if (pArray == NULL) { if (pArray == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno; if (terrno != 0) code = terrno;
@ -3140,7 +3157,7 @@ int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgro
int32_t code = -1; int32_t code = -1;
STrans *pTrans = NULL; STrans *pTrans = NULL;
SDbObj dbObj = {0}; SDbObj dbObj = {0};
SArray *pArray = mndBuildDnodesArray(pMnode, 0); SArray *pArray = mndBuildDnodesArray(pMnode, 0, NULL);
int32_t numOfStreams = 0; int32_t numOfStreams = 0;
if ((code = mndGetNumOfStreams(pMnode, pDb->name, &numOfStreams)) != 0) { if ((code = mndGetNumOfStreams(pMnode, pDb->name, &numOfStreams)) != 0) {
@ -3508,7 +3525,7 @@ static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) {
sdbRelease(pMnode->pSdb, pDnode); sdbRelease(pMnode->pSdb, pDnode);
} }
pArray = mndBuildDnodesArray(pMnode, 0); pArray = mndBuildDnodesArray(pMnode, 0, NULL);
if (pArray == NULL) { if (pArray == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno; if (terrno != 0) code = terrno;

130
source/dnode/mnode/impl/test/arbgroup/arbgroup.cpp
View File

@ -80,17 +80,17 @@ TEST_F(ArbgroupTest, 01_encode_decode_sdb) {
SArbGroup* pNewGroup = (SArbGroup*)sdbGetRowObj(pRow); SArbGroup* pNewGroup = (SArbGroup*)sdbGetRowObj(pRow);
EXPECT_EQ(group.vgId, pNewGroup->vgId); ASSERT_EQ(group.vgId, pNewGroup->vgId);
EXPECT_EQ(group.dbUid, pNewGroup->dbUid); ASSERT_EQ(group.dbUid, pNewGroup->dbUid);
EXPECT_EQ(group.members[0].info.dnodeId, pNewGroup->members[0].info.dnodeId); ASSERT_EQ(group.members[0].info.dnodeId, pNewGroup->members[0].info.dnodeId);
EXPECT_EQ(group.members[1].info.dnodeId, pNewGroup->members[1].info.dnodeId); ASSERT_EQ(group.members[1].info.dnodeId, pNewGroup->members[1].info.dnodeId);
EXPECT_EQ(group.isSync, pNewGroup->isSync); ASSERT_EQ(group.isSync, pNewGroup->isSync);
EXPECT_EQ(group.assignedLeader.dnodeId, pNewGroup->assignedLeader.dnodeId); ASSERT_EQ(group.assignedLeader.dnodeId, pNewGroup->assignedLeader.dnodeId);
EXPECT_EQ(std::string(group.members[0].state.token), std::string(pNewGroup->members[0].state.token)); ASSERT_EQ(std::string(group.members[0].state.token), std::string(pNewGroup->members[0].state.token));
EXPECT_EQ(std::string(group.members[1].state.token), std::string(pNewGroup->members[1].state.token)); ASSERT_EQ(std::string(group.members[1].state.token), std::string(pNewGroup->members[1].state.token));
EXPECT_EQ(std::string(group.assignedLeader.token), std::string(pNewGroup->assignedLeader.token)); ASSERT_EQ(std::string(group.assignedLeader.token), std::string(pNewGroup->assignedLeader.token));
EXPECT_EQ(group.version, pNewGroup->version); ASSERT_EQ(group.version, pNewGroup->version);
taosMemoryFree(pRow); taosMemoryFree(pRow);
taosMemoryFree(pRaw); taosMemoryFree(pRaw);
@ -129,9 +129,9 @@ TEST_F(ArbgroupTest, 02_process_heart_beat_rsp) {
SArbGroup newGroup = {0}; SArbGroup newGroup = {0};
bool updateToken = mndUpdateArbGroupByHeartBeat(&group, &rspMember, nowMs, dnodeId, &newGroup); bool updateToken = mndUpdateArbGroupByHeartBeat(&group, &rspMember, nowMs, dnodeId, &newGroup);
EXPECT_FALSE(updateToken); ASSERT_EQ(updateToken, false);
EXPECT_NE(group.members[0].state.responsedHbSeq, rspMember.hbSeq); ASSERT_NE(group.members[0].state.responsedHbSeq, rspMember.hbSeq);
EXPECT_NE(group.members[0].state.lastHbMs, nowMs); ASSERT_NE(group.members[0].state.lastHbMs, nowMs);
} }
{ // old token { // old token
@ -144,9 +144,9 @@ TEST_F(ArbgroupTest, 02_process_heart_beat_rsp) {
SArbGroup newGroup = {0}; SArbGroup newGroup = {0};
bool updateToken = mndUpdateArbGroupByHeartBeat(&group, &rspMember, nowMs, dnodeId, &newGroup); bool updateToken = mndUpdateArbGroupByHeartBeat(&group, &rspMember, nowMs, dnodeId, &newGroup);
EXPECT_FALSE(updateToken); ASSERT_EQ(updateToken, false);
EXPECT_EQ(group.members[0].state.responsedHbSeq, rspMember.hbSeq); ASSERT_EQ(group.members[0].state.responsedHbSeq, rspMember.hbSeq);
EXPECT_EQ(group.members[0].state.lastHbMs, nowMs); ASSERT_EQ(group.members[0].state.lastHbMs, nowMs);
} }
{ // new token { // new token
@ -159,14 +159,14 @@ TEST_F(ArbgroupTest, 02_process_heart_beat_rsp) {
SArbGroup newGroup = {0}; SArbGroup newGroup = {0};
bool updateToken = mndUpdateArbGroupByHeartBeat(&group, &rspMember, nowMs, dnodeId, &newGroup); bool updateToken = mndUpdateArbGroupByHeartBeat(&group, &rspMember, nowMs, dnodeId, &newGroup);
EXPECT_TRUE(updateToken); ASSERT_EQ(updateToken, true);
EXPECT_EQ(group.members[0].state.responsedHbSeq, rspMember.hbSeq); ASSERT_EQ(group.members[0].state.responsedHbSeq, rspMember.hbSeq);
EXPECT_EQ(group.members[0].state.lastHbMs, nowMs); ASSERT_EQ(group.members[0].state.lastHbMs, nowMs);
EXPECT_EQ(std::string(newGroup.members[0].state.token), std::string(rspMember.memberToken)); ASSERT_EQ(std::string(newGroup.members[0].state.token), std::string(rspMember.memberToken));
EXPECT_FALSE(newGroup.isSync); ASSERT_EQ(newGroup.isSync, false);
EXPECT_EQ(newGroup.assignedLeader.dnodeId, 0); ASSERT_EQ(newGroup.assignedLeader.dnodeId, 0);
EXPECT_EQ(std::string(newGroup.assignedLeader.token).size(), 0); ASSERT_EQ(std::string(newGroup.assignedLeader.token).size(), 0);
} }
taosThreadMutexDestroy(&group.mutex); taosThreadMutexDestroy(&group.mutex);
@ -203,7 +203,7 @@ TEST_F(ArbgroupTest, 03_process_check_sync_rsp) {
SArbGroup newGroup = {0}; SArbGroup newGroup = {0};
bool updateIsSync = mndUpdateArbGroupByCheckSync(&group, vgId, member0Token, member1Token, newIsSync, &newGroup); bool updateIsSync = mndUpdateArbGroupByCheckSync(&group, vgId, member0Token, member1Token, newIsSync, &newGroup);
EXPECT_FALSE(updateIsSync); ASSERT_EQ(updateIsSync, false);
} }
{ // newIsSync { // newIsSync
@ -216,8 +216,8 @@ TEST_F(ArbgroupTest, 03_process_check_sync_rsp) {
SArbGroup newGroup = {0}; SArbGroup newGroup = {0};
bool updateIsSync = mndUpdateArbGroupByCheckSync(&group, vgId, member0Token, member1Token, newIsSync, &newGroup); bool updateIsSync = mndUpdateArbGroupByCheckSync(&group, vgId, member0Token, member1Token, newIsSync, &newGroup);
EXPECT_TRUE(updateIsSync); ASSERT_EQ(updateIsSync, true);
EXPECT_TRUE(newGroup.isSync); ASSERT_EQ(newGroup.isSync, true);
} }
taosThreadMutexDestroy(&group.mutex); taosThreadMutexDestroy(&group.mutex);
@ -254,7 +254,7 @@ TEST_F(ArbgroupTest, 04_process_set_assigned_leader){
SArbGroup newGroup = {0}; SArbGroup newGroup = {0};
bool updateAssigned = mndUpdateArbGroupBySetAssignedLeader(&group, vgId, memberToken, errcode, &newGroup); bool updateAssigned = mndUpdateArbGroupBySetAssignedLeader(&group, vgId, memberToken, errcode, &newGroup);
EXPECT_FALSE(updateAssigned); ASSERT_EQ(updateAssigned, false);
} }
{ // errcode != TSDB_CODE_SUCCESS { // errcode != TSDB_CODE_SUCCESS
@ -265,7 +265,7 @@ TEST_F(ArbgroupTest, 04_process_set_assigned_leader){
SArbGroup newGroup = {0}; SArbGroup newGroup = {0};
bool updateAssigned = mndUpdateArbGroupBySetAssignedLeader(&group, vgId, memberToken, errcode, &newGroup); bool updateAssigned = mndUpdateArbGroupBySetAssignedLeader(&group, vgId, memberToken, errcode, &newGroup);
EXPECT_FALSE(updateAssigned); ASSERT_EQ(updateAssigned, false);
} }
{ // errcode == TSDB_CODE_SUCCESS { // errcode == TSDB_CODE_SUCCESS
@ -276,11 +276,81 @@ TEST_F(ArbgroupTest, 04_process_set_assigned_leader){
SArbGroup newGroup = {0}; SArbGroup newGroup = {0};
bool updateAssigned = mndUpdateArbGroupBySetAssignedLeader(&group, vgId, memberToken, errcode, &newGroup); bool updateAssigned = mndUpdateArbGroupBySetAssignedLeader(&group, vgId, memberToken, errcode, &newGroup);
EXPECT_TRUE(updateAssigned); ASSERT_EQ(updateAssigned, true);
EXPECT_FALSE(newGroup.isSync); ASSERT_EQ(newGroup.isSync, false);
} }
taosThreadMutexDestroy(&group.mutex); taosThreadMutexDestroy(&group.mutex);
} }
TEST_F(ArbgroupTest, 05_check_sync_timer) {
const int32_t assgndDnodeId = 1;
const int32_t vgId = 5;
const int64_t nowMs = 173044838300;
SArbGroup group = {0};
group.vgId = vgId;
group.dbUid = 1234;
group.members[0].info.dnodeId = assgndDnodeId;
group.members[0].state.lastHbMs = nowMs - 10;
group.members[1].info.dnodeId = 2;
group.members[1].state.lastHbMs = nowMs - 10;
group.isSync = 1;
taosThreadMutexInit(&group.mutex, NULL);
SArbAssignedLeader assgnedLeader = {0};
assgnedLeader.dnodeId = assgndDnodeId;
assgnedLeader.acked = false;
strncpy(assgnedLeader.token, group.members[0].state.token, TSDB_ARB_TOKEN_SIZE);
SArbAssignedLeader noneAsgndLeader = {0};
noneAsgndLeader.dnodeId = 0;
noneAsgndLeader.acked = false;
ECheckSyncOp op = CHECK_SYNC_NONE;
SArbGroup newGroup = {0};
// 1. asgnd,sync,noAck --> send set assigned
group.assignedLeader = assgnedLeader;
group.assignedLeader.acked = false;
group.isSync = true;
mndArbCheckSync(&group, nowMs, &op, &newGroup);
ASSERT_EQ(op, CHECK_SYNC_SET_ASSIGNED_LEADER);
// 2. asgnd,notSync,noAck --> send set assgnd
newGroup = {0};
group.assignedLeader = assgnedLeader;
group.isSync = false;
group.assignedLeader.acked = false;
mndArbCheckSync(&group, nowMs, &op, &newGroup);
ASSERT_EQ(op, CHECK_SYNC_SET_ASSIGNED_LEADER);
// 3. noAsgnd,notSync,noAck(init) --> check sync
newGroup = {0};
group.assignedLeader = noneAsgndLeader;
group.isSync = false;
group.assignedLeader.acked = false;
mndArbCheckSync(&group, nowMs, &op, &newGroup);
ASSERT_EQ(op, CHECK_SYNC_CHECK_SYNC);
// 4. noAsgnd,sync,noAck,one timeout--> update arbgroup (asgnd,sync,noAck)
newGroup = {0};
group.assignedLeader = noneAsgndLeader;
group.isSync = true;
group.assignedLeader.acked = false;
group.members[1].state.lastHbMs = nowMs - 2 * tsArbSetAssignedTimeoutSec * 1000; // member1 timeout
mndArbCheckSync(&group, nowMs, &op, &newGroup);
ASSERT_EQ(op, CHECK_SYNC_UPDATE);
ASSERT_EQ(newGroup.assignedLeader.dnodeId, assgndDnodeId);
ASSERT_EQ(std::string(newGroup.assignedLeader.token), std::string(group.members[0].state.token));
ASSERT_EQ(newGroup.isSync, true);
ASSERT_EQ(newGroup.assignedLeader.acked, false);
}
#pragma GCC diagnostic pop #pragma GCC diagnostic pop

4
source/dnode/mnode/impl/test/profile/profile.cpp
View File

@ -39,7 +39,7 @@ TEST_F(MndTestProfile, 01_ConnectMsg) {
strcpy(connectReq.db, ""); strcpy(connectReq.db, "");
strcpy(connectReq.user, "root"); strcpy(connectReq.user, "root");
strcpy(connectReq.passwd, secretEncrypt); strcpy(connectReq.passwd, secretEncrypt);
strcpy(connectReq.sVer, version); strcpy(connectReq.sVer, td_version);
int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq);
void* pReq = rpcMallocCont(contLen); void* pReq = rpcMallocCont(contLen);
@ -76,7 +76,7 @@ TEST_F(MndTestProfile, 02_ConnectMsg_NotExistDB) {
strcpy(connectReq.db, "not_exist_db"); strcpy(connectReq.db, "not_exist_db");
strcpy(connectReq.user, "root"); strcpy(connectReq.user, "root");
strcpy(connectReq.passwd, secretEncrypt); strcpy(connectReq.passwd, secretEncrypt);
strcpy(connectReq.sVer, version); strcpy(connectReq.sVer, td_version);
int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq);
void* pReq = rpcMallocCont(contLen); void* pReq = rpcMallocCont(contLen);

2
source/dnode/mnode/impl/test/show/show.cpp
View File

@ -64,7 +64,7 @@ TEST_F(MndTestShow, 03_ShowMsg_Conn) {
strcpy(connectReq.db, ""); strcpy(connectReq.db, "");
strcpy(connectReq.user, "root"); strcpy(connectReq.user, "root");
strcpy(connectReq.passwd, secretEncrypt); strcpy(connectReq.passwd, secretEncrypt);
strcpy(connectReq.sVer, version); strcpy(connectReq.sVer, td_version);
int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq);
void* pReq = rpcMallocCont(contLen); void* pReq = rpcMallocCont(contLen);

9
source/dnode/snode/src/snode.c
View File

@ -38,24 +38,23 @@ int32_t sndBuildStreamTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProce
streamTaskOpenAllUpstreamInput(pTask); streamTaskOpenAllUpstreamInput(pTask);
streamTaskResetUpstreamStageInfo(pTask); streamTaskResetUpstreamStageInfo(pTask);
streamSetupScheduleTrigger(pTask);
SCheckpointInfo *pChkInfo = &pTask->chkInfo; SCheckpointInfo *pChkInfo = &pTask->chkInfo;
tqSetRestoreVersionInfo(pTask); tqSetRestoreVersionInfo(pTask);
char *p = streamTaskGetStatus(pTask).name; char *p = streamTaskGetStatus(pTask).name;
if (pTask->info.fillHistory) { if (pTask->info.fillHistory) {
sndInfo("vgId:%d build stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 sndInfo("vgId:%d build stream task, s-task:%s, %p checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64 " nextProcessVer:%" PRId64
" child id:%d, level:%d, status:%s fill-history:%d, related stream task:0x%x trigger:%" PRId64 " ms", " child id:%d, level:%d, status:%s fill-history:%d, related stream task:0x%x trigger:%" PRId64 " ms",
SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, SNODE_HANDLE, pTask->id.idStr, pTask, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory, pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory,
(int32_t)pTask->streamTaskId.taskId, pTask->info.delaySchedParam); (int32_t)pTask->streamTaskId.taskId, pTask->info.delaySchedParam);
} else { } else {
sndInfo("vgId:%d build stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 sndInfo("vgId:%d build stream task, s-task:%s, %p checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64 " nextProcessVer:%" PRId64
" child id:%d, level:%d, status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64 " ms", " child id:%d, level:%d, status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64 " ms",
SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, SNODE_HANDLE, pTask->id.idStr, pTask, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory, pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory,
(int32_t)pTask->hTaskInfo.id.taskId, pTask->info.delaySchedParam); (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.delaySchedParam);
} }

5
source/dnode/vnode/src/inc/tsdb.h
View File

@ -342,7 +342,10 @@ typedef struct {
rocksdb_writeoptions_t *writeoptions; rocksdb_writeoptions_t *writeoptions;
rocksdb_readoptions_t *readoptions; rocksdb_readoptions_t *readoptions;
rocksdb_writebatch_t *writebatch; rocksdb_writebatch_t *writebatch;
TdThreadMutex writeBatchMutex; TdThreadMutex writeBatchMutex;
int32_t sver;
tb_uid_t suid;
tb_uid_t uid;
STSchema *pTSchema; STSchema *pTSchema;
} SRocksCache; } SRocksCache;

1
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -222,6 +222,7 @@ int32_t tsdbCacheNewSTableColumn(STsdb* pTsdb, SArray* uids, int16_t cid, int8_t
int32_t tsdbCacheDropSTableColumn(STsdb* pTsdb, SArray* uids, int16_t cid, bool hasPrimayKey); int32_t tsdbCacheDropSTableColumn(STsdb* pTsdb, SArray* uids, int16_t cid, bool hasPrimayKey);
int32_t tsdbCacheNewNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, int8_t col_type); int32_t tsdbCacheNewNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, int8_t col_type);
int32_t tsdbCacheDropNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, bool hasPrimayKey); int32_t tsdbCacheDropNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, bool hasPrimayKey);
void tsdbCacheInvalidateSchema(STsdb* pTsdb, tb_uid_t suid, tb_uid_t uid, int32_t sver);
int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg);
int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp);
int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows); int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows);

6
source/dnode/vnode/src/meta/metaTable.c
View File

@ -620,6 +620,8 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
} }
} }
if (uids) taosArrayDestroy(uids); if (uids) taosArrayDestroy(uids);
tsdbCacheInvalidateSchema(pTsdb, pReq->suid, -1, pReq->schemaRow.version);
} }
metaWLock(pMeta); metaWLock(pMeta);
@ -1945,6 +1947,10 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
break; break;
} }
if (!TSDB_CACHE_NO(pMeta->pVnode->config)) {
tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, entry.uid, pSchema->version);
}
entry.version = version; entry.version = version;
// do actual write // do actual write

15
source/dnode/vnode/src/sma/smaRollup.c
View File

@ -238,13 +238,18 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui
} }
static void tdRSmaTaskInit(SStreamMeta *pMeta, SRSmaInfoItem *pItem, SStreamTaskId *pId) { static void tdRSmaTaskInit(SStreamMeta *pMeta, SRSmaInfoItem *pItem, SStreamTaskId *pId) {
STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId};
SStreamTask *pTask = NULL;
streamMetaRLock(pMeta); streamMetaRLock(pMeta);
SStreamTask **ppTask = (SStreamTask **)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
if (ppTask && *ppTask) { int32_t code = streamMetaAcquireTaskUnsafe(pMeta, &id, &pTask);
pItem->submitReqVer = (*ppTask)->chkInfo.checkpointVer; if (code == 0) {
pItem->fetchResultVer = (*ppTask)->info.delaySchedParam; pItem->submitReqVer = pTask->chkInfo.checkpointVer;
pItem->fetchResultVer = pTask->info.delaySchedParam;
streamMetaReleaseTask(pMeta, pTask);
} }
streamMetaRUnLock(pMeta); streamMetaRUnLock(pMeta);
} }

9
source/dnode/vnode/src/tq/tq.c
View File

@ -765,7 +765,6 @@ int32_t tqBuildStreamTask(void* pTqObj, SStreamTask* pTask, int64_t nextProcessV
} }
streamTaskResetUpstreamStageInfo(pTask); streamTaskResetUpstreamStageInfo(pTask);
streamSetupScheduleTrigger(pTask);
SCheckpointInfo* pChkInfo = &pTask->chkInfo; SCheckpointInfo* pChkInfo = &pTask->chkInfo;
tqSetRestoreVersionInfo(pTask); tqSetRestoreVersionInfo(pTask);
@ -774,19 +773,19 @@ int32_t tqBuildStreamTask(void* pTqObj, SStreamTask* pTask, int64_t nextProcessV
const char* pNext = streamTaskGetStatusStr(pTask->status.taskStatus); const char* pNext = streamTaskGetStatusStr(pTask->status.taskStatus);
if (pTask->info.fillHistory) { if (pTask->info.fillHistory) {
tqInfo("vgId:%d build stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 tqInfo("vgId:%d build stream task, s-task:%s, %p checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64 " nextProcessVer:%" PRId64
" child id:%d, level:%d, cur-status:%s, next-status:%s fill-history:%d, related stream task:0x%x " " child id:%d, level:%d, cur-status:%s, next-status:%s fill-history:%d, related stream task:0x%x "
"delaySched:%" PRId64 " ms, inputVer:%" PRId64, "delaySched:%" PRId64 " ms, inputVer:%" PRId64,
vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, vgId, pTask->id.idStr, pTask, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
pTask->info.selfChildId, pTask->info.taskLevel, p, pNext, pTask->info.fillHistory, pTask->info.selfChildId, pTask->info.taskLevel, p, pNext, pTask->info.fillHistory,
(int32_t)pTask->streamTaskId.taskId, pTask->info.delaySchedParam, nextProcessVer); (int32_t)pTask->streamTaskId.taskId, pTask->info.delaySchedParam, nextProcessVer);
} else { } else {
tqInfo("vgId:%d build stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 tqInfo("vgId:%d build stream task, s-task:%s, %p checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64 " nextProcessVer:%" PRId64
" child id:%d, level:%d, cur-status:%s next-status:%s fill-history:%d, related fill-task:0x%x " " child id:%d, level:%d, cur-status:%s next-status:%s fill-history:%d, related fill-task:0x%x "
"delaySched:%" PRId64 " ms, inputVer:%" PRId64, "delaySched:%" PRId64 " ms, inputVer:%" PRId64,
vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, vgId, pTask->id.idStr, pTask, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
pTask->info.selfChildId, pTask->info.taskLevel, p, pNext, pTask->info.fillHistory, pTask->info.selfChildId, pTask->info.taskLevel, p, pNext, pTask->info.fillHistory,
(int32_t)pTask->hTaskInfo.id.taskId, pTask->info.delaySchedParam, nextProcessVer); (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.delaySchedParam, nextProcessVer);

20
source/dnode/vnode/src/tq/tqRead.c
View File

@ -1113,12 +1113,20 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
break; break;
} }
SStreamTask* pTask = *(SStreamTask**)pIter; int64_t refId = *(int64_t*)pIter;
if ((pTask->info.taskLevel == TASK_LEVEL__SOURCE) && (pTask->exec.pExecutor != NULL)) { SStreamTask* pTask = taosAcquireRef(streamTaskRefPool, refId);
int32_t code = qUpdateTableListForStreamScanner(pTask->exec.pExecutor, tbUidList, isAdd); if (pTask != NULL) {
if (code != 0) { int32_t taskId = pTask->id.taskId;
tqError("vgId:%d, s-task:%s update qualified table error for stream task", vgId, pTask->id.idStr);
continue; if ((pTask->info.taskLevel == TASK_LEVEL__SOURCE) && (pTask->exec.pExecutor != NULL)) {
int32_t code = qUpdateTableListForStreamScanner(pTask->exec.pExecutor, tbUidList, isAdd);
if (code != 0) {
tqError("vgId:%d, s-task:0x%x update qualified table error for stream task", vgId, taskId);
}
}
int32_t ret = taosReleaseRef(streamTaskRefPool, refId);
if (ret) {
tqError("vgId:%d release task refId failed, refId:%" PRId64, vgId, refId);
} }
} }
} }

4
source/dnode/vnode/src/tq/tqStreamTask.c
View File

@ -79,7 +79,7 @@ static void doStartScanWal(void* param, void* tmrId) {
SBuildScanWalMsgParam* pParam = (SBuildScanWalMsgParam*)param; SBuildScanWalMsgParam* pParam = (SBuildScanWalMsgParam*)param;
SStreamMeta* pMeta = taosAcquireRef(streamMetaId, pParam->metaId); SStreamMeta* pMeta = taosAcquireRef(streamMetaRefPool, pParam->metaId);
if (pMeta == NULL) { if (pMeta == NULL) {
tqError("metaRid:%" PRId64 " not valid now, stream meta has been freed", pParam->metaId); tqError("metaRid:%" PRId64 " not valid now, stream meta has been freed", pParam->metaId);
taosMemoryFree(pParam); taosMemoryFree(pParam);
@ -97,7 +97,7 @@ static void doStartScanWal(void* param, void* tmrId) {
tqError("vgId:%d failed sched task to scan wal, code:%s", vgId, tstrerror(code)); tqError("vgId:%d failed sched task to scan wal, code:%s", vgId, tstrerror(code));
} }
code = taosReleaseRef(streamMetaId, pParam->metaId); code = taosReleaseRef(streamMetaRefPool, pParam->metaId);
if (code) { if (code) {
tqError("vgId:%d failed to release ref for streamMeta, rid:%" PRId64 " code:%s", vgId, pParam->metaId, tqError("vgId:%d failed to release ref for streamMeta, rid:%" PRId64 " code:%s", vgId, pParam->metaId,
tstrerror(code)); tstrerror(code));

14
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -685,19 +685,21 @@ int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, b
continue; continue;
} }
STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId};
SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); SStreamTask* pTask = NULL;
if (ppTask == NULL) {
code = streamMetaAcquireTaskUnsafe(pMeta, &id, &pTask);
if (code != 0) {
tqError("vgId:%d failed to acquire task:0x%x in retrieving progress", pMeta->vgId, pId->taskId); tqError("vgId:%d failed to acquire task:0x%x in retrieving progress", pMeta->vgId, pId->taskId);
continue; continue;
} }
if ((*ppTask)->info.taskLevel != TASK_LEVEL__SOURCE) { if (pTask->info.taskLevel != TASK_LEVEL__SOURCE) {
streamMetaReleaseTask(pMeta, pTask);
continue; continue;
} }
// here we get the required stream source task // here we get the required stream source task
SStreamTask* pTask = *ppTask;
*fhFinished = !HAS_RELATED_FILLHISTORY_TASK(pTask); *fhFinished = !HAS_RELATED_FILLHISTORY_TASK(pTask);
int64_t ver = walReaderGetCurrentVer(pTask->exec.pWalReader); int64_t ver = walReaderGetCurrentVer(pTask->exec.pWalReader);
@ -713,6 +715,7 @@ int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, b
SWalReader* pReader = walOpenReader(pTask->exec.pWalReader->pWal, NULL, 0); SWalReader* pReader = walOpenReader(pTask->exec.pWalReader->pWal, NULL, 0);
if (pReader == NULL) { if (pReader == NULL) {
tqError("failed to open wal reader to extract exec progress, vgId:%d", pMeta->vgId); tqError("failed to open wal reader to extract exec progress, vgId:%d", pMeta->vgId);
streamMetaReleaseTask(pMeta, pTask);
continue; continue;
} }
@ -738,6 +741,7 @@ int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, b
} }
walCloseReader(pReader); walCloseReader(pReader);
streamMetaReleaseTask(pMeta, pTask);
} }
streamMetaRUnLock(pMeta); streamMetaRUnLock(pMeta);

98
source/dnode/vnode/src/tqCommon/tqCommon.c
View File

@ -138,13 +138,15 @@ int32_t tqStreamStartOneTaskAsync(SStreamMeta* pMeta, SMsgCb* cb, int64_t stream
// this is to process request from transaction, always return true. // this is to process request from transaction, always return true.
int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored) { int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored) {
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
int32_t len = pMsg->contLen - sizeof(SMsgHead); int32_t len = pMsg->contLen - sizeof(SMsgHead);
SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS}; SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS};
int64_t st = taosGetTimestampMs(); int64_t st = taosGetTimestampMs();
bool updated = false; bool updated = false;
int32_t code = 0; int32_t code = 0;
SStreamTask* pTask = NULL;
SStreamTask* pHTask = NULL;
SStreamTaskNodeUpdateMsg req = {0}; SStreamTaskNodeUpdateMsg req = {0};
@ -170,9 +172,9 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
streamMetaWLock(pMeta); streamMetaWLock(pMeta);
// the task epset may be updated again and again, when replaying the WAL, the task may be in stop status. // the task epset may be updated again and again, when replaying the WAL, the task may be in stop status.
STaskId id = {.streamId = req.streamId, .taskId = req.taskId}; STaskId id = {.streamId = req.streamId, .taskId = req.taskId};
SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); code = streamMetaAcquireTaskUnsafe(pMeta, &id, &pTask);
if (ppTask == NULL || *ppTask == NULL) { if (code != 0) {
tqError("vgId:%d failed to acquire task:0x%x when handling update task epset, it may have been dropped", vgId, tqError("vgId:%d failed to acquire task:0x%x when handling update task epset, it may have been dropped", vgId,
req.taskId); req.taskId);
rsp.code = TSDB_CODE_SUCCESS; rsp.code = TSDB_CODE_SUCCESS;
@ -181,12 +183,13 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
return rsp.code; return rsp.code;
} }
SStreamTask* pTask = *ppTask; const char* idstr = pTask->id.idStr;
const char* idstr = pTask->id.idStr;
if (req.transId <= 0) { if (req.transId <= 0) {
tqError("vgId:%d invalid update nodeEp task, transId:%d, discard", vgId, req.taskId); tqError("vgId:%d invalid update nodeEp task, transId:%d, discard", vgId, req.taskId);
rsp.code = TSDB_CODE_SUCCESS; rsp.code = TSDB_CODE_SUCCESS;
streamMetaReleaseTask(pMeta, pTask);
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
taosArrayDestroy(req.pNodeList); taosArrayDestroy(req.pNodeList);
@ -197,6 +200,8 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
bool update = streamMetaInitUpdateTaskList(pMeta, req.transId); bool update = streamMetaInitUpdateTaskList(pMeta, req.transId);
if (!update) { if (!update) {
rsp.code = TSDB_CODE_SUCCESS; rsp.code = TSDB_CODE_SUCCESS;
streamMetaReleaseTask(pMeta, pTask);
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
taosArrayDestroy(req.pNodeList); taosArrayDestroy(req.pNodeList);
@ -211,7 +216,10 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
tqDebug("s-task:%s (vgId:%d) already update in transId:%d, discard the nodeEp update msg", idstr, vgId, tqDebug("s-task:%s (vgId:%d) already update in transId:%d, discard the nodeEp update msg", idstr, vgId,
req.transId); req.transId);
rsp.code = TSDB_CODE_SUCCESS; rsp.code = TSDB_CODE_SUCCESS;
streamMetaReleaseTask(pMeta, pTask);
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
taosArrayDestroy(req.pNodeList); taosArrayDestroy(req.pNodeList);
return rsp.code; return rsp.code;
} }
@ -227,24 +235,23 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
streamTaskStopMonitorCheckRsp(&pTask->taskCheckInfo, pTask->id.idStr); streamTaskStopMonitorCheckRsp(&pTask->taskCheckInfo, pTask->id.idStr);
SStreamTask** ppHTask = NULL;
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
ppHTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id)); code = streamMetaAcquireTaskUnsafe(pMeta, &pTask->hTaskInfo.id, &pHTask);
if (ppHTask == NULL || *ppHTask == NULL) { if (code != 0) {
tqError( tqError(
"vgId:%d failed to acquire fill-history task:0x%x when handling update, may have been dropped already, rel " "vgId:%d failed to acquire fill-history task:0x%x when handling update, may have been dropped already, rel "
"stream task:0x%x", "stream task:0x%x",
vgId, (uint32_t)pTask->hTaskInfo.id.taskId, req.taskId); vgId, (uint32_t)pTask->hTaskInfo.id.taskId, req.taskId);
CLEAR_RELATED_FILLHISTORY_TASK(pTask); CLEAR_RELATED_FILLHISTORY_TASK(pTask);
} else { } else {
tqDebug("s-task:%s fill-history task update nodeEp along with stream task", (*ppHTask)->id.idStr); tqDebug("s-task:%s fill-history task update nodeEp along with stream task", pHTask->id.idStr);
bool updateEpSet = streamTaskUpdateEpsetInfo(*ppHTask, req.pNodeList); bool updateEpSet = streamTaskUpdateEpsetInfo(pHTask, req.pNodeList);
if (updateEpSet) { if (updateEpSet) {
updated = updateEpSet; updated = updateEpSet;
} }
streamTaskResetStatus(*ppHTask); streamTaskResetStatus(pHTask);
streamTaskStopMonitorCheckRsp(&(*ppHTask)->taskCheckInfo, (*ppHTask)->id.idStr); streamTaskStopMonitorCheckRsp(&pHTask->taskCheckInfo, pHTask->id.idStr);
} }
} }
@ -256,8 +263,8 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
tqError("s-task:%s vgId:%d failed to save task, code:%s", idstr, vgId, tstrerror(code)); tqError("s-task:%s vgId:%d failed to save task, code:%s", idstr, vgId, tstrerror(code));
} }
if (ppHTask != NULL) { if (pHTask != NULL) {
code = streamMetaSaveTask(pMeta, *ppHTask); code = streamMetaSaveTask(pMeta, pHTask);
if (code) { if (code) {
tqError("s-task:%s vgId:%d failed to save related history task, code:%s", idstr, vgId, tstrerror(code)); tqError("s-task:%s vgId:%d failed to save related history task, code:%s", idstr, vgId, tstrerror(code));
} }
@ -271,15 +278,17 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
tqError("s-task:%s vgId:%d failed to stop task, code:%s", idstr, vgId, tstrerror(code)); tqError("s-task:%s vgId:%d failed to stop task, code:%s", idstr, vgId, tstrerror(code));
} }
if (ppHTask != NULL) { if (pHTask != NULL) {
code = streamTaskStop(*ppHTask); code = streamTaskStop(pHTask);
if (code) { if (code) {
tqError("s-task:%s vgId:%d failed to stop related history task, code:%s", idstr, vgId, tstrerror(code)); tqError("s-task:%s vgId:%d failed to stop related history task, code:%s", idstr, vgId, tstrerror(code));
} }
} }
// keep info // keep info
streamMetaAddIntoUpdateTaskList(pMeta, pTask, (ppHTask != NULL) ? (*ppHTask) : NULL, req.transId, st); streamMetaAddIntoUpdateTaskList(pMeta, pTask, (pHTask != NULL) ? (pHTask) : NULL, req.transId, st);
streamMetaReleaseTask(pMeta, pTask);
streamMetaReleaseTask(pMeta, pHTask);
rsp.code = TSDB_CODE_SUCCESS; rsp.code = TSDB_CODE_SUCCESS;
@ -643,7 +652,6 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve
if (code < 0) { if (code < 0) {
tqError("failed to add s-task:0x%x into vgId:%d meta, existed:%d, code:%s", vgId, taskId, numOfTasks, tqError("failed to add s-task:0x%x into vgId:%d meta, existed:%d, code:%s", vgId, taskId, numOfTasks,
tstrerror(code)); tstrerror(code));
tFreeStreamTask(pTask);
return code; return code;
} }
@ -673,7 +681,6 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve
} }
} else { } else {
tqWarn("vgId:%d failed to add s-task:0x%x, since already exists in meta store, total:%d", vgId, taskId, numOfTasks); tqWarn("vgId:%d failed to add s-task:0x%x, since already exists in meta store, total:%d", vgId, taskId, numOfTasks);
tFreeStreamTask(pTask);
} }
return code; return code;
@ -681,25 +688,25 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve
int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) { int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) {
SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg;
int32_t code = 0;
int32_t vgId = pMeta->vgId;
STaskId hTaskId = {0};
SStreamTask* pTask = NULL;
int32_t code = 0;
int32_t vgId = pMeta->vgId;
STaskId hTaskId = {0};
tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId); tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId);
streamMetaWLock(pMeta); streamMetaWLock(pMeta);
STaskId id = {.streamId = pReq->streamId, .taskId = pReq->taskId}; STaskId id = {.streamId = pReq->streamId, .taskId = pReq->taskId};
SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); code = streamMetaAcquireTaskUnsafe(pMeta, &id, &pTask);
if ((ppTask != NULL) && ((*ppTask) != NULL)) { if (code == 0) {
int32_t unusedRetRef = streamMetaAcquireOneTask(*ppTask);
SStreamTask* pTask = *ppTask;
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
hTaskId.streamId = pTask->hTaskInfo.id.streamId; hTaskId.streamId = pTask->hTaskInfo.id.streamId;
hTaskId.taskId = pTask->hTaskInfo.id.taskId; hTaskId.taskId = pTask->hTaskInfo.id.taskId;
} }
// clear the relationship, and then release the stream tasks, to avoid invalid accessing of already freed
// related stream(history) task
streamTaskSetRemoveBackendFiles(pTask); streamTaskSetRemoveBackendFiles(pTask);
code = streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt); code = streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
@ -742,18 +749,19 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored, char* msg) { int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored, char* msg) {
SVUpdateCheckpointInfoReq* pReq = (SVUpdateCheckpointInfoReq*)msg; SVUpdateCheckpointInfoReq* pReq = (SVUpdateCheckpointInfoReq*)msg;
int32_t code = 0;
int32_t vgId = pMeta->vgId;
SStreamTask* pTask = NULL;
int32_t code = 0;
int32_t vgId = pMeta->vgId;
tqDebug("vgId:%d receive msg to update-checkpoint-info for s-task:0x%x", vgId, pReq->taskId); tqDebug("vgId:%d receive msg to update-checkpoint-info for s-task:0x%x", vgId, pReq->taskId);
streamMetaWLock(pMeta); streamMetaWLock(pMeta);
STaskId id = {.streamId = pReq->streamId, .taskId = pReq->taskId}; STaskId id = {.streamId = pReq->streamId, .taskId = pReq->taskId};
SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); code = streamMetaAcquireTaskUnsafe(pMeta, &id, &pTask);
if (code == 0) {
if (ppTask != NULL && (*ppTask) != NULL) { code = streamTaskUpdateTaskCheckpointInfo(pTask, restored, pReq);
code = streamTaskUpdateTaskCheckpointInfo(*ppTask, restored, pReq); streamMetaReleaseTask(pMeta, pTask);
} else { // failed to get the task. } else { // failed to get the task.
int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
tqError( tqError(
@ -763,7 +771,6 @@ int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored
} }
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
// always return success when handling the requirement issued by mnode during transaction. // always return success when handling the requirement issued by mnode during transaction.
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -789,11 +796,6 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
tqInfo("vgId:%d tasks are all updated and stopped, restart all tasks, triggered by transId:%d, ts:%" PRId64, vgId, tqInfo("vgId:%d tasks are all updated and stopped, restart all tasks, triggered by transId:%d, ts:%" PRId64, vgId,
pMeta->updateInfo.completeTransId, pMeta->updateInfo.completeTs); pMeta->updateInfo.completeTransId, pMeta->updateInfo.completeTs);
while (streamMetaTaskInTimer(pMeta)) {
tqDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId);
taosMsleep(100);
}
streamMetaWLock(pMeta); streamMetaWLock(pMeta);
streamMetaClear(pMeta); streamMetaClear(pMeta);

97
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -221,7 +221,7 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) {
rocksdb_writebatch_t *writebatch = rocksdb_writebatch_create(); rocksdb_writebatch_t *writebatch = rocksdb_writebatch_create();
TAOS_CHECK_GOTO(taosThreadMutexInit(&pTsdb->rCache.writeBatchMutex, NULL), &lino, _err6) ; TAOS_CHECK_GOTO(taosThreadMutexInit(&pTsdb->rCache.writeBatchMutex, NULL), &lino, _err6);
pTsdb->rCache.writebatch = writebatch; pTsdb->rCache.writebatch = writebatch;
pTsdb->rCache.my_comparator = cmp; pTsdb->rCache.my_comparator = cmp;
@ -230,6 +230,9 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) {
pTsdb->rCache.readoptions = readoptions; pTsdb->rCache.readoptions = readoptions;
pTsdb->rCache.flushoptions = flushoptions; pTsdb->rCache.flushoptions = flushoptions;
pTsdb->rCache.db = db; pTsdb->rCache.db = db;
pTsdb->rCache.sver = -1;
pTsdb->rCache.suid = -1;
pTsdb->rCache.uid = -1;
pTsdb->rCache.pTSchema = NULL; pTsdb->rCache.pTSchema = NULL;
TAOS_RETURN(code); TAOS_RETURN(code);
@ -1132,19 +1135,17 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray
(void)taosThreadMutexLock(&pTsdb->lruMutex); (void)taosThreadMutexLock(&pTsdb->lruMutex);
for (int i = 0; i < num_keys; ++i) { for (int i = 0; i < num_keys; ++i) {
SLastUpdateCtx *updCtx = (SLastUpdateCtx *)taosArrayGet(updCtxArray, i); SLastUpdateCtx *updCtx = &((SLastUpdateCtx *)TARRAY_DATA(updCtxArray))[i];
int8_t lflag = updCtx->lflag;
int8_t lflag = updCtx->lflag; SRowKey *pRowKey = &updCtx->tsdbRowKey.key;
SRowKey *pRowKey = &updCtx->tsdbRowKey.key; SColVal *pColVal = &updCtx->colVal;
SColVal *pColVal = &updCtx->colVal;
if (lflag == LFLAG_LAST && !COL_VAL_IS_VALUE(pColVal)) { if (lflag == LFLAG_LAST && !COL_VAL_IS_VALUE(pColVal)) {
continue; continue;
} }
SLastKey *key = &(SLastKey){.lflag = lflag, .uid = uid, .cid = pColVal->cid}; SLastKey *key = &(SLastKey){.lflag = lflag, .uid = uid, .cid = pColVal->cid};
size_t klen = ROCKS_KEY_LEN; LRUHandle *h = taosLRUCacheLookup(pCache, key, ROCKS_KEY_LEN);
LRUHandle *h = taosLRUCacheLookup(pCache, key, klen);
if (h) { if (h) {
SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pCache, h); SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pCache, h);
if (pLastCol->cacheStatus != TSDB_LAST_CACHE_NO_CACHE) { if (pLastCol->cacheStatus != TSDB_LAST_CACHE_NO_CACHE) {
@ -1299,53 +1300,94 @@ _exit:
TAOS_RETURN(code); TAOS_RETURN(code);
} }
void tsdbCacheInvalidateSchema(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int32_t sver) {
SRocksCache *pRCache = &pTsdb->rCache;
if (!pRCache->pTSchema || sver <= pTsdb->rCache.sver) return;
if (suid > 0 && suid == pRCache->suid) {
pRCache->sver = -1;
pRCache->suid = -1;
}
if (suid == 0 && uid == pRCache->uid) {
pRCache->sver = -1;
pRCache->uid = -1;
}
}
static int32_t tsdbUpdateSkm(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int32_t sver) {
SRocksCache *pRCache = &pTsdb->rCache;
if (pRCache->pTSchema && sver == pRCache->sver) {
if (suid > 0 && suid == pRCache->suid) {
return 0;
}
if (suid == 0 && uid == pRCache->uid) {
return 0;
}
}
pRCache->suid = suid;
pRCache->uid = uid;
pRCache->sver = sver;
tDestroyTSchema(pRCache->pTSchema);
return metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, -1, &pRCache->pTSchema);
}
int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int64_t version, int32_t nRow, int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int64_t version, int32_t nRow,
SRow **aRow) { SRow **aRow) {
int32_t code = 0, lino = 0; int32_t code = 0, lino = 0;
// 1. prepare last // 1. prepare last
TSDBROW lRow = {.type = TSDBROW_ROW_FMT, .pTSRow = aRow[nRow - 1], .version = version}; TSDBROW lRow = {.type = TSDBROW_ROW_FMT, .pTSRow = aRow[nRow - 1], .version = version};
STSchema *pTSchema = NULL; STSchema *pTSchema = NULL;
int32_t sver = TSDBROW_SVERSION(&lRow); int32_t sver = TSDBROW_SVERSION(&lRow);
SArray *ctxArray = NULL; SArray *ctxArray = NULL;
SSHashObj *iColHash = NULL; SSHashObj *iColHash = NULL;
TAOS_CHECK_GOTO(metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema), &lino, _exit); TAOS_CHECK_GOTO(tsdbUpdateSkm(pTsdb, suid, uid, sver), &lino, _exit);
pTSchema = pTsdb->rCache.pTSchema;
TSDBROW tRow = {.type = TSDBROW_ROW_FMT, .version = version}; TSDBROW tRow = {.type = TSDBROW_ROW_FMT, .version = version};
int32_t nCol = pTSchema->numOfCols; int32_t nCol = pTSchema->numOfCols;
ctxArray = taosArrayInit(nCol, sizeof(SLastUpdateCtx)); ctxArray = taosArrayInit(nCol * 2, sizeof(SLastUpdateCtx));
iColHash = tSimpleHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT)); if (ctxArray == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
// 1. prepare by lrow // 1. prepare by lrow
STsdbRowKey tsdbRowKey = {0}; STsdbRowKey tsdbRowKey = {0};
tsdbRowGetKey(&lRow, &tsdbRowKey); tsdbRowGetKey(&lRow, &tsdbRowKey);
STSDBRowIter iter = {0}; STSDBRowIter iter = {0};
code = tsdbRowIterOpen(&iter, &lRow, pTSchema); TAOS_CHECK_GOTO(tsdbRowIterOpen(&iter, &lRow, pTSchema), &lino, _exit);
if (code != TSDB_CODE_SUCCESS) {
tsdbError("vgId:%d, %s tsdbRowIterOpen failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__,
tstrerror(code));
TAOS_CHECK_GOTO(code, &lino, _exit);
}
int32_t iCol = 0; int32_t iCol = 0;
for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal && iCol < nCol; pColVal = tsdbRowIterNext(&iter), iCol++) { for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal && iCol < nCol; pColVal = tsdbRowIterNext(&iter), iCol++) {
SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST_ROW, .tsdbRowKey = tsdbRowKey, .colVal = *pColVal}; SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST_ROW, .tsdbRowKey = tsdbRowKey, .colVal = *pColVal};
if (!taosArrayPush(ctxArray, &updateCtx)) { if (!taosArrayPush(ctxArray, &updateCtx)) {
tsdbRowClose(&iter);
TAOS_CHECK_GOTO(terrno, &lino, _exit); TAOS_CHECK_GOTO(terrno, &lino, _exit);
} }
if (!COL_VAL_IS_VALUE(pColVal)) { if (COL_VAL_IS_VALUE(pColVal)) {
updateCtx.lflag = LFLAG_LAST;
if (!taosArrayPush(ctxArray, &updateCtx)) {
tsdbRowClose(&iter);
TAOS_CHECK_GOTO(terrno, &lino, _exit);
}
} else {
if (!iColHash) {
iColHash = tSimpleHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT));
if (iColHash == NULL) {
tsdbRowClose(&iter);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
}
if (tSimpleHashPut(iColHash, &iCol, sizeof(iCol), NULL, 0)) { if (tSimpleHashPut(iColHash, &iCol, sizeof(iCol), NULL, 0)) {
tsdbRowClose(&iter);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
} }
continue;
}
updateCtx.lflag = LFLAG_LAST;
if (!taosArrayPush(ctxArray, &updateCtx)) {
TAOS_CHECK_GOTO(terrno, &lino, _exit);
} }
} }
tsdbRowClose(&iter); tsdbRowClose(&iter);
@ -1390,7 +1432,10 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6
} }
_exit: _exit:
taosMemoryFreeClear(pTSchema); if (code) {
tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code));
}
taosArrayDestroy(ctxArray); taosArrayDestroy(ctxArray);
tSimpleHashCleanup(iColHash); tSimpleHashCleanup(iColHash);

7
source/libs/executor/inc/executorInt.h
View File

@ -553,6 +553,7 @@ typedef struct SIntervalAggOperatorInfo {
EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model]
STimeWindowAggSupp twAggSup; STimeWindowAggSupp twAggSup;
SArray* pPrevValues; // SArray<SGroupKeys> used to keep the previous not null value for interpolation. SArray* pPrevValues; // SArray<SGroupKeys> used to keep the previous not null value for interpolation.
bool cleanGroupResInfo;
struct SOperatorInfo* pOperator; struct SOperatorInfo* pOperator;
// for limit optimization // for limit optimization
bool limited; bool limited;
@ -831,8 +832,10 @@ void cleanupExprSupp(SExprSupp* pSup);
void cleanupResultInfoInStream(SExecTaskInfo* pTaskInfo, void* pState, SExprSupp* pSup, void cleanupResultInfoInStream(SExecTaskInfo* pTaskInfo, void* pState, SExprSupp* pSup,
SGroupResInfo* pGroupResInfo); SGroupResInfo* pGroupResInfo);
void cleanupResultInfo(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SDiskbasedBuf* pBuf, void cleanupResultInfoInHashMap(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SDiskbasedBuf* pBuf,
SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap); SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap);
void cleanupResultInfo(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SGroupResInfo* pGroupResInfo,
SAggSupporter *pAggSup, bool cleanHashmap);
void cleanupResultInfoWithoutHash(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SDiskbasedBuf* pBuf, void cleanupResultInfoWithoutHash(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SDiskbasedBuf* pBuf,
SGroupResInfo* pGroupResInfo); SGroupResInfo* pGroupResInfo);

2
source/libs/executor/inc/operator.h
View File

@ -85,6 +85,8 @@ int32_t createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNo
int32_t createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableListInfo* pTableList, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo); int32_t createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableListInfo* pTableList, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);
int32_t createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo);
int32_t createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo); int32_t createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);
int32_t createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SNode* pTagCond, SNode*pTagIndexCond, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo); int32_t createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SNode* pTagCond, SNode*pTagIndexCond, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);

20
source/libs/executor/src/aggregateoperator.c
View File

@ -49,6 +49,7 @@ typedef struct SAggOperatorInfo {
SSDataBlock* pNewGroupBlock; SSDataBlock* pNewGroupBlock;
bool hasCountFunc; bool hasCountFunc;
SOperatorInfo* pOperator; SOperatorInfo* pOperator;
bool cleanGroupResInfo;
} SAggOperatorInfo; } SAggOperatorInfo;
static void destroyAggOperatorInfo(void* param); static void destroyAggOperatorInfo(void* param);
@ -121,6 +122,7 @@ int32_t createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pA
pInfo->binfo.outputTsOrder = pAggNode->node.outputTsOrder; pInfo->binfo.outputTsOrder = pAggNode->node.outputTsOrder;
pInfo->hasCountFunc = pAggNode->hasCountLikeFunc; pInfo->hasCountFunc = pAggNode->hasCountLikeFunc;
pInfo->pOperator = pOperator; pInfo->pOperator = pOperator;
pInfo->cleanGroupResInfo = false;
setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG,
!pAggNode->node.forceCreateNonBlockingOptr, OP_NOT_OPENED, pInfo, pTaskInfo); !pAggNode->node.forceCreateNonBlockingOptr, OP_NOT_OPENED, pInfo, pTaskInfo);
@ -159,8 +161,8 @@ void destroyAggOperatorInfo(void* param) {
cleanupBasicInfo(&pInfo->binfo); cleanupBasicInfo(&pInfo->binfo);
if (pInfo->pOperator) { if (pInfo->pOperator) {
cleanupResultInfoWithoutHash(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf, cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, &pInfo->groupResInfo, &pInfo->aggSup,
&pInfo->groupResInfo); pInfo->cleanGroupResInfo);
pInfo->pOperator = NULL; pInfo->pOperator = NULL;
} }
cleanupAggSup(&pInfo->aggSup); cleanupAggSup(&pInfo->aggSup);
@ -191,6 +193,7 @@ static bool nextGroupedResult(SOperatorInfo* pOperator) {
int32_t order = pAggInfo->binfo.inputTsOrder; int32_t order = pAggInfo->binfo.inputTsOrder;
SSDataBlock* pBlock = pAggInfo->pNewGroupBlock; SSDataBlock* pBlock = pAggInfo->pNewGroupBlock;
pAggInfo->cleanGroupResInfo = false;
if (pBlock) { if (pBlock) {
pAggInfo->pNewGroupBlock = NULL; pAggInfo->pNewGroupBlock = NULL;
tSimpleHashClear(pAggInfo->aggSup.pResultRowHashTable); tSimpleHashClear(pAggInfo->aggSup.pResultRowHashTable);
@ -263,6 +266,7 @@ static bool nextGroupedResult(SOperatorInfo* pOperator) {
code = initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, 0); code = initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, 0);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
pAggInfo->cleanGroupResInfo = true;
_end: _end:
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -627,7 +631,7 @@ void cleanupResultInfoInStream(SExecTaskInfo* pTaskInfo, void* pState, SExprSupp
} }
} }
void cleanupResultInfoWithoutHash(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SDiskbasedBuf* pBuf, void cleanupResultInfoInGroupResInfo(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SDiskbasedBuf* pBuf,
SGroupResInfo* pGroupResInfo) { SGroupResInfo* pGroupResInfo) {
int32_t numOfExprs = pSup->numOfExprs; int32_t numOfExprs = pSup->numOfExprs;
int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; int32_t* rowEntryOffset = pSup->rowEntryInfoOffset;
@ -663,7 +667,7 @@ void cleanupResultInfoWithoutHash(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SDi
} }
} }
void cleanupResultInfo(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SDiskbasedBuf* pBuf, void cleanupResultInfoInHashMap(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SDiskbasedBuf* pBuf,
SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap) { SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap) {
int32_t numOfExprs = pSup->numOfExprs; int32_t numOfExprs = pSup->numOfExprs;
int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; int32_t* rowEntryOffset = pSup->rowEntryInfoOffset;
@ -701,6 +705,14 @@ void cleanupResultInfo(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SDiskbasedBuf*
} }
} }
void cleanupResultInfo(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SGroupResInfo* pGroupResInfo,
SAggSupporter *pAggSup, bool cleanGroupResInfo) {
if (cleanGroupResInfo) {
cleanupResultInfoInGroupResInfo(pTaskInfo, pSup, pAggSup->pResultBuf, pGroupResInfo);
} else {
cleanupResultInfoInHashMap(pTaskInfo, pSup, pAggSup->pResultBuf, pGroupResInfo, pAggSup->pResultRowHashTable);
}
}
void cleanupAggSup(SAggSupporter* pAggSup) { void cleanupAggSup(SAggSupporter* pAggSup) {
taosMemoryFreeClear(pAggSup->keyBuf); taosMemoryFreeClear(pAggSup->keyBuf);
tSimpleHashCleanup(pAggSup->pResultRowHashTable); tSimpleHashCleanup(pAggSup->pResultRowHashTable);

9
source/libs/executor/src/groupoperator.c
View File

@ -88,8 +88,8 @@ static void destroyGroupOperatorInfo(void* param) {
cleanupExprSupp(&pInfo->scalarSup); cleanupExprSupp(&pInfo->scalarSup);
if (pInfo->pOperator != NULL) { if (pInfo->pOperator != NULL) {
cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf, cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, &pInfo->groupResInfo, &pInfo->aggSup,
&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable); false);
pInfo->pOperator = NULL; pInfo->pOperator = NULL;
} }
@ -1263,7 +1263,10 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
QUERY_CHECK_CONDITION((hasRemainPartion(pInfo)), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); QUERY_CHECK_CONDITION((hasRemainPartion(pInfo)), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->parIte; SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->parIte;
blockDataCleanup(pDest); blockDataCleanup(pDest);
int32_t rows = taosArrayGetSize(pParInfo->rowIds); int32_t rows = taosArrayGetSize(pParInfo->rowIds);
code = blockDataEnsureCapacity(pDest, rows);
QUERY_CHECK_CODE(code, lino, _end);
SSDataBlock* pSrc = pInfo->pInputDataBlock; SSDataBlock* pSrc = pInfo->pInputDataBlock;
for (int32_t i = 0; i < rows; i++) { for (int32_t i = 0; i < rows; i++) {
int32_t rowIndex = *(int32_t*)taosArrayGet(pParInfo->rowIds, i); int32_t rowIndex = *(int32_t*)taosArrayGet(pParInfo->rowIds, i);

4
source/libs/executor/src/streamtimewindowoperator.c
View File

@ -474,8 +474,8 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) {
SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)param; SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)param;
cleanupBasicInfo(&pInfo->binfo); cleanupBasicInfo(&pInfo->binfo);
if (pInfo->pOperator) { if (pInfo->pOperator) {
cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf, cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, &pInfo->groupResInfo, &pInfo->aggSup,
&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable); false);
pInfo->pOperator = NULL; pInfo->pOperator = NULL;
} }
cleanupAggSup(&pInfo->aggSup); cleanupAggSup(&pInfo->aggSup);

64
source/libs/executor/src/timesliceoperator.c
View File

@ -42,7 +42,7 @@ typedef struct STimeSliceOperatorInfo {
SRowKey prevKey; SRowKey prevKey;
bool prevTsSet; bool prevTsSet;
uint64_t groupId; uint64_t groupId;
SGroupKeys* pPrevGroupKey; SArray* pPrevGroupKeys;
SSDataBlock* pNextGroupRes; SSDataBlock* pNextGroupRes;
SSDataBlock* pRemainRes; // save block unfinished processing SSDataBlock* pRemainRes; // save block unfinished processing
int32_t remainIndex; // the remaining index in the block to be processed int32_t remainIndex; // the remaining index in the block to be processed
@ -288,6 +288,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
// output the result // output the result
int32_t fillColIndex = 0; int32_t fillColIndex = 0;
int32_t groupKeyIndex = 0;
bool hasInterp = true; bool hasInterp = true;
for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) { for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
SExprInfo* pExprInfo = &pExprSup->pExprInfo[j]; SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];
@ -320,7 +321,9 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
} else if (!isSelectGroupConstValueFunc(pExprInfo)) { } else if (!isSelectGroupConstValueFunc(pExprInfo)) {
// use stored group key // use stored group key
SGroupKeys* pkey = pSliceInfo->pPrevGroupKey; SGroupKeys *pkey = taosArrayGet(pSliceInfo->pPrevGroupKeys, groupKeyIndex);
QUERY_CHECK_NULL(pkey, code, lino, _end, terrno);
groupKeyIndex++;
if (pkey->isNull == false) { if (pkey->isNull == false) {
code = colDataSetVal(pDst, rows, pkey->pData, false); code = colDataSetVal(pDst, rows, pkey->pData, false);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
@ -645,13 +648,20 @@ _end:
return code; return code;
} }
static void destroyGroupKey(void* pKey) {
SGroupKeys* key = (SGroupKeys*)pKey;
if (key->pData != NULL) {
taosMemoryFreeClear(key->pData);
}
}
static int32_t initGroupKeyKeeper(STimeSliceOperatorInfo* pInfo, SExprSupp* pExprSup) { static int32_t initGroupKeyKeeper(STimeSliceOperatorInfo* pInfo, SExprSupp* pExprSup) {
if (pInfo->pPrevGroupKey != NULL) { if (pInfo->pPrevGroupKeys != NULL) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
pInfo->pPrevGroupKey = taosMemoryCalloc(1, sizeof(SGroupKeys)); pInfo->pPrevGroupKeys = taosArrayInit(pExprSup->numOfExprs, sizeof(SGroupKeys));
if (pInfo->pPrevGroupKey == NULL) { if (pInfo->pPrevGroupKeys == NULL) {
return terrno; return terrno;
} }
@ -659,11 +669,19 @@ static int32_t initGroupKeyKeeper(STimeSliceOperatorInfo* pInfo, SExprSupp* pExp
SExprInfo* pExprInfo = &pExprSup->pExprInfo[i]; SExprInfo* pExprInfo = &pExprSup->pExprInfo[i];
if (isGroupKeyFunc(pExprInfo)) { if (isGroupKeyFunc(pExprInfo)) {
pInfo->pPrevGroupKey->bytes = pExprInfo->base.resSchema.bytes; SGroupKeys key = {.bytes = pExprInfo->base.resSchema.bytes,
pInfo->pPrevGroupKey->type = pExprInfo->base.resSchema.type; .type = pExprInfo->base.resSchema.type,
pInfo->pPrevGroupKey->isNull = false; .isNull = false,
pInfo->pPrevGroupKey->pData = taosMemoryCalloc(1, pInfo->pPrevGroupKey->bytes); .pData = taosMemoryCalloc(1, pExprInfo->base.resSchema.bytes)};
if (!pInfo->pPrevGroupKey->pData) { if (!key.pData) {
taosArrayDestroyEx(pInfo->pPrevGroupKeys, destroyGroupKey);
pInfo->pPrevGroupKeys = NULL;
return terrno;
}
if (NULL == taosArrayPush(pInfo->pPrevGroupKeys, &key)) {
taosMemoryFree(key.pData);
taosArrayDestroyEx(pInfo->pPrevGroupKeys, destroyGroupKey);
pInfo->pPrevGroupKeys = NULL;
return terrno; return terrno;
} }
} }
@ -910,7 +928,7 @@ static void genInterpAfterDataBlock(STimeSliceOperatorInfo* pSliceInfo, SOperato
SInterval* pInterval = &pSliceInfo->interval; SInterval* pInterval = &pSliceInfo->interval;
if (pSliceInfo->fillType == TSDB_FILL_NEXT || pSliceInfo->fillType == TSDB_FILL_LINEAR || if (pSliceInfo->fillType == TSDB_FILL_NEXT || pSliceInfo->fillType == TSDB_FILL_LINEAR ||
pSliceInfo->pPrevGroupKey == NULL) { pSliceInfo->pPrevGroupKeys == NULL) {
return; return;
} }
@ -921,12 +939,18 @@ static void genInterpAfterDataBlock(STimeSliceOperatorInfo* pSliceInfo, SOperato
} }
} }
static void copyPrevGroupKey(SExprSupp* pExprSup, SGroupKeys* pGroupKey, SSDataBlock* pSrcBlock) { static int32_t copyPrevGroupKey(SExprSupp* pExprSup, SArray * pGroupKeys, SSDataBlock* pSrcBlock) {
int32_t groupKeyIdx = 0;
for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) { for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
SExprInfo* pExprInfo = &pExprSup->pExprInfo[j]; SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];
if (isGroupKeyFunc(pExprInfo)) { if (isGroupKeyFunc(pExprInfo)) {
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
SGroupKeys *pGroupKey = taosArrayGet(pGroupKeys, groupKeyIdx);
if (pGroupKey == NULL) {
return terrno;
}
groupKeyIdx++;
SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot); SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot);
if (colDataIsNull_s(pSrc, 0)) { if (colDataIsNull_s(pSrc, 0)) {
@ -942,9 +966,9 @@ static void copyPrevGroupKey(SExprSupp* pExprSup, SGroupKeys* pGroupKey, SSDataB
} }
pGroupKey->isNull = false; pGroupKey->isNull = false;
break;
} }
} }
return TSDB_CODE_SUCCESS;
} }
static void resetTimesliceInfo(STimeSliceOperatorInfo* pSliceInfo) { static void resetTimesliceInfo(STimeSliceOperatorInfo* pSliceInfo) {
@ -986,7 +1010,11 @@ static void doHandleTimeslice(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
T_LONG_JMP(pTaskInfo->env, code); T_LONG_JMP(pTaskInfo->env, code);
} }
doTimesliceImpl(pOperator, pSliceInfo, pBlock, pTaskInfo, ignoreNull); doTimesliceImpl(pOperator, pSliceInfo, pBlock, pTaskInfo, ignoreNull);
copyPrevGroupKey(&pOperator->exprSupp, pSliceInfo->pPrevGroupKey, pBlock); code = copyPrevGroupKey(&pOperator->exprSupp, pSliceInfo->pPrevGroupKeys, pBlock);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
T_LONG_JMP(pTaskInfo->env, code);
}
} }
static int32_t doTimesliceNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { static int32_t doTimesliceNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
@ -1160,7 +1188,7 @@ int32_t createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyN
pInfo->prevTsSet = false; pInfo->prevTsSet = false;
pInfo->prevKey.ts = INT64_MIN; pInfo->prevKey.ts = INT64_MIN;
pInfo->groupId = 0; pInfo->groupId = 0;
pInfo->pPrevGroupKey = NULL; pInfo->pPrevGroupKeys = NULL;
pInfo->pNextGroupRes = NULL; pInfo->pNextGroupRes = NULL;
pInfo->pRemainRes = NULL; pInfo->pRemainRes = NULL;
pInfo->remainIndex = 0; pInfo->remainIndex = 0;
@ -1233,9 +1261,9 @@ void destroyTimeSliceOperatorInfo(void* param) {
} }
taosArrayDestroy(pInfo->pLinearInfo); taosArrayDestroy(pInfo->pLinearInfo);
if (pInfo->pPrevGroupKey) { if (pInfo->pPrevGroupKeys) {
taosMemoryFree(pInfo->pPrevGroupKey->pData); taosArrayDestroyEx(pInfo->pPrevGroupKeys, destroyGroupKey);
taosMemoryFree(pInfo->pPrevGroupKey); pInfo->pPrevGroupKeys = NULL;
} }
if (pInfo->hasPk && IS_VAR_DATA_TYPE(pInfo->pkCol.type)) { if (pInfo->hasPk && IS_VAR_DATA_TYPE(pInfo->pkCol.type)) {
taosMemoryFreeClear(pInfo->prevKey.pks[0].pData); taosMemoryFreeClear(pInfo->prevKey.pks[0].pData);

24
source/libs/executor/src/timewindowoperator.c
View File

@ -38,6 +38,7 @@ typedef struct SSessionAggOperatorInfo {
int32_t tsSlotId; // primary timestamp slot id int32_t tsSlotId; // primary timestamp slot id
STimeWindowAggSupp twAggSup; STimeWindowAggSupp twAggSup;
SOperatorInfo* pOperator; SOperatorInfo* pOperator;
bool cleanGroupResInfo;
} SSessionAggOperatorInfo; } SSessionAggOperatorInfo;
typedef struct SStateWindowOperatorInfo { typedef struct SStateWindowOperatorInfo {
@ -52,6 +53,7 @@ typedef struct SStateWindowOperatorInfo {
int32_t tsSlotId; // primary timestamp column slot id int32_t tsSlotId; // primary timestamp column slot id
STimeWindowAggSupp twAggSup; STimeWindowAggSupp twAggSup;
SOperatorInfo* pOperator; SOperatorInfo* pOperator;
bool cleanGroupResInfo;
} SStateWindowOperatorInfo; } SStateWindowOperatorInfo;
typedef enum SResultTsInterpType { typedef enum SResultTsInterpType {
@ -943,6 +945,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
int32_t scanFlag = MAIN_SCAN; int32_t scanFlag = MAIN_SCAN;
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
pInfo->cleanGroupResInfo = false;
while (1) { while (1) {
SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0); SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
if (pBlock == NULL) { if (pBlock == NULL) {
@ -965,6 +968,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
code = initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->binfo.outputTsOrder); code = initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->binfo.outputTsOrder);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
pInfo->cleanGroupResInfo = true;
OPTR_SET_OPENED(pOperator); OPTR_SET_OPENED(pOperator);
@ -1092,6 +1096,7 @@ static int32_t openStateWindowAggOptr(SOperatorInfo* pOperator) {
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
SOperatorInfo* downstream = pOperator->pDownstream[0]; SOperatorInfo* downstream = pOperator->pDownstream[0];
pInfo->cleanGroupResInfo = false;
while (1) { while (1) {
SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0); SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
if (pBlock == NULL) { if (pBlock == NULL) {
@ -1120,7 +1125,7 @@ static int32_t openStateWindowAggOptr(SOperatorInfo* pOperator) {
pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
code = initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC); code = initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
pInfo->cleanGroupResInfo = true;
pOperator->status = OP_RES_TO_RETURN; pOperator->status = OP_RES_TO_RETURN;
_end: _end:
@ -1230,8 +1235,8 @@ static void destroyStateWindowOperatorInfo(void* param) {
cleanupBasicInfo(&pInfo->binfo); cleanupBasicInfo(&pInfo->binfo);
taosMemoryFreeClear(pInfo->stateKey.pData); taosMemoryFreeClear(pInfo->stateKey.pData);
if (pInfo->pOperator) { if (pInfo->pOperator) {
cleanupResultInfoWithoutHash(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf, cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, &pInfo->groupResInfo, &pInfo->aggSup,
&pInfo->groupResInfo); pInfo->cleanGroupResInfo);
pInfo->pOperator = NULL; pInfo->pOperator = NULL;
} }
@ -1257,8 +1262,8 @@ void destroyIntervalOperatorInfo(void* param) {
cleanupBasicInfo(&pInfo->binfo); cleanupBasicInfo(&pInfo->binfo);
if (pInfo->pOperator) { if (pInfo->pOperator) {
cleanupResultInfoWithoutHash(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf, cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, &pInfo->groupResInfo, &pInfo->aggSup,
&pInfo->groupResInfo); pInfo->cleanGroupResInfo);
pInfo->pOperator = NULL; pInfo->pOperator = NULL;
} }
@ -1452,6 +1457,7 @@ int32_t createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode
} }
pInfo->pOperator = pOperator; pInfo->pOperator = pOperator;
pInfo->cleanGroupResInfo = false;
initResultRowInfo(&pInfo->binfo.resultRowInfo); initResultRowInfo(&pInfo->binfo.resultRowInfo);
setOperatorInfo(pOperator, "TimeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, true, OP_NOT_OPENED, setOperatorInfo(pOperator, "TimeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, true, OP_NOT_OPENED,
pInfo, pTaskInfo); pInfo, pTaskInfo);
@ -1573,6 +1579,7 @@ static int32_t doSessionWindowAggNext(SOperatorInfo* pOperator, SSDataBlock** pp
SOptrBasicInfo* pBInfo = &pInfo->binfo; SOptrBasicInfo* pBInfo = &pInfo->binfo;
SExprSupp* pSup = &pOperator->exprSupp; SExprSupp* pSup = &pOperator->exprSupp;
pInfo->cleanGroupResInfo = false;
if (pOperator->status == OP_RES_TO_RETURN) { if (pOperator->status == OP_RES_TO_RETURN) {
while (1) { while (1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
@ -1628,6 +1635,7 @@ static int32_t doSessionWindowAggNext(SOperatorInfo* pOperator, SSDataBlock** pp
code = initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC); code = initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
pInfo->cleanGroupResInfo = true;
code = blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); code = blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
@ -1731,6 +1739,7 @@ int32_t createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhy
pInfo->tsSlotId = tsSlotId; pInfo->tsSlotId = tsSlotId;
pInfo->pOperator = pOperator; pInfo->pOperator = pOperator;
pInfo->cleanGroupResInfo = false;
setOperatorInfo(pOperator, "StateWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, true, OP_NOT_OPENED, pInfo, setOperatorInfo(pOperator, "StateWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, true, OP_NOT_OPENED, pInfo,
pTaskInfo); pTaskInfo);
pOperator->fpSet = createOperatorFpSet(openStateWindowAggOptr, doStateWindowAggNext, NULL, destroyStateWindowOperatorInfo, pOperator->fpSet = createOperatorFpSet(openStateWindowAggOptr, doStateWindowAggNext, NULL, destroyStateWindowOperatorInfo,
@ -1763,8 +1772,8 @@ void destroySWindowOperatorInfo(void* param) {
cleanupBasicInfo(&pInfo->binfo); cleanupBasicInfo(&pInfo->binfo);
colDataDestroy(&pInfo->twAggSup.timeWindowData); colDataDestroy(&pInfo->twAggSup.timeWindowData);
if (pInfo->pOperator) { if (pInfo->pOperator) {
cleanupResultInfoWithoutHash(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf, cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, &pInfo->groupResInfo, &pInfo->aggSup,
&pInfo->groupResInfo); pInfo->cleanGroupResInfo);
pInfo->pOperator = NULL; pInfo->pOperator = NULL;
} }
@ -1835,6 +1844,7 @@ int32_t createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPh
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);
pInfo->pOperator = pOperator; pInfo->pOperator = pOperator;
pInfo->cleanGroupResInfo = false;
setOperatorInfo(pOperator, "SessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, true, OP_NOT_OPENED, setOperatorInfo(pOperator, "SessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, true, OP_NOT_OPENED,
pInfo, pTaskInfo); pInfo, pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doSessionWindowAggNext, NULL, destroySWindowOperatorInfo, pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doSessionWindowAggNext, NULL, destroySWindowOperatorInfo,

Some files were not shown because too many files have changed in this diff Show More