xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into feat/TS-4917-3.0

This commit is contained in:
Shungang Li 2024-06-25 10:33:12 +08:00
parent b3368f6472 42beba30a8
commit 17e84e1fca
290 changed files with 15964 additions and 9980 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
cmake/cmake.define
View File

@ -180,18 +180,20 @@ ELSE ()
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2")
ENDIF()
MESSAGE(STATUS "SIMD instructions (FMA/AVX/AVX2) is ACTIVATED")
ENDIF()
# IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI)
# SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi")
# SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi")
# MESSAGE(STATUS "avx512f/avx512bmi supported by compiler")
# ENDIF()
#
# IF (COMPILER_SUPPORT_AVX512VL)
# SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512vl")
# SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512vl")
# MESSAGE(STATUS "avx512vl supported by compiler")
# ENDIF()
IF ("${SIMD_AVX512_SUPPORT}" MATCHES "true")
IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi")
MESSAGE(STATUS "avx512f/avx512bmi enabled by compiler")
ENDIF()
IF (COMPILER_SUPPORT_AVX512VL)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512vl")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512vl")
MESSAGE(STATUS "avx512vl enabled by compiler")
ENDIF()
ENDIF()
# build mode

2
cmake/cmake.version
View File

@ -2,7 +2,7 @@
IF (DEFINED VERNUMBER)
SET(TD_VER_NUMBER ${VERNUMBER})
ELSE ()
SET(TD_VER_NUMBER "3.3.1.0.alpha")
SET(TD_VER_NUMBER "3.3.2.0.alpha")
ENDIF ()
IF (DEFINED VERCOMPATIBLE)

2
cmake/libuv_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# libuv
ExternalProject_Add(libuv
GIT_REPOSITORY https://github.com/libuv/libuv.git
GIT_TAG v1.44.2
GIT_TAG v1.48.0
SOURCE_DIR "${TD_CONTRIB_DIR}/libuv"
BINARY_DIR "${TD_CONTRIB_DIR}/libuv"
CONFIGURE_COMMAND ""

2
cmake/taostools_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taos-tools
ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG main
GIT_TAG 3.0
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE

4
docs/en/10-deployment/03-k8s.md
View File

@ -243,7 +243,7 @@ curl -u root:taosdata -d "show databases" 127.0.0.1:6041/rest/sql
{"code":0,"column_meta":[["name","VARCHAR",64]],"data":[["information_schema"],["performance_schema"],["test"],["test1"]],"rows":4}
```
## Test cluster
## Test cluster
### Data preparation
@ -335,7 +335,7 @@ tdengine-1 1/1 Running 1 (6m48s ago) 20m 10.244.0.59 node84
tdengine-2 1/1 Running 0 21m 10.244.1.223 node85 <none> <none>
```
At this time, the cluster mnode has a re-election, and the monde on dnode1 becomes the leader.
At this time, the cluster mnode has a re-election, and the monde on dnode2 becomes the leader.
```Bash
kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show mnodes\G"

40
docs/en/12-taos-sql/03-table.md
View File

@ -25,7 +25,7 @@ create_definition:
col_name column_definition
column_definition:
type_name [comment 'string_value'] [PRIMARY KEY] [ENCODE 'encode_type'] [COMPRESS 'compress_type'] [LEVEL 'level_type']
type_name [comment 'string_value'] [PRIMARY KEY] [ENCODE 'encode_type'] [COMPRESS 'compress_type'] [LEVEL 'level_type']
table_options:
table_option ...
@ -52,9 +52,9 @@ table_option: {
**Parameter description**
1. COMMENT: specifies comments for the table. This parameter can be used with supertables, standard tables, and subtables.
1. COMMENT: specifies comments for the table. This parameter can be used with supertables, standard tables, and subtables. The maximum length of the comment is 1024 bytes.
2. SMA: specifies functions on which to enable small materialized aggregates (SMA). SMA is user-defined precomputation of aggregates based on data blocks. Enter one of the following values: max, min, or sum This parameter can be used with supertables and standard tables.
3. TTL: specifies the time to live (TTL) for the table. If TTL is specified when creatinga table, after the time period for which the table has been existing is over TTL, TDengine will automatically delete the table. Please be noted that the system may not delete the table at the exact moment that the TTL expires but guarantee there is such a system and finally the table will be deleted. The unit of TTL is in days. The default value is 0, i.e. never expire.
3. TTL: specifies the time to live (TTL) for the table. If TTL is specified when creatinga table, after the time period for which the table has been existing is over TTL, TDengine will automatically delete the table. Please be noted that the system may not delete the table at the exact moment that the TTL expires but guarantee there is such a system and finally the table will be deleted. The unit of TTL is in days. The value range is [0, 2147483647]. The default value is 0, i.e. never expire.
## Create Subtables
@ -112,6 +112,11 @@ You can perform the following modifications on existing tables:
4. RENAME COLUMN: renames a specified column in the table.
5. The primary key column of a table cannot be modified or added or deleted using ADD/DROP COLUMN.
**Parameter description**
1. COMMENT: specifies comments for the table. This parameter can be used with supertables, standard tables, and subtables. The maximum length of the comment is 1024 bytes.
2. TTL: specifies the time to live (TTL) for the table. If TTL is specified when creatinga table, after the time period for which the table has been existing is over TTL, TDengine will automatically delete the table. Please be noted that the system may not delete the table at the exact moment that the TTL expires but guarantee there is such a system and finally the table will be deleted. The unit of TTL is in days. The value range is [0, 2147483647]. The default value is 0, i.e. never expire.
### Add a Column
```sql
@ -136,6 +141,18 @@ ALTER TABLE tb_name MODIFY COLUMN field_name data_type(length);
ALTER TABLE tb_name RENAME COLUMN old_col_name new_col_name
```
### Alter Table TTL
```sql
ALTER TABLE tb_name TTL value
```
### Alter Table Comment
```sql
ALTER TABLE tb_name COMMENT 'string_value'
```
## Modify a Subtable
```sql
@ -159,12 +176,29 @@ alter_table_option: {
1. Only the value of a tag can be modified directly. For all other modifications, you must modify the supertable from which the subtable was created.
**Parameter description**
1. COMMENT: specifies comments for the table. This parameter can be used with supertables, standard tables, and subtables. The maximum length of the comment is 1024 bytes.
2. TTL: specifies the time to live (TTL) for the table. If TTL is specified when creatinga table, after the time period for which the table has been existing is over TTL, TDengine will automatically delete the table. Please be noted that the system may not delete the table at the exact moment that the TTL expires but guarantee there is such a system and finally the table will be deleted. The unit of TTL is in days. The value range is [0, 2147483647]. The default value is 0, i.e. never expire.
### Change Tag Value Of Sub Table
```
ALTER TABLE tb_name SET TAG tag_name=new_tag_value;
```
### Alter Table TTL
```sql
ALTER TABLE tb_name TTL value
```
### Alter Table Comment
```sql
ALTER TABLE tb_name COMMENT 'string_value'
```
## Delete a Table
The following SQL statement deletes one or more tables.

2
docs/en/12-taos-sql/14-stream.md
View File

@ -47,7 +47,7 @@ window_clause: {
}
```
`SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically.
`SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically.The `_wend` of this window is the time of the last data plus `tol_val`.
`EVENT_WINDOW` is determined according to the window start condition and the window close condition. The window is started when `start_trigger_condition` is evaluated to true, the window is closed when `end_trigger_condition` is evaluated to true. `start_trigger_condition` and `end_trigger_condition` can be any conditional expressions supported by TDengine and can include multiple columns.

4
docs/en/12-taos-sql/20-keywords.md
View File

@ -151,7 +151,7 @@ The following list shows all reserved keywords:
- INTERVAL
- INTO
- IS
- ISNULL
- IS NULL
### J
@ -197,7 +197,7 @@ The following list shows all reserved keywords:
- NMATCH
- NONE
- NOT
- NOTNULL
- NOT NULL
- NOW
- NULL
- NULLS

20
docs/en/12-taos-sql/25-grant.md
View File

@ -39,10 +39,10 @@ This is an example:
```sql
taos> show users;
name | super | enable | sysinfo | create_time |
================================================================================
test | 0 | 1 | 1 | 2022-08-29 15:10:27.315 |
root | 1 | 1 | 1 | 2022-08-29 15:03:34.710 |
name | super | enable | sysinfo | createdb | create_time | allowed_host |
=========================================================================================================
test | 0 | 1 | 1 | 0 |2022-08-29 15:10:27.315 | 127.0.0.1 |
root | 1 | 1 | 1 | 1 |2022-08-29 15:03:34.710 | 127.0.0.1 |
Query OK, 2 rows in database (0.001657s)
```
@ -50,10 +50,10 @@ Alternatively, you can get the user information by querying a built-in table, IN
```sql
taos> select * from information_schema.ins_users;
name | super | enable | sysinfo | create_time |
================================================================================
test | 0 | 1 | 1 | 2022-08-29 15:10:27.315 |
root | 1 | 1 | 1 | 2022-08-29 15:03:34.710 |
name | super | enable | sysinfo | createdb | create_time | allowed_host |
=========================================================================================================
test | 0 | 1 | 1 | 0 |2022-08-29 15:10:27.315 | 127.0.0.1 |
root | 1 | 1 | 1 | 1 |2022-08-29 15:03:34.710 | 127.0.0.1 |
Query OK, 2 rows in database (0.001953s)
```
@ -67,17 +67,19 @@ DROP USER user_name;
```sql
ALTER USER user_name alter_user_clause
alter_user_clause: {
PASS 'literal'
| ENABLE value
| SYSINFO value
| CREATEDB value
}
```
- PASS: Modify the user password.
- ENABLE: Specify whether the user is enabled or disabled. 1 indicates enabled and 0 indicates disabled.
- SYSINFO: Specify whether the user can query system information. 1 indicates that the user can query system information and 0 indicates that the user cannot query system information.
- CREATEDB: Specify whether the user can create databases. 1 indicates that the user can create databases and 0 indicates that the user cannot create databases.
For example, you can use below command to disable user `test`:

2
docs/en/14-reference/12-config/index.md
View File

@ -432,7 +432,7 @@ The charset that takes effect is UTF-8.
| Applicable | Server Only |
| Meaning | Maximum number of threads to commit |
| Value Range | 0-1024 |
| Default Value | |
| Default Value | 4 |
## Log Parameters

14
docs/en/21-tdinternal/08-compress.md
View File

@ -1,12 +1,8 @@
---
title: Configurable Column Compression
description: Configurable column storage compression method
---
# Configurable Storage Compression
Since TDengine 3.3.0.0, more advanced compression feature is introduced, you can specify compression or not, the compression method and compression level for each column.
## Compression Terminology Definition
@ -32,16 +28,14 @@ In this article, it specifically refers to the level within the secondary compre
- Default compression algorithm list and applicable range for each data type
| Data Type | Optional Encoding Algorithm | Default Encoding Algorithm | Optional Compression Algorithm|Default Compression Algorithm| Default Compression Level|
| Data Type | Optional Encoding Algorithm | Default Encoding Algorithm | Optional Compression Algorithm|Default Compression Algorithm| Default Compression Level|
| :-----------:|:----------:|:-------:|:-------:|:----------:|:----:|
tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium|
| tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium|
| bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium|
|float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|tsz| medium|
|float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|lz4| medium|
|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| medium|
|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| medium|
Note: For floating point types, if configured as tsz, its precision is determined by the global configuration of taosd. If configured as tsz, but the lossy compression flag is not configured, lz4 is used for compression by default.
## SQL
### Create Table with Compression
@ -76,7 +70,7 @@ ALTER TABLE [db_name.]tabName MODIFY COLUMN colName [ENCODE 'ecode_type'] [COMPR
- Change the compression method of the column
### View Compression Dethod
### View Compression Method
```sql
DESCRIBE [dbname.]tabName

4
docs/en/28-releases/01-tdengine.md
View File

@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://t
import Release from "/components/ReleaseV3";
## 3.3.1.0
<Release type="tdengine" version="3.3.1.0" />
## 3.3.0.3
<Release type="tdengine" version="3.3.0.3" />

40
docs/zh/10-deployment/03-k8s.md
View File

@ -10,9 +10,9 @@ description: 利用 Kubernetes 部署 TDengine 集群的详细指南
为了满足[高可用](https://docs.taosdata.com/tdinternal/high-availability/)的需求，集群需要满足如下要求：
- 3个及以上 dnode ：TDengine 的同一个 vgroup 中的多个 vnode ，不允许同时分布在一个 dnode ，所以如果创建3副本的数据库，则 dnode 数大于等于3
- 3个 mnode ：mnode 负责整个集群的管理工作，TDengine 默认是一个 mnode。如果这个 mnode 所在的 dnode 掉线，则整个集群不可用。
- 数据库的3副本：TDengine 的副本配置是数据库级别，所以数据库3副本可满足在3个 dnode 的集群中，任意一个 dnode 下线，都不影响集群的正常使用。**如果下线** **dnode** **个数为2时，此时集群不可用，****因为****RAFT无法完成选举****。**（企业版：在灾难恢复场景，任一节点数据文件损坏，都可以通过重新拉起dnode进行恢复）
- 3 个及以上 dnode ：TDengine 的同一个 vgroup 中的多个 vnode ，不允许同时分布在一个 dnode ，所以如果创建 3 副本的数据库，则 dnode 数大于等于 3
- 3 个 mnode ：mnode 负责整个集群的管理工作，TDengine 默认是一个 mnode。如果这个 mnode 所在的 dnode 掉线，则整个集群不可用。
- 数据库的 3 副本：TDengine 的副本配置是数据库级别，所以数据库 3 副本可满足在 3 个 dnode 的集群中，任意一个 dnode 下线，都不影响集群的正常使用。**如果下线** **dnode** **个数为 2 时，此时集群不可用，\*\***因为\***\*RAFT 无法完成选举\*\***。\*\*（企业版：在灾难恢复场景，任一节点数据文件损坏，都可以通过重新拉起 dnode 进行恢复）
## 前置条件
@ -52,7 +52,7 @@ spec:
根据 Kubernetes 对各类部署的说明，我们将使用 StatefulSet 作为 TDengine 的部署资源类型。 创建文件 `tdengine.yaml`，其中 replicas 定义集群节点的数量为 3。节点时区为中国（Asia/Shanghai），每个节点分配 5G 标准（standard）存储（参考[Storage Classes](https://kubernetes.io/docs/concepts/storage/storage-classes/) 配置 storage class ）。你也可以根据实际情况进行相应修改。
请特别注意startupProbe的配置，在 dnode 的 Pod 掉线一段时间后，再重新启动，这个时候新上线的 dnode 会短暂不可用。如果startupProbe配置过小，Kubernetes 会认为该 Pod 处于不正常的状态，并尝试重启该 Pod，该 dnode 的 Pod 会频繁重启，始终无法恢复到正常状态。参考 [Configure Liveness, Readiness and Startup Probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/)
请特别注意 startupProbe 的配置，在 dnode 的 Pod 掉线一段时间后，再重新启动，这个时候新上线的 dnode 会短暂不可用。如果 startupProbe 配置过小，Kubernetes 会认为该 Pod 处于不正常的状态，并尝试重启该 Pod，该 dnode 的 Pod 会频繁重启，始终无法恢复到正常状态。参考 [Configure Liveness, Readiness and Startup Probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/)
```YAML
---
@ -176,7 +176,7 @@ taos> show dnodes
Query OK, 3 row(s) in set (0.001853s)
```
查看当前mnode
查看当前 mnode
```Bash
kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show mnodes\G"
@ -191,14 +191,14 @@ reboot_time: 2023-07-19 17:54:19.520
Query OK, 1 row(s) in set (0.001282s)
```
## 创建mnode
## 创建 mnode
```Bash
kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "create mnode on dnode 2"
kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "create mnode on dnode 3"
```
查看mnode
查看 mnode
```Bash
kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show mnodes\G"
@ -249,7 +249,7 @@ curl -u root:taosdata -d "show databases" 127.0.0.1:6041/rest/sql
#### taosBenchmark
通过taosBenchmark 创建一个3副本的数据库，同时写入1亿条数据，同时查看数据
通过 taosBenchmark 创建一个 3 副本的数据库，同时写入 1 亿条数据，同时查看数据
```Bash
kubectl exec -it tdengine-0 -n tdengine-test -- taosBenchmark -I stmt -d test -n 10000 -t 10000 -a 3
@ -264,7 +264,7 @@ taos> select count(*) from test.meters;
Query OK, 1 row(s) in set (0.103537s)
```
查看vnode分布，通过show dnodes
查看 vnode 分布，通过 show dnodes
```Bash
kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes"
@ -278,7 +278,7 @@ taos> show dnodes
Query OK, 3 row(s) in set (0.001357s)
```
通过show vgroup 查看 vnode 分布情况
通过 show vgroup 查看 vnode 分布情况
```Bash
kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show test.vgroups"
@ -299,7 +299,7 @@ Query OK, 8 row(s) in set (0.001488s)
#### 手工创建
常见一个三副本的test1，并创建一张表，写入2条数据
常见一个三副本的 test1，并创建一张表，写入 2 条数据
```Bash
kubectl exec -it tdengine-0 -n tdengine-test -- \
@ -310,7 +310,7 @@ kubectl exec -it tdengine-0 -n tdengine-test -- \
insert into t1 values(now, 1)(now+1s, 2);"
```
通过show test1.vgroup 查看xnode分布情况
通过 show test1.vgroup 查看 xnode 分布情况
```Bash
kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show test1.vgroups"
@ -335,7 +335,7 @@ tdengine-1 1/1 Running 1 (6m48s ago) 20m 10.244.0.59 node84
tdengine-2 1/1 Running 0 21m 10.244.1.223 node85 <none> <none>
```
此时集群mnode发生重新选举，dnode1上的monde 成为leader
此时集群 mnode 发生重新选举，dnode2 上的 monde 成为 leader
```Bash
kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show mnodes\G"
@ -389,7 +389,7 @@ taos> select *from test1.t1
Query OK, 4 row(s) in set (0.001994s)
```
同理，至于非leader得mnode掉线，读写当然可以正常进行，这里就不做过多的展示。
同理，至于非 leader 得 mnode 掉线，读写当然可以正常进行，这里就不做过多的展示。
## 集群扩容
@ -436,7 +436,7 @@ Query OK, 4 row(s) in set (0.003628s)
## 集群缩容
由于 TDengine 集群在扩缩容时会对数据进行节点间迁移，使用 kubectl 命令进行缩容需要首先使用 "drop dnodes" 命令（**如果集群中存在3副本的db，那么缩容后的** **dnode** **个数也要必须大于等于3，否则drop dnode操作会被中止**），然后再节点删除完成后再进行 Kubernetes 集群缩容。
由于 TDengine 集群在扩缩容时会对数据进行节点间迁移，使用 kubectl 命令进行缩容需要首先使用 "drop dnodes" 命令（**如果集群中存在 3 副本的 db，那么缩容后的** **dnode** **个数也要必须大于等于 3，否则 drop dnode 操作会被中止**），然后再节点删除完成后再进行 Kubernetes 集群缩容。
注意：由于 Kubernetes Statefulset 中 Pod 的只能按创建顺序逆序移除，所以 TDengine drop dnode 也需要按照创建顺序逆序移除，否则会导致 Pod 处于错误状态。
@ -459,7 +459,7 @@ Query OK, 3 row(s) in set (0.003324s)
kubectl scale statefulsets tdengine --replicas=3 -n tdengine-test
```
最后一个 POD 将会被删除。使用命令 kubectl get pods -l app=tdengine 查看POD状态：
最后一个 POD 将会被删除。使用命令 kubectl get pods -l app=tdengine 查看 POD 状态：
```Plain
kubectl get pod -l app=tdengine -n tdengine-test -o wide
@ -469,7 +469,7 @@ tdengine-1 1/1 Running 1 (7h9m ago) 7h23m 10.244.0.59 node84 <
tdengine-2 1/1 Running 0 5h45m 10.244.1.224 node85 <none> <none>
```
POD删除后，需要手动删除PVC，否则下次扩容时会继续使用以前的数据导致无法正常加入集群。
POD 删除后，需要手动删除 PVC，否则下次扩容时会继续使用以前的数据导致无法正常加入集群。
```Bash
kubectl delete pvc aosdata-tdengine-3 -n tdengine-test
@ -502,7 +502,7 @@ Query OK, 4 row(s) in set (0.003881s)
## 清理 TDengine 集群
> **删除pvc时需要注意下pv persistentVolumeReclaimPolicy策略，建议改为Delete，这样在删除pvc时才会自动清理pv，同时会清理底层的csi存储资源，如果没有配置删除pvc自动清理pv的策略，再删除pvc后，在手动清理pv时，pv对应的csi存储资源可能不会被释放。**
> **删除 pvc 时需要注意下 pv persistentVolumeReclaimPolicy 策略，建议改为 Delete，这样在删除 pvc 时才会自动清理 pv，同时会清理底层的 csi 存储资源，如果没有配置删除 pvc 自动清理 pv 的策略，再删除 pvc 后，在手动清理 pv 时，pv 对应的 csi 存储资源可能不会被释放。**
完整移除 TDengine 集群，需要分别清理 statefulset、svc、configmap、pvc。
@ -537,8 +537,8 @@ Query OK, 4 row(s) in set (0.003862s)
对于在 Kubernetes 环境下 TDengine 的高可用和高可靠来说，对于硬件损坏、灾难恢复，分为两个层面来讲：
1. 底层的分布式块存储具备的灾难恢复能力，块存储的多副本，当下流行的分布式块存储如 Ceph，就具备多副本能力，将存储副本扩展到不同的机架、机柜、机房、数据中心（或者直接使用公有云厂商提供的块存储服务）
2. TDengine的灾难恢复，在 TDengine Enterprise 中，本身具备了当一个 dnode 永久下线（物理机磁盘损坏，数据分拣丢失）后，重新拉起一个空白的dnode来恢复原dnode的工作。
2. TDengine 的灾难恢复，在 TDengine Enterprise 中，本身具备了当一个 dnode 永久下线（物理机磁盘损坏，数据分拣丢失）后，重新拉起一个空白的 dnode 来恢复原 dnode 的工作。
最后，欢迎使用[TDengine Cloud](https://cloud.taosdata.com/)，来体验一站式全托管的TDengine云服务。
最后，欢迎使用[TDengine Cloud](https://cloud.taosdata.com/)，来体验一站式全托管的 TDengine 云服务。
> TDengine Cloud 是一个极简的全托管时序数据处理云服务平台，它是基于开源的时序数据库 TDengine 而开发的。除高性能的时序数据库之外，它还具有缓存、订阅和流计算等系统功能，而且提供了便利而又安全的数据分享、以及众多的企业级功能。它可以让物联网、工业互联网、金融、IT 运维监控等领域企业在时序数据的管理上大幅降低人力成本和运营成本。

42
docs/zh/12-taos-sql/03-table.md
View File

@ -23,10 +23,10 @@ create_subtable_clause: {
}
create_definition:
col_name column_definition
col_name column_definition
column_definition:
type_name [comment 'string_value'] [PRIMARY KEY] [ENCODE 'encode_type'] [COMPRESS 'compress_type'] [LEVEL 'level_type']
type_name [comment 'string_value'] [PRIMARY KEY] [ENCODE 'encode_type'] [COMPRESS 'compress_type'] [LEVEL 'level_type']
table_options:
table_option ...
@ -52,9 +52,9 @@ table_option: {
**参数说明**
1. COMMENT：表注释。可用于超级表、子表和普通表。
1. COMMENT：表注释。可用于超级表、子表和普通表。最大长度为 1024 个字节。
2. SMA：Small Materialized Aggregates，提供基于数据块的自定义预计算功能。预计算类型包括 MAX、MIN 和 SUM。可用于超级表/普通表。
3. TTL：Time to Live，是用户用来指定表的生命周期的参数。如果创建表时指定了这个参数，当该表的存在时间超过 TTL 指定的时间后，TDengine 自动删除该表。这个 TTL 的时间只是一个大概时间，系统不保证到了时间一定会将其删除，而只保证存在这样一个机制且最终一定会删除。TTL 单位是天，默认为 0，表示不限制，到期时间为表创建时间加上 TTL 时间。TTL 与数据库 KEEP 参数没有关联，如果 KEEP 比 TTL 小，在表被删除之前数据也可能已经被删除。
3. TTL：Time to Live，是用户用来指定表的生命周期的参数。如果创建表时指定了这个参数，当该表的存在时间超过 TTL 指定的时间后，TDengine 自动删除该表。这个 TTL 的时间只是一个大概时间，系统不保证到了时间一定会将其删除，而只保证存在这样一个机制且最终一定会删除。TTL 单位是天，取值范围为[0, 2147483647]，默认为 0，表示不限制，到期时间为表创建时间加上 TTL 时间。TTL 与数据库 KEEP 参数没有关联，如果 KEEP 比 TTL 小，在表被删除之前数据也可能已经被删除。
## 创建子表
@ -112,6 +112,11 @@ alter_table_option: {
4. RENAME COLUMN：修改列名称。
5. 普通表的主键列不能被修改，也不能通过 ADD/DROP COLUMN 来添加/删除主键列。
**参数说明**
1. COMMENT：表注释。可用于超级表、子表和普通表。最大长度为 1024 个字节。
2. TTL：Time to Live，是用户用来指定表的生命周期的参数。如果创建表时指定了这个参数，当该表的存在时间超过 TTL 指定的时间后，TDengine 自动删除该表。这个 TTL 的时间只是一个大概时间，系统不保证到了时间一定会将其删除，而只保证存在这样一个机制且最终一定会删除。TTL 单位是天，取值范围为[0, 2147483647]，默认为 0，表示不限制，到期时间为表创建时间加上 TTL 时间。TTL 与数据库 KEEP 参数没有关联，如果 KEEP 比 TTL 小，在表被删除之前数据也可能已经被删除。
### 增加列
```sql
@ -136,6 +141,18 @@ ALTER TABLE tb_name MODIFY COLUMN field_name data_type(length);
ALTER TABLE tb_name RENAME COLUMN old_col_name new_col_name
```
### 修改表生命周期
```sql
ALTER TABLE tb_name TTL value
```
### 修改表注释
```sql
ALTER TABLE tb_name COMMENT 'string_value'
```
## 修改子表
```sql
@ -159,12 +176,29 @@ alter_table_option: {
1. 对子表的列和标签的修改，除了更改标签值以外，都要通过超级表才能进行。
**参数说明**
1. COMMENT：表注释。可用于超级表、子表和普通表。最大长度为 1024 个字节。
2. TTL：Time to Live，是用户用来指定表的生命周期的参数。如果创建表时指定了这个参数，当该表的存在时间超过 TTL 指定的时间后，TDengine 自动删除该表。这个 TTL 的时间只是一个大概时间，系统不保证到了时间一定会将其删除，而只保证存在这样一个机制且最终一定会删除。TTL 单位是天，取值范围为[0, 2147483647]，默认为 0，表示不限制，到期时间为表创建时间加上 TTL 时间。TTL 与数据库 KEEP 参数没有关联，如果 KEEP 比 TTL 小，在表被删除之前数据也可能已经被删除。
### 修改子表标签值
```
ALTER TABLE tb_name SET TAG tag_name=new_tag_value;
```
### 修改表生命周期
```sql
ALTER TABLE tb_name TTL value
```
### 修改表注释
```sql
ALTER TABLE tb_name COMMENT 'string_value'
```
## 删除表
可以在一条 SQL 语句中删除一个或多个普通表或子表。

4
docs/zh/12-taos-sql/14-stream.md
View File

@ -54,8 +54,10 @@ window_clause: {
}
```
其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。
其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。该窗口的 _wend 等于最后一条数据的时间加上 tol_val。
EVENT_WINDOW 是事件窗口，根据开始条件和结束条件来划定窗口。当 start_trigger_condition 满足时则窗口开始，直到 end_trigger_condition 满足时窗口关闭。 start_trigger_condition 和 end_trigger_condition 可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。
COUNT_WINDOW 是计数窗口,按固定的数据行数来划分窗口。 count_val 是常量，是正整数，必须大于等于2，小于2147483648。 count_val 表示每个 COUNT_WINDOW 包含的最大数据行数，总数据行数不能整除 count_val 时，最后一个窗口的行数会小于 count_val 。 sliding_val 是常量，表示窗口滑动的数量，类似于 INTERVAL 的 SLIDING 。
窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished)

4
docs/zh/12-taos-sql/20-keywords.md
View File

@ -151,7 +151,7 @@ description: TDengine 保留关键字的详细列表
- INTERVAL
- INTO
- IS
- ISNULL
- IS NULL
### J
@ -197,7 +197,7 @@ description: TDengine 保留关键字的详细列表
- NMATCH
- NONE
- NOT
- NOTNULL
- NOT NULL
- NOW
- NULL
- NULLS

91
docs/zh/12-taos-sql/25-grant.md Normal file
View File

@ -0,0 +1,91 @@
---
title: 用户管理
sidebar_label: 用户管理
description: 本节讲述基本的用户管理功能
---
用户和权限管理是 TDengine 企业版的功能，本节只讲述基本的用户管理部分。要想了解和获取全面的权限管理功能，请联系 TDengine 销售团队。
## 创建用户
```sql
CREATE USER user_name PASS 'password' [SYSINFO {1|0}];
```
用户名最长不超过 23 个字节。
密码最长不超过 31 个字节。密码可以包含字母、数字以及除单引号、双引号、反引号、反斜杠和空格以外的特殊字符，密码不能为空字符串。
`SYSINFO` 表示该用户是否能够查看系统信息。`1` 表示可以查看，`0` 表示无权查看。系统信息包括服务配置、dnode、vnode、存储等信息。缺省值为 `1`。
在下面的示例中，我们创建一个密码为 `123456` 且可以查看系统信息的用户。
```sql
taos> create user test pass '123456' sysinfo 1;
Query OK, 0 of 0 rows affected (0.001254s)
```
## 查看用户
可以使用如下命令查看系统中的用户。
```sql
SHOW USERS;
```
以下是示例：
```sql
taos> show users;
name | super | enable | sysinfo | createdb | create_time | allowed_host |
=========================================================================================================
test | 0 | 1 | 1 | 0 |2022-08-29 15:10:27.315 | 127.0.0.1 |
root | 1 | 1 | 1 | 1 |2022-08-29 15:03:34.710 | 127.0.0.1 |
Query OK, 2 rows in database (0.001657s)
```
或者，可以查询内置系统表 INFORMATION_SCHEMA.INS_USERS 来获取用户信息。
```sql
taos> select * from information_schema.ins_users;
name | super | enable | sysinfo | createdb | create_time | allowed_host |
=========================================================================================================
test | 0 | 1 | 1 | 0 |2022-08-29 15:10:27.315 | 127.0.0.1 |
root | 1 | 1 | 1 | 1 |2022-08-29 15:03:34.710 | 127.0.0.1 |
Query OK, 2 rows in database (0.001953s)
```
## 删除用户
```sql
DROP USER user_name;
```
## 修改用户配置
```sql
ALTER USER user_name alter_user_clause
alter_user_clause: {
PASS 'literal'
| ENABLE value
| SYSINFO value
| CREATEDB value
}
```
- PASS: 修改密码，后跟新密码
- ENABLE: 启用或禁用该用户，`1` 表示启用，`0` 表示禁用
- SYSINFO: 允许或禁止查看系统信息，`1` 表示允许，`0` 表示禁止
- CREATEDB: 允许或禁止创建数据库，`1` 表示允许，`0` 表示禁止
下面的示例禁用了名为 `test` 的用户:
```sql
taos> alter user test enable 0;
Query OK, 0 of 0 rows affected (0.001160s)
```
## 授权管理
授权管理仅在 TDengine 企业版中可用，请联系 TDengine 销售团队。

2
docs/zh/14-reference/12-config/index.md
View File

@ -430,7 +430,7 @@ charset 的有效值是 UTF-8。
| 适用范围 | 仅服务端适用 |
| 含义 | 设置写入线程的最大数量 |
| 取值范围 | 0-1024 |
| 缺省值 | |
| 缺省值 | 4 |
## 日志相关

10
docs/zh/21-tdinternal/08-compress.md
View File

@ -3,8 +3,6 @@ title: 可配置压缩算法
description: 可配置压缩算法
---
# 可配置存储压缩
从 TDengine 3.3.0.0 版本开始，TDengine 提供了更高级的压缩功能，用户可以在建表时针对每一列配置是否进行压缩、以及使用的压缩算法和压缩级别。
## 压缩术语定义
@ -30,16 +28,14 @@ description: 可配置压缩算法
- 各个数据类型的默认压缩算法列表和适用范围
| 数据类型 | 可选编码算法 | 编码算法默认值 | 可选压缩算法|可选压缩算法| 压缩等级默认值|
| 数据类型 | 可选编码算法 | 编码算法默认值 | 可选压缩算法|压缩算法默认值| 压缩等级默认值|
| :-----------:|:----------:|:-------:|:-------:|:----------:|:----:|
tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium|
| tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium|
| bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium|
|float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|tsz| medium|
|float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|lz4| medium|
|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| medium|
|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| medium|
注意: 针对浮点类型，如果配置为tsz, 其精度由taosd的全局配置决定，如果配置为tsz, 但是没有配置有损压缩标志, 则使用lz4进行压缩
## SQL 语法
### 建表时指定压缩

4
docs/zh/28-releases/01-tdengine.md
View File

@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do
import Release from "/components/ReleaseV3";
## 3.3.1.0
<Release type="tdengine" version="3.3.1.0" />
## 3.3.0.3
<Release type="tdengine" version="3.3.0.3" />

9
include/client/taos.h
View File

@ -150,6 +150,12 @@ typedef struct TAOS_DB_ROUTE_INFO {
TAOS_VGROUP_HASH_INFO *vgHash;
} TAOS_DB_ROUTE_INFO;
typedef struct TAOS_STMT_OPTIONS {
int64_t reqId;
bool singleStbInsert;
bool singleTableBindOnce;
} TAOS_STMT_OPTIONS;
DLL_EXPORT void taos_cleanup(void);
DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...);
DLL_EXPORT setConfRet taos_set_config(const char *config);
@ -162,6 +168,7 @@ DLL_EXPORT const char *taos_data_type(int type);
DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos);
DLL_EXPORT TAOS_STMT *taos_stmt_init_with_reqid(TAOS *taos, int64_t reqid);
DLL_EXPORT TAOS_STMT *taos_stmt_init_with_options(TAOS *taos, TAOS_STMT_OPTIONS* options);
DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length);
DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_MULTI_BIND *tags);
DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name);
@ -359,7 +366,7 @@ typedef enum {
} TSDB_SERVER_STATUS;
DLL_EXPORT TSDB_SERVER_STATUS taos_check_server_status(const char *fqdn, int port, char *details, int maxlen);
DLL_EXPORT char* getBuildInfo();
#ifdef __cplusplus
}
#endif

1
include/common/systable.h
View File

@ -40,6 +40,7 @@ extern "C" {
#define TSDB_INS_TABLE_COLS "ins_columns"
#define TSDB_INS_TABLE_TABLE_DISTRIBUTED "ins_table_distributed"
#define TSDB_INS_TABLE_USERS "ins_users"
#define TSDB_INS_TABLE_USERS_FULL "ins_users_full"
#define TSDB_INS_TABLE_LICENCES "ins_grants"
#define TSDB_INS_TABLE_VGROUPS "ins_vgroups"
#define TSDB_INS_TABLE_VNODES "ins_vnodes"

4
include/common/taosdef.h
View File

@ -30,10 +30,6 @@ typedef int64_t tb_uid_t;
#define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX))
#define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey))
//define show cluster alive and show db.alive
#define SHOW_STATUS_NOT_AVAILABLE 0
#define SHOW_STATUS_AVAILABLE 1
#define SHOW_STATUS_HALF_AVAILABLE 2
typedef enum {
TSDB_SUPER_TABLE = 1, // super table

2
include/common/tcommon.h
View File

@ -240,7 +240,7 @@ typedef struct SDataBlockInfo {
} SDataBlockInfo;
typedef struct SSDataBlock {
SColumnDataAgg** pBlockAgg;
SColumnDataAgg* pBlockAgg;
SArray* pDataBlock; // SArray<SColumnInfoData>
SDataBlockInfo info;
} SSDataBlock;

4
include/common/tdatablock.h
View File

@ -102,7 +102,7 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
return false;
}
if (pColAgg != NULL) {
if (pColAgg != NULL && pColAgg->colId != -1) {
if (pColAgg->numOfNull == totalRows) {
ASSERT(pColumnInfoData->nullbitmap == NULL);
return true;
@ -282,6 +282,8 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbName, uint64_t groupId, char* p
void trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList);
void copyPkVal(SDataBlockInfo* pDst, const SDataBlockInfo* pSrc);
#ifdef __cplusplus
}
#endif

8
include/common/tdataformat.h
View File

@ -370,6 +370,14 @@ int32_t tDecompressData(void *input, // input
int32_t tCompressDataToBuffer(void *input, SCompressInfo *info, SBuffer *output, SBuffer *assist);
int32_t tDecompressDataToBuffer(void *input, SCompressInfo *info, SBuffer *output, SBuffer *assist);
typedef struct {
int32_t columnId;
int32_t type;
TAOS_MULTI_BIND *bind;
} SBindInfo;
int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted, const STSchema *pTSchema,
SArray *rowArray);
#endif
#ifdef __cplusplus

1
include/common/tglobal.h
View File

@ -86,6 +86,7 @@ extern int32_t tsNumOfQnodeFetchThreads;
extern int32_t tsNumOfSnodeStreamThreads;
extern int32_t tsNumOfSnodeWriteThreads;
extern int64_t tsRpcQueueMemoryAllowed;
extern int32_t tsRetentionSpeedLimitMB;
// sync raft
extern int32_t tsElectInterval;

1
include/common/tgrant.h
View File

@ -29,6 +29,7 @@ extern "C" {
#endif
#define GRANT_HEART_BEAT_MIN 2
#define GRANT_EXPIRE_VALUE (31556995201)
#define GRANT_ACTIVE_CODE "activeCode"
#define GRANT_FLAG_ALL (0x01)
#define GRANT_FLAG_AUDIT (0x02)

48
include/common/tmsg.h
View File

@ -37,6 +37,7 @@ extern "C" {
#define TD_MSG_NUMBER_
#undef TD_MSG_DICT_
#undef TD_MSG_INFO_
#undef TD_MSG_TYPE_INFO_
#undef TD_MSG_RANGE_CODE_
#undef TD_MSG_SEG_CODE_
#include "tmsgdef.h"
@ -44,6 +45,7 @@ extern "C" {
#undef TD_MSG_NUMBER_
#undef TD_MSG_DICT_
#undef TD_MSG_INFO_
#undef TD_MSG_TYPE_INFO_
#undef TD_MSG_RANGE_CODE_
#define TD_MSG_SEG_CODE_
#include "tmsgdef.h"
@ -51,6 +53,7 @@ extern "C" {
#undef TD_MSG_NUMBER_
#undef TD_MSG_DICT_
#undef TD_MSG_INFO_
#undef TD_MSG_TYPE_INFO_
#undef TD_MSG_SEG_CODE_
#undef TD_MSG_RANGE_CODE_
#include "tmsgdef.h"
@ -67,7 +70,7 @@ typedef uint16_t tmsg_t;
static inline bool tmsgIsValid(tmsg_t type) {
// static int8_t sz = sizeof(tMsgRangeDict) / sizeof(tMsgRangeDict[0]);
int8_t maxSegIdx = TMSG_SEG_CODE(TDMT_MAX_MSG);
int8_t maxSegIdx = TMSG_SEG_CODE(TDMT_MAX_MSG_MIN);
int segIdx = TMSG_SEG_CODE(type);
if (segIdx >= 0 && segIdx < maxSegIdx) {
return type < tMsgRangeDict[segIdx];
@ -155,6 +158,7 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_MACHINES,
TSDB_MGMT_TABLE_ARBGROUP,
TSDB_MGMT_TABLE_ENCRYPTIONS,
TSDB_MGMT_TABLE_USER_FULL,
TSDB_MGMT_TABLE_MAX,
} EShowType;
@ -190,6 +194,7 @@ typedef enum _mgmt_table {
#define TSDB_ALTER_USER_DEL_PRIVILEGES 0x6
#define TSDB_ALTER_USER_ADD_WHITE_LIST 0x7
#define TSDB_ALTER_USER_DROP_WHITE_LIST 0x8
#define TSDB_ALTER_USER_CREATEDB 0x9
#define TSDB_KILL_MSG_LEN 30
@ -359,6 +364,7 @@ typedef enum ENodeType {
QUERY_NODE_SHOW_TABLES_STMT,
QUERY_NODE_SHOW_TAGS_STMT,
QUERY_NODE_SHOW_USERS_STMT,
QUERY_NODE_SHOW_USERS_FULL_STMT,
QUERY_NODE_SHOW_LICENCES_STMT,
QUERY_NODE_SHOW_VGROUPS_STMT,
QUERY_NODE_SHOW_TOPICS_STMT,
@ -1014,6 +1020,8 @@ typedef struct {
SIpV4Range* pIpRanges;
int32_t sqlLen;
char* sql;
int8_t isImport;
int8_t createDb;
} SCreateUserReq;
int32_t tSerializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq);
@ -1045,11 +1053,18 @@ int32_t tSerializeRetrieveIpWhite(void* buf, int32_t bufLen, SRetrieveIpWhiteReq
int32_t tDeserializeRetrieveIpWhite(void* buf, int32_t bufLen, SRetrieveIpWhiteReq* pReq);
typedef struct {
int8_t alterType;
int8_t superUser;
int8_t sysInfo;
int8_t enable;
int8_t isView;
int8_t alterType;
int8_t superUser;
int8_t sysInfo;
int8_t enable;
int8_t isView;
union {
uint8_t flag;
struct {
uint8_t createdb : 1;
uint8_t reserve : 7;
};
};
char user[TSDB_USER_LEN];
char pass[TSDB_USET_PASSWORD_LEN];
char objname[TSDB_DB_FNAME_LEN]; // db or topic
@ -2101,6 +2116,7 @@ typedef struct {
char filterTb[TSDB_TABLE_NAME_LEN]; // for ins_columns
int64_t showId;
int64_t compactId; // for compact
bool withFull; // for show users full
} SRetrieveTableReq;
typedef struct SSysTableSchema {
@ -2118,6 +2134,7 @@ typedef struct {
int8_t precision;
int8_t compressed;
int8_t streamBlockType;
int32_t payloadLen;
int32_t compLen;
int32_t numOfBlocks;
int64_t numOfRows; // from int32_t change to int64_t
@ -2130,6 +2147,14 @@ typedef struct {
char data[];
} SRetrieveTableRsp;
#define PAYLOAD_PREFIX_LEN ((sizeof(int32_t)) << 1)
#define SET_PAYLOAD_LEN(_p, _compLen, _fullLen) \
do { \
((int32_t*)(_p))[0] = (_compLen); \
((int32_t*)(_p))[1] = (_fullLen); \
} while (0);
typedef struct {
int64_t version;
int64_t numOfRows;
@ -2146,6 +2171,7 @@ typedef struct {
int8_t compressed;
int32_t compLen;
int32_t numOfRows;
int32_t fullLen;
char data[];
} SRetrieveMetaTableRsp;
@ -2501,6 +2527,7 @@ typedef struct SSubQueryMsg {
int8_t taskType;
int8_t explain;
int8_t needFetch;
int8_t compress;
uint32_t sqlLen;
char* sql;
uint32_t msgLen;
@ -3219,6 +3246,7 @@ typedef struct {
typedef struct {
int64_t reqId;
SArray* reqs; // SArray<SClientHbReq>
int64_t ipWhiteList;
} SClientHbBatchReq;
typedef struct {
@ -3473,9 +3501,9 @@ typedef struct SVUpdateCheckpointInfoReq {
int64_t checkpointVer;
int64_t checkpointTs;
int32_t transId;
int8_t dropRelHTask;
int64_t hStreamId;
int64_t hStreamId; // add encode/decode
int64_t hTaskId;
int8_t dropRelHTask;
} SVUpdateCheckpointInfoReq;
typedef struct {
@ -3628,10 +3656,6 @@ typedef struct {
int32_t taskId;
} SVPauseStreamTaskReq, SVResetStreamTaskReq;
typedef struct {
int8_t reserved;
} SVPauseStreamTaskRsp;
typedef struct {
char name[TSDB_STREAM_FNAME_LEN];
int8_t igNotExists;

63
include/common/tmsgdef.h
View File

@ -17,6 +17,7 @@
#if 0
#undef TD_MSG_INFO_
#undef TD_MSG_TYPE_INFO_
#undef TD_MSG_NUMBER_
#undef TD_MSG_DICT_
#undef TD_MSG_SEG_CODE_
@ -40,7 +41,7 @@
#undef TD_CLOSE_MSG_SEG
#define TD_NEW_MSG_SEG(TYPE)
#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP)
#define TD_CLOSE_MSG_SEG(TYPE) TYPE,
#define TD_CLOSE_MSG_SEG(TYPE) TYPE##_MAX,
int32_t tMsgRangeDict[] = {
#elif defined(TD_MSG_NUMBER_)
@ -48,7 +49,7 @@
#undef TD_NEW_MSG_SEG
#undef TD_DEF_MSG_TYPE
#undef TD_CLOSE_MSG_SEG
#define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM,
#define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM_MIN,
#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE##_NUM, TYPE##_RSP_NUM,
#define TD_CLOSE_MSG_SEG(TYPE)
@ -59,7 +60,7 @@
#undef TD_NEW_MSG_SEG
#undef TD_DEF_MSG_TYPE
#undef TD_CLOSE_MSG_SEG
#define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM,
#define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM_MIN,
#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP)
#define TD_CLOSE_MSG_SEG(type)
@ -76,14 +77,31 @@
#define TD_CLOSE_MSG_SEG(TYPE)
enum {
#elif defined(TD_MSG_TYPE_INFO_)
typedef struct {
const char *name;
const char *rspName;
int32_t type;
int32_t rspType;
} SMsgTypeInfo;
#undef TD_NEW_MSG_SEG
#undef TD_DEF_MSG_TYPE
#undef TD_CLOSE_MSG_SEG
#define TD_NEW_MSG_SEG(TYPE)
#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) { #TYPE, #TYPE "_RSP", TYPE, TYPE##_RSP },
#define TD_CLOSE_MSG_SEG(TYPE)
SMsgTypeInfo tMsgTypeInfo[] = {
#else
#undef TD_NEW_MSG_SEG
#undef TD_DEF_MSG_TYPE
#undef TD_CLOSE_MSG_SEG
#define TD_NEW_MSG_SEG(TYPE) TYPE = ((TYPE##_SEG_CODE) << 8),
#define TD_NEW_MSG_SEG(TYPE) TYPE##_MIN = ((TYPE##_SEG_CODE) << 8),
#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE, TYPE##_RSP,
#define TD_CLOSE_MSG_SEG(TYPE) TYPE,
#define TD_CLOSE_MSG_SEG(TYPE) TYPE##_MAX,
enum { // WARN: new msg should be appended to segment tail
#endif
@ -107,8 +125,7 @@
TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE_TYPE, "dnode-alter-vnode-type", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, "dnode-check-vnode-learner-catchup", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_CREATE_ENCRYPT_KEY, "create-encrypt-key", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_MAX_MSG, "dnd-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_DND_MSG)
TD_CLOSE_MSG_SEG(TDMT_DND_MSG)
TD_NEW_MSG_SEG(TDMT_MND_MSG) // 1<<8
TD_DEF_MSG_TYPE(TDMT_MND_CONNECT, "connect", NULL, NULL)
@ -231,8 +248,9 @@
TD_DEF_MSG_TYPE(TDMT_MND_GET_TABLE_TSMA, "get-table-tsma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_GET_TSMA, "get-tsma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_TB_WITH_TSMA, "drop-tb-with-tsma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_UPDATE_CHKPT_EVT, "stream-update-chkpt-evt", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHKPT_REPORT, "stream-chkpt-report", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_MND_MSG)
TD_NEW_MSG_SEG(TDMT_VND_MSG) // 2<<8
TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp)
@ -287,8 +305,7 @@
TD_DEF_MSG_TYPE(TDMT_VND_ARB_HEARTBEAT, "vnode-arb-hb", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ARB_CHECK_SYNC, "vnode-arb-check-sync", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_FETCH_TTL_EXPIRED_TBS, "vnode-fetch-ttl-expired-tbs", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_VND_MSG)
TD_CLOSE_MSG_SEG(TDMT_VND_MSG)
TD_NEW_MSG_SEG(TDMT_SCH_MSG) // 3<<8
TD_DEF_MSG_TYPE(TDMT_SCH_QUERY, "query", NULL, NULL)
@ -302,8 +319,7 @@
TD_DEF_MSG_TYPE(TDMT_SCH_EXPLAIN, "explain", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SCH_TASK_NOTIFY, "task-notify", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SCH_MAX_MSG, "sch-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_SCH_MSG)
TD_CLOSE_MSG_SEG(TDMT_SCH_MSG)
TD_NEW_MSG_SEG(TDMT_STREAM_MSG) //4 << 8
@ -323,13 +339,10 @@
TD_DEF_MSG_TYPE(TDMT_STREAM_CREATE, "stream-create", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_DROP, "stream-drop", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE_TRIGGER, "stream-retri-trigger", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_STREAM_MSG)
TD_CLOSE_MSG_SEG(TDMT_STREAM_MSG)
TD_NEW_MSG_SEG(TDMT_MON_MSG) //5 << 8
TD_DEF_MSG_TYPE(TDMT_MON_MAX_MSG, "monitor-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_MON_MSG)
TD_CLOSE_MSG_SEG(TDMT_MON_MSG)
TD_NEW_MSG_SEG(TDMT_SYNC_MSG) //6 << 8
TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT, "sync-timer", NULL, NULL)
@ -362,8 +375,7 @@
TD_DEF_MSG_TYPE(TDMT_SYNC_UNUSED_CODE, "sync-unused", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SYNC_FORCE_FOLLOWER, "sync-force-become-follower", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SYNC_SET_ASSIGNED_LEADER, "sync-set-assigned-leader", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_SYNC_MSG)
TD_CLOSE_MSG_SEG(TDMT_SYNC_MSG)
TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG) //7 << 8
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_SCAN_HISTORY, "vnode-stream-scan-history", NULL, NULL)
@ -373,8 +385,7 @@
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_UNUSED, "vnd-stream-unused", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_GET_STREAM_PROGRESS, "vnd-stream-progress", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_VND_STREAM_MSG)
TD_CLOSE_MSG_SEG(TDMT_VND_STREAM_MSG)
TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG) //8 << 8
TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SUBSCRIBE, "vnode-tmq-subscribe", SMqRebVgReq, SMqRebVgRsp)
@ -387,19 +398,17 @@
TD_DEF_MSG_TYPE(TDMT_VND_TMQ_CONSUME_PUSH, "vnode-tmq-consume-push", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_WALINFO, "vnode-tmq-vg-walinfo", SMqPollReq, SMqDataBlkRsp)
TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_COMMITTEDINFO, "vnode-tmq-committedinfo", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_TMQ_MAX_MSG, "vnd-tmq-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_TMQ_MSG)
TD_CLOSE_MSG_SEG(TDMT_VND_TMQ_MSG)
TD_NEW_MSG_SEG(TDMT_MND_ARB_MSG) //9 << 8
TD_DEF_MSG_TYPE(TDMT_MND_ARB_HEARTBEAT_TIMER, "mnd-arb-hb-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_ARB_CHECK_SYNC_TIMER, "mnd-arb-check-sync-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP, "mnd-arb-update-group", NULL, NULL) // no longer used
TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP_BATCH, "mnd-arb-update-group-batch", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_ARB_MAX_MSG, "mnd-arb-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_ARB_MSG)
TD_CLOSE_MSG_SEG(TDMT_MND_ARB_MSG)
TD_NEW_MSG_SEG(TDMT_MAX_MSG) // msg end mark
TD_CLOSE_MSG_SEG(TDMT_END_MAX_MSG)
TD_CLOSE_MSG_SEG(TDMT_MAX_MSG)
#if defined(TD_MSG_NUMBER_)
TDMT_MAX

1
include/common/trow.h
View File

@ -289,7 +289,6 @@ int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValT
*/
static FORCE_INLINE void tdSRowInit(SRowBuilder *pBuilder, int16_t sver) {
pBuilder->rowType = pBuilder->rowType;
pBuilder->sver = sver;
}
int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen);

752
include/common/ttokendef.h
View File

@ -16,381 +16,383 @@
#ifndef _TD_COMMON_TOKEN_H_
#define _TD_COMMON_TOKEN_H_
#define TK_OR 1
#define TK_AND 2
#define TK_UNION 3
#define TK_ALL 4
#define TK_MINUS 5
#define TK_EXCEPT 6
#define TK_INTERSECT 7
#define TK_NK_BITAND 8
#define TK_NK_BITOR 9
#define TK_NK_LSHIFT 10
#define TK_NK_RSHIFT 11
#define TK_NK_PLUS 12
#define TK_NK_MINUS 13
#define TK_NK_STAR 14
#define TK_NK_SLASH 15
#define TK_NK_REM 16
#define TK_NK_CONCAT 17
#define TK_CREATE 18
#define TK_ACCOUNT 19
#define TK_NK_ID 20
#define TK_PASS 21
#define TK_NK_STRING 22
#define TK_ALTER 23
#define TK_PPS 24
#define TK_TSERIES 25
#define TK_STORAGE 26
#define TK_STREAMS 27
#define TK_QTIME 28
#define TK_DBS 29
#define TK_USERS 30
#define TK_CONNS 31
#define TK_STATE 32
#define TK_NK_COMMA 33
#define TK_HOST 34
#define TK_USER 35
#define TK_ENABLE 36
#define TK_NK_INTEGER 37
#define TK_SYSINFO 38
#define TK_ADD 39
#define TK_DROP 40
#define TK_GRANT 41
#define TK_ON 42
#define TK_TO 43
#define TK_REVOKE 44
#define TK_FROM 45
#define TK_SUBSCRIBE 46
#define TK_READ 47
#define TK_WRITE 48
#define TK_NK_DOT 49
#define TK_WITH 50
#define TK_ENCRYPT_KEY 51
#define TK_DNODE 52
#define TK_PORT 53
#define TK_DNODES 54
#define TK_RESTORE 55
#define TK_NK_IPTOKEN 56
#define TK_FORCE 57
#define TK_UNSAFE 58
#define TK_CLUSTER 59
#define TK_LOCAL 60
#define TK_QNODE 61
#define TK_BNODE 62
#define TK_SNODE 63
#define TK_MNODE 64
#define TK_VNODE 65
#define TK_DATABASE 66
#define TK_USE 67
#define TK_FLUSH 68
#define TK_TRIM 69
#define TK_S3MIGRATE 70
#define TK_COMPACT 71
#define TK_IF 72
#define TK_NOT 73
#define TK_EXISTS 74
#define TK_BUFFER 75
#define TK_CACHEMODEL 76
#define TK_CACHESIZE 77
#define TK_COMP 78
#define TK_DURATION 79
#define TK_NK_VARIABLE 80
#define TK_MAXROWS 81
#define TK_MINROWS 82
#define TK_KEEP 83
#define TK_PAGES 84
#define TK_PAGESIZE 85
#define TK_TSDB_PAGESIZE 86
#define TK_PRECISION 87
#define TK_REPLICA 88
#define TK_VGROUPS 89
#define TK_SINGLE_STABLE 90
#define TK_RETENTIONS 91
#define TK_SCHEMALESS 92
#define TK_WAL_LEVEL 93
#define TK_WAL_FSYNC_PERIOD 94
#define TK_WAL_RETENTION_PERIOD 95
#define TK_WAL_RETENTION_SIZE 96
#define TK_WAL_ROLL_PERIOD 97
#define TK_WAL_SEGMENT_SIZE 98
#define TK_STT_TRIGGER 99
#define TK_TABLE_PREFIX 100
#define TK_TABLE_SUFFIX 101
#define TK_S3_CHUNKSIZE 102
#define TK_S3_KEEPLOCAL 103
#define TK_S3_COMPACT 104
#define TK_KEEP_TIME_OFFSET 105
#define TK_ENCRYPT_ALGORITHM 106
#define TK_NK_COLON 107
#define TK_BWLIMIT 108
#define TK_START 109
#define TK_TIMESTAMP 110
#define TK_END 111
#define TK_TABLE 112
#define TK_NK_LP 113
#define TK_NK_RP 114
#define TK_STABLE 115
#define TK_COLUMN 116
#define TK_MODIFY 117
#define TK_RENAME 118
#define TK_TAG 119
#define TK_SET 120
#define TK_NK_EQ 121
#define TK_USING 122
#define TK_TAGS 123
#define TK_BOOL 124
#define TK_TINYINT 125
#define TK_SMALLINT 126
#define TK_INT 127
#define TK_INTEGER 128
#define TK_BIGINT 129
#define TK_FLOAT 130
#define TK_DOUBLE 131
#define TK_BINARY 132
#define TK_NCHAR 133
#define TK_UNSIGNED 134
#define TK_JSON 135
#define TK_VARCHAR 136
#define TK_MEDIUMBLOB 137
#define TK_BLOB 138
#define TK_VARBINARY 139
#define TK_GEOMETRY 140
#define TK_DECIMAL 141
#define TK_COMMENT 142
#define TK_MAX_DELAY 143
#define TK_WATERMARK 144
#define TK_ROLLUP 145
#define TK_TTL 146
#define TK_SMA 147
#define TK_DELETE_MARK 148
#define TK_FIRST 149
#define TK_LAST 150
#define TK_SHOW 151
#define TK_PRIVILEGES 152
#define TK_DATABASES 153
#define TK_TABLES 154
#define TK_STABLES 155
#define TK_MNODES 156
#define TK_QNODES 157
#define TK_ARBGROUPS 158
#define TK_FUNCTIONS 159
#define TK_INDEXES 160
#define TK_ACCOUNTS 161
#define TK_APPS 162
#define TK_CONNECTIONS 163
#define TK_LICENCES 164
#define TK_GRANTS 165
#define TK_FULL 166
#define TK_LOGS 167
#define TK_MACHINES 168
#define TK_ENCRYPTIONS 169
#define TK_QUERIES 170
#define TK_SCORES 171
#define TK_TOPICS 172
#define TK_VARIABLES 173
#define TK_BNODES 174
#define TK_SNODES 175
#define TK_TRANSACTIONS 176
#define TK_DISTRIBUTED 177
#define TK_CONSUMERS 178
#define TK_SUBSCRIPTIONS 179
#define TK_VNODES 180
#define TK_ALIVE 181
#define TK_VIEWS 182
#define TK_VIEW 183
#define TK_COMPACTS 184
#define TK_NORMAL 185
#define TK_CHILD 186
#define TK_LIKE 187
#define TK_TBNAME 188
#define TK_QTAGS 189
#define TK_AS 190
#define TK_SYSTEM 191
#define TK_TSMA 192
#define TK_INTERVAL 193
#define TK_RECURSIVE 194
#define TK_TSMAS 195
#define TK_FUNCTION 196
#define TK_INDEX 197
#define TK_COUNT 198
#define TK_LAST_ROW 199
#define TK_META 200
#define TK_ONLY 201
#define TK_TOPIC 202
#define TK_CONSUMER 203
#define TK_GROUP 204
#define TK_DESC 205
#define TK_DESCRIBE 206
#define TK_RESET 207
#define TK_QUERY 208
#define TK_CACHE 209
#define TK_EXPLAIN 210
#define TK_ANALYZE 211
#define TK_VERBOSE 212
#define TK_NK_BOOL 213
#define TK_RATIO 214
#define TK_NK_FLOAT 215
#define TK_OUTPUTTYPE 216
#define TK_AGGREGATE 217
#define TK_BUFSIZE 218
#define TK_LANGUAGE 219
#define TK_REPLACE 220
#define TK_STREAM 221
#define TK_INTO 222
#define TK_PAUSE 223
#define TK_RESUME 224
#define TK_PRIMARY 225
#define TK_KEY 226
#define TK_TRIGGER 227
#define TK_AT_ONCE 228
#define TK_WINDOW_CLOSE 229
#define TK_IGNORE 230
#define TK_EXPIRED 231
#define TK_FILL_HISTORY 232
#define TK_UPDATE 233
#define TK_SUBTABLE 234
#define TK_UNTREATED 235
#define TK_KILL 236
#define TK_CONNECTION 237
#define TK_TRANSACTION 238
#define TK_BALANCE 239
#define TK_VGROUP 240
#define TK_LEADER 241
#define TK_MERGE 242
#define TK_REDISTRIBUTE 243
#define TK_SPLIT 244
#define TK_DELETE 245
#define TK_INSERT 246
#define TK_NK_BIN 247
#define TK_NK_HEX 248
#define TK_NULL 249
#define TK_NK_QUESTION 250
#define TK_NK_ALIAS 251
#define TK_NK_ARROW 252
#define TK_ROWTS 253
#define TK_QSTART 254
#define TK_QEND 255
#define TK_QDURATION 256
#define TK_WSTART 257
#define TK_WEND 258
#define TK_WDURATION 259
#define TK_IROWTS 260
#define TK_ISFILLED 261
#define TK_CAST 262
#define TK_NOW 263
#define TK_TODAY 264
#define TK_TIMEZONE 265
#define TK_CLIENT_VERSION 266
#define TK_SERVER_VERSION 267
#define TK_SERVER_STATUS 268
#define TK_CURRENT_USER 269
#define TK_CASE 270
#define TK_WHEN 271
#define TK_THEN 272
#define TK_ELSE 273
#define TK_BETWEEN 274
#define TK_IS 275
#define TK_NK_LT 276
#define TK_NK_GT 277
#define TK_NK_LE 278
#define TK_NK_GE 279
#define TK_NK_NE 280
#define TK_MATCH 281
#define TK_NMATCH 282
#define TK_CONTAINS 283
#define TK_IN 284
#define TK_JOIN 285
#define TK_INNER 286
#define TK_LEFT 287
#define TK_RIGHT 288
#define TK_OUTER 289
#define TK_SEMI 290
#define TK_ANTI 291
#define TK_ASOF 292
#define TK_WINDOW 293
#define TK_WINDOW_OFFSET 294
#define TK_JLIMIT 295
#define TK_SELECT 296
#define TK_NK_HINT 297
#define TK_DISTINCT 298
#define TK_WHERE 299
#define TK_PARTITION 300
#define TK_BY 301
#define TK_SESSION 302
#define TK_STATE_WINDOW 303
#define TK_EVENT_WINDOW 304
#define TK_COUNT_WINDOW 305
#define TK_SLIDING 306
#define TK_FILL 307
#define TK_VALUE 308
#define TK_VALUE_F 309
#define TK_NONE 310
#define TK_PREV 311
#define TK_NULL_F 312
#define TK_LINEAR 313
#define TK_NEXT 314
#define TK_HAVING 315
#define TK_RANGE 316
#define TK_EVERY 317
#define TK_ORDER 318
#define TK_SLIMIT 319
#define TK_SOFFSET 320
#define TK_LIMIT 321
#define TK_OFFSET 322
#define TK_ASC 323
#define TK_NULLS 324
#define TK_ABORT 325
#define TK_AFTER 326
#define TK_ATTACH 327
#define TK_BEFORE 328
#define TK_BEGIN 329
#define TK_BITAND 330
#define TK_BITNOT 331
#define TK_BITOR 332
#define TK_BLOCKS 333
#define TK_CHANGE 334
#define TK_COMMA 335
#define TK_CONCAT 336
#define TK_CONFLICT 337
#define TK_COPY 338
#define TK_DEFERRED 339
#define TK_DELIMITERS 340
#define TK_DETACH 341
#define TK_DIVIDE 342
#define TK_DOT 343
#define TK_EACH 344
#define TK_FAIL 345
#define TK_FILE 346
#define TK_FOR 347
#define TK_GLOB 348
#define TK_ID 349
#define TK_IMMEDIATE 350
#define TK_IMPORT 351
#define TK_INITIALLY 352
#define TK_INSTEAD 353
#define TK_ISNULL 354
#define TK_MODULES 355
#define TK_NK_BITNOT 356
#define TK_NK_SEMI 357
#define TK_NOTNULL 358
#define TK_OF 359
#define TK_PLUS 360
#define TK_PRIVILEGE 361
#define TK_RAISE 362
#define TK_RESTRICT 363
#define TK_ROW 364
#define TK_STAR 365
#define TK_STATEMENT 366
#define TK_STRICT 367
#define TK_STRING 368
#define TK_TIMES 369
#define TK_VALUES 370
#define TK_VARIABLE 371
#define TK_WAL 372
#define TK_ENCODE 373
#define TK_COMPRESS 374
#define TK_LEVEL 375
#define TK_OR 1
#define TK_AND 2
#define TK_UNION 3
#define TK_ALL 4
#define TK_MINUS 5
#define TK_EXCEPT 6
#define TK_INTERSECT 7
#define TK_NK_BITAND 8
#define TK_NK_BITOR 9
#define TK_NK_LSHIFT 10
#define TK_NK_RSHIFT 11
#define TK_NK_PLUS 12
#define TK_NK_MINUS 13
#define TK_NK_STAR 14
#define TK_NK_SLASH 15
#define TK_NK_REM 16
#define TK_NK_CONCAT 17
#define TK_CREATE 18
#define TK_ACCOUNT 19
#define TK_NK_ID 20
#define TK_PASS 21
#define TK_NK_STRING 22
#define TK_ALTER 23
#define TK_PPS 24
#define TK_TSERIES 25
#define TK_STORAGE 26
#define TK_STREAMS 27
#define TK_QTIME 28
#define TK_DBS 29
#define TK_USERS 30
#define TK_CONNS 31
#define TK_STATE 32
#define TK_NK_COMMA 33
#define TK_HOST 34
#define TK_IS_IMPORT 35
#define TK_NK_INTEGER 36
#define TK_CREATEDB 37
#define TK_USER 38
#define TK_ENABLE 39
#define TK_SYSINFO 40
#define TK_ADD 41
#define TK_DROP 42
#define TK_GRANT 43
#define TK_ON 44
#define TK_TO 45
#define TK_REVOKE 46
#define TK_FROM 47
#define TK_SUBSCRIBE 48
#define TK_READ 49
#define TK_WRITE 50
#define TK_NK_DOT 51
#define TK_WITH 52
#define TK_ENCRYPT_KEY 53
#define TK_DNODE 54
#define TK_PORT 55
#define TK_DNODES 56
#define TK_RESTORE 57
#define TK_NK_IPTOKEN 58
#define TK_FORCE 59
#define TK_UNSAFE 60
#define TK_CLUSTER 61
#define TK_LOCAL 62
#define TK_QNODE 63
#define TK_BNODE 64
#define TK_SNODE 65
#define TK_MNODE 66
#define TK_VNODE 67
#define TK_DATABASE 68
#define TK_USE 69
#define TK_FLUSH 70
#define TK_TRIM 71
#define TK_S3MIGRATE 72
#define TK_COMPACT 73
#define TK_IF 74
#define TK_NOT 75
#define TK_EXISTS 76
#define TK_BUFFER 77
#define TK_CACHEMODEL 78
#define TK_CACHESIZE 79
#define TK_COMP 80
#define TK_DURATION 81
#define TK_NK_VARIABLE 82
#define TK_MAXROWS 83
#define TK_MINROWS 84
#define TK_KEEP 85
#define TK_PAGES 86
#define TK_PAGESIZE 87
#define TK_TSDB_PAGESIZE 88
#define TK_PRECISION 89
#define TK_REPLICA 90
#define TK_VGROUPS 91
#define TK_SINGLE_STABLE 92
#define TK_RETENTIONS 93
#define TK_SCHEMALESS 94
#define TK_WAL_LEVEL 95
#define TK_WAL_FSYNC_PERIOD 96
#define TK_WAL_RETENTION_PERIOD 97
#define TK_WAL_RETENTION_SIZE 98
#define TK_WAL_ROLL_PERIOD 99
#define TK_WAL_SEGMENT_SIZE 100
#define TK_STT_TRIGGER 101
#define TK_TABLE_PREFIX 102
#define TK_TABLE_SUFFIX 103
#define TK_S3_CHUNKSIZE 104
#define TK_S3_KEEPLOCAL 105
#define TK_S3_COMPACT 106
#define TK_KEEP_TIME_OFFSET 107
#define TK_ENCRYPT_ALGORITHM 108
#define TK_NK_COLON 109
#define TK_BWLIMIT 110
#define TK_START 111
#define TK_TIMESTAMP 112
#define TK_END 113
#define TK_TABLE 114
#define TK_NK_LP 115
#define TK_NK_RP 116
#define TK_STABLE 117
#define TK_COLUMN 118
#define TK_MODIFY 119
#define TK_RENAME 120
#define TK_TAG 121
#define TK_SET 122
#define TK_NK_EQ 123
#define TK_USING 124
#define TK_TAGS 125
#define TK_BOOL 126
#define TK_TINYINT 127
#define TK_SMALLINT 128
#define TK_INT 129
#define TK_INTEGER 130
#define TK_BIGINT 131
#define TK_FLOAT 132
#define TK_DOUBLE 133
#define TK_BINARY 134
#define TK_NCHAR 135
#define TK_UNSIGNED 136
#define TK_JSON 137
#define TK_VARCHAR 138
#define TK_MEDIUMBLOB 139
#define TK_BLOB 140
#define TK_VARBINARY 141
#define TK_GEOMETRY 142
#define TK_DECIMAL 143
#define TK_COMMENT 144
#define TK_MAX_DELAY 145
#define TK_WATERMARK 146
#define TK_ROLLUP 147
#define TK_TTL 148
#define TK_SMA 149
#define TK_DELETE_MARK 150
#define TK_FIRST 151
#define TK_LAST 152
#define TK_SHOW 153
#define TK_FULL 154
#define TK_PRIVILEGES 155
#define TK_DATABASES 156
#define TK_TABLES 157
#define TK_STABLES 158
#define TK_MNODES 159
#define TK_QNODES 160
#define TK_ARBGROUPS 161
#define TK_FUNCTIONS 162
#define TK_INDEXES 163
#define TK_ACCOUNTS 164
#define TK_APPS 165
#define TK_CONNECTIONS 166
#define TK_LICENCES 167
#define TK_GRANTS 168
#define TK_LOGS 169
#define TK_MACHINES 170
#define TK_ENCRYPTIONS 171
#define TK_QUERIES 172
#define TK_SCORES 173
#define TK_TOPICS 174
#define TK_VARIABLES 175
#define TK_BNODES 176
#define TK_SNODES 177
#define TK_TRANSACTIONS 178
#define TK_DISTRIBUTED 179
#define TK_CONSUMERS 180
#define TK_SUBSCRIPTIONS 181
#define TK_VNODES 182
#define TK_ALIVE 183
#define TK_VIEWS 184
#define TK_VIEW 185
#define TK_COMPACTS 186
#define TK_NORMAL 187
#define TK_CHILD 188
#define TK_LIKE 189
#define TK_TBNAME 190
#define TK_QTAGS 191
#define TK_AS 192
#define TK_SYSTEM 193
#define TK_TSMA 194
#define TK_INTERVAL 195
#define TK_RECURSIVE 196
#define TK_TSMAS 197
#define TK_FUNCTION 198
#define TK_INDEX 199
#define TK_COUNT 200
#define TK_LAST_ROW 201
#define TK_META 202
#define TK_ONLY 203
#define TK_TOPIC 204
#define TK_CONSUMER 205
#define TK_GROUP 206
#define TK_DESC 207
#define TK_DESCRIBE 208
#define TK_RESET 209
#define TK_QUERY 210
#define TK_CACHE 211
#define TK_EXPLAIN 212
#define TK_ANALYZE 213
#define TK_VERBOSE 214
#define TK_NK_BOOL 215
#define TK_RATIO 216
#define TK_NK_FLOAT 217
#define TK_OUTPUTTYPE 218
#define TK_AGGREGATE 219
#define TK_BUFSIZE 220
#define TK_LANGUAGE 221
#define TK_REPLACE 222
#define TK_STREAM 223
#define TK_INTO 224
#define TK_PAUSE 225
#define TK_RESUME 226
#define TK_PRIMARY 227
#define TK_KEY 228
#define TK_TRIGGER 229
#define TK_AT_ONCE 230
#define TK_WINDOW_CLOSE 231
#define TK_IGNORE 232
#define TK_EXPIRED 233
#define TK_FILL_HISTORY 234
#define TK_UPDATE 235
#define TK_SUBTABLE 236
#define TK_UNTREATED 237
#define TK_KILL 238
#define TK_CONNECTION 239
#define TK_TRANSACTION 240
#define TK_BALANCE 241
#define TK_VGROUP 242
#define TK_LEADER 243
#define TK_MERGE 244
#define TK_REDISTRIBUTE 245
#define TK_SPLIT 246
#define TK_DELETE 247
#define TK_INSERT 248
#define TK_NK_BIN 249
#define TK_NK_HEX 250
#define TK_NULL 251
#define TK_NK_QUESTION 252
#define TK_NK_ALIAS 253
#define TK_NK_ARROW 254
#define TK_ROWTS 255
#define TK_QSTART 256
#define TK_QEND 257
#define TK_QDURATION 258
#define TK_WSTART 259
#define TK_WEND 260
#define TK_WDURATION 261
#define TK_IROWTS 262
#define TK_ISFILLED 263
#define TK_CAST 264
#define TK_NOW 265
#define TK_TODAY 266
#define TK_TIMEZONE 267
#define TK_CLIENT_VERSION 268
#define TK_SERVER_VERSION 269
#define TK_SERVER_STATUS 270
#define TK_CURRENT_USER 271
#define TK_CASE 272
#define TK_WHEN 273
#define TK_THEN 274
#define TK_ELSE 275
#define TK_BETWEEN 276
#define TK_IS 277
#define TK_NK_LT 278
#define TK_NK_GT 279
#define TK_NK_LE 280
#define TK_NK_GE 281
#define TK_NK_NE 282
#define TK_MATCH 283
#define TK_NMATCH 284
#define TK_CONTAINS 285
#define TK_IN 286
#define TK_JOIN 287
#define TK_INNER 288
#define TK_LEFT 289
#define TK_RIGHT 290
#define TK_OUTER 291
#define TK_SEMI 292
#define TK_ANTI 293
#define TK_ASOF 294
#define TK_WINDOW 295
#define TK_WINDOW_OFFSET 296
#define TK_JLIMIT 297
#define TK_SELECT 298
#define TK_NK_HINT 299
#define TK_DISTINCT 300
#define TK_WHERE 301
#define TK_PARTITION 302
#define TK_BY 303
#define TK_SESSION 304
#define TK_STATE_WINDOW 305
#define TK_EVENT_WINDOW 306
#define TK_COUNT_WINDOW 307
#define TK_SLIDING 308
#define TK_FILL 309
#define TK_VALUE 310
#define TK_VALUE_F 311
#define TK_NONE 312
#define TK_PREV 313
#define TK_NULL_F 314
#define TK_LINEAR 315
#define TK_NEXT 316
#define TK_HAVING 317
#define TK_RANGE 318
#define TK_EVERY 319
#define TK_ORDER 320
#define TK_SLIMIT 321
#define TK_SOFFSET 322
#define TK_LIMIT 323
#define TK_OFFSET 324
#define TK_ASC 325
#define TK_NULLS 326
#define TK_ABORT 327
#define TK_AFTER 328
#define TK_ATTACH 329
#define TK_BEFORE 330
#define TK_BEGIN 331
#define TK_BITAND 332
#define TK_BITNOT 333
#define TK_BITOR 334
#define TK_BLOCKS 335
#define TK_CHANGE 336
#define TK_COMMA 337
#define TK_CONCAT 338
#define TK_CONFLICT 339
#define TK_COPY 340
#define TK_DEFERRED 341
#define TK_DELIMITERS 342
#define TK_DETACH 343
#define TK_DIVIDE 344
#define TK_DOT 345
#define TK_EACH 346
#define TK_FAIL 347
#define TK_FILE 348
#define TK_FOR 349
#define TK_GLOB 350
#define TK_ID 351
#define TK_IMMEDIATE 352
#define TK_IMPORT 353
#define TK_INITIALLY 354
#define TK_INSTEAD 355
#define TK_ISNULL 356
#define TK_MODULES 357
#define TK_NK_BITNOT 358
#define TK_NK_SEMI 359
#define TK_NOTNULL 360
#define TK_OF 361
#define TK_PLUS 362
#define TK_PRIVILEGE 363
#define TK_RAISE 364
#define TK_RESTRICT 365
#define TK_ROW 366
#define TK_STAR 367
#define TK_STATEMENT 368
#define TK_STRICT 369
#define TK_STRING 370
#define TK_TIMES 371
#define TK_VALUES 372
#define TK_VARIABLE 373
#define TK_WAL 374
#define TK_ENCODE 375
#define TK_COMPRESS 376
#define TK_LEVEL 377
#define TK_NK_SPACE 600
#define TK_NK_COMMENT 601

5
include/dnode/vnode/tqCommon.h
View File

@ -28,6 +28,7 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe
int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sversion, char* msg, int32_t msgLen,
bool isLeader, bool restored);
@ -40,9 +41,9 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg);
int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* pMsg, bool fromVnode);
int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, char* msg, int32_t msgLen);
int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored, char* msg, int32_t msgLen);
int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta);
void tqSetRestoreVersionInfo(SStreamTask* pTask);
int32_t tqExpandStreamTask(SStreamTask* pTask);
#endif // TDENGINE_TQ_COMMON_H

3
include/libs/executor/dataSinkMgt.h
View File

@ -55,6 +55,7 @@ typedef struct SDataSinkStat {
} SDataSinkStat;
typedef struct SDataSinkMgtCfg {
int8_t compress;
uint32_t maxDataBlockNum; // todo: this should be numOfRows?
uint32_t maxDataBlockNumPerQuery;
} SDataSinkMgtCfg;
@ -104,7 +105,7 @@ void dsReset(DataSinkHandle handle);
* @param handle
* @param pLen data length
*/
void dsGetDataLength(DataSinkHandle handle, int64_t* pLen, bool* pQueryEnd);
void dsGetDataLength(DataSinkHandle handle, int64_t* pLen, int64_t* pRawLen, bool* pQueryEnd);
/**
* Get data, the caller needs to allocate data memory.

5
include/libs/executor/executor.h
View File

@ -139,8 +139,9 @@ void qUpdateOperatorParam(qTaskInfo_t tinfo, void* pParam);
* @param qId
* @return
*/
int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan,
qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, char* sql, EOPTR_EXEC_MODEL model);
int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pSubplan,
qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, int8_t compressResult, char* sql,
EOPTR_EXEC_MODEL model);
/**
*

4
include/libs/executor/storageapi.h
View File

@ -329,7 +329,7 @@ typedef struct {
typedef struct SStateStore {
int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname);
int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal);
int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache);
int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t (*streamStateReleaseBuf)(SStreamState* pState, void* pVal, bool used);
@ -410,7 +410,7 @@ typedef struct SStateStore {
void (*streamFileStateClear)(struct SStreamFileState* pFileState);
bool (*needClearDiskBuff)(struct SStreamFileState* pFileState);
SStreamState* (*streamStateOpen)(const char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages);
SStreamState* (*streamStateOpen)(const char* path, void* pTask, int64_t streamId, int32_t taskId);
void (*streamStateClose)(SStreamState* pState, bool remove);
int32_t (*streamStateBegin)(SStreamState* pState);
int32_t (*streamStateCommit)(SStreamState* pState);

4
include/libs/nodes/cmdnodes.h
View File

@ -270,6 +270,8 @@ typedef struct SCreateUserStmt {
char userName[TSDB_USER_LEN];
char password[TSDB_USET_PASSWORD_LEN];
int8_t sysinfo;
int8_t createDb;
int8_t isImport;
int32_t numIpRanges;
SIpV4Range* pIpRanges;
@ -283,6 +285,7 @@ typedef struct SAlterUserStmt {
char password[TSDB_USET_PASSWORD_LEN];
int8_t enable;
int8_t sysinfo;
int8_t createdb;
int32_t numIpRanges;
SIpV4Range* pIpRanges;
@ -322,6 +325,7 @@ typedef struct SShowStmt {
SNode* pTbName; // SValueNode
EOperatorType tableCondType;
EShowKind showKind; // show databases: user/system, show tables: normal/child, others NULL
bool withFull; // for show users full;
} SShowStmt;
typedef struct SShowCreateDatabaseStmt {

1
include/libs/nodes/querynodes.h
View File

@ -628,6 +628,7 @@ char* nodesGetStrValueFromNode(SValueNode* pNode);
void nodesValueNodeToVariant(const SValueNode* pNode, SVariant* pVal);
SValueNode* nodesMakeValueNodeFromString(char* literal);
SValueNode* nodesMakeValueNodeFromBool(bool b);
SNode* nodesMakeValueNodeFromInt32(int32_t value);
char* nodesGetFillModeString(EFillMode mode);
int32_t nodesMergeConds(SNode** pDst, SNodeList** pSrc);

10
include/libs/parser/parser.h
View File

@ -118,7 +118,11 @@ int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** p
int32_t qSetSTableIdForRsma(SNode* pStmt, int64_t uid);
void qCleanupKeywordsTable();
int32_t qAppendStmtTableOutput(SQuery* pQuery, SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx, SStbInterlaceInfo* pBuildInfo);
int32_t qBuildStmtFinOutput(SQuery* pQuery, SHashObj* pAllVgHash, SArray* pVgDataBlocks);
//int32_t qBuildStmtOutputFromTbList(SQuery* pQuery, SHashObj* pVgHash, SArray* pBlockList, STableDataCxt* pTbCtx, int32_t tbNum);
int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash);
int32_t qResetStmtColumns(SArray* pCols, bool deepClear);
int32_t qResetStmtDataBlock(STableDataCxt* block, bool keepBuf);
int32_t qCloneStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, bool reset);
int32_t qRebuildStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_t uid, uint64_t suid, int32_t vgId,
@ -129,8 +133,9 @@ int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData** pData
int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx);
int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery);
int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen);
int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx,
int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, STSchema** pTSchema, SBindInfo* pBindInfos);
int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen);
int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx,
int32_t rowNum);
int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD_E** fields);
int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields);
@ -160,6 +165,7 @@ SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap);
SArray* serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap);
void destoryCatalogReq(SCatalogReq *pCatalogReq);
bool isPrimaryKeyImpl(SNode* pExpr);
int32_t insAppendStmtTableDataCxt(SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx, SStbInterlaceInfo* pBuildInfo);
#ifdef __cplusplus
}

49
include/libs/qcom/query.h
View File

@ -25,6 +25,7 @@ extern "C" {
#include "tarray.h"
#include "thash.h"
#include "tlog.h"
#include "tsimplehash.h"
#include "tmsg.h"
#include "tmsgcb.h"
@ -193,6 +194,27 @@ typedef struct SBoundColInfo {
int32_t numOfBound;
} SBoundColInfo;
typedef struct STableColsData {
char tbName[TSDB_TABLE_NAME_LEN];
SArray* aCol;
bool getFromHash;
} STableColsData;
typedef struct STableVgUid {
uint64_t uid;
int32_t vgid;
} STableVgUid;
typedef struct STableBufInfo {
void* pCurBuff;
SArray* pBufList;
int64_t buffUnit;
int64_t buffSize;
int64_t buffIdx;
int64_t buffOffset;
} STableBufInfo;
typedef struct STableDataCxt {
STableMeta* pMeta;
STSchema* pSchema;
@ -204,6 +226,33 @@ typedef struct STableDataCxt {
bool duplicateTs;
} STableDataCxt;
typedef struct SStbInterlaceInfo {
void* pCatalog;
void* pQuery;
int32_t acctId;
char* dbname;
void* transport;
SEpSet mgmtEpSet;
void* pRequest;
uint64_t requestId;
int64_t requestSelf;
bool tbFromHash;
SHashObj* pVgroupHash;
SArray* pVgroupList;
SSHashObj* pTableHash;
int64_t tbRemainNum;
STableBufInfo tbBuf;
char firstName[TSDB_TABLE_NAME_LEN];
STSchema *pTSchema;
STableDataCxt *pDataCtx;
void *boundTags;
bool tableColsReady;
SArray *pTableCols;
int32_t pTableColsIdx;
} SStbInterlaceInfo;
typedef int32_t (*__async_send_cb_fn_t)(void* param, SDataBuf* pMsg, int32_t code);
typedef int32_t (*__async_exec_fn_t)(void* param);

1
include/libs/qworker/qworker.h
View File

@ -61,6 +61,7 @@ typedef struct SQWMsgInfo {
int8_t taskType;
int8_t explain;
int8_t needFetch;
int8_t compressMsg;
} SQWMsgInfo;
typedef struct SQWMsg {

2
include/libs/scalar/filter.h
View File

@ -58,7 +58,7 @@ extern int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict
extern int32_t filterConverNcharColumns(SFilterInfo *pFilterInfo, int32_t rows, bool *gotNchar);
extern int32_t filterFreeNcharColumns(SFilterInfo *pFilterInfo);
extern void filterFreeInfo(SFilterInfo *info);
extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pColsAgg, int32_t numOfCols, int32_t numOfRows);
extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pColsAgg, int32_t numOfCols, int32_t numOfRows);
/* condition split interface */
int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond,

10
include/libs/scheduler/scheduler.h
View File

@ -48,16 +48,6 @@ typedef struct SQueryProfileSummary {
uint64_t resultSize; // generated result size in Kb.
} SQueryProfileSummary;
typedef struct STaskInfo {
SQueryNodeAddr addr;
SSubQueryMsg* msg;
} STaskInfo;
typedef struct SSchdFetchParam {
void** pData;
int32_t* code;
} SSchdFetchParam;
typedef void (*schedulerExecFp)(SExecResult* pResult, void* param, int32_t code);
typedef void (*schedulerFetchFp)(void* pResult, void* param, int32_t code);
typedef bool (*schedulerChkKillFp)(void* param);

4
include/libs/stream/streamState.h
View File

@ -29,7 +29,7 @@ extern "C" {
#include "storageapi.h"
SStreamState* streamStateOpen(const char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages);
SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId);
void streamStateClose(SStreamState* pState, bool remove);
int32_t streamStateBegin(SStreamState* pState);
int32_t streamStateCommit(SStreamState* pState);
@ -101,7 +101,7 @@ int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname);
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal);
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache);
void streamStateReloadInfo(SStreamState* pState, TSKEY ts);

14
include/libs/stream/streammsg.h
View File

@ -190,6 +190,20 @@ typedef struct SCheckpointTriggerRsp {
int32_t rspCode;
} SCheckpointTriggerRsp;
typedef struct SCheckpointReport {
int64_t streamId;
int32_t taskId;
int32_t nodeId;
int64_t checkpointId;
int64_t checkpointVer;
int64_t checkpointTs;
int32_t transId;
int8_t dropHTask;
} SCheckpointReport;
int32_t tEncodeStreamTaskChkptReport(SEncoder* pEncoder, const SCheckpointReport* pReq);
int32_t tDecodeStreamTaskChkptReport(SDecoder* pDecoder, SCheckpointReport* pReq);
typedef struct {
SMsgHead head;
int64_t streamId;

43
include/libs/stream/tstream.h
View File

@ -157,7 +157,8 @@ typedef enum EStreamTaskEvent {
typedef void FTbSink(SStreamTask* pTask, void* vnode, void* data);
typedef void FSmaSink(void* vnode, int64_t smaId, const SArray* data);
typedef int32_t FTaskExpand(void* ahandle, SStreamTask* pTask, int64_t ver);
typedef int32_t FTaskBuild(void* ahandle, SStreamTask* pTask, int64_t ver);
typedef int32_t FTaskExpand(SStreamTask* pTask);
typedef struct {
int8_t type;
@ -205,7 +206,6 @@ typedef struct {
typedef struct {
char stbFullName[TSDB_TABLE_FNAME_LEN];
int32_t waitingRspCnt;
SUseDbRsp dbInfo;
} STaskDispatcherShuffle;
@ -312,15 +312,18 @@ typedef struct SMetaHbInfo SMetaHbInfo;
typedef struct SDispatchMsgInfo {
SStreamDispatchReq* pData; // current dispatch data
int8_t dispatchMsgType;
int64_t checkpointId;// checkpoint id msg
int32_t transId; // transId for current checkpoint
int16_t msgType; // dispatch msg type
int32_t retryCount; // retry send data count
int64_t startTs; // dispatch start time, record total elapsed time for dispatch
SArray* pRetryList; // current dispatch successfully completed node of downstream
void* pRetryTmr; // used to dispatch data after a given time duration
void* pRspTmr; // used to dispatch data after a given time duration
int8_t dispatchMsgType;
int64_t checkpointId; // checkpoint id msg
int32_t transId; // transId for current checkpoint
int16_t msgType; // dispatch msg type
int32_t msgId;
int64_t startTs; // dispatch start time, record total elapsed time for dispatch
int64_t rspTs; // latest rsp time
void* pRetryTmr; // used to dispatch data after a given time duration
TdThreadMutex lock;
int8_t inMonitor;
SArray* pSendInfo; // SArray<SDispatchEntry>
} SDispatchMsgInfo;
typedef struct STaskQueue {
@ -484,7 +487,8 @@ typedef struct SStreamMeta {
SArray* pTaskList; // SArray<STaskId*>
void* ahandle;
TXN* txn;
FTaskExpand* expandFunc;
FTaskBuild* buildTaskFn;
FTaskExpand* expandTaskFn;
int32_t vgId;
int64_t stage;
int32_t role;
@ -677,7 +681,8 @@ bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeI
void streamTaskGetTriggerRecvStatus(SStreamTask* pTask, int32_t* pRecved, int32_t* pTotal);
void streamTaskInitTriggerDispatchInfo(SStreamTask* pTask);
void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId);
int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId, SRpcHandleInfo* pInfo, int32_t code);
int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId, int32_t downstreamNodeId,
SRpcHandleInfo* pInfo, int32_t code);
int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue);
int32_t streamQueueGetNumOfUnAccessedItems(const SStreamQueue* pQueue);
@ -708,8 +713,8 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st);
// stream task meta
void streamMetaInit();
void streamMetaCleanup();
SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage,
startComplete_fn_t fn);
SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskBuild expandFunc, FTaskExpand expandTaskFn,
int32_t vgId, int64_t stage, startComplete_fn_t fn);
void streamMetaClose(SStreamMeta* streamMeta);
int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); // save to stream meta store
int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey);
@ -734,6 +739,9 @@ int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32
void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs);
void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SStreamTask* pHTask, int32_t transId,
int64_t startTs);
void streamMetaClearUpdateTaskList(SStreamMeta* pMeta);
void streamMetaInitUpdateTaskList(SStreamMeta* pMeta, int32_t transId);
void streamMetaRLock(SStreamMeta* pMeta);
void streamMetaRUnLock(SStreamMeta* pMeta);
void streamMetaWLock(SStreamMeta* pMeta);
@ -762,9 +770,8 @@ int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskI
int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask);
int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SRpcMsg* pMsg,
int32_t setCode);
int32_t streamBuildAndSendCheckpointUpdateMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId, STaskId* pHTaskId,
SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask);
int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpointInfoReq* pReq);
int32_t streamSendChkptReportMsg(SStreamTask* pTask, SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask);
int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SVUpdateCheckpointInfoReq* pReq);
SActiveCheckpointInfo* streamTaskCreateActiveChkptInfo();
// stream task state machine, and event handling

1
include/libs/transport/trpc.h
View File

@ -62,6 +62,7 @@ typedef struct SRpcHandleInfo {
SRpcConnInfo conn;
int8_t forbiddenIp;
int8_t notFreeAhandle;
int8_t compressed;
} SRpcHandleInfo;
typedef struct SRpcMsg {

1
include/libs/wal/wal.h
View File

@ -60,6 +60,7 @@ typedef struct {
EWalType level; // wal level
int32_t encryptAlgorithm;
char encryptKey[ENCRYPT_KEY_LEN + 1];
int8_t clearFiles;
} SWalCfg;
typedef struct {

11
include/os/osEnv.h
View File

@ -37,11 +37,12 @@ extern float tsNumOfCores;
extern int64_t tsTotalMemoryKB;
extern char *tsProcPath;
extern char tsSIMDEnable;
extern char tsSSE42Enable;
extern char tsAVXEnable;
extern char tsAVX2Enable;
extern char tsFMAEnable;
extern char tsAVX512Enable;
extern char tsSSE42Supported;
extern char tsAVXSupported;
extern char tsAVX2Supported;
extern char tsFMASupported;
extern char tsAVX512Supported;
extern char tsAVX512Enable;
extern char tsTagFilterCache;
extern char configDir[];

26
include/util/taoserror.h
View File

@ -24,6 +24,14 @@ extern "C" {
// clang-format off
typedef struct {
int32_t val;
const char* str;
const char* macro;
} STaosError;
extern STaosError errors[];
#define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code))))
#define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code))
@ -38,6 +46,7 @@ const char* terrstr();
char* taosGetErrMsgReturn();
char* taosGetErrMsg();
int32_t* taosGetErrno();
int32_t taosGetErrSize();
#define terrno (*taosGetErrno())
#define terrMsg (taosGetErrMsg())
@ -76,8 +85,8 @@ int32_t* taosGetErrno();
#define TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED TAOS_DEF_ERROR_CODE(0, 0x0020) // "Vgroup could not be connected"
#define TSDB_CODE_RPC_SOMENODE_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0021) //
#define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x0022) //
#define TSDB_CODE_RPC_NETWORK_ERROR TAOS_DEF_ERROR_CODE(0, 0x0023)
#define TSDB_CODE_RPC_NETWORK_BUSY TAOS_DEF_ERROR_CODE(0, 0x0024)
#define TSDB_CODE_RPC_NETWORK_ERROR TAOS_DEF_ERROR_CODE(0, 0x0023)
#define TSDB_CODE_RPC_NETWORK_BUSY TAOS_DEF_ERROR_CODE(0, 0x0024)
@ -265,8 +274,8 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_PRIVILEDGE_EXIST TAOS_DEF_ERROR_CODE(0, 0x0359)
#define TSDB_CODE_MND_USER_HOST_EXIST TAOS_DEF_ERROR_CODE(0, 0x035A)
#define TSDB_CODE_MND_USER_HOST_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x035B)
#define TSDB_CODE_MND_TOO_MANY_USER_HOST TAOS_DEF_ERROR_CODE(0, 0x035C)
#define TSDB_CODE_MND_USER_LOCAL_HOST_NOT_DROP TAOS_DEF_ERROR_CODE(0, 0x035D)
#define TSDB_CODE_MND_TOO_MANY_USER_HOST TAOS_DEF_ERROR_CODE(0, 0x035C)
#define TSDB_CODE_MND_USER_LOCAL_HOST_NOT_DROP TAOS_DEF_ERROR_CODE(0, 0x035D)
// mnode-stable-part1
#define TSDB_CODE_MND_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360)
@ -285,7 +294,7 @@ int32_t* taosGetErrno();
// #define TSDB_CODE_MND_INVALID_STABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x036D) // 2.x
#define TSDB_CODE_MND_INVALID_STB_OPTION TAOS_DEF_ERROR_CODE(0, 0x036E)
#define TSDB_CODE_MND_INVALID_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x036F)
#define TSDB_CODE_MND_FIELD_VALUE_OVERFLOW TAOS_DEF_ERROR_CODE(0, 0x0370)
// #define TSDB_CODE_MND_FIELD_VALUE_OVERFLOW TAOS_DEF_ERROR_CODE(0, 0x0370) // unused
// mnode-func
@ -316,7 +325,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x038A) //
#define TSDB_CODE_MND_DB_INDEX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x038B)
#define TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO TAOS_DEF_ERROR_CODE(0, 0x038C)
#define TSDB_CODE_MND_INCONSIST_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x038D)
// #define TSDB_CODE_MND_INCONSIST_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x038D) // unused
#define TSDB_CODE_MND_INVALID_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x038E)
// #define TSDB_CODE_MND_INVALID_DB_OPTION_DAYS TAOS_DEF_ERROR_CODE(0, 0x0390) // 2.x
// #define TSDB_CODE_MND_INVALID_DB_OPTION_KEEP TAOS_DEF_ERROR_CODE(0, 0x0391) // 2.x
@ -327,6 +336,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_DB_IN_CREATING TAOS_DEF_ERROR_CODE(0, 0x0396) //
#define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x039A)
#define TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE TAOS_DEF_ERROR_CODE(0, 0x039B)
#define TSDB_CODE_MND_INVALID_WAL_LEVEL TAOS_DEF_ERROR_CODE(0, 0x039C)
// mnode-node
#define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0)
@ -506,7 +516,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_VND_DIR_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0534)
#define TSDB_CODE_VND_META_DATA_UNSAFE_DELETE TAOS_DEF_ERROR_CODE(0, 0x0535)
#define TSDB_CODE_VND_COLUMN_COMPRESS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0536)
#define TSDB_CODE_VND_ARB_NOT_SYNCED TAOS_DEF_ERROR_CODE(0, 0x0536) // internal
#define TSDB_CODE_VND_ARB_NOT_SYNCED TAOS_DEF_ERROR_CODE(0, 0x0537) // internal
// tsdb
#define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600)
@ -920,7 +930,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_TDLITE_IVLD_OPEN_DIR TAOS_DEF_ERROR_CODE(0, 0x5101)
// UTIL
#define TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x6000)
#define TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x6000)
#ifdef __cplusplus
}

1
include/util/tdef.h
View File

@ -549,6 +549,7 @@ typedef struct {
char dir[TSDB_FILENAME_LEN];
int32_t level;
int32_t primary;
int8_t disable; // disable create new file
} SDiskCfg;
typedef struct {

4
packaging/tools/install.sh
View File

@ -795,10 +795,10 @@ function is_version_compatible() {
if [ -f ${script_dir}/driver/vercomp.txt ]; then
min_compatible_version=$(cat ${script_dir}/driver/vercomp.txt)
else
min_compatible_version=$(${script_dir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 5)
min_compatible_version=$(${script_dir}/bin/${serverName} -V | grep version | head -1 | cut -d ' ' -f 5)
fi
exist_version=$(${installDir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 3)
exist_version=$(${installDir}/bin/${serverName} -V | grep version | head -1 | cut -d ' ' -f 3)
vercomp $exist_version "3.0.0.0"
case $? in
2)

5
source/client/inc/clientInt.h
View File

@ -124,7 +124,6 @@ struct SAppInstInfo {
typedef struct SAppInfo {
int64_t startTime;
char appName[TSDB_APP_NAME_LEN];
char* ep;
int32_t pid;
int32_t numOfThreads;
SHashObj* pInstMap;
@ -197,8 +196,10 @@ typedef struct SReqResultInfo {
uint64_t current;
bool localResultFetched;
bool completed;
int32_t precision;
bool convertUcs4;
char* decompBuf;
int32_t decompBufSize;
int32_t precision;
int32_t payloadLen;
char* convertJson;
} SReqResultInfo;

74
source/client/inc/clientStmt.h
View File

@ -40,6 +40,8 @@ typedef enum {
STMT_MAX,
} STMT_STATUS;
#define STMT_TABLE_COLS_NUM 1000
typedef struct SStmtTableCache {
STableDataCxt *pDataCtx;
void *boundTags;
@ -57,6 +59,7 @@ typedef struct SStmtBindInfo {
bool inExecCache;
uint64_t tbUid;
uint64_t tbSuid;
int32_t tbVgId;
int32_t sBindRowNum;
int32_t sBindLastIdx;
int8_t tbType;
@ -66,8 +69,15 @@ typedef struct SStmtBindInfo {
char tbFName[TSDB_TABLE_FNAME_LEN];
char stbFName[TSDB_TABLE_FNAME_LEN];
SName sname;
char statbName[TSDB_TABLE_FNAME_LEN];
} SStmtBindInfo;
typedef struct SStmtAsyncParam {
STableColsData *pTbData;
void* pStmt;
} SStmtAsyncParam;
typedef struct SStmtExecInfo {
int32_t affectedRows;
SRequestObj *pRequest;
@ -77,8 +87,10 @@ typedef struct SStmtExecInfo {
} SStmtExecInfo;
typedef struct SStmtSQLInfo {
bool stbInterlaceMode;
STMT_TYPE type;
STMT_STATUS status;
uint64_t suid;
uint64_t runTimes;
SHashObj *pTableCache; // SHash<SStmtTableCache>
SQuery *pQuery;
@ -88,21 +100,60 @@ typedef struct SStmtSQLInfo {
SStmtQueryResInfo queryRes;
bool autoCreateTbl;
SHashObj *pVgHash;
SBindInfo *pBindInfo;
SStbInterlaceInfo siInfo;
} SStmtSQLInfo;
typedef struct SStmtStatInfo {
int64_t ctgGetTbMetaNum;
int64_t getCacheTbInfo;
int64_t parseSqlNum;
int64_t bindDataNum;
int64_t setTbNameUs;
int64_t bindDataUs1;
int64_t bindDataUs2;
int64_t bindDataUs3;
int64_t bindDataUs4;
int64_t addBatchUs;
int64_t execWaitUs;
int64_t execUseUs;
} SStmtStatInfo;
typedef struct SStmtQNode {
bool restoreTbCols;
STableColsData tblData;
struct SStmtQNode* next;
} SStmtQNode;
typedef struct SStmtQueue {
bool stopQueue;
SStmtQNode* head;
SStmtQNode* tail;
uint64_t qRemainNum;
} SStmtQueue;
typedef struct STscStmt {
STscObj *taos;
SCatalog *pCatalog;
int32_t affectedRows;
uint32_t seqId;
uint32_t seqIds[STMT_MAX];
STscObj *taos;
SCatalog *pCatalog;
int32_t affectedRows;
uint32_t seqId;
uint32_t seqIds[STMT_MAX];
bool bindThreadInUse;
TdThread bindThread;
TAOS_STMT_OPTIONS options;
bool stbInterlaceMode;
SStmtQueue queue;
SStmtSQLInfo sql;
SStmtExecInfo exec;
SStmtBindInfo bInfo;
SStmtSQLInfo sql;
SStmtExecInfo exec;
SStmtBindInfo bInfo;
int64_t reqid;
int32_t errCode;
int64_t reqid;
int32_t errCode;
SStmtStatInfo stat;
} STscStmt;
extern char *gStmtStatusStr[];
@ -154,13 +205,14 @@ extern char *gStmtStatusStr[];
} while (0)
#define STMT_FLOG(param, ...) qFatal("stmt:%p " param, pStmt, __VA_ARGS__)
#define STMT_ELOG(param, ...) qError("stmt:%p " param, pStmt, __VA_ARGS__)
#define STMT_DLOG(param, ...) qDebug("stmt:%p " param, pStmt, __VA_ARGS__)
#define STMT_ELOG_E(param) qError("stmt:%p " param, pStmt)
#define STMT_DLOG_E(param) qDebug("stmt:%p " param, pStmt)
TAOS_STMT *stmtInit(STscObj *taos, int64_t reqid);
TAOS_STMT *stmtInit(STscObj* taos, int64_t reqid, TAOS_STMT_OPTIONS* pOptions);
int stmtClose(TAOS_STMT *stmt);
int stmtExec(TAOS_STMT *stmt);
const char *stmtErrstr(TAOS_STMT *stmt);

3
source/client/src/clientEnv.c
View File

@ -334,6 +334,7 @@ void *createRequest(uint64_t connId, int32_t type, int64_t reqid) {
}
SSyncQueryParam *interParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
if (interParam == NULL) {
releaseTscObj(connId);
doDestroyRequest(pRequest);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
@ -374,6 +375,7 @@ void doFreeReqResultInfo(SReqResultInfo *pResInfo) {
taosMemoryFreeClear(pResInfo->fields);
taosMemoryFreeClear(pResInfo->userFields);
taosMemoryFreeClear(pResInfo->convertJson);
taosMemoryFreeClear(pResInfo->decompBuf);
if (pResInfo->convertBuf != NULL) {
for (int32_t i = 0; i < pResInfo->numOfCols; ++i) {
@ -763,7 +765,6 @@ void taos_init_imp(void) {
clientConnRefPool = taosOpenRef(200, destroyTscObj);
clientReqRefPool = taosOpenRef(40960, doDestroyRequest);
// transDestroyBuffer(&conn->readBuf);
taosGetAppName(appInfo.appName, NULL);
taosThreadMutexInit(&appInfo.mutex, NULL);

67
source/client/src/clientHb.c
View File

@ -133,15 +133,23 @@ static int32_t hbUpdateUserAuthInfo(SAppHbMgr *pAppHbMgr, SUserAuthBatchRsp *bat
if (pTscObj->whiteListInfo.fp) {
SWhiteListInfo *whiteListInfo = &pTscObj->whiteListInfo;
int64_t oldVer = atomic_load_64(&whiteListInfo->ver);
if (oldVer < pRsp->whiteListVer) {
int64_t oldVer = atomic_load_64(&whiteListInfo->ver);
if (oldVer != pRsp->whiteListVer) {
atomic_store_64(&whiteListInfo->ver, pRsp->whiteListVer);
if (whiteListInfo->fp) {
(*whiteListInfo->fp)(whiteListInfo->param, &pRsp->whiteListVer, TAOS_NOTIFY_WHITELIST_VER);
}
tscDebug("update whitelist version of user %s from %"PRId64" to %"PRId64", tscRid:%" PRIi64, pRsp->user, oldVer,
atomic_load_64(&whiteListInfo->ver), pTscObj->id);
tscDebug("update whitelist version of user %s from %" PRId64 " to %" PRId64 ", tscRid:%" PRIi64, pRsp->user,
oldVer, atomic_load_64(&whiteListInfo->ver), pTscObj->id);
}
} else {
// Need to update version information to prevent frequent fetching of authentication
// information.
SWhiteListInfo *whiteListInfo = &pTscObj->whiteListInfo;
int64_t oldVer = atomic_load_64(&whiteListInfo->ver);
atomic_store_64(&whiteListInfo->ver, pRsp->whiteListVer);
tscDebug("update whitelist version of user %s from %" PRId64 " to %" PRId64 ", tscRid:%" PRIi64, pRsp->user,
oldVer, atomic_load_64(&whiteListInfo->ver), pTscObj->id);
}
releaseTscObj(pReq->connKey.tscRid);
}
@ -202,8 +210,8 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
for (int32_t i = 0; i < numOfBatchs; ++i) {
SDbHbRsp *rsp = taosArrayGet(batchRsp.pArray, i);
if (rsp->useDbRsp) {
tscDebug("hb use db rsp, db:%s, vgVersion:%d, stateTs:%" PRId64 ", uid:%" PRIx64,
rsp->useDbRsp->db, rsp->useDbRsp->vgVersion, rsp->useDbRsp->stateTs, rsp->useDbRsp->uid);
tscDebug("hb use db rsp, db:%s, vgVersion:%d, stateTs:%" PRId64 ", uid:%" PRIx64, rsp->useDbRsp->db,
rsp->useDbRsp->vgVersion, rsp->useDbRsp->stateTs, rsp->useDbRsp->uid);
if (rsp->useDbRsp->vgVersion < 0) {
tscDebug("hb to remove db, db:%s", rsp->useDbRsp->db);
@ -225,7 +233,9 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
goto _return;
}
catalogUpdateDBVgInfo(pCatalog, (rsp->useDbRsp->db[0] == 'i') ? TSDB_PERFORMANCE_SCHEMA_DB : TSDB_INFORMATION_SCHEMA_DB, rsp->useDbRsp->uid, vgInfo);
catalogUpdateDBVgInfo(pCatalog,
(rsp->useDbRsp->db[0] == 'i') ? TSDB_PERFORMANCE_SCHEMA_DB : TSDB_INFORMATION_SCHEMA_DB,
rsp->useDbRsp->uid, vgInfo);
}
}
}
@ -238,7 +248,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
if (rsp->pTsmaRsp) {
if (rsp->pTsmaRsp->pTsmas) {
for (int32_t i = 0; i < rsp->pTsmaRsp->pTsmas->size; ++i) {
STableTSMAInfo* pTsma = taosArrayGetP(rsp->pTsmaRsp->pTsmas, i);
STableTSMAInfo *pTsma = taosArrayGetP(rsp->pTsmaRsp->pTsmas, i);
catalogAsyncUpdateTSMA(pCatalog, &pTsma, rsp->dbTsmaVersion);
}
taosArrayClear(rsp->pTsmaRsp->pTsmas);
@ -294,16 +304,15 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo
return TSDB_CODE_SUCCESS;
}
static int32_t hbProcessDynViewRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) {
return catalogUpdateDynViewVer(pCatalog, (SDynViewVersion*)value);
return catalogUpdateDynViewVer(pCatalog, (SDynViewVersion *)value);
}
static void hbFreeSViewMetaInRsp(void* p) {
if (NULL == p || NULL == *(void**)p) {
static void hbFreeSViewMetaInRsp(void *p) {
if (NULL == p || NULL == *(void **)p) {
return;
}
SViewMetaRsp *pRsp = *(SViewMetaRsp**)p;
SViewMetaRsp *pRsp = *(SViewMetaRsp **)p;
tFreeSViewMetaRsp(pRsp);
taosMemoryFreeClear(pRsp);
}
@ -337,7 +346,7 @@ static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatal
return TSDB_CODE_SUCCESS;
}
static int32_t hbprocessTSMARsp(void* value, int32_t valueLen, struct SCatalog* pCatalog) {
static int32_t hbprocessTSMARsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) {
int32_t code = 0;
STSMAHbRsp hbRsp = {0};
@ -348,7 +357,7 @@ static int32_t hbprocessTSMARsp(void* value, int32_t valueLen, struct SCatalog*
int32_t numOfTsma = taosArrayGetSize(hbRsp.pTsmas);
for (int32_t i = 0; i < numOfTsma; ++i) {
STableTSMAInfo* pTsmaInfo = taosArrayGetP(hbRsp.pTsmas, i);
STableTSMAInfo *pTsmaInfo = taosArrayGetP(hbRsp.pTsmas, i);
if (!pTsmaInfo->pFuncs) {
tscDebug("hb to remove tsma: %s.%s", pTsmaInfo->dbFName, pTsmaInfo->name);
@ -365,7 +374,7 @@ static int32_t hbprocessTSMARsp(void* value, int32_t valueLen, struct SCatalog*
return TSDB_CODE_SUCCESS;
}
static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) {
static void hbProcessQueryRspKvs(int32_t kvNum, SArray *pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) {
for (int32_t i = 0; i < kvNum; ++i) {
SKv *kv = taosArrayGet(pKvs, i);
switch (kv->key) {
@ -489,14 +498,14 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
if (kvNum > 0) {
struct SCatalog *pCatalog = NULL;
int32_t code = catalogGetHandle(pReq->clusterId, &pCatalog);
int32_t code = catalogGetHandle(pReq->clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) {
tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pReq->clusterId, tstrerror(code));
} else {
hbProcessQueryRspKvs(kvNum, pRsp->info, pCatalog, pAppHbMgr);
}
}
taosHashRelease(pAppHbMgr->activeInfo, pReq);
return TSDB_CODE_SUCCESS;
@ -799,8 +808,9 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl
for (int32_t i = 0; i < dbNum; ++i) {
SDbCacheInfo *db = &dbs[i];
tscDebug("the %dth expired dbFName:%s, dbId:%" PRId64 ", vgVersion:%d, cfgVersion:%d, numOfTable:%d, startTs:%" PRId64,
i, db->dbFName, db->dbId, db->vgVersion, db->cfgVersion, db->numOfTable, db->stateTs);
tscDebug("the %dth expired dbFName:%s, dbId:%" PRId64
", vgVersion:%d, cfgVersion:%d, numOfTable:%d, startTs:%" PRId64,
i, db->dbFName, db->dbId, db->vgVersion, db->cfgVersion, db->numOfTable, db->stateTs);
db->dbId = htobe64(db->dbId);
db->vgVersion = htonl(db->vgVersion);
@ -916,7 +926,7 @@ int32_t hbGetExpiredViewInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S
return TSDB_CODE_SUCCESS;
}
int32_t hbGetExpiredTSMAInfo(SClientHbKey* connKey, struct SCatalog* pCatalog, SClientHbReq* pReq) {
int32_t hbGetExpiredTSMAInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *pReq) {
int32_t code = 0;
uint32_t tsmaNum = 0;
STSMAVersion *tsmas = NULL;
@ -933,7 +943,7 @@ int32_t hbGetExpiredTSMAInfo(SClientHbKey* connKey, struct SCatalog* pCatalog, S
}
for (int32_t i = 0; i < tsmaNum; ++i) {
STSMAVersion* tsma = &tsmas[i];
STSMAVersion *tsma = &tsmas[i];
tsma->dbId = htobe64(tsma->dbId);
tsma->tsmaId = htobe64(tsma->tsmaId);
tsma->version = htonl(tsma->version);
@ -1012,7 +1022,7 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req
if (TSDB_CODE_SUCCESS != code) {
return code;
}
#endif
#endif
code = hbGetExpiredTSMAInfo(connKey, pCatalog, req);
} else {
req->app.appId = 0;
@ -1045,6 +1055,7 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) {
return NULL;
}
int64_t maxIpWhiteVer = 0;
void *pIter = NULL;
SHbParam param = {0};
while ((pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter))) {
@ -1080,8 +1091,11 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) {
}
}
int64_t ver = atomic_load_64(&pTscObj->whiteListInfo.ver);
maxIpWhiteVer = TMAX(maxIpWhiteVer, ver);
releaseTscObj(connKey->tscRid);
}
pBatchReq->ipWhiteList = maxIpWhiteVer;
return pBatchReq;
}
@ -1151,7 +1165,8 @@ static void *hbThreadFunc(void *param) {
if (sz > 0) {
hbGatherAppInfo();
if (sz > 1 && !clientHbMgr.appHbHash) {
clientHbMgr.appHbHash = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
clientHbMgr.appHbHash =
taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
}
taosHashClear(clientHbMgr.appHbHash);
}
@ -1433,6 +1448,4 @@ void hbDeregisterConn(STscObj *pTscObj, SClientHbKey connKey) {
}
// set heart beat thread quit mode , if quicByKill 1 then kill thread else quit from inner
void taos_set_hb_quit(int8_t quitByKill) {
clientHbMgr.quitByKill = quitByKill;
}
void taos_set_hb_quit(int8_t quitByKill) { clientHbMgr.quitByKill = quitByKill; }

56
source/client/src/clientImpl.c
View File

@ -1711,10 +1711,8 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
}
SReqResultInfo* pResInfo = &pRequest->body.resInfo;
SSchedulerReq req = {
.syncReq = true,
.pFetchRes = (void**)&pResInfo->pData,
};
SSchedulerReq req = { .syncReq = true, .pFetchRes = (void**)&pResInfo->pData };
pRequest->code = schedulerFetchRows(pRequest->body.queryJob, &req);
if (pRequest->code != TSDB_CODE_SUCCESS) {
pResultInfo->numOfRows = 0;
@ -2065,6 +2063,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
tscError("setResultDataPtr paras error");
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
if (numOfRows == 0) {
return TSDB_CODE_SUCCESS;
}
@ -2195,17 +2194,58 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR
taosMemoryFreeClear(pResultInfo->pRspMsg);
pResultInfo->pRspMsg = (const char*)pRsp;
pResultInfo->pData = (void*)pRsp->data;
pResultInfo->numOfRows = htobe64(pRsp->numOfRows);
pResultInfo->current = 0;
pResultInfo->completed = (pRsp->completed == 1);
pResultInfo->payloadLen = htonl(pRsp->compLen);
pResultInfo->precision = pRsp->precision;
// decompress data if needed
int32_t payloadLen = htonl(pRsp->payloadLen);
if (pRsp->compressed) {
if (pResultInfo->decompBuf == NULL) {
pResultInfo->decompBuf = taosMemoryMalloc(payloadLen);
pResultInfo->decompBufSize = payloadLen;
} else {
if (pResultInfo->decompBufSize < payloadLen) {
char* p = taosMemoryRealloc(pResultInfo->decompBuf, payloadLen);
if (p == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
tscError("failed to prepare the decompress buffer, size:%d", payloadLen);
return terrno;
}
pResultInfo->decompBuf = p;
pResultInfo->decompBufSize = payloadLen;
}
}
}
if (payloadLen > 0) {
int32_t compLen = *(int32_t*)pRsp->data;
int32_t rawLen = *(int32_t*)(pRsp->data + sizeof(int32_t));
char* pStart = (char*)pRsp->data + sizeof(int32_t) * 2;
if (pRsp->compressed && compLen < rawLen) {
int32_t len = tsDecompressString(pStart, compLen, 1, pResultInfo->decompBuf, rawLen, ONE_STAGE_COMP, NULL, 0);
ASSERT(len == rawLen);
pResultInfo->pData = pResultInfo->decompBuf;
pResultInfo->payloadLen = rawLen;
} else {
pResultInfo->pData = pStart;
pResultInfo->payloadLen = htonl(pRsp->compLen);
ASSERT(pRsp->compLen == pRsp->payloadLen);
}
}
// TODO handle the compressed case
pResultInfo->totalRows += pResultInfo->numOfRows;
return setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows,
convertUcs4);
int32_t code =
setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows, convertUcs4);
return code;
}
TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* details, int maxlen) {

24
source/client/src/clientMain.c
View File

@ -1552,7 +1552,7 @@ TAOS_STMT *taos_stmt_init(TAOS *taos) {
return NULL;
}
TAOS_STMT *pStmt = stmtInit(pObj, 0);
TAOS_STMT *pStmt = stmtInit(pObj, 0, NULL);
releaseTscObj(*(int64_t *)taos);
@ -1567,13 +1567,29 @@ TAOS_STMT *taos_stmt_init_with_reqid(TAOS *taos, int64_t reqid) {
return NULL;
}
TAOS_STMT *pStmt = stmtInit(pObj, reqid);
TAOS_STMT *pStmt = stmtInit(pObj, reqid, NULL);
releaseTscObj(*(int64_t *)taos);
return pStmt;
}
TAOS_STMT *taos_stmt_init_with_options(TAOS *taos, TAOS_STMT_OPTIONS *options) {
STscObj *pObj = acquireTscObj(*(int64_t *)taos);
if (NULL == pObj) {
tscError("invalid parameter for %s", __FUNCTION__);
terrno = TSDB_CODE_TSC_DISCONNECTED;
return NULL;
}
TAOS_STMT *pStmt = stmtInit(pObj, options->reqId, options);
releaseTscObj(*(int64_t *)taos);
return pStmt;
}
int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) {
if (stmt == NULL || sql == NULL) {
tscError("NULL parameter for %s", __FUNCTION__);
@ -1830,3 +1846,7 @@ int taos_set_conn_mode(TAOS* taos, int mode, int value) {
}
return 0;
}
char* getBuildInfo(){
return buildinfo;
}

28
source/client/src/clientMsgHandler.c
View File

@ -25,6 +25,7 @@
#include "tglobal.h"
#include "tname.h"
#include "tversion.h"
#include "command.h"
extern SClientHbMgr clientHbMgr;
@ -499,7 +500,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) {
return code;
}
size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN;
*pRsp = taosMemoryCalloc(1, rspSize);
if (NULL == *pRsp) {
blockDataDestroy(pBlock);
@ -510,14 +511,20 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) {
(*pRsp)->completed = 1;
(*pRsp)->precision = 0;
(*pRsp)->compressed = 0;
(*pRsp)->compLen = 0;
(*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows);
(*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS);
int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS);
int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, SHOW_VARIABLES_RESULT_COLS);
blockDataDestroy(pBlock);
if (len != rspSize - sizeof(SRetrieveTableRsp)) {
SET_PAYLOAD_LEN((*pRsp)->data, len, len);
int32_t payloadLen = len + PAYLOAD_PREFIX_LEN;
(*pRsp)->payloadLen = htonl(payloadLen);
(*pRsp)->compLen = htonl(payloadLen);
if (payloadLen != rspSize - sizeof(SRetrieveTableRsp)) {
uError("buildShowVariablesRsp error, len:%d != rspSize - sizeof(SRetrieveTableRsp):%" PRIu64, len,
(uint64_t)(rspSize - sizeof(SRetrieveTableRsp)));
return TSDB_CODE_TSC_INVALID_INPUT;
@ -611,7 +618,7 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr
return code;
}
size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN;
*pRsp = taosMemoryCalloc(1, rspSize);
if (NULL == *pRsp) {
blockDataDestroy(pBlock);
@ -623,13 +630,20 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr
(*pRsp)->precision = 0;
(*pRsp)->compressed = 0;
(*pRsp)->compLen = 0;
(*pRsp)->payloadLen = 0;
(*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows);
(*pRsp)->numOfCols = htonl(COMPACT_DB_RESULT_COLS);
int32_t len = blockEncode(pBlock, (*pRsp)->data, COMPACT_DB_RESULT_COLS);
int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, COMPACT_DB_RESULT_COLS);
blockDataDestroy(pBlock);
if (len != rspSize - sizeof(SRetrieveTableRsp)) {
SET_PAYLOAD_LEN((*pRsp)->data, len, len);
int32_t payloadLen = len + PAYLOAD_PREFIX_LEN;
(*pRsp)->payloadLen = htonl(payloadLen);
(*pRsp)->compLen = htonl(payloadLen);
if (payloadLen != rspSize - sizeof(SRetrieveTableRsp)) {
uError("buildRetriveTableRspForCompactDb error, len:%d != rspSize - sizeof(SRetrieveTableRsp):%" PRIu64, len,
(uint64_t)(rspSize - sizeof(SRetrieveTableRsp)));
return TSDB_CODE_TSC_INVALID_INPUT;

748
source/client/src/clientStmt.c
View File

File diff suppressed because it is too large Load Diff

99
source/client/test/clientTests.cpp
View File

@ -821,32 +821,17 @@ TEST(clientCase, projection_query_tables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
TAOS_RES* pRes = NULL;
// TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1");
// if (taos_errno(pRes) != 0) {
// printf("error in create db, reason:%s\n", taos_errstr(pRes));
// }
// taos_free_result(pRes);
TAOS_RES* pRes = taos_query(pConn, "alter local 'fqdn 127.0.0.1'");
if (taos_errno(pRes) != 0) {
printf("failed to exec query, %s\n", taos_errstr(pRes));
}
pRes= taos_query(pConn, "use abc1");
taos_free_result(pRes);
pRes = taos_query(pConn, "select last(ts), ts from cache_1.t1");
// pRes = taos_query(pConn, "select last(ts), ts from cache_1.no_pk_t1");
if (taos_errno(pRes) != 0) {
printf("failed to exec query, %s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
// pRes = taos_query(pConn, "create stream stream_1 trigger at_once fill_history 1 ignore expired 0 into str_res1 as select _wstart as ts, count(*) from stable_1 interval(10s);");
// if (taos_errno(pRes) != 0) {
// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
// }
// taos_free_result(pRes);
pRes = taos_query(pConn, "create table tu using st2 tags(2)");
if (taos_errno(pRes) != 0) {
printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
@ -876,27 +861,26 @@ TEST(clientCase, projection_query_tables) {
for(int32_t j = 0; j < 1; ++j) {
start += 20;
for (int32_t i = 0; i < 1; ++i) {
createNewTable(pConn, i, 100, start, pstr);
createNewTable(pConn, i, 100000, 0, pstr);
}
}
//
// pRes = taos_query(pConn, "select * from tu");
// if (taos_errno(pRes) != 0) {
// printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
// taos_free_result(pRes);
// ASSERT_TRUE(false);
// }
//
// TAOS_ROW pRow = NULL;
// TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// int32_t numOfFields = taos_num_fields(pRes);
//
// char str[512] = {0};
// while ((pRow = taos_fetch_row(pRes)) != NULL) {
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
// printf("%s\n", str);
// }
pRes = taos_query(pConn, "select * from abc1.st2");
if (taos_errno(pRes) != 0) {
printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes);
ASSERT_TRUE(false);
}
TAOS_ROW pRow = NULL;
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes);
char str[512] = {0};
while ((pRow = taos_fetch_row(pRes)) != NULL) {
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
// printf("%s\n", str);
}
taos_free_result(pRes);
taos_close(pConn);
@ -915,25 +899,44 @@ TEST(clientCase, projection_query_stables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
TAOS_RES* pRes = taos_query(pConn, "use test");
taos_free_result(pRes);
TAOS_RES* pRes = taos_query(pConn, "explain select * from dbvg.st where tbname='ct1'");
// taos_free_result(pRes);
pRes = taos_query(pConn, "select * from meters limit 50000000");
if (taos_errno(pRes) != 0) {
printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes);
ASSERT_TRUE(false);
}
// pRes = taos_query(pConn, "select * from st2");
// if (taos_errno(pRes) != 0) {
// printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
// taos_free_result(pRes);
// ASSERT_TRUE(false);
// }
TAOS_ROW pRow = NULL;
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes);
char str[512] = {0};
while ((pRow = taos_fetch_row(pRes)) != NULL) {
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
// printf("%s\n", str);
int32_t numOfRows = 0;
int32_t i = 0;
int32_t prev = 0;
char str[512] = {0};
while (1) {
pRow = taos_fetch_row(pRes);
if (pRow == NULL) {
break;
}
i += numOfRows;
if ( (i / 1000000) > prev) {
printf("%d\n", i);
prev = i/1000000;
}
//printf("%d\n", i);
}
// while ((pRow = taos_fetch_row(pRes)) != NULL) {
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
// if (i++ % 100000 == 0) {
// printf("%d\n", i);
// }
// }
taos_free_result(pRes);
taos_close(pConn);

24
source/common/src/rsync.c
View File

@ -169,15 +169,22 @@ int32_t uploadByRsync(const char* id, const char* path) {
#else
if (path[strlen(path) - 1] != '/') {
#endif
snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 --bwlimit=100000 %s/ rsync://%s/checkpoint/%s/",
snprintf(command, PATH_MAX,
"rsync -av --debug=all --log-file=%s/rsynclog --delete --timeout=10 --bwlimit=100000 %s/ "
"rsync://%s/checkpoint/%s/",
tsLogDir,
#ifdef WINDOWS
pathTransform
#else
path
#endif
, tsSnodeAddress, id);
,
tsSnodeAddress, id);
} else {
snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 --bwlimit=100000 %s rsync://%s/checkpoint/%s/",
snprintf(command, PATH_MAX,
"rsync -av --debug=all --log-file=%s/rsynclog --delete --timeout=10 --bwlimit=100000 %s "
"rsync://%s/checkpoint/%s/",
tsLogDir,
#ifdef WINDOWS
pathTransform
#else
@ -213,14 +220,15 @@ int32_t downloadRsync(const char* id, const char* path) {
#endif
char command[PATH_MAX] = {0};
snprintf(command, PATH_MAX, "rsync -av --debug=all --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/ %s",
tsSnodeAddress, id,
snprintf(command, PATH_MAX,
"rsync -av --debug=all --log-file=%s/rsynclog --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/ %s",
tsLogDir, tsSnodeAddress, id,
#ifdef WINDOWS
pathTransform
#else
path
#endif
);
);
uDebug("[rsync] %s start to sync data from remote to:%s, %s", id, path, command);
@ -249,7 +257,9 @@ int32_t deleteRsync(const char* id) {
}
char command[PATH_MAX] = {0};
snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 %s rsync://%s/checkpoint/%s/", tmp, tsSnodeAddress, id);
snprintf(command, PATH_MAX,
"rsync -av --debug=all --log-file=%s/rsynclog --delete --timeout=10 %s rsync://%s/checkpoint/%s/", tsLogDir,
tmp, tsSnodeAddress, id);
code = execCommand(command);
taosRemoveDir(tmp);

12
source/common/src/systable.c
View File

@ -264,10 +264,21 @@ static const SSysDbTableSchema userUsersSchema[] = {
{.name = "super", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "enable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "sysinfo", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "createdb", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
{.name = "allowed_host", .bytes = TSDB_PRIVILEDGE_HOST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};
static const SSysDbTableSchema userUsersFullSchema[] = {
{.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "super", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "enable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "sysinfo", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "createdb", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "encrypted_pass", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "allowed_host", .bytes = TSDB_PRIVILEDGE_HOST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};
GRANTS_SCHEMA;
static const SSysDbTableSchema vgroupsSchema[] = {
@ -437,6 +448,7 @@ static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_COLS, userColsSchema, tListLen(userColsSchema), false},
// {TSDB_INS_TABLE_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)},
{TSDB_INS_TABLE_USERS, userUsersSchema, tListLen(userUsersSchema), true},
{TSDB_INS_TABLE_USERS_FULL, userUsersFullSchema, tListLen(userUsersFullSchema), true},
{TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema), true},
{TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema), true},
{TSDB_INS_TABLE_CONFIGS, configSchema, tListLen(configSchema), false},

4
source/common/src/tcol.c
View File

@ -327,7 +327,9 @@ int32_t setColCompressByOption(uint8_t type, uint8_t encode, uint16_t compressTy
return TSDB_CODE_SUCCESS;
}
bool useCompress(uint8_t tableType) { return TSDB_SUPER_TABLE == tableType || TSDB_NORMAL_TABLE == tableType; }
bool useCompress(uint8_t tableType) {
return TSDB_SUPER_TABLE == tableType || TSDB_NORMAL_TABLE == tableType || TSDB_CHILD_TABLE == tableType;
}
int8_t validColCompressLevel(uint8_t type, uint8_t level) {
if (level == TSDB_COLVAL_LEVEL_DISABLED) return 1;

6
source/common/src/tdatablock.c
View File

@ -21,7 +21,7 @@
#define MALLOC_ALIGN_BYTES 32
static void copyPkVal(SDataBlockInfo* pDst, const SDataBlockInfo* pSrc);
int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) {
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
@ -848,7 +848,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
if (pBlock->pBlockAgg == NULL) {
isNull = colDataIsNull_s(pColData, j);
} else {
isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg[i]);
isNull = colDataIsNull(pColData, pBlock->info.rows, j, &pBlock->pBlockAgg[i]);
}
if (isNull) {
@ -1362,6 +1362,8 @@ void blockDataEmpty(SSDataBlock* pDataBlock) {
return;
}
taosMemoryFreeClear(pDataBlock->pBlockAgg);
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);

73
source/common/src/tdataformat.c
View File

@ -426,6 +426,79 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) {
return code;
}
static int32_t tBindInfoCompare(const void *p1, const void *p2, const void *param) {
if (((SBindInfo *)p1)->columnId < ((SBindInfo *)p2)->columnId) {
return -1;
} else if (((SBindInfo *)p1)->columnId > ((SBindInfo *)p2)->columnId) {
return 1;
}
return 0;
}
/* build rows to `rowArray` from bind
* `infos` is the bind information array
* `numOfInfos` is the number of bind information
* `infoSorted` is whether the bind information is sorted by column id
* `pTSchema` is the schema of the table
* `rowArray` is the array to store the rows
*/
int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted, const STSchema *pTSchema,
SArray *rowArray) {
if (infos == NULL || numOfInfos <= 0 || numOfInfos > pTSchema->numOfCols || pTSchema == NULL || rowArray == NULL) {
return TSDB_CODE_INVALID_PARA;
}
if (!infoSorted) {
taosqsort_r(infos, numOfInfos, sizeof(SBindInfo), NULL, tBindInfoCompare);
}
int32_t code = 0;
int32_t numOfRows = infos[0].bind->num;
SArray *colValArray;
SColVal colVal;
if ((colValArray = taosArrayInit(numOfInfos, sizeof(SColVal))) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t iRow = 0; iRow < numOfRows; iRow++) {
taosArrayClear(colValArray);
for (int32_t iInfo = 0; iInfo < numOfInfos; iInfo++) {
if (infos[iInfo].bind->is_null && infos[iInfo].bind->is_null[iRow]) {
colVal = COL_VAL_NULL(infos[iInfo].columnId, infos[iInfo].type);
} else {
SValue value = {
.type = infos[iInfo].type,
};
if (IS_VAR_DATA_TYPE(infos[iInfo].type)) {
value.nData = infos[iInfo].bind->length[iRow];
value.pData = (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow;
} else {
memcpy(&value.val, (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow,
infos[iInfo].bind->buffer_length);
}
colVal = COL_VAL_VALUE(infos[iInfo].columnId, value);
}
taosArrayPush(colValArray, &colVal);
}
SRow *row;
if ((code = tRowBuild(colValArray, pTSchema, &row))) {
goto _exit;
}
if ((taosArrayPush(rowArray, &row)) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
}
_exit:
taosArrayDestroy(colValArray);
return code;
}
int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) {
ASSERT(iCol < pTSchema->numOfCols);
ASSERT(pRow->sver == pTSchema->version);

21
source/common/src/tglobal.c
View File

@ -74,6 +74,7 @@ int32_t tsNumOfSnodeStreamThreads = 4;
int32_t tsNumOfSnodeWriteThreads = 1;
int32_t tsMaxStreamBackendCache = 128; // M
int32_t tsPQSortMemThreshold = 16; // M
int32_t tsRetentionSpeedLimitMB = 0; // unlimited
// sync raft
int32_t tsElectInterval = 25 * 1000;
@ -325,6 +326,7 @@ int32_t taosSetTfsCfg(SConfig *pCfg) {
tstrncpy(tsDiskCfg[0].dir, pItem->str, TSDB_FILENAME_LEN);
tsDiskCfg[0].level = 0;
tsDiskCfg[0].primary = 1;
tsDiskCfg[0].disable = 0;
tstrncpy(tsDataDir, pItem->str, PATH_MAX);
if (taosMulMkDir(tsDataDir) != 0) {
uError("failed to create dataDir:%s", tsDataDir);
@ -594,12 +596,13 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "enableCoreFile", 1, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1;
if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "ssd42", tsSSE42Enable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "avx", tsAVXEnable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "avx2", tsAVX2Enable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "fma", tsFMAEnable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "avx512", tsAVX512Enable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "ssd42", tsSSE42Supported, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "avx", tsAVXSupported, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "avx2", tsAVX2Supported, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "fma", tsFMASupported, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "avx512", tsAVX512Supported, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "simdEnable", tsSIMDEnable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "AVX512Enable", tsAVX512Enable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
@ -667,6 +670,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "retentionSpeedLimitMB", tsRetentionSpeedLimitMB, 0, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
@ -776,7 +780,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
// min free disk space used to check if the disk is full [50MB, 1GB]
if (cfgAddInt64(pCfg, "minDiskFreeSize", tsMinDiskFreeSize, TFS_MIN_DISK_FREE_SIZE, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
// clang-format on
@ -1117,6 +1121,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsTimeToGetAvailableConn = cfgGetItem(pCfg, "timeToGetAvailableConn")->i32;
tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32;
tsRetentionSpeedLimitMB = cfgGetItem(pCfg, "retentionSpeedLimitMB")->i32;
tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32;
tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32;
tsRatioOfVnodeStreamThreads = cfgGetItem(pCfg, "ratioOfVnodeStreamThreads")->fval;
@ -1299,8 +1304,8 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
return 0;
}
int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd,
const char *envFile, char *apolloUrl, SArray *pArgs) {
int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl,
SArray *pArgs) {
if (tsCfg == NULL) osDefaultInit();
SConfig *pCfg = cfgInit();

32
source/common/src/tmsg.c
View File

@ -20,11 +20,13 @@
#undef TD_MSG_DICT_
#undef TD_MSG_RANGE_CODE_
#define TD_MSG_INFO_
#undef TD_MSG_TYPE_INFO_
#undef TD_MSG_SEG_CODE_
#include "tmsgdef.h"
#undef TD_MSG_NUMBER_
#undef TD_MSG_INFO_
#undef TD_MSG_TYPE_INFO_
#undef TD_MSG_RANGE_CODE_
#define TD_MSG_DICT_
#undef TD_MSG_SEG_CODE_
@ -32,6 +34,7 @@
#undef TD_MSG_NUMBER_
#undef TD_MSG_INFO_
#undef TD_MSG_TYPE_INFO_
#undef TD_MSG_DICT_
#undef TD_MSG_SEG_CODE_
#define TD_MSG_RANGE_CODE_
@ -464,6 +467,8 @@ int32_t tSerializeSClientHbBatchReq(void *buf, int32_t bufLen, const SClientHbBa
SClientHbReq *pReq = taosArrayGet(pBatchReq->reqs, i);
if (tSerializeSClientHbReq(&encoder, pReq) < 0) return -1;
}
if (tEncodeI64(&encoder, pBatchReq->ipWhiteList) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -490,6 +495,10 @@ int32_t tDeserializeSClientHbBatchReq(void *buf, int32_t bufLen, SClientHbBatchR
taosArrayPush(pBatchReq->reqs, &req);
}
if (!tDecodeIsEnd(&decoder)) {
tDecodeI64(&decoder, &pBatchReq->ipWhiteList);
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
@ -814,7 +823,6 @@ int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq
for (int32_t i = 0; i < pReq->numOfFields; ++i) {
if (pReq->alterType == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION) {
taosArrayDestroy(pReq->pFields);
pReq->pFields = taosArrayInit(pReq->numOfFields, sizeof(SFieldWithOptions));
SFieldWithOptions field = {0};
@ -1634,8 +1642,10 @@ int32_t tSerializeSCreateUserReq(void *buf, int32_t bufLen, SCreateUserReq *pReq
if (tEncodeU32(&encoder, pReq->pIpRanges[i].ip) < 0) return -1;
if (tEncodeU32(&encoder, pReq->pIpRanges[i].mask) < 0) return -1;
}
ENCODESQL();
if (tEncodeI8(&encoder, pReq->isImport) < 0) return -1;
if (tEncodeI8(&encoder, pReq->createDb) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -1661,8 +1671,12 @@ int32_t tDeserializeSCreateUserReq(void *buf, int32_t bufLen, SCreateUserReq *pR
if (tDecodeU32(&decoder, &(pReq->pIpRanges[i].ip)) < 0) return -1;
if (tDecodeU32(&decoder, &(pReq->pIpRanges[i].mask)) < 0) return -1;
}
DECODESQL();
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI8(&decoder, &pReq->createDb) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->isImport) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
@ -1810,6 +1824,7 @@ int32_t tSerializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq)
}
if (tEncodeI64(&encoder, pReq->privileges) < 0) return -1;
ENCODESQL();
if (tEncodeU8(&encoder, pReq->flag) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -1849,6 +1864,9 @@ int32_t tDeserializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq
}
if (tDecodeI64(&decoder, &pReq->privileges) < 0) return -1;
DECODESQL();
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeU8(&decoder, &pReq->flag) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
@ -4597,6 +4615,7 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq
if (tEncodeCStr(&encoder, pReq->filterTb) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
if (tEncodeI64(&encoder, pReq->compactId) < 0) return -1;
if (tEncodeI8(&encoder, pReq->withFull) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -4619,7 +4638,9 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR
} else {
pReq->compactId = -1;
}
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI8(&decoder, (int8_t *)&pReq->withFull) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
@ -7046,6 +7067,7 @@ int32_t tSerializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq) {
if (tEncodeI8(&encoder, pReq->taskType) < 0) return -1;
if (tEncodeI8(&encoder, pReq->explain) < 0) return -1;
if (tEncodeI8(&encoder, pReq->needFetch) < 0) return -1;
if (tEncodeI8(&encoder, pReq->compress) < 0) return -1;
if (tEncodeU32(&encoder, pReq->sqlLen) < 0) return -1;
if (tEncodeCStrWithLen(&encoder, pReq->sql, pReq->sqlLen) < 0) return -1;
if (tEncodeU32(&encoder, pReq->msgLen) < 0) return -1;
@ -7086,6 +7108,7 @@ int32_t tDeserializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq)
if (tDecodeI8(&decoder, &pReq->taskType) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->explain) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->needFetch) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->compress) < 0) return -1;
if (tDecodeU32(&decoder, &pReq->sqlLen) < 0) return -1;
if (tDecodeCStrAlloc(&decoder, &pReq->sql) < 0) return -1;
if (tDecodeU32(&decoder, &pReq->msgLen) < 0) return -1;
@ -9728,6 +9751,7 @@ void tDestroySubmitTbData(SSubmitTbData *pTbData, int32_t flag) {
for (int32_t i = 0; i < nRow; ++i) {
tRowDestroy(rows[i]);
rows[i] = NULL;
}
taosArrayDestroy(pTbData->aRowP);
}

6
source/common/src/tvariant.c
View File

@ -88,7 +88,7 @@ static int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint
if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
return TSDB_CODE_FAILED;
}
if (val > UINT64_MAX) {
if (val > (double)UINT64_MAX) {
errno = ERANGE;
return TSDB_CODE_FAILED;
}
@ -172,7 +172,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
} break;
case TK_NK_FLOAT: {
double val = round(taosStr2Double(z, &endPtr));
if (!IS_VALID_INT64(val)) {
if(val < (double)INT64_MIN || val > (double)INT64_MAX){
return TSDB_CODE_FAILED;
}
if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
@ -271,7 +271,7 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) {
} break;
case TK_NK_FLOAT: {
double val = round(taosStr2Double(p, &endPtr));
if (!IS_VALID_UINT64(val)) {
if (val < 0 || val > (double)UINT64_MAX) {
return TSDB_CODE_FAILED;
}
if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {

30
source/common/test/CMakeLists.txt
View File

@ -40,12 +40,24 @@ add_test(
COMMAND dataformatTest
)
# tmsg test
# add_executable(tmsgTest "")
# target_sources(tmsgTest
# PRIVATE
# "tmsgTest.cpp"
# "../src/tmsg.c"
# )
# target_include_directories(tmsgTest PUBLIC "${TD_SOURCE_DIR}/include/common/")
# target_link_libraries(tmsgTest PUBLIC os util gtest gtest_main)
if (${TD_LINUX})
# tmsg test
add_executable(tmsgTest "")
target_sources(tmsgTest
PRIVATE
"tmsgTest.cpp"
"../src/tmsg.c"
)
target_include_directories(tmsgTest PUBLIC "${TD_SOURCE_DIR}/include/common/")
target_link_libraries(tmsgTest PUBLIC os util gtest gtest_main)
add_test(
NAME tmsgTest
COMMAND tmsgTest
)
# config file for msg type table
SET(MSG_TBL_FILE ${CMAKE_CURRENT_SOURCE_DIR}/msgTypeTable.ini)
add_custom_command(TARGET tmsgTest POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${MSG_TBL_FILE} $<TARGET_FILE_DIR:tmsgTest>
)
endif ()

540
source/common/test/msgTypeTable.ini Normal file
View File

@ -0,0 +1,540 @@
TDMT_DND_CREATE_MNODE = 1
TDMT_DND_CREATE_MNODE_RSP = 2
TDMT_DND_DROP_MNODE = 3
TDMT_DND_DROP_MNODE_RSP = 4
TDMT_DND_CREATE_QNODE = 5
TDMT_DND_CREATE_QNODE_RSP = 6
TDMT_DND_DROP_QNODE = 7
TDMT_DND_DROP_QNODE_RSP = 8
TDMT_DND_CREATE_SNODE = 9
TDMT_DND_CREATE_SNODE_RSP = 10
TDMT_DND_DROP_SNODE = 11
TDMT_DND_DROP_SNODE_RSP = 12
TDMT_DND_CREATE_BNODE = 13
TDMT_DND_CREATE_BNODE_RSP = 14
TDMT_DND_DROP_BNODE = 15
TDMT_DND_DROP_BNODE_RSP = 16
TDMT_DND_CREATE_VNODE = 17
TDMT_DND_CREATE_VNODE_RSP = 18
TDMT_DND_DROP_VNODE = 19
TDMT_DND_DROP_VNODE_RSP = 20
TDMT_DND_SERVER_STATUS = 21
TDMT_DND_SERVER_STATUS_RSP = 22
TDMT_DND_NET_TEST = 23
TDMT_DND_NET_TEST_RSP = 24
TDMT_DND_CONFIG_DNODE = 25
TDMT_DND_CONFIG_DNODE_RSP = 26
TDMT_DND_SYSTABLE_RETRIEVE = 27
TDMT_DND_SYSTABLE_RETRIEVE_RSP = 28
TDMT_DND_UNUSED_CODE = 29
TDMT_DND_UNUSED_CODE_RSP = 30
TDMT_DND_ALTER_MNODE_TYPE = 31
TDMT_DND_ALTER_MNODE_TYPE_RSP = 32
TDMT_DND_ALTER_VNODE_TYPE = 33
TDMT_DND_ALTER_VNODE_TYPE_RSP = 34
TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP = 35
TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP_RSP = 36
TDMT_DND_CREATE_ENCRYPT_KEY = 37
TDMT_DND_CREATE_ENCRYPT_KEY_RSP = 38
TDMT_MND_CONNECT = 257
TDMT_MND_CONNECT_RSP = 258
TDMT_MND_CREATE_ACCT = 259
TDMT_MND_CREATE_ACCT_RSP = 260
TDMT_MND_ALTER_ACCT = 261
TDMT_MND_ALTER_ACCT_RSP = 262
TDMT_MND_DROP_ACCT = 263
TDMT_MND_DROP_ACCT_RSP = 264
TDMT_MND_CREATE_USER = 265
TDMT_MND_CREATE_USER_RSP = 266
TDMT_MND_ALTER_USER = 267
TDMT_MND_ALTER_USER_RSP = 268
TDMT_MND_DROP_USER = 269
TDMT_MND_DROP_USER_RSP = 270
TDMT_MND_GET_USER_AUTH = 271
TDMT_MND_GET_USER_AUTH_RSP = 272
TDMT_MND_CREATE_DNODE = 273
TDMT_MND_CREATE_DNODE_RSP = 274
TDMT_MND_CONFIG_DNODE = 275
TDMT_MND_CONFIG_DNODE_RSP = 276
TDMT_MND_DROP_DNODE = 277
TDMT_MND_DROP_DNODE_RSP = 278
TDMT_MND_CREATE_MNODE = 279
TDMT_MND_CREATE_MNODE_RSP = 280
TDMT_MND_ALTER_MNODE = 281
TDMT_MND_ALTER_MNODE_RSP = 282
TDMT_MND_DROP_MNODE = 283
TDMT_MND_DROP_MNODE_RSP = 284
TDMT_MND_CREATE_QNODE = 285
TDMT_MND_CREATE_QNODE_RSP = 286
TDMT_MND_ALTER_QNODE = 287
TDMT_MND_ALTER_QNODE_RSP = 288
TDMT_MND_DROP_QNODE = 289
TDMT_MND_DROP_QNODE_RSP = 290
TDMT_MND_QNODE_LIST = 291
TDMT_MND_QNODE_LIST_RSP = 292
TDMT_MND_DNODE_LIST = 293
TDMT_MND_DNODE_LIST_RSP = 294
TDMT_MND_CREATE_SNODE = 295
TDMT_MND_CREATE_SNODE_RSP = 296
TDMT_MND_ALTER_SNODE = 297
TDMT_MND_ALTER_SNODE_RSP = 298
TDMT_MND_DROP_SNODE = 299
TDMT_MND_DROP_SNODE_RSP = 300
TDMT_MND_CREATE_BNODE = 301
TDMT_MND_CREATE_BNODE_RSP = 302
TDMT_MND_ALTER_BNODE = 303
TDMT_MND_ALTER_BNODE_RSP = 304
TDMT_MND_DROP_BNODE = 305
TDMT_MND_DROP_BNODE_RSP = 306
TDMT_MND_CREATE_DB = 307
TDMT_MND_CREATE_DB_RSP = 308
TDMT_MND_DROP_DB = 309
TDMT_MND_DROP_DB_RSP = 310
TDMT_MND_USE_DB = 311
TDMT_MND_USE_DB_RSP = 312
TDMT_MND_ALTER_DB = 313
TDMT_MND_ALTER_DB_RSP = 314
TDMT_MND_SYNC_DB = 315
TDMT_MND_SYNC_DB_RSP = 316
TDMT_MND_COMPACT_DB = 317
TDMT_MND_COMPACT_DB_RSP = 318
TDMT_MND_TRIM_DB = 319
TDMT_MND_TRIM_DB_RSP = 320
TDMT_MND_GET_DB_CFG = 321
TDMT_MND_GET_DB_CFG_RSP = 322
TDMT_MND_VGROUP_LIST = 323
TDMT_MND_VGROUP_LIST_RSP = 324
TDMT_MND_CREATE_FUNC = 325
TDMT_MND_CREATE_FUNC_RSP = 326
TDMT_MND_RETRIEVE_FUNC = 327
TDMT_MND_RETRIEVE_FUNC_RSP = 328
TDMT_MND_DROP_FUNC = 329
TDMT_MND_DROP_FUNC_RSP = 330
TDMT_MND_CREATE_STB = 331
TDMT_MND_CREATE_STB_RSP = 332
TDMT_MND_ALTER_STB = 333
TDMT_MND_ALTER_STB_RSP = 334
TDMT_MND_DROP_STB = 335
TDMT_MND_DROP_STB_RSP = 336
TDMT_MND_TABLE_META = 337
TDMT_MND_TABLE_META_RSP = 338
TDMT_MND_CREATE_SMA = 339
TDMT_MND_CREATE_SMA_RSP = 340
TDMT_MND_DROP_SMA = 341
TDMT_MND_DROP_SMA_RSP = 342
TDMT_MND_CREATE_STREAM = 343
TDMT_MND_CREATE_STREAM_RSP = 344
TDMT_MND_ALTER_STREAM = 345
TDMT_MND_ALTER_STREAM_RSP = 346
TDMT_MND_DROP_STREAM = 347
TDMT_MND_DROP_STREAM_RSP = 348
TDMT_MND_RECOVER_STREAM = 349
TDMT_MND_RECOVER_STREAM_RSP = 350
TDMT_MND_CREATE_INDEX = 351
TDMT_MND_CREATE_INDEX_RSP = 352
TDMT_MND_DROP_INDEX = 353
TDMT_MND_DROP_INDEX_RSP = 354
TDMT_MND_GET_INDEX = 355
TDMT_MND_GET_INDEX_RSP = 356
TDMT_MND_GET_TABLE_INDEX = 357
TDMT_MND_GET_TABLE_INDEX_RSP = 358
TDMT_MND_BATCH_META = 359
TDMT_MND_BATCH_META_RSP = 360
TDMT_MND_TABLE_CFG = 361
TDMT_MND_TABLE_CFG_RSP = 362
TDMT_MND_TMQ_CREATE_TOPIC = 363
TDMT_MND_TMQ_CREATE_TOPIC_RSP = 364
TDMT_MND_UNUSED1 = 365
TDMT_MND_UNUSED1_RSP = 366
TDMT_MND_TMQ_DROP_TOPIC = 367
TDMT_MND_TMQ_DROP_TOPIC_RSP = 368
TDMT_MND_TMQ_SUBSCRIBE = 369
TDMT_MND_TMQ_SUBSCRIBE_RSP = 370
TDMT_MND_TMQ_ASK_EP = 371
TDMT_MND_TMQ_ASK_EP_RSP = 372
TDMT_MND_TMQ_CONSUMER_RECOVER = 373
TDMT_MND_TMQ_CONSUMER_RECOVER_RSP = 374
TDMT_MND_TMQ_HB = 375
TDMT_MND_TMQ_HB_RSP = 376
TDMT_MND_TMQ_DO_REBALANCE = 377
TDMT_MND_TMQ_DO_REBALANCE_RSP = 378
TDMT_MND_TMQ_DROP_CGROUP = 379
TDMT_MND_TMQ_DROP_CGROUP_RSP = 380
TDMT_MND_CREATE_VG = 381
TDMT_MND_CREATE_VG_RSP = 382
TDMT_MND_TMQ_TIMER = 383
TDMT_MND_TMQ_TIMER_RSP = 384
TDMT_MND_TELEM_TIMER = 385
TDMT_MND_TELEM_TIMER_RSP = 386
TDMT_MND_TRANS_TIMER = 387
TDMT_MND_TRANS_TIMER_RSP = 388
TDMT_MND_TTL_TIMER = 389
TDMT_MND_TTL_TIMER_RSP = 390
TDMT_MND_GRANT_HB_TIMER = 391
TDMT_MND_GRANT_HB_TIMER_RSP = 392
TDMT_MND_NODECHECK_TIMER = 393
TDMT_MND_NODECHECK_TIMER_RSP = 394
TDMT_MND_KILL_TRANS = 395
TDMT_MND_KILL_TRANS_RSP = 396
TDMT_MND_KILL_QUERY = 397
TDMT_MND_KILL_QUERY_RSP = 398
TDMT_MND_KILL_CONN = 399
TDMT_MND_KILL_CONN_RSP = 400
TDMT_MND_HEARTBEAT = 401
TDMT_MND_HEARTBEAT_RSP = 402
TDMT_MND_STATUS = 403
TDMT_MND_STATUS_RSP = 404
TDMT_MND_SHOW = 405
TDMT_MND_SHOW_RSP = 406
TDMT_MND_SYSTABLE_RETRIEVE = 407
TDMT_MND_SYSTABLE_RETRIEVE_RSP = 408
TDMT_MND_GRANT = 409
TDMT_MND_GRANT_RSP = 410
TDMT_MND_AUTH = 411
TDMT_MND_AUTH_RSP = 412
TDMT_MND_APPLY_MSG = 413
TDMT_MND_APPLY_MSG_RSP = 414
TDMT_MND_BALANCE_VGROUP = 415
TDMT_MND_BALANCE_VGROUP_RSP = 416
TDMT_MND_MERGE_VGROUP = 417
TDMT_MND_MERGE_VGROUP_RSP = 418
TDMT_MND_REDISTRIBUTE_VGROUP = 419
TDMT_MND_REDISTRIBUTE_VGROUP_RSP = 420
TDMT_MND_SPLIT_VGROUP = 421
TDMT_MND_SPLIT_VGROUP_RSP = 422
TDMT_MND_SHOW_VARIABLES = 423
TDMT_MND_SHOW_VARIABLES_RSP = 424
TDMT_MND_SERVER_VERSION = 425
TDMT_MND_SERVER_VERSION_RSP = 426
TDMT_MND_UPTIME_TIMER = 427
TDMT_MND_UPTIME_TIMER_RSP = 428
TDMT_MND_TMQ_LOST_CONSUMER_CLEAR = 429
TDMT_MND_TMQ_LOST_CONSUMER_CLEAR_RSP = 430
TDMT_MND_STREAM_HEARTBEAT = 431
TDMT_MND_STREAM_HEARTBEAT_RSP = 432
TDMT_MND_RETRIEVE_IP_WHITE = 433
TDMT_MND_RETRIEVE_IP_WHITE_RSP = 434
TDMT_MND_GET_USER_WHITELIST = 435
TDMT_MND_GET_USER_WHITELIST_RSP = 436
TDMT_MND_NOTIFY = 437
TDMT_MND_NOTIFY_RSP = 438
TDMT_MND_BALANCE_VGROUP_LEADER = 439
TDMT_MND_BALANCE_VGROUP_LEADER_RSP = 440
TDMT_MND_RESTORE_DNODE = 441
TDMT_MND_RESTORE_DNODE_RSP = 442
TDMT_MND_PAUSE_STREAM = 443
TDMT_MND_PAUSE_STREAM_RSP = 444
TDMT_MND_RESUME_STREAM = 445
TDMT_MND_RESUME_STREAM_RSP = 446
TDMT_MND_STREAM_CHECKPOINT_TIMER = 447
TDMT_MND_STREAM_CHECKPOINT_TIMER_RSP = 448
TDMT_MND_STREAM_BEGIN_CHECKPOINT = 449
TDMT_MND_STREAM_BEGIN_CHECKPOINT_RSP = 450
TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE = 451
TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE_RSP = 452
TDMT_MND_STREAM_NODECHANGE_CHECK = 453
TDMT_MND_STREAM_NODECHANGE_CHECK_RSP = 454
TDMT_MND_TRIM_DB_TIMER = 455
TDMT_MND_TRIM_DB_TIMER_RSP = 456
TDMT_MND_GRANT_NOTIFY = 457
TDMT_MND_GRANT_NOTIFY_RSP = 458
TDMT_MND_CREATE_VIEW = 459
TDMT_MND_CREATE_VIEW_RSP = 460
TDMT_MND_DROP_VIEW = 461
TDMT_MND_DROP_VIEW_RSP = 462
TDMT_MND_VIEW_META = 463
TDMT_MND_VIEW_META_RSP = 464
TDMT_MND_STATIS = 465
TDMT_MND_STATIS_RSP = 466
TDMT_MND_KILL_COMPACT = 467
TDMT_MND_KILL_COMPACT_RSP = 468
TDMT_MND_COMPACT_TIMER = 469
TDMT_MND_COMPACT_TIMER_RSP = 470
TDMT_MND_STREAM_REQ_CHKPT = 471
TDMT_MND_STREAM_REQ_CHKPT_RSP = 472
TDMT_MND_CONFIG_CLUSTER = 473
TDMT_MND_CONFIG_CLUSTER_RSP = 474
TDMT_MND_CREATE_ENCRYPT_KEY = 475
TDMT_MND_CREATE_ENCRYPT_KEY_RSP = 476
TDMT_MND_S3MIGRATE_DB = 477
TDMT_MND_S3MIGRATE_DB_RSP = 478
TDMT_MND_S3MIGRATE_DB_TIMER = 479
TDMT_MND_S3MIGRATE_DB_TIMER_RSP = 480
TDMT_MND_UNUSED2 = 481
TDMT_MND_UNUSED2_RSP = 482
TDMT_MND_CREATE_TSMA = 483
TDMT_MND_CREATE_TSMA_RSP = 484
TDMT_MND_DROP_TSMA = 485
TDMT_MND_DROP_TSMA_RSP = 486
TDMT_MND_STB_DROP = 487
TDMT_MND_STB_DROP_RSP = 488
TDMT_MND_GET_TABLE_TSMA = 489
TDMT_MND_GET_TABLE_TSMA_RSP = 490
TDMT_MND_GET_TSMA = 491
TDMT_MND_GET_TSMA_RSP = 492
TDMT_MND_DROP_TB_WITH_TSMA = 493
TDMT_MND_DROP_TB_WITH_TSMA_RSP = 494
TDMT_MND_STREAM_UPDATE_CHKPT_EVT = 495
TDMT_MND_STREAM_UPDATE_CHKPT_EVT_RSP = 496
TDMT_MND_STREAM_CHKPT_REPORT = 497
TDMT_MND_STREAM_CHKPT_REPORT_RSP = 498
TDMT_VND_SUBMIT = 513
TDMT_VND_SUBMIT_RSP = 514
TDMT_VND_CREATE_TABLE = 515
TDMT_VND_CREATE_TABLE_RSP = 516
TDMT_VND_ALTER_TABLE = 517
TDMT_VND_ALTER_TABLE_RSP = 518
TDMT_VND_DROP_TABLE = 519
TDMT_VND_DROP_TABLE_RSP = 520
TDMT_VND_UPDATE_TAG_VAL = 521
TDMT_VND_UPDATE_TAG_VAL_RSP = 522
TDMT_VND_TABLE_META = 523
TDMT_VND_TABLE_META_RSP = 524
TDMT_VND_TABLES_META = 525
TDMT_VND_TABLES_META_RSP = 526
TDMT_VND_TABLE_CFG = 527
TDMT_VND_TABLE_CFG_RSP = 528
TDMT_VND_BATCH_META = 529
TDMT_VND_BATCH_META_RSP = 530
TDMT_VND_CREATE_STB = 531
TDMT_VND_CREATE_STB_RSP = 532
TDMT_VND_ALTER_STB = 533
TDMT_VND_ALTER_STB_RSP = 534
TDMT_VND_DROP_STB = 535
TDMT_VND_DROP_STB_RSP = 536
TDMT_VND_UNUSED1 = 537
TDMT_VND_UNUSED1_RSP = 538
TDMT_VND_UNUSED2 = 539
TDMT_VND_UNUSED2_RSP = 540
TDMT_VND_UNUSED3 = 541
TDMT_VND_UNUSED3_RSP = 542
TDMT_VND_UNUSED4 = 543
TDMT_VND_UNUSED4_RSP = 544
TDMT_VND_UNUSED5 = 545
TDMT_VND_UNUSED5_RSP = 546
TDMT_VND_UNUSED6 = 547
TDMT_VND_UNUSED6_RSP = 548
TDMT_VND_UNUSED7 = 549
TDMT_VND_UNUSED7_RSP = 550
TDMT_VND_UNUSED8 = 551
TDMT_VND_UNUSED8_RSP = 552
TDMT_VND_UNUSED9 = 553
TDMT_VND_UNUSED9_RSP = 554
TDMT_VND_UNUSED10 = 555
TDMT_VND_UNUSED10_RSP = 556
TDMT_VND_UNUSED11 = 557
TDMT_VND_UNUSED11_RSP = 558
TDMT_VND_UNUSED12 = 559
TDMT_VND_UNUSED12_RSP = 560
TDMT_VND_UNUSED13 = 561
TDMT_VND_UNUSED13_RSP = 562
TDMT_VND_UNUSED14 = 563
TDMT_VND_UNUSED14_RSP = 564
TDMT_VND_UNUSED15 = 565
TDMT_VND_UNUSED15_RSP = 566
TDMT_VND_CREATE_SMA = 567
TDMT_VND_CREATE_SMA_RSP = 568
TDMT_VND_CANCEL_SMA = 569
TDMT_VND_CANCEL_SMA_RSP = 570
TDMT_VND_DROP_SMA = 571
TDMT_VND_DROP_SMA_RSP = 572
TDMT_VND_SUBMIT_RSMA = 573
TDMT_VND_SUBMIT_RSMA_RSP = 574
TDMT_VND_FETCH_RSMA = 575
TDMT_VND_FETCH_RSMA_RSP = 576
TDMT_VND_EXEC_RSMA = 577
TDMT_VND_EXEC_RSMA_RSP = 578
TDMT_VND_DELETE = 579
TDMT_VND_DELETE_RSP = 580
TDMT_VND_BATCH_DEL = 581
TDMT_VND_BATCH_DEL_RSP = 582
TDMT_VND_ALTER_CONFIG = 583
TDMT_VND_ALTER_CONFIG_RSP = 584
TDMT_VND_ALTER_REPLICA = 585
TDMT_VND_ALTER_REPLICA_RSP = 586
TDMT_VND_ALTER_CONFIRM = 587
TDMT_VND_ALTER_CONFIRM_RSP = 588
TDMT_VND_ALTER_HASHRANGE = 589
TDMT_VND_ALTER_HASHRANGE_RSP = 590
TDMT_VND_COMPACT = 591
TDMT_VND_COMPACT_RSP = 592
TDMT_VND_DROP_TTL_TABLE = 593
TDMT_VND_DROP_TTL_TABLE_RSP = 594
TDMT_VND_TRIM = 595
TDMT_VND_TRIM_RSP = 596
TDMT_VND_COMMIT = 597
TDMT_VND_COMMIT_RSP = 598
TDMT_VND_CREATE_INDEX = 599
TDMT_VND_CREATE_INDEX_RSP = 600
TDMT_VND_DROP_INDEX = 601
TDMT_VND_DROP_INDEX_RSP = 602
TDMT_VND_DISABLE_WRITE = 603
TDMT_VND_DISABLE_WRITE_RSP = 604
TDMT_VND_QUERY_COMPACT_PROGRESS = 605
TDMT_VND_QUERY_COMPACT_PROGRESS_RSP = 606
TDMT_VND_KILL_COMPACT = 607
TDMT_VND_KILL_COMPACT_RSP = 608
TDMT_VND_S3MIGRATE = 609
TDMT_VND_S3MIGRATE_RSP = 610
TDMT_VND_ARB_HEARTBEAT = 611
TDMT_VND_ARB_HEARTBEAT_RSP = 612
TDMT_VND_ARB_CHECK_SYNC = 613
TDMT_VND_ARB_CHECK_SYNC_RSP = 614
TDMT_VND_FETCH_TTL_EXPIRED_TBS = 615
TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP = 616
TDMT_SCH_QUERY = 769
TDMT_SCH_QUERY_RSP = 770
TDMT_SCH_MERGE_QUERY = 771
TDMT_SCH_MERGE_QUERY_RSP = 772
TDMT_SCH_QUERY_CONTINUE = 773
TDMT_SCH_QUERY_CONTINUE_RSP = 774
TDMT_SCH_QUERY_HEARTBEAT = 775
TDMT_SCH_QUERY_HEARTBEAT_RSP = 776
TDMT_SCH_FETCH = 777
TDMT_SCH_FETCH_RSP = 778
TDMT_SCH_MERGE_FETCH = 779
TDMT_SCH_MERGE_FETCH_RSP = 780
TDMT_SCH_CANCEL_TASK = 781
TDMT_SCH_CANCEL_TASK_RSP = 782
TDMT_SCH_DROP_TASK = 783
TDMT_SCH_DROP_TASK_RSP = 784
TDMT_SCH_EXPLAIN = 785
TDMT_SCH_EXPLAIN_RSP = 786
TDMT_SCH_LINK_BROKEN = 787
TDMT_SCH_LINK_BROKEN_RSP = 788
TDMT_SCH_TASK_NOTIFY = 789
TDMT_SCH_TASK_NOTIFY_RSP = 790
TDMT_STREAM_TASK_DEPLOY = 1025
TDMT_STREAM_TASK_DEPLOY_RSP = 1026
TDMT_STREAM_TASK_DROP = 1027
TDMT_STREAM_TASK_DROP_RSP = 1028
TDMT_STREAM_TASK_RUN = 1029
TDMT_STREAM_TASK_RUN_RSP = 1030
TDMT_STREAM_TASK_DISPATCH = 1031
TDMT_STREAM_TASK_DISPATCH_RSP = 1032
TDMT_STREAM_TASK_UPDATE_CHKPT = 1033
TDMT_STREAM_TASK_UPDATE_CHKPT_RSP = 1034
TDMT_STREAM_RETRIEVE = 1035
TDMT_STREAM_RETRIEVE_RSP = 1036
TDMT_STREAM_TASK_CHECKPOINT_READY = 1037
TDMT_STREAM_TASK_CHECKPOINT_READY_RSP = 1038
TDMT_STREAM_TASK_REPORT_CHECKPOINT = 1039
TDMT_STREAM_TASK_REPORT_CHECKPOINT_RSP = 1040
TDMT_STREAM_TASK_RESTORE_CHECKPOINT = 1041
TDMT_STREAM_TASK_RESTORE_CHECKPOINT_RSP = 1042
TDMT_STREAM_TASK_PAUSE = 1043
TDMT_STREAM_TASK_PAUSE_RSP = 1044
TDMT_STREAM_TASK_RESUME = 1045
TDMT_STREAM_TASK_RESUME_RSP = 1046
TDMT_STREAM_TASK_STOP = 1047
TDMT_STREAM_TASK_STOP_RSP = 1048
TDMT_STREAM_UNUSED = 1049
TDMT_STREAM_UNUSED_RSP = 1050
TDMT_STREAM_CREATE = 1051
TDMT_STREAM_CREATE_RSP = 1052
TDMT_STREAM_DROP = 1053
TDMT_STREAM_DROP_RSP = 1054
TDMT_STREAM_RETRIEVE_TRIGGER = 1055
TDMT_STREAM_RETRIEVE_TRIGGER_RSP = 1056
TDMT_SYNC_TIMEOUT = 1537
TDMT_SYNC_TIMEOUT_RSP = 1538
TDMT_SYNC_TIMEOUT_ELECTION = 1539
TDMT_SYNC_TIMEOUT_ELECTION_RSP = 1540
TDMT_SYNC_PING_REPLY = 1541
TDMT_SYNC_PING_REPLY_RSP = 1542
TDMT_SYNC_CLIENT_REQUEST = 1543
TDMT_SYNC_CLIENT_REQUEST_RSP = 1544
TDMT_SYNC_CLIENT_REQUEST_BATCH = 1545
TDMT_SYNC_CLIENT_REQUEST_BATCH_RSP = 1546
TDMT_SYNC_CLIENT_REQUEST_REPLY = 1547
TDMT_SYNC_CLIENT_REQUEST_REPLY_RSP = 1548
TDMT_SYNC_REQUEST_VOTE = 1549
TDMT_SYNC_REQUEST_VOTE_RSP = 1550
TDMT_SYNC_REQUEST_VOTE_REPLY = 1551
TDMT_SYNC_REQUEST_VOTE_REPLY_RSP = 1552
TDMT_SYNC_APPEND_ENTRIES = 1553
TDMT_SYNC_APPEND_ENTRIES_RSP = 1554
TDMT_SYNC_APPEND_ENTRIES_BATCH = 1555
TDMT_SYNC_APPEND_ENTRIES_BATCH_RSP = 1556
TDMT_SYNC_APPEND_ENTRIES_REPLY = 1557
TDMT_SYNC_APPEND_ENTRIES_REPLY_RSP = 1558
TDMT_SYNC_NOOP = 1559
TDMT_SYNC_NOOP_RSP = 1560
TDMT_SYNC_UNKNOWN = 1561
TDMT_SYNC_UNKNOWN_RSP = 1562
TDMT_SYNC_COMMON_RESPONSE = 1563
TDMT_SYNC_COMMON_RESPONSE_RSP = 1564
TDMT_SYNC_APPLY_MSG = 1565
TDMT_SYNC_APPLY_MSG_RSP = 1566
TDMT_SYNC_CONFIG_CHANGE = 1567
TDMT_SYNC_CONFIG_CHANGE_RSP = 1568
TDMT_SYNC_CONFIG_CHANGE_FINISH = 1569
TDMT_SYNC_CONFIG_CHANGE_FINISH_RSP = 1570
TDMT_SYNC_SNAPSHOT_SEND = 1571
TDMT_SYNC_SNAPSHOT_SEND_RSP = 1572
TDMT_SYNC_SNAPSHOT_RSP = 1573
TDMT_SYNC_SNAPSHOT_RSP_RSP = 1574
TDMT_SYNC_LEADER_TRANSFER = 1575
TDMT_SYNC_LEADER_TRANSFER_RSP = 1576
TDMT_SYNC_SET_MNODE_STANDBY = 1577
TDMT_SYNC_SET_MNODE_STANDBY_RSP = 1578
TDMT_SYNC_SET_VNODE_STANDBY = 1579
TDMT_SYNC_SET_VNODE_STANDBY_RSP = 1580
TDMT_SYNC_HEARTBEAT = 1581
TDMT_SYNC_HEARTBEAT_RSP = 1582
TDMT_SYNC_HEARTBEAT_REPLY = 1583
TDMT_SYNC_HEARTBEAT_REPLY_RSP = 1584
TDMT_SYNC_LOCAL_CMD = 1585
TDMT_SYNC_LOCAL_CMD_RSP = 1586
TDMT_SYNC_PREP_SNAPSHOT = 1587
TDMT_SYNC_PREP_SNAPSHOT_RSP = 1588
TDMT_SYNC_PREP_SNAPSHOT_REPLY = 1589
TDMT_SYNC_PREP_SNAPSHOT_REPLY_RSP = 1590
TDMT_SYNC_UNUSED_CODE = 1591
TDMT_SYNC_UNUSED_CODE_RSP = 1592
TDMT_SYNC_FORCE_FOLLOWER = 1593
TDMT_SYNC_FORCE_FOLLOWER_RSP = 1594
TDMT_SYNC_SET_ASSIGNED_LEADER = 1595
TDMT_SYNC_SET_ASSIGNED_LEADER_RSP = 1596
TDMT_VND_STREAM_SCAN_HISTORY = 1793
TDMT_VND_STREAM_SCAN_HISTORY_RSP = 1794
TDMT_VND_STREAM_CHECK_POINT_SOURCE = 1795
TDMT_VND_STREAM_CHECK_POINT_SOURCE_RSP = 1796
TDMT_VND_STREAM_TASK_UPDATE = 1797
TDMT_VND_STREAM_TASK_UPDATE_RSP = 1798
TDMT_VND_STREAM_TASK_RESET = 1799
TDMT_VND_STREAM_TASK_RESET_RSP = 1800
TDMT_VND_STREAM_TASK_CHECK = 1801
TDMT_VND_STREAM_TASK_CHECK_RSP = 1802
TDMT_VND_STREAM_UNUSED = 1803
TDMT_VND_STREAM_UNUSED_RSP = 1804
TDMT_VND_GET_STREAM_PROGRESS = 1805
TDMT_VND_GET_STREAM_PROGRESS_RSP = 1806
TDMT_VND_TMQ_SUBSCRIBE = 2049
TDMT_VND_TMQ_SUBSCRIBE_RSP = 2050
TDMT_VND_TMQ_DELETE_SUB = 2051
TDMT_VND_TMQ_DELETE_SUB_RSP = 2052
TDMT_VND_TMQ_COMMIT_OFFSET = 2053
TDMT_VND_TMQ_COMMIT_OFFSET_RSP = 2054
TDMT_VND_TMQ_SEEK = 2055
TDMT_VND_TMQ_SEEK_RSP = 2056
TDMT_VND_TMQ_ADD_CHECKINFO = 2057
TDMT_VND_TMQ_ADD_CHECKINFO_RSP = 2058
TDMT_VND_TMQ_DEL_CHECKINFO = 2059
TDMT_VND_TMQ_DEL_CHECKINFO_RSP = 2060
TDMT_VND_TMQ_CONSUME = 2061
TDMT_VND_TMQ_CONSUME_RSP = 2062
TDMT_VND_TMQ_CONSUME_PUSH = 2063
TDMT_VND_TMQ_CONSUME_PUSH_RSP = 2064
TDMT_VND_TMQ_VG_WALINFO = 2065
TDMT_VND_TMQ_VG_WALINFO_RSP = 2066
TDMT_VND_TMQ_VG_COMMITTEDINFO = 2067
TDMT_VND_TMQ_VG_COMMITTEDINFO_RSP = 2068
TDMT_MND_ARB_HEARTBEAT_TIMER = 2305
TDMT_MND_ARB_HEARTBEAT_TIMER_RSP = 2306
TDMT_MND_ARB_CHECK_SYNC_TIMER = 2307
TDMT_MND_ARB_CHECK_SYNC_TIMER_RSP = 2308
TDMT_MND_ARB_UPDATE_GROUP = 2309
TDMT_MND_ARB_UPDATE_GROUP_RSP = 2310
TDMT_MND_ARB_UPDATE_GROUP_BATCH = 2311
TDMT_MND_ARB_UPDATE_GROUP_BATCH_RSP = 2312

279
source/common/test/tmsgTest.cpp
View File

@ -1,11 +1,280 @@
#include <iostream>
#include <iomanip>
#include <fstream>
#include <sstream>
#include <string>
#include <vector>
#include <unistd.h>
#include <limits.h>
#include <sys/stat.h>
#include <algorithm>
#include <unordered_map>
#include <gtest/gtest.h>
#include "tmsg.h"
TEST(td_msg_test, simple_msg_test) {
// std::cout << TMSG_INFO(TDMT_VND_DROP_TABLE) << std::endl;
// std::cout << TMSG_INFO(TDMT_MND_DROP_SUPER_TABLE) << std::endl;
// std::cout << TMSG_INFO(TDMT_MND_CREATE_SUPER_TABLE) << std::endl;
#undef TD_MSG_NUMBER_
#undef TD_MSG_DICT_
#undef TD_MSG_INFO_
#define TD_MSG_TYPE_INFO_
#undef TD_MSG_RANGE_CODE_
#undef TD_MSG_SEG_CODE_
#include "tmsgdef.h"
#undef getline
#undef close
using namespace std;
enum class ParseStatus {
Success,
FileNotExist,
FileNotOpen,
ResponseWithoutRequest,
RequestWithoutResponse
};
typedef struct {
string name;
string rspName;
int32_t type;
int32_t rspType;
} STestMsgTypeInfo;
string getExecutableDirectory() {
char result[PATH_MAX];
ssize_t count = readlink("/proc/self/exe", result, PATH_MAX);
if (count != -1) {
result[count] = '\0';
string path(result);
size_t pos = path.rfind('/');
if (pos != string::npos) {
path.erase(pos + 1);
}
return path;
} else {
throw std::runtime_error("Failed to get the executable's directory");
}
}
// parses key-value pairs from strings
pair<string, int32_t> parseKeyValuePair(const string &line, char delim = '=') {
size_t pos = line.find(delim);
if (pos == string::npos)
return make_pair("", 0);
string key = line.substr(0, pos);
// remove leading spaces
size_t firstNotSpace = key.find_first_not_of(" ");
if (firstNotSpace != string::npos) {
key = key.substr(firstNotSpace);
} else {
key.clear();
}
// remove ending spaces
size_t lastNotSpace = key.find_last_not_of(" ");
if (lastNotSpace != string::npos) {
key = key.substr(0, lastNotSpace + 1);
}
if (key.front() == '"' && key.back() == '"')
key = key.substr(1, key.size() - 2);
if (key.front() == '\'' && key.back() == '\'')
key = key.substr(1, key.size() - 2);
string valStr = line.substr(pos + 1);
int32_t val = stoi(valStr);
return make_pair(key, val);
}
// read the configuration file and parse it into the STestMsgTypeInfo array
ParseStatus readConfig(const string& filePath, vector<STestMsgTypeInfo>& msgTypes) {
ifstream file(filePath);
if (!file.is_open()) {
if (file.fail() && errno == ENOENT) {
cerr << "Error: The file does not exist, file: " << filePath << endl;
return ParseStatus::FileNotExist;
} else {
cerr << "Error: Could not open the file, file: " << filePath << endl;
return ParseStatus::FileNotOpen;
}
}
auto endsWith = [](const string& str, const string& suffix) {
if (str.length() < suffix.length()) {
return false;
}
return equal(str.end() - suffix.length(), str.end(), suffix.begin());
};
bool evenLine = true;
string line;
string suffix("_RSP");
pair<string, int32_t> reqKwInfo;
while (std::getline(file, line)) {
char delim = '#';
if (line.find('=') != string::npos) {
delim = '=';
} else if (line.find(':') != string::npos) {
delim = ':';
} else if (line.find('{') != string::npos || line.find('}') != string::npos) {
// TODO: parse json format
continue;
} else {
continue;
}
auto curKwInfo = parseKeyValuePair(line, delim);
evenLine = ! evenLine;
// check message type
if (evenLine == false) { // req msg
reqKwInfo = curKwInfo;
} else { // rsp msg
if (reqKwInfo.first.empty()) {
cerr << "Error: Found a response message without a matching request, rsp: " << curKwInfo.first << endl;
return ParseStatus::ResponseWithoutRequest;
} else if (!endsWith(curKwInfo.first, suffix)) {
cerr << "Error: A request message was not followed by a matching response, req: " << reqKwInfo.first << endl;
return ParseStatus::RequestWithoutResponse;
} else {
STestMsgTypeInfo msgInfo;
msgInfo.name = reqKwInfo.first;
msgInfo.rspName = curKwInfo.first;
msgInfo.type = reqKwInfo.second;
msgInfo.rspType = curKwInfo.second;
msgTypes.push_back(msgInfo);
// reset req info
reqKwInfo = make_pair("", -1);
}
}
}
if (!reqKwInfo.first.empty()) {
cerr << "Error: A request message was not followed by a matching response, req: " << reqKwInfo.first << endl;
return ParseStatus::RequestWithoutResponse;
}
return ParseStatus::Success;
}
TEST(td_msg_test, msg_type_compatibility_test) {
// cout << TMSG_INFO(TDMT_VND_DROP_TABLE) << endl;
// cout << TMSG_INFO(TDMT_MND_DROP_SUPER_TABLE) << endl;
// cout << TMSG_INFO(TDMT_MND_CREATE_SUPER_TABLE) << endl;
// int32_t msgSize = sizeof(tMsgTypeInfo) / sizeof(SMsgTypeInfo);
// for (int32_t i = 0; i < msgSize; ++i) {
// SMsgTypeInfo *pInfo = &tMsgTypeInfo[i];
// cout << i * 2 + 1 << " " << pInfo->name << " " << pInfo->type << endl;
// cout << i * 2 + 2 << " " << pInfo->rspName << " " << pInfo->rspType << endl;
// }
// current msgs: to map
unordered_map<string, const SMsgTypeInfo*> map;
for (const auto& info : tMsgTypeInfo) {
map[info.name] = &info;
}
string configFileName = "msgTypeTable.ini";
string execDir = getExecutableDirectory();
string configFilePath(execDir + configFileName);
vector<STestMsgTypeInfo> msgTypes;
ParseStatus status = readConfig(configFilePath, msgTypes);
switch (status) {
case ParseStatus::Success:
for (const auto& stdInfo : msgTypes) {
auto it = map.find(stdInfo.name);
if (it == map.end()) {
FAIL() << "Error: Could not find msg: " << stdInfo.name << ".";
} else {
auto newInfo = it->second;
ASSERT_STREQ(stdInfo.name.c_str(), newInfo->name);
ASSERT_STREQ(stdInfo.rspName.c_str(), newInfo->rspName);
ASSERT_EQ(stdInfo.type, newInfo->type)
<< "Message type mismatch(" << stdInfo.name << "): expected " << stdInfo.type << ", got " << newInfo->type << ".";
ASSERT_EQ(stdInfo.rspType, newInfo->rspType)
<< "Message response type mismatch(" << stdInfo.rspName << "): expected " << stdInfo.rspType << ", got " << newInfo->rspType << ".";
}
}
break;
case ParseStatus::FileNotExist:
FAIL() << "Error: The file does not exist, file: " << configFileName << ".";
break;
case ParseStatus::FileNotOpen:
FAIL() << "Error: Could not open the file, file: " << configFileName << ".";
break;
case ParseStatus::ResponseWithoutRequest:
FAIL() << "Error: Found a response message without a matching request.";
break;
case ParseStatus::RequestWithoutResponse:
FAIL() << "Error: A request message was not followed by a matching response.";
break;
default:
FAIL() << "Unknown Error.";
break;
}
}
size_t maxLengthOfMsgType() {
size_t maxLen = 0;
for (const auto& info : tMsgTypeInfo) {
maxLen = std::max(maxLen, strlen(info.name));
maxLen = std::max(maxLen, strlen(info.rspName));
}
return (maxLen / 4 + 1) * 4;
}
void generateConfigFile(const string& filePath) {
size_t maxStringLength = maxLengthOfMsgType();
std::ofstream file(filePath);
if (!file.is_open()) {
cerr << "Failed to open file for writing, at: " << filePath << "." << endl;
return;
}
for (const auto& info : tMsgTypeInfo) {
file << std::left << std::setw(maxStringLength) << info.name << "= " << info.type << endl;
file << std::left << std::setw(maxStringLength) << info.rspName << "= " << info.rspType << endl;
}
if (file.fail()) {
cerr << "An error occurred while writing to the file." << endl;
} else {
cout << "Data successfully written to file: " << filePath << endl;
}
file.close();
}
void processCommandArgs(int argc, char** argv) {
for (int i = 1; i < argc; ++i) {
if (string(argv[i]) == "--output-config") {
string configFile = (i + 1 < argc) ? argv[++i] : "./msgTypeTable.ini";
generateConfigFile(configFile);
exit(0);
}
}
}
int main(int argc, char **argv) {
processCommandArgs(argc, argv);
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

2
source/dnode/mgmt/exe/dmMain.c
View File

@ -261,7 +261,7 @@ static void dmPrintVersion() {
printf("%s\ntaosd version: %s compatible_version: %s\n", TD_PRODUCT_NAME, version, compatible_version);
printf("git: %s\n", gitinfo);
#ifdef TD_ENTERPRISE
printf("git: %s\n", gitinfoOfInternal);
printf("gitOfInternal: %s\n", gitinfoOfInternal);
#endif
printf("build: %s\n", buildinfo);
}

2
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -232,6 +232,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_PAUSE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_CREATE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_DROP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_CREATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
@ -240,6 +241,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_KILL_COMPACT_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

9
source/dnode/mgmt/mgmt_snode/src/smHandle.c
View File

@ -75,26 +75,27 @@ SArray *smGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_PAUSE, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
code = 0;
_OVER:

21
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -144,7 +144,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
}
#if defined(TD_ENTERPRISE)
pCfg->tsdbCfg.encryptAlgorithm = pCreate->encryptAlgorithm;
if(pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4){
if (pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4) {
strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
#else
@ -160,7 +160,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->walCfg.level = pCreate->walLevel;
#if defined(TD_ENTERPRISE)
pCfg->walCfg.encryptAlgorithm = pCreate->encryptAlgorithm;
if(pCfg->walCfg.encryptAlgorithm == DND_CA_SM4){
if (pCfg->walCfg.encryptAlgorithm == DND_CA_SM4) {
strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
#else
@ -169,7 +169,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
#if defined(TD_ENTERPRISE)
pCfg->tdbEncryptAlgorithm = pCreate->encryptAlgorithm;
if(pCfg->tdbEncryptAlgorithm == DND_CA_SM4){
if (pCfg->tdbEncryptAlgorithm == DND_CA_SM4) {
strncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
#else
@ -278,7 +278,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
req.keepTimeOffset, req.s3ChunkSize, req.s3KeepLocal, req.s3Compact, req.isTsma, req.precision, req.compression,
req.minRows, req.maxRows, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize,
req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix,
req.replica, req.selfIndex, req.learnerReplica, req.learnerSelfIndex, req.strict, req.changeVersion,
req.replica, req.selfIndex, req.learnerReplica, req.learnerSelfIndex, req.strict, req.changeVersion,
req.encryptAlgorithm);
for (int32_t i = 0; i < req.replica; ++i) {
@ -304,8 +304,8 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return -1;
}
if(req.encryptAlgorithm == DND_CA_SM4){
if(strlen(tsEncryptKey) == 0){
if (req.encryptAlgorithm == DND_CA_SM4) {
if (strlen(tsEncryptKey) == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
dError("vgId:%d, failed to create vnode since encrypt key is empty", req.vgId);
return -1;
@ -482,7 +482,9 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
.diskPrimary = pVnode->diskPrimary,
};
tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path));
vmCloseVnode(pMgmt, pVnode, false);
bool commitAndRemoveWal = vnodeShouldRemoveWal(pVnode->pImpl);
vmCloseVnode(pMgmt, pVnode, commitAndRemoveWal);
int32_t diskPrimary = wrapperCfg.diskPrimary;
char path[TSDB_FILENAME_LEN] = {0};
@ -737,7 +739,9 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
.diskPrimary = pVnode->diskPrimary,
};
tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path));
vmCloseVnode(pMgmt, pVnode, false);
bool commitAndRemoveWal = vnodeShouldRemoveWal(pVnode->pImpl);
vmCloseVnode(pMgmt, pVnode, commitAndRemoveWal);
int32_t diskPrimary = wrapperCfg.diskPrimary;
char path[TSDB_FILENAME_LEN] = {0};
@ -967,6 +971,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

4
source/dnode/mgmt/node_mgmt/src/dmEnv.c
View File

@ -110,10 +110,8 @@ static bool dmCheckDiskSpace() {
int32_t dmDiskInit() {
SDnode *pDnode = dmInstance();
SDiskCfg dCfg = {0};
SDiskCfg dCfg = {.level = 0, .primary = 1, .disable = 0};
tstrncpy(dCfg.dir, tsDataDir, TSDB_FILENAME_LEN);
dCfg.level = 0;
dCfg.primary = 1;
SDiskCfg *pDisks = tsDiskCfg;
int32_t numOfDisks = tsDiskCfgNum;
if (numOfDisks <= 0 || pDisks == NULL) {

8
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -58,8 +58,8 @@ static void dmConvertErrCode(tmsg_t msgType) {
if (terrno != TSDB_CODE_APP_IS_STOPPING) {
return;
}
if ((msgType > TDMT_VND_MSG && msgType < TDMT_VND_MAX_MSG) ||
(msgType > TDMT_SCH_MSG && msgType < TDMT_SCH_MAX_MSG)) {
if ((msgType > TDMT_VND_MSG_MIN && msgType < TDMT_VND_MSG_MAX) ||
(msgType > TDMT_SCH_MSG_MIN && msgType < TDMT_SCH_MSG_MAX)) {
terrno = TSDB_CODE_VND_STOPPED;
}
}
@ -279,7 +279,7 @@ int32_t dmInitMsgHandle(SDnode *pDnode) {
static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
SDnode *pDnode = dmInstance();
if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG) {
if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG_MIN) {
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
if (pDnode->status == DND_STAT_INIT) {
@ -296,7 +296,7 @@ static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
}
static inline int32_t dmSendSyncReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
SDnode *pDnode = dmInstance();
if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG) {
if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG_MIN) {
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
if (pDnode->status == DND_STAT_INIT) {

29
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -77,6 +77,7 @@ typedef enum {
MND_OPER_CREATE_VIEW,
MND_OPER_DROP_VIEW,
MND_OPER_CONFIG_CLUSTER,
MND_OPER_BALANCE_VGROUP_LEADER,
} EOperType;
typedef enum {
@ -102,8 +103,8 @@ typedef enum {
TRN_CONFLICT_GLOBAL = 1,
TRN_CONFLICT_DB = 2,
TRN_CONFLICT_DB_INSIDE = 3,
TRN_CONFLICT_TOPIC = 4,
TRN_CONFLICT_TOPIC_INSIDE = 5,
// TRN_CONFLICT_TOPIC = 4,
// TRN_CONFLICT_TOPIC_INSIDE = 5,
TRN_CONFLICT_ARBGROUP = 6,
} ETrnConflct;
@ -322,15 +323,21 @@ typedef struct {
} SAcctObj;
typedef struct {
char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN];
char acct[TSDB_USER_LEN];
int64_t createdTime;
int64_t updateTime;
int8_t superUser;
int8_t sysInfo;
int8_t enable;
int8_t reserve;
char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN];
char acct[TSDB_USER_LEN];
int64_t createdTime;
int64_t updateTime;
int8_t superUser;
int8_t sysInfo;
int8_t enable;
union {
uint8_t flag;
struct {
uint8_t createdb : 1;
uint8_t reserve : 7;
};
};
int32_t acctId;
int32_t authVersion;
int32_t passVersion;

8
source/dnode/mnode/impl/inc/mndShow.h
View File

@ -23,6 +23,14 @@
extern "C" {
#endif
#define COL_DATA_SET_VAL_RET(pData, isNull, pObj) \
do { \
if ((code = colDataSetVal(pColInfo, numOfRows, (pData), (isNull))) != 0) { \
if (pObj) sdbRelease(pSdb, (pObj)); \
return code; \
} \
} while (0)
int32_t mndInitShow(SMnode *pMnode);
void mndCleanupShow(SMnode *pMnode);
void mndAddShowRetrieveHandle(SMnode *pMnode, EShowType showType, ShowRetrieveFp fp);

41
source/dnode/mnode/impl/inc/mndStream.h
View File

@ -26,13 +26,14 @@ extern "C" {
#define MND_STREAM_RESERVE_SIZE 64
#define MND_STREAM_VER_NUMBER 5
#define MND_STREAM_CREATE_NAME "stream-create"
#define MND_STREAM_CHECKPOINT_NAME "stream-checkpoint"
#define MND_STREAM_PAUSE_NAME "stream-pause"
#define MND_STREAM_RESUME_NAME "stream-resume"
#define MND_STREAM_DROP_NAME "stream-drop"
#define MND_STREAM_TASK_RESET_NAME "stream-task-reset"
#define MND_STREAM_TASK_UPDATE_NAME "stream-task-update"
#define MND_STREAM_CREATE_NAME "stream-create"
#define MND_STREAM_CHECKPOINT_NAME "stream-checkpoint"
#define MND_STREAM_PAUSE_NAME "stream-pause"
#define MND_STREAM_RESUME_NAME "stream-resume"
#define MND_STREAM_DROP_NAME "stream-drop"
#define MND_STREAM_TASK_RESET_NAME "stream-task-reset"
#define MND_STREAM_TASK_UPDATE_NAME "stream-task-update"
#define MND_STREAM_CHKPT_UPDATE_NAME "stream-chkpt-update"
typedef struct SStreamTransInfo {
int64_t startTime;
@ -51,6 +52,7 @@ typedef struct SStreamTransMgmt {
} SStreamTransMgmt;
typedef struct SStreamExecInfo {
bool initTaskList;
SArray *pNodeList;
int64_t ts; // snapshot ts
SStreamTransMgmt transMgmt;
@ -58,6 +60,7 @@ typedef struct SStreamExecInfo {
SArray *pTaskList;
TdThreadMutex lock;
SHashObj *pTransferStateStreams;
SHashObj *pChkptStreams;
} SStreamExecInfo;
extern SStreamExecInfo execInfo;
@ -78,7 +81,18 @@ typedef struct SOrphanTask {
typedef struct {
SMsgHead head;
} SMStreamHbRspMsg, SMStreamReqCheckpointRspMsg;
} SMStreamHbRspMsg, SMStreamReqCheckpointRsp, SMStreamUpdateChkptRsp;
typedef struct STaskChkptInfo {
int32_t nodeId;
int32_t taskId;
int64_t streamId;
int64_t checkpointId;
int64_t version;
int64_t ts;
int32_t transId;
int8_t dropHTask;
}STaskChkptInfo;
int32_t mndInitStream(SMnode *pMnode);
void mndCleanupStream(SMnode *pMnode);
@ -96,7 +110,7 @@ int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream);
SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady);
void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName);
int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
int32_t retryCode);
int32_t retryCode, int32_t acceptCode);
STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, const char *pMsg);
int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status);
SSdbRaw *mndStreamActionEncode(SStreamObj *pStream);
@ -114,14 +128,19 @@ int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *p
int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList);
int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream);
int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList);
int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq);
void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo *pExecInfo);
SStreamTaskIter *createStreamTaskIter(SStreamObj *pStream);
void destroyStreamTaskIter(SStreamTaskIter *pIter);
bool streamTaskIterNextTask(SStreamTaskIter *pIter);
SStreamTask *streamTaskIterGetCurrent(SStreamTaskIter *pIter);
void mndInitExecInfo();
void removeExpiredNodeInfo(const SArray *pNodeSnapshot);
void removeTasksInBuf(SArray* pTaskIds, SStreamExecInfo* pExecInfo);
void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo);
int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot);
void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
#ifdef __cplusplus
}

3
source/dnode/mnode/impl/inc/mndUser.h
View File

@ -38,7 +38,7 @@ SHashObj *mndDupDbHash(SHashObj *pOld);
SHashObj *mndDupTableHash(SHashObj *pOld);
SHashObj *mndDupTopicHash(SHashObj *pOld);
int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp,
int32_t *pRspLen);
int32_t *pRspLen, int64_t ipWhiteListVer);
int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db);
int32_t mndUserRemoveStb(SMnode *pMnode, STrans *pTrans, char *stb);
int32_t mndUserRemoveView(SMnode *pMnode, STrans *pTrans, char *view);
@ -53,6 +53,7 @@ void mndUpdateIpWhiteForAllUser(SMnode *pMnode, char *user, char *fqdn, int8_t t
int32_t mndRefreshUserIpWhiteList(SMnode *pMnode);
int64_t mndGetUserIpWhiteListVer(SMnode *pMnode, SUserObj *pUser);
#ifdef __cplusplus
}
#endif

13
source/dnode/mnode/impl/src/mndCluster.c
View File

@ -280,6 +280,7 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) {
static int32_t mndRetrieveClusters(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pMsg->info.node;
SSdb *pSdb = pMnode->pSdb;
int32_t code = 0;
int32_t numOfRows = 0;
int32_t cols = 0;
SClusterObj *pCluster = NULL;
@ -290,31 +291,31 @@ static int32_t mndRetrieveClusters(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock *
cols = 0;
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCluster->id, false);
COL_DATA_SET_VAL_RET((const char *)&pCluster->id, false, pCluster);
char buf[tListLen(pCluster->name) + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(buf, pCluster->name, pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, buf, false);
COL_DATA_SET_VAL_RET(buf, false, pCluster);
int32_t upTime = mndGetClusterUpTimeImp(pCluster);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&upTime, false);
COL_DATA_SET_VAL_RET((const char *)&upTime, false, pCluster);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCluster->createdTime, false);
COL_DATA_SET_VAL_RET((const char *)&pCluster->createdTime, false, pCluster);
char ver[12] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(ver, tsVersionName, pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)ver, false);
COL_DATA_SET_VAL_RET((const char *)ver, false, pCluster);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
if (tsExpireTime <= 0) {
colDataSetNULL(pColInfo, numOfRows);
} else {
colDataSetVal(pColInfo, numOfRows, (const char *)&tsExpireTime, false);
COL_DATA_SET_VAL_RET((const char *)&tsExpireTime, false, pCluster);
}
sdbRelease(pSdb, pCluster);

246
source/dnode/mnode/impl/src/mndCompact.c
View File

@ -13,20 +13,19 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "mndCompact.h"
#include "mndTrans.h"
#include "mndShow.h"
#include "mndDb.h"
#include "mndCompactDetail.h"
#include "mndVgroup.h"
#include "tmsgcb.h"
#include "mndDnode.h"
#include "tmisce.h"
#include "audit.h"
#include "mndCompactDetail.h"
#include "mndDb.h"
#include "mndDnode.h"
#include "mndPrivilege.h"
#include "mndShow.h"
#include "mndTrans.h"
#include "mndVgroup.h"
#include "tmisce.h"
#include "tmsgcb.h"
#define MND_COMPACT_VER_NUMBER 1
#define MND_COMPACT_ID_LEN 11
#define MND_COMPACT_ID_LEN 11
static int32_t mndProcessCompactTimer(SRpcMsg *pReq);
@ -50,12 +49,9 @@ int32_t mndInitCompact(SMnode *pMnode) {
return sdbSetTable(pMnode->pSdb, table);
}
void mndCleanupCompact(SMnode *pMnode) {
mDebug("mnd compact cleanup");
}
void mndCleanupCompact(SMnode *pMnode) { mDebug("mnd compact cleanup"); }
void tFreeCompactObj(SCompactObj *pCompact) {
}
void tFreeCompactObj(SCompactObj *pCompact) {}
int32_t tSerializeSCompactObj(void *buf, int32_t bufLen, const SCompactObj *pObj) {
SEncoder encoder = {0};
@ -75,7 +71,7 @@ int32_t tSerializeSCompactObj(void *buf, int32_t bufLen, const SCompactObj *pObj
}
int32_t tDeserializeSCompactObj(void *buf, int32_t bufLen, SCompactObj *pObj) {
int8_t ex = 0;
int8_t ex = 0;
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
@ -94,7 +90,7 @@ int32_t tDeserializeSCompactObj(void *buf, int32_t bufLen, SCompactObj *pObj) {
SSdbRaw *mndCompactActionEncode(SCompactObj *pCompact) {
terrno = TSDB_CODE_SUCCESS;
void *buf = NULL;
void *buf = NULL;
SSdbRaw *pRaw = NULL;
int32_t tlen = tSerializeSCompactObj(NULL, 0, pCompact);
@ -102,8 +98,8 @@ SSdbRaw *mndCompactActionEncode(SCompactObj *pCompact) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
int32_t size = sizeof(int32_t) + tlen;
int32_t size = sizeof(int32_t) + tlen;
pRaw = sdbAllocRaw(SDB_COMPACT, MND_COMPACT_VER_NUMBER, size);
if (pRaw == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -127,7 +123,6 @@ SSdbRaw *mndCompactActionEncode(SCompactObj *pCompact) {
SDB_SET_BINARY(pRaw, dataPos, buf, tlen, OVER);
SDB_SET_DATALEN(pRaw, dataPos, OVER);
OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
@ -141,9 +136,9 @@ OVER:
}
SSdbRow *mndCompactActionDecode(SSdbRaw *pRaw) {
SSdbRow *pRow = NULL;
SCompactObj *pCompact = NULL;
void *buf = NULL;
SSdbRow *pRow = NULL;
SCompactObj *pCompact = NULL;
void *buf = NULL;
terrno = TSDB_CODE_SUCCESS;
int8_t sver = 0;
@ -184,7 +179,7 @@ SSdbRow *mndCompactActionDecode(SSdbRaw *pRaw) {
goto OVER;
}
//taosInitRWLatch(&pView->lock);
// taosInitRWLatch(&pView->lock);
OVER:
taosMemoryFreeClear(buf);
@ -210,15 +205,15 @@ int32_t mndCompactActionDelete(SSdb *pSdb, SCompactObj *pCompact) {
}
int32_t mndCompactActionUpdate(SSdb *pSdb, SCompactObj *pOldCompact, SCompactObj *pNewCompact) {
mTrace("compact:%" PRId32 ", perform update action, old row:%p new row:%p",
pOldCompact->compactId, pOldCompact, pNewCompact);
mTrace("compact:%" PRId32 ", perform update action, old row:%p new row:%p", pOldCompact->compactId, pOldCompact,
pNewCompact);
return 0;
}
SCompactObj *mndAcquireCompact(SMnode *pMnode, int64_t compactId) {
SSdb *pSdb = pMnode->pSdb;
SCompactObj *pCompact = sdbAcquire(pSdb, SDB_COMPACT, &compactId);
SSdb *pSdb = pMnode->pSdb;
SCompactObj *pCompact = sdbAcquire(pSdb, SDB_COMPACT, &compactId);
if (pCompact == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
terrno = TSDB_CODE_SUCCESS;
}
@ -230,8 +225,8 @@ void mndReleaseCompact(SMnode *pMnode, SCompactObj *pCompact) {
sdbRelease(pSdb, pCompact);
}
//compact db
int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompact, SDbObj *pDb, SCompactDbRsp *rsp){
// compact db
int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj *pCompact, SDbObj *pDb, SCompactDbRsp *rsp) {
pCompact->compactId = tGenIdPI32();
strcpy(pCompact->dbname, pDb->name);
@ -251,18 +246,19 @@ int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompac
return 0;
}
//retrieve compact
int32_t mndRetrieveCompact(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows){
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
SCompactObj *pCompact = NULL;
char *sep = NULL;
SDbObj *pDb = NULL;
// retrieve compact
int32_t mndRetrieveCompact(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
SCompactObj *pCompact = NULL;
char *sep = NULL;
SDbObj *pDb = NULL;
if (strlen(pShow->db) > 0) {
sep = strchr(pShow->db, '.');
if (sep && ((0 == strcmp(sep + 1, TSDB_INFORMATION_SCHEMA_DB) || (0 == strcmp(sep + 1, TSDB_PERFORMANCE_SCHEMA_DB))))) {
if (sep &&
((0 == strcmp(sep + 1, TSDB_INFORMATION_SCHEMA_DB) || (0 == strcmp(sep + 1, TSDB_PERFORMANCE_SCHEMA_DB))))) {
sep++;
} else {
pDb = mndAcquireDb(pMnode, pShow->db);
@ -306,9 +302,9 @@ int32_t mndRetrieveCompact(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock,
return numOfRows;
}
//kill compact
static void *mndBuildKillCompactReq(SMnode *pMnode, SVgObj *pVgroup, int32_t *pContLen,
int32_t compactId, int32_t dnodeid) {
// kill compact
static void *mndBuildKillCompactReq(SMnode *pMnode, SVgObj *pVgroup, int32_t *pContLen, int32_t compactId,
int32_t dnodeid) {
SVKillCompactReq req = {0};
req.compactId = compactId;
req.vgId = pVgroup->vgId;
@ -337,8 +333,8 @@ static void *mndBuildKillCompactReq(SMnode *pMnode, SVgObj *pVgroup, int32_t *pC
return pReq;
}
static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup,
int32_t compactId, int32_t dnodeid) {
static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, int32_t compactId,
int32_t dnodeid) {
STransAction action = {0};
SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeid);
@ -365,7 +361,7 @@ static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *p
static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompact) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq, "kill-compact");
if (pTrans == NULL) {
mError("compact:%" PRId32 ", failed to drop since %s" , pCompact->compactId, terrstr());
mError("compact:%" PRId32 ", failed to drop since %s", pCompact->compactId, terrstr());
return -1;
}
mInfo("trans:%d, used to kill compact:%" PRId32, pTrans->id, pCompact->compactId);
@ -388,13 +384,13 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa
if (pDetail->compactId == pCompact->compactId) {
SVgObj *pVgroup = mndAcquireVgroup(pMnode, pDetail->vgId);
if(pVgroup == NULL){
if (pVgroup == NULL) {
mError("trans:%d, failed to append redo action since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
if(mndAddKillCompactAction(pMnode, pTrans, pVgroup, pCompact->compactId, pDetail->dnodeId) != 0){
if (mndAddKillCompactAction(pMnode, pTrans, pVgroup, pCompact->compactId, pDetail->dnodeId) != 0) {
mError("trans:%d, failed to append redo action since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
@ -426,7 +422,7 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa
return 0;
}
int32_t mndProcessKillCompactReq(SRpcMsg *pReq){
int32_t mndProcessKillCompactReq(SRpcMsg *pReq) {
SKillCompactReq killCompactReq = {0};
if (tDeserializeSKillCompactReq(pReq->pCont, pReq->contLen, &killCompactReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
@ -435,10 +431,10 @@ int32_t mndProcessKillCompactReq(SRpcMsg *pReq){
mInfo("start to kill compact:%" PRId32, killCompactReq.compactId);
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SCompactObj *pCompact = mndAcquireCompact(pMnode, killCompactReq.compactId);
if(pCompact == NULL){
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SCompactObj *pCompact = mndAcquireCompact(pMnode, killCompactReq.compactId);
if (pCompact == NULL) {
terrno = TSDB_CODE_MND_INVALID_COMPACT_ID;
tFreeSKillCompactReq(&killCompactReq);
return -1;
@ -470,9 +466,10 @@ _OVER:
return code;
}
//update progress
static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t compactId, SQueryCompactProgressRsp* rsp) {
void* pIter = NULL;
// update progress
static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t compactId,
SQueryCompactProgressRsp *rsp) {
void *pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
@ -493,36 +490,36 @@ static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t c
return TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST;
}
int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq){
int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq) {
SQueryCompactProgressRsp req = {0};
int32_t code = 0;
int32_t code = 0;
code = tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req);
if (code != 0) {
terrno = TSDB_CODE_INVALID_MSG;
mError("failed to deserialize vnode-query-compact-progress-rsp, ret:%d, pCont:%p, len:%d",
code, pReq->pCont, pReq->contLen);
mError("failed to deserialize vnode-query-compact-progress-rsp, ret:%d, pCont:%p, len:%d", code, pReq->pCont,
pReq->contLen);
return -1;
}
mDebug("compact:%d, receive query response, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d",
req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished);
mDebug("compact:%d, receive query response, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d", req.compactId,
req.vgId, req.dnodeId, req.numberFileset, req.finished);
SMnode *pMnode = pReq->info.node;
SMnode *pMnode = pReq->info.node;
code = mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req);
if(code != 0){
if (code != 0) {
terrno = code;
mError("compact:%d, failed to update progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d",
req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished);
mError("compact:%d, failed to update progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d", req.compactId,
req.vgId, req.dnodeId, req.numberFileset, req.finished);
return -1;
}
return 0;
}
//timer
void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact){
void *pIter = NULL;
// timer
void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) {
void *pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
@ -532,8 +529,8 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact){
if (pDetail->compactId == pCompact->compactId) {
SEpSet epSet = {0};
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pDetail->dnodeId);
if(pDnode == NULL) break;
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pDetail->dnodeId);
if (pDnode == NULL) break;
addEpIntoEpSet(&epSet, pDnode->fqdn, pDnode->port);
mndReleaseDnode(pMnode, pDnode);
@ -545,6 +542,8 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact){
int32_t contLen = tSerializeSQueryCompactProgressReq(NULL, 0, &req);
if (contLen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
sdbCancelFetch(pMnode->pSdb, pDetail);
sdbRelease(pMnode->pSdb, pDetail);
continue;
}
@ -555,31 +554,29 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact){
sdbCancelFetch(pMnode->pSdb, pDetail);
sdbRelease(pMnode->pSdb, pDetail);
continue;
}
}
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pDetail->vgId);
tSerializeSQueryCompactProgressReq((char *)pHead + sizeof(SMsgHead), contLen - sizeof(SMsgHead), &req);
SRpcMsg rpcMsg = {.msgType = TDMT_VND_QUERY_COMPACT_PROGRESS,
.contLen = contLen};
//rpcMsg.pCont = rpcMallocCont(contLen);
//if (rpcMsg.pCont == NULL) {
// return;
//}
SRpcMsg rpcMsg = {.msgType = TDMT_VND_QUERY_COMPACT_PROGRESS, .contLen = contLen};
//memcpy(rpcMsg.pCont, pHead, contLen);
// rpcMsg.pCont = rpcMallocCont(contLen);
// if (rpcMsg.pCont == NULL) {
// return;
// }
// memcpy(rpcMsg.pCont, pHead, contLen);
rpcMsg.pCont = pHead;
char detail[1024] = {0};
int32_t len = snprintf(detail, sizeof(detail), "msgType:%s numOfEps:%d inUse:%d", TMSG_INFO(TDMT_VND_QUERY_COMPACT_PROGRESS),
epSet.numOfEps, epSet.inUse);
int32_t len = snprintf(detail, sizeof(detail), "msgType:%s numOfEps:%d inUse:%d",
TMSG_INFO(TDMT_VND_QUERY_COMPACT_PROGRESS), epSet.numOfEps, epSet.inUse);
for (int32_t i = 0; i < epSet.numOfEps; ++i) {
len += snprintf(detail + len, sizeof(detail) - len, " ep:%d-%s:%u", i, epSet.eps[i].fqdn,
epSet.eps[i].port);
len += snprintf(detail + len, sizeof(detail) - len, " ep:%d-%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);
}
mDebug("compact:%d, send update progress msg to %s", pDetail->compactId, detail);
@ -592,40 +589,51 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact){
}
static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
bool needSave = false;
void* pIter = NULL;
bool needSave = false;
void *pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == compactId) {
mDebug("compact:%d, check save progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d, "
"newNumberFileset:%d, newFinished:%d",
pDetail->compactId, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished,
pDetail->newNumberFileset, pDetail->newFinished);
mDebug(
"compact:%d, check save progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d, "
"newNumberFileset:%d, newFinished:%d",
pDetail->compactId, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished,
pDetail->newNumberFileset, pDetail->newFinished);
//these 2 number will jump back after dnode restart, so < is not used here
if(pDetail->numberFileset != pDetail->newNumberFileset || pDetail->finished != pDetail->newFinished)
// these 2 number will jump back after dnode restart, so < is not used here
if (pDetail->numberFileset != pDetail->newNumberFileset || pDetail->finished != pDetail->newFinished)
needSave = true;
}
sdbRelease(pMnode->pSdb, pDetail);
}
if(!needSave) {
mDebug("compact:%" PRId32 ", no need to save" , compactId);
SCompactObj *pCompact = mndAcquireCompact(pMnode, compactId);
if (pCompact == NULL) return 0;
SDbObj *pDb = mndAcquireDb(pMnode, pCompact->dbname);
if (pDb == NULL) {
needSave = true;
mWarn("compact:%" PRId32 ", no db exist, set needSave:%s", compactId, pCompact->dbname);
} else {
mndReleaseDb(pMnode, pDb);
pDb = NULL;
}
if (!needSave) {
mDebug("compact:%" PRId32 ", no need to save", compactId);
return 0;
}
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, NULL, "update-compact-progress");
if (pTrans == NULL) {
mError("trans:%" PRId32 ", failed to create since %s" , pTrans->id, terrstr());
mError("trans:%" PRId32 ", failed to create since %s", pTrans->id, terrstr());
return -1;
}
mInfo("compact:%d, trans:%d, used to update compact progress.", compactId, pTrans->id);
SCompactObj *pCompact = mndAcquireCompact(pMnode, compactId);
mndTransSetDbName(pTrans, pCompact->dbname, NULL);
@ -636,14 +644,15 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
if (pIter == NULL) break;
if (pDetail->compactId == compactId) {
mInfo("compact:%d, trans:%d, check compact progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d, "
"newNumberFileset:%d, newFinished:%d",
pDetail->compactId, pTrans->id, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished,
pDetail->newNumberFileset, pDetail->newFinished);
mInfo(
"compact:%d, trans:%d, check compact progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d, "
"newNumberFileset:%d, newFinished:%d",
pDetail->compactId, pTrans->id, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished,
pDetail->newNumberFileset, pDetail->newFinished);
pDetail->numberFileset = pDetail->newNumberFileset;
pDetail->finished = pDetail->newFinished;
SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id, terrstr());
@ -657,22 +666,22 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
}
bool allFinished = true;
pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if(pDetail->compactId == compactId){
mInfo("compact:%d, trans:%d, check compact finished, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d",
pDetail->compactId, pTrans->id, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished);
if (pDetail->compactId == compactId) {
mInfo("compact:%d, trans:%d, check compact finished, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d",
pDetail->compactId, pTrans->id, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished);
if(pDetail->numberFileset == -1 && pDetail->finished == -1){
if (pDetail->numberFileset == -1 && pDetail->finished == -1) {
allFinished = false;
sdbRelease(pMnode->pSdb, pDetail);
break;
}
if (pDetail->numberFileset != -1 && pDetail->finished != -1 &&
pDetail->numberFileset != pDetail->finished) {
if (pDetail->numberFileset != -1 && pDetail->finished != -1 && pDetail->numberFileset != pDetail->finished) {
allFinished = false;
sdbRelease(pMnode->pSdb, pDetail);
break;
@ -682,21 +691,33 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
sdbRelease(pMnode->pSdb, pDetail);
}
if(allFinished){
pDb = mndAcquireDb(pMnode, pCompact->dbname);
if (pDb == NULL) {
allFinished = true;
mWarn("compact:%" PRId32 ", no db exist, set all finished:%s", compactId, pCompact->dbname);
} else {
mndReleaseDb(pMnode, pDb);
pDb = NULL;
}
if (allFinished) {
mInfo("compact:%d, all finished", pCompact->compactId);
pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == pCompact->compactId) {
if (pDetail->compactId == pCompact->compactId) {
SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id, terrstr());
mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id,
terrstr());
mndTransDrop(pTrans);
return -1;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
mInfo("compact:%d, add drop compactdetail action", pDetail->compactDetailId);
}
sdbRelease(pMnode->pSdb, pDetail);
@ -709,6 +730,7 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
return -1;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
mInfo("compact:%d, add drop compact action", pCompact->compactId);
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
@ -739,8 +761,8 @@ void mndCompactPullup(SMnode *pMnode) {
for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
mInfo("begin to pull up");
int32_t *pCompactId = taosArrayGet(pArray, i);
SCompactObj *pCompact = mndAcquireCompact(pMnode, *pCompactId);
int32_t *pCompactId = taosArrayGet(pArray, i);
SCompactObj *pCompact = mndAcquireCompact(pMnode, *pCompactId);
if (pCompact != NULL) {
mInfo("compact:%d, begin to pull up", pCompact->compactId);
mndCompactSendProgressReq(pMnode, pCompact);

15
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -91,7 +91,7 @@ void mndSendConsumerMsg(SMnode *pMnode, int64_t consumerId, uint16_t msgType, SR
}
}
static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *pMnode, const char *pUser,
static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const char *pUser,
bool enableReplay) {
SMqTopicObj *pTopic = NULL;
int32_t code = 0;
@ -135,11 +135,6 @@ static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *
}
}
mndTransSetDbName(pTrans, pOneTopic, NULL);
if (mndTransCheckConflict(pMnode, pTrans) != 0) {
code = -1;
goto FAILED;
}
mndReleaseTopic(pMnode, pTopic);
}
@ -177,12 +172,12 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
goto END;
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_TOPIC, pMsg, "recover-csm");
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "recover-csm");
if (pTrans == NULL) {
code = -1;
goto END;
}
code = validateTopics(pTrans, pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false);
code = validateTopics(pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false);
if (code != 0) {
goto END;
}
@ -675,13 +670,13 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
goto _over;
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_TOPIC_INSIDE, pMsg, "subscribe");
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe");
if (pTrans == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _over;
}
code = validateTopics(pTrans, subscribe.topicNames, pMnode, pMsg->info.conn.user, subscribe.enableReplay);
code = validateTopics(subscribe.topicNames, pMnode, pMsg->info.conn.user, subscribe.enableReplay);
if (code != TSDB_CODE_SUCCESS) {
goto _over;
}

32
source/dnode/mnode/impl/src/mndDb.c
View File

@ -495,6 +495,16 @@ static int32_t mndCheckInChangeDbCfg(SMnode *pMnode, SDbCfg *pOldCfg, SDbCfg *pN
#else
if (pNewCfg->replications != 1 && pNewCfg->replications != 3) return -1;
#endif
if (pNewCfg->walLevel == 0 && pOldCfg->replications > 1) {
terrno = TSDB_CODE_MND_INVALID_WAL_LEVEL;
return -1;
}
if (pNewCfg->replications > 1 && pOldCfg->walLevel == 0) {
terrno = TSDB_CODE_MND_INVALID_WAL_LEVEL;
return -1;
}
if (pNewCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pNewCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1;
if (pNewCfg->minRows < TSDB_MIN_MINROWS_FBLOCK || pNewCfg->minRows > TSDB_MAX_MINROWS_FBLOCK) return -1;
if (pNewCfg->maxRows < TSDB_MIN_MAXROWS_FBLOCK || pNewCfg->maxRows > TSDB_MAX_MAXROWS_FBLOCK) return -1;
@ -874,9 +884,6 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
}
#endif
mInfo("db:%s, start to create, vgroups:%d", createReq.db, createReq.numOfVgroups);
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DB, NULL) != 0) {
goto _OVER;
}
pDb = mndAcquireDb(pMnode, createReq.db);
if (pDb != NULL) {
@ -901,6 +908,10 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
}
}
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DB, NULL) != 0) {
goto _OVER;
}
if ((terrno = grantCheck(TSDB_GRANT_DB)) != 0) {
code = terrno;
goto _OVER;
@ -1114,11 +1125,22 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *
if (pNewDb->cfg.withArbitrator) {
SArbGroup arbGroup = {0};
mndArbGroupInitFromVgObj(&newVgroup, &arbGroup);
if (mndSetCreateArbGroupCommitLogs(pTrans, &arbGroup) != 0) return -1;
if (mndSetCreateArbGroupCommitLogs(pTrans, &arbGroup) != 0) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup);
taosArrayDestroy(pArray);
return -1;
}
} else {
SArbGroup arbGroup = {0};
mndArbGroupInitFromVgObj(pVgroup, &arbGroup);
if (mndSetDropArbGroupCommitLogs(pTrans, &arbGroup) != 0) return -1;
if (mndSetDropArbGroupCommitLogs(pTrans, &arbGroup) != 0) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup);
taosArrayDestroy(pArray);
return -1;
}
}
}
}

4
source/dnode/mnode/impl/src/mndDef.c
View File

@ -537,7 +537,7 @@ SMqSubscribeObj *tNewSubscribeObj(const char *key) {
memcpy(pSubObj->key, key, TSDB_SUBSCRIBE_KEY_LEN);
taosInitRWLatch(&pSubObj->lock);
pSubObj->vgNum = 0;
pSubObj->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
pSubObj->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
// TODO set hash free fp
/*taosHashSetFreeFp(pSubObj->consumerHash, tDeleteSMqConsumerEp);*/
@ -557,7 +557,7 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) {
pSubNew->withMeta = pSub->withMeta;
pSubNew->vgNum = pSub->vgNum;
pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
// TODO set hash free fp
/*taosHashSetFreeFp(pSubNew->consumerHash, tDeleteSMqConsumerEp);*/
void *pIter = NULL;

231
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -14,9 +14,8 @@
*/
#define _DEFAULT_SOURCE
#include <stdio.h>
#include "tjson.h"
#include "mndDnode.h"
#include <stdio.h>
#include "audit.h"
#include "mndCluster.h"
#include "mndDb.h"
@ -28,9 +27,10 @@
#include "mndTrans.h"
#include "mndUser.h"
#include "mndVgroup.h"
#include "taos_monitor.h"
#include "tjson.h"
#include "tmisce.h"
#include "tunit.h"
#include "taos_monitor.h"
#define TSDB_DNODE_VER_NUMBER 2
#define TSDB_DNODE_RESERVE_SIZE 40
@ -191,8 +191,8 @@ static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) {
SDB_SET_BINARY(pRaw, dataPos, pDnode->fqdn, TSDB_FQDN_LEN, _OVER)
SDB_SET_BINARY(pRaw, dataPos, pDnode->machineId, TSDB_MACHINE_ID_LEN, _OVER)
SDB_SET_RESERVE(pRaw, dataPos, TSDB_DNODE_RESERVE_SIZE, _OVER)
SDB_SET_INT16(pRaw, dataPos, 0, _OVER) // forward/backward compatible
SDB_SET_INT16(pRaw, dataPos, 0, _OVER) // forward/backward compatible
SDB_SET_INT16(pRaw, dataPos, 0, _OVER) // forward/backward compatible
SDB_SET_INT16(pRaw, dataPos, 0, _OVER) // forward/backward compatible
SDB_SET_DATALEN(pRaw, dataPos, _OVER);
terrno = 0;
@ -536,49 +536,49 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
int32_t code = -1;
char strClusterId[TSDB_CLUSTER_ID_LEN] = {0};
sprintf(strClusterId, "%"PRId64, pMnode->clusterId);
sprintf(strClusterId, "%" PRId64, pMnode->clusterId);
if (tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return code;
}
if(tsMonitorLogProtocol){
if (tsMonitorLogProtocol) {
mInfo("process statis req,\n %s", statisReq.pCont);
}
SJson* pJson = tjsonParse(statisReq.pCont);
SJson *pJson = tjsonParse(statisReq.pCont);
int32_t ts_size = tjsonGetArraySize(pJson);
for(int32_t i = 0; i < ts_size; i++){
SJson* item = tjsonGetArrayItem(pJson, i);
for (int32_t i = 0; i < ts_size; i++) {
SJson *item = tjsonGetArrayItem(pJson, i);
SJson* tables = tjsonGetObjectItem(item, "tables");
SJson *tables = tjsonGetObjectItem(item, "tables");
int32_t tableSize = tjsonGetArraySize(tables);
for(int32_t i = 0; i < tableSize; i++){
SJson* table = tjsonGetArrayItem(tables, i);
for (int32_t i = 0; i < tableSize; i++) {
SJson *table = tjsonGetArrayItem(tables, i);
char tableName[MONITOR_TABLENAME_LEN] = {0};
tjsonGetStringValue(table, "name", tableName);
SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups");
SJson *metricGroups = tjsonGetObjectItem(table, "metric_groups");
int32_t size = tjsonGetArraySize(metricGroups);
for(int32_t i = 0; i < size; i++){
SJson* item = tjsonGetArrayItem(metricGroups, i);
for (int32_t i = 0; i < size; i++) {
SJson *item = tjsonGetArrayItem(metricGroups, i);
SJson* arrayTag = tjsonGetObjectItem(item, "tags");
SJson *arrayTag = tjsonGetObjectItem(item, "tags");
int32_t tagSize = tjsonGetArraySize(arrayTag);
for(int32_t j = 0; j < tagSize; j++){
SJson* item = tjsonGetArrayItem(arrayTag, j);
for (int32_t j = 0; j < tagSize; j++) {
SJson *item = tjsonGetArrayItem(arrayTag, j);
char tagName[MONITOR_TAG_NAME_LEN] = {0};
tjsonGetStringValue(item, "name", tagName);
if(strncmp(tagName, "cluster_id", MONITOR_TAG_NAME_LEN) == 0) {
if (strncmp(tagName, "cluster_id", MONITOR_TAG_NAME_LEN) == 0) {
tjsonDeleteItemFromObject(item, "value");
tjsonAddStringToObject(item, "value", strClusterId);
}
@ -590,12 +590,12 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
char *pCont = tjsonToString(pJson);
monSendContent(pCont);
if(pJson != NULL){
if (pJson != NULL) {
tjsonDelete(pJson);
pJson = NULL;
}
if(pCont != NULL){
if (pCont != NULL) {
taosMemoryFree(pCont);
pCont = NULL;
}
@ -603,132 +603,132 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
tFreeSStatisReq(&statisReq);
return 0;
/*
SJson* pJson = tjsonParse(statisReq.pCont);
/*
SJson* pJson = tjsonParse(statisReq.pCont);
int32_t ts_size = tjsonGetArraySize(pJson);
int32_t ts_size = tjsonGetArraySize(pJson);
for(int32_t i = 0; i < ts_size; i++){
SJson* item = tjsonGetArrayItem(pJson, i);
for(int32_t i = 0; i < ts_size; i++){
SJson* item = tjsonGetArrayItem(pJson, i);
SJson* tables = tjsonGetObjectItem(item, "tables");
SJson* tables = tjsonGetObjectItem(item, "tables");
int32_t tableSize = tjsonGetArraySize(tables);
for(int32_t i = 0; i < tableSize; i++){
SJson* table = tjsonGetArrayItem(tables, i);
int32_t tableSize = tjsonGetArraySize(tables);
for(int32_t i = 0; i < tableSize; i++){
SJson* table = tjsonGetArrayItem(tables, i);
char tableName[MONITOR_TABLENAME_LEN] = {0};
tjsonGetStringValue(table, "name", tableName);
char tableName[MONITOR_TABLENAME_LEN] = {0};
tjsonGetStringValue(table, "name", tableName);
SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups");
SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups");
int32_t size = tjsonGetArraySize(metricGroups);
for(int32_t i = 0; i < size; i++){
SJson* item = tjsonGetArrayItem(metricGroups, i);
int32_t size = tjsonGetArraySize(metricGroups);
for(int32_t i = 0; i < size; i++){
SJson* item = tjsonGetArrayItem(metricGroups, i);
SJson* arrayTag = tjsonGetObjectItem(item, "tags");
SJson* arrayTag = tjsonGetObjectItem(item, "tags");
int32_t tagSize = tjsonGetArraySize(arrayTag);
int32_t tagSize = tjsonGetArraySize(arrayTag);
char** labels = taosMemoryMalloc(sizeof(char*) * tagSize);
char** sample_labels = taosMemoryMalloc(sizeof(char*) * tagSize);
char** labels = taosMemoryMalloc(sizeof(char*) * tagSize);
char** sample_labels = taosMemoryMalloc(sizeof(char*) * tagSize);
for(int32_t j = 0; j < tagSize; j++){
SJson* item = tjsonGetArrayItem(arrayTag, j);
for(int32_t j = 0; j < tagSize; j++){
SJson* item = tjsonGetArrayItem(arrayTag, j);
*(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN);
tjsonGetStringValue(item, "name", *(labels + j));
*(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN);
tjsonGetStringValue(item, "name", *(labels + j));
*(sample_labels + j) = taosMemoryMalloc(MONITOR_TAG_VALUE_LEN);
tjsonGetStringValue(item, "value", *(sample_labels + j));
if(strncmp(*(labels + j), "cluster_id", MONITOR_TAG_NAME_LEN) == 0) {
strncpy(*(sample_labels + j), strClusterId, MONITOR_TAG_VALUE_LEN);
*(sample_labels + j) = taosMemoryMalloc(MONITOR_TAG_VALUE_LEN);
tjsonGetStringValue(item, "value", *(sample_labels + j));
if(strncmp(*(labels + j), "cluster_id", MONITOR_TAG_NAME_LEN) == 0) {
strncpy(*(sample_labels + j), strClusterId, MONITOR_TAG_VALUE_LEN);
}
}
}
SJson* metrics = tjsonGetObjectItem(item, "metrics");
SJson* metrics = tjsonGetObjectItem(item, "metrics");
int32_t metricLen = tjsonGetArraySize(metrics);
for(int32_t j = 0; j < metricLen; j++){
SJson *item = tjsonGetArrayItem(metrics, j);
int32_t metricLen = tjsonGetArraySize(metrics);
for(int32_t j = 0; j < metricLen; j++){
SJson *item = tjsonGetArrayItem(metrics, j);
char name[MONITOR_METRIC_NAME_LEN] = {0};
tjsonGetStringValue(item, "name", name);
char name[MONITOR_METRIC_NAME_LEN] = {0};
tjsonGetStringValue(item, "name", name);
double value = 0;
tjsonGetDoubleValue(item, "value", &value);
double value = 0;
tjsonGetDoubleValue(item, "value", &value);
double type = 0;
tjsonGetDoubleValue(item, "type", &type);
double type = 0;
tjsonGetDoubleValue(item, "type", &type);
int32_t metricNameLen = strlen(name) + strlen(tableName) + 2;
char* metricName = taosMemoryMalloc(metricNameLen);
memset(metricName, 0, metricNameLen);
sprintf(metricName, "%s:%s", tableName, name);
int32_t metricNameLen = strlen(name) + strlen(tableName) + 2;
char* metricName = taosMemoryMalloc(metricNameLen);
memset(metricName, 0, metricNameLen);
sprintf(metricName, "%s:%s", tableName, name);
taos_metric_t* metric = taos_collector_registry_get_metric(metricName);
if(metric == NULL){
if(type == 0){
metric = taos_counter_new(metricName, "", tagSize, (const char**)labels);
}
if(type == 1){
metric = taos_gauge_new(metricName, "", tagSize, (const char**)labels);
}
mTrace("fail to get metric from registry, new one metric:%p", metric);
if(taos_collector_registry_register_metric(metric) == 1){
taos_metric_t* metric = taos_collector_registry_get_metric(metricName);
if(metric == NULL){
if(type == 0){
taos_counter_destroy(metric);
metric = taos_counter_new(metricName, "", tagSize, (const char**)labels);
}
if(type == 1){
taos_gauge_destroy(metric);
metric = taos_gauge_new(metricName, "", tagSize, (const char**)labels);
}
mTrace("fail to get metric from registry, new one metric:%p", metric);
metric = taos_collector_registry_get_metric(metricName);
if(taos_collector_registry_register_metric(metric) == 1){
if(type == 0){
taos_counter_destroy(metric);
}
if(type == 1){
taos_gauge_destroy(metric);
}
mTrace("fail to register metric, get metric from registry:%p", metric);
metric = taos_collector_registry_get_metric(metricName);
mTrace("fail to register metric, get metric from registry:%p", metric);
}
else{
mTrace("succeed to register metric:%p", metric);
}
}
else{
mTrace("succeed to register metric:%p", metric);
mTrace("get metric from registry:%p", metric);
}
}
else{
mTrace("get metric from registry:%p", metric);
if(type == 0){
taos_counter_add(metric, value, (const char**)sample_labels);
}
if(type == 1){
taos_gauge_set(metric, value, (const char**)sample_labels);
}
taosMemoryFreeClear(metricName);
}
if(type == 0){
taos_counter_add(metric, value, (const char**)sample_labels);
}
if(type == 1){
taos_gauge_set(metric, value, (const char**)sample_labels);
for(int32_t j = 0; j < tagSize; j++){
taosMemoryFreeClear(*(labels + j));
taosMemoryFreeClear(*(sample_labels + j));
}
taosMemoryFreeClear(metricName);
taosMemoryFreeClear(sample_labels);
taosMemoryFreeClear(labels);
}
for(int32_t j = 0; j < tagSize; j++){
taosMemoryFreeClear(*(labels + j));
taosMemoryFreeClear(*(sample_labels + j));
}
taosMemoryFreeClear(sample_labels);
taosMemoryFreeClear(labels);
}
}
}
code = 0;
code = 0;
_OVER:
if(pJson != NULL){
tjsonDelete(pJson);
pJson = NULL;
}
_OVER:
if(pJson != NULL){
tjsonDelete(pJson);
pJson = NULL;
}
tFreeSStatisReq(&statisReq);
return code;
*/
tFreeSStatisReq(&statisReq);
return code;
*/
}
static int32_t mndUpdateDnodeObj(SMnode *pMnode, SDnodeObj *pDnode) {
@ -816,8 +816,9 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
bool reboot = (pDnode->rebootTime != statusReq.rebootTime);
bool supportVnodesChanged = pDnode->numOfSupportVnodes != statusReq.numOfSupportVnodes;
bool encryptKeyChanged = pDnode->encryptionKeyChksum != statusReq.clusterCfg.encryptionKeyChksum;
bool enableWhiteListChanged = statusReq.clusterCfg.enableWhiteList != (tsEnableWhiteList ? 1 : 0);
bool needCheck = !online || dnodeChanged || reboot || supportVnodesChanged ||
pMnode->ipWhiteVer != statusReq.ipWhiteVer || encryptKeyChanged;
pMnode->ipWhiteVer != statusReq.ipWhiteVer || encryptKeyChanged || enableWhiteListChanged;
const STraceId *trace = &pReq->info.traceId;
mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id,
@ -1148,7 +1149,6 @@ _OVER:
}
tFreeSShowVariablesRsp(&rsp);
return code;
}
@ -1457,7 +1457,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
int8_t updateIpWhiteList = 0;
mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) {
tFreeSMCfgDnodeReq(&cfgReq);
@ -1492,6 +1492,9 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
terrno = TSDB_CODE_INVALID_CFG;
goto _err_out;
}
if (strncasecmp(dcfgReq.config, "enableWhiteList", strlen("enableWhiteList")) == 0) {
updateIpWhiteList = 1;
}
if (cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true) != 0) goto _err_out;
}
@ -1505,7 +1508,11 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
tFreeSMCfgDnodeReq(&cfgReq);
return mndSendCfgDnodeReq(pMnode, cfgReq.dnodeId, &dcfgReq);
int32_t code = mndSendCfgDnodeReq(pMnode, cfgReq.dnodeId, &dcfgReq);
// dont care suss or succ;
if (updateIpWhiteList) mndRefreshUserIpWhiteList(pMnode);
return code;
_err_out:
tFreeSMCfgDnodeReq(&cfgReq);

8
source/dnode/mnode/impl/src/mndInfoSchema.c
View File

@ -75,7 +75,13 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *
return -1;
}
STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName));
STableMetaRsp *pMeta = NULL;
if (strcmp(tbName, TSDB_INS_TABLE_USERS_FULL) == 0) {
pMeta = taosHashGet(pMnode->infosMeta, TSDB_INS_TABLE_USERS_FULL, strlen(tbName));
} else {
pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName));
}
if (NULL == pMeta) {
mError("invalid information schema table name:%s", tbName);
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;

22
source/dnode/mnode/impl/src/mndPrivilege.c
View File

@ -31,7 +31,6 @@ int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType op
int32_t mndCheckTopicPrivilege(SMnode *pMnode, const char *user, EOperType operType, SMqTopicObj *pTopic) { return 0; }
int32_t mndSetUserWhiteListRsp(SMnode *pMnode, SUserObj *pUser, SGetUserWhiteListRsp *pWhiteListRsp) {
memcpy(pWhiteListRsp->user, pUser->user, TSDB_USER_LEN);
pWhiteListRsp->numWhiteLists = 1;
@ -41,25 +40,6 @@ int32_t mndSetUserWhiteListRsp(SMnode *pMnode, SUserObj *pUser, SGetUserWhiteLis
}
memset(pWhiteListRsp->pWhiteLists, 0, pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range));
// if (tsEnableWhiteList) {
// memcpy(pWhiteListRsp->user, pUser->user, TSDB_USER_LEN);
// pWhiteListRsp->numWhiteLists = pUser->pIpWhiteList->num;
// pWhiteListRsp->pWhiteLists = taosMemoryMalloc(pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range));
// if (pWhiteListRsp->pWhiteLists == NULL) {
// return TSDB_CODE_OUT_OF_MEMORY;
// }
// memcpy(pWhiteListRsp->pWhiteLists, pUser->pIpWhiteList->pIpRange,
// pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range));
// } else {
// memcpy(pWhiteListRsp->user, pUser->user, TSDB_USER_LEN);
// pWhiteListRsp->numWhiteLists = 1;
// pWhiteListRsp->pWhiteLists = taosMemoryMalloc(pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range));
// if (pWhiteListRsp->pWhiteLists == NULL) {
// return TSDB_CODE_OUT_OF_MEMORY;
// }
// memset(pWhiteListRsp->pWhiteLists, 0, pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range));
// }
return 0;
}
@ -70,7 +50,7 @@ int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp
pRsp->sysInfo = pUser->sysInfo;
pRsp->version = pUser->authVersion;
pRsp->passVer = pUser->passVersion;
pRsp->whiteListVer = pUser->ipWhiteListVer;
pRsp->whiteListVer = pMnode->ipWhiteVer;
return 0;
}

5
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -62,6 +62,7 @@ typedef struct {
int32_t onlineDnodes;
SEpSet epSet;
SArray *pQnodeList;
int64_t ipWhiteListVer;
} SConnPreparedObj;
static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType, uint32_t ip, uint16_t port,
@ -568,7 +569,8 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
case HEARTBEAT_KEY_USER_AUTHINFO: {
void *rspMsg = NULL;
int32_t rspLen = 0;
mndValidateUserAuthInfo(pMnode, kv->value, kv->valueLen / sizeof(SUserAuthVersion), &rspMsg, &rspLen);
mndValidateUserAuthInfo(pMnode, kv->value, kv->valueLen / sizeof(SUserAuthVersion), &rspMsg, &rspLen,
pObj->ipWhiteListVer);
if (rspMsg && rspLen > 0) {
SKv kv1 = {.key = HEARTBEAT_KEY_USER_AUTHINFO, .valueLen = rspLen, .value = rspMsg};
taosArrayPush(hbRsp.info, &kv1);
@ -650,6 +652,7 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) {
SConnPreparedObj obj = {0};
obj.totalDnodes = mndGetDnodeSize(pMnode);
obj.ipWhiteListVer = batchReq.ipWhiteList;
mndGetOnlineDnodeNum(pMnode, &obj.onlineDnodes);
mndGetMnodeEpSet(pMnode, &obj.epSet);
mndCreateQnodeList(pMnode, &obj.pQnodeList, -1);

9
source/dnode/mnode/impl/src/mndShow.c
View File

@ -89,6 +89,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
// type = TSDB_MGMT_TABLE_DIST;
} else if (strncasecmp(name, TSDB_INS_TABLE_USERS, len) == 0) {
type = TSDB_MGMT_TABLE_USER;
} else if (strncasecmp(name, TSDB_INS_TABLE_USERS_FULL, len) == 0) {
type = TSDB_MGMT_TABLE_USER_FULL;
} else if (strncasecmp(name, TSDB_INS_TABLE_LICENCES, len) == 0) {
type = TSDB_MGMT_TABLE_GRANTS;
} else if (strncasecmp(name, TSDB_INS_TABLE_VGROUPS, len) == 0) {
@ -276,6 +278,13 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
if (retrieveReq.db[0] && mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db) != 0) {
return -1;
}
if (pShow->type == TSDB_MGMT_TABLE_USER_FULL) {
if(strcmp(pReq->info.conn.user, "root") != 0){
mError("The operation is not permitted, user:%s, pShow->type:%d", pReq->info.conn.user, pShow->type);
terrno = TSDB_CODE_MND_NO_RIGHTS;
return -1;
}
}
int32_t numOfCols = pShow->pMeta->numOfColumns;

14
source/dnode/mnode/impl/src/mndStb.c
View File

@ -884,7 +884,7 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
}
if (pDst->nextColId < 0 || pDst->nextColId >= 0x7fff - pDst->numOfColumns - pDst->numOfTags) {
terrno = TSDB_CODE_MND_FIELD_VALUE_OVERFLOW;
terrno = TSDB_CODE_OUT_OF_RANGE;
return -1;
}
@ -1148,7 +1148,7 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq
}
if (pDst->nextColId < 0 || pDst->nextColId >= 0x7fff - pDst->numOfColumns - pDst->numOfTags) {
terrno = TSDB_CODE_MND_FIELD_VALUE_OVERFLOW;
terrno = TSDB_CODE_OUT_OF_RANGE;
return -1;
}
@ -1414,7 +1414,7 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p
}
if (pNew->nextColId < 0 || pNew->nextColId >= 0x7fff - ntags) {
terrno = TSDB_CODE_MND_FIELD_VALUE_OVERFLOW;
terrno = TSDB_CODE_OUT_OF_RANGE;
return -1;
}
@ -1766,7 +1766,7 @@ static int32_t mndUpdateSuperTableColumnCompress(SMnode *pMnode, const SStbObj *
uint32_t dst = 0;
updated = tUpdateCompress(pCmpr->alg, p->bytes, TSDB_COLVAL_COMPRESS_DISABLED, TSDB_COLVAL_LEVEL_DISABLED,
TSDB_COLVAL_LEVEL_MEDIUM, &dst);
if (updated) pCmpr->alg = dst;
if (updated > 0) pCmpr->alg = dst;
break;
}
}
@ -1774,7 +1774,11 @@ static int32_t mndUpdateSuperTableColumnCompress(SMnode *pMnode, const SStbObj *
if (updated == 0) {
terrno = TSDB_CODE_MND_COLUMN_COMPRESS_ALREADY_EXIST;
return -1;
} else if (updated == -1) {
terrno = TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR;
return -1;
}
pNew->colVer++;
return 0;
@ -1802,7 +1806,7 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray
}
if (pNew->nextColId < 0 || pNew->nextColId >= 0x7fff - ncols) {
terrno = TSDB_CODE_MND_FIELD_VALUE_OVERFLOW;
terrno = TSDB_CODE_OUT_OF_RANGE;
return -1;
}

320
source/dnode/mnode/impl/src/mndStream.c
View File

@ -56,13 +56,14 @@ static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, int
int64_t streamId, int32_t taskId, int32_t transId, int8_t mndTrigger);
static int32_t mndProcessNodeCheck(SRpcMsg *pReq);
static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg);
static SArray *extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList);
static int32_t extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList);
static int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq);
static int32_t mndProcessCheckpointReport(SRpcMsg *pReq);
static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList);
void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
static int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot);
static void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo *pExecInfo);
static void removeExpiredNodeInfo(const SArray *pNodeSnapshot);
static int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDbName, size_t len);
static SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw);
@ -103,6 +104,7 @@ int32_t mndInitStream(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_STOP_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_VND_STREAM_TASK_UPDATE_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_VND_STREAM_TASK_RESET_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_UPDATE_CHKPT_RSP, mndTransProcessRsp);
// for msgs inside mnode
// TODO change the name
@ -114,8 +116,10 @@ int32_t mndInitStream(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_VND_STREAM_CHECK_POINT_SOURCE_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamCheckpoint);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_REQ_CHKPT, mndProcessStreamReqCheckpoint);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_HEARTBEAT, mndProcessStreamHb);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHKPT_REPORT, mndProcessCheckpointReport);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_UPDATE_CHKPT_EVT, mndScanCheckpointReportInfo);
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_REPORT_CHECKPOINT, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_HEARTBEAT, mndProcessStreamHb);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_NODECHANGE_CHECK, mndProcessNodeCheckReq);
mndSetMsgHandle(pMnode, TDMT_MND_PAUSE_STREAM, mndProcessPauseStreamReq);
@ -131,9 +135,11 @@ int32_t mndInitStream(SMnode *pMnode) {
if (sdbSetTable(pMnode->pSdb, table) != 0) {
return -1;
}
if (sdbSetTable(pMnode->pSdb, tableSeq) != 0) {
return -1;
}
return 0;
}
@ -143,6 +149,7 @@ void mndCleanupStream(SMnode *pMnode) {
taosHashCleanup(execInfo.pTaskMap);
taosHashCleanup(execInfo.transMgmt.pDBTrans);
taosHashCleanup(execInfo.pTransferStateStreams);
taosHashCleanup(execInfo.pChkptStreams);
taosThreadMutexDestroy(&execInfo.lock);
mDebug("mnd stream exec info cleanup");
}
@ -508,7 +515,7 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) {
tEncodeStreamTask(&encoder, pTask);
tEncoderClear(&encoder);
int32_t code = setTransAction(pTrans, buf, tlen, TDMT_STREAM_TASK_DEPLOY, &pTask->info.epSet, 0);
int32_t code = setTransAction(pTrans, buf, tlen, TDMT_STREAM_TASK_DEPLOY, &pTask->info.epSet, 0, 0);
if (code != 0) {
taosMemoryFree(buf);
return -1;
@ -952,7 +959,7 @@ static int32_t doSetCheckpointAction(SMnode *pMnode, STrans *pTrans, SStreamTask
return -1;
}
code = setTransAction(pTrans, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY);
code = setTransAction(pTrans, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY, 0);
if (code != 0) {
taosMemoryFree(buf);
}
@ -1068,7 +1075,7 @@ static bool taskNodeIsUpdated(SMnode *pMnode) {
mWarn("not all vnodes ready, quit from vnodes status check");
taosArrayDestroy(pNodeSnapshot);
taosThreadMutexUnlock(&execInfo.lock);
return 0;
return true;
}
SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execInfo.pNodeList, pNodeSnapshot);
@ -1088,7 +1095,7 @@ static bool taskNodeIsUpdated(SMnode *pMnode) {
}
static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
bool ready = true;
bool ready = true;
if (taskNodeIsUpdated(pMnode)) {
return -1;
}
@ -1099,6 +1106,8 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
ASSERT(taosArrayGetSize(execInfo.pTaskList) == 0);
}
SArray* pInvalidList = taosArrayInit(4, sizeof(STaskId));
for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
STaskId *p = taosArrayGet(execInfo.pTaskList, i);
STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p));
@ -1106,11 +1115,20 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
continue;
}
if (pEntry->status == TASK_STATUS__STOP) {
for(int32_t j = 0; j < taosArrayGetSize(pInvalidList); ++j) {
STaskId* pId = taosArrayGet(pInvalidList, j);
if (pEntry->id.streamId == pId->streamId) {
taosArrayPush(pInvalidList, &pEntry->id);
break;
}
}
}
if (pEntry->status != TASK_STATUS__READY) {
mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s not ready, checkpoint msg not issued",
mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s, checkpoint not issued",
pEntry->id.streamId, (int32_t)pEntry->id.taskId, pEntry->nodeId, streamTaskGetStatusStr(pEntry->status));
ready = false;
break;
}
if (pEntry->hTaskId != 0) {
@ -1123,6 +1141,9 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
}
}
removeTasksInBuf(pInvalidList, &execInfo);
taosArrayDestroy(pInvalidList);
taosThreadMutexUnlock(&execInfo.lock);
return ready ? 0 : -1;
}
@ -1151,7 +1172,8 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
int32_t numOfCheckpointTrans = 0;
if ((code = mndCheckTaskAndNodeStatus(pMnode)) != 0) {
return code;
terrno = TSDB_CODE_STREAM_TASK_IVLD_STATUS;
return -1;
}
SArray* pList = taosArrayInit(4, sizeof(SCheckpointInterval));
@ -1798,6 +1820,10 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
int32_t numOfRows = 0;
SStreamObj *pStream = NULL;
taosThreadMutexLock(&execInfo.lock);
mndInitStreamExecInfo(pMnode, &execInfo);
taosThreadMutexUnlock(&execInfo.lock);
while (numOfRows < rowsCapacity) {
pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream);
if (pShow->pIter == NULL) {
@ -1885,9 +1911,51 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
bool updated = taskNodeIsUpdated(pMnode);
if (updated) {
mError("tasks are not ready for pause, node update detected");
sdbRelease(pMnode->pSdb, pStream);
return -1;
}
{ // check for tasks, if tasks are not ready, not allowed to pause
bool found = false;
bool readyToPause = true;
taosThreadMutexLock(&execInfo.lock);
for(int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
STaskId *p = taosArrayGet(execInfo.pTaskList, i);
STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p));
if (pEntry == NULL) {
continue;
}
if (pEntry->id.streamId != pStream->uid) {
continue;
}
if (pEntry->status == TASK_STATUS__UNINIT || pEntry->status == TASK_STATUS__CK) {
mError("stream:%s uid:0x%" PRIx64 " vgId:%d task:0x%" PRIx64 " status:%s, not ready for pause", pStream->name,
pStream->uid, pEntry->nodeId, pEntry->id.taskId, streamTaskGetStatusStr(pEntry->status));
readyToPause = false;
}
found = true;
}
taosThreadMutexUnlock(&execInfo.lock);
if (!found) {
mError("stream:%s task not report status yet, not ready for pause", pauseReq.name);
sdbRelease(pMnode->pSdb, pStream);
return -1;
}
if (!readyToPause) {
mError("stream:%s task not ready for pause yet", pauseReq.name);
sdbRelease(pMnode->pSdb, pStream);
return -1;
}
}
STrans *pTrans =
doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_PAUSE_NAME, "pause the stream");
if (pTrans == NULL) {
@ -2169,7 +2237,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
return 0;
}
static SArray *extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList) {
static int32_t extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList) {
SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = NULL;
void *pIter = NULL;
@ -2215,48 +2283,6 @@ static SArray *extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList) {
return TSDB_CODE_SUCCESS;
}
static bool taskNodeExists(SArray *pList, int32_t nodeId) {
size_t num = taosArrayGetSize(pList);
for (int32_t i = 0; i < num; ++i) {
SNodeEntry *pEntry = taosArrayGet(pList, i);
if (pEntry->nodeId == nodeId) {
return true;
}
}
return false;
}
int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot) {
SArray *pRemovedTasks = taosArrayInit(4, sizeof(STaskId));
int32_t numOfTask = taosArrayGetSize(execInfo.pTaskList);
for (int32_t i = 0; i < numOfTask; ++i) {
STaskId *pId = taosArrayGet(execInfo.pTaskList, i);
STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId));
if (pEntry->nodeId == SNODE_HANDLE) {
continue;
}
bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId);
if (!existed) {
taosArrayPush(pRemovedTasks, pId);
}
}
removeTasksInBuf(pRemovedTasks, &execInfo);
mDebug("remove invalid stream tasks:%d, remain:%d", (int32_t)taosArrayGetSize(pRemovedTasks),
(int32_t)taosArrayGetSize(execInfo.pTaskList));
removeExpiredNodeInfo(pNodeSnapshot);
taosArrayDestroy(pRemovedTasks);
return 0;
}
// this function runs by only one thread, so it is not multi-thread safe
static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
int32_t code = 0;
@ -2476,13 +2502,137 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
taosThreadMutexUnlock(&execInfo.lock);
{
SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRspMsg)};
SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRsp)};
rsp.pCont = rpcMallocCont(rsp.contLen);
SMsgHead *pHead = rsp.pCont;
pHead->vgId = htonl(req.nodeId);
tmsgSendRsp(&rsp);
pReq->info.handle = NULL; // disable auto rsp
}
return 0;
}
static void doAddTaskInfo(SArray* pList, SCheckpointReport* pReport) {
bool existed = false;
for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
STaskChkptInfo* p = taosArrayGet(pList ,i);
if (p->taskId == pReport->taskId) {
existed = true;
break;
}
}
if (!existed) {
STaskChkptInfo info = {
.streamId = pReport->streamId,
.taskId = pReport->taskId,
.transId = pReport->transId,
.dropHTask = pReport->dropHTask,
.version = pReport->checkpointVer,
.ts = pReport->checkpointTs,
.checkpointId = pReport->checkpointId,
.nodeId = pReport->nodeId,
};
taosArrayPush(pList, &info);
}
}
int32_t mndProcessCheckpointReport(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SCheckpointReport req = {0};
SDecoder decoder = {0};
tDecoderInit(&decoder, pReq->pCont, pReq->contLen);
if (tDecodeStreamTaskChkptReport(&decoder, &req)) {
tDecoderClear(&decoder);
terrno = TSDB_CODE_INVALID_MSG;
mError("invalid task checkpoint-report msg received");
return -1;
}
tDecoderClear(&decoder);
mDebug("receive stream task checkpoint-report msg, vgId:%d, s-task:0x%x, checkpointId:%" PRId64
" checkpointVer:%" PRId64 " transId:%d",
req.nodeId, req.taskId, req.checkpointId, req.checkpointVer, req.transId);
// register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans.
taosThreadMutexLock(&execInfo.lock);
SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId);
if (pStream == NULL) {
mWarn("failed to find the stream:0x%" PRIx64 ", not handle checkpoint-report, try to acquire in buf",
req.streamId);
// not in meta-store yet, try to acquire the task in exec buffer
// the checkpoint req arrives too soon before the completion of the create stream trans.
STaskId id = {.streamId = req.streamId, .taskId = req.taskId};
void *p = taosHashGet(execInfo.pTaskMap, &id, sizeof(id));
if (p == NULL) {
mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint-report", req.streamId);
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
taosThreadMutexUnlock(&execInfo.lock);
return -1;
} else {
mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet",
req.streamId, req.taskId);
}
}
int32_t numOfTasks = (pStream == NULL) ? 0 : mndGetNumOfStreamTasks(pStream);
SArray **pReqTaskList = (SArray **)taosHashGet(execInfo.pChkptStreams, &req.streamId, sizeof(req.streamId));
if (pReqTaskList == NULL) {
SArray *pList = taosArrayInit(4, sizeof(STaskChkptInfo));
doAddTaskInfo(pList, &req);
taosHashPut(execInfo.pChkptStreams, &req.streamId, sizeof(req.streamId), &pList, POINTER_BYTES);
pReqTaskList = (SArray **)taosHashGet(execInfo.pChkptStreams, &req.streamId, sizeof(req.streamId));
} else {
doAddTaskInfo(*pReqTaskList, &req);
}
int32_t total = taosArrayGetSize(*pReqTaskList);
if (total == numOfTasks) { // all tasks has send the reqs
mInfo("stream:0x%" PRIx64 " %s all %d tasks send checkpoint-report, checkpoint meta-info for checkpointId:%" PRId64
" will be issued soon",
req.streamId, pStream->name, total, req.checkpointId);
// if (pStream != NULL) {
// bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHKPT_UPDATE_NAME, false);
// if (conflict) {
// mDebug("stream:0x%"PRIx64" active checkpoint trans not finished yet, wait", req.streamId);
// } else {
// int32_t code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, *pReqTaskList);
// if (code == TSDB_CODE_SUCCESS) { // remove this entry
// taosHashRemove(execInfo.pChkptStreams, &req.streamId, sizeof(req.streamId));
//
// int32_t numOfStreams = taosHashGetSize(execInfo.pChkptStreams);
// mDebug("stream:0x%" PRIx64 " removed, remain streams:%d in checkpoint procedure", req.streamId,
// numOfStreams);
// } else {
// mDebug("stream:0x%" PRIx64 " not launch chkpt update trans, due to checkpoint not finished yet",
// req.streamId);
// }
// }
// }
}
if (pStream != NULL) {
mndReleaseStream(pMnode, pStream);
}
taosThreadMutexUnlock(&execInfo.lock);
{
SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamUpdateChkptRsp)};
rsp.pCont = rpcMallocCont(rsp.contLen);
SMsgHead *pHead = rsp.pCont;
pHead->vgId = htonl(req.nodeId);
tmsgSendRsp(&rsp);
pReq->info.handle = NULL; // disable auto rsp
}
@ -2510,3 +2660,63 @@ static int32_t mndProcessDropStreamReqFromMNode(SRpcMsg *pReq) {
}
return code;
}
void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo) {
if (pExecInfo->initTaskList || pMnode == NULL) {
return;
}
addAllStreamTasksIntoBuf(pMnode, pExecInfo);
extractNodeListFromStream(pMnode, pExecInfo->pNodeList);
pExecInfo->initTaskList = true;
}
void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo* pExecInfo) {
SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = NULL;
void *pIter = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
if (pIter == NULL) {
break;
}
saveTaskAndNodeInfoIntoBuf(pStream, pExecInfo);
sdbRelease(pSdb, pStream);
}
}
int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray* pChkptInfoList) {
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_UPDATE_NAME, "update checkpoint-info");
if (pTrans == NULL) {
return terrno;
}
/*int32_t code = */mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_UPDATE_NAME, pStream->uid);
int32_t code = mndStreamSetUpdateChkptAction(pMnode, pTrans, pStream);
if (code != 0) {
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return code;
}
code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY);
if (code != TSDB_CODE_SUCCESS) {
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return -1;
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare update checkpoint-info meta trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return -1;
}
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return TSDB_CODE_ACTION_IN_PROGRESS;
}

88
source/dnode/mnode/impl/src/mndStreamHb.c
View File

@ -22,54 +22,7 @@ typedef struct SFailedCheckpointInfo {
int32_t transId;
} SFailedCheckpointInfo;
static void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo* pExecInfo) {
SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = NULL;
void *pIter = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
if (pIter == NULL) {
break;
}
saveTaskAndNodeInfoIntoBuf(pStream, pExecInfo);
sdbRelease(pSdb, pStream);
}
}
static void removeDroppedStreamTasksInBuf(SMnode *pMnode, SStreamExecInfo *pExecInfo) {
if (pMnode == NULL) {
return;
}
int32_t num = taosArrayGetSize(pExecInfo->pTaskList);
SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
SArray *pIdList = taosArrayInit(4, sizeof(STaskId));
for (int32_t i = 0; i < num; ++i) {
STaskId* pId = taosArrayGet(pExecInfo->pTaskList, i);
void* p = taosHashGet(pHash, &pId->streamId, sizeof(int64_t));
if (p != NULL) {
continue;
}
void* pObj = mndGetStreamObj(pMnode, pId->streamId);
if (pObj != NULL) {
mndReleaseStream(pMnode, pObj);
taosHashPut(pHash, &pId->streamId, sizeof(int64_t), NULL, 0);
} else {
taosArrayPush(pIdList, pId);
}
}
removeTasksInBuf(pIdList, &execInfo);
taosArrayDestroy(pIdList);
taosHashCleanup(pHash);
}
static void mndStreamStartUpdateCheckpointInfo(SMnode *pMnode);
static void updateStageInfo(STaskStatusEntry *pTaskEntry, int64_t stage) {
int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeList);
@ -290,16 +243,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
taosThreadMutexLock(&execInfo.lock);
// extract stream task list
if (taosHashGetSize(execInfo.pTaskMap) == 0) {
addAllStreamTasksIntoBuf(pMnode, &execInfo);
} else {
// the already dropped tasks may be added by hb from vnode at the time when the pTaskMap happens to be empty.
// let's drop them here.
removeDroppedStreamTasksInBuf(pMnode, &execInfo);
}
extractStreamNodeList(pMnode);
mndInitStreamExecInfo(pMnode, &execInfo);
int32_t numOfUpdated = taosArrayGetSize(req.pUpdateNodes);
if (numOfUpdated > 0) {
@ -326,18 +270,6 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
snodeChanged = true;
}
} else {
// task is idle for more than 50 sec.
// if (fabs(pTaskEntry->inputQUsed - p->inputQUsed) <= DBL_EPSILON) {
// if (!pTaskEntry->inputQChanging) {
// pTaskEntry->inputQUnchangeCounter++;
// } else {
// pTaskEntry->inputQChanging = false;
// }
// } else {
// pTaskEntry->inputQChanging = true;
// pTaskEntry->inputQUnchangeCounter = 0;
// }
streamTaskStatusCopy(pTaskEntry, p);
STaskCkptInfo *pChkInfo = &p->checkpointInfo;
@ -348,6 +280,9 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
SFailedCheckpointInfo info = {
.transId = pChkInfo->activeTransId, .checkpointId = pChkInfo->activeId, .streamUid = p->id.streamId};
addIntoCheckpointList(pFailedChkpt, &info);
// remove failed trans from pChkptStreams
taosHashRemove(execInfo.pChkptStreams, &p->id.streamId, sizeof(p->id.streamId));
}
}
@ -393,6 +328,10 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
mndDropOrphanTasks(pMnode, pOrphanTasks);
}
if (pMnode != NULL) { // make sure that the unit test case can work
mndStreamStartUpdateCheckpointInfo(pMnode);
}
taosThreadMutexUnlock(&execInfo.lock);
tCleanupStreamHbMsg(&req);
@ -411,3 +350,12 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
return TSDB_CODE_SUCCESS;
}
void mndStreamStartUpdateCheckpointInfo(SMnode *pMnode) { // here reuse the doCheckpointmsg
SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg));
if (pMsg != NULL) {
int32_t size = sizeof(SMStreamDoCheckpointMsg);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_UPDATE_CHKPT_EVT, .pCont = pMsg, .contLen = size};
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
}
}

14
source/dnode/mnode/impl/src/mndStreamTrans.c
View File

@ -127,7 +127,7 @@ bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamId, const char* p
return false;
}
int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamUid) {
int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamId) {
taosThreadMutexLock(&execInfo.lock);
int32_t num = taosHashGetSize(execInfo.transMgmt.pDBTrans);
if (num <= 0) {
@ -136,12 +136,13 @@ int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamUid) {
}
mndStreamClearFinishedTrans(pMnode, NULL);
SStreamTransInfo* pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamUid, sizeof(streamUid));
SStreamTransInfo* pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId));
if (pEntry != NULL) {
SStreamTransInfo tInfo = *pEntry;
taosThreadMutexUnlock(&execInfo.lock);
if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0 || strcmp(tInfo.name, MND_STREAM_TASK_UPDATE_NAME) == 0) {
if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0 || strcmp(tInfo.name, MND_STREAM_TASK_UPDATE_NAME) == 0 ||
strcmp(tInfo.name, MND_STREAM_CHKPT_UPDATE_NAME) == 0) {
return tInfo.transId;
}
} else {
@ -159,7 +160,7 @@ STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnCo
return NULL;
}
mInfo("s-task:0x%" PRIx64 " start to build trans %s, transId:%d", pStream->uid, pMsg, pTrans->id);
mInfo("stream:0x%" PRIx64 " start to build trans %s, transId:%d", pStream->uid, pMsg, pTrans->id);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetSTbName);
if (mndTransCheckConflict(pMnode, pTrans) != 0) {
@ -246,8 +247,9 @@ int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status)
}
int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
int32_t retryCode) {
STransAction action = {.epSet = *pEpset, .contLen = contLen, .pCont = pCont, .msgType = msgType, .retryCode = retryCode};
int32_t retryCode, int32_t acceptCode) {
STransAction action = {.epSet = *pEpset, .contLen = contLen, .pCont = pCont, .msgType = msgType, .retryCode = retryCode,
.acceptableCode = acceptCode};
return mndTransAppendRedoAction(pTrans, &action);
}

184
source/dnode/mnode/impl/src/mndStreamUtil.c
View File

@ -230,7 +230,7 @@ static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pT
return -1;
}
code = setTransAction(pTrans, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0);
code = setTransAction(pTrans, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0, 0);
if (code != 0) {
taosMemoryFree(pReq);
return -1;
@ -308,7 +308,7 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
epsetToStr(&epset, buf, tListLen(buf));
mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf);
code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0);
code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0, 0);
if (code != 0) {
taosMemoryFree(pReq);
return -1;
@ -356,7 +356,7 @@ static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTas
}
// The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode.
code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0);
code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0);
if (code != 0) {
taosMemoryFree(pReq);
return -1;
@ -400,7 +400,7 @@ static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask
}
// The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode.
code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0);
code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0);
if (code != 0) {
taosMemoryFree(pReq);
return -1;
@ -484,7 +484,7 @@ static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask
return code;
}
code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &epset, TSDB_CODE_VND_INVALID_VGROUP_ID);
code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &epset, TSDB_CODE_VND_INVALID_VGROUP_ID, 0);
if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(pBuf);
}
@ -534,7 +534,7 @@ static int32_t doSetResetAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
return code;
}
code = setTransAction(pTrans, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &epset, 0);
code = setTransAction(pTrans, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &epset, 0, 0);
if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(pReq);
}
@ -574,9 +574,11 @@ void mndInitExecInfo() {
execInfo.pTaskMap = taosHashInit(64, fn, true, HASH_NO_LOCK);
execInfo.transMgmt.pDBTrans = taosHashInit(32, fn, true, HASH_NO_LOCK);
execInfo.pTransferStateStreams = taosHashInit(32, fn, true, HASH_NO_LOCK);
execInfo.pChkptStreams = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
execInfo.pNodeList = taosArrayInit(4, sizeof(SNodeEntry));
taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList);
taosHashSetFreeFp(execInfo.pChkptStreams, freeTaskList);
}
void removeExpiredNodeInfo(const SArray *pNodeSnapshot) {
@ -645,4 +647,174 @@ void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
taosThreadMutexUnlock(&pExecNode->lock);
destroyStreamTaskIter(pIter);
}
static bool taskNodeExists(SArray *pList, int32_t nodeId) {
size_t num = taosArrayGetSize(pList);
for (int32_t i = 0; i < num; ++i) {
SNodeEntry *pEntry = taosArrayGet(pList, i);
if (pEntry->nodeId == nodeId) {
return true;
}
}
return false;
}
int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot) {
SArray *pRemovedTasks = taosArrayInit(4, sizeof(STaskId));
int32_t numOfTask = taosArrayGetSize(execInfo.pTaskList);
for (int32_t i = 0; i < numOfTask; ++i) {
STaskId *pId = taosArrayGet(execInfo.pTaskList, i);
STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId));
if (pEntry->nodeId == SNODE_HANDLE) {
continue;
}
bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId);
if (!existed) {
taosArrayPush(pRemovedTasks, pId);
}
}
removeTasksInBuf(pRemovedTasks, &execInfo);
mDebug("remove invalid stream tasks:%d, remain:%d", (int32_t)taosArrayGetSize(pRemovedTasks),
(int32_t)taosArrayGetSize(execInfo.pTaskList));
removeExpiredNodeInfo(pNodeSnapshot);
taosArrayDestroy(pRemovedTasks);
return 0;
}
static int32_t doSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) {
SVUpdateCheckpointInfoReq *pReq = taosMemoryCalloc(1, sizeof(SVUpdateCheckpointInfoReq));
if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to malloc in reset stream, size:%" PRIzu ", code:%s", sizeof(SVUpdateCheckpointInfoReq),
tstrerror(TSDB_CODE_OUT_OF_MEMORY));
return terrno;
}
pReq->head.vgId = htonl(pTask->info.nodeId);
pReq->taskId = pTask->id.taskId;
pReq->streamId = pTask->id.streamId;
SArray **pReqTaskList = (SArray **)taosHashGet(execInfo.pChkptStreams, &pTask->id.streamId, sizeof(pTask->id.streamId));
ASSERT(pReqTaskList);
int32_t size = taosArrayGetSize(*pReqTaskList);
for(int32_t i = 0; i < size; ++i) {
STaskChkptInfo* pInfo = taosArrayGet(*pReqTaskList, i);
if (pInfo->taskId == pTask->id.taskId) {
pReq->checkpointId = pInfo->checkpointId;
pReq->checkpointVer = pInfo->version;
pReq->checkpointTs = pInfo->ts;
pReq->dropRelHTask = pInfo->dropHTask;
pReq->transId = pInfo->transId;
pReq->hStreamId = pTask->hTaskInfo.id.streamId;
pReq->hTaskId = pTask->hTaskInfo.id.taskId;
}
}
SEpSet epset = {0};
bool hasEpset = false;
int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
if (code != TSDB_CODE_SUCCESS || !hasEpset) {
taosMemoryFree(pReq);
return code;
}
code = setTransAction(pTrans, pReq, sizeof(SVUpdateCheckpointInfoReq), TDMT_STREAM_TASK_UPDATE_CHKPT, &epset, 0, TSDB_CODE_VND_INVALID_VGROUP_ID);
if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(pReq);
}
return code;
}
int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
taosWLockLatch(&pStream->lock);
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
int32_t code = doSetUpdateChkptAction(pMnode, pTrans, pTask);
if (code != TSDB_CODE_SUCCESS) {
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return -1;
}
}
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return 0;
}
int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
void *pIter = NULL;
SArray *pDropped = taosArrayInit(4, sizeof(int64_t));
mDebug("start to scan checkpoint report info");
while ((pIter = taosHashIterate(execInfo.pChkptStreams, pIter)) != NULL) {
SArray *pList = *(SArray **)pIter;
STaskChkptInfo* pInfo = taosArrayGet(pList, 0);
SStreamObj* pStream = mndGetStreamObj(pMnode, pInfo->streamId);
if (pStream == NULL) {
mDebug("failed to acquire stream:0x%" PRIx64 " remove it from checkpoint-report list", pInfo->streamId);
taosArrayPush(pDropped, &pInfo->streamId);
continue;
}
int32_t total = mndGetNumOfStreamTasks(pStream);
int32_t existed = (int32_t) taosArrayGetSize(pList);
if (total == existed) {
mDebug("stream:0x%" PRIx64 " %s all %d tasks send checkpoint-report, start to update checkpoint-info",
pStream->uid, pStream->name, total);
bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHKPT_UPDATE_NAME, false);
if (!conflict) {
int32_t code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, pList);
if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) { // remove this entry
taosArrayPush(pDropped, &pInfo->streamId);
mDebug("stream:0x%" PRIx64 " removed", pInfo->streamId);
} else {
mDebug("stream:0x%" PRIx64 " not launch chkpt-meta update trans, due to checkpoint not finished yet",
pInfo->streamId);
}
break;
} else {
mDebug("stream:0x%"PRIx64" active checkpoint trans not finished yet, wait", pInfo->streamId);
}
} else {
mDebug("stream:0x%" PRIx64 " %s %d/%d tasks send checkpoint-report, %d not send", pInfo->streamId, pStream->name,
existed, total, total - existed);
}
sdbRelease(pMnode->pSdb, pStream);
}
int32_t size = taosArrayGetSize(pDropped);
if (size > 0) {
for (int32_t i = 0; i < size; ++i) {
int64_t streamId = *(int64_t *)taosArrayGet(pDropped, i);
taosHashRemove(execInfo.pChkptStreams, &streamId, sizeof(streamId));
}
int32_t numOfStreams = taosHashGetSize(execInfo.pChkptStreams);
mDebug("drop %d stream(s) in checkpoint-report list, remain:%d", size, numOfStreams);
}
taosArrayDestroy(pDropped);
return TSDB_CODE_SUCCESS;
}

104
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -618,13 +618,13 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
char cgroup[TSDB_CGROUP_LEN] = {0};
mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup, true);
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TOPIC_INSIDE, pMsg, "tmq-reb");
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "tmq-reb");
if (pTrans == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
mndTransSetDbName(pTrans, topic, cgroup);
mndTransSetDbName(pTrans, pOutput->pSub->dbName, cgroup);
code = mndTransCheckConflict(pMnode, pTrans);
if (code != 0) {
goto END;
@ -799,6 +799,29 @@ static int32_t initRebOutput(SMqRebOutputObj *rebOutput) {
return 0;
}
// This function only works when there are dirty consumers
static void checkConsumer(SMnode *pMnode, SMqSubscribeObj* pSub){
void *pIter = NULL;
while (1) {
pIter = taosHashIterate(pSub->consumerHash, pIter);
if (pIter == NULL) {
break;
}
SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pConsumerEp->consumerId);
if (pConsumer != NULL) {
mndReleaseConsumer(pMnode, pConsumer);
continue;
}
mError("consumer:0x%" PRIx64 " not exists in sdb for exception", pConsumerEp->consumerId);
taosArrayAddAll(pSub->unassignedVgs, pConsumerEp->vgs);
taosArrayDestroy(pConsumerEp->vgs);
taosHashRemove(pSub->consumerHash, &pConsumerEp->consumerId, sizeof(int64_t));
}
}
static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOutputObj *rebOutput){
const char *key = rebInput->pRebInfo->key;
SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, key);
@ -834,8 +857,9 @@ static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOu
mInfo("[rebalance] sub topic:%s has no consumers sub yet", key);
} else {
taosRLockLatch(&pSub->lock);
rebInput->oldConsumerNum = taosHashGetSize(pSub->consumerHash);
rebOutput->pSub = tCloneSubscribeObj(pSub);
checkConsumer(pMnode, rebOutput->pSub);
rebInput->oldConsumerNum = taosHashGetSize(rebOutput->pSub->consumerHash);
taosRUnLockLatch(&pSub->lock);
mInfo("[rebalance] sub topic:%s has %d consumers sub till now", key, rebInput->oldConsumerNum);
@ -908,61 +932,83 @@ END:
}
static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STrans *pTrans){
// iter all vnode to delete handle
int32_t sz = taosArrayGetSize(pSub->unassignedVgs);
for (int32_t i = 0; i < sz; i++) {
SMqVgEp *pVgEp = taosArrayGetP(pSub->unassignedVgs, i);
void* pIter = NULL;
SVgObj* pVgObj = NULL;
int32_t ret = 0;
while (1) {
pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void**)&pVgObj);
if (pIter == NULL) {
break;
}
if (!mndVgroupInDb(pVgObj, pSub->dbUid)) {
sdbRelease(pMnode->pSdb, pVgObj);
continue;
}
SMqVDeleteReq *pReq = taosMemoryCalloc(1, sizeof(SMqVDeleteReq));
if(pReq == NULL){
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
ret = -1;
goto END;
}
pReq->head.vgId = htonl(pVgEp->vgId);
pReq->vgId = pVgEp->vgId;
pReq->head.vgId = htonl(pVgObj->vgId);
pReq->vgId = pVgObj->vgId;
pReq->consumerId = -1;
memcpy(pReq->subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN);
SVgObj *pVgObj = mndAcquireVgroup(pMnode, pVgEp->vgId);
if (pVgObj == NULL) {
taosMemoryFree(pReq);
terrno = TSDB_CODE_MND_VGROUP_NOT_EXIST;
return -1;
}
STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgObj);;
action.pCont = pReq;
action.contLen = sizeof(SMqVDeleteReq);
action.msgType = TDMT_VND_TMQ_DELETE_SUB;
action.acceptableCode = TSDB_CODE_MND_VGROUP_NOT_EXIST;
mndReleaseVgroup(pMnode, pVgObj);
sdbRelease(pMnode->pSdb, pVgObj);
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
return -1;
ret = -1;
goto END;
}
}
return 0;
END:
sdbRelease(pMnode->pSdb, pVgObj);
sdbCancelFetch(pMnode->pSdb, pIter);
return ret;
}
static int32_t mndDropConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgroup, char *topic){
void *pIter = NULL;
SMqConsumerObj *pConsumer = NULL;
int ret = 0;
while (1) {
pIter = sdbFetch(pMnode->pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer);
if (pIter == NULL) {
break;
}
if (strcmp(cgroup, pConsumer->cgroup) == 0 && taosArrayGetSize(pConsumer->currentTopics) == 0) {
int32_t code = mndSetConsumerDropLogs(pTrans, pConsumer);
if (code != 0) {
sdbRelease(pMnode->pSdb, pConsumer);
sdbCancelFetch(pMnode->pSdb, pIter);
return code;
// drop consumer in lost status, other consumers not in lost status already deleted by rebalance
if (pConsumer->status != MQ_CONSUMER_STATUS_LOST || strcmp(cgroup, pConsumer->cgroup) != 0) {
sdbRelease(pMnode->pSdb, pConsumer);
continue;
}
int32_t sz = taosArrayGetSize(pConsumer->assignedTopics);
for (int32_t i = 0; i < sz; i++) {
char *name = taosArrayGetP(pConsumer->assignedTopics, i);
if (strcmp(topic, name) == 0) {
int32_t code = mndSetConsumerDropLogs(pTrans, pConsumer);
if (code != 0) {
ret = code;
goto END;
}
}
}
sdbRelease(pMnode->pSdb, pConsumer);
}
return 0;
END:
sdbRelease(pMnode->pSdb, pConsumer);
sdbCancelFetch(pMnode->pSdb, pIter);
return ret;
}
static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) {
@ -996,7 +1042,7 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) {
goto end;
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TOPIC_INSIDE, pMsg, "drop-cgroup");
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "drop-cgroup");
if (pTrans == NULL) {
mError("cgroup: %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr());
code = -1;
@ -1004,7 +1050,7 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) {
}
mInfo("trans:%d, used to drop cgroup:%s on topic %s", pTrans->id, dropReq.cgroup, dropReq.topic);
mndTransSetDbName(pTrans, dropReq.topic, dropReq.cgroup);
mndTransSetDbName(pTrans, pSub->dbName, dropReq.cgroup);
code = mndTransCheckConflict(pMnode, pTrans);
if (code != 0) {
goto end;

8
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -422,14 +422,14 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
SQueryPlan *pPlan = NULL;
SMqTopicObj topicObj = {0};
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TOPIC, pReq, "create-topic");
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq, "create-topic");
if (pTrans == NULL) {
mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
code = -1;
goto _OUT;
}
mndTransSetDbName(pTrans, pCreate->name, NULL);
mndTransSetDbName(pTrans, pDb->name, NULL);
code = mndTransCheckConflict(pMnode, pTrans);
if (code != 0) {
goto _OUT;
@ -779,14 +779,14 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
}
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TOPIC, pReq, "drop-topic");
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-topic");
if (pTrans == NULL) {
mError("topic:%s, failed to drop since %s", pTopic->name, terrstr());
code = -1;
goto end;
}
mndTransSetDbName(pTrans, pTopic->name, NULL);
mndTransSetDbName(pTrans, pTopic->db, NULL);
code = mndTransCheckConflict(pMnode, pTrans);
if (code != 0) {
goto end;

34
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -836,26 +836,26 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) {
}
}
if (pNew->conflict == TRN_CONFLICT_TOPIC) {
if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true;
if (pTrans->conflict == TRN_CONFLICT_TOPIC || pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) {
if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true;
}
}
if (pNew->conflict == TRN_CONFLICT_TOPIC_INSIDE) {
if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true;
if (pTrans->conflict == TRN_CONFLICT_TOPIC) {
if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true;
}
if (pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) {
if (strcasecmp(pNew->dbname, pTrans->dbname) == 0 && strcasecmp(pNew->stbname, pTrans->stbname) == 0)
conflict = true;
}
}
// if (pNew->conflict == TRN_CONFLICT_TOPIC) {
// if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true;
// if (pTrans->conflict == TRN_CONFLICT_TOPIC || pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) {
// if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true;
// }
// }
// if (pNew->conflict == TRN_CONFLICT_TOPIC_INSIDE) {
// if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true;
// if (pTrans->conflict == TRN_CONFLICT_TOPIC) {
// if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true;
// }
// if (pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) {
// if (strcasecmp(pNew->dbname, pTrans->dbname) == 0 && strcasecmp(pNew->stbname, pTrans->stbname) == 0)
// conflict = true;
// }
// }
if (pNew->conflict == TRN_CONFLICT_ARBGROUP) {
if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true;
if (pTrans->conflict == TRN_CONFLICT_ARBGROUP) {
void *pIter = taosHashIterate(pNew->arbGroupIds, NULL);
pIter = taosHashIterate(pNew->arbGroupIds, NULL);
while (pIter != NULL) {
int32_t groupId = *(int32_t *)pIter;
if (taosHashGet(pTrans->arbGroupIds, &groupId, sizeof(int32_t)) != NULL) {

Some files were not shown because too many files have changed in this diff Show More