xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix:[TS-4921] merge from 3.0

This commit is contained in:
wangmm0220 2024-06-22 15:02:34 +08:00
parent bc4f55407d 7c748a9ebe
commit 65fac20367
232 changed files with 12051 additions and 7864 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
cmake/cmake.define
View File

@ -180,18 +180,20 @@ ELSE ()
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2")
ENDIF()
MESSAGE(STATUS "SIMD instructions (FMA/AVX/AVX2) is ACTIVATED")
ENDIF()
# IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI)
# SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi")
# SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi")
# MESSAGE(STATUS "avx512f/avx512bmi supported by compiler")
# ENDIF()
#
# IF (COMPILER_SUPPORT_AVX512VL)
# SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512vl")
# SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512vl")
# MESSAGE(STATUS "avx512vl supported by compiler")
# ENDIF()
IF ("${SIMD_AVX512_SUPPORT}" MATCHES "true")
IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi")
MESSAGE(STATUS "avx512f/avx512bmi enabled by compiler")
ENDIF()
IF (COMPILER_SUPPORT_AVX512VL)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512vl")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512vl")
MESSAGE(STATUS "avx512vl enabled by compiler")
ENDIF()
ENDIF()
# build mode

2
cmake/libuv_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# libuv
ExternalProject_Add(libuv
GIT_REPOSITORY https://github.com/libuv/libuv.git
GIT_TAG v1.44.2
GIT_TAG v1.48.0
SOURCE_DIR "${TD_CONTRIB_DIR}/libuv"
BINARY_DIR "${TD_CONTRIB_DIR}/libuv"
CONFIGURE_COMMAND ""

2
cmake/taostools_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taos-tools
ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG main
GIT_TAG 3.0
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE

4
docs/en/10-deployment/03-k8s.md
View File

@ -243,7 +243,7 @@ curl -u root:taosdata -d "show databases" 127.0.0.1:6041/rest/sql
{"code":0,"column_meta":[["name","VARCHAR",64]],"data":[["information_schema"],["performance_schema"],["test"],["test1"]],"rows":4}
```
## Test cluster
## Test cluster
### Data preparation
@ -335,7 +335,7 @@ tdengine-1 1/1 Running 1 (6m48s ago) 20m 10.244.0.59 node84
tdengine-2 1/1 Running 0 21m 10.244.1.223 node85 <none> <none>
```
At this time, the cluster mnode has a re-election, and the monde on dnode1 becomes the leader.
At this time, the cluster mnode has a re-election, and the monde on dnode2 becomes the leader.
```Bash
kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show mnodes\G"

40
docs/en/12-taos-sql/03-table.md
View File

@ -25,7 +25,7 @@ create_definition:
col_name column_definition
column_definition:
type_name [comment 'string_value'] [PRIMARY KEY] [ENCODE 'encode_type'] [COMPRESS 'compress_type'] [LEVEL 'level_type']
type_name [comment 'string_value'] [PRIMARY KEY] [ENCODE 'encode_type'] [COMPRESS 'compress_type'] [LEVEL 'level_type']
table_options:
table_option ...
@ -52,9 +52,9 @@ table_option: {
**Parameter description**
1. COMMENT: specifies comments for the table. This parameter can be used with supertables, standard tables, and subtables.
1. COMMENT: specifies comments for the table. This parameter can be used with supertables, standard tables, and subtables. The maximum length of the comment is 1024 bytes.
2. SMA: specifies functions on which to enable small materialized aggregates (SMA). SMA is user-defined precomputation of aggregates based on data blocks. Enter one of the following values: max, min, or sum This parameter can be used with supertables and standard tables.
3. TTL: specifies the time to live (TTL) for the table. If TTL is specified when creatinga table, after the time period for which the table has been existing is over TTL, TDengine will automatically delete the table. Please be noted that the system may not delete the table at the exact moment that the TTL expires but guarantee there is such a system and finally the table will be deleted. The unit of TTL is in days. The default value is 0, i.e. never expire.
3. TTL: specifies the time to live (TTL) for the table. If TTL is specified when creatinga table, after the time period for which the table has been existing is over TTL, TDengine will automatically delete the table. Please be noted that the system may not delete the table at the exact moment that the TTL expires but guarantee there is such a system and finally the table will be deleted. The unit of TTL is in days. The value range is [0, 2147483647]. The default value is 0, i.e. never expire.
## Create Subtables
@ -112,6 +112,11 @@ You can perform the following modifications on existing tables:
4. RENAME COLUMN: renames a specified column in the table.
5. The primary key column of a table cannot be modified or added or deleted using ADD/DROP COLUMN.
**Parameter description**
1. COMMENT: specifies comments for the table. This parameter can be used with supertables, standard tables, and subtables. The maximum length of the comment is 1024 bytes.
2. TTL: specifies the time to live (TTL) for the table. If TTL is specified when creatinga table, after the time period for which the table has been existing is over TTL, TDengine will automatically delete the table. Please be noted that the system may not delete the table at the exact moment that the TTL expires but guarantee there is such a system and finally the table will be deleted. The unit of TTL is in days. The value range is [0, 2147483647]. The default value is 0, i.e. never expire.
### Add a Column
```sql
@ -136,6 +141,18 @@ ALTER TABLE tb_name MODIFY COLUMN field_name data_type(length);
ALTER TABLE tb_name RENAME COLUMN old_col_name new_col_name
```
### Alter Table TTL
```sql
ALTER TABLE tb_name TTL value
```
### Alter Table Comment
```sql
ALTER TABLE tb_name COMMENT 'string_value'
```
## Modify a Subtable
```sql
@ -159,12 +176,29 @@ alter_table_option: {
1. Only the value of a tag can be modified directly. For all other modifications, you must modify the supertable from which the subtable was created.
**Parameter description**
1. COMMENT: specifies comments for the table. This parameter can be used with supertables, standard tables, and subtables. The maximum length of the comment is 1024 bytes.
2. TTL: specifies the time to live (TTL) for the table. If TTL is specified when creatinga table, after the time period for which the table has been existing is over TTL, TDengine will automatically delete the table. Please be noted that the system may not delete the table at the exact moment that the TTL expires but guarantee there is such a system and finally the table will be deleted. The unit of TTL is in days. The value range is [0, 2147483647]. The default value is 0, i.e. never expire.
### Change Tag Value Of Sub Table
```
ALTER TABLE tb_name SET TAG tag_name=new_tag_value;
```
### Alter Table TTL
```sql
ALTER TABLE tb_name TTL value
```
### Alter Table Comment
```sql
ALTER TABLE tb_name COMMENT 'string_value'
```
## Delete a Table
The following SQL statement deletes one or more tables.

2
docs/en/12-taos-sql/14-stream.md
View File

@ -47,7 +47,7 @@ window_clause: {
}
```
`SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically.
`SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically.The `_wend` of this window is the time of the last data plus `tol_val`.
`EVENT_WINDOW` is determined according to the window start condition and the window close condition. The window is started when `start_trigger_condition` is evaluated to true, the window is closed when `end_trigger_condition` is evaluated to true. `start_trigger_condition` and `end_trigger_condition` can be any conditional expressions supported by TDengine and can include multiple columns.

4
docs/en/12-taos-sql/20-keywords.md
View File

@ -151,7 +151,7 @@ The following list shows all reserved keywords:
- INTERVAL
- INTO
- IS
- ISNULL
- IS NULL
### J
@ -197,7 +197,7 @@ The following list shows all reserved keywords:
- NMATCH
- NONE
- NOT
- NOTNULL
- NOT NULL
- NOW
- NULL
- NULLS

20
docs/en/12-taos-sql/25-grant.md
View File

@ -39,10 +39,10 @@ This is an example:
```sql
taos> show users;
name | super | enable | sysinfo | create_time |
================================================================================
test | 0 | 1 | 1 | 2022-08-29 15:10:27.315 |
root | 1 | 1 | 1 | 2022-08-29 15:03:34.710 |
name | super | enable | sysinfo | createdb | create_time | allowed_host |
=========================================================================================================
test | 0 | 1 | 1 | 0 |2022-08-29 15:10:27.315 | 127.0.0.1 |
root | 1 | 1 | 1 | 1 |2022-08-29 15:03:34.710 | 127.0.0.1 |
Query OK, 2 rows in database (0.001657s)
```
@ -50,10 +50,10 @@ Alternatively, you can get the user information by querying a built-in table, IN
```sql
taos> select * from information_schema.ins_users;
name | super | enable | sysinfo | create_time |
================================================================================
test | 0 | 1 | 1 | 2022-08-29 15:10:27.315 |
root | 1 | 1 | 1 | 2022-08-29 15:03:34.710 |
name | super | enable | sysinfo | createdb | create_time | allowed_host |
=========================================================================================================
test | 0 | 1 | 1 | 0 |2022-08-29 15:10:27.315 | 127.0.0.1 |
root | 1 | 1 | 1 | 1 |2022-08-29 15:03:34.710 | 127.0.0.1 |
Query OK, 2 rows in database (0.001953s)
```
@ -67,17 +67,19 @@ DROP USER user_name;
```sql
ALTER USER user_name alter_user_clause
alter_user_clause: {
PASS 'literal'
| ENABLE value
| SYSINFO value
| CREATEDB value
}
```
- PASS: Modify the user password.
- ENABLE: Specify whether the user is enabled or disabled. 1 indicates enabled and 0 indicates disabled.
- SYSINFO: Specify whether the user can query system information. 1 indicates that the user can query system information and 0 indicates that the user cannot query system information.
- CREATEDB: Specify whether the user can create databases. 1 indicates that the user can create databases and 0 indicates that the user cannot create databases.
For example, you can use below command to disable user `test`:

2
docs/en/14-reference/12-config/index.md
View File

@ -432,7 +432,7 @@ The charset that takes effect is UTF-8.
| Applicable | Server Only |
| Meaning | Maximum number of threads to commit |
| Value Range | 0-1024 |
| Default Value | |
| Default Value | 4 |
## Log Parameters

14
docs/en/21-tdinternal/08-compress.md
View File

@ -1,12 +1,8 @@
---
title: Configurable Column Compression
description: Configurable column storage compression method
---
# Configurable Storage Compression
Since TDengine 3.3.0.0, more advanced compression feature is introduced, you can specify compression or not, the compression method and compression level for each column.
## Compression Terminology Definition
@ -32,16 +28,14 @@ In this article, it specifically refers to the level within the secondary compre
- Default compression algorithm list and applicable range for each data type
| Data Type | Optional Encoding Algorithm | Default Encoding Algorithm | Optional Compression Algorithm|Default Compression Algorithm| Default Compression Level|
| Data Type | Optional Encoding Algorithm | Default Encoding Algorithm | Optional Compression Algorithm|Default Compression Algorithm| Default Compression Level|
| :-----------:|:----------:|:-------:|:-------:|:----------:|:----:|
tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium|
| tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium|
| bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium|
|float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|tsz| medium|
|float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|lz4| medium|
|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| medium|
|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| medium|
Note: For floating point types, if configured as tsz, its precision is determined by the global configuration of taosd. If configured as tsz, but the lossy compression flag is not configured, lz4 is used for compression by default.
## SQL
### Create Table with Compression
@ -76,7 +70,7 @@ ALTER TABLE [db_name.]tabName MODIFY COLUMN colName [ENCODE 'ecode_type'] [COMPR
- Change the compression method of the column
### View Compression Dethod
### View Compression Method
```sql
DESCRIBE [dbname.]tabName

40
docs/zh/10-deployment/03-k8s.md
View File

@ -10,9 +10,9 @@ description: 利用 Kubernetes 部署 TDengine 集群的详细指南
为了满足[高可用](https://docs.taosdata.com/tdinternal/high-availability/)的需求，集群需要满足如下要求：
- 3个及以上 dnode ：TDengine 的同一个 vgroup 中的多个 vnode ，不允许同时分布在一个 dnode ，所以如果创建3副本的数据库，则 dnode 数大于等于3
- 3个 mnode ：mnode 负责整个集群的管理工作，TDengine 默认是一个 mnode。如果这个 mnode 所在的 dnode 掉线，则整个集群不可用。
- 数据库的3副本：TDengine 的副本配置是数据库级别，所以数据库3副本可满足在3个 dnode 的集群中，任意一个 dnode 下线，都不影响集群的正常使用。**如果下线** **dnode** **个数为2时，此时集群不可用，****因为****RAFT无法完成选举****。**（企业版：在灾难恢复场景，任一节点数据文件损坏，都可以通过重新拉起dnode进行恢复）
- 3 个及以上 dnode ：TDengine 的同一个 vgroup 中的多个 vnode ，不允许同时分布在一个 dnode ，所以如果创建 3 副本的数据库，则 dnode 数大于等于 3
- 3 个 mnode ：mnode 负责整个集群的管理工作，TDengine 默认是一个 mnode。如果这个 mnode 所在的 dnode 掉线，则整个集群不可用。
- 数据库的 3 副本：TDengine 的副本配置是数据库级别，所以数据库 3 副本可满足在 3 个 dnode 的集群中，任意一个 dnode 下线，都不影响集群的正常使用。**如果下线** **dnode** **个数为 2 时，此时集群不可用，\*\***因为\***\*RAFT 无法完成选举\*\***。\*\*（企业版：在灾难恢复场景，任一节点数据文件损坏，都可以通过重新拉起 dnode 进行恢复）
## 前置条件
@ -52,7 +52,7 @@ spec:
根据 Kubernetes 对各类部署的说明，我们将使用 StatefulSet 作为 TDengine 的部署资源类型。 创建文件 `tdengine.yaml`，其中 replicas 定义集群节点的数量为 3。节点时区为中国（Asia/Shanghai），每个节点分配 5G 标准（standard）存储（参考[Storage Classes](https://kubernetes.io/docs/concepts/storage/storage-classes/) 配置 storage class ）。你也可以根据实际情况进行相应修改。
请特别注意startupProbe的配置，在 dnode 的 Pod 掉线一段时间后，再重新启动，这个时候新上线的 dnode 会短暂不可用。如果startupProbe配置过小，Kubernetes 会认为该 Pod 处于不正常的状态，并尝试重启该 Pod，该 dnode 的 Pod 会频繁重启，始终无法恢复到正常状态。参考 [Configure Liveness, Readiness and Startup Probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/)
请特别注意 startupProbe 的配置，在 dnode 的 Pod 掉线一段时间后，再重新启动，这个时候新上线的 dnode 会短暂不可用。如果 startupProbe 配置过小，Kubernetes 会认为该 Pod 处于不正常的状态，并尝试重启该 Pod，该 dnode 的 Pod 会频繁重启，始终无法恢复到正常状态。参考 [Configure Liveness, Readiness and Startup Probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/)
```YAML
---
@ -176,7 +176,7 @@ taos> show dnodes
Query OK, 3 row(s) in set (0.001853s)
```
查看当前mnode
查看当前 mnode
```Bash
kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show mnodes\G"
@ -191,14 +191,14 @@ reboot_time: 2023-07-19 17:54:19.520
Query OK, 1 row(s) in set (0.001282s)
```
## 创建mnode
## 创建 mnode
```Bash
kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "create mnode on dnode 2"
kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "create mnode on dnode 3"
```
查看mnode
查看 mnode
```Bash
kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show mnodes\G"
@ -249,7 +249,7 @@ curl -u root:taosdata -d "show databases" 127.0.0.1:6041/rest/sql
#### taosBenchmark
通过taosBenchmark 创建一个3副本的数据库，同时写入1亿条数据，同时查看数据
通过 taosBenchmark 创建一个 3 副本的数据库，同时写入 1 亿条数据，同时查看数据
```Bash
kubectl exec -it tdengine-0 -n tdengine-test -- taosBenchmark -I stmt -d test -n 10000 -t 10000 -a 3
@ -264,7 +264,7 @@ taos> select count(*) from test.meters;
Query OK, 1 row(s) in set (0.103537s)
```
查看vnode分布，通过show dnodes
查看 vnode 分布，通过 show dnodes
```Bash
kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes"
@ -278,7 +278,7 @@ taos> show dnodes
Query OK, 3 row(s) in set (0.001357s)
```
通过show vgroup 查看 vnode 分布情况
通过 show vgroup 查看 vnode 分布情况
```Bash
kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show test.vgroups"
@ -299,7 +299,7 @@ Query OK, 8 row(s) in set (0.001488s)
#### 手工创建
常见一个三副本的test1，并创建一张表，写入2条数据
常见一个三副本的 test1，并创建一张表，写入 2 条数据
```Bash
kubectl exec -it tdengine-0 -n tdengine-test -- \
@ -310,7 +310,7 @@ kubectl exec -it tdengine-0 -n tdengine-test -- \
insert into t1 values(now, 1)(now+1s, 2);"
```
通过show test1.vgroup 查看xnode分布情况
通过 show test1.vgroup 查看 xnode 分布情况
```Bash
kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show test1.vgroups"
@ -335,7 +335,7 @@ tdengine-1 1/1 Running 1 (6m48s ago) 20m 10.244.0.59 node84
tdengine-2 1/1 Running 0 21m 10.244.1.223 node85 <none> <none>
```
此时集群mnode发生重新选举，dnode1上的monde 成为leader
此时集群 mnode 发生重新选举，dnode2 上的 monde 成为 leader
```Bash
kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show mnodes\G"
@ -389,7 +389,7 @@ taos> select *from test1.t1
Query OK, 4 row(s) in set (0.001994s)
```
同理，至于非leader得mnode掉线，读写当然可以正常进行，这里就不做过多的展示。
同理，至于非 leader 得 mnode 掉线，读写当然可以正常进行，这里就不做过多的展示。
## 集群扩容
@ -436,7 +436,7 @@ Query OK, 4 row(s) in set (0.003628s)
## 集群缩容
由于 TDengine 集群在扩缩容时会对数据进行节点间迁移，使用 kubectl 命令进行缩容需要首先使用 "drop dnodes" 命令（**如果集群中存在3副本的db，那么缩容后的** **dnode** **个数也要必须大于等于3，否则drop dnode操作会被中止**），然后再节点删除完成后再进行 Kubernetes 集群缩容。
由于 TDengine 集群在扩缩容时会对数据进行节点间迁移，使用 kubectl 命令进行缩容需要首先使用 "drop dnodes" 命令（**如果集群中存在 3 副本的 db，那么缩容后的** **dnode** **个数也要必须大于等于 3，否则 drop dnode 操作会被中止**），然后再节点删除完成后再进行 Kubernetes 集群缩容。
注意：由于 Kubernetes Statefulset 中 Pod 的只能按创建顺序逆序移除，所以 TDengine drop dnode 也需要按照创建顺序逆序移除，否则会导致 Pod 处于错误状态。
@ -459,7 +459,7 @@ Query OK, 3 row(s) in set (0.003324s)
kubectl scale statefulsets tdengine --replicas=3 -n tdengine-test
```
最后一个 POD 将会被删除。使用命令 kubectl get pods -l app=tdengine 查看POD状态：
最后一个 POD 将会被删除。使用命令 kubectl get pods -l app=tdengine 查看 POD 状态：
```Plain
kubectl get pod -l app=tdengine -n tdengine-test -o wide
@ -469,7 +469,7 @@ tdengine-1 1/1 Running 1 (7h9m ago) 7h23m 10.244.0.59 node84 <
tdengine-2 1/1 Running 0 5h45m 10.244.1.224 node85 <none> <none>
```
POD删除后，需要手动删除PVC，否则下次扩容时会继续使用以前的数据导致无法正常加入集群。
POD 删除后，需要手动删除 PVC，否则下次扩容时会继续使用以前的数据导致无法正常加入集群。
```Bash
kubectl delete pvc aosdata-tdengine-3 -n tdengine-test
@ -502,7 +502,7 @@ Query OK, 4 row(s) in set (0.003881s)
## 清理 TDengine 集群
> **删除pvc时需要注意下pv persistentVolumeReclaimPolicy策略，建议改为Delete，这样在删除pvc时才会自动清理pv，同时会清理底层的csi存储资源，如果没有配置删除pvc自动清理pv的策略，再删除pvc后，在手动清理pv时，pv对应的csi存储资源可能不会被释放。**
> **删除 pvc 时需要注意下 pv persistentVolumeReclaimPolicy 策略，建议改为 Delete，这样在删除 pvc 时才会自动清理 pv，同时会清理底层的 csi 存储资源，如果没有配置删除 pvc 自动清理 pv 的策略，再删除 pvc 后，在手动清理 pv 时，pv 对应的 csi 存储资源可能不会被释放。**
完整移除 TDengine 集群，需要分别清理 statefulset、svc、configmap、pvc。
@ -537,8 +537,8 @@ Query OK, 4 row(s) in set (0.003862s)
对于在 Kubernetes 环境下 TDengine 的高可用和高可靠来说，对于硬件损坏、灾难恢复，分为两个层面来讲：
1. 底层的分布式块存储具备的灾难恢复能力，块存储的多副本，当下流行的分布式块存储如 Ceph，就具备多副本能力，将存储副本扩展到不同的机架、机柜、机房、数据中心（或者直接使用公有云厂商提供的块存储服务）
2. TDengine的灾难恢复，在 TDengine Enterprise 中，本身具备了当一个 dnode 永久下线（物理机磁盘损坏，数据分拣丢失）后，重新拉起一个空白的dnode来恢复原dnode的工作。
2. TDengine 的灾难恢复，在 TDengine Enterprise 中，本身具备了当一个 dnode 永久下线（物理机磁盘损坏，数据分拣丢失）后，重新拉起一个空白的 dnode 来恢复原 dnode 的工作。
最后，欢迎使用[TDengine Cloud](https://cloud.taosdata.com/)，来体验一站式全托管的TDengine云服务。
最后，欢迎使用[TDengine Cloud](https://cloud.taosdata.com/)，来体验一站式全托管的 TDengine 云服务。
> TDengine Cloud 是一个极简的全托管时序数据处理云服务平台，它是基于开源的时序数据库 TDengine 而开发的。除高性能的时序数据库之外，它还具有缓存、订阅和流计算等系统功能，而且提供了便利而又安全的数据分享、以及众多的企业级功能。它可以让物联网、工业互联网、金融、IT 运维监控等领域企业在时序数据的管理上大幅降低人力成本和运营成本。

42
docs/zh/12-taos-sql/03-table.md
View File

@ -23,10 +23,10 @@ create_subtable_clause: {
}
create_definition:
col_name column_definition
col_name column_definition
column_definition:
type_name [comment 'string_value'] [PRIMARY KEY] [ENCODE 'encode_type'] [COMPRESS 'compress_type'] [LEVEL 'level_type']
type_name [comment 'string_value'] [PRIMARY KEY] [ENCODE 'encode_type'] [COMPRESS 'compress_type'] [LEVEL 'level_type']
table_options:
table_option ...
@ -52,9 +52,9 @@ table_option: {
**参数说明**
1. COMMENT：表注释。可用于超级表、子表和普通表。
1. COMMENT：表注释。可用于超级表、子表和普通表。最大长度为 1024 个字节。
2. SMA：Small Materialized Aggregates，提供基于数据块的自定义预计算功能。预计算类型包括 MAX、MIN 和 SUM。可用于超级表/普通表。
3. TTL：Time to Live，是用户用来指定表的生命周期的参数。如果创建表时指定了这个参数，当该表的存在时间超过 TTL 指定的时间后，TDengine 自动删除该表。这个 TTL 的时间只是一个大概时间，系统不保证到了时间一定会将其删除，而只保证存在这样一个机制且最终一定会删除。TTL 单位是天，默认为 0，表示不限制，到期时间为表创建时间加上 TTL 时间。TTL 与数据库 KEEP 参数没有关联，如果 KEEP 比 TTL 小，在表被删除之前数据也可能已经被删除。
3. TTL：Time to Live，是用户用来指定表的生命周期的参数。如果创建表时指定了这个参数，当该表的存在时间超过 TTL 指定的时间后，TDengine 自动删除该表。这个 TTL 的时间只是一个大概时间，系统不保证到了时间一定会将其删除，而只保证存在这样一个机制且最终一定会删除。TTL 单位是天，取值范围为[0, 2147483647]，默认为 0，表示不限制，到期时间为表创建时间加上 TTL 时间。TTL 与数据库 KEEP 参数没有关联，如果 KEEP 比 TTL 小，在表被删除之前数据也可能已经被删除。
## 创建子表
@ -112,6 +112,11 @@ alter_table_option: {
4. RENAME COLUMN：修改列名称。
5. 普通表的主键列不能被修改，也不能通过 ADD/DROP COLUMN 来添加/删除主键列。
**参数说明**
1. COMMENT：表注释。可用于超级表、子表和普通表。最大长度为 1024 个字节。
2. TTL：Time to Live，是用户用来指定表的生命周期的参数。如果创建表时指定了这个参数，当该表的存在时间超过 TTL 指定的时间后，TDengine 自动删除该表。这个 TTL 的时间只是一个大概时间，系统不保证到了时间一定会将其删除，而只保证存在这样一个机制且最终一定会删除。TTL 单位是天，取值范围为[0, 2147483647]，默认为 0，表示不限制，到期时间为表创建时间加上 TTL 时间。TTL 与数据库 KEEP 参数没有关联，如果 KEEP 比 TTL 小，在表被删除之前数据也可能已经被删除。
### 增加列
```sql
@ -136,6 +141,18 @@ ALTER TABLE tb_name MODIFY COLUMN field_name data_type(length);
ALTER TABLE tb_name RENAME COLUMN old_col_name new_col_name
```
### 修改表生命周期
```sql
ALTER TABLE tb_name TTL value
```
### 修改表注释
```sql
ALTER TABLE tb_name COMMENT 'string_value'
```
## 修改子表
```sql
@ -159,12 +176,29 @@ alter_table_option: {
1. 对子表的列和标签的修改，除了更改标签值以外，都要通过超级表才能进行。
**参数说明**
1. COMMENT：表注释。可用于超级表、子表和普通表。最大长度为 1024 个字节。
2. TTL：Time to Live，是用户用来指定表的生命周期的参数。如果创建表时指定了这个参数，当该表的存在时间超过 TTL 指定的时间后，TDengine 自动删除该表。这个 TTL 的时间只是一个大概时间，系统不保证到了时间一定会将其删除，而只保证存在这样一个机制且最终一定会删除。TTL 单位是天，取值范围为[0, 2147483647]，默认为 0，表示不限制，到期时间为表创建时间加上 TTL 时间。TTL 与数据库 KEEP 参数没有关联，如果 KEEP 比 TTL 小，在表被删除之前数据也可能已经被删除。
### 修改子表标签值
```
ALTER TABLE tb_name SET TAG tag_name=new_tag_value;
```
### 修改表生命周期
```sql
ALTER TABLE tb_name TTL value
```
### 修改表注释
```sql
ALTER TABLE tb_name COMMENT 'string_value'
```
## 删除表
可以在一条 SQL 语句中删除一个或多个普通表或子表。

4
docs/zh/12-taos-sql/14-stream.md
View File

@ -54,8 +54,10 @@ window_clause: {
}
```
其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。
其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。该窗口的 _wend 等于最后一条数据的时间加上 tol_val。
EVENT_WINDOW 是事件窗口，根据开始条件和结束条件来划定窗口。当 start_trigger_condition 满足时则窗口开始，直到 end_trigger_condition 满足时窗口关闭。 start_trigger_condition 和 end_trigger_condition 可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。
COUNT_WINDOW 是计数窗口,按固定的数据行数来划分窗口。 count_val 是常量，是正整数，必须大于等于2，小于2147483648。 count_val 表示每个 COUNT_WINDOW 包含的最大数据行数，总数据行数不能整除 count_val 时，最后一个窗口的行数会小于 count_val 。 sliding_val 是常量，表示窗口滑动的数量，类似于 INTERVAL 的 SLIDING 。
窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished)

4
docs/zh/12-taos-sql/20-keywords.md
View File

@ -151,7 +151,7 @@ description: TDengine 保留关键字的详细列表
- INTERVAL
- INTO
- IS
- ISNULL
- IS NULL
### J
@ -197,7 +197,7 @@ description: TDengine 保留关键字的详细列表
- NMATCH
- NONE
- NOT
- NOTNULL
- NOT NULL
- NOW
- NULL
- NULLS

91
docs/zh/12-taos-sql/25-grant.md Normal file
View File

@ -0,0 +1,91 @@
---
title: 用户管理
sidebar_label: 用户管理
description: 本节讲述基本的用户管理功能
---
用户和权限管理是 TDengine 企业版的功能，本节只讲述基本的用户管理部分。要想了解和获取全面的权限管理功能，请联系 TDengine 销售团队。
## 创建用户
```sql
CREATE USER user_name PASS 'password' [SYSINFO {1|0}];
```
用户名最长不超过 23 个字节。
密码最长不超过 31 个字节。密码可以包含字母、数字以及除单引号、双引号、反引号、反斜杠和空格以外的特殊字符，密码不能为空字符串。
`SYSINFO` 表示该用户是否能够查看系统信息。`1` 表示可以查看，`0` 表示无权查看。系统信息包括服务配置、dnode、vnode、存储等信息。缺省值为 `1`。
在下面的示例中，我们创建一个密码为 `123456` 且可以查看系统信息的用户。
```sql
taos> create user test pass '123456' sysinfo 1;
Query OK, 0 of 0 rows affected (0.001254s)
```
## 查看用户
可以使用如下命令查看系统中的用户。
```sql
SHOW USERS;
```
以下是示例：
```sql
taos> show users;
name | super | enable | sysinfo | createdb | create_time | allowed_host |
=========================================================================================================
test | 0 | 1 | 1 | 0 |2022-08-29 15:10:27.315 | 127.0.0.1 |
root | 1 | 1 | 1 | 1 |2022-08-29 15:03:34.710 | 127.0.0.1 |
Query OK, 2 rows in database (0.001657s)
```
或者，可以查询内置系统表 INFORMATION_SCHEMA.INS_USERS 来获取用户信息。
```sql
taos> select * from information_schema.ins_users;
name | super | enable | sysinfo | createdb | create_time | allowed_host |
=========================================================================================================
test | 0 | 1 | 1 | 0 |2022-08-29 15:10:27.315 | 127.0.0.1 |
root | 1 | 1 | 1 | 1 |2022-08-29 15:03:34.710 | 127.0.0.1 |
Query OK, 2 rows in database (0.001953s)
```
## 删除用户
```sql
DROP USER user_name;
```
## 修改用户配置
```sql
ALTER USER user_name alter_user_clause
alter_user_clause: {
PASS 'literal'
| ENABLE value
| SYSINFO value
| CREATEDB value
}
```
- PASS: 修改密码，后跟新密码
- ENABLE: 启用或禁用该用户，`1` 表示启用，`0` 表示禁用
- SYSINFO: 允许或禁止查看系统信息，`1` 表示允许，`0` 表示禁止
- CREATEDB: 允许或禁止创建数据库，`1` 表示允许，`0` 表示禁止
下面的示例禁用了名为 `test` 的用户:
```sql
taos> alter user test enable 0;
Query OK, 0 of 0 rows affected (0.001160s)
```
## 授权管理
授权管理仅在 TDengine 企业版中可用，请联系 TDengine 销售团队。

2
docs/zh/14-reference/12-config/index.md
View File

@ -430,7 +430,7 @@ charset 的有效值是 UTF-8。
| 适用范围 | 仅服务端适用 |
| 含义 | 设置写入线程的最大数量 |
| 取值范围 | 0-1024 |
| 缺省值 | |
| 缺省值 | 4 |
## 日志相关

10
docs/zh/21-tdinternal/08-compress.md
View File

@ -3,8 +3,6 @@ title: 可配置压缩算法
description: 可配置压缩算法
---
# 可配置存储压缩
从 TDengine 3.3.0.0 版本开始，TDengine 提供了更高级的压缩功能，用户可以在建表时针对每一列配置是否进行压缩、以及使用的压缩算法和压缩级别。
## 压缩术语定义
@ -30,16 +28,14 @@ description: 可配置压缩算法
- 各个数据类型的默认压缩算法列表和适用范围
| 数据类型 | 可选编码算法 | 编码算法默认值 | 可选压缩算法|可选压缩算法| 压缩等级默认值|
| 数据类型 | 可选编码算法 | 编码算法默认值 | 可选压缩算法|压缩算法默认值| 压缩等级默认值|
| :-----------:|:----------:|:-------:|:-------:|:----------:|:----:|
tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium|
| tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium|
| bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium|
|float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|tsz| medium|
|float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|lz4| medium|
|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| medium|
|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| medium|
注意: 针对浮点类型，如果配置为tsz, 其精度由taosd的全局配置决定，如果配置为tsz, 但是没有配置有损压缩标志, 则使用lz4进行压缩
## SQL 语法
### 建表时指定压缩

2
include/client/taos.h
View File

@ -366,7 +366,7 @@ typedef enum {
} TSDB_SERVER_STATUS;
DLL_EXPORT TSDB_SERVER_STATUS taos_check_server_status(const char *fqdn, int port, char *details, int maxlen);
DLL_EXPORT char* getBuildInfo();
#ifdef __cplusplus
}
#endif

1
include/common/systable.h
View File

@ -40,6 +40,7 @@ extern "C" {
#define TSDB_INS_TABLE_COLS "ins_columns"
#define TSDB_INS_TABLE_TABLE_DISTRIBUTED "ins_table_distributed"
#define TSDB_INS_TABLE_USERS "ins_users"
#define TSDB_INS_TABLE_USERS_FULL "ins_users_full"
#define TSDB_INS_TABLE_LICENCES "ins_grants"
#define TSDB_INS_TABLE_VGROUPS "ins_vgroups"
#define TSDB_INS_TABLE_VNODES "ins_vnodes"

4
include/common/taosdef.h
View File

@ -30,10 +30,6 @@ typedef int64_t tb_uid_t;
#define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX))
#define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey))
//define show cluster alive and show db.alive
#define SHOW_STATUS_NOT_AVAILABLE 0
#define SHOW_STATUS_AVAILABLE 1
#define SHOW_STATUS_HALF_AVAILABLE 2
typedef enum {
TSDB_SUPER_TABLE = 1, // super table

2
include/common/tcommon.h
View File

@ -240,7 +240,7 @@ typedef struct SDataBlockInfo {
} SDataBlockInfo;
typedef struct SSDataBlock {
SColumnDataAgg** pBlockAgg;
SColumnDataAgg* pBlockAgg;
SArray* pDataBlock; // SArray<SColumnInfoData>
SDataBlockInfo info;
} SSDataBlock;

4
include/common/tdatablock.h
View File

@ -102,7 +102,7 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
return false;
}
if (pColAgg != NULL) {
if (pColAgg != NULL && pColAgg->colId != -1) {
if (pColAgg->numOfNull == totalRows) {
ASSERT(pColumnInfoData->nullbitmap == NULL);
return true;
@ -282,6 +282,8 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbName, uint64_t groupId, char* p
void trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList);
void copyPkVal(SDataBlockInfo* pDst, const SDataBlockInfo* pSrc);
#ifdef __cplusplus
}
#endif

1
include/common/tglobal.h
View File

@ -87,6 +87,7 @@ extern int32_t tsNumOfQnodeFetchThreads;
extern int32_t tsNumOfSnodeStreamThreads;
extern int32_t tsNumOfSnodeWriteThreads;
extern int64_t tsRpcQueueMemoryAllowed;
extern int32_t tsRetentionSpeedLimitMB;
// sync raft
extern int32_t tsElectInterval;

12
include/common/tgrant.h
View File

@ -28,11 +28,13 @@ extern "C" {
#define GRANTS_COL_MAX_LEN 196
#endif
#define GRANT_HEART_BEAT_MIN 2
#define GRANT_ACTIVE_CODE "activeCode"
#define GRANT_FLAG_ALL (0x01)
#define GRANT_FLAG_AUDIT (0x02)
#define GRANT_FLAG_VIEW (0x04)
#define GRANT_HEART_BEAT_MIN 2
#define GRANT_EXPIRE_VALUE (31556995201)
#define GRANT_EXPIRE_UNLIMITED(v) ((v) == GRANT_EXPIRE_VALUE)
#define GRANT_ACTIVE_CODE "activeCode"
#define GRANT_FLAG_ALL (0x01)
#define GRANT_FLAG_AUDIT (0x02)
#define GRANT_FLAG_VIEW (0x04)
typedef enum {
TSDB_GRANT_ALL,

19
include/common/tmsg.h
View File

@ -158,6 +158,7 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_MACHINES,
TSDB_MGMT_TABLE_ARBGROUP,
TSDB_MGMT_TABLE_ENCRYPTIONS,
TSDB_MGMT_TABLE_USER_FULL,
TSDB_MGMT_TABLE_MAX,
} EShowType;
@ -362,6 +363,7 @@ typedef enum ENodeType {
QUERY_NODE_SHOW_TABLES_STMT,
QUERY_NODE_SHOW_TAGS_STMT,
QUERY_NODE_SHOW_USERS_STMT,
QUERY_NODE_SHOW_USERS_FULL_STMT,
QUERY_NODE_SHOW_LICENCES_STMT,
QUERY_NODE_SHOW_VGROUPS_STMT,
QUERY_NODE_SHOW_TOPICS_STMT,
@ -1026,6 +1028,8 @@ typedef struct {
SIpV4Range* pIpRanges;
int32_t sqlLen;
char* sql;
int8_t isImport;
int8_t createDb;
} SCreateUserReq;
int32_t tSerializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq);
@ -1063,10 +1067,10 @@ typedef struct {
int8_t enable;
int8_t isView;
union {
int8_t flag;
uint8_t flag;
struct {
int8_t createdb : 1;
int8_t reserve : 7;
uint8_t createdb : 1;
uint8_t reserve : 7;
};
};
char user[TSDB_USER_LEN];
@ -2127,6 +2131,7 @@ typedef struct {
char filterTb[TSDB_TABLE_NAME_LEN]; // for ins_columns
int64_t showId;
int64_t compactId; // for compact
bool withFull; // for show users full
} SRetrieveTableReq;
typedef struct SSysTableSchema {
@ -3511,9 +3516,9 @@ typedef struct SVUpdateCheckpointInfoReq {
int64_t checkpointVer;
int64_t checkpointTs;
int32_t transId;
int8_t dropRelHTask;
int64_t hStreamId;
int64_t hStreamId; // add encode/decode
int64_t hTaskId;
int8_t dropRelHTask;
} SVUpdateCheckpointInfoReq;
typedef struct {
@ -3666,10 +3671,6 @@ typedef struct {
int32_t taskId;
} SVPauseStreamTaskReq, SVResetStreamTaskReq;
typedef struct {
int8_t reserved;
} SVPauseStreamTaskRsp;
typedef struct {
char name[TSDB_STREAM_FNAME_LEN];
int8_t igNotExists;

5
include/common/tmsgdef.h
View File

@ -225,9 +225,9 @@
TD_DEF_MSG_TYPE(TDMT_MND_RESTORE_DNODE, "restore-dnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_PAUSE_STREAM, "pause-stream", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_RESUME_STREAM, "resume-stream", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_TIMER, "stream-checkpoint-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_UPDATE_CHKPT_EVT, "stream-update-chkpt-evt", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_BEGIN_CHECKPOINT, "stream-begin-checkpoint", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE, "stream-checkpoint-remain", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHKPT_REPORT, "stream-chkpt-report", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_NODECHANGE_CHECK, "stream-nodechange-check", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_TRIM_DB_TIMER, "trim-db-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_GRANT_NOTIFY, "grant-notify", NULL, NULL)
@ -390,6 +390,7 @@
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_RESET, "vnode-stream-reset", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_UNUSED, "vnd-stream-unused", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_GET_STREAM_PROGRESS, "vnd-stream-progress", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_VND_STREAM_MSG)

685
include/common/ttokendef.h
View File

@ -50,348 +50,349 @@
#define TK_STATE 32
#define TK_NK_COMMA 33
#define TK_HOST 34
#define TK_USER 35
#define TK_ENABLE 36
#define TK_NK_INTEGER 37
#define TK_SYSINFO 38
#define TK_CREATEDB 39
#define TK_ADD 40
#define TK_DROP 41
#define TK_GRANT 42
#define TK_ON 43
#define TK_TO 44
#define TK_REVOKE 45
#define TK_FROM 46
#define TK_SUBSCRIBE 47
#define TK_READ 48
#define TK_WRITE 49
#define TK_NK_DOT 50
#define TK_WITH 51
#define TK_ENCRYPT_KEY 52
#define TK_DNODE 53
#define TK_PORT 54
#define TK_DNODES 55
#define TK_RESTORE 56
#define TK_NK_IPTOKEN 57
#define TK_FORCE 58
#define TK_UNSAFE 59
#define TK_CLUSTER 60
#define TK_LOCAL 61
#define TK_QNODE 62
#define TK_BNODE 63
#define TK_SNODE 64
#define TK_MNODE 65
#define TK_VNODE 66
#define TK_DATABASE 67
#define TK_USE 68
#define TK_FLUSH 69
#define TK_TRIM 70
#define TK_S3MIGRATE 71
#define TK_COMPACT 72
#define TK_IF 73
#define TK_NOT 74
#define TK_EXISTS 75
#define TK_BUFFER 76
#define TK_CACHEMODEL 77
#define TK_CACHESIZE 78
#define TK_COMP 79
#define TK_DURATION 80
#define TK_NK_VARIABLE 81
#define TK_MAXROWS 82
#define TK_MINROWS 83
#define TK_KEEP 84
#define TK_PAGES 85
#define TK_PAGESIZE 86
#define TK_TSDB_PAGESIZE 87
#define TK_PRECISION 88
#define TK_REPLICA 89
#define TK_VGROUPS 90
#define TK_SINGLE_STABLE 91
#define TK_RETENTIONS 92
#define TK_SCHEMALESS 93
#define TK_WAL_LEVEL 94
#define TK_WAL_FSYNC_PERIOD 95
#define TK_WAL_RETENTION_PERIOD 96
#define TK_WAL_RETENTION_SIZE 97
#define TK_WAL_ROLL_PERIOD 98
#define TK_WAL_SEGMENT_SIZE 99
#define TK_STT_TRIGGER 100
#define TK_TABLE_PREFIX 101
#define TK_TABLE_SUFFIX 102
#define TK_S3_CHUNKSIZE 103
#define TK_S3_KEEPLOCAL 104
#define TK_S3_COMPACT 105
#define TK_KEEP_TIME_OFFSET 106
#define TK_ENCRYPT_ALGORITHM 107
#define TK_NK_COLON 108
#define TK_BWLIMIT 109
#define TK_START 110
#define TK_TIMESTAMP 111
#define TK_END 112
#define TK_TABLE 113
#define TK_NK_LP 114
#define TK_NK_RP 115
#define TK_STABLE 116
#define TK_COLUMN 117
#define TK_MODIFY 118
#define TK_RENAME 119
#define TK_TAG 120
#define TK_SET 121
#define TK_NK_EQ 122
#define TK_USING 123
#define TK_TAGS 124
#define TK_BOOL 125
#define TK_TINYINT 126
#define TK_SMALLINT 127
#define TK_INT 128
#define TK_INTEGER 129
#define TK_BIGINT 130
#define TK_FLOAT 131
#define TK_DOUBLE 132
#define TK_BINARY 133
#define TK_NCHAR 134
#define TK_UNSIGNED 135
#define TK_JSON 136
#define TK_VARCHAR 137
#define TK_MEDIUMBLOB 138
#define TK_BLOB 139
#define TK_VARBINARY 140
#define TK_GEOMETRY 141
#define TK_DECIMAL 142
#define TK_COMMENT 143
#define TK_MAX_DELAY 144
#define TK_WATERMARK 145
#define TK_ROLLUP 146
#define TK_TTL 147
#define TK_SMA 148
#define TK_DELETE_MARK 149
#define TK_FIRST 150
#define TK_LAST 151
#define TK_SHOW 152
#define TK_PRIVILEGES 153
#define TK_DATABASES 154
#define TK_TABLES 155
#define TK_STABLES 156
#define TK_MNODES 157
#define TK_QNODES 158
#define TK_ARBGROUPS 159
#define TK_FUNCTIONS 160
#define TK_INDEXES 161
#define TK_ACCOUNTS 162
#define TK_APPS 163
#define TK_CONNECTIONS 164
#define TK_LICENCES 165
#define TK_GRANTS 166
#define TK_FULL 167
#define TK_LOGS 168
#define TK_MACHINES 169
#define TK_ENCRYPTIONS 170
#define TK_QUERIES 171
#define TK_SCORES 172
#define TK_TOPICS 173
#define TK_VARIABLES 174
#define TK_BNODES 175
#define TK_SNODES 176
#define TK_TRANSACTIONS 177
#define TK_DISTRIBUTED 178
#define TK_CONSUMERS 179
#define TK_SUBSCRIPTIONS 180
#define TK_VNODES 181
#define TK_ALIVE 182
#define TK_VIEWS 183
#define TK_VIEW 184
#define TK_COMPACTS 185
#define TK_NORMAL 186
#define TK_CHILD 187
#define TK_LIKE 188
#define TK_TBNAME 189
#define TK_QTAGS 190
#define TK_AS 191
#define TK_SYSTEM 192
#define TK_TSMA 193
#define TK_INTERVAL 194
#define TK_RECURSIVE 195
#define TK_TSMAS 196
#define TK_FUNCTION 197
#define TK_INDEX 198
#define TK_COUNT 199
#define TK_LAST_ROW 200
#define TK_META 201
#define TK_ONLY 202
#define TK_TOPIC 203
#define TK_CONSUMER 204
#define TK_GROUP 205
#define TK_DESC 206
#define TK_DESCRIBE 207
#define TK_RESET 208
#define TK_QUERY 209
#define TK_CACHE 210
#define TK_EXPLAIN 211
#define TK_ANALYZE 212
#define TK_VERBOSE 213
#define TK_NK_BOOL 214
#define TK_RATIO 215
#define TK_NK_FLOAT 216
#define TK_OUTPUTTYPE 217
#define TK_AGGREGATE 218
#define TK_BUFSIZE 219
#define TK_LANGUAGE 220
#define TK_REPLACE 221
#define TK_STREAM 222
#define TK_INTO 223
#define TK_PAUSE 224
#define TK_RESUME 225
#define TK_PRIMARY 226
#define TK_KEY 227
#define TK_TRIGGER 228
#define TK_AT_ONCE 229
#define TK_WINDOW_CLOSE 230
#define TK_IGNORE 231
#define TK_EXPIRED 232
#define TK_FILL_HISTORY 233
#define TK_UPDATE 234
#define TK_SUBTABLE 235
#define TK_UNTREATED 236
#define TK_KILL 237
#define TK_CONNECTION 238
#define TK_TRANSACTION 239
#define TK_BALANCE 240
#define TK_VGROUP 241
#define TK_LEADER 242
#define TK_MERGE 243
#define TK_REDISTRIBUTE 244
#define TK_SPLIT 245
#define TK_DELETE 246
#define TK_INSERT 247
#define TK_NK_BIN 248
#define TK_NK_HEX 249
#define TK_NULL 250
#define TK_NK_QUESTION 251
#define TK_NK_ALIAS 252
#define TK_NK_ARROW 253
#define TK_ROWTS 254
#define TK_QSTART 255
#define TK_QEND 256
#define TK_QDURATION 257
#define TK_WSTART 258
#define TK_WEND 259
#define TK_WDURATION 260
#define TK_IROWTS 261
#define TK_ISFILLED 262
#define TK_CAST 263
#define TK_NOW 264
#define TK_TODAY 265
#define TK_TIMEZONE 266
#define TK_CLIENT_VERSION 267
#define TK_SERVER_VERSION 268
#define TK_SERVER_STATUS 269
#define TK_CURRENT_USER 270
#define TK_CASE 271
#define TK_WHEN 272
#define TK_THEN 273
#define TK_ELSE 274
#define TK_BETWEEN 275
#define TK_IS 276
#define TK_NK_LT 277
#define TK_NK_GT 278
#define TK_NK_LE 279
#define TK_NK_GE 280
#define TK_NK_NE 281
#define TK_MATCH 282
#define TK_NMATCH 283
#define TK_CONTAINS 284
#define TK_IN 285
#define TK_JOIN 286
#define TK_INNER 287
#define TK_LEFT 288
#define TK_RIGHT 289
#define TK_OUTER 290
#define TK_SEMI 291
#define TK_ANTI 292
#define TK_ASOF 293
#define TK_WINDOW 294
#define TK_WINDOW_OFFSET 295
#define TK_JLIMIT 296
#define TK_SELECT 297
#define TK_NK_HINT 298
#define TK_DISTINCT 299
#define TK_WHERE 300
#define TK_PARTITION 301
#define TK_BY 302
#define TK_SESSION 303
#define TK_STATE_WINDOW 304
#define TK_EVENT_WINDOW 305
#define TK_COUNT_WINDOW 306
#define TK_SLIDING 307
#define TK_FILL 308
#define TK_VALUE 309
#define TK_VALUE_F 310
#define TK_NONE 311
#define TK_PREV 312
#define TK_NULL_F 313
#define TK_LINEAR 314
#define TK_NEXT 315
#define TK_HAVING 316
#define TK_RANGE 317
#define TK_EVERY 318
#define TK_ORDER 319
#define TK_SLIMIT 320
#define TK_SOFFSET 321
#define TK_LIMIT 322
#define TK_OFFSET 323
#define TK_ASC 324
#define TK_NULLS 325
#define TK_ABORT 326
#define TK_AFTER 327
#define TK_ATTACH 328
#define TK_BEFORE 329
#define TK_BEGIN 330
#define TK_BITAND 331
#define TK_BITNOT 332
#define TK_BITOR 333
#define TK_BLOCKS 334
#define TK_CHANGE 335
#define TK_COMMA 336
#define TK_CONCAT 337
#define TK_CONFLICT 338
#define TK_COPY 339
#define TK_DEFERRED 340
#define TK_DELIMITERS 341
#define TK_DETACH 342
#define TK_DIVIDE 343
#define TK_DOT 344
#define TK_EACH 345
#define TK_FAIL 346
#define TK_FILE 347
#define TK_FOR 348
#define TK_GLOB 349
#define TK_ID 350
#define TK_IMMEDIATE 351
#define TK_IMPORT 352
#define TK_INITIALLY 353
#define TK_INSTEAD 354
#define TK_ISNULL 355
#define TK_MODULES 356
#define TK_NK_BITNOT 357
#define TK_NK_SEMI 358
#define TK_NOTNULL 359
#define TK_OF 360
#define TK_PLUS 361
#define TK_PRIVILEGE 362
#define TK_RAISE 363
#define TK_RESTRICT 364
#define TK_ROW 365
#define TK_STAR 366
#define TK_STATEMENT 367
#define TK_STRICT 368
#define TK_STRING 369
#define TK_TIMES 370
#define TK_VALUES 371
#define TK_VARIABLE 372
#define TK_WAL 373
#define TK_ENCODE 374
#define TK_COMPRESS 375
#define TK_LEVEL 376
#define TK_IS_IMPORT 35
#define TK_NK_INTEGER 36
#define TK_CREATEDB 37
#define TK_USER 38
#define TK_ENABLE 39
#define TK_SYSINFO 40
#define TK_ADD 41
#define TK_DROP 42
#define TK_GRANT 43
#define TK_ON 44
#define TK_TO 45
#define TK_REVOKE 46
#define TK_FROM 47
#define TK_SUBSCRIBE 48
#define TK_READ 49
#define TK_WRITE 50
#define TK_NK_DOT 51
#define TK_WITH 52
#define TK_ENCRYPT_KEY 53
#define TK_DNODE 54
#define TK_PORT 55
#define TK_DNODES 56
#define TK_RESTORE 57
#define TK_NK_IPTOKEN 58
#define TK_FORCE 59
#define TK_UNSAFE 60
#define TK_CLUSTER 61
#define TK_LOCAL 62
#define TK_QNODE 63
#define TK_BNODE 64
#define TK_SNODE 65
#define TK_MNODE 66
#define TK_VNODE 67
#define TK_DATABASE 68
#define TK_USE 69
#define TK_FLUSH 70
#define TK_TRIM 71
#define TK_S3MIGRATE 72
#define TK_COMPACT 73
#define TK_IF 74
#define TK_NOT 75
#define TK_EXISTS 76
#define TK_BUFFER 77
#define TK_CACHEMODEL 78
#define TK_CACHESIZE 79
#define TK_COMP 80
#define TK_DURATION 81
#define TK_NK_VARIABLE 82
#define TK_MAXROWS 83
#define TK_MINROWS 84
#define TK_KEEP 85
#define TK_PAGES 86
#define TK_PAGESIZE 87
#define TK_TSDB_PAGESIZE 88
#define TK_PRECISION 89
#define TK_REPLICA 90
#define TK_VGROUPS 91
#define TK_SINGLE_STABLE 92
#define TK_RETENTIONS 93
#define TK_SCHEMALESS 94
#define TK_WAL_LEVEL 95
#define TK_WAL_FSYNC_PERIOD 96
#define TK_WAL_RETENTION_PERIOD 97
#define TK_WAL_RETENTION_SIZE 98
#define TK_WAL_ROLL_PERIOD 99
#define TK_WAL_SEGMENT_SIZE 100
#define TK_STT_TRIGGER 101
#define TK_TABLE_PREFIX 102
#define TK_TABLE_SUFFIX 103
#define TK_S3_CHUNKSIZE 104
#define TK_S3_KEEPLOCAL 105
#define TK_S3_COMPACT 106
#define TK_KEEP_TIME_OFFSET 107
#define TK_ENCRYPT_ALGORITHM 108
#define TK_NK_COLON 109
#define TK_BWLIMIT 110
#define TK_START 111
#define TK_TIMESTAMP 112
#define TK_END 113
#define TK_TABLE 114
#define TK_NK_LP 115
#define TK_NK_RP 116
#define TK_STABLE 117
#define TK_COLUMN 118
#define TK_MODIFY 119
#define TK_RENAME 120
#define TK_TAG 121
#define TK_SET 122
#define TK_NK_EQ 123
#define TK_USING 124
#define TK_TAGS 125
#define TK_BOOL 126
#define TK_TINYINT 127
#define TK_SMALLINT 128
#define TK_INT 129
#define TK_INTEGER 130
#define TK_BIGINT 131
#define TK_FLOAT 132
#define TK_DOUBLE 133
#define TK_BINARY 134
#define TK_NCHAR 135
#define TK_UNSIGNED 136
#define TK_JSON 137
#define TK_VARCHAR 138
#define TK_MEDIUMBLOB 139
#define TK_BLOB 140
#define TK_VARBINARY 141
#define TK_GEOMETRY 142
#define TK_DECIMAL 143
#define TK_COMMENT 144
#define TK_MAX_DELAY 145
#define TK_WATERMARK 146
#define TK_ROLLUP 147
#define TK_TTL 148
#define TK_SMA 149
#define TK_DELETE_MARK 150
#define TK_FIRST 151
#define TK_LAST 152
#define TK_SHOW 153
#define TK_FULL 154
#define TK_PRIVILEGES 155
#define TK_DATABASES 156
#define TK_TABLES 157
#define TK_STABLES 158
#define TK_MNODES 159
#define TK_QNODES 160
#define TK_ARBGROUPS 161
#define TK_FUNCTIONS 162
#define TK_INDEXES 163
#define TK_ACCOUNTS 164
#define TK_APPS 165
#define TK_CONNECTIONS 166
#define TK_LICENCES 167
#define TK_GRANTS 168
#define TK_LOGS 169
#define TK_MACHINES 170
#define TK_ENCRYPTIONS 171
#define TK_QUERIES 172
#define TK_SCORES 173
#define TK_TOPICS 174
#define TK_VARIABLES 175
#define TK_BNODES 176
#define TK_SNODES 177
#define TK_TRANSACTIONS 178
#define TK_DISTRIBUTED 179
#define TK_CONSUMERS 180
#define TK_SUBSCRIPTIONS 181
#define TK_VNODES 182
#define TK_ALIVE 183
#define TK_VIEWS 184
#define TK_VIEW 185
#define TK_COMPACTS 186
#define TK_NORMAL 187
#define TK_CHILD 188
#define TK_LIKE 189
#define TK_TBNAME 190
#define TK_QTAGS 191
#define TK_AS 192
#define TK_SYSTEM 193
#define TK_TSMA 194
#define TK_INTERVAL 195
#define TK_RECURSIVE 196
#define TK_TSMAS 197
#define TK_FUNCTION 198
#define TK_INDEX 199
#define TK_COUNT 200
#define TK_LAST_ROW 201
#define TK_META 202
#define TK_ONLY 203
#define TK_TOPIC 204
#define TK_CONSUMER 205
#define TK_GROUP 206
#define TK_DESC 207
#define TK_DESCRIBE 208
#define TK_RESET 209
#define TK_QUERY 210
#define TK_CACHE 211
#define TK_EXPLAIN 212
#define TK_ANALYZE 213
#define TK_VERBOSE 214
#define TK_NK_BOOL 215
#define TK_RATIO 216
#define TK_NK_FLOAT 217
#define TK_OUTPUTTYPE 218
#define TK_AGGREGATE 219
#define TK_BUFSIZE 220
#define TK_LANGUAGE 221
#define TK_REPLACE 222
#define TK_STREAM 223
#define TK_INTO 224
#define TK_PAUSE 225
#define TK_RESUME 226
#define TK_PRIMARY 227
#define TK_KEY 228
#define TK_TRIGGER 229
#define TK_AT_ONCE 230
#define TK_WINDOW_CLOSE 231
#define TK_IGNORE 232
#define TK_EXPIRED 233
#define TK_FILL_HISTORY 234
#define TK_UPDATE 235
#define TK_SUBTABLE 236
#define TK_UNTREATED 237
#define TK_KILL 238
#define TK_CONNECTION 239
#define TK_TRANSACTION 240
#define TK_BALANCE 241
#define TK_VGROUP 242
#define TK_LEADER 243
#define TK_MERGE 244
#define TK_REDISTRIBUTE 245
#define TK_SPLIT 246
#define TK_DELETE 247
#define TK_INSERT 248
#define TK_NK_BIN 249
#define TK_NK_HEX 250
#define TK_NULL 251
#define TK_NK_QUESTION 252
#define TK_NK_ALIAS 253
#define TK_NK_ARROW 254
#define TK_ROWTS 255
#define TK_QSTART 256
#define TK_QEND 257
#define TK_QDURATION 258
#define TK_WSTART 259
#define TK_WEND 260
#define TK_WDURATION 261
#define TK_IROWTS 262
#define TK_ISFILLED 263
#define TK_CAST 264
#define TK_NOW 265
#define TK_TODAY 266
#define TK_TIMEZONE 267
#define TK_CLIENT_VERSION 268
#define TK_SERVER_VERSION 269
#define TK_SERVER_STATUS 270
#define TK_CURRENT_USER 271
#define TK_CASE 272
#define TK_WHEN 273
#define TK_THEN 274
#define TK_ELSE 275
#define TK_BETWEEN 276
#define TK_IS 277
#define TK_NK_LT 278
#define TK_NK_GT 279
#define TK_NK_LE 280
#define TK_NK_GE 281
#define TK_NK_NE 282
#define TK_MATCH 283
#define TK_NMATCH 284
#define TK_CONTAINS 285
#define TK_IN 286
#define TK_JOIN 287
#define TK_INNER 288
#define TK_LEFT 289
#define TK_RIGHT 290
#define TK_OUTER 291
#define TK_SEMI 292
#define TK_ANTI 293
#define TK_ASOF 294
#define TK_WINDOW 295
#define TK_WINDOW_OFFSET 296
#define TK_JLIMIT 297
#define TK_SELECT 298
#define TK_NK_HINT 299
#define TK_DISTINCT 300
#define TK_WHERE 301
#define TK_PARTITION 302
#define TK_BY 303
#define TK_SESSION 304
#define TK_STATE_WINDOW 305
#define TK_EVENT_WINDOW 306
#define TK_COUNT_WINDOW 307
#define TK_SLIDING 308
#define TK_FILL 309
#define TK_VALUE 310
#define TK_VALUE_F 311
#define TK_NONE 312
#define TK_PREV 313
#define TK_NULL_F 314
#define TK_LINEAR 315
#define TK_NEXT 316
#define TK_HAVING 317
#define TK_RANGE 318
#define TK_EVERY 319
#define TK_ORDER 320
#define TK_SLIMIT 321
#define TK_SOFFSET 322
#define TK_LIMIT 323
#define TK_OFFSET 324
#define TK_ASC 325
#define TK_NULLS 326
#define TK_ABORT 327
#define TK_AFTER 328
#define TK_ATTACH 329
#define TK_BEFORE 330
#define TK_BEGIN 331
#define TK_BITAND 332
#define TK_BITNOT 333
#define TK_BITOR 334
#define TK_BLOCKS 335
#define TK_CHANGE 336
#define TK_COMMA 337
#define TK_CONCAT 338
#define TK_CONFLICT 339
#define TK_COPY 340
#define TK_DEFERRED 341
#define TK_DELIMITERS 342
#define TK_DETACH 343
#define TK_DIVIDE 344
#define TK_DOT 345
#define TK_EACH 346
#define TK_FAIL 347
#define TK_FILE 348
#define TK_FOR 349
#define TK_GLOB 350
#define TK_ID 351
#define TK_IMMEDIATE 352
#define TK_IMPORT 353
#define TK_INITIALLY 354
#define TK_INSTEAD 355
#define TK_ISNULL 356
#define TK_MODULES 357
#define TK_NK_BITNOT 358
#define TK_NK_SEMI 359
#define TK_NOTNULL 360
#define TK_OF 361
#define TK_PLUS 362
#define TK_PRIVILEGE 363
#define TK_RAISE 364
#define TK_RESTRICT 365
#define TK_ROW 366
#define TK_STAR 367
#define TK_STATEMENT 368
#define TK_STRICT 369
#define TK_STRING 370
#define TK_TIMES 371
#define TK_VALUES 372
#define TK_VARIABLE 373
#define TK_WAL 374
#define TK_ENCODE 375
#define TK_COMPRESS 376
#define TK_LEVEL 377
#define TK_NK_SPACE 600
#define TK_NK_COMMENT 601

3
include/dnode/vnode/tqCommon.h
View File

@ -28,6 +28,7 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe
int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sversion, char* msg, int32_t msgLen,
bool isLeader, bool restored);
@ -42,7 +43,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg);
int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* pMsg, bool fromVnode);
int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, char* msg, int32_t msgLen);
int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta);
void tqSetRestoreVersionInfo(SStreamTask* pTask);
int32_t tqExpandStreamTask(SStreamTask* pTask);
#endif // TDENGINE_TQ_COMMON_H

4
include/libs/executor/storageapi.h
View File

@ -329,7 +329,7 @@ typedef struct {
typedef struct SStateStore {
int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname);
int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal);
int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache);
int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t (*streamStateReleaseBuf)(SStreamState* pState, void* pVal, bool used);
@ -410,7 +410,7 @@ typedef struct SStateStore {
void (*streamFileStateClear)(struct SStreamFileState* pFileState);
bool (*needClearDiskBuff)(struct SStreamFileState* pFileState);
SStreamState* (*streamStateOpen)(const char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages);
SStreamState* (*streamStateOpen)(const char* path, void* pTask, int64_t streamId, int32_t taskId);
void (*streamStateClose)(SStreamState* pState, bool remove);
int32_t (*streamStateBegin)(SStreamState* pState);
int32_t (*streamStateCommit)(SStreamState* pState);

3
include/libs/nodes/cmdnodes.h
View File

@ -258,6 +258,8 @@ typedef struct SCreateUserStmt {
char userName[TSDB_USER_LEN];
char password[TSDB_USET_PASSWORD_LEN];
int8_t sysinfo;
int8_t createDb;
int8_t isImport;
int32_t numIpRanges;
SIpV4Range* pIpRanges;
@ -311,6 +313,7 @@ typedef struct SShowStmt {
SNode* pTbName; // SValueNode
EOperatorType tableCondType;
EShowKind showKind; // show databases: user/system, show tables: normal/child, others NULL
bool withFull; // for show users full;
} SShowStmt;
typedef struct SShowCreateDatabaseStmt {

1
include/libs/nodes/querynodes.h
View File

@ -628,6 +628,7 @@ char* nodesGetStrValueFromNode(SValueNode* pNode);
void nodesValueNodeToVariant(const SValueNode* pNode, SVariant* pVal);
SValueNode* nodesMakeValueNodeFromString(char* literal);
SValueNode* nodesMakeValueNodeFromBool(bool b);
SNode* nodesMakeValueNodeFromInt32(int32_t value);
char* nodesGetFillModeString(EFillMode mode);
int32_t nodesMergeConds(SNode** pDst, SNodeList** pSrc);

2
include/libs/scalar/filter.h
View File

@ -58,7 +58,7 @@ extern int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict
extern int32_t filterConverNcharColumns(SFilterInfo *pFilterInfo, int32_t rows, bool *gotNchar);
extern int32_t filterFreeNcharColumns(SFilterInfo *pFilterInfo);
extern void filterFreeInfo(SFilterInfo *info);
extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pColsAgg, int32_t numOfCols, int32_t numOfRows);
extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pColsAgg, int32_t numOfCols, int32_t numOfRows);
/* condition split interface */
int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond,

4
include/libs/stream/streamState.h
View File

@ -29,7 +29,7 @@ extern "C" {
#include "storageapi.h"
SStreamState* streamStateOpen(const char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages);
SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId);
void streamStateClose(SStreamState* pState, bool remove);
int32_t streamStateBegin(SStreamState* pState);
int32_t streamStateCommit(SStreamState* pState);
@ -101,7 +101,7 @@ int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname);
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal);
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache);
void streamStateReloadInfo(SStreamState* pState, TSKEY ts);

14
include/libs/stream/streammsg.h
View File

@ -190,6 +190,20 @@ typedef struct SCheckpointTriggerRsp {
int32_t rspCode;
} SCheckpointTriggerRsp;
typedef struct SCheckpointReport {
int64_t streamId;
int32_t taskId;
int32_t nodeId;
int64_t checkpointId;
int64_t checkpointVer;
int64_t checkpointTs;
int32_t transId;
int8_t dropHTask;
} SCheckpointReport;
int32_t tEncodeStreamTaskChkptReport(SEncoder* pEncoder, const SCheckpointReport* pReq);
int32_t tDecodeStreamTaskChkptReport(SDecoder* pDecoder, SCheckpointReport* pReq);
typedef struct {
SMsgHead head;
int64_t streamId;

38
include/libs/stream/tstream.h
View File

@ -157,7 +157,8 @@ typedef enum EStreamTaskEvent {
typedef void FTbSink(SStreamTask* pTask, void* vnode, void* data);
typedef void FSmaSink(void* vnode, int64_t smaId, const SArray* data);
typedef int32_t FTaskExpand(void* ahandle, SStreamTask* pTask, int64_t ver);
typedef int32_t FTaskBuild(void* ahandle, SStreamTask* pTask, int64_t ver);
typedef int32_t FTaskExpand(SStreamTask* pTask);
typedef struct {
int8_t type;
@ -205,7 +206,6 @@ typedef struct {
typedef struct {
char stbFullName[TSDB_TABLE_FNAME_LEN];
int32_t waitingRspCnt;
SUseDbRsp dbInfo;
} STaskDispatcherShuffle;
@ -312,15 +312,18 @@ typedef struct SMetaHbInfo SMetaHbInfo;
typedef struct SDispatchMsgInfo {
SStreamDispatchReq* pData; // current dispatch data
int8_t dispatchMsgType;
int64_t checkpointId;// checkpoint id msg
int32_t transId; // transId for current checkpoint
int16_t msgType; // dispatch msg type
int32_t retryCount; // retry send data count
int64_t startTs; // dispatch start time, record total elapsed time for dispatch
SArray* pRetryList; // current dispatch successfully completed node of downstream
void* pRetryTmr; // used to dispatch data after a given time duration
void* pRspTmr; // used to dispatch data after a given time duration
int8_t dispatchMsgType;
int64_t checkpointId; // checkpoint id msg
int32_t transId; // transId for current checkpoint
int16_t msgType; // dispatch msg type
int32_t msgId;
int64_t startTs; // dispatch start time, record total elapsed time for dispatch
int64_t rspTs; // latest rsp time
void* pRetryTmr; // used to dispatch data after a given time duration
TdThreadMutex lock;
int8_t inMonitor;
SArray* pSendInfo; // SArray<SDispatchEntry>
} SDispatchMsgInfo;
typedef struct STaskQueue {
@ -484,7 +487,8 @@ typedef struct SStreamMeta {
SArray* pTaskList; // SArray<STaskId*>
void* ahandle;
TXN* txn;
FTaskExpand* expandFunc;
FTaskBuild* buildTaskFn;
FTaskExpand* expandTaskFn;
int32_t vgId;
int64_t stage;
int32_t role;
@ -708,8 +712,8 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st);
// stream task meta
void streamMetaInit();
void streamMetaCleanup();
SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage,
startComplete_fn_t fn);
SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskBuild expandFunc, FTaskExpand expandTaskFn,
int32_t vgId, int64_t stage, startComplete_fn_t fn);
void streamMetaClose(SStreamMeta* streamMeta);
int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); // save to stream meta store
int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey);
@ -734,6 +738,9 @@ int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32
void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs);
void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SStreamTask* pHTask, int32_t transId,
int64_t startTs);
void streamMetaClearUpdateTaskList(SStreamMeta* pMeta);
void streamMetaInitUpdateTaskList(SStreamMeta* pMeta, int32_t transId);
void streamMetaRLock(SStreamMeta* pMeta);
void streamMetaRUnLock(SStreamMeta* pMeta);
void streamMetaWLock(SStreamMeta* pMeta);
@ -762,8 +769,7 @@ int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskI
int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask);
int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SRpcMsg* pMsg,
int32_t setCode);
int32_t streamBuildAndSendCheckpointUpdateMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId, STaskId* pHTaskId,
SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask);
int32_t streamSendChkptReportMsg(SStreamTask* pTask, SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask);
int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpointInfoReq* pReq);
SActiveCheckpointInfo* streamTaskCreateActiveChkptInfo();

1
include/libs/wal/wal.h
View File

@ -60,6 +60,7 @@ typedef struct {
EWalType level; // wal level
int32_t encryptAlgorithm;
char encryptKey[ENCRYPT_KEY_LEN + 1];
int8_t clearFiles;
} SWalCfg;
typedef struct {

11
include/os/osEnv.h
View File

@ -37,11 +37,12 @@ extern float tsNumOfCores;
extern int64_t tsTotalMemoryKB;
extern char *tsProcPath;
extern char tsSIMDEnable;
extern char tsSSE42Enable;
extern char tsAVXEnable;
extern char tsAVX2Enable;
extern char tsFMAEnable;
extern char tsAVX512Enable;
extern char tsSSE42Supported;
extern char tsAVXSupported;
extern char tsAVX2Supported;
extern char tsFMASupported;
extern char tsAVX512Supported;
extern char tsAVX512Enable;
extern char tsTagFilterCache;
extern char configDir[];

12
include/util/taoserror.h
View File

@ -24,6 +24,14 @@ extern "C" {
// clang-format off
typedef struct {
int32_t val;
const char* str;
const char* macro;
} STaosError;
extern STaosError errors[];
#define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code))))
#define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code))
@ -38,6 +46,7 @@ const char* terrstr();
char* taosGetErrMsgReturn();
char* taosGetErrMsg();
int32_t* taosGetErrno();
int32_t taosGetErrSize();
#define terrno (*taosGetErrno())
#define terrMsg (taosGetErrMsg())
@ -316,7 +325,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x038A) //
#define TSDB_CODE_MND_DB_INDEX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x038B)
#define TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO TAOS_DEF_ERROR_CODE(0, 0x038C)
#define TSDB_CODE_MND_INCONSIST_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x038D)
// #define TSDB_CODE_MND_INCONSIST_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x038D) // unused
#define TSDB_CODE_MND_INVALID_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x038E)
// #define TSDB_CODE_MND_INVALID_DB_OPTION_DAYS TAOS_DEF_ERROR_CODE(0, 0x0390) // 2.x
// #define TSDB_CODE_MND_INVALID_DB_OPTION_KEEP TAOS_DEF_ERROR_CODE(0, 0x0391) // 2.x
@ -327,6 +336,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_DB_IN_CREATING TAOS_DEF_ERROR_CODE(0, 0x0396) //
#define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x039A)
#define TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE TAOS_DEF_ERROR_CODE(0, 0x039B)
#define TSDB_CODE_MND_INVALID_WAL_LEVEL TAOS_DEF_ERROR_CODE(0, 0x039C)
// mnode-node
#define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0)

1
include/util/tdef.h
View File

@ -549,6 +549,7 @@ typedef struct {
char dir[TSDB_FILENAME_LEN];
int32_t level;
int32_t primary;
int8_t disable; // disable create new file
} SDiskCfg;
typedef struct {

4
packaging/tools/install.sh
View File

@ -795,10 +795,10 @@ function is_version_compatible() {
if [ -f ${script_dir}/driver/vercomp.txt ]; then
min_compatible_version=$(cat ${script_dir}/driver/vercomp.txt)
else
min_compatible_version=$(${script_dir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 5)
min_compatible_version=$(${script_dir}/bin/${serverName} -V | grep version | head -1 | cut -d ' ' -f 5)
fi
exist_version=$(${installDir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 3)
exist_version=$(${installDir}/bin/${serverName} -V | grep version | head -1 | cut -d ' ' -f 3)
vercomp $exist_version "3.0.0.0"
case $? in
2)

56
source/client/src/clientHb.c
View File

@ -134,14 +134,15 @@ static int32_t hbUpdateUserAuthInfo(SAppHbMgr *pAppHbMgr, SUserAuthBatchRsp *bat
if (pTscObj->whiteListInfo.fp) {
SWhiteListInfo *whiteListInfo = &pTscObj->whiteListInfo;
int64_t oldVer = atomic_load_64(&whiteListInfo->ver);
if (oldVer < pRsp->whiteListVer) {
int64_t oldVer = atomic_load_64(&whiteListInfo->ver);
if (oldVer < pRsp->whiteListVer || pRsp->whiteListVer == 0) {
atomic_store_64(&whiteListInfo->ver, pRsp->whiteListVer);
if (whiteListInfo->fp) {
(*whiteListInfo->fp)(whiteListInfo->param, &pRsp->whiteListVer, TAOS_NOTIFY_WHITELIST_VER);
}
tscDebug("update whitelist version of user %s from %"PRId64" to %"PRId64", tscRid:%" PRIi64, pRsp->user, oldVer,
atomic_load_64(&whiteListInfo->ver), pTscObj->id);
tscDebug("update whitelist version of user %s from %" PRId64 " to %" PRId64 ", tscRid:%" PRIi64, pRsp->user,
oldVer, atomic_load_64(&whiteListInfo->ver), pTscObj->id);
}
}
releaseTscObj(pReq->connKey.tscRid);
@ -203,8 +204,8 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
for (int32_t i = 0; i < numOfBatchs; ++i) {
SDbHbRsp *rsp = taosArrayGet(batchRsp.pArray, i);
if (rsp->useDbRsp) {
tscDebug("hb use db rsp, db:%s, vgVersion:%d, stateTs:%" PRId64 ", uid:%" PRIx64,
rsp->useDbRsp->db, rsp->useDbRsp->vgVersion, rsp->useDbRsp->stateTs, rsp->useDbRsp->uid);
tscDebug("hb use db rsp, db:%s, vgVersion:%d, stateTs:%" PRId64 ", uid:%" PRIx64, rsp->useDbRsp->db,
rsp->useDbRsp->vgVersion, rsp->useDbRsp->stateTs, rsp->useDbRsp->uid);
if (rsp->useDbRsp->vgVersion < 0) {
tscDebug("hb to remove db, db:%s", rsp->useDbRsp->db);
@ -226,7 +227,9 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
goto _return;
}
catalogUpdateDBVgInfo(pCatalog, (rsp->useDbRsp->db[0] == 'i') ? TSDB_PERFORMANCE_SCHEMA_DB : TSDB_INFORMATION_SCHEMA_DB, rsp->useDbRsp->uid, vgInfo);
catalogUpdateDBVgInfo(pCatalog,
(rsp->useDbRsp->db[0] == 'i') ? TSDB_PERFORMANCE_SCHEMA_DB : TSDB_INFORMATION_SCHEMA_DB,
rsp->useDbRsp->uid, vgInfo);
}
}
}
@ -239,7 +242,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
if (rsp->pTsmaRsp) {
if (rsp->pTsmaRsp->pTsmas) {
for (int32_t i = 0; i < rsp->pTsmaRsp->pTsmas->size; ++i) {
STableTSMAInfo* pTsma = taosArrayGetP(rsp->pTsmaRsp->pTsmas, i);
STableTSMAInfo *pTsma = taosArrayGetP(rsp->pTsmaRsp->pTsmas, i);
catalogAsyncUpdateTSMA(pCatalog, &pTsma, rsp->dbTsmaVersion);
}
taosArrayClear(rsp->pTsmaRsp->pTsmas);
@ -295,16 +298,15 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo
return TSDB_CODE_SUCCESS;
}
static int32_t hbProcessDynViewRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) {
return catalogUpdateDynViewVer(pCatalog, (SDynViewVersion*)value);
return catalogUpdateDynViewVer(pCatalog, (SDynViewVersion *)value);
}
static void hbFreeSViewMetaInRsp(void* p) {
if (NULL == p || NULL == *(void**)p) {
static void hbFreeSViewMetaInRsp(void *p) {
if (NULL == p || NULL == *(void **)p) {
return;
}
SViewMetaRsp *pRsp = *(SViewMetaRsp**)p;
SViewMetaRsp *pRsp = *(SViewMetaRsp **)p;
tFreeSViewMetaRsp(pRsp);
taosMemoryFreeClear(pRsp);
}
@ -338,7 +340,7 @@ static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatal
return TSDB_CODE_SUCCESS;
}
static int32_t hbprocessTSMARsp(void* value, int32_t valueLen, struct SCatalog* pCatalog) {
static int32_t hbprocessTSMARsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) {
int32_t code = 0;
STSMAHbRsp hbRsp = {0};
@ -349,7 +351,7 @@ static int32_t hbprocessTSMARsp(void* value, int32_t valueLen, struct SCatalog*
int32_t numOfTsma = taosArrayGetSize(hbRsp.pTsmas);
for (int32_t i = 0; i < numOfTsma; ++i) {
STableTSMAInfo* pTsmaInfo = taosArrayGetP(hbRsp.pTsmas, i);
STableTSMAInfo *pTsmaInfo = taosArrayGetP(hbRsp.pTsmas, i);
if (!pTsmaInfo->pFuncs) {
tscDebug("hb to remove tsma: %s.%s", pTsmaInfo->dbFName, pTsmaInfo->name);
@ -366,7 +368,7 @@ static int32_t hbprocessTSMARsp(void* value, int32_t valueLen, struct SCatalog*
return TSDB_CODE_SUCCESS;
}
static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) {
static void hbProcessQueryRspKvs(int32_t kvNum, SArray *pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) {
for (int32_t i = 0; i < kvNum; ++i) {
SKv *kv = taosArrayGet(pKvs, i);
switch (kv->key) {
@ -490,14 +492,14 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
if (kvNum > 0) {
struct SCatalog *pCatalog = NULL;
int32_t code = catalogGetHandle(pReq->clusterId, &pCatalog);
int32_t code = catalogGetHandle(pReq->clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) {
tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pReq->clusterId, tstrerror(code));
} else {
hbProcessQueryRspKvs(kvNum, pRsp->info, pCatalog, pAppHbMgr);
}
}
taosHashRelease(pAppHbMgr->activeInfo, pReq);
return TSDB_CODE_SUCCESS;
@ -802,8 +804,9 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl
for (int32_t i = 0; i < dbNum; ++i) {
SDbCacheInfo *db = &dbs[i];
tscDebug("the %dth expired dbFName:%s, dbId:%" PRId64 ", vgVersion:%d, cfgVersion:%d, numOfTable:%d, startTs:%" PRId64,
i, db->dbFName, db->dbId, db->vgVersion, db->cfgVersion, db->numOfTable, db->stateTs);
tscDebug("the %dth expired dbFName:%s, dbId:%" PRId64
", vgVersion:%d, cfgVersion:%d, numOfTable:%d, startTs:%" PRId64,
i, db->dbFName, db->dbId, db->vgVersion, db->cfgVersion, db->numOfTable, db->stateTs);
db->dbId = htobe64(db->dbId);
db->vgVersion = htonl(db->vgVersion);
@ -919,7 +922,7 @@ int32_t hbGetExpiredViewInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S
return TSDB_CODE_SUCCESS;
}
int32_t hbGetExpiredTSMAInfo(SClientHbKey* connKey, struct SCatalog* pCatalog, SClientHbReq* pReq) {
int32_t hbGetExpiredTSMAInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *pReq) {
int32_t code = 0;
uint32_t tsmaNum = 0;
STSMAVersion *tsmas = NULL;
@ -936,7 +939,7 @@ int32_t hbGetExpiredTSMAInfo(SClientHbKey* connKey, struct SCatalog* pCatalog, S
}
for (int32_t i = 0; i < tsmaNum; ++i) {
STSMAVersion* tsma = &tsmas[i];
STSMAVersion *tsma = &tsmas[i];
tsma->dbId = htobe64(tsma->dbId);
tsma->tsmaId = htobe64(tsma->tsmaId);
tsma->version = htonl(tsma->version);
@ -1015,7 +1018,7 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req
if (TSDB_CODE_SUCCESS != code) {
return code;
}
#endif
#endif
code = hbGetExpiredTSMAInfo(connKey, pCatalog, req);
} else {
req->app.appId = 0;
@ -1154,7 +1157,8 @@ static void *hbThreadFunc(void *param) {
if (sz > 0) {
hbGatherAppInfo();
if (sz > 1 && !clientHbMgr.appHbHash) {
clientHbMgr.appHbHash = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
clientHbMgr.appHbHash =
taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
}
taosHashClear(clientHbMgr.appHbHash);
}
@ -1436,6 +1440,4 @@ void hbDeregisterConn(STscObj *pTscObj, SClientHbKey connKey) {
}
// set heart beat thread quit mode , if quicByKill 1 then kill thread else quit from inner
void taos_set_hb_quit(int8_t quitByKill) {
clientHbMgr.quitByKill = quitByKill;
}
void taos_set_hb_quit(int8_t quitByKill) { clientHbMgr.quitByKill = quitByKill; }

4
source/client/src/clientMain.c
View File

@ -1849,3 +1849,7 @@ int taos_set_conn_mode(TAOS* taos, int mode, int value) {
}
return 0;
}
char* getBuildInfo(){
return buildinfo;
}

9
source/client/test/clientTests.cpp
View File

@ -828,12 +828,8 @@ TEST(clientCase, projection_query_tables) {
// printf("error in create db, reason:%s\n", taos_errstr(pRes));
// }
// taos_free_result(pRes);
/*
TAOS_RES* pRes = taos_query(pConn, "select last(ts), ts from cache_1.t1");
// pRes = taos_query(pConn, "select last(ts), ts from cache_1.no_pk_t1");
if (taos_errno(pRes) != 0) {
printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
}
pRes= taos_query(pConn, "use abc1");
taos_free_result(pRes);
pRes = taos_query(pConn, "create table tu using st2 tags(2)");
@ -868,7 +864,6 @@ TEST(clientCase, projection_query_tables) {
createNewTable(pConn, i, 100000, 0, pstr);
}
}
*/
pRes = taos_query(pConn, "select * from abc1.st2");
if (taos_errno(pRes) != 0) {

24
source/common/src/rsync.c
View File

@ -169,15 +169,22 @@ int32_t uploadByRsync(const char* id, const char* path) {
#else
if (path[strlen(path) - 1] != '/') {
#endif
snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 --bwlimit=100000 %s/ rsync://%s/checkpoint/%s/",
snprintf(command, PATH_MAX,
"rsync -av --debug=all --log-file=%s/rsynclog --delete --timeout=10 --bwlimit=100000 %s/ "
"rsync://%s/checkpoint/%s/",
tsLogDir,
#ifdef WINDOWS
pathTransform
#else
path
#endif
, tsSnodeAddress, id);
,
tsSnodeAddress, id);
} else {
snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 --bwlimit=100000 %s rsync://%s/checkpoint/%s/",
snprintf(command, PATH_MAX,
"rsync -av --debug=all --log-file=%s/rsynclog --delete --timeout=10 --bwlimit=100000 %s "
"rsync://%s/checkpoint/%s/",
tsLogDir,
#ifdef WINDOWS
pathTransform
#else
@ -213,14 +220,15 @@ int32_t downloadRsync(const char* id, const char* path) {
#endif
char command[PATH_MAX] = {0};
snprintf(command, PATH_MAX, "rsync -av --debug=all --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/ %s",
tsSnodeAddress, id,
snprintf(command, PATH_MAX,
"rsync -av --debug=all --log-file=%s/rsynclog --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/ %s",
tsLogDir, tsSnodeAddress, id,
#ifdef WINDOWS
pathTransform
#else
path
#endif
);
);
uDebug("[rsync] %s start to sync data from remote to:%s, %s", id, path, command);
@ -249,7 +257,9 @@ int32_t deleteRsync(const char* id) {
}
char command[PATH_MAX] = {0};
snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 %s rsync://%s/checkpoint/%s/", tmp, tsSnodeAddress, id);
snprintf(command, PATH_MAX,
"rsync -av --debug=all --log-file=%s/rsynclog --delete --timeout=10 %s rsync://%s/checkpoint/%s/", tsLogDir,
tmp, tsSnodeAddress, id);
code = execCommand(command);
taosRemoveDir(tmp);

13
source/common/src/systable.c
View File

@ -85,7 +85,7 @@ static const SSysDbTableSchema clusterSchema[] = {
{.name = "uptime", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
{.name = "version", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "expire_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
{.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};
static const SSysDbTableSchema userDBSchema[] = {
@ -269,6 +269,16 @@ static const SSysDbTableSchema userUsersSchema[] = {
{.name = "allowed_host", .bytes = TSDB_PRIVILEDGE_HOST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};
static const SSysDbTableSchema userUsersFullSchema[] = {
{.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "super", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "enable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "sysinfo", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "createdb", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "encrypted_pass", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "allowed_host", .bytes = TSDB_PRIVILEDGE_HOST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};
GRANTS_SCHEMA;
static const SSysDbTableSchema vgroupsSchema[] = {
@ -438,6 +448,7 @@ static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_COLS, userColsSchema, tListLen(userColsSchema), false},
// {TSDB_INS_TABLE_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)},
{TSDB_INS_TABLE_USERS, userUsersSchema, tListLen(userUsersSchema), true},
{TSDB_INS_TABLE_USERS_FULL, userUsersFullSchema, tListLen(userUsersFullSchema), true},
{TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema), true},
{TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema), true},
{TSDB_INS_TABLE_CONFIGS, configSchema, tListLen(configSchema), false},

4
source/common/src/tcol.c
View File

@ -327,7 +327,9 @@ int32_t setColCompressByOption(uint8_t type, uint8_t encode, uint16_t compressTy
return TSDB_CODE_SUCCESS;
}
bool useCompress(uint8_t tableType) { return TSDB_SUPER_TABLE == tableType || TSDB_NORMAL_TABLE == tableType; }
bool useCompress(uint8_t tableType) {
return TSDB_SUPER_TABLE == tableType || TSDB_NORMAL_TABLE == tableType || TSDB_CHILD_TABLE == tableType;
}
int8_t validColCompressLevel(uint8_t type, uint8_t level) {
if (level == TSDB_COLVAL_LEVEL_DISABLED) return 1;

6
source/common/src/tdatablock.c
View File

@ -21,7 +21,7 @@
#define MALLOC_ALIGN_BYTES 32
static void copyPkVal(SDataBlockInfo* pDst, const SDataBlockInfo* pSrc);
int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) {
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
@ -848,7 +848,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
if (pBlock->pBlockAgg == NULL) {
isNull = colDataIsNull_s(pColData, j);
} else {
isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg[i]);
isNull = colDataIsNull(pColData, pBlock->info.rows, j, &pBlock->pBlockAgg[i]);
}
if (isNull) {
@ -1362,6 +1362,8 @@ void blockDataEmpty(SSDataBlock* pDataBlock) {
return;
}
taosMemoryFreeClear(pDataBlock->pBlockAgg);
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);

15
source/common/src/tglobal.c
View File

@ -74,6 +74,7 @@ int32_t tsNumOfSnodeStreamThreads = 4;
int32_t tsNumOfSnodeWriteThreads = 1;
int32_t tsMaxStreamBackendCache = 128; // M
int32_t tsPQSortMemThreshold = 16; // M
int32_t tsRetentionSpeedLimitMB = 0; // unlimited
// sync raft
int32_t tsElectInterval = 25 * 1000;
@ -328,6 +329,7 @@ int32_t taosSetTfsCfg(SConfig *pCfg) {
tstrncpy(tsDiskCfg[0].dir, pItem->str, TSDB_FILENAME_LEN);
tsDiskCfg[0].level = 0;
tsDiskCfg[0].primary = 1;
tsDiskCfg[0].disable = 0;
tstrncpy(tsDataDir, pItem->str, PATH_MAX);
if (taosMulMkDir(tsDataDir) != 0) {
uError("failed to create dataDir:%s", tsDataDir);
@ -591,12 +593,13 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "enableCoreFile", 1, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1;
if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "ssd42", tsSSE42Enable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "avx", tsAVXEnable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "avx2", tsAVX2Enable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "fma", tsFMAEnable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "avx512", tsAVX512Enable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "ssd42", tsSSE42Supported, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "avx", tsAVXSupported, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "avx2", tsAVX2Supported, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "fma", tsFMASupported, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "avx512", tsAVX512Supported, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "simdEnable", tsSIMDEnable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "AVX512Enable", tsAVX512Enable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
@ -664,6 +667,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "retentionSpeedLimitMB", tsRetentionSpeedLimitMB, 0, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
@ -1128,6 +1132,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsTimeToGetAvailableConn = cfgGetItem(pCfg, "timeToGetAvailableConn")->i32;
tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32;
tsRetentionSpeedLimitMB = cfgGetItem(pCfg, "retentionSpeedLimitMB")->i32;
tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32;
tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32;
tsRatioOfVnodeStreamThreads = cfgGetItem(pCfg, "ratioOfVnodeStreamThreads")->fval;

20
source/common/src/tmsg.c
View File

@ -840,7 +840,6 @@ int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq
for (int32_t i = 0; i < pReq->numOfFields; ++i) {
if (pReq->alterType == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION) {
taosArrayDestroy(pReq->pFields);
pReq->pFields = taosArrayInit(pReq->numOfFields, sizeof(SFieldWithOptions));
SFieldWithOptions field = {0};
@ -1670,8 +1669,10 @@ int32_t tSerializeSCreateUserReq(void *buf, int32_t bufLen, SCreateUserReq *pReq
if (tEncodeU32(&encoder, pReq->pIpRanges[i].ip) < 0) return -1;
if (tEncodeU32(&encoder, pReq->pIpRanges[i].mask) < 0) return -1;
}
ENCODESQL();
if (tEncodeI8(&encoder, pReq->isImport) < 0) return -1;
if (tEncodeI8(&encoder, pReq->createDb) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -1697,8 +1698,12 @@ int32_t tDeserializeSCreateUserReq(void *buf, int32_t bufLen, SCreateUserReq *pR
if (tDecodeU32(&decoder, &(pReq->pIpRanges[i].ip)) < 0) return -1;
if (tDecodeU32(&decoder, &(pReq->pIpRanges[i].mask)) < 0) return -1;
}
DECODESQL();
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI8(&decoder, &pReq->createDb) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->isImport) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
@ -1846,7 +1851,7 @@ int32_t tSerializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq)
}
if (tEncodeI64(&encoder, pReq->privileges) < 0) return -1;
ENCODESQL();
if (tEncodeI8(&encoder, pReq->flag) < 0) return -1;
if (tEncodeU8(&encoder, pReq->flag) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -1887,7 +1892,7 @@ int32_t tDeserializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq
if (tDecodeI64(&decoder, &pReq->privileges) < 0) return -1;
DECODESQL();
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI8(&decoder, &pReq->flag) < 0) return -1;
if (tDecodeU8(&decoder, &pReq->flag) < 0) return -1;
}
tEndDecode(&decoder);
@ -4637,6 +4642,7 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq
if (tEncodeCStr(&encoder, pReq->filterTb) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
if (tEncodeI64(&encoder, pReq->compactId) < 0) return -1;
if (tEncodeI8(&encoder, pReq->withFull) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -4659,7 +4665,9 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR
} else {
pReq->compactId = -1;
}
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI8(&decoder, (int8_t *)&pReq->withFull) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;

6
source/common/src/tvariant.c
View File

@ -88,7 +88,7 @@ static int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint
if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
return TSDB_CODE_FAILED;
}
if (val > UINT64_MAX) {
if (val > (double)UINT64_MAX) {
errno = ERANGE;
return TSDB_CODE_FAILED;
}
@ -172,7 +172,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
} break;
case TK_NK_FLOAT: {
double val = round(taosStr2Double(z, &endPtr));
if (!IS_VALID_INT64(val)) {
if(val < (double)INT64_MIN || val > (double)INT64_MAX){
return TSDB_CODE_FAILED;
}
if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
@ -271,7 +271,7 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) {
} break;
case TK_NK_FLOAT: {
double val = round(taosStr2Double(p, &endPtr));
if (!IS_VALID_UINT64(val)) {
if (val < 0 || val > (double)UINT64_MAX) {
return TSDB_CODE_FAILED;
}
if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {

24
source/common/test/CMakeLists.txt
View File

@ -40,12 +40,24 @@ add_test(
COMMAND dataformatTest
)
# tmsg test
add_executable(tmsgTest "")
target_sources(tmsgTest
if (${TD_LINUX})
# tmsg test
add_executable(tmsgTest "")
target_sources(tmsgTest
PRIVATE
"tmsgTest.cpp"
"../src/tmsg.c"
)
target_include_directories(tmsgTest PUBLIC "${TD_SOURCE_DIR}/include/common/")
target_link_libraries(tmsgTest PUBLIC os util gtest gtest_main)
)
target_include_directories(tmsgTest PUBLIC "${TD_SOURCE_DIR}/include/common/")
target_link_libraries(tmsgTest PUBLIC os util gtest gtest_main)
add_test(
NAME tmsgTest
COMMAND tmsgTest
)
# config file for msg type table
SET(MSG_TBL_FILE ${CMAKE_CURRENT_SOURCE_DIR}/msgTypeTable.ini)
add_custom_command(TARGET tmsgTest POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${MSG_TBL_FILE} $<TARGET_FILE_DIR:tmsgTest>
)
endif ()

556
source/common/test/msgTypeTable.ini Normal file
View File

@ -0,0 +1,556 @@
TDMT_DND_CREATE_MNODE = 1
TDMT_DND_CREATE_MNODE_RSP = 2
TDMT_DND_DROP_MNODE = 3
TDMT_DND_DROP_MNODE_RSP = 4
TDMT_DND_CREATE_QNODE = 5
TDMT_DND_CREATE_QNODE_RSP = 6
TDMT_DND_DROP_QNODE = 7
TDMT_DND_DROP_QNODE_RSP = 8
TDMT_DND_CREATE_SNODE = 9
TDMT_DND_CREATE_SNODE_RSP = 10
TDMT_DND_DROP_SNODE = 11
TDMT_DND_DROP_SNODE_RSP = 12
TDMT_DND_CREATE_BNODE = 13
TDMT_DND_CREATE_BNODE_RSP = 14
TDMT_DND_DROP_BNODE = 15
TDMT_DND_DROP_BNODE_RSP = 16
TDMT_DND_CREATE_VNODE = 17
TDMT_DND_CREATE_VNODE_RSP = 18
TDMT_DND_DROP_VNODE = 19
TDMT_DND_DROP_VNODE_RSP = 20
TDMT_DND_SERVER_STATUS = 21
TDMT_DND_SERVER_STATUS_RSP = 22
TDMT_DND_NET_TEST = 23
TDMT_DND_NET_TEST_RSP = 24
TDMT_DND_CONFIG_DNODE = 25
TDMT_DND_CONFIG_DNODE_RSP = 26
TDMT_DND_SYSTABLE_RETRIEVE = 27
TDMT_DND_SYSTABLE_RETRIEVE_RSP = 28
TDMT_DND_UNUSED_CODE = 29
TDMT_DND_UNUSED_CODE_RSP = 30
TDMT_DND_ALTER_MNODE_TYPE = 31
TDMT_DND_ALTER_MNODE_TYPE_RSP = 32
TDMT_DND_ALTER_VNODE_TYPE = 33
TDMT_DND_ALTER_VNODE_TYPE_RSP = 34
TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP = 35
TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP_RSP = 36
TDMT_DND_CREATE_ENCRYPT_KEY = 37
TDMT_DND_CREATE_ENCRYPT_KEY_RSP = 38
TDMT_DND_MAX_MSG = 39
TDMT_DND_MAX_MSG_RSP = 40
TDMT_MND_CONNECT = 257
TDMT_MND_CONNECT_RSP = 258
TDMT_MND_CREATE_ACCT = 259
TDMT_MND_CREATE_ACCT_RSP = 260
TDMT_MND_ALTER_ACCT = 261
TDMT_MND_ALTER_ACCT_RSP = 262
TDMT_MND_DROP_ACCT = 263
TDMT_MND_DROP_ACCT_RSP = 264
TDMT_MND_CREATE_USER = 265
TDMT_MND_CREATE_USER_RSP = 266
TDMT_MND_ALTER_USER = 267
TDMT_MND_ALTER_USER_RSP = 268
TDMT_MND_DROP_USER = 269
TDMT_MND_DROP_USER_RSP = 270
TDMT_MND_GET_USER_AUTH = 271
TDMT_MND_GET_USER_AUTH_RSP = 272
TDMT_MND_CREATE_DNODE = 273
TDMT_MND_CREATE_DNODE_RSP = 274
TDMT_MND_CONFIG_DNODE = 275
TDMT_MND_CONFIG_DNODE_RSP = 276
TDMT_MND_DROP_DNODE = 277
TDMT_MND_DROP_DNODE_RSP = 278
TDMT_MND_CREATE_MNODE = 279
TDMT_MND_CREATE_MNODE_RSP = 280
TDMT_MND_ALTER_MNODE = 281
TDMT_MND_ALTER_MNODE_RSP = 282
TDMT_MND_DROP_MNODE = 283
TDMT_MND_DROP_MNODE_RSP = 284
TDMT_MND_CREATE_QNODE = 285
TDMT_MND_CREATE_QNODE_RSP = 286
TDMT_MND_ALTER_QNODE = 287
TDMT_MND_ALTER_QNODE_RSP = 288
TDMT_MND_DROP_QNODE = 289
TDMT_MND_DROP_QNODE_RSP = 290
TDMT_MND_QNODE_LIST = 291
TDMT_MND_QNODE_LIST_RSP = 292
TDMT_MND_DNODE_LIST = 293
TDMT_MND_DNODE_LIST_RSP = 294
TDMT_MND_CREATE_SNODE = 295
TDMT_MND_CREATE_SNODE_RSP = 296
TDMT_MND_ALTER_SNODE = 297
TDMT_MND_ALTER_SNODE_RSP = 298
TDMT_MND_DROP_SNODE = 299
TDMT_MND_DROP_SNODE_RSP = 300
TDMT_MND_CREATE_BNODE = 301
TDMT_MND_CREATE_BNODE_RSP = 302
TDMT_MND_ALTER_BNODE = 303
TDMT_MND_ALTER_BNODE_RSP = 304
TDMT_MND_DROP_BNODE = 305
TDMT_MND_DROP_BNODE_RSP = 306
TDMT_MND_CREATE_DB = 307
TDMT_MND_CREATE_DB_RSP = 308
TDMT_MND_DROP_DB = 309
TDMT_MND_DROP_DB_RSP = 310
TDMT_MND_USE_DB = 311
TDMT_MND_USE_DB_RSP = 312
TDMT_MND_ALTER_DB = 313
TDMT_MND_ALTER_DB_RSP = 314
TDMT_MND_SYNC_DB = 315
TDMT_MND_SYNC_DB_RSP = 316
TDMT_MND_COMPACT_DB = 317
TDMT_MND_COMPACT_DB_RSP = 318
TDMT_MND_TRIM_DB = 319
TDMT_MND_TRIM_DB_RSP = 320
TDMT_MND_GET_DB_CFG = 321
TDMT_MND_GET_DB_CFG_RSP = 322
TDMT_MND_VGROUP_LIST = 323
TDMT_MND_VGROUP_LIST_RSP = 324
TDMT_MND_CREATE_FUNC = 325
TDMT_MND_CREATE_FUNC_RSP = 326
TDMT_MND_RETRIEVE_FUNC = 327
TDMT_MND_RETRIEVE_FUNC_RSP = 328
TDMT_MND_DROP_FUNC = 329
TDMT_MND_DROP_FUNC_RSP = 330
TDMT_MND_CREATE_STB = 331
TDMT_MND_CREATE_STB_RSP = 332
TDMT_MND_ALTER_STB = 333
TDMT_MND_ALTER_STB_RSP = 334
TDMT_MND_DROP_STB = 335
TDMT_MND_DROP_STB_RSP = 336
TDMT_MND_TABLE_META = 337
TDMT_MND_TABLE_META_RSP = 338
TDMT_MND_CREATE_SMA = 339
TDMT_MND_CREATE_SMA_RSP = 340
TDMT_MND_DROP_SMA = 341
TDMT_MND_DROP_SMA_RSP = 342
TDMT_MND_CREATE_STREAM = 343
TDMT_MND_CREATE_STREAM_RSP = 344
TDMT_MND_ALTER_STREAM = 345
TDMT_MND_ALTER_STREAM_RSP = 346
TDMT_MND_DROP_STREAM = 347
TDMT_MND_DROP_STREAM_RSP = 348
TDMT_MND_RECOVER_STREAM = 349
TDMT_MND_RECOVER_STREAM_RSP = 350
TDMT_MND_CREATE_INDEX = 351
TDMT_MND_CREATE_INDEX_RSP = 352
TDMT_MND_DROP_INDEX = 353
TDMT_MND_DROP_INDEX_RSP = 354
TDMT_MND_GET_INDEX = 355
TDMT_MND_GET_INDEX_RSP = 356
TDMT_MND_GET_TABLE_INDEX = 357
TDMT_MND_GET_TABLE_INDEX_RSP = 358
TDMT_MND_BATCH_META = 359
TDMT_MND_BATCH_META_RSP = 360
TDMT_MND_TABLE_CFG = 361
TDMT_MND_TABLE_CFG_RSP = 362
TDMT_MND_TMQ_CREATE_TOPIC = 363
TDMT_MND_TMQ_CREATE_TOPIC_RSP = 364
TDMT_MND_UNUSED1 = 365
TDMT_MND_UNUSED1_RSP = 366
TDMT_MND_TMQ_DROP_TOPIC = 367
TDMT_MND_TMQ_DROP_TOPIC_RSP = 368
TDMT_MND_TMQ_SUBSCRIBE = 369
TDMT_MND_TMQ_SUBSCRIBE_RSP = 370
TDMT_MND_TMQ_ASK_EP = 371
TDMT_MND_TMQ_ASK_EP_RSP = 372
TDMT_MND_TMQ_CONSUMER_RECOVER = 373
TDMT_MND_TMQ_CONSUMER_RECOVER_RSP = 374
TDMT_MND_TMQ_HB = 375
TDMT_MND_TMQ_HB_RSP = 376
TDMT_MND_TMQ_DO_REBALANCE = 377
TDMT_MND_TMQ_DO_REBALANCE_RSP = 378
TDMT_MND_TMQ_DROP_CGROUP = 379
TDMT_MND_TMQ_DROP_CGROUP_RSP = 380
TDMT_MND_CREATE_VG = 381
TDMT_MND_CREATE_VG_RSP = 382
TDMT_MND_TMQ_TIMER = 383
TDMT_MND_TMQ_TIMER_RSP = 384
TDMT_MND_TELEM_TIMER = 385
TDMT_MND_TELEM_TIMER_RSP = 386
TDMT_MND_TRANS_TIMER = 387
TDMT_MND_TRANS_TIMER_RSP = 388
TDMT_MND_TTL_TIMER = 389
TDMT_MND_TTL_TIMER_RSP = 390
TDMT_MND_GRANT_HB_TIMER = 391
TDMT_MND_GRANT_HB_TIMER_RSP = 392
TDMT_MND_NODECHECK_TIMER = 393
TDMT_MND_NODECHECK_TIMER_RSP = 394
TDMT_MND_KILL_TRANS = 395
TDMT_MND_KILL_TRANS_RSP = 396
TDMT_MND_KILL_QUERY = 397
TDMT_MND_KILL_QUERY_RSP = 398
TDMT_MND_KILL_CONN = 399
TDMT_MND_KILL_CONN_RSP = 400
TDMT_MND_HEARTBEAT = 401
TDMT_MND_HEARTBEAT_RSP = 402
TDMT_MND_STATUS = 403
TDMT_MND_STATUS_RSP = 404
TDMT_MND_SHOW = 405
TDMT_MND_SHOW_RSP = 406
TDMT_MND_SYSTABLE_RETRIEVE = 407
TDMT_MND_SYSTABLE_RETRIEVE_RSP = 408
TDMT_MND_GRANT = 409
TDMT_MND_GRANT_RSP = 410
TDMT_MND_AUTH = 411
TDMT_MND_AUTH_RSP = 412
TDMT_MND_APPLY_MSG = 413
TDMT_MND_APPLY_MSG_RSP = 414
TDMT_MND_BALANCE_VGROUP = 415
TDMT_MND_BALANCE_VGROUP_RSP = 416
TDMT_MND_MERGE_VGROUP = 417
TDMT_MND_MERGE_VGROUP_RSP = 418
TDMT_MND_REDISTRIBUTE_VGROUP = 419
TDMT_MND_REDISTRIBUTE_VGROUP_RSP = 420
TDMT_MND_SPLIT_VGROUP = 421
TDMT_MND_SPLIT_VGROUP_RSP = 422
TDMT_MND_SHOW_VARIABLES = 423
TDMT_MND_SHOW_VARIABLES_RSP = 424
TDMT_MND_SERVER_VERSION = 425
TDMT_MND_SERVER_VERSION_RSP = 426
TDMT_MND_UPTIME_TIMER = 427
TDMT_MND_UPTIME_TIMER_RSP = 428
TDMT_MND_TMQ_LOST_CONSUMER_CLEAR = 429
TDMT_MND_TMQ_LOST_CONSUMER_CLEAR_RSP = 430
TDMT_MND_STREAM_HEARTBEAT = 431
TDMT_MND_STREAM_HEARTBEAT_RSP = 432
TDMT_MND_RETRIEVE_IP_WHITE = 433
TDMT_MND_RETRIEVE_IP_WHITE_RSP = 434
TDMT_MND_GET_USER_WHITELIST = 435
TDMT_MND_GET_USER_WHITELIST_RSP = 436
TDMT_MND_NOTIFY = 437
TDMT_MND_NOTIFY_RSP = 438
TDMT_MND_BALANCE_VGROUP_LEADER = 439
TDMT_MND_BALANCE_VGROUP_LEADER_RSP = 440
TDMT_MND_RESTORE_DNODE = 441
TDMT_MND_RESTORE_DNODE_RSP = 442
TDMT_MND_PAUSE_STREAM = 443
TDMT_MND_PAUSE_STREAM_RSP = 444
TDMT_MND_RESUME_STREAM = 445
TDMT_MND_RESUME_STREAM_RSP = 446
TDMT_MND_STREAM_UPDATE_CHKPT_EVT = 447
TDMT_MND_STREAM_UPDATE_CHKPT_EVT_RSP = 448
TDMT_MND_STREAM_BEGIN_CHECKPOINT = 449
TDMT_MND_STREAM_BEGIN_CHECKPOINT_RSP = 450
TDMT_MND_STREAM_CHKPT_REPORT = 451
TDMT_MND_STREAM_CHKPT_REPORT_RSP = 452
TDMT_MND_STREAM_NODECHANGE_CHECK = 453
TDMT_MND_STREAM_NODECHANGE_CHECK_RSP = 454
TDMT_MND_TRIM_DB_TIMER = 455
TDMT_MND_TRIM_DB_TIMER_RSP = 456
TDMT_MND_GRANT_NOTIFY = 457
TDMT_MND_GRANT_NOTIFY_RSP = 458
TDMT_MND_CREATE_VIEW = 459
TDMT_MND_CREATE_VIEW_RSP = 460
TDMT_MND_DROP_VIEW = 461
TDMT_MND_DROP_VIEW_RSP = 462
TDMT_MND_VIEW_META = 463
TDMT_MND_VIEW_META_RSP = 464
TDMT_MND_STATIS = 465
TDMT_MND_STATIS_RSP = 466
TDMT_MND_KILL_COMPACT = 467
TDMT_MND_KILL_COMPACT_RSP = 468
TDMT_MND_COMPACT_TIMER = 469
TDMT_MND_COMPACT_TIMER_RSP = 470
TDMT_MND_STREAM_REQ_CHKPT = 471
TDMT_MND_STREAM_REQ_CHKPT_RSP = 472
TDMT_MND_CONFIG_CLUSTER = 473
TDMT_MND_CONFIG_CLUSTER_RSP = 474
TDMT_MND_CREATE_ENCRYPT_KEY = 475
TDMT_MND_CREATE_ENCRYPT_KEY_RSP = 476
TDMT_MND_S3MIGRATE_DB = 477
TDMT_MND_S3MIGRATE_DB_RSP = 478
TDMT_MND_S3MIGRATE_DB_TIMER = 479
TDMT_MND_S3MIGRATE_DB_TIMER_RSP = 480
TDMT_MND_UNUSED2 = 481
TDMT_MND_UNUSED2_RSP = 482
TDMT_MND_CREATE_TSMA = 483
TDMT_MND_CREATE_TSMA_RSP = 484
TDMT_MND_DROP_TSMA = 485
TDMT_MND_DROP_TSMA_RSP = 486
TDMT_MND_STB_DROP = 487
TDMT_MND_STB_DROP_RSP = 488
TDMT_MND_GET_TABLE_TSMA = 489
TDMT_MND_GET_TABLE_TSMA_RSP = 490
TDMT_MND_GET_TSMA = 491
TDMT_MND_GET_TSMA_RSP = 492
TDMT_MND_DROP_TB_WITH_TSMA = 493
TDMT_MND_DROP_TB_WITH_TSMA_RSP = 494
TDMT_MND_MAX_MSG = 495
TDMT_MND_MAX_MSG_RSP = 496
TDMT_VND_SUBMIT = 513
TDMT_VND_SUBMIT_RSP = 514
TDMT_VND_CREATE_TABLE = 515
TDMT_VND_CREATE_TABLE_RSP = 516
TDMT_VND_ALTER_TABLE = 517
TDMT_VND_ALTER_TABLE_RSP = 518
TDMT_VND_DROP_TABLE = 519
TDMT_VND_DROP_TABLE_RSP = 520
TDMT_VND_UPDATE_TAG_VAL = 521
TDMT_VND_UPDATE_TAG_VAL_RSP = 522
TDMT_VND_TABLE_META = 523
TDMT_VND_TABLE_META_RSP = 524
TDMT_VND_TABLES_META = 525
TDMT_VND_TABLES_META_RSP = 526
TDMT_VND_TABLE_CFG = 527
TDMT_VND_TABLE_CFG_RSP = 528
TDMT_VND_BATCH_META = 529
TDMT_VND_BATCH_META_RSP = 530
TDMT_VND_CREATE_STB = 531
TDMT_VND_CREATE_STB_RSP = 532
TDMT_VND_ALTER_STB = 533
TDMT_VND_ALTER_STB_RSP = 534
TDMT_VND_DROP_STB = 535
TDMT_VND_DROP_STB_RSP = 536
TDMT_VND_UNUSED1 = 537
TDMT_VND_UNUSED1_RSP = 538
TDMT_VND_UNUSED2 = 539
TDMT_VND_UNUSED2_RSP = 540
TDMT_VND_UNUSED3 = 541
TDMT_VND_UNUSED3_RSP = 542
TDMT_VND_UNUSED4 = 543
TDMT_VND_UNUSED4_RSP = 544
TDMT_VND_UNUSED5 = 545
TDMT_VND_UNUSED5_RSP = 546
TDMT_VND_UNUSED6 = 547
TDMT_VND_UNUSED6_RSP = 548
TDMT_VND_UNUSED7 = 549
TDMT_VND_UNUSED7_RSP = 550
TDMT_VND_UNUSED8 = 551
TDMT_VND_UNUSED8_RSP = 552
TDMT_VND_UNUSED9 = 553
TDMT_VND_UNUSED9_RSP = 554
TDMT_VND_UNUSED10 = 555
TDMT_VND_UNUSED10_RSP = 556
TDMT_VND_UNUSED11 = 557
TDMT_VND_UNUSED11_RSP = 558
TDMT_VND_UNUSED12 = 559
TDMT_VND_UNUSED12_RSP = 560
TDMT_VND_UNUSED13 = 561
TDMT_VND_UNUSED13_RSP = 562
TDMT_VND_UNUSED14 = 563
TDMT_VND_UNUSED14_RSP = 564
TDMT_VND_UNUSED15 = 565
TDMT_VND_UNUSED15_RSP = 566
TDMT_VND_CREATE_SMA = 567
TDMT_VND_CREATE_SMA_RSP = 568
TDMT_VND_CANCEL_SMA = 569
TDMT_VND_CANCEL_SMA_RSP = 570
TDMT_VND_DROP_SMA = 571
TDMT_VND_DROP_SMA_RSP = 572
TDMT_VND_SUBMIT_RSMA = 573
TDMT_VND_SUBMIT_RSMA_RSP = 574
TDMT_VND_FETCH_RSMA = 575
TDMT_VND_FETCH_RSMA_RSP = 576
TDMT_VND_EXEC_RSMA = 577
TDMT_VND_EXEC_RSMA_RSP = 578
TDMT_VND_DELETE = 579
TDMT_VND_DELETE_RSP = 580
TDMT_VND_BATCH_DEL = 581
TDMT_VND_BATCH_DEL_RSP = 582
TDMT_VND_ALTER_CONFIG = 583
TDMT_VND_ALTER_CONFIG_RSP = 584
TDMT_VND_ALTER_REPLICA = 585
TDMT_VND_ALTER_REPLICA_RSP = 586
TDMT_VND_ALTER_CONFIRM = 587
TDMT_VND_ALTER_CONFIRM_RSP = 588
TDMT_VND_ALTER_HASHRANGE = 589
TDMT_VND_ALTER_HASHRANGE_RSP = 590
TDMT_VND_COMPACT = 591
TDMT_VND_COMPACT_RSP = 592
TDMT_VND_DROP_TTL_TABLE = 593
TDMT_VND_DROP_TTL_TABLE_RSP = 594
TDMT_VND_TRIM = 595
TDMT_VND_TRIM_RSP = 596
TDMT_VND_COMMIT = 597
TDMT_VND_COMMIT_RSP = 598
TDMT_VND_CREATE_INDEX = 599
TDMT_VND_CREATE_INDEX_RSP = 600
TDMT_VND_DROP_INDEX = 601
TDMT_VND_DROP_INDEX_RSP = 602
TDMT_VND_DISABLE_WRITE = 603
TDMT_VND_DISABLE_WRITE_RSP = 604
TDMT_VND_QUERY_COMPACT_PROGRESS = 605
TDMT_VND_QUERY_COMPACT_PROGRESS_RSP = 606
TDMT_VND_KILL_COMPACT = 607
TDMT_VND_KILL_COMPACT_RSP = 608
TDMT_VND_S3MIGRATE = 609
TDMT_VND_S3MIGRATE_RSP = 610
TDMT_VND_ARB_HEARTBEAT = 611
TDMT_VND_ARB_HEARTBEAT_RSP = 612
TDMT_VND_ARB_CHECK_SYNC = 613
TDMT_VND_ARB_CHECK_SYNC_RSP = 614
TDMT_VND_FETCH_TTL_EXPIRED_TBS = 615
TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP = 616
TDMT_VND_MAX_MSG = 617
TDMT_VND_MAX_MSG_RSP = 618
TDMT_SCH_QUERY = 769
TDMT_SCH_QUERY_RSP = 770
TDMT_SCH_MERGE_QUERY = 771
TDMT_SCH_MERGE_QUERY_RSP = 772
TDMT_SCH_QUERY_CONTINUE = 773
TDMT_SCH_QUERY_CONTINUE_RSP = 774
TDMT_SCH_QUERY_HEARTBEAT = 775
TDMT_SCH_QUERY_HEARTBEAT_RSP = 776
TDMT_SCH_FETCH = 777
TDMT_SCH_FETCH_RSP = 778
TDMT_SCH_MERGE_FETCH = 779
TDMT_SCH_MERGE_FETCH_RSP = 780
TDMT_SCH_CANCEL_TASK = 781
TDMT_SCH_CANCEL_TASK_RSP = 782
TDMT_SCH_DROP_TASK = 783
TDMT_SCH_DROP_TASK_RSP = 784
TDMT_SCH_EXPLAIN = 785
TDMT_SCH_EXPLAIN_RSP = 786
TDMT_SCH_LINK_BROKEN = 787
TDMT_SCH_LINK_BROKEN_RSP = 788
TDMT_SCH_TASK_NOTIFY = 789
TDMT_SCH_TASK_NOTIFY_RSP = 790
TDMT_SCH_MAX_MSG = 791
TDMT_SCH_MAX_MSG_RSP = 792
TDMT_STREAM_TASK_DEPLOY = 1025
TDMT_STREAM_TASK_DEPLOY_RSP = 1026
TDMT_STREAM_TASK_DROP = 1027
TDMT_STREAM_TASK_DROP_RSP = 1028
TDMT_STREAM_TASK_RUN = 1029
TDMT_STREAM_TASK_RUN_RSP = 1030
TDMT_STREAM_TASK_DISPATCH = 1031
TDMT_STREAM_TASK_DISPATCH_RSP = 1032
TDMT_STREAM_TASK_UPDATE_CHKPT = 1033
TDMT_STREAM_TASK_UPDATE_CHKPT_RSP = 1034
TDMT_STREAM_RETRIEVE = 1035
TDMT_STREAM_RETRIEVE_RSP = 1036
TDMT_STREAM_TASK_CHECKPOINT_READY = 1037
TDMT_STREAM_TASK_CHECKPOINT_READY_RSP = 1038
TDMT_STREAM_TASK_REPORT_CHECKPOINT = 1039
TDMT_STREAM_TASK_REPORT_CHECKPOINT_RSP = 1040
TDMT_STREAM_TASK_RESTORE_CHECKPOINT = 1041
TDMT_STREAM_TASK_RESTORE_CHECKPOINT_RSP = 1042
TDMT_STREAM_TASK_PAUSE = 1043
TDMT_STREAM_TASK_PAUSE_RSP = 1044
TDMT_STREAM_TASK_RESUME = 1045
TDMT_STREAM_TASK_RESUME_RSP = 1046
TDMT_STREAM_TASK_STOP = 1047
TDMT_STREAM_TASK_STOP_RSP = 1048
TDMT_STREAM_UNUSED = 1049
TDMT_STREAM_UNUSED_RSP = 1050
TDMT_STREAM_CREATE = 1051
TDMT_STREAM_CREATE_RSP = 1052
TDMT_STREAM_DROP = 1053
TDMT_STREAM_DROP_RSP = 1054
TDMT_STREAM_RETRIEVE_TRIGGER = 1055
TDMT_STREAM_RETRIEVE_TRIGGER_RSP = 1056
TDMT_STREAM_MAX_MSG = 1057
TDMT_STREAM_MAX_MSG_RSP = 1058
TDMT_MON_MAX_MSG = 1281
TDMT_MON_MAX_MSG_RSP = 1282
TDMT_SYNC_TIMEOUT = 1537
TDMT_SYNC_TIMEOUT_RSP = 1538
TDMT_SYNC_TIMEOUT_ELECTION = 1539
TDMT_SYNC_TIMEOUT_ELECTION_RSP = 1540
TDMT_SYNC_PING_REPLY = 1541
TDMT_SYNC_PING_REPLY_RSP = 1542
TDMT_SYNC_CLIENT_REQUEST = 1543
TDMT_SYNC_CLIENT_REQUEST_RSP = 1544
TDMT_SYNC_CLIENT_REQUEST_BATCH = 1545
TDMT_SYNC_CLIENT_REQUEST_BATCH_RSP = 1546
TDMT_SYNC_CLIENT_REQUEST_REPLY = 1547
TDMT_SYNC_CLIENT_REQUEST_REPLY_RSP = 1548
TDMT_SYNC_REQUEST_VOTE = 1549
TDMT_SYNC_REQUEST_VOTE_RSP = 1550
TDMT_SYNC_REQUEST_VOTE_REPLY = 1551
TDMT_SYNC_REQUEST_VOTE_REPLY_RSP = 1552
TDMT_SYNC_APPEND_ENTRIES = 1553
TDMT_SYNC_APPEND_ENTRIES_RSP = 1554
TDMT_SYNC_APPEND_ENTRIES_BATCH = 1555
TDMT_SYNC_APPEND_ENTRIES_BATCH_RSP = 1556
TDMT_SYNC_APPEND_ENTRIES_REPLY = 1557
TDMT_SYNC_APPEND_ENTRIES_REPLY_RSP = 1558
TDMT_SYNC_NOOP = 1559
TDMT_SYNC_NOOP_RSP = 1560
TDMT_SYNC_UNKNOWN = 1561
TDMT_SYNC_UNKNOWN_RSP = 1562
TDMT_SYNC_COMMON_RESPONSE = 1563
TDMT_SYNC_COMMON_RESPONSE_RSP = 1564
TDMT_SYNC_APPLY_MSG = 1565
TDMT_SYNC_APPLY_MSG_RSP = 1566
TDMT_SYNC_CONFIG_CHANGE = 1567
TDMT_SYNC_CONFIG_CHANGE_RSP = 1568
TDMT_SYNC_CONFIG_CHANGE_FINISH = 1569
TDMT_SYNC_CONFIG_CHANGE_FINISH_RSP = 1570
TDMT_SYNC_SNAPSHOT_SEND = 1571
TDMT_SYNC_SNAPSHOT_SEND_RSP = 1572
TDMT_SYNC_SNAPSHOT_RSP = 1573
TDMT_SYNC_SNAPSHOT_RSP_RSP = 1574
TDMT_SYNC_LEADER_TRANSFER = 1575
TDMT_SYNC_LEADER_TRANSFER_RSP = 1576
TDMT_SYNC_SET_MNODE_STANDBY = 1577
TDMT_SYNC_SET_MNODE_STANDBY_RSP = 1578
TDMT_SYNC_SET_VNODE_STANDBY = 1579
TDMT_SYNC_SET_VNODE_STANDBY_RSP = 1580
TDMT_SYNC_HEARTBEAT = 1581
TDMT_SYNC_HEARTBEAT_RSP = 1582
TDMT_SYNC_HEARTBEAT_REPLY = 1583
TDMT_SYNC_HEARTBEAT_REPLY_RSP = 1584
TDMT_SYNC_LOCAL_CMD = 1585
TDMT_SYNC_LOCAL_CMD_RSP = 1586
TDMT_SYNC_PREP_SNAPSHOT = 1587
TDMT_SYNC_PREP_SNAPSHOT_RSP = 1588
TDMT_SYNC_PREP_SNAPSHOT_REPLY = 1589
TDMT_SYNC_PREP_SNAPSHOT_REPLY_RSP = 1590
TDMT_SYNC_UNUSED_CODE = 1591
TDMT_SYNC_UNUSED_CODE_RSP = 1592
TDMT_SYNC_FORCE_FOLLOWER = 1593
TDMT_SYNC_FORCE_FOLLOWER_RSP = 1594
TDMT_SYNC_SET_ASSIGNED_LEADER = 1595
TDMT_SYNC_SET_ASSIGNED_LEADER_RSP = 1596
TDMT_SYNC_MAX_MSG = 1597
TDMT_SYNC_MAX_MSG_RSP = 1598
TDMT_VND_STREAM_SCAN_HISTORY = 1793
TDMT_VND_STREAM_SCAN_HISTORY_RSP = 1794
TDMT_VND_STREAM_CHECK_POINT_SOURCE = 1795
TDMT_VND_STREAM_CHECK_POINT_SOURCE_RSP = 1796
TDMT_VND_STREAM_TASK_UPDATE = 1797
TDMT_VND_STREAM_TASK_UPDATE_RSP = 1798
TDMT_VND_STREAM_TASK_RESET = 1799
TDMT_VND_STREAM_TASK_RESET_RSP = 1800
TDMT_VND_STREAM_TASK_CHECK = 1801
TDMT_VND_STREAM_TASK_CHECK_RSP = 1802
TDMT_VND_STREAM_UNUSED = 1803
TDMT_VND_STREAM_UNUSED_RSP = 1804
TDMT_VND_GET_STREAM_PROGRESS = 1805
TDMT_VND_GET_STREAM_PROGRESS_RSP = 1806
TDMT_VND_STREAM_MAX_MSG = 1807
TDMT_VND_STREAM_MAX_MSG_RSP = 1808
TDMT_VND_TMQ_SUBSCRIBE = 2049
TDMT_VND_TMQ_SUBSCRIBE_RSP = 2050
TDMT_VND_TMQ_DELETE_SUB = 2051
TDMT_VND_TMQ_DELETE_SUB_RSP = 2052
TDMT_VND_TMQ_COMMIT_OFFSET = 2053
TDMT_VND_TMQ_COMMIT_OFFSET_RSP = 2054
TDMT_VND_TMQ_SEEK = 2055
TDMT_VND_TMQ_SEEK_RSP = 2056
TDMT_VND_TMQ_ADD_CHECKINFO = 2057
TDMT_VND_TMQ_ADD_CHECKINFO_RSP = 2058
TDMT_VND_TMQ_DEL_CHECKINFO = 2059
TDMT_VND_TMQ_DEL_CHECKINFO_RSP = 2060
TDMT_VND_TMQ_CONSUME = 2061
TDMT_VND_TMQ_CONSUME_RSP = 2062
TDMT_VND_TMQ_CONSUME_PUSH = 2063
TDMT_VND_TMQ_CONSUME_PUSH_RSP = 2064
TDMT_VND_TMQ_VG_WALINFO = 2065
TDMT_VND_TMQ_VG_WALINFO_RSP = 2066
TDMT_VND_TMQ_VG_COMMITTEDINFO = 2067
TDMT_VND_TMQ_VG_COMMITTEDINFO_RSP = 2068
TDMT_VND_TMQ_MAX_MSG = 2069
TDMT_VND_TMQ_MAX_MSG_RSP = 2070
TDMT_MND_ARB_HEARTBEAT_TIMER = 2305
TDMT_MND_ARB_HEARTBEAT_TIMER_RSP = 2306
TDMT_MND_ARB_CHECK_SYNC_TIMER = 2307
TDMT_MND_ARB_CHECK_SYNC_TIMER_RSP = 2308
TDMT_MND_ARB_UPDATE_GROUP = 2309
TDMT_MND_ARB_UPDATE_GROUP_RSP = 2310
TDMT_MND_ARB_UPDATE_GROUP_BATCH = 2311
TDMT_MND_ARB_UPDATE_GROUP_BATCH_RSP = 2312
TDMT_MND_ARB_MAX_MSG = 2313
TDMT_MND_ARB_MAX_MSG_RSP = 2314

274
source/common/test/tmsgTest.cpp
View File

@ -1,5 +1,14 @@
#include <iostream>
#include <iomanip>
#include <fstream>
#include <sstream>
#include <string>
#include <vector>
#include <unistd.h>
#include <limits.h>
#include <sys/stat.h>
#include <algorithm>
#include <unordered_map>
#include <gtest/gtest.h>
#include "tmsg.h"
@ -12,15 +21,260 @@
#undef TD_MSG_SEG_CODE_
#include "tmsgdef.h"
TEST(td_msg_test, simple_msg_test) {
// std::cout << TMSG_INFO(TDMT_VND_DROP_TABLE) << std::endl;
// std::cout << TMSG_INFO(TDMT_MND_DROP_SUPER_TABLE) << std::endl;
// std::cout << TMSG_INFO(TDMT_MND_CREATE_SUPER_TABLE) << std::endl;
#undef getline
#undef close
int32_t msgSize = sizeof(tMsgTypeInfo) / sizeof(SMsgTypeInfo);
for (int32_t i = 0; i < msgSize; ++i) {
SMsgTypeInfo *pInfo = &tMsgTypeInfo[i];
std::cout << i * 2 + 1 << " " << pInfo->name << " " << pInfo->type << std::endl;
std::cout << i * 2 + 2 << " " << pInfo->rspName << " " << pInfo->rspType << std::endl;
using namespace std;
enum class ParseStatus {
Success,
FileNotExist,
FileNotOpen,
ResponseWithoutRequest,
RequestWithoutResponse
};
typedef struct {
string name;
string rspName;
int32_t type;
int32_t rspType;
} STestMsgTypeInfo;
string getExecutableDirectory() {
char result[PATH_MAX];
ssize_t count = readlink("/proc/self/exe", result, PATH_MAX);
if (count != -1) {
result[count] = '\0';
string path(result);
size_t pos = path.rfind('/');
if (pos != string::npos) {
path.erase(pos + 1);
}
return path;
} else {
throw std::runtime_error("Failed to get the executable's directory");
}
}
// parses key-value pairs from strings
pair<string, int32_t> parseKeyValuePair(const string &line, char delim = '=') {
size_t pos = line.find(delim);
if (pos == string::npos)
return make_pair("", 0);
string key = line.substr(0, pos);
// remove leading spaces
size_t firstNotSpace = key.find_first_not_of(" ");
if (firstNotSpace != string::npos) {
key = key.substr(firstNotSpace);
} else {
key.clear();
}
// remove ending spaces
size_t lastNotSpace = key.find_last_not_of(" ");
if (lastNotSpace != string::npos) {
key = key.substr(0, lastNotSpace + 1);
}
if (key.front() == '"' && key.back() == '"')
key = key.substr(1, key.size() - 2);
if (key.front() == '\'' && key.back() == '\'')
key = key.substr(1, key.size() - 2);
string valStr = line.substr(pos + 1);
int32_t val = stoi(valStr);
return make_pair(key, val);
}
// read the configuration file and parse it into the STestMsgTypeInfo array
ParseStatus readConfig(const string& filePath, vector<STestMsgTypeInfo>& msgTypes) {
ifstream file(filePath);
if (!file.is_open()) {
if (file.fail() && errno == ENOENT) {
cerr << "Error: The file does not exist, file: " << filePath << endl;
return ParseStatus::FileNotExist;
} else {
cerr << "Error: Could not open the file, file: " << filePath << endl;
return ParseStatus::FileNotOpen;
}
}
auto endsWith = [](const string& str, const string& suffix) {
if (str.length() < suffix.length()) {
return false;
}
return equal(str.end() - suffix.length(), str.end(), suffix.begin());
};
bool evenLine = true;
string line;
string suffix("_RSP");
pair<string, int32_t> reqKwInfo;
while (std::getline(file, line)) {
char delim = '#';
if (line.find('=') != string::npos) {
delim = '=';
} else if (line.find(':') != string::npos) {
delim = ':';
} else if (line.find('{') != string::npos || line.find('}') != string::npos) {
// TODO: parse json format
continue;
} else {
continue;
}
auto curKwInfo = parseKeyValuePair(line, delim);
evenLine = ! evenLine;
// check message type
if (evenLine == false) { // req msg
reqKwInfo = curKwInfo;
} else { // rsp msg
if (reqKwInfo.first.empty()) {
cerr << "Error: Found a response message without a matching request, rsp: " << curKwInfo.first << endl;
return ParseStatus::ResponseWithoutRequest;
} else if (!endsWith(curKwInfo.first, suffix)) {
cerr << "Error: A request message was not followed by a matching response, req: " << reqKwInfo.first << endl;
return ParseStatus::RequestWithoutResponse;
} else {
STestMsgTypeInfo msgInfo;
msgInfo.name = reqKwInfo.first;
msgInfo.rspName = curKwInfo.first;
msgInfo.type = reqKwInfo.second;
msgInfo.rspType = curKwInfo.second;
msgTypes.push_back(msgInfo);
// reset req info
reqKwInfo = make_pair("", -1);
}
}
}
if (!reqKwInfo.first.empty()) {
cerr << "Error: A request message was not followed by a matching response, req: " << reqKwInfo.first << endl;
return ParseStatus::RequestWithoutResponse;
}
return ParseStatus::Success;
}
TEST(td_msg_test, msg_type_compatibility_test) {
// cout << TMSG_INFO(TDMT_VND_DROP_TABLE) << endl;
// cout << TMSG_INFO(TDMT_MND_DROP_SUPER_TABLE) << endl;
// cout << TMSG_INFO(TDMT_MND_CREATE_SUPER_TABLE) << endl;
// int32_t msgSize = sizeof(tMsgTypeInfo) / sizeof(SMsgTypeInfo);
// for (int32_t i = 0; i < msgSize; ++i) {
// SMsgTypeInfo *pInfo = &tMsgTypeInfo[i];
// cout << i * 2 + 1 << " " << pInfo->name << " " << pInfo->type << endl;
// cout << i * 2 + 2 << " " << pInfo->rspName << " " << pInfo->rspType << endl;
// }
// current msgs: to map
unordered_map<string, const SMsgTypeInfo*> map;
for (const auto& info : tMsgTypeInfo) {
map[info.name] = &info;
}
string configFileName = "msgTypeTable.ini";
string execDir = getExecutableDirectory();
string configFilePath(execDir + configFileName);
vector<STestMsgTypeInfo> msgTypes;
ParseStatus status = readConfig(configFilePath, msgTypes);
switch (status) {
case ParseStatus::Success:
for (const auto& stdInfo : msgTypes) {
auto it = map.find(stdInfo.name);
if (it == map.end()) {
FAIL() << "Error: Could not find msg: " << stdInfo.name << ".";
} else {
auto newInfo = it->second;
ASSERT_STREQ(stdInfo.name.c_str(), newInfo->name);
ASSERT_STREQ(stdInfo.rspName.c_str(), newInfo->rspName);
ASSERT_EQ(stdInfo.type, newInfo->type)
<< "Message type mismatch(" << stdInfo.name << "): expected " << stdInfo.type << ", got " << newInfo->type << ".";
ASSERT_EQ(stdInfo.rspType, newInfo->rspType)
<< "Message response type mismatch(" << stdInfo.rspName << "): expected " << stdInfo.rspType << ", got " << newInfo->rspType << ".";
}
}
break;
case ParseStatus::FileNotExist:
FAIL() << "Error: The file does not exist, file: " << configFileName << ".";
break;
case ParseStatus::FileNotOpen:
FAIL() << "Error: Could not open the file, file: " << configFileName << ".";
break;
case ParseStatus::ResponseWithoutRequest:
FAIL() << "Error: Found a response message without a matching request.";
break;
case ParseStatus::RequestWithoutResponse:
FAIL() << "Error: A request message was not followed by a matching response.";
break;
default:
FAIL() << "Unknown Error.";
break;
}
}
size_t maxLengthOfMsgType() {
size_t maxLen = 0;
for (const auto& info : tMsgTypeInfo) {
maxLen = std::max(maxLen, strlen(info.name));
maxLen = std::max(maxLen, strlen(info.rspName));
}
return (maxLen / 4 + 1) * 4;
}
void generateConfigFile(const string& filePath) {
size_t maxStringLength = maxLengthOfMsgType();
std::ofstream file(filePath);
if (!file.is_open()) {
cerr << "Failed to open file for writing, at: " << filePath << "." << endl;
return;
}
for (const auto& info : tMsgTypeInfo) {
file << std::left << std::setw(maxStringLength) << info.name << "= " << info.type << endl;
file << std::left << std::setw(maxStringLength) << info.rspName << "= " << info.rspType << endl;
}
if (file.fail()) {
cerr << "An error occurred while writing to the file." << endl;
} else {
cout << "Data successfully written to file: " << filePath << endl;
}
file.close();
}
void processCommandArgs(int argc, char** argv) {
for (int i = 1; i < argc; ++i) {
if (string(argv[i]) == "--output-config") {
string configFile = (i + 1 < argc) ? argv[++i] : "./msgTypeTable.ini";
generateConfigFile(configFile);
exit(0);
}
}
}
int main(int argc, char **argv) {
processCommandArgs(argc, argv);
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

2
source/dnode/mgmt/exe/dmMain.c
View File

@ -261,7 +261,7 @@ static void dmPrintVersion() {
printf("%s\ntaosd version: %s compatible_version: %s\n", TD_PRODUCT_NAME, version, compatible_version);
printf("git: %s\n", gitinfo);
#ifdef TD_ENTERPRISE
printf("git: %s\n", gitinfoOfInternal);
printf("gitOfInternal: %s\n", gitinfoOfInternal);
#endif
printf("build: %s\n", buildinfo);
}

2
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -232,6 +232,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_PAUSE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_CREATE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_DROP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_CREATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
@ -240,6 +241,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_KILL_COMPACT_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

9
source/dnode/mgmt/mgmt_snode/src/smHandle.c
View File

@ -75,26 +75,27 @@ SArray *smGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_PAUSE, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
code = 0;
_OVER:

21
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -144,7 +144,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
}
#if defined(TD_ENTERPRISE)
pCfg->tsdbCfg.encryptAlgorithm = pCreate->encryptAlgorithm;
if(pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4){
if (pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4) {
strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
#else
@ -160,7 +160,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->walCfg.level = pCreate->walLevel;
#if defined(TD_ENTERPRISE)
pCfg->walCfg.encryptAlgorithm = pCreate->encryptAlgorithm;
if(pCfg->walCfg.encryptAlgorithm == DND_CA_SM4){
if (pCfg->walCfg.encryptAlgorithm == DND_CA_SM4) {
strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
#else
@ -169,7 +169,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
#if defined(TD_ENTERPRISE)
pCfg->tdbEncryptAlgorithm = pCreate->encryptAlgorithm;
if(pCfg->tdbEncryptAlgorithm == DND_CA_SM4){
if (pCfg->tdbEncryptAlgorithm == DND_CA_SM4) {
strncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
#else
@ -278,7 +278,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
req.keepTimeOffset, req.s3ChunkSize, req.s3KeepLocal, req.s3Compact, req.isTsma, req.precision, req.compression,
req.minRows, req.maxRows, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize,
req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix,
req.replica, req.selfIndex, req.learnerReplica, req.learnerSelfIndex, req.strict, req.changeVersion,
req.replica, req.selfIndex, req.learnerReplica, req.learnerSelfIndex, req.strict, req.changeVersion,
req.encryptAlgorithm);
for (int32_t i = 0; i < req.replica; ++i) {
@ -304,8 +304,8 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return -1;
}
if(req.encryptAlgorithm == DND_CA_SM4){
if(strlen(tsEncryptKey) == 0){
if (req.encryptAlgorithm == DND_CA_SM4) {
if (strlen(tsEncryptKey) == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
dError("vgId:%d, failed to create vnode since encrypt key is empty", req.vgId);
return -1;
@ -482,7 +482,9 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
.diskPrimary = pVnode->diskPrimary,
};
tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path));
vmCloseVnode(pMgmt, pVnode, false);
bool commitAndRemoveWal = vnodeShouldRemoveWal(pVnode->pImpl);
vmCloseVnode(pMgmt, pVnode, commitAndRemoveWal);
int32_t diskPrimary = wrapperCfg.diskPrimary;
char path[TSDB_FILENAME_LEN] = {0};
@ -737,7 +739,9 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
.diskPrimary = pVnode->diskPrimary,
};
tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path));
vmCloseVnode(pMgmt, pVnode, false);
bool commitAndRemoveWal = vnodeShouldRemoveWal(pVnode->pImpl);
vmCloseVnode(pMgmt, pVnode, commitAndRemoveWal);
int32_t diskPrimary = wrapperCfg.diskPrimary;
char path[TSDB_FILENAME_LEN] = {0};
@ -967,6 +971,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

4
source/dnode/mgmt/node_mgmt/src/dmEnv.c
View File

@ -110,10 +110,8 @@ static bool dmCheckDiskSpace() {
int32_t dmDiskInit() {
SDnode *pDnode = dmInstance();
SDiskCfg dCfg = {0};
SDiskCfg dCfg = {.level = 0, .primary = 1, .disable = 0};
tstrncpy(dCfg.dir, tsDataDir, TSDB_FILENAME_LEN);
dCfg.level = 0;
dCfg.primary = 1;
SDiskCfg *pDisks = tsDiskCfg;
int32_t numOfDisks = tsDiskCfgNum;
if (numOfDisks <= 0 || pDisks == NULL) {

10
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -103,8 +103,8 @@ typedef enum {
TRN_CONFLICT_GLOBAL = 1,
TRN_CONFLICT_DB = 2,
TRN_CONFLICT_DB_INSIDE = 3,
TRN_CONFLICT_TOPIC = 4,
TRN_CONFLICT_TOPIC_INSIDE = 5,
// TRN_CONFLICT_TOPIC = 4,
// TRN_CONFLICT_TOPIC_INSIDE = 5,
TRN_CONFLICT_ARBGROUP = 6,
} ETrnConflct;
@ -332,10 +332,10 @@ typedef struct {
int8_t sysInfo;
int8_t enable;
union {
int8_t flag;
uint8_t flag;
struct {
int8_t createdb : 1;
int8_t reserve : 7;
uint8_t createdb : 1;
uint8_t reserve : 7;
};
};
int32_t acctId;

8
source/dnode/mnode/impl/inc/mndShow.h
View File

@ -23,6 +23,14 @@
extern "C" {
#endif
#define COL_DATA_SET_VAL_RET(pData, isNull, pObj) \
do { \
if ((code = colDataSetVal(pColInfo, numOfRows, (pData), (isNull))) != 0) { \
if (pObj) sdbRelease(pSdb, (pObj)); \
return code; \
} \
} while (0)
int32_t mndInitShow(SMnode *pMnode);
void mndCleanupShow(SMnode *pMnode);
void mndAddShowRetrieveHandle(SMnode *pMnode, EShowType showType, ShowRetrieveFp fp);

41
source/dnode/mnode/impl/inc/mndStream.h
View File

@ -26,13 +26,14 @@ extern "C" {
#define MND_STREAM_RESERVE_SIZE 64
#define MND_STREAM_VER_NUMBER 5
#define MND_STREAM_CREATE_NAME "stream-create"
#define MND_STREAM_CHECKPOINT_NAME "stream-checkpoint"
#define MND_STREAM_PAUSE_NAME "stream-pause"
#define MND_STREAM_RESUME_NAME "stream-resume"
#define MND_STREAM_DROP_NAME "stream-drop"
#define MND_STREAM_TASK_RESET_NAME "stream-task-reset"
#define MND_STREAM_TASK_UPDATE_NAME "stream-task-update"
#define MND_STREAM_CREATE_NAME "stream-create"
#define MND_STREAM_CHECKPOINT_NAME "stream-checkpoint"
#define MND_STREAM_PAUSE_NAME "stream-pause"
#define MND_STREAM_RESUME_NAME "stream-resume"
#define MND_STREAM_DROP_NAME "stream-drop"
#define MND_STREAM_TASK_RESET_NAME "stream-task-reset"
#define MND_STREAM_TASK_UPDATE_NAME "stream-task-update"
#define MND_STREAM_CHKPT_UPDATE_NAME "stream-chkpt-update"
typedef struct SStreamTransInfo {
int64_t startTime;
@ -51,6 +52,7 @@ typedef struct SStreamTransMgmt {
} SStreamTransMgmt;
typedef struct SStreamExecInfo {
bool initTaskList;
SArray *pNodeList;
int64_t ts; // snapshot ts
SStreamTransMgmt transMgmt;
@ -58,6 +60,7 @@ typedef struct SStreamExecInfo {
SArray *pTaskList;
TdThreadMutex lock;
SHashObj *pTransferStateStreams;
SHashObj *pChkptStreams;
} SStreamExecInfo;
extern SStreamExecInfo execInfo;
@ -78,7 +81,18 @@ typedef struct SOrphanTask {
typedef struct {
SMsgHead head;
} SMStreamHbRspMsg, SMStreamReqCheckpointRspMsg;
} SMStreamHbRspMsg, SMStreamReqCheckpointRsp, SMStreamUpdateChkptRsp;
typedef struct STaskChkptInfo {
int32_t nodeId;
int32_t taskId;
int64_t streamId;
int64_t checkpointId;
int64_t version;
int64_t ts;
int32_t transId;
int8_t dropHTask;
}STaskChkptInfo;
int32_t mndInitStream(SMnode *pMnode);
void mndCleanupStream(SMnode *pMnode);
@ -96,7 +110,7 @@ int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream);
SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady);
void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName);
int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
int32_t retryCode);
int32_t retryCode, int32_t acceptCode);
STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, const char *pMsg);
int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status);
SSdbRaw *mndStreamActionEncode(SStreamObj *pStream);
@ -114,14 +128,19 @@ int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *p
int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList);
int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream);
int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList);
int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq);
void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo *pExecInfo);
SStreamTaskIter *createStreamTaskIter(SStreamObj *pStream);
void destroyStreamTaskIter(SStreamTaskIter *pIter);
bool streamTaskIterNextTask(SStreamTaskIter *pIter);
SStreamTask *streamTaskIterGetCurrent(SStreamTaskIter *pIter);
void mndInitExecInfo();
void removeExpiredNodeInfo(const SArray *pNodeSnapshot);
void removeTasksInBuf(SArray* pTaskIds, SStreamExecInfo* pExecInfo);
void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo);
int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot);
void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
#ifdef __cplusplus
}

1
source/dnode/mnode/impl/inc/mndUser.h
View File

@ -53,6 +53,7 @@ void mndUpdateIpWhiteForAllUser(SMnode *pMnode, char *user, char *fqdn, int8_t t
int32_t mndRefreshUserIpWhiteList(SMnode *pMnode);
int64_t mndGetUserIpWhiteListVer(SMnode *pMnode, SUserObj *pUser);
#ifdef __cplusplus
}
#endif

30
source/dnode/mnode/impl/src/mndCluster.c
View File

@ -280,6 +280,7 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) {
static int32_t mndRetrieveClusters(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pMsg->info.node;
SSdb *pSdb = pMnode->pSdb;
int32_t code = 0;
int32_t numOfRows = 0;
int32_t cols = 0;
SClusterObj *pCluster = NULL;
@ -290,31 +291,44 @@ static int32_t mndRetrieveClusters(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock *
cols = 0;
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCluster->id, false);
COL_DATA_SET_VAL_RET((const char *)&pCluster->id, false, pCluster);
char buf[tListLen(pCluster->name) + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(buf, pCluster->name, pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, buf, false);
COL_DATA_SET_VAL_RET(buf, false, pCluster);
int32_t upTime = mndGetClusterUpTimeImp(pCluster);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&upTime, false);
COL_DATA_SET_VAL_RET((const char *)&upTime, false, pCluster);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pCluster->createdTime, false);
COL_DATA_SET_VAL_RET((const char *)&pCluster->createdTime, false, pCluster);
char ver[12] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(ver, tsVersionName, pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)ver, false);
COL_DATA_SET_VAL_RET((const char *)ver, false, pCluster);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
if (tsExpireTime <= 0) {
char expireTime[25] = {0};
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
if (GRANT_EXPIRE_UNLIMITED(tsExpireTime / 1000)) {
STR_WITH_MAXSIZE_TO_VARSTR(expireTime, "unlimited", pShow->pMeta->pSchemas[cols].bytes);
COL_DATA_SET_VAL_RET(expireTime, false, pCluster);
} else if (tsExpireTime <= 0) {
colDataSetNULL(pColInfo, numOfRows);
} else {
colDataSetVal(pColInfo, numOfRows, (const char *)&tsExpireTime, false);
char ts[20] = {0};
time_t expireSec = tsExpireTime / 1000;
struct tm ptm;
if (taosLocalTime(&expireSec, &ptm, ts) != NULL) {
strftime(ts, 20, "%Y-%m-%d %H:%M:%S", &ptm);
} else {
ts[0] = 0;
}
STR_WITH_MAXSIZE_TO_VARSTR(expireTime, ts, pShow->pMeta->pSchemas[cols].bytes);
COL_DATA_SET_VAL_RET(expireTime, false, pCluster);
}
sdbRelease(pSdb, pCluster);

244
source/dnode/mnode/impl/src/mndCompact.c
View File

@ -13,20 +13,19 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "mndCompact.h"
#include "mndTrans.h"
#include "mndShow.h"
#include "mndDb.h"
#include "mndCompactDetail.h"
#include "mndVgroup.h"
#include "tmsgcb.h"
#include "mndDnode.h"
#include "tmisce.h"
#include "audit.h"
#include "mndCompactDetail.h"
#include "mndDb.h"
#include "mndDnode.h"
#include "mndPrivilege.h"
#include "mndShow.h"
#include "mndTrans.h"
#include "mndVgroup.h"
#include "tmisce.h"
#include "tmsgcb.h"
#define MND_COMPACT_VER_NUMBER 1
#define MND_COMPACT_ID_LEN 11
#define MND_COMPACT_ID_LEN 11
static int32_t mndProcessCompactTimer(SRpcMsg *pReq);
@ -50,12 +49,9 @@ int32_t mndInitCompact(SMnode *pMnode) {
return sdbSetTable(pMnode->pSdb, table);
}
void mndCleanupCompact(SMnode *pMnode) {
mDebug("mnd compact cleanup");
}
void mndCleanupCompact(SMnode *pMnode) { mDebug("mnd compact cleanup"); }
void tFreeCompactObj(SCompactObj *pCompact) {
}
void tFreeCompactObj(SCompactObj *pCompact) {}
int32_t tSerializeSCompactObj(void *buf, int32_t bufLen, const SCompactObj *pObj) {
SEncoder encoder = {0};
@ -75,7 +71,7 @@ int32_t tSerializeSCompactObj(void *buf, int32_t bufLen, const SCompactObj *pObj
}
int32_t tDeserializeSCompactObj(void *buf, int32_t bufLen, SCompactObj *pObj) {
int8_t ex = 0;
int8_t ex = 0;
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
@ -94,7 +90,7 @@ int32_t tDeserializeSCompactObj(void *buf, int32_t bufLen, SCompactObj *pObj) {
SSdbRaw *mndCompactActionEncode(SCompactObj *pCompact) {
terrno = TSDB_CODE_SUCCESS;
void *buf = NULL;
void *buf = NULL;
SSdbRaw *pRaw = NULL;
int32_t tlen = tSerializeSCompactObj(NULL, 0, pCompact);
@ -102,8 +98,8 @@ SSdbRaw *mndCompactActionEncode(SCompactObj *pCompact) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER;
}
int32_t size = sizeof(int32_t) + tlen;
int32_t size = sizeof(int32_t) + tlen;
pRaw = sdbAllocRaw(SDB_COMPACT, MND_COMPACT_VER_NUMBER, size);
if (pRaw == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -127,7 +123,6 @@ SSdbRaw *mndCompactActionEncode(SCompactObj *pCompact) {
SDB_SET_BINARY(pRaw, dataPos, buf, tlen, OVER);
SDB_SET_DATALEN(pRaw, dataPos, OVER);
OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
@ -141,9 +136,9 @@ OVER:
}
SSdbRow *mndCompactActionDecode(SSdbRaw *pRaw) {
SSdbRow *pRow = NULL;
SCompactObj *pCompact = NULL;
void *buf = NULL;
SSdbRow *pRow = NULL;
SCompactObj *pCompact = NULL;
void *buf = NULL;
terrno = TSDB_CODE_SUCCESS;
int8_t sver = 0;
@ -184,7 +179,7 @@ SSdbRow *mndCompactActionDecode(SSdbRaw *pRaw) {
goto OVER;
}
//taosInitRWLatch(&pView->lock);
// taosInitRWLatch(&pView->lock);
OVER:
taosMemoryFreeClear(buf);
@ -210,15 +205,15 @@ int32_t mndCompactActionDelete(SSdb *pSdb, SCompactObj *pCompact) {
}
int32_t mndCompactActionUpdate(SSdb *pSdb, SCompactObj *pOldCompact, SCompactObj *pNewCompact) {
mTrace("compact:%" PRId32 ", perform update action, old row:%p new row:%p",
pOldCompact->compactId, pOldCompact, pNewCompact);
mTrace("compact:%" PRId32 ", perform update action, old row:%p new row:%p", pOldCompact->compactId, pOldCompact,
pNewCompact);
return 0;
}
SCompactObj *mndAcquireCompact(SMnode *pMnode, int64_t compactId) {
SSdb *pSdb = pMnode->pSdb;
SCompactObj *pCompact = sdbAcquire(pSdb, SDB_COMPACT, &compactId);
SSdb *pSdb = pMnode->pSdb;
SCompactObj *pCompact = sdbAcquire(pSdb, SDB_COMPACT, &compactId);
if (pCompact == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
terrno = TSDB_CODE_SUCCESS;
}
@ -230,8 +225,8 @@ void mndReleaseCompact(SMnode *pMnode, SCompactObj *pCompact) {
sdbRelease(pSdb, pCompact);
}
//compact db
int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompact, SDbObj *pDb, SCompactDbRsp *rsp){
// compact db
int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj *pCompact, SDbObj *pDb, SCompactDbRsp *rsp) {
pCompact->compactId = tGenIdPI32();
strcpy(pCompact->dbname, pDb->name);
@ -251,18 +246,19 @@ int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompac
return 0;
}
//retrieve compact
int32_t mndRetrieveCompact(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows){
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
SCompactObj *pCompact = NULL;
char *sep = NULL;
SDbObj *pDb = NULL;
// retrieve compact
int32_t mndRetrieveCompact(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
SCompactObj *pCompact = NULL;
char *sep = NULL;
SDbObj *pDb = NULL;
if (strlen(pShow->db) > 0) {
sep = strchr(pShow->db, '.');
if (sep && ((0 == strcmp(sep + 1, TSDB_INFORMATION_SCHEMA_DB) || (0 == strcmp(sep + 1, TSDB_PERFORMANCE_SCHEMA_DB))))) {
if (sep &&
((0 == strcmp(sep + 1, TSDB_INFORMATION_SCHEMA_DB) || (0 == strcmp(sep + 1, TSDB_PERFORMANCE_SCHEMA_DB))))) {
sep++;
} else {
pDb = mndAcquireDb(pMnode, pShow->db);
@ -306,9 +302,9 @@ int32_t mndRetrieveCompact(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock,
return numOfRows;
}
//kill compact
static void *mndBuildKillCompactReq(SMnode *pMnode, SVgObj *pVgroup, int32_t *pContLen,
int32_t compactId, int32_t dnodeid) {
// kill compact
static void *mndBuildKillCompactReq(SMnode *pMnode, SVgObj *pVgroup, int32_t *pContLen, int32_t compactId,
int32_t dnodeid) {
SVKillCompactReq req = {0};
req.compactId = compactId;
req.vgId = pVgroup->vgId;
@ -337,8 +333,8 @@ static void *mndBuildKillCompactReq(SMnode *pMnode, SVgObj *pVgroup, int32_t *pC
return pReq;
}
static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup,
int32_t compactId, int32_t dnodeid) {
static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, int32_t compactId,
int32_t dnodeid) {
STransAction action = {0};
SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeid);
@ -365,7 +361,7 @@ static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *p
static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompact) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq, "kill-compact");
if (pTrans == NULL) {
mError("compact:%" PRId32 ", failed to drop since %s" , pCompact->compactId, terrstr());
mError("compact:%" PRId32 ", failed to drop since %s", pCompact->compactId, terrstr());
return -1;
}
mInfo("trans:%d, used to kill compact:%" PRId32, pTrans->id, pCompact->compactId);
@ -388,13 +384,13 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa
if (pDetail->compactId == pCompact->compactId) {
SVgObj *pVgroup = mndAcquireVgroup(pMnode, pDetail->vgId);
if(pVgroup == NULL){
if (pVgroup == NULL) {
mError("trans:%d, failed to append redo action since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
if(mndAddKillCompactAction(pMnode, pTrans, pVgroup, pCompact->compactId, pDetail->dnodeId) != 0){
if (mndAddKillCompactAction(pMnode, pTrans, pVgroup, pCompact->compactId, pDetail->dnodeId) != 0) {
mError("trans:%d, failed to append redo action since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
@ -426,7 +422,7 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa
return 0;
}
int32_t mndProcessKillCompactReq(SRpcMsg *pReq){
int32_t mndProcessKillCompactReq(SRpcMsg *pReq) {
SKillCompactReq killCompactReq = {0};
if (tDeserializeSKillCompactReq(pReq->pCont, pReq->contLen, &killCompactReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
@ -435,10 +431,10 @@ int32_t mndProcessKillCompactReq(SRpcMsg *pReq){
mInfo("start to kill compact:%" PRId32, killCompactReq.compactId);
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SCompactObj *pCompact = mndAcquireCompact(pMnode, killCompactReq.compactId);
if(pCompact == NULL){
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SCompactObj *pCompact = mndAcquireCompact(pMnode, killCompactReq.compactId);
if (pCompact == NULL) {
terrno = TSDB_CODE_MND_INVALID_COMPACT_ID;
tFreeSKillCompactReq(&killCompactReq);
return -1;
@ -470,9 +466,10 @@ _OVER:
return code;
}
//update progress
static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t compactId, SQueryCompactProgressRsp* rsp) {
void* pIter = NULL;
// update progress
static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t compactId,
SQueryCompactProgressRsp *rsp) {
void *pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
@ -493,36 +490,36 @@ static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t c
return TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST;
}
int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq){
int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq) {
SQueryCompactProgressRsp req = {0};
int32_t code = 0;
int32_t code = 0;
code = tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req);
if (code != 0) {
terrno = TSDB_CODE_INVALID_MSG;
mError("failed to deserialize vnode-query-compact-progress-rsp, ret:%d, pCont:%p, len:%d",
code, pReq->pCont, pReq->contLen);
mError("failed to deserialize vnode-query-compact-progress-rsp, ret:%d, pCont:%p, len:%d", code, pReq->pCont,
pReq->contLen);
return -1;
}
mDebug("compact:%d, receive query response, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d",
req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished);
mDebug("compact:%d, receive query response, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d", req.compactId,
req.vgId, req.dnodeId, req.numberFileset, req.finished);
SMnode *pMnode = pReq->info.node;
SMnode *pMnode = pReq->info.node;
code = mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req);
if(code != 0){
if (code != 0) {
terrno = code;
mError("compact:%d, failed to update progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d",
req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished);
mError("compact:%d, failed to update progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d", req.compactId,
req.vgId, req.dnodeId, req.numberFileset, req.finished);
return -1;
}
return 0;
}
//timer
void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact){
void *pIter = NULL;
// timer
void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) {
void *pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
@ -532,8 +529,8 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact){
if (pDetail->compactId == pCompact->compactId) {
SEpSet epSet = {0};
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pDetail->dnodeId);
if(pDnode == NULL) break;
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pDetail->dnodeId);
if (pDnode == NULL) break;
addEpIntoEpSet(&epSet, pDnode->fqdn, pDnode->port);
mndReleaseDnode(pMnode, pDnode);
@ -555,31 +552,29 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact){
sdbCancelFetch(pMnode->pSdb, pDetail);
sdbRelease(pMnode->pSdb, pDetail);
continue;
}
}
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pDetail->vgId);
tSerializeSQueryCompactProgressReq((char *)pHead + sizeof(SMsgHead), contLen - sizeof(SMsgHead), &req);
SRpcMsg rpcMsg = {.msgType = TDMT_VND_QUERY_COMPACT_PROGRESS,
.contLen = contLen};
//rpcMsg.pCont = rpcMallocCont(contLen);
//if (rpcMsg.pCont == NULL) {
// return;
//}
SRpcMsg rpcMsg = {.msgType = TDMT_VND_QUERY_COMPACT_PROGRESS, .contLen = contLen};
//memcpy(rpcMsg.pCont, pHead, contLen);
// rpcMsg.pCont = rpcMallocCont(contLen);
// if (rpcMsg.pCont == NULL) {
// return;
// }
// memcpy(rpcMsg.pCont, pHead, contLen);
rpcMsg.pCont = pHead;
char detail[1024] = {0};
int32_t len = snprintf(detail, sizeof(detail), "msgType:%s numOfEps:%d inUse:%d", TMSG_INFO(TDMT_VND_QUERY_COMPACT_PROGRESS),
epSet.numOfEps, epSet.inUse);
int32_t len = snprintf(detail, sizeof(detail), "msgType:%s numOfEps:%d inUse:%d",
TMSG_INFO(TDMT_VND_QUERY_COMPACT_PROGRESS), epSet.numOfEps, epSet.inUse);
for (int32_t i = 0; i < epSet.numOfEps; ++i) {
len += snprintf(detail + len, sizeof(detail) - len, " ep:%d-%s:%u", i, epSet.eps[i].fqdn,
epSet.eps[i].port);
len += snprintf(detail + len, sizeof(detail) - len, " ep:%d-%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);
}
mDebug("compact:%d, send update progress msg to %s", pDetail->compactId, detail);
@ -592,40 +587,51 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact){
}
static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
bool needSave = false;
void* pIter = NULL;
bool needSave = false;
void *pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == compactId) {
mDebug("compact:%d, check save progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d, "
"newNumberFileset:%d, newFinished:%d",
pDetail->compactId, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished,
pDetail->newNumberFileset, pDetail->newFinished);
mDebug(
"compact:%d, check save progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d, "
"newNumberFileset:%d, newFinished:%d",
pDetail->compactId, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished,
pDetail->newNumberFileset, pDetail->newFinished);
//these 2 number will jump back after dnode restart, so < is not used here
if(pDetail->numberFileset != pDetail->newNumberFileset || pDetail->finished != pDetail->newFinished)
// these 2 number will jump back after dnode restart, so < is not used here
if (pDetail->numberFileset != pDetail->newNumberFileset || pDetail->finished != pDetail->newFinished)
needSave = true;
}
sdbRelease(pMnode->pSdb, pDetail);
}
if(!needSave) {
mDebug("compact:%" PRId32 ", no need to save" , compactId);
SCompactObj *pCompact = mndAcquireCompact(pMnode, compactId);
if (pCompact == NULL) return 0;
SDbObj *pDb = mndAcquireDb(pMnode, pCompact->dbname);
if (pDb == NULL) {
needSave = true;
mWarn("compact:%" PRId32 ", no db exist, set needSave:%s", compactId, pCompact->dbname);
} else {
mndReleaseDb(pMnode, pDb);
pDb = NULL;
}
if (!needSave) {
mDebug("compact:%" PRId32 ", no need to save", compactId);
return 0;
}
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, NULL, "update-compact-progress");
if (pTrans == NULL) {
mError("trans:%" PRId32 ", failed to create since %s" , pTrans->id, terrstr());
mError("trans:%" PRId32 ", failed to create since %s", pTrans->id, terrstr());
return -1;
}
mInfo("compact:%d, trans:%d, used to update compact progress.", compactId, pTrans->id);
SCompactObj *pCompact = mndAcquireCompact(pMnode, compactId);
mndTransSetDbName(pTrans, pCompact->dbname, NULL);
@ -636,14 +642,15 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
if (pIter == NULL) break;
if (pDetail->compactId == compactId) {
mInfo("compact:%d, trans:%d, check compact progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d, "
"newNumberFileset:%d, newFinished:%d",
pDetail->compactId, pTrans->id, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished,
pDetail->newNumberFileset, pDetail->newFinished);
mInfo(
"compact:%d, trans:%d, check compact progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d, "
"newNumberFileset:%d, newFinished:%d",
pDetail->compactId, pTrans->id, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished,
pDetail->newNumberFileset, pDetail->newFinished);
pDetail->numberFileset = pDetail->newNumberFileset;
pDetail->finished = pDetail->newFinished;
SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id, terrstr());
@ -657,22 +664,22 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
}
bool allFinished = true;
pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if(pDetail->compactId == compactId){
mInfo("compact:%d, trans:%d, check compact finished, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d",
pDetail->compactId, pTrans->id, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished);
if (pDetail->compactId == compactId) {
mInfo("compact:%d, trans:%d, check compact finished, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d",
pDetail->compactId, pTrans->id, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished);
if(pDetail->numberFileset == -1 && pDetail->finished == -1){
if (pDetail->numberFileset == -1 && pDetail->finished == -1) {
allFinished = false;
sdbRelease(pMnode->pSdb, pDetail);
break;
}
if (pDetail->numberFileset != -1 && pDetail->finished != -1 &&
pDetail->numberFileset != pDetail->finished) {
if (pDetail->numberFileset != -1 && pDetail->finished != -1 && pDetail->numberFileset != pDetail->finished) {
allFinished = false;
sdbRelease(pMnode->pSdb, pDetail);
break;
@ -682,21 +689,33 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
sdbRelease(pMnode->pSdb, pDetail);
}
if(allFinished){
pDb = mndAcquireDb(pMnode, pCompact->dbname);
if (pDb == NULL) {
allFinished = true;
mWarn("compact:%" PRId32 ", no db exist, set all finished:%s", compactId, pCompact->dbname);
} else {
mndReleaseDb(pMnode, pDb);
pDb = NULL;
}
if (allFinished) {
mInfo("compact:%d, all finished", pCompact->compactId);
pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
if (pIter == NULL) break;
if (pDetail->compactId == pCompact->compactId) {
if (pDetail->compactId == pCompact->compactId) {
SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id, terrstr());
mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id,
terrstr());
mndTransDrop(pTrans);
return -1;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
mInfo("compact:%d, add drop compactdetail action", pDetail->compactDetailId);
}
sdbRelease(pMnode->pSdb, pDetail);
@ -709,6 +728,7 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
return -1;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
mInfo("compact:%d, add drop compact action", pCompact->compactId);
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
@ -739,8 +759,8 @@ void mndCompactPullup(SMnode *pMnode) {
for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
mInfo("begin to pull up");
int32_t *pCompactId = taosArrayGet(pArray, i);
SCompactObj *pCompact = mndAcquireCompact(pMnode, *pCompactId);
int32_t *pCompactId = taosArrayGet(pArray, i);
SCompactObj *pCompact = mndAcquireCompact(pMnode, *pCompactId);
if (pCompact != NULL) {
mInfo("compact:%d, begin to pull up", pCompact->compactId);
mndCompactSendProgressReq(pMnode, pCompact);

15
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -91,7 +91,7 @@ void mndSendConsumerMsg(SMnode *pMnode, int64_t consumerId, uint16_t msgType, SR
}
}
static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *pMnode, const char *pUser,
static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const char *pUser,
bool enableReplay) {
SMqTopicObj *pTopic = NULL;
int32_t code = 0;
@ -135,11 +135,6 @@ static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *
}
}
mndTransSetDbName(pTrans, pOneTopic, NULL);
if (mndTransCheckConflict(pMnode, pTrans) != 0) {
code = -1;
goto FAILED;
}
mndReleaseTopic(pMnode, pTopic);
}
@ -177,12 +172,12 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
goto END;
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_TOPIC, pMsg, "recover-csm");
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "recover-csm");
if (pTrans == NULL) {
code = -1;
goto END;
}
code = validateTopics(pTrans, pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false);
code = validateTopics(pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false);
if (code != 0) {
goto END;
}
@ -675,13 +670,13 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
goto _over;
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_TOPIC_INSIDE, pMsg, "subscribe");
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe");
if (pTrans == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _over;
}
code = validateTopics(pTrans, subscribe.topicNames, pMnode, pMsg->info.conn.user, subscribe.enableReplay);
code = validateTopics(subscribe.topicNames, pMnode, pMsg->info.conn.user, subscribe.enableReplay);
if (code != TSDB_CODE_SUCCESS) {
goto _over;
}

10
source/dnode/mnode/impl/src/mndDb.c
View File

@ -495,6 +495,16 @@ static int32_t mndCheckInChangeDbCfg(SMnode *pMnode, SDbCfg *pOldCfg, SDbCfg *pN
#else
if (pNewCfg->replications != 1 && pNewCfg->replications != 3) return -1;
#endif
if (pNewCfg->walLevel == 0 && pOldCfg->replications > 1) {
terrno = TSDB_CODE_MND_INVALID_WAL_LEVEL;
return -1;
}
if (pNewCfg->replications > 1 && pOldCfg->walLevel == 0) {
terrno = TSDB_CODE_MND_INVALID_WAL_LEVEL;
return -1;
}
if (pNewCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pNewCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1;
if (pNewCfg->minRows < TSDB_MIN_MINROWS_FBLOCK || pNewCfg->minRows > TSDB_MAX_MINROWS_FBLOCK) return -1;
if (pNewCfg->maxRows < TSDB_MIN_MAXROWS_FBLOCK || pNewCfg->maxRows > TSDB_MAX_MAXROWS_FBLOCK) return -1;

8
source/dnode/mnode/impl/src/mndInfoSchema.c
View File

@ -75,7 +75,13 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *
return -1;
}
STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName));
STableMetaRsp *pMeta = NULL;
if (strcmp(tbName, TSDB_INS_TABLE_USERS_FULL) == 0) {
pMeta = taosHashGet(pMnode->infosMeta, TSDB_INS_TABLE_USERS_FULL, strlen(tbName));
} else {
pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName));
}
if (NULL == pMeta) {
mError("invalid information schema table name:%s", tbName);
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;

22
source/dnode/mnode/impl/src/mndPrivilege.c
View File

@ -31,7 +31,6 @@ int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType op
int32_t mndCheckTopicPrivilege(SMnode *pMnode, const char *user, EOperType operType, SMqTopicObj *pTopic) { return 0; }
int32_t mndSetUserWhiteListRsp(SMnode *pMnode, SUserObj *pUser, SGetUserWhiteListRsp *pWhiteListRsp) {
memcpy(pWhiteListRsp->user, pUser->user, TSDB_USER_LEN);
pWhiteListRsp->numWhiteLists = 1;
@ -41,25 +40,6 @@ int32_t mndSetUserWhiteListRsp(SMnode *pMnode, SUserObj *pUser, SGetUserWhiteLis
}
memset(pWhiteListRsp->pWhiteLists, 0, pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range));
// if (tsEnableWhiteList) {
// memcpy(pWhiteListRsp->user, pUser->user, TSDB_USER_LEN);
// pWhiteListRsp->numWhiteLists = pUser->pIpWhiteList->num;
// pWhiteListRsp->pWhiteLists = taosMemoryMalloc(pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range));
// if (pWhiteListRsp->pWhiteLists == NULL) {
// return TSDB_CODE_OUT_OF_MEMORY;
// }
// memcpy(pWhiteListRsp->pWhiteLists, pUser->pIpWhiteList->pIpRange,
// pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range));
// } else {
// memcpy(pWhiteListRsp->user, pUser->user, TSDB_USER_LEN);
// pWhiteListRsp->numWhiteLists = 1;
// pWhiteListRsp->pWhiteLists = taosMemoryMalloc(pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range));
// if (pWhiteListRsp->pWhiteLists == NULL) {
// return TSDB_CODE_OUT_OF_MEMORY;
// }
// memset(pWhiteListRsp->pWhiteLists, 0, pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range));
// }
return 0;
}
@ -70,7 +50,7 @@ int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp
pRsp->sysInfo = pUser->sysInfo;
pRsp->version = pUser->authVersion;
pRsp->passVer = pUser->passVersion;
pRsp->whiteListVer = pUser->ipWhiteListVer;
pRsp->whiteListVer = mndGetUserIpWhiteListVer(pMnode, pUser);
return 0;
}

2
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -299,7 +299,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
connectRsp.svrTimestamp = taosGetTimestampSec();
connectRsp.passVer = pUser->passVersion;
connectRsp.authVer = pUser->authVersion;
connectRsp.whiteListVer = pUser->ipWhiteListVer;
connectRsp.whiteListVer = mndGetUserIpWhiteListVer(pMnode, pUser);
connectRsp.monitorParas.tsEnableMonitor = tsEnableMonitor;
connectRsp.monitorParas.tsMonitorInterval = tsMonitorInterval;
connectRsp.monitorParas.tsSlowLogScope = tsSlowLogScope;

9
source/dnode/mnode/impl/src/mndShow.c
View File

@ -89,6 +89,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
// type = TSDB_MGMT_TABLE_DIST;
} else if (strncasecmp(name, TSDB_INS_TABLE_USERS, len) == 0) {
type = TSDB_MGMT_TABLE_USER;
} else if (strncasecmp(name, TSDB_INS_TABLE_USERS_FULL, len) == 0) {
type = TSDB_MGMT_TABLE_USER_FULL;
} else if (strncasecmp(name, TSDB_INS_TABLE_LICENCES, len) == 0) {
type = TSDB_MGMT_TABLE_GRANTS;
} else if (strncasecmp(name, TSDB_INS_TABLE_VGROUPS, len) == 0) {
@ -276,6 +278,13 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
if (retrieveReq.db[0] && mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db) != 0) {
return -1;
}
if (pShow->type == TSDB_MGMT_TABLE_USER_FULL) {
if(strcmp(pReq->info.conn.user, "root") != 0){
mError("The operation is not permitted, user:%s, pShow->type:%d", pReq->info.conn.user, pShow->type);
terrno = TSDB_CODE_MND_NO_RIGHTS;
return -1;
}
}
int32_t numOfCols = pShow->pMeta->numOfColumns;

6
source/dnode/mnode/impl/src/mndStb.c
View File

@ -1766,7 +1766,7 @@ static int32_t mndUpdateSuperTableColumnCompress(SMnode *pMnode, const SStbObj *
uint32_t dst = 0;
updated = tUpdateCompress(pCmpr->alg, p->bytes, TSDB_COLVAL_COMPRESS_DISABLED, TSDB_COLVAL_LEVEL_DISABLED,
TSDB_COLVAL_LEVEL_MEDIUM, &dst);
if (updated) pCmpr->alg = dst;
if (updated > 0) pCmpr->alg = dst;
break;
}
}
@ -1774,7 +1774,11 @@ static int32_t mndUpdateSuperTableColumnCompress(SMnode *pMnode, const SStbObj *
if (updated == 0) {
terrno = TSDB_CODE_MND_COLUMN_COMPRESS_ALREADY_EXIST;
return -1;
} else if (updated == -1) {
terrno = TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR;
return -1;
}
pNew->colVer++;
return 0;

276
source/dnode/mnode/impl/src/mndStream.c
View File

@ -56,13 +56,14 @@ static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, int
int64_t streamId, int32_t taskId, int32_t transId, int8_t mndTrigger);
static int32_t mndProcessNodeCheck(SRpcMsg *pReq);
static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg);
static SArray *extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList);
static int32_t extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList);
static int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq);
static int32_t mndProcessCheckpointReport(SRpcMsg *pReq);
static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList);
void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
static int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot);
static void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo *pExecInfo);
static void removeExpiredNodeInfo(const SArray *pNodeSnapshot);
static int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDbName, size_t len);
static SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw);
@ -103,6 +104,7 @@ int32_t mndInitStream(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_STOP_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_VND_STREAM_TASK_UPDATE_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_VND_STREAM_TASK_RESET_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_UPDATE_CHKPT_RSP, mndTransProcessRsp);
// for msgs inside mnode
// TODO change the name
@ -114,8 +116,10 @@ int32_t mndInitStream(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_VND_STREAM_CHECK_POINT_SOURCE_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamCheckpoint);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_REQ_CHKPT, mndProcessStreamReqCheckpoint);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_HEARTBEAT, mndProcessStreamHb);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHKPT_REPORT, mndProcessCheckpointReport);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_UPDATE_CHKPT_EVT, mndScanCheckpointReportInfo);
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_REPORT_CHECKPOINT, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_HEARTBEAT, mndProcessStreamHb);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_NODECHANGE_CHECK, mndProcessNodeCheckReq);
mndSetMsgHandle(pMnode, TDMT_MND_PAUSE_STREAM, mndProcessPauseStreamReq);
@ -131,9 +135,11 @@ int32_t mndInitStream(SMnode *pMnode) {
if (sdbSetTable(pMnode->pSdb, table) != 0) {
return -1;
}
if (sdbSetTable(pMnode->pSdb, tableSeq) != 0) {
return -1;
}
return 0;
}
@ -143,6 +149,7 @@ void mndCleanupStream(SMnode *pMnode) {
taosHashCleanup(execInfo.pTaskMap);
taosHashCleanup(execInfo.transMgmt.pDBTrans);
taosHashCleanup(execInfo.pTransferStateStreams);
taosHashCleanup(execInfo.pChkptStreams);
taosThreadMutexDestroy(&execInfo.lock);
mDebug("mnd stream exec info cleanup");
}
@ -508,7 +515,7 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) {
tEncodeStreamTask(&encoder, pTask);
tEncoderClear(&encoder);
int32_t code = setTransAction(pTrans, buf, tlen, TDMT_STREAM_TASK_DEPLOY, &pTask->info.epSet, 0);
int32_t code = setTransAction(pTrans, buf, tlen, TDMT_STREAM_TASK_DEPLOY, &pTask->info.epSet, 0, 0);
if (code != 0) {
taosMemoryFree(buf);
return -1;
@ -952,7 +959,7 @@ static int32_t doSetCheckpointAction(SMnode *pMnode, STrans *pTrans, SStreamTask
return -1;
}
code = setTransAction(pTrans, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY);
code = setTransAction(pTrans, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY, 0);
if (code != 0) {
taosMemoryFree(buf);
}
@ -1088,7 +1095,7 @@ static bool taskNodeIsUpdated(SMnode *pMnode) {
}
static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
bool ready = true;
bool ready = true;
if (taskNodeIsUpdated(pMnode)) {
return -1;
}
@ -1099,6 +1106,8 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
ASSERT(taosArrayGetSize(execInfo.pTaskList) == 0);
}
SArray* pInvalidList = taosArrayInit(4, sizeof(STaskId));
for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
STaskId *p = taosArrayGet(execInfo.pTaskList, i);
STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p));
@ -1106,11 +1115,20 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
continue;
}
if (pEntry->status == TASK_STATUS__STOP) {
for(int32_t j = 0; j < taosArrayGetSize(pInvalidList); ++j) {
STaskId* pId = taosArrayGet(pInvalidList, j);
if (pEntry->id.streamId == pId->streamId) {
taosArrayPush(pInvalidList, &pEntry->id);
break;
}
}
}
if (pEntry->status != TASK_STATUS__READY) {
mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s not ready, checkpoint msg not issued",
mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s, checkpoint not issued",
pEntry->id.streamId, (int32_t)pEntry->id.taskId, pEntry->nodeId, streamTaskGetStatusStr(pEntry->status));
ready = false;
break;
}
if (pEntry->hTaskId != 0) {
@ -1123,6 +1141,9 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
}
}
removeTasksInBuf(pInvalidList, &execInfo);
taosArrayDestroy(pInvalidList);
taosThreadMutexUnlock(&execInfo.lock);
return ready ? 0 : -1;
}
@ -1151,7 +1172,8 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
int32_t numOfCheckpointTrans = 0;
if ((code = mndCheckTaskAndNodeStatus(pMnode)) != 0) {
return code;
terrno = TSDB_CODE_STREAM_TASK_IVLD_STATUS;
return -1;
}
SArray* pList = taosArrayInit(4, sizeof(SCheckpointInterval));
@ -1798,6 +1820,10 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
int32_t numOfRows = 0;
SStreamObj *pStream = NULL;
taosThreadMutexLock(&execInfo.lock);
mndInitStreamExecInfo(pMnode, &execInfo);
taosThreadMutexUnlock(&execInfo.lock);
while (numOfRows < rowsCapacity) {
pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream);
if (pShow->pIter == NULL) {
@ -2169,7 +2195,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
return 0;
}
static SArray *extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList) {
static int32_t extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList) {
SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = NULL;
void *pIter = NULL;
@ -2215,48 +2241,6 @@ static SArray *extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList) {
return TSDB_CODE_SUCCESS;
}
static bool taskNodeExists(SArray *pList, int32_t nodeId) {
size_t num = taosArrayGetSize(pList);
for (int32_t i = 0; i < num; ++i) {
SNodeEntry *pEntry = taosArrayGet(pList, i);
if (pEntry->nodeId == nodeId) {
return true;
}
}
return false;
}
int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot) {
SArray *pRemovedTasks = taosArrayInit(4, sizeof(STaskId));
int32_t numOfTask = taosArrayGetSize(execInfo.pTaskList);
for (int32_t i = 0; i < numOfTask; ++i) {
STaskId *pId = taosArrayGet(execInfo.pTaskList, i);
STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId));
if (pEntry->nodeId == SNODE_HANDLE) {
continue;
}
bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId);
if (!existed) {
taosArrayPush(pRemovedTasks, pId);
}
}
removeTasksInBuf(pRemovedTasks, &execInfo);
mDebug("remove invalid stream tasks:%d, remain:%d", (int32_t)taosArrayGetSize(pRemovedTasks),
(int32_t)taosArrayGetSize(execInfo.pTaskList));
removeExpiredNodeInfo(pNodeSnapshot);
taosArrayDestroy(pRemovedTasks);
return 0;
}
// this function runs by only one thread, so it is not multi-thread safe
static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
int32_t code = 0;
@ -2476,13 +2460,137 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
taosThreadMutexUnlock(&execInfo.lock);
{
SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRspMsg)};
SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRsp)};
rsp.pCont = rpcMallocCont(rsp.contLen);
SMsgHead *pHead = rsp.pCont;
pHead->vgId = htonl(req.nodeId);
tmsgSendRsp(&rsp);
pReq->info.handle = NULL; // disable auto rsp
}
return 0;
}
static void doAddTaskInfo(SArray* pList, SCheckpointReport* pReport) {
bool existed = false;
for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
STaskChkptInfo* p = taosArrayGet(pList ,i);
if (p->taskId == pReport->taskId) {
existed = true;
break;
}
}
if (!existed) {
STaskChkptInfo info = {
.streamId = pReport->streamId,
.taskId = pReport->taskId,
.transId = pReport->transId,
.dropHTask = pReport->dropHTask,
.version = pReport->checkpointVer,
.ts = pReport->checkpointTs,
.checkpointId = pReport->checkpointId,
.nodeId = pReport->nodeId,
};
taosArrayPush(pList, &info);
}
}
int32_t mndProcessCheckpointReport(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SCheckpointReport req = {0};
SDecoder decoder = {0};
tDecoderInit(&decoder, pReq->pCont, pReq->contLen);
if (tDecodeStreamTaskChkptReport(&decoder, &req)) {
tDecoderClear(&decoder);
terrno = TSDB_CODE_INVALID_MSG;
mError("invalid task checkpoint-report msg received");
return -1;
}
tDecoderClear(&decoder);
mDebug("receive stream task checkpoint-report msg, vgId:%d, s-task:0x%x, checkpointId:%" PRId64
" checkpointVer:%" PRId64 " transId:%d",
req.nodeId, req.taskId, req.checkpointId, req.checkpointVer, req.transId);
// register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans.
taosThreadMutexLock(&execInfo.lock);
SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId);
if (pStream == NULL) {
mWarn("failed to find the stream:0x%" PRIx64 ", not handle checkpoint-report, try to acquire in buf",
req.streamId);
// not in meta-store yet, try to acquire the task in exec buffer
// the checkpoint req arrives too soon before the completion of the create stream trans.
STaskId id = {.streamId = req.streamId, .taskId = req.taskId};
void *p = taosHashGet(execInfo.pTaskMap, &id, sizeof(id));
if (p == NULL) {
mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint-report", req.streamId);
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
taosThreadMutexUnlock(&execInfo.lock);
return -1;
} else {
mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet",
req.streamId, req.taskId);
}
}
int32_t numOfTasks = (pStream == NULL) ? 0 : mndGetNumOfStreamTasks(pStream);
SArray **pReqTaskList = (SArray **)taosHashGet(execInfo.pChkptStreams, &req.streamId, sizeof(req.streamId));
if (pReqTaskList == NULL) {
SArray *pList = taosArrayInit(4, sizeof(STaskChkptInfo));
doAddTaskInfo(pList, &req);
taosHashPut(execInfo.pChkptStreams, &req.streamId, sizeof(req.streamId), &pList, POINTER_BYTES);
pReqTaskList = (SArray **)taosHashGet(execInfo.pChkptStreams, &req.streamId, sizeof(req.streamId));
} else {
doAddTaskInfo(*pReqTaskList, &req);
}
int32_t total = taosArrayGetSize(*pReqTaskList);
if (total == numOfTasks) { // all tasks has send the reqs
mInfo("stream:0x%" PRIx64 " %s all %d tasks send checkpoint-report, checkpoint meta-info for checkpointId:%" PRId64
" will be issued soon",
req.streamId, pStream->name, total, req.checkpointId);
// if (pStream != NULL) {
// bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHKPT_UPDATE_NAME, false);
// if (conflict) {
// mDebug("stream:0x%"PRIx64" active checkpoint trans not finished yet, wait", req.streamId);
// } else {
// int32_t code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, *pReqTaskList);
// if (code == TSDB_CODE_SUCCESS) { // remove this entry
// taosHashRemove(execInfo.pChkptStreams, &req.streamId, sizeof(req.streamId));
//
// int32_t numOfStreams = taosHashGetSize(execInfo.pChkptStreams);
// mDebug("stream:0x%" PRIx64 " removed, remain streams:%d in checkpoint procedure", req.streamId,
// numOfStreams);
// } else {
// mDebug("stream:0x%" PRIx64 " not launch chkpt update trans, due to checkpoint not finished yet",
// req.streamId);
// }
// }
// }
}
if (pStream != NULL) {
mndReleaseStream(pMnode, pStream);
}
taosThreadMutexUnlock(&execInfo.lock);
{
SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamUpdateChkptRsp)};
rsp.pCont = rpcMallocCont(rsp.contLen);
SMsgHead *pHead = rsp.pCont;
pHead->vgId = htonl(req.nodeId);
tmsgSendRsp(&rsp);
pReq->info.handle = NULL; // disable auto rsp
}
@ -2510,3 +2618,63 @@ static int32_t mndProcessDropStreamReqFromMNode(SRpcMsg *pReq) {
}
return code;
}
void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo) {
if (pExecInfo->initTaskList || pMnode == NULL) {
return;
}
addAllStreamTasksIntoBuf(pMnode, pExecInfo);
extractNodeListFromStream(pMnode, pExecInfo->pNodeList);
pExecInfo->initTaskList = true;
}
void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo* pExecInfo) {
SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = NULL;
void *pIter = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
if (pIter == NULL) {
break;
}
saveTaskAndNodeInfoIntoBuf(pStream, pExecInfo);
sdbRelease(pSdb, pStream);
}
}
int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray* pChkptInfoList) {
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_UPDATE_NAME, "update checkpoint-info");
if (pTrans == NULL) {
return terrno;
}
/*int32_t code = */mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_UPDATE_NAME, pStream->uid);
int32_t code = mndStreamSetUpdateChkptAction(pMnode, pTrans, pStream);
if (code != 0) {
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return code;
}
code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY);
if (code != TSDB_CODE_SUCCESS) {
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return -1;
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare update checkpoint-info meta trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return -1;
}
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return TSDB_CODE_ACTION_IN_PROGRESS;
}

88
source/dnode/mnode/impl/src/mndStreamHb.c
View File

@ -22,54 +22,7 @@ typedef struct SFailedCheckpointInfo {
int32_t transId;
} SFailedCheckpointInfo;
static void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo* pExecInfo) {
SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = NULL;
void *pIter = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
if (pIter == NULL) {
break;
}
saveTaskAndNodeInfoIntoBuf(pStream, pExecInfo);
sdbRelease(pSdb, pStream);
}
}
static void removeDroppedStreamTasksInBuf(SMnode *pMnode, SStreamExecInfo *pExecInfo) {
if (pMnode == NULL) {
return;
}
int32_t num = taosArrayGetSize(pExecInfo->pTaskList);
SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
SArray *pIdList = taosArrayInit(4, sizeof(STaskId));
for (int32_t i = 0; i < num; ++i) {
STaskId* pId = taosArrayGet(pExecInfo->pTaskList, i);
void* p = taosHashGet(pHash, &pId->streamId, sizeof(int64_t));
if (p != NULL) {
continue;
}
void* pObj = mndGetStreamObj(pMnode, pId->streamId);
if (pObj != NULL) {
mndReleaseStream(pMnode, pObj);
taosHashPut(pHash, &pId->streamId, sizeof(int64_t), NULL, 0);
} else {
taosArrayPush(pIdList, pId);
}
}
removeTasksInBuf(pIdList, &execInfo);
taosArrayDestroy(pIdList);
taosHashCleanup(pHash);
}
static void mndStreamStartUpdateCheckpointInfo(SMnode *pMnode);
static void updateStageInfo(STaskStatusEntry *pTaskEntry, int64_t stage) {
int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeList);
@ -290,16 +243,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
taosThreadMutexLock(&execInfo.lock);
// extract stream task list
if (taosHashGetSize(execInfo.pTaskMap) == 0) {
addAllStreamTasksIntoBuf(pMnode, &execInfo);
} else {
// the already dropped tasks may be added by hb from vnode at the time when the pTaskMap happens to be empty.
// let's drop them here.
removeDroppedStreamTasksInBuf(pMnode, &execInfo);
}
extractStreamNodeList(pMnode);
mndInitStreamExecInfo(pMnode, &execInfo);
int32_t numOfUpdated = taosArrayGetSize(req.pUpdateNodes);
if (numOfUpdated > 0) {
@ -326,18 +270,6 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
snodeChanged = true;
}
} else {
// task is idle for more than 50 sec.
// if (fabs(pTaskEntry->inputQUsed - p->inputQUsed) <= DBL_EPSILON) {
// if (!pTaskEntry->inputQChanging) {
// pTaskEntry->inputQUnchangeCounter++;
// } else {
// pTaskEntry->inputQChanging = false;
// }
// } else {
// pTaskEntry->inputQChanging = true;
// pTaskEntry->inputQUnchangeCounter = 0;
// }
streamTaskStatusCopy(pTaskEntry, p);
STaskCkptInfo *pChkInfo = &p->checkpointInfo;
@ -348,6 +280,9 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
SFailedCheckpointInfo info = {
.transId = pChkInfo->activeTransId, .checkpointId = pChkInfo->activeId, .streamUid = p->id.streamId};
addIntoCheckpointList(pFailedChkpt, &info);
// remove failed trans from pChkptStreams
taosHashRemove(execInfo.pChkptStreams, &p->id.streamId, sizeof(p->id.streamId));
}
}
@ -393,6 +328,10 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
mndDropOrphanTasks(pMnode, pOrphanTasks);
}
if (pMnode != NULL) { // make sure that the unit test case can work
mndStreamStartUpdateCheckpointInfo(pMnode);
}
taosThreadMutexUnlock(&execInfo.lock);
tCleanupStreamHbMsg(&req);
@ -411,3 +350,12 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
return TSDB_CODE_SUCCESS;
}
void mndStreamStartUpdateCheckpointInfo(SMnode *pMnode) { // here reuse the doCheckpointmsg
SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg));
if (pMsg != NULL) {
int32_t size = sizeof(SMStreamDoCheckpointMsg);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_UPDATE_CHKPT_EVT, .pCont = pMsg, .contLen = size};
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
}
}

14
source/dnode/mnode/impl/src/mndStreamTrans.c
View File

@ -127,7 +127,7 @@ bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamId, const char* p
return false;
}
int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamUid) {
int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamId) {
taosThreadMutexLock(&execInfo.lock);
int32_t num = taosHashGetSize(execInfo.transMgmt.pDBTrans);
if (num <= 0) {
@ -136,12 +136,13 @@ int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamUid) {
}
mndStreamClearFinishedTrans(pMnode, NULL);
SStreamTransInfo* pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamUid, sizeof(streamUid));
SStreamTransInfo* pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId));
if (pEntry != NULL) {
SStreamTransInfo tInfo = *pEntry;
taosThreadMutexUnlock(&execInfo.lock);
if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0 || strcmp(tInfo.name, MND_STREAM_TASK_UPDATE_NAME) == 0) {
if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0 || strcmp(tInfo.name, MND_STREAM_TASK_UPDATE_NAME) == 0 ||
strcmp(tInfo.name, MND_STREAM_CHKPT_UPDATE_NAME) == 0) {
return tInfo.transId;
}
} else {
@ -159,7 +160,7 @@ STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnCo
return NULL;
}
mInfo("s-task:0x%" PRIx64 " start to build trans %s, transId:%d", pStream->uid, pMsg, pTrans->id);
mInfo("stream:0x%" PRIx64 " start to build trans %s, transId:%d", pStream->uid, pMsg, pTrans->id);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetSTbName);
if (mndTransCheckConflict(pMnode, pTrans) != 0) {
@ -246,8 +247,9 @@ int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status)
}
int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
int32_t retryCode) {
STransAction action = {.epSet = *pEpset, .contLen = contLen, .pCont = pCont, .msgType = msgType, .retryCode = retryCode};
int32_t retryCode, int32_t acceptCode) {
STransAction action = {.epSet = *pEpset, .contLen = contLen, .pCont = pCont, .msgType = msgType, .retryCode = retryCode,
.acceptableCode = acceptCode};
return mndTransAppendRedoAction(pTrans, &action);
}

184
source/dnode/mnode/impl/src/mndStreamUtil.c
View File

@ -230,7 +230,7 @@ static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pT
return -1;
}
code = setTransAction(pTrans, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0);
code = setTransAction(pTrans, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0, 0);
if (code != 0) {
taosMemoryFree(pReq);
return -1;
@ -308,7 +308,7 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
epsetToStr(&epset, buf, tListLen(buf));
mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf);
code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0);
code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0, 0);
if (code != 0) {
taosMemoryFree(pReq);
return -1;
@ -356,7 +356,7 @@ static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTas
}
// The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode.
code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0);
code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0);
if (code != 0) {
taosMemoryFree(pReq);
return -1;
@ -400,7 +400,7 @@ static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask
}
// The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode.
code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0);
code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0);
if (code != 0) {
taosMemoryFree(pReq);
return -1;
@ -484,7 +484,7 @@ static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask
return code;
}
code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &epset, TSDB_CODE_VND_INVALID_VGROUP_ID);
code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &epset, TSDB_CODE_VND_INVALID_VGROUP_ID, 0);
if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(pBuf);
}
@ -534,7 +534,7 @@ static int32_t doSetResetAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
return code;
}
code = setTransAction(pTrans, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &epset, 0);
code = setTransAction(pTrans, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &epset, 0, 0);
if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(pReq);
}
@ -574,9 +574,11 @@ void mndInitExecInfo() {
execInfo.pTaskMap = taosHashInit(64, fn, true, HASH_NO_LOCK);
execInfo.transMgmt.pDBTrans = taosHashInit(32, fn, true, HASH_NO_LOCK);
execInfo.pTransferStateStreams = taosHashInit(32, fn, true, HASH_NO_LOCK);
execInfo.pChkptStreams = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
execInfo.pNodeList = taosArrayInit(4, sizeof(SNodeEntry));
taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList);
taosHashSetFreeFp(execInfo.pChkptStreams, freeTaskList);
}
void removeExpiredNodeInfo(const SArray *pNodeSnapshot) {
@ -645,4 +647,174 @@ void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
taosThreadMutexUnlock(&pExecNode->lock);
destroyStreamTaskIter(pIter);
}
static bool taskNodeExists(SArray *pList, int32_t nodeId) {
size_t num = taosArrayGetSize(pList);
for (int32_t i = 0; i < num; ++i) {
SNodeEntry *pEntry = taosArrayGet(pList, i);
if (pEntry->nodeId == nodeId) {
return true;
}
}
return false;
}
int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot) {
SArray *pRemovedTasks = taosArrayInit(4, sizeof(STaskId));
int32_t numOfTask = taosArrayGetSize(execInfo.pTaskList);
for (int32_t i = 0; i < numOfTask; ++i) {
STaskId *pId = taosArrayGet(execInfo.pTaskList, i);
STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId));
if (pEntry->nodeId == SNODE_HANDLE) {
continue;
}
bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId);
if (!existed) {
taosArrayPush(pRemovedTasks, pId);
}
}
removeTasksInBuf(pRemovedTasks, &execInfo);
mDebug("remove invalid stream tasks:%d, remain:%d", (int32_t)taosArrayGetSize(pRemovedTasks),
(int32_t)taosArrayGetSize(execInfo.pTaskList));
removeExpiredNodeInfo(pNodeSnapshot);
taosArrayDestroy(pRemovedTasks);
return 0;
}
static int32_t doSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) {
SVUpdateCheckpointInfoReq *pReq = taosMemoryCalloc(1, sizeof(SVUpdateCheckpointInfoReq));
if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to malloc in reset stream, size:%" PRIzu ", code:%s", sizeof(SVUpdateCheckpointInfoReq),
tstrerror(TSDB_CODE_OUT_OF_MEMORY));
return terrno;
}
pReq->head.vgId = htonl(pTask->info.nodeId);
pReq->taskId = pTask->id.taskId;
pReq->streamId = pTask->id.streamId;
SArray **pReqTaskList = (SArray **)taosHashGet(execInfo.pChkptStreams, &pTask->id.streamId, sizeof(pTask->id.streamId));
ASSERT(pReqTaskList);
int32_t size = taosArrayGetSize(*pReqTaskList);
for(int32_t i = 0; i < size; ++i) {
STaskChkptInfo* pInfo = taosArrayGet(*pReqTaskList, i);
if (pInfo->taskId == pTask->id.taskId) {
pReq->checkpointId = pInfo->checkpointId;
pReq->checkpointVer = pInfo->version;
pReq->checkpointTs = pInfo->ts;
pReq->dropRelHTask = pInfo->dropHTask;
pReq->transId = pInfo->transId;
pReq->hStreamId = pTask->hTaskInfo.id.streamId;
pReq->hTaskId = pTask->hTaskInfo.id.taskId;
}
}
SEpSet epset = {0};
bool hasEpset = false;
int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
if (code != TSDB_CODE_SUCCESS || !hasEpset) {
taosMemoryFree(pReq);
return code;
}
code = setTransAction(pTrans, pReq, sizeof(SVUpdateCheckpointInfoReq), TDMT_STREAM_TASK_UPDATE_CHKPT, &epset, 0, TSDB_CODE_VND_INVALID_VGROUP_ID);
if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(pReq);
}
return code;
}
int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
taosWLockLatch(&pStream->lock);
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
int32_t code = doSetUpdateChkptAction(pMnode, pTrans, pTask);
if (code != TSDB_CODE_SUCCESS) {
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return -1;
}
}
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return 0;
}
int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
void *pIter = NULL;
SArray *pDropped = taosArrayInit(4, sizeof(int64_t));
mDebug("start to scan checkpoint report info");
while ((pIter = taosHashIterate(execInfo.pChkptStreams, pIter)) != NULL) {
SArray *pList = *(SArray **)pIter;
STaskChkptInfo* pInfo = taosArrayGet(pList, 0);
SStreamObj* pStream = mndGetStreamObj(pMnode, pInfo->streamId);
if (pStream == NULL) {
mDebug("failed to acquire stream:0x%" PRIx64 " remove it from checkpoint-report list", pInfo->streamId);
taosArrayPush(pDropped, &pInfo->streamId);
continue;
}
int32_t total = mndGetNumOfStreamTasks(pStream);
int32_t existed = (int32_t) taosArrayGetSize(pList);
if (total == existed) {
mDebug("stream:0x%" PRIx64 " %s all %d tasks send checkpoint-report, start to update checkpoint-info",
pStream->uid, pStream->name, total);
bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHKPT_UPDATE_NAME, false);
if (!conflict) {
int32_t code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, pList);
if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) { // remove this entry
taosArrayPush(pDropped, &pInfo->streamId);
mDebug("stream:0x%" PRIx64 " removed", pInfo->streamId);
} else {
mDebug("stream:0x%" PRIx64 " not launch chkpt-meta update trans, due to checkpoint not finished yet",
pInfo->streamId);
}
break;
} else {
mDebug("stream:0x%"PRIx64" active checkpoint trans not finished yet, wait", pInfo->streamId);
}
} else {
mDebug("stream:0x%" PRIx64 " %s %d/%d tasks send checkpoint-report, %d not send", pInfo->streamId, pStream->name,
existed, total, total - existed);
}
sdbRelease(pMnode->pSdb, pStream);
}
int32_t size = taosArrayGetSize(pDropped);
if (size > 0) {
for (int32_t i = 0; i < size; ++i) {
int64_t streamId = *(int64_t *)taosArrayGet(pDropped, i);
taosHashRemove(execInfo.pChkptStreams, &streamId, sizeof(streamId));
}
int32_t numOfStreams = taosHashGetSize(execInfo.pChkptStreams);
mDebug("drop %d stream(s) in checkpoint-report list, remain:%d", size, numOfStreams);
}
taosArrayDestroy(pDropped);
return TSDB_CODE_SUCCESS;
}

38
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -618,13 +618,13 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
char cgroup[TSDB_CGROUP_LEN] = {0};
mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup, true);
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TOPIC_INSIDE, pMsg, "tmq-reb");
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "tmq-reb");
if (pTrans == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
mndTransSetDbName(pTrans, topic, cgroup);
mndTransSetDbName(pTrans, pOutput->pSub->dbName, cgroup);
code = mndTransCheckConflict(pMnode, pTrans);
if (code != 0) {
goto END;
@ -908,33 +908,37 @@ END:
}
static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STrans *pTrans){
// iter all vnode to delete handle
int32_t sz = taosArrayGetSize(pSub->unassignedVgs);
for (int32_t i = 0; i < sz; i++) {
SMqVgEp *pVgEp = taosArrayGetP(pSub->unassignedVgs, i);
void* pIter = NULL;
SVgObj* pVgObj = NULL;
while (1) {
pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void**)&pVgObj);
if (pIter == NULL) {
break;
}
if (!mndVgroupInDb(pVgObj, pSub->dbUid)) {
sdbRelease(pMnode->pSdb, pVgObj);
continue;
}
SMqVDeleteReq *pReq = taosMemoryCalloc(1, sizeof(SMqVDeleteReq));
if(pReq == NULL){
terrno = TSDB_CODE_OUT_OF_MEMORY;
sdbRelease(pMnode->pSdb, pVgObj);
return -1;
}
pReq->head.vgId = htonl(pVgEp->vgId);
pReq->vgId = pVgEp->vgId;
pReq->head.vgId = htonl(pVgObj->vgId);
pReq->vgId = pVgObj->vgId;
pReq->consumerId = -1;
memcpy(pReq->subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN);
SVgObj *pVgObj = mndAcquireVgroup(pMnode, pVgEp->vgId);
if (pVgObj == NULL) {
taosMemoryFree(pReq);
terrno = TSDB_CODE_MND_VGROUP_NOT_EXIST;
return -1;
}
STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgObj);;
action.pCont = pReq;
action.contLen = sizeof(SMqVDeleteReq);
action.msgType = TDMT_VND_TMQ_DELETE_SUB;
action.acceptableCode = TSDB_CODE_MND_VGROUP_NOT_EXIST;
mndReleaseVgroup(pMnode, pVgObj);
sdbRelease(pMnode->pSdb, pVgObj);
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
return -1;
@ -996,7 +1000,7 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) {
goto end;
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TOPIC_INSIDE, pMsg, "drop-cgroup");
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "drop-cgroup");
if (pTrans == NULL) {
mError("cgroup: %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr());
code = -1;
@ -1004,7 +1008,7 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) {
}
mInfo("trans:%d, used to drop cgroup:%s on topic %s", pTrans->id, dropReq.cgroup, dropReq.topic);
mndTransSetDbName(pTrans, dropReq.topic, dropReq.cgroup);
mndTransSetDbName(pTrans, pSub->dbName, dropReq.cgroup);
code = mndTransCheckConflict(pMnode, pTrans);
if (code != 0) {
goto end;

8
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -422,14 +422,14 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
SQueryPlan *pPlan = NULL;
SMqTopicObj topicObj = {0};
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TOPIC, pReq, "create-topic");
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq, "create-topic");
if (pTrans == NULL) {
mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
code = -1;
goto _OUT;
}
mndTransSetDbName(pTrans, pCreate->name, NULL);
mndTransSetDbName(pTrans, pDb->name, NULL);
code = mndTransCheckConflict(pMnode, pTrans);
if (code != 0) {
goto _OUT;
@ -779,14 +779,14 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
}
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TOPIC, pReq, "drop-topic");
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-topic");
if (pTrans == NULL) {
mError("topic:%s, failed to drop since %s", pTopic->name, terrstr());
code = -1;
goto end;
}
mndTransSetDbName(pTrans, pTopic->name, NULL);
mndTransSetDbName(pTrans, pTopic->db, NULL);
code = mndTransCheckConflict(pMnode, pTrans);
if (code != 0) {
goto end;

34
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -836,26 +836,26 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) {
}
}
if (pNew->conflict == TRN_CONFLICT_TOPIC) {
if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true;
if (pTrans->conflict == TRN_CONFLICT_TOPIC || pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) {
if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true;
}
}
if (pNew->conflict == TRN_CONFLICT_TOPIC_INSIDE) {
if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true;
if (pTrans->conflict == TRN_CONFLICT_TOPIC) {
if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true;
}
if (pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) {
if (strcasecmp(pNew->dbname, pTrans->dbname) == 0 && strcasecmp(pNew->stbname, pTrans->stbname) == 0)
conflict = true;
}
}
// if (pNew->conflict == TRN_CONFLICT_TOPIC) {
// if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true;
// if (pTrans->conflict == TRN_CONFLICT_TOPIC || pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) {
// if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true;
// }
// }
// if (pNew->conflict == TRN_CONFLICT_TOPIC_INSIDE) {
// if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true;
// if (pTrans->conflict == TRN_CONFLICT_TOPIC) {
// if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true;
// }
// if (pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) {
// if (strcasecmp(pNew->dbname, pTrans->dbname) == 0 && strcasecmp(pNew->stbname, pTrans->stbname) == 0)
// conflict = true;
// }
// }
if (pNew->conflict == TRN_CONFLICT_ARBGROUP) {
if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true;
if (pTrans->conflict == TRN_CONFLICT_ARBGROUP) {
void *pIter = taosHashIterate(pNew->arbGroupIds, NULL);
pIter = taosHashIterate(pNew->arbGroupIds, NULL);
while (pIter != NULL) {
int32_t groupId = *(int32_t *)pIter;
if (taosHashGet(pTrans->arbGroupIds, &groupId, sizeof(int32_t)) != NULL) {

314
source/dnode/mnode/impl/src/mndUser.c
View File

@ -45,35 +45,55 @@
#define ALTER_USER_DEL_PRIVS(_type) ((_type) == TSDB_ALTER_USER_DEL_PRIVILEGES)
#define ALTER_USER_ALL_PRIV(_priv) (BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_ALL))
#define ALTER_USER_READ_PRIV(_priv) (BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_READ) || BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_ALL))
#define ALTER_USER_WRITE_PRIV(_priv) (BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_WRITE) || BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_ALL))
#define ALTER_USER_ALTER_PRIV(_priv) (BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_ALTER) || BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_ALL))
#define ALTER_USER_READ_PRIV(_priv) \
(BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_READ) || BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_ALL))
#define ALTER_USER_WRITE_PRIV(_priv) \
(BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_WRITE) || BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_ALL))
#define ALTER_USER_ALTER_PRIV(_priv) \
(BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_ALTER) || BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_ALL))
#define ALTER_USER_SUBSCRIBE_PRIV(_priv) (BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_SUBSCRIBE))
#define ALTER_USER_TARGET_DB(_tbname) (0 == (_tbname)[0])
#define ALTER_USER_TARGET_TB(_tbname) (0 != (_tbname)[0])
#define ALTER_USER_ADD_READ_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_READ_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_DEL_READ_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_READ_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_ADD_WRITE_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_WRITE_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_DEL_WRITE_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_WRITE_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_ADD_ALTER_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_ALTER_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_DEL_ALTER_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_ALTER_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_ADD_ALL_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_ALL_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_DEL_ALL_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_ALL_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_ADD_READ_DB_PRIV(_type, _priv, _tbname) \
(ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_READ_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_DEL_READ_DB_PRIV(_type, _priv, _tbname) \
(ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_READ_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_ADD_WRITE_DB_PRIV(_type, _priv, _tbname) \
(ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_WRITE_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_DEL_WRITE_DB_PRIV(_type, _priv, _tbname) \
(ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_WRITE_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_ADD_ALTER_DB_PRIV(_type, _priv, _tbname) \
(ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_ALTER_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_DEL_ALTER_DB_PRIV(_type, _priv, _tbname) \
(ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_ALTER_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_ADD_ALL_DB_PRIV(_type, _priv, _tbname) \
(ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_ALL_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_DEL_ALL_DB_PRIV(_type, _priv, _tbname) \
(ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_ALL_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_ADD_READ_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_READ_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_DEL_READ_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_READ_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_ADD_WRITE_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_WRITE_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_DEL_WRITE_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_WRITE_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_ADD_ALTER_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_ALTER_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_DEL_ALTER_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_ALTER_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_ADD_ALL_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_ALL_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_DEL_ALL_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_ALL_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_ADD_SUBSCRIBE_TOPIC_PRIV(_type, _priv) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_SUBSCRIBE_PRIV(_priv))
#define ALTER_USER_DEL_SUBSCRIBE_TOPIC_PRIV(_type, _priv) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_SUBSCRIBE_PRIV(_priv))
#define ALTER_USER_ADD_READ_TB_PRIV(_type, _priv, _tbname) \
(ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_READ_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_DEL_READ_TB_PRIV(_type, _priv, _tbname) \
(ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_READ_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_ADD_WRITE_TB_PRIV(_type, _priv, _tbname) \
(ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_WRITE_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_DEL_WRITE_TB_PRIV(_type, _priv, _tbname) \
(ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_WRITE_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_ADD_ALTER_TB_PRIV(_type, _priv, _tbname) \
(ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_ALTER_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_DEL_ALTER_TB_PRIV(_type, _priv, _tbname) \
(ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_ALTER_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_ADD_ALL_TB_PRIV(_type, _priv, _tbname) \
(ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_ALL_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_DEL_ALL_TB_PRIV(_type, _priv, _tbname) \
(ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_ALL_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_ADD_SUBSCRIBE_TOPIC_PRIV(_type, _priv) \
(ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_SUBSCRIBE_PRIV(_priv))
#define ALTER_USER_DEL_SUBSCRIBE_TOPIC_PRIV(_type, _priv) \
(ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_SUBSCRIBE_PRIV(_priv))
static SIpWhiteList *createDefaultIpWhiteList();
SIpWhiteList *createIpWhiteList(void *buf, int32_t len);
@ -97,6 +117,7 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq);
static int32_t mndProcessGetUserAuthReq(SRpcMsg *pReq);
static int32_t mndProcessGetUserWhiteListReq(SRpcMsg *pReq);
static int32_t mndRetrieveUsers(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static int32_t mndRetrieveUsersFull(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextUser(SMnode *pMnode, void *pIter);
static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextPrivileges(SMnode *pMnode, void *pIter);
@ -502,6 +523,8 @@ int32_t mndInitUser(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_USER, mndRetrieveUsers);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_USER, mndCancelGetNextUser);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_USER_FULL, mndRetrieveUsersFull);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_USER_FULL, mndCancelGetNextUser);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_PRIVILEGES, mndRetrievePrivileges);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_PRIVILEGES, mndCancelGetNextPrivileges);
return sdbSetTable(pMnode->pSdb, table);
@ -818,7 +841,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
SDB_SET_INT8(pRaw, dataPos, pUser->superUser, _OVER)
SDB_SET_INT8(pRaw, dataPos, pUser->sysInfo, _OVER)
SDB_SET_INT8(pRaw, dataPos, pUser->enable, _OVER)
SDB_SET_INT8(pRaw, dataPos, pUser->flag, _OVER)
SDB_SET_UINT8(pRaw, dataPos, pUser->flag, _OVER)
SDB_SET_INT32(pRaw, dataPos, pUser->authVersion, _OVER)
SDB_SET_INT32(pRaw, dataPos, pUser->passVersion, _OVER)
SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER)
@ -1002,7 +1025,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pUser->sysInfo, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pUser->enable, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pUser->flag, _OVER)
SDB_GET_UINT8(pRaw, dataPos, &pUser->flag, _OVER)
if (pUser->superUser) pUser->createdb = 1;
SDB_GET_INT32(pRaw, dataPos, &pUser->authVersion, _OVER)
if (sver >= 4) {
@ -1440,7 +1463,12 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) {
static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate, SRpcMsg *pReq) {
SUserObj userObj = {0};
taosEncryptPass_c((uint8_t *)pCreate->pass, strlen(pCreate->pass), userObj.pass);
if (pCreate->isImport != 1) {
taosEncryptPass_c((uint8_t *)pCreate->pass, strlen(pCreate->pass), userObj.pass);
} else {
// mInfo("pCreate->pass:%s", pCreate->pass)
strncpy(userObj.pass, pCreate->pass, TSDB_PASSWORD_LEN);
}
tstrncpy(userObj.user, pCreate->user, TSDB_USER_LEN);
tstrncpy(userObj.acct, acct, TSDB_USER_LEN);
userObj.createdTime = taosGetTimestampMs();
@ -1448,7 +1476,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate
userObj.superUser = 0; // pCreate->superUser;
userObj.sysInfo = pCreate->sysInfo;
userObj.enable = pCreate->enable;
userObj.createdb = 0;
userObj.createdb = pCreate->createDb;
if (pCreate->numIpRanges == 0) {
userObj.pIpWhiteList = createDefaultIpWhiteList();
@ -1534,10 +1562,25 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) {
goto _OVER;
}
mInfo("user:%s, start to create", createReq.user);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_USER) != 0) {
mInfo("user:%s, start to create, createdb:%d, is_import:%d", createReq.user, createReq.isImport, createReq.createDb);
#ifndef TD_ENTERPRISE
if (createReq.isImport == 1) {
goto _OVER;
}
#endif
if (createReq.isImport != 1) {
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_USER) != 0) {
goto _OVER;
}
} else {
if (strcmp(pReq->info.conn.user, "root") != 0) {
mError("The operation is not permitted, user:%s", pReq->info.conn.user);
terrno = TSDB_CODE_MND_NO_RIGHTS;
goto _OVER;
}
}
if (createReq.user[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_USER_FORMAT;
@ -1549,9 +1592,11 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) {
goto _OVER;
}
if (strlen(createReq.pass) >= TSDB_PASSWORD_LEN) {
terrno = TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG;
goto _OVER;
if (createReq.isImport != 1) {
if (strlen(createReq.pass) >= TSDB_PASSWORD_LEN) {
terrno = TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG;
goto _OVER;
}
}
pUser = mndAcquireUser(pMnode, createReq.user);
@ -1575,10 +1620,16 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) {
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
char detail[1000] = {0};
sprintf(detail, "enable:%d, superUser:%d, sysInfo:%d, password:xxx",
createReq.enable, createReq.superUser, createReq.sysInfo);
sprintf(detail, "enable:%d, superUser:%d, sysInfo:%d, password:xxx", createReq.enable, createReq.superUser,
createReq.sysInfo);
char operation[15] = {0};
if (createReq.isImport == 1) {
strcpy(operation, "importUser");
} else {
strcpy(operation, "createUser");
}
auditRecord(pReq, pMnode->clusterId, "createUser", "", createReq.user, detail, strlen(detail));
auditRecord(pReq, pMnode->clusterId, operation, "", createReq.user, detail, strlen(detail));
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
@ -1766,7 +1817,7 @@ static int32_t mndRemoveTablePriviledge(SMnode *pMnode, SHashObj *hash, SHashObj
if (NULL == currRef) {
return 0;
}
if (1 == *currRef) {
if (taosHashRemove(useDbHash, alterReq->objname, dbKeyLen) != 0) {
return -1;
@ -1800,12 +1851,12 @@ static char *mndUserAuditTypeStr(int32_t type) {
return "error";
}
static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode *pMnode, SUserObj* pNewUser) {
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode *pMnode, SUserObj *pNewUser) {
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
if (ALTER_USER_ADD_READ_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) ||
ALTER_USER_ADD_ALL_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (ALTER_USER_ADD_READ_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) ||
ALTER_USER_ADD_ALL_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (strcmp(pAlterReq->objname, "1.*") != 0) {
int32_t len = strlen(pAlterReq->objname) + 1;
SDbObj *pDb = mndAcquireDb(pMnode, pAlterReq->objname);
@ -1830,7 +1881,8 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode
}
}
if (ALTER_USER_ADD_WRITE_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_ADD_ALL_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (ALTER_USER_ADD_WRITE_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) ||
ALTER_USER_ADD_ALL_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (strcmp(pAlterReq->objname, "1.*") != 0) {
int32_t len = strlen(pAlterReq->objname) + 1;
SDbObj *pDb = mndAcquireDb(pMnode, pAlterReq->objname);
@ -1855,7 +1907,8 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode
}
}
if (ALTER_USER_DEL_READ_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_DEL_ALL_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (ALTER_USER_DEL_READ_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) ||
ALTER_USER_DEL_ALL_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (strcmp(pAlterReq->objname, "1.*") != 0) {
int32_t len = strlen(pAlterReq->objname) + 1;
SDbObj *pDb = mndAcquireDb(pMnode, pAlterReq->objname);
@ -1870,7 +1923,8 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode
}
}
if (ALTER_USER_DEL_WRITE_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_DEL_ALL_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (ALTER_USER_DEL_WRITE_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) ||
ALTER_USER_DEL_ALL_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (strcmp(pAlterReq->objname, "1.*") != 0) {
int32_t len = strlen(pAlterReq->objname) + 1;
SDbObj *pDb = mndAcquireDb(pMnode, pAlterReq->objname);
@ -1885,9 +1939,9 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode
}
}
SHashObj* pReadTbs = pNewUser->readTbs;
SHashObj* pWriteTbs = pNewUser->writeTbs;
SHashObj* pAlterTbs = pNewUser->alterTbs;
SHashObj *pReadTbs = pNewUser->readTbs;
SHashObj *pWriteTbs = pNewUser->writeTbs;
SHashObj *pAlterTbs = pNewUser->alterTbs;
#ifdef TD_ENTERPRISE
if (pAlterReq->isView) {
@ -1897,15 +1951,18 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode
}
#endif
if (ALTER_USER_ADD_READ_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_ADD_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (ALTER_USER_ADD_READ_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) ||
ALTER_USER_ADD_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (mndTablePriviledge(pMnode, pReadTbs, pNewUser->useDbs, pAlterReq, pSdb) != 0) return -1;
}
if (ALTER_USER_ADD_WRITE_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_ADD_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (ALTER_USER_ADD_WRITE_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) ||
ALTER_USER_ADD_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (mndTablePriviledge(pMnode, pWriteTbs, pNewUser->useDbs, pAlterReq, pSdb) != 0) return -1;
}
if (ALTER_USER_ADD_ALTER_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_ADD_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (ALTER_USER_ADD_ALTER_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) ||
ALTER_USER_ADD_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (mndTablePriviledge(pMnode, pAlterTbs, pNewUser->useDbs, pAlterReq, pSdb) != 0) return -1;
}
@ -2016,7 +2073,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
newUser.sysInfo = alterReq.sysInfo;
}
if(alterReq.alterType == TSDB_ALTER_USER_CREATEDB) {
if (alterReq.alterType == TSDB_ALTER_USER_CREATEDB) {
newUser.createdb = alterReq.createdb;
}
@ -2119,52 +2176,43 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
code = mndAlterUser(pMnode, pUser, &newUser, pReq);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
if(alterReq.alterType == TSDB_ALTER_USER_PASSWD){
if (alterReq.alterType == TSDB_ALTER_USER_PASSWD) {
char detail[1000] = {0};
sprintf(detail, "alterType:%s, enable:%d, superUser:%d, sysInfo:%d, createdb:%d, tabName:%s, password:xxx",
mndUserAuditTypeStr(alterReq.alterType), alterReq.enable, alterReq.superUser, alterReq.sysInfo,
alterReq.createdb ? 1 : 0, alterReq.tabName);
auditRecord(pReq, pMnode->clusterId, "alterUser", "", alterReq.user, detail, strlen(detail));
}
else if(alterReq.alterType == TSDB_ALTER_USER_SUPERUSER ||
alterReq.alterType == TSDB_ALTER_USER_ENABLE ||
alterReq.alterType == TSDB_ALTER_USER_SYSINFO ||
alterReq.alterType == TSDB_ALTER_USER_CREATEDB){
} else if (alterReq.alterType == TSDB_ALTER_USER_SUPERUSER || alterReq.alterType == TSDB_ALTER_USER_ENABLE ||
alterReq.alterType == TSDB_ALTER_USER_SYSINFO || alterReq.alterType == TSDB_ALTER_USER_CREATEDB) {
auditRecord(pReq, pMnode->clusterId, "alterUser", "", alterReq.user, alterReq.sql, alterReq.sqlLen);
}
else if(ALTER_USER_ADD_READ_DB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName)||
ALTER_USER_ADD_WRITE_DB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName)||
ALTER_USER_ADD_ALL_DB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName)||
ALTER_USER_ADD_READ_TB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName)||
ALTER_USER_ADD_WRITE_TB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName)||
ALTER_USER_ADD_ALL_TB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName)){
if (strcmp(alterReq.objname, "1.*") != 0){
} else if (ALTER_USER_ADD_READ_DB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName) ||
ALTER_USER_ADD_WRITE_DB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName) ||
ALTER_USER_ADD_ALL_DB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName) ||
ALTER_USER_ADD_READ_TB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName) ||
ALTER_USER_ADD_WRITE_TB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName) ||
ALTER_USER_ADD_ALL_TB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName)) {
if (strcmp(alterReq.objname, "1.*") != 0) {
SName name = {0};
tNameFromString(&name, alterReq.objname, T_NAME_ACCT | T_NAME_DB);
auditRecord(pReq, pMnode->clusterId, "GrantPrivileges", name.dbname, alterReq.user,
alterReq.sql, alterReq.sqlLen);
}else{
auditRecord(pReq, pMnode->clusterId, "GrantPrivileges", "", alterReq.user,
alterReq.sql, alterReq.sqlLen);
auditRecord(pReq, pMnode->clusterId, "GrantPrivileges", name.dbname, alterReq.user, alterReq.sql,
alterReq.sqlLen);
} else {
auditRecord(pReq, pMnode->clusterId, "GrantPrivileges", "", alterReq.user, alterReq.sql, alterReq.sqlLen);
}
}
else if(ALTER_USER_ADD_SUBSCRIBE_TOPIC_PRIV(alterReq.alterType, alterReq.privileges)){
auditRecord(pReq, pMnode->clusterId, "GrantPrivileges", alterReq.objname, alterReq.user,
alterReq.sql, alterReq.sqlLen);
}
else if(ALTER_USER_DEL_SUBSCRIBE_TOPIC_PRIV(alterReq.alterType, alterReq.privileges)){
auditRecord(pReq, pMnode->clusterId, "RevokePrivileges", alterReq.objname, alterReq.user,
alterReq.sql, alterReq.sqlLen);
}
else{
if (strcmp(alterReq.objname, "1.*") != 0){
} else if (ALTER_USER_ADD_SUBSCRIBE_TOPIC_PRIV(alterReq.alterType, alterReq.privileges)) {
auditRecord(pReq, pMnode->clusterId, "GrantPrivileges", alterReq.objname, alterReq.user, alterReq.sql,
alterReq.sqlLen);
} else if (ALTER_USER_DEL_SUBSCRIBE_TOPIC_PRIV(alterReq.alterType, alterReq.privileges)) {
auditRecord(pReq, pMnode->clusterId, "RevokePrivileges", alterReq.objname, alterReq.user, alterReq.sql,
alterReq.sqlLen);
} else {
if (strcmp(alterReq.objname, "1.*") != 0) {
SName name = {0};
tNameFromString(&name, alterReq.objname, T_NAME_ACCT | T_NAME_DB);
auditRecord(pReq, pMnode->clusterId, "RevokePrivileges", name.dbname, alterReq.user,
alterReq.sql, alterReq.sqlLen);
}else{
auditRecord(pReq, pMnode->clusterId, "RevokePrivileges", "", alterReq.user,
alterReq.sql, alterReq.sqlLen);
auditRecord(pReq, pMnode->clusterId, "RevokePrivileges", name.dbname, alterReq.user, alterReq.sql,
alterReq.sqlLen);
} else {
auditRecord(pReq, pMnode->clusterId, "RevokePrivileges", "", alterReq.user, alterReq.sql, alterReq.sqlLen);
}
}
@ -2364,6 +2412,88 @@ static int32_t mndRetrieveUsers(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
return numOfRows;
}
static int32_t mndRetrieveUsersFull(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
int32_t numOfRows = 0;
#ifdef TD_ENTERPRISE
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
SUserObj *pUser = NULL;
int32_t cols = 0;
int8_t flag = 0;
char *pWrite;
int32_t code = 0;
while (numOfRows < rows) {
pShow->pIter = sdbFetch(pSdb, SDB_USER, pShow->pIter, (void **)&pUser);
if (pShow->pIter == NULL) break;
cols = 0;
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
char name[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->user, pShow->pMeta->pSchemas[cols].bytes);
code = colDataSetVal(pColInfo, numOfRows, (const char *)name, false);
if (code != 0) mError("User:%s, failed to retrieve at columns:%d, cause %s", pUser->acct, cols, tstrerror(code));
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
code = colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->superUser, false);
if (code != 0) mError("User:%s, failed to retrieve at columns:%d, cause %s", pUser->acct, cols, tstrerror(code));
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
code = colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->enable, false);
if (code != 0) mError("User:%s, failed to retrieve at columns:%d, cause %s", pUser->acct, cols, tstrerror(code));
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
code = colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->sysInfo, false);
if (code != 0) mError("User:%s, failed to retrieve at columns:%d, cause %s", pUser->acct, cols, tstrerror(code));
cols++;
flag = pUser->createdb ? 1 : 0;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
code = colDataSetVal(pColInfo, numOfRows, (const char *)&flag, false);
if (code != 0) mError("User:%s, failed to retrieve at columns:%d, cause %s", pUser->acct, cols, tstrerror(code));
// mInfo("pUser->pass:%s", pUser->pass);
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
char pass[TSDB_PASSWORD_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(pass, pUser->pass, pShow->pMeta->pSchemas[cols].bytes);
code = colDataSetVal(pColInfo, numOfRows, (const char *)pass, false);
if (code != 0) mError("User:%s, failed to retrieve at columns:%d, cause %s", pUser->acct, cols, tstrerror(code));
cols++;
char *buf = NULL;
int32_t tlen = convertIpWhiteListToStr(pUser->pIpWhiteList, &buf);
// int32_t tlen = mndFetchIpWhiteList(pUser->pIpWhiteList, &buf);
if (tlen != 0) {
char *varstr = taosMemoryCalloc(1, VARSTR_HEADER_SIZE + tlen);
varDataSetLen(varstr, tlen);
memcpy(varDataVal(varstr), buf, tlen);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
code = colDataSetVal(pColInfo, numOfRows, (const char *)varstr, false);
if (code != 0) mError("User:%s, failed to retrieve at columns:%d, cause %s", pUser->acct, cols, tstrerror(code));
taosMemoryFree(varstr);
taosMemoryFree(buf);
} else {
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
code = colDataSetVal(pColInfo, numOfRows, (const char *)NULL, true);
if (code != 0) mError("User:%s, failed to retrieve at columns:%d, cause %s", pUser->acct, cols, tstrerror(code));
}
numOfRows++;
sdbRelease(pSdb, pUser);
}
pShow->numOfRows += numOfRows;
#endif
return numOfRows;
}
static void mndCancelGetNextUser(SMnode *pMnode, void *pIter) {
SSdb *pSdb = pMnode->pSdb;
sdbCancelFetch(pSdb, pIter);
@ -2480,11 +2610,14 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
int32_t numOfReadViews = taosHashGetSize(pUser->readViews);
int32_t numOfWriteViews = taosHashGetSize(pUser->writeViews);
int32_t numOfAlterViews = taosHashGetSize(pUser->alterViews);
if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics + numOfReadTbs + numOfWriteTbs + numOfAlterTbs + numOfReadViews + numOfWriteViews + numOfAlterViews >= rows) {
if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics + numOfReadTbs + numOfWriteTbs + numOfAlterTbs +
numOfReadViews + numOfWriteViews + numOfAlterViews >=
rows) {
mInfo(
"will restore. current num of rows: %d, read dbs %d, write dbs %d, topics %d, read tables %d, write tables "
"%d, alter tables %d, read views %d, write views %d, alter views %d",
numOfRows, numOfReadDbs, numOfWriteDbs, numOfTopics, numOfReadTbs, numOfWriteTbs, numOfAlterTbs, numOfReadViews, numOfWriteViews, numOfAlterViews);
numOfRows, numOfReadDbs, numOfWriteDbs, numOfTopics, numOfReadTbs, numOfWriteTbs, numOfAlterTbs,
numOfReadViews, numOfWriteViews, numOfAlterViews);
pShow->restore = true;
sdbRelease(pSdb, pUser);
break;
@ -2870,7 +3003,6 @@ int32_t mndUserRemoveView(SMnode *pMnode, STrans *pTrans, char *view) {
return code;
}
int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
@ -2910,3 +3042,9 @@ int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic) {
mndUserFreeObj(&newUser);
return code;
}
int64_t mndGetUserIpWhiteListVer(SMnode *pMnode, SUserObj *pUser) {
// ver = 0, disable ip white list
// ver > 0, enable ip white list
return tsEnableWhiteList ? pUser->ipWhiteListVer : 0;
}

27
source/dnode/mnode/impl/test/func/func.cpp
View File

@ -169,7 +169,8 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 1;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
taosArrayPush(retrieveReq.pFuncNames, "f1");
char name[TSDB_FUNC_NAME_LEN] = "f1";
taosArrayPush(retrieveReq.pFuncNames, name);
int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
void* pReq = rpcMallocCont(contLen);
@ -220,7 +221,8 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
retrieveReq.numOfFuncs = TSDB_FUNC_MAX_RETRIEVE + 1;
retrieveReq.pFuncNames = taosArrayInit(TSDB_FUNC_MAX_RETRIEVE + 1, TSDB_FUNC_NAME_LEN);
for (int32_t i = 0; i < TSDB_FUNC_MAX_RETRIEVE + 1; ++i) {
taosArrayPush(retrieveReq.pFuncNames, "1");
char name[TSDB_FUNC_NAME_LEN] = "1";
taosArrayPush(retrieveReq.pFuncNames, name);
}
int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
@ -237,7 +239,8 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 1;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
taosArrayPush(retrieveReq.pFuncNames, "f2");
char name[TSDB_FUNC_NAME_LEN] = "f2";
taosArrayPush(retrieveReq.pFuncNames, name);
int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
void* pReq = rpcMallocCont(contLen);
@ -279,7 +282,8 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 1;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
taosArrayPush(retrieveReq.pFuncNames, "f2");
char name[TSDB_FUNC_NAME_LEN] = "f2";
taosArrayPush(retrieveReq.pFuncNames, name);
int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
void* pReq = rpcMallocCont(contLen);
@ -316,8 +320,10 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 2;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
taosArrayPush(retrieveReq.pFuncNames, "f2");
taosArrayPush(retrieveReq.pFuncNames, "f1");
char name1[TSDB_FUNC_NAME_LEN] = "f2";
taosArrayPush(retrieveReq.pFuncNames, name1);
char name2[TSDB_FUNC_NAME_LEN] = "f1";
taosArrayPush(retrieveReq.pFuncNames, name2);
int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
void* pReq = rpcMallocCont(contLen);
@ -367,8 +373,10 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 2;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
taosArrayPush(retrieveReq.pFuncNames, "f2");
taosArrayPush(retrieveReq.pFuncNames, "f3");
char name1[TSDB_FUNC_NAME_LEN] = "f2";
taosArrayPush(retrieveReq.pFuncNames, name1);
char name2[TSDB_FUNC_NAME_LEN] = "f3";
taosArrayPush(retrieveReq.pFuncNames, name2);
int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
void* pReq = rpcMallocCont(contLen);
@ -483,7 +491,8 @@ TEST_F(MndTestFunc, 05_Actual_code) {
SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 1;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
taosArrayPush(retrieveReq.pFuncNames, "udf1");
char name[TSDB_FUNC_NAME_LEN] = "udf1";
taosArrayPush(retrieveReq.pFuncNames, name);
int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
void* pReq = rpcMallocCont(contLen);

4
source/dnode/mnode/sdb/inc/sdb.h
View File

@ -57,6 +57,7 @@ extern "C" {
#define SDB_GET_INT32(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt32, int32_t)
#define SDB_GET_INT16(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt16, int16_t)
#define SDB_GET_INT8(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt8, int8_t)
#define SDB_GET_UINT8(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawUInt8, uint8_t)
#define SDB_GET_RESERVE(pRaw, dataPos, valLen, pos) \
{ \
@ -76,6 +77,7 @@ extern "C" {
#define SDB_SET_INT32(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawInt32, int32_t)
#define SDB_SET_INT16(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawInt16, int16_t)
#define SDB_SET_INT8(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawInt8, int8_t)
#define SDB_SET_UINT8(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawUInt8, uint8_t)
#define SDB_SET_BINARY(pRaw, dataPos, val, valLen, pos) \
{ \
@ -388,6 +390,7 @@ void sdbGetCommitInfo(SSdb *pSdb, int64_t *index, int64_t *term, int64_t *config
SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen);
void sdbFreeRaw(SSdbRaw *pRaw);
int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val);
int32_t sdbSetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t val);
int32_t sdbSetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t val);
int32_t sdbSetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t val);
int32_t sdbSetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t val);
@ -395,6 +398,7 @@ int32_t sdbSetRawBinary(SSdbRaw *pRaw, int32_t dataPos, const char *pVal, int32
int32_t sdbSetRawDataLen(SSdbRaw *pRaw, int32_t dataLen);
int32_t sdbSetRawStatus(SSdbRaw *pRaw, ESdbStatus status);
int32_t sdbGetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t *val);
int32_t sdbGetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t *val);
int32_t sdbGetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t *val);
int32_t sdbGetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t *val);
int32_t sdbGetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t *val);

30
source/dnode/mnode/sdb/src/sdbRaw.c
View File

@ -67,6 +67,21 @@ int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val) {
return 0;
}
int32_t sdbSetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t val) {
if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
}
if (dataPos + sizeof(uint8_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1;
}
*(uint8_t *)(pRaw->pData + dataPos) = val;
return 0;
}
int32_t sdbSetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t val) {
if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR;
@ -174,6 +189,21 @@ int32_t sdbGetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t *val) {
return 0;
}
int32_t sdbGetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t *val) {
if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
}
if (dataPos + sizeof(uint8_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1;
}
*val = *(uint8_t *)(pRaw->pData + dataPos);
return 0;
}
int32_t sdbGetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t *val) {
if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR;

18
source/dnode/snode/src/snode.c
View File

@ -25,20 +25,6 @@
#define sndDebug(...) do { if (sndDebugFlag & DEBUG_DEBUG) { taosPrintLog("SND ", DEBUG_DEBUG, sndDebugFlag, __VA_ARGS__);}} while (0)
// clang-format on
static STaskId replaceStreamTaskId(SStreamTask *pTask) {
ASSERT(pTask->info.fillHistory);
STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
pTask->id.streamId = pTask->streamTaskId.streamId;
pTask->id.taskId = pTask->streamTaskId.taskId;
return id;
}
static void restoreStreamTaskId(SStreamTask *pTask, STaskId *pId) {
ASSERT(pTask->info.fillHistory);
pTask->id.taskId = pId->taskId;
pTask->id.streamId = pId->streamId;
}
int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer) {
ASSERT(pTask->info.taskLevel == TASK_LEVEL__AGG && taosArrayGetSize(pTask->upstreamInfo.pList) != 0);
int32_t code = streamTaskInit(pTask, pSnode->pMeta, &pSnode->msgCb, nextProcessVer);
@ -85,7 +71,7 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
startRsync();
pSnode->msgCb = pOption->msgCb;
pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskExpand *)sndExpandTask, SNODE_HANDLE, taosGetTimestampMs(), tqStartTaskCompleteCallback);
pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskBuild *)sndExpandTask, tqExpandStreamTask, SNODE_HANDLE, taosGetTimestampMs(), tqStartTaskCompleteCallback);
if (pSnode->pMeta == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto FAIL;
@ -136,6 +122,8 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) {
return tqStreamProcessReqCheckpointRsp(pSnode->pMeta, pMsg);
case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP:
return tqStreamProcessCheckpointReadyRsp(pSnode->pMeta, pMsg);
case TDMT_MND_STREAM_CHKPT_REPORT_RSP:
return tqStreamProcessChkptReportRsp(pSnode->pMeta, pMsg);
case TDMT_STREAM_RETRIEVE_TRIGGER:
return tqStreamTaskProcessRetrieveTriggerReq(pSnode->pMeta, pMsg);
case TDMT_STREAM_RETRIEVE_TRIGGER_RSP:

11
source/dnode/vnode/inc/vnode.h
View File

@ -52,6 +52,7 @@ extern const SVnodeCfg vnodeCfgDefault;
int32_t vnodeInit(int32_t nthreads);
void vnodeCleanup();
int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, int32_t diskPrimary, STfs *pTfs);
bool vnodeShouldRemoveWal(SVnode *pVnode);
int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, int32_t diskPrimary, STfs *pTfs);
int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnodeHashRangeReq *pReq,
int32_t diskPrimary, STfs *pTfs);
@ -181,7 +182,7 @@ void tsdbReaderSetNotifyCb(STsdbReader *pReader, TsdReaderNotifyCbFn not
int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables);
int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr,
SArray *pFuncTypeList, SColumnInfo* pkCol, int32_t numOfPks);
SArray *pFuncTypeList, SColumnInfo *pkCol, int32_t numOfPks);
int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds,
SArray *pTableUids);
void *tsdbCacherowsReaderClose(void *pReader);
@ -218,8 +219,8 @@ typedef struct STqReader {
STqReader *tqReaderOpen(SVnode *pVnode);
void tqReaderClose(STqReader *);
bool tqGetTablePrimaryKey(STqReader* pReader);
void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid);
bool tqGetTablePrimaryKey(STqReader *pReader);
void tqSetTablePrimaryKey(STqReader *pReader, int64_t uid);
void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList);
int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList, const char *id);
@ -278,8 +279,8 @@ struct STsdbCfg {
int32_t keep2; // just for save config, don't use in tsdbRead/tsdbCommit/..., and use STsdbKeepCfg in STsdb instead
int32_t keepTimeOffset; // just for save config, use STsdbKeepCfg in STsdb instead
SRetention retentions[TSDB_RETENTION_MAX];
int32_t encryptAlgorithm;
char encryptKey[ENCRYPT_KEY_LEN + 1];
int32_t encryptAlgorithm;
char encryptKey[ENCRYPT_KEY_LEN + 1];
};
typedef struct {

3
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -261,9 +261,10 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessStreamHbRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessStreamReqCheckpointRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskChkptReportRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskCheckpointReadyRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver);
int32_t tqBuildStreamTask(void* pTq, SStreamTask* pTask, int64_t ver);
int32_t tqScanWal(STQ* pTq);
int tqCommit(STQ*);

7
source/dnode/vnode/src/meta/metaTable.c
View File

@ -2240,7 +2240,7 @@ int32_t metaUpdateTableColCompress(SMeta *pMeta, int64_t version, SVAlterTbReq *
uint32_t dst = 0;
updated = tUpdateCompress(p->alg, pReq->compress, TSDB_COLVAL_COMPRESS_DISABLED, TSDB_COLVAL_LEVEL_DISABLED,
TSDB_COLVAL_LEVEL_MEDIUM, &dst);
if (updated) {
if (updated > 0) {
p->alg = dst;
}
}
@ -2250,6 +2250,11 @@ int32_t metaUpdateTableColCompress(SMeta *pMeta, int64_t version, SVAlterTbReq *
tDecoderClear(&dc);
terrno = TSDB_CODE_VND_COLUMN_COMPRESS_ALREADY_EXIST;
goto _err;
} else if (updated < 0) {
tdbFree(pVal);
tDecoderClear(&dc);
terrno = TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR;
goto _err;
}
tbEntry.version = version;

2
source/dnode/vnode/src/sma/smaRollup.c
View File

@ -299,7 +299,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat
tdRSmaTaskInit(pStreamTask->pMeta, pItem, &pStreamTask->id);
pStreamTask->status.pSM = streamCreateStateMachine(pStreamTask);
pStreamTask->chkInfo.pActiveInfo = streamTaskCreateActiveChkptInfo();
pStreamState = streamStateOpen(taskInfDir, pStreamTask, true, -1, -1);
pStreamState = streamStateOpen(taskInfDir, pStreamTask, pStreamTask->id.streamId, pStreamTask->id.taskId);
if (!pStreamState) {
terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN;
return TSDB_CODE_FAILED;

31
source/dnode/vnode/src/tq/tq.c
View File

@ -50,6 +50,9 @@ void tqDestroyTqHandle(void* data) {
if (pData->block != NULL) {
blockDataDestroy(pData->block);
}
if (pData->pRef) {
walCloseRef(pData->pRef->pWal, pData->pRef->refId);
}
}
static bool tqOffsetEqual(const STqOffset* pLeft, const STqOffset* pRight) {
@ -87,7 +90,8 @@ STQ* tqOpen(const char* path, SVnode* pVnode) {
int32_t tqInitialize(STQ* pTq) {
int32_t vgId = TD_VID(pTq->pVnode);
pTq->pStreamMeta = streamMetaOpen(pTq->path, pTq, (FTaskExpand*)tqExpandTask, vgId, -1, tqStartTaskCompleteCallback);
pTq->pStreamMeta =
streamMetaOpen(pTq->path, pTq, tqBuildStreamTask, tqExpandStreamTask, vgId, -1, tqStartTaskCompleteCallback);
if (pTq->pStreamMeta == NULL) {
return -1;
}
@ -571,9 +575,6 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
taosMsleep(10);
continue;
}
if (pHandle->pRef) {
walCloseRef(pTq->pVnode->pWal, pHandle->pRef->refId);
}
tqUnregisterPushHandle(pTq, pHandle);
@ -658,12 +659,10 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
taosRLockLatch(&pTq->lock);
ret = tqMetaGetHandle(pTq, req.subKey);
taosRUnLockLatch(&pTq->lock);
if (ret < 0) {
break;
}
}
if (pHandle == NULL) {
if (req.oldConsumerId != -1) {
tqError("vgId:%d, build new consumer handle %s for consumer:0x%" PRIx64 ", but old consumerId:0x%" PRIx64,
@ -708,14 +707,16 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
}
}
end:
end:
tDecoderClear(&dc);
return ret;
}
static void freePtr(void* ptr) { taosMemoryFree(*(void**)ptr); }
int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
int32_t tqBuildStreamTask(void* pTqObj, SStreamTask* pTask, int64_t nextProcessVer) {
STQ* pTq = (STQ*) pTqObj;
int32_t vgId = TD_VID(pTq->pVnode);
tqDebug("s-task:0x%x start to build task", pTask->id.taskId);
@ -1012,16 +1013,6 @@ int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) {
}
int32_t tqProcessTaskUpdateCheckpointReq(STQ* pTq, char* msg, int32_t msgLen) {
int32_t vgId = TD_VID(pTq->pVnode);
SVUpdateCheckpointInfoReq* pReq = (SVUpdateCheckpointInfoReq*)msg;
// if (!pTq->pVnode->restored) {
// tqDebug("vgId:%d update-checkpoint-info msg received during restoring, checkpointId:%" PRId64
// ", transId:%d s-task:0x%x ignore it",
// vgId, pReq->checkpointId, pReq->transId, pReq->taskId);
// return TSDB_CODE_SUCCESS;
// }
return tqStreamTaskProcessUpdateCheckpointReq(pTq->pStreamMeta, msg, msgLen);
}
@ -1278,3 +1269,7 @@ int32_t tqProcessStreamReqCheckpointRsp(STQ* pTq, SRpcMsg* pMsg) {
int32_t tqProcessTaskCheckpointReadyRsp(STQ* pTq, SRpcMsg* pMsg) {
return tqStreamProcessCheckpointReadyRsp(pTq->pStreamMeta, pMsg);
}
int32_t tqProcessTaskChkptReportRsp(STQ* pTq, SRpcMsg* pMsg) {
return tqStreamProcessChkptReportRsp(pTq->pStreamMeta, pMsg);
}

3
source/dnode/vnode/src/tq/tqMeta.c
View File

@ -352,7 +352,6 @@ int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){
memcpy(handle->subKey, req->subKey, TSDB_SUBSCRIBE_KEY_LEN);
handle->consumerId = req->newConsumerId;
handle->epoch = -1;
handle->execHandle.subType = req->subType;
handle->fetchMeta = req->withMeta;
@ -371,7 +370,7 @@ int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){
if(buildHandle(pTq, handle) < 0){
return -1;
}
tqInfo("tqCreateHandle %s consumer 0x%" PRIx64 " vgId:%d", handle->subKey, handle->consumerId, vgId);
tqInfo("tqCreateHandle %s consumer 0x%" PRIx64 " vgId:%d, snapshotVer:%" PRId64, handle->subKey, handle->consumerId, vgId, handle->snapshotVer);
return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle));
}

2
source/dnode/vnode/src/tq/tqPush.c
View File

@ -87,7 +87,7 @@ int tqUnregisterPushHandle(STQ* pTq, void *handle) {
int32_t ret = taosHashRemove(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey));
tqInfo("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId);
if(pHandle->msg != NULL) {
if(ret == 0 && pHandle->msg != NULL) {
// tqPushDataRsp(pHandle, vgId);
tqPushEmptyDataRsp(pHandle, vgId);

7
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -605,14 +605,15 @@ int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, b
numOfTasks = taosArrayGetSize(pMeta->pTaskList);
for (int32_t i = 0; i < numOfTasks; ++i) {
STaskId* pId = taosArrayGet(pMeta->pTaskList, i);
SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i);
if (pId->streamId != streamId) {
continue;
}
SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, pId, sizeof(*pId));
STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId};
SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
if (ppTask == NULL) {
tqError("vgId:%d failed to acquire task:0x%" PRIx64 " in retrieving progress", pMeta->vgId, pId->taskId);
tqError("vgId:%d failed to acquire task:0x%x in retrieving progress", pMeta->vgId, pId->taskId);
continue;
}

53
source/dnode/vnode/src/tqCommon/tqCommon.c
View File

@ -30,37 +30,26 @@ typedef struct SMStreamCheckpointReadyRspMsg {
static int32_t doProcessDummyRspMsg(SStreamMeta* pMeta, SRpcMsg* pMsg);
static STaskId replaceStreamTaskId(SStreamTask* pTask) {
ASSERT(pTask->info.fillHistory);
STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
pTask->id.streamId = pTask->streamTaskId.streamId;
pTask->id.taskId = pTask->streamTaskId.taskId;
return id;
}
static void restoreStreamTaskId(SStreamTask* pTask, STaskId* pId) {
ASSERT(pTask->info.fillHistory);
pTask->id.taskId = pId->taskId;
pTask->id.streamId = pId->streamId;
}
int32_t tqExpandStreamTask(SStreamTask* pTask) {
SStreamMeta* pMeta = pTask->pMeta;
int32_t vgId = pMeta->vgId;
STaskId taskId = {0};
int64_t st = taosGetTimestampMs();
int64_t streamId = 0;
int32_t taskId = 0;
tqDebug("s-task:%s vgId:%d start to expand stream task", pTask->id.idStr, vgId);
if (pTask->info.fillHistory) {
taskId = replaceStreamTaskId(pTask);
streamId = pTask->streamTaskId.streamId;
taskId = pTask->streamTaskId.taskId;
} else {
streamId = pTask->id.streamId;
taskId = pTask->id.taskId;
}
// sink task does not need the pState
if (pTask->info.taskLevel != TASK_LEVEL__SINK) {
pTask->pState = streamStateOpen(pMeta->path, pTask, false, -1, -1);
pTask->pState = streamStateOpen(pMeta->path, pTask, streamId, taskId);
if (pTask->pState == NULL) {
tqError("s-task:%s (vgId:%d) failed to open state for task, expand task failed", pTask->id.idStr, vgId);
return -1;
@ -69,10 +58,6 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) {
}
}
if (pTask->info.fillHistory) {
restoreStreamTaskId(pTask, &taskId);
}
SReadHandle handle = {
.checkpointId = pTask->chkInfo.checkpointId,
.pStateBackend = pTask->pState,
@ -185,7 +170,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
SStreamTask* pTask = *ppTask;
const char* idstr = pTask->id.idStr;
if (pMeta->updateInfo.transId != req.transId) {
if ((pMeta->updateInfo.transId != req.transId) && (pMeta->updateInfo.transId != -1)) {
if (req.transId < pMeta->updateInfo.transId) {
tqError("s-task:%s vgId:%d disorder update nodeEp msg recv, discarded, newest transId:%d, recv:%d", idstr, vgId,
pMeta->updateInfo.transId, req.transId);
@ -197,10 +182,8 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
} else {
tqInfo("s-task:%s vgId:%d receive new trans to update nodeEp msg from mnode, transId:%d, prev transId:%d", idstr,
vgId, req.transId, pMeta->updateInfo.transId);
// info needs to be kept till the new trans to update the nodeEp arrived.
taosHashClear(pMeta->updateInfo.pTasks);
pMeta->updateInfo.transId = req.transId;
streamMetaInitUpdateTaskList(pMeta, req.transId);
}
} else {
tqDebug("s-task:%s vgId:%d recv trans to update nodeEp from mnode, transId:%d", idstr, vgId, req.transId);
@ -280,6 +263,8 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
// persist to disk
}
streamMetaClearUpdateTaskList(pMeta);
if (!restored) {
tqDebug("vgId:%d vnode restore not completed, not start the tasks, clear the start after nodeUpdate flag", vgId);
pMeta->startInfo.tasksWillRestart = 0;
@ -362,6 +347,7 @@ int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
int32_t vgId = pMeta->vgId;
pRsp->upstreamNodeId = htonl(pRsp->upstreamNodeId);
pRsp->upstreamTaskId = htonl(pRsp->upstreamTaskId);
pRsp->streamId = htobe64(pRsp->streamId);
pRsp->downstreamTaskId = htonl(pRsp->downstreamTaskId);
@ -369,6 +355,9 @@ int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
pRsp->stage = htobe64(pRsp->stage);
pRsp->msgId = htonl(pRsp->msgId);
tqDebug("s-task:0x%x vgId:%d recv dispatch-rsp from 0x%x vgId:%d", pRsp->upstreamTaskId, pRsp->upstreamNodeId,
pRsp->downstreamTaskId, pRsp->downstreamNodeId);
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId);
if (pTask) {
streamProcessDispatchRsp(pTask, pRsp, pMsg->code);
@ -414,7 +403,9 @@ int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
streamMetaReleaseTask(pMeta, pTask);
tCleanupStreamRetrieveReq(&req);
return code;
// always return success, to disable the auto rsp
return TSDB_CODE_SUCCESS;
}
int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
@ -629,8 +620,8 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
// drop the related fill-history task firstly
if (hTaskId.taskId != 0 && hTaskId.streamId != 0) {
streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId);
tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId);
streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId);
}
// drop the stream task now
@ -646,8 +637,6 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
}
streamMetaWUnLock(pMeta);
// tqStreamRemoveTaskBackend(pMeta, &id);
return 0;
}
@ -1078,6 +1067,8 @@ int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return d
int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); }
int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {return doProcessDummyRspMsg(pMeta, pMsg);}
int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SMStreamCheckpointReadyRspMsg* pRsp = pMsg->pCont;

24
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
View File

@ -46,6 +46,12 @@ SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList,
}
pLoadInfo->aSttBlk = taosArrayInit(4, sizeof(SSttBlk));
if (pLoadInfo->aSttBlk == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFreeClear(pLoadInfo);
return NULL;
}
pLoadInfo->pSchema = pSchema;
pLoadInfo->colIds = colList;
pLoadInfo->numOfCols = numOfCols;
@ -107,15 +113,21 @@ void *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoa
SArray *pList = taosArrayGetP(pLDataIterArray, i);
for (int32_t j = 0; j < taosArrayGetSize(pList); ++j) {
SLDataIter *pIter = taosArrayGetP(pList, j);
if (pIter->pBlockLoadInfo == NULL) {
continue;
}
SSttBlockLoadCostInfo* pCost = &pIter->pBlockLoadInfo->cost;
if (pLoadCost != NULL) {
pLoadCost->loadBlocks += pIter->pBlockLoadInfo->cost.loadBlocks;
pLoadCost->loadStatisBlocks += pIter->pBlockLoadInfo->cost.loadStatisBlocks;
pLoadCost->blockElapsedTime += pIter->pBlockLoadInfo->cost.blockElapsedTime;
pLoadCost->statisElapsedTime += pIter->pBlockLoadInfo->cost.statisElapsedTime;
pLoadCost->loadBlocks += pCost->loadBlocks;
pLoadCost->loadStatisBlocks += pCost->loadStatisBlocks;
pLoadCost->blockElapsedTime += pCost->blockElapsedTime;
pLoadCost->statisElapsedTime += pCost->statisElapsedTime;
}
destroyLDataIter(pIter);
}
taosArrayDestroy(pList);
}
@ -903,6 +915,10 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoF
if (pLoadInfo == NULL) {
pLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols);
if (pLoadInfo == NULL) {
code = terrno;
goto _end;
}
}
memset(pIter, 0, sizeof(SLDataIter));

16
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -2240,7 +2240,8 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan
};
SSttDataInfoForTable info = {.pKeyRangeList = taosArrayInit(4, sizeof(SSttKeyRange))};
int32_t code = tMergeTreeOpen2(&pSttBlockReader->mergeTree, &conf, &info);
int32_t code = tMergeTreeOpen2(&pSttBlockReader->mergeTree, &conf, &info);
if (code != TSDB_CODE_SUCCESS) {
return false;
}
@ -4896,7 +4897,7 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_
}
int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool* hasNullSMA) {
SColumnDataAgg*** pBlockSMA = &pDataBlock->pBlockAgg;
SColumnDataAgg** pBlockSMA = &pDataBlock->pBlockAgg;
int32_t code = 0;
*allHave = false;
@ -4951,7 +4952,13 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock,
if (pResBlock->pBlockAgg == NULL) {
size_t num = taosArrayGetSize(pResBlock->pDataBlock);
pResBlock->pBlockAgg = taosMemoryCalloc(num, POINTER_BYTES);
pResBlock->pBlockAgg = taosMemoryCalloc(num, sizeof(SColumnDataAgg));
if (pResBlock->pBlockAgg == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for(int i = 0; i < num; ++i) {
pResBlock->pBlockAgg[i].colId = -1;
}
}
// do fill all null column value SMA info
@ -4963,13 +4970,12 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock,
while (j < numOfCols && i < size) {
SColumnDataAgg* pAgg = &pSup->colAggArray.data[i];
if (pAgg->colId == pSup->colId[j]) {
pResBlock->pBlockAgg[pSup->slotId[j]] = pAgg;
pResBlock->pBlockAgg[pSup->slotId[j]] = *pAgg;
i += 1;
j += 1;
} else if (pAgg->colId < pSup->colId[j]) {
i += 1;
} else if (pSup->colId[j] < pAgg->colId) {
pResBlock->pBlockAgg[pSup->slotId[j]] = NULL;
*allHave = false;
j += 1;
}

4
source/dnode/vnode/src/tsdb/tsdbReadUtil.c
View File

@ -985,6 +985,10 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra
if (pIter->pBlockLoadInfo == NULL) {
pIter->pBlockLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols);
if (pIter->pBlockLoadInfo == NULL) {
tsdbError("failed to create block load info, code: out of memory, %s", pstr);
continue;
}
}
// load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file

31
source/dnode/vnode/src/tsdb/tsdbRetention.c
View File

@ -38,6 +38,34 @@ static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) {
return TARRAY2_APPEND(&rtner->fopArr, op);
}
static int64_t tsdbCopyFileWithLimitedSpeed(TdFilePtr from, TdFilePtr to, int64_t size, uint32_t limitMB) {
int64_t total = 0;
int64_t interval = 1000; // 1s
int64_t limit = limitMB ? limitMB * 1024 * 1024 : INT64_MAX;
int64_t offset = 0;
int64_t remain = size;
while (remain > 0) {
int64_t n;
int64_t last = taosGetTimestampMs();
if ((n = taosFSendFile(to, from, &offset, TMIN(limit, remain))) < 0) {
return -1;
}
total += n;
remain -= n;
if (remain > 0) {
int64_t elapsed = taosGetTimestampMs() - last;
if (elapsed < interval) {
taosMsleep(interval - elapsed);
}
}
}
return total;
}
static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFile *to) {
int32_t code = 0;
int32_t lino = 0;
@ -98,7 +126,8 @@ static int32_t tsdbDoCopyFile(SRTNer *rtner, const STFileObj *from, const STFile
if (fdTo == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
int64_t n = taosFSendFile(fdTo, fdFrom, 0, tsdbLogicToFileSize(from->f->size, rtner->szPage));
int64_t n = tsdbCopyFileWithLimitedSpeed(fdFrom, fdTo, tsdbLogicToFileSize(from->f->size, rtner->szPage),
tsRetentionSpeedLimitMB);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit);

27
source/dnode/vnode/src/vnd/vnodeCfg.c
View File

@ -13,9 +13,9 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tglobal.h"
#include "tutil.h"
#include "vnd.h"
#include "tglobal.h"
const SVnodeCfg vnodeCfgDefault = {.vgId = -1,
.dbname = "",
@ -47,6 +47,7 @@ const SVnodeCfg vnodeCfgDefault = {.vgId = -1,
.segSize = 0,
.retentionSize = -1,
.level = TAOS_WAL_WRITE,
.clearFiles = 0,
},
.hashBegin = 0,
.hashEnd = 0,
@ -142,6 +143,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
if (tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.clearFiles", pCfg->walCfg.clearFiles) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sttTrigger) < 0) return -1;
@ -249,12 +251,11 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
tjsonGetNumberValue(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm, code);
if (code < 0) return -1;
#if defined(TD_ENTERPRISE)
if(pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4){
if(tsEncryptKey[0] == 0){
if (pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4) {
if (tsEncryptKey[0] == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
}
else{
} else {
strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
@ -273,15 +274,16 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
if (code < 0) return -1;
tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code);
if (code < 0) return -1;
tjsonGetNumberValue(pJson, "wal.clearFiles", pCfg->walCfg.clearFiles, code);
if (code < 0) return -1;
tjsonGetNumberValue(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm, code);
if (code < 0) return -1;
#if defined(TD_ENTERPRISE)
if(pCfg->walCfg.encryptAlgorithm == DND_CA_SM4){
if(tsEncryptKey[0] == 0){
if (pCfg->walCfg.encryptAlgorithm == DND_CA_SM4) {
if (tsEncryptKey[0] == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
}
else{
} else {
strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
@ -289,12 +291,11 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
tjsonGetNumberValue(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm, code);
if (code < 0) return -1;
#if defined(TD_ENTERPRISE)
if(pCfg->tdbEncryptAlgorithm == DND_CA_SM4){
if(tsEncryptKey[0] == 0){
if (pCfg->tdbEncryptAlgorithm == DND_CA_SM4) {
if (tsEncryptKey[0] == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
}
else{
} else {
strncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}

25
source/dnode/vnode/src/vnd/vnodeOpen.c
View File

@ -81,6 +81,8 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, int32_t diskPrimary, STfs
return 0;
}
bool vnodeShouldRemoveWal(SVnode *pVnode) { return pVnode->config.walCfg.clearFiles == 1; }
int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, int32_t diskPrimary, STfs *pTfs) {
SVnodeInfo info = {0};
char dir[TSDB_FILENAME_LEN] = {0};
@ -129,6 +131,12 @@ int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, int32_t
}
pCfg->changeVersion = pReq->changeVersion;
if (info.config.walCfg.clearFiles) {
info.config.walCfg.clearFiles = 0;
vInfo("vgId:%d, reset wal clearFiles", pReq->vgId);
}
vInfo("vgId:%d, save config while alter, replicas:%d totalReplicas:%d selfIndex:%d changeVersion:%d", pReq->vgId,
pCfg->replicaNum, pCfg->totalReplicaNum, pCfg->myIndex, pCfg->changeVersion);
@ -486,15 +494,14 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC
if (tsEnableMonitor && pVnode->monitor.insertCounter == NULL) {
taos_counter_t *counter = NULL;
int32_t label_count = 7;
const char *sample_labels[] = {VNODE_METRIC_TAG_NAME_SQL_TYPE, VNODE_METRIC_TAG_NAME_CLUSTER_ID,
VNODE_METRIC_TAG_NAME_DNODE_ID, VNODE_METRIC_TAG_NAME_DNODE_EP,
VNODE_METRIC_TAG_NAME_VGROUP_ID, VNODE_METRIC_TAG_NAME_USERNAME,
VNODE_METRIC_TAG_NAME_RESULT};
counter = taos_counter_new(VNODE_METRIC_SQL_COUNT, "counter for insert sql",
label_count, sample_labels);
vInfo("vgId:%d, new metric:%p",TD_VID(pVnode), counter);
if(taos_collector_registry_register_metric(counter) == 1){
int32_t label_count = 7;
const char *sample_labels[] = {VNODE_METRIC_TAG_NAME_SQL_TYPE, VNODE_METRIC_TAG_NAME_CLUSTER_ID,
VNODE_METRIC_TAG_NAME_DNODE_ID, VNODE_METRIC_TAG_NAME_DNODE_EP,
VNODE_METRIC_TAG_NAME_VGROUP_ID, VNODE_METRIC_TAG_NAME_USERNAME,
VNODE_METRIC_TAG_NAME_RESULT};
counter = taos_counter_new(VNODE_METRIC_SQL_COUNT, "counter for insert sql", label_count, sample_labels);
vInfo("vgId:%d, new metric:%p", TD_VID(pVnode), counter);
if (taos_collector_registry_register_metric(counter) == 1) {
taos_counter_destroy(counter);
counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT);
vInfo("vgId:%d, get metric from registry:%p", TD_VID(pVnode), counter);

36
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -120,7 +120,8 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
memcpy(metaRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols);
}
if (metaRsp.pSchemaExt) {
code = fillTableColCmpr(&mer1, metaRsp.pSchemaExt, metaRsp.numOfColumns);
SMetaReader *pReader = mer1.me.type == TSDB_CHILD_TABLE ? &mer2 : &mer1;
code = fillTableColCmpr(pReader, metaRsp.pSchemaExt, metaRsp.numOfColumns);
if (code < 0) {
code = TSDB_CODE_INVALID_MSG;
goto _exit;
@ -254,15 +255,18 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
memcpy(cfgRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols);
}
if (useCompress(cfgRsp.tableType)) {
SColCmprWrapper *pColCmpr = &mer1.me.colCmpr;
for (int32_t i = 0; i < cfgRsp.numOfColumns; i++) {
SColCmpr *pCmpr = &pColCmpr->pColCmpr[i];
SSchemaExt *pSchExt = cfgRsp.pSchemaExt + i;
pSchExt->colId = pCmpr->id;
pSchExt->compress = pCmpr->alg;
}
// if (useCompress(cfgRsp.tableType)) {
SMetaReader *pReader = mer1.me.type == TSDB_CHILD_TABLE ? &mer2 : &mer1;
SColCmprWrapper *pColCmpr = &pReader->me.colCmpr;
for (int32_t i = 0; i < cfgRsp.numOfColumns; i++) {
SColCmpr *pCmpr = &pColCmpr->pColCmpr[i];
SSchemaExt *pSchExt = cfgRsp.pSchemaExt + i;
pSchExt->colId = pCmpr->id;
pSchExt->compress = pCmpr->alg;
}
//}
// encode and send response
rspLen = tSerializeSTableCfgRsp(NULL, 0, &cfgRsp);
@ -752,13 +756,13 @@ int32_t vnodeGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64
return tsdbGetTableSchema(((SVnode *)pVnode)->pMeta, uid, pSchema, suid);
}
int32_t vnodeGetStreamProgress(SVnode* pVnode, SRpcMsg* pMsg, bool direct) {
int32_t code = 0;
SStreamProgressReq req;
SStreamProgressRsp rsp = {0};
SRpcMsg rpcMsg = {.info = pMsg->info, .code = 0};
char * buf = NULL;
int32_t rspLen = 0;
int32_t vnodeGetStreamProgress(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
int32_t code = 0;
SStreamProgressReq req;
SStreamProgressRsp rsp = {0};
SRpcMsg rpcMsg = {.info = pMsg->info, .code = 0};
char *buf = NULL;
int32_t rspLen = 0;
code = tDeserializeStreamProgressReq(pMsg->pCont, pMsg->contLen, &req);
if (code == TSDB_CODE_SUCCESS) {

5
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -854,6 +854,8 @@ int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo)
return tqProcessStreamReqCheckpointRsp(pVnode->pTq, pMsg);
case TDMT_VND_GET_STREAM_PROGRESS:
return tqStreamProgressRetrieveReq(pVnode->pTq, pMsg);
case TDMT_MND_STREAM_CHKPT_REPORT_RSP:
return tqProcessTaskChkptReportRsp(pVnode->pTq, pMsg);
default:
vError("unknown msg type:%d in stream queue", pMsg->msgType);
return TSDB_CODE_APP_ERROR;
@ -2021,6 +2023,9 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pRe
}
if (pVnode->config.walCfg.level != req.walLevel) {
if (pVnode->config.walCfg.level == 0) {
pVnode->config.walCfg.clearFiles = 1;
}
pVnode->config.walCfg.level = req.walLevel;
walChanged = true;
}

Some files were not shown because too many files have changed in this diff Show More