xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #28674 from taosdata/main 

merge: from main to 3.0
This commit is contained in:
Shengliang Guan 2024-11-07 09:04:54 +08:00 committed by GitHub
parent 1154f7be1a 60da49c09e
commit 05075cb2a0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 7296 additions and 9007 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -159,4 +159,6 @@ pcre2.h
zconf.h zconf.h
version.h version.h
geos_c.h geos_c.h
source/libs/parser/src/sql.c
include/common/ttokenauto.h

11
cmake/lemon_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,11 @@
# lemon
ExternalProject_Add(
lemon
SOURCE_DIR ${TD_CONTRIB_DIR}/lemon
CONFIGURE_COMMAND ""
BUILD_COMMAND "${C_COMPILER_LEMON}" -o ${TD_CONTRIB_DIR}/lemon/lemon ${TD_CONTRIB_DIR}/lemon/lemon.c
INSTALL_COMMAND ""
BUILD_IN_SOURCE 1
BUILD_ALWAYS 1
)

11
contrib/CMakeLists.txt
View File

@ -184,6 +184,17 @@ if(${BUILD_PCRE2})
cat("${TD_SUPPORT_DIR}/pcre2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/pcre2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif() endif()
find_program(C_COMPILER_LEMON NAMES gcc)
if(C_COMPILER_LEMON)
message(STATUS "LEMON C compiler: ${C_COMPILER_LEMON}")
else()
set(C_COMPILER_LEMON ${CMAKE_C_COMPILER})
message(STATUS "LEMON C compiler: ${C_COMPILER_LEMON}")
endif()
# lemon
cat("${TD_SUPPORT_DIR}/lemon_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
# download dependencies # download dependencies
configure_file(${CONTRIB_TMP_FILE} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") configure_file(${CONTRIB_TMP_FILE} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .

6038
contrib/lemon/lemon.c Normal file
View File

File diff suppressed because it is too large Load Diff

1086
contrib/lemon/lempar.c Normal file
View File

File diff suppressed because it is too large Load Diff

123
docs/zh/14-reference/01-components/06-taoskeeper.md
View File

@ -22,34 +22,45 @@ taosKeeper 有两种安装方式：
taosKeeper 需要在操作系统终端执行，该工具支持三种配置方式：命令行参数、环境变量 和 配置文件。优先级为：命令行参数、环境变量、配置文件参数。 一般我们推荐使用配置文件。 taosKeeper 需要在操作系统终端执行，该工具支持三种配置方式：命令行参数、环境变量 和 配置文件。优先级为：命令行参数、环境变量、配置文件参数。 一般我们推荐使用配置文件。
### 命令行参数和环境变量 ### 命令行参数和环境变量
命令行参数 和 环境变量说明可以参考命令 `taoskeeper --help` 的输出。下面是一个例子： 命令行参数 和 环境变量说明可以参考命令 `taoskeeper --help` 的输出。下面是一个例子：
```shell ```shell
Usage of taosKeeper v3.3.2.0: Usage of taoskeeper v3.3.3.0:
--debug enable debug mode. Env "TAOS_KEEPER_DEBUG"
-P, --port int http port. Env "TAOS_KEEPER_PORT" (default 6043)
--logLevel string log level (panic fatal error warn warning info debug trace). Env "TAOS_KEEPER_LOG_LEVEL" (default "info")
--gopoolsize int coroutine size. Env "TAOS_KEEPER_POOL_SIZE" (default 50000)
-R, --RotationInterval string interval for refresh metrics, such as "300ms", Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". Env "TAOS_KEEPER_ROTATION_INTERVAL" (default "15s") -R, --RotationInterval string interval for refresh metrics, such as "300ms", Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". Env "TAOS_KEEPER_ROTATION_INTERVAL" (default "15s")
-c, --config string config path default /etc/taos/taoskeeper.toml
--drop string run taoskeeper in command mode, only support old_taosd_metric_stables.
--environment.incgroup whether running in cgroup. Env "TAOS_KEEPER_ENVIRONMENT_INCGROUP"
--fromTime string parameter of transfer, example: 2020-01-01T00:00:00+08:00 (default "2020-01-01T00:00:00+08:00")
--gopoolsize int coroutine size. Env "TAOS_KEEPER_POOL_SIZE" (default 50000)
-h, --help Print this help message and exit
--instanceId int instance ID. Env "TAOS_KEEPER_INSTANCE_ID" (default 64)
--log.compress whether to compress old log. Env "TAOS_KEEPER_LOG_COMPRESS"
--log.keepDays uint log retention days, must be a positive integer. Env "TAOS_KEEPER_LOG_KEEP_DAYS" (default 30)
--log.level string log level (trace debug info warning error). Env "TAOS_KEEPER_LOG_LEVEL" (default "info")
--log.path string log path. Env "TAOS_KEEPER_LOG_PATH" (default "/var/log/taos")
--log.reservedDiskSize string reserved disk size for log dir (KB MB GB), must be a positive integer. Env "TAOS_KEEPER_LOG_RESERVED_DISK_SIZE" (default "1GB")
--log.rotationCount uint log rotation count. Env "TAOS_KEEPER_LOG_ROTATION_COUNT" (default 5)
--log.rotationSize string log rotation size(KB MB GB), must be a positive integer. Env "TAOS_KEEPER_LOG_ROTATION_SIZE" (default "1GB")
--log.rotationTime duration deprecated: log rotation time always 24 hours. Env "TAOS_KEEPER_LOG_ROTATION_TIME" (default 24h0m0s)
--logLevel string log level (trace debug info warning error). Env "TAOS_KEEPER_LOG_LEVEL" (default "info")
--metrics.database.name string database for storing metrics data. Env "TAOS_KEEPER_METRICS_DATABASE" (default "log")
--metrics.database.options.buffer int database option buffer for audit database. Env "TAOS_KEEPER_METRICS_BUFFER" (default 64)
--metrics.database.options.cachemodel string database option cachemodel for audit database. Env "TAOS_KEEPER_METRICS_CACHEMODEL" (default "both")
--metrics.database.options.keep int database option buffer for audit database. Env "TAOS_KEEPER_METRICS_KEEP" (default 90)
--metrics.database.options.vgroups int database option vgroups for audit database. Env "TAOS_KEEPER_METRICS_VGROUPS" (default 1)
--metrics.prefix string prefix in metrics names. Env "TAOS_KEEPER_METRICS_PREFIX"
--metrics.tables stringArray export some tables that are not super table, multiple values split with white space. Env "TAOS_KEEPER_METRICS_TABLES"
-P, --port int http port. Env "TAOS_KEEPER_PORT" (default 6043)
--tdengine.host string TDengine server's ip. Env "TAOS_KEEPER_TDENGINE_HOST" (default "127.0.0.1") --tdengine.host string TDengine server's ip. Env "TAOS_KEEPER_TDENGINE_HOST" (default "127.0.0.1")
--tdengine.password string TDengine server's password. Env "TAOS_KEEPER_TDENGINE_PASSWORD" (default "taosdata")
--tdengine.port int TDengine REST server(taosAdapter)'s port. Env "TAOS_KEEPER_TDENGINE_PORT" (default 6041) --tdengine.port int TDengine REST server(taosAdapter)'s port. Env "TAOS_KEEPER_TDENGINE_PORT" (default 6041)
--tdengine.username string TDengine server's username. Env "TAOS_KEEPER_TDENGINE_USERNAME" (default "root") --tdengine.username string TDengine server's username. Env "TAOS_KEEPER_TDENGINE_USERNAME" (default "root")
--tdengine.password string TDengine server's password. Env "TAOS_KEEPER_TDENGINE_PASSWORD" (default "taosdata")
--tdengine.usessl TDengine server use ssl or not. Env "TAOS_KEEPER_TDENGINE_USESSL" --tdengine.usessl TDengine server use ssl or not. Env "TAOS_KEEPER_TDENGINE_USESSL"
--metrics.prefix string prefix in metrics names. Env "TAOS_KEEPER_METRICS_PREFIX" --transfer string run taoskeeper in command mode, only support old_taosd_metric. transfer old metrics data to new tables and exit
--metrics.database.name string database for storing metrics data. Env "TAOS_KEEPER_METRICS_DATABASE" (default "log")
--metrics.tables stringArray export some tables that are not super table, multiple values split with white space. Env "TAOS_KEEPER_METRICS_TABLES"
--environment.incgroup whether running in cgroup. Env "TAOS_KEEPER_ENVIRONMENT_INCGROUP"
--log.path string log path. Env "TAOS_KEEPER_LOG_PATH" (default "/var/log/taos")
--log.rotationCount uint log rotation count. Env "TAOS_KEEPER_LOG_ROTATION_COUNT" (default 5)
--log.rotationTime duration log rotation time. Env "TAOS_KEEPER_LOG_ROTATION_TIME" (default 24h0m0s)
--log.rotationSize string log rotation size(KB MB GB), must be a positive integer. Env "TAOS_KEEPER_LOG_ROTATION_SIZE" (default "100000000")
-c, --config string config path default /etc/taos/taoskeeper.toml
-V, --version Print the version and exit -V, --version Print the version and exit
-h, --help Print this help message and exit
``` ```
### 配置文件 ### 配置文件
taosKeeper 支持用 `taoskeeper -c <keeper config file>` 命令来指定配置文件。 taosKeeper 支持用 `taoskeeper -c <keeper config file>` 命令来指定配置文件。
@ -57,20 +68,18 @@ taosKeeper 支持用 `taoskeeper -c <keeper config file>` 命令来指定配置
若既不指定 taosKeeper 配置文件，且 `/etc/taos/taoskeeper.toml` 也不存在，将使用默认配置。 若既不指定 taosKeeper 配置文件，且 `/etc/taos/taoskeeper.toml` 也不存在，将使用默认配置。
**下面是配置文件的示例：** **下面是配置文件的示例：**
```toml
# Start with debug middleware for gin
debug = false
# Listen port, default is 6043 ```toml
# The ID of the currently running taoskeeper instance, default is 64.
instanceId = 64
# Listening port, default is 6043.
port = 6043 port = 6043
# log level # Go pool size
loglevel = "info"
# go pool size
gopoolsize = 50000 gopoolsize = 50000
# interval for metrics # Interval for metrics
RotationInterval = "15s" RotationInterval = "15s"
[tdengine] [tdengine]
@ -81,20 +90,21 @@ password = "taosdata"
usessl = false usessl = false
[metrics] [metrics]
# metrics prefix in metrics names. # Metrics prefix in metrics names.
prefix = "taos" prefix = "taos"
# export some tables that are not super table # Export some tables that are not super table.
tables = [] tables = []
# database for storing metrics data # Database for storing metrics data.
[metrics.database] [metrics.database]
name = "log" name = "log"
# database options for db storing metrics data
# Database options for db storing metrics data.
[metrics.database.options] [metrics.database.options]
vgroups = 1 vgroups = 1
buffer = 64 buffer = 64
KEEP = 90 keep = 90
cachemodel = "both" cachemodel = "both"
[environment] [environment]
@ -102,9 +112,19 @@ cachemodel = "both"
incgroup = false incgroup = false
[log] [log]
rotationCount = 5 # The directory where log files are stored.
rotationTime = "24h" # path = "/var/log/taos"
rotationSize = 100000000 level = "info"
# Number of log file rotations before deletion.
rotationCount = 30
# The number of days to retain log files.
keepDays = 30
# The maximum size of a log file before rotation.
rotationSize = "1GB"
# If set to true, log files will be compressed.
compress = false
# Minimum disk space to reserve. Log files will not be written if disk space falls below this limit.
reservedDiskSize = "1GB"
``` ```
## 启动 ## 启动
@ -118,7 +138,6 @@ monitorFqdn localhost # taoskeeper 服务的 FQDN
TDengine 监控配置相关，具体请参考：[TDengine 监控配置](../../../operation/monitor)。 TDengine 监控配置相关，具体请参考：[TDengine 监控配置](../../../operation/monitor)。
<Tabs> <Tabs>
<TabItem label="Linux" value="linux"> <TabItem label="Linux" value="linux">
@ -188,7 +207,6 @@ Active: inactive (dead)
</TabItem> </TabItem>
</Tabs> </Tabs>
## 健康检查 ## 健康检查
可以访问 taosKeeper 的 `check_health` 接口来判断服务是否存活，如果服务正常则会返回 HTTP 200 状态码： 可以访问 taosKeeper 的 `check_health` 接口来判断服务是否存活，如果服务正常则会返回 HTTP 200 状态码：
@ -208,7 +226,6 @@ Content-Length: 21
{"version":"3.3.2.3"} {"version":"3.3.2.3"}
``` ```
## 数据收集与监控 ## 数据收集与监控
taosKeeper 作为 TDengine 监控指标的导出工具，可以将 TDengine 产生的监控数据记录在指定数据库中（默认的监控数据是 `log`），这些监控数据可以用来配置 TDengine 监控。 taosKeeper 作为 TDengine 监控指标的导出工具，可以将 TDengine 产生的监控数据记录在指定数据库中（默认的监控数据是 `log`），这些监控数据可以用来配置 TDengine 监控。
@ -216,6 +233,7 @@ taosKeeper 作为 TDengine 监控指标的导出工具，可以将 TDengine 产
### 查看监控数据 ### 查看监控数据
可以查看 `log` 库下的超级表，每个超级表都对应一组监控指标，具体指标不再赘述。 可以查看 `log` 库下的超级表，每个超级表都对应一组监控指标，具体指标不再赘述。
```shell ```shell
taos> use log; taos> use log;
Database changed. Database changed.
@ -251,17 +269,14 @@ taos> select last_row(*) from taosd_dnodes_info;
Query OK, 1 row(s) in set (0.003168s) Query OK, 1 row(s) in set (0.003168s)
``` ```
### 使用 TDInsight 配置监控 ### 使用 TDInsight 配置监控
收集到监控数据以后，就可以使用 TDInsight 来配置 TDengine 的监控，具体请参考 [TDinsight 参考手册](../tdinsight/) 收集到监控数据以后，就可以使用 TDInsight 来配置 TDengine 的监控，具体请参考 [TDinsight 参考手册](../tdinsight/)。
## 集成 Prometheus ## 集成 Prometheus
taoskeeper 提供了 `/metrics` 接口，返回了 Prometheus 格式的监控数据，Prometheus 可以从 taoskeeper 抽取监控数据，实现通过 Prometheus 监控 TDengine 的目的。 taoskeeper 提供了 `/metrics` 接口，返回了 Prometheus 格式的监控数据，Prometheus 可以从 taoskeeper 抽取监控数据，实现通过 Prometheus 监控 TDengine 的目的。
### 导出监控指标 ### 导出监控指标
下面通过 `curl` 命令展示 `/metrics` 接口返回的数据格式： 下面通过 `curl` 命令展示 `/metrics` 接口返回的数据格式：
@ -298,9 +313,11 @@ taos_cluster_info_first_ep_dnode_id{cluster_id="554014120921134497"} 1
#### taosd 集群 #### taosd 集群
##### 监控信息支持的标签 ##### 监控信息支持的标签
- `cluster_id`： 集群 id - `cluster_id`： 集群 id
##### 相关指标及其含义 ##### 相关指标及其含义
| 指标名称 | 类型 | 含义 | | 指标名称 | 类型 | 含义 |
| ----------------------------------- | ------- | ------------------------------------- | | ----------------------------------- | ------- | ------------------------------------- |
| taos_cluster_info_connections_total | counter | 总连接数 | | taos_cluster_info_connections_total | counter | 总连接数 |
@ -328,11 +345,13 @@ taos_cluster_info_first_ep_dnode_id{cluster_id="554014120921134497"} 1
#### dnode #### dnode
##### 监控信息支持的标签 ##### 监控信息支持的标签
- `cluster_id`： 集群 id - `cluster_id`： 集群 id
- `dnode_ep`： dnode 端点 - `dnode_ep`： dnode 端点
- `dnode_id`：dnode id - `dnode_id`：dnode id
##### 相关指标及其含义 ##### 相关指标及其含义
| 指标名称 | 类型 | 含义 | | 指标名称 | 类型 | 含义 |
| ------------------------------ | ------- | ---------------------------------------------------------------------------------------- | | ------------------------------ | ------- | ---------------------------------------------------------------------------------------- |
| taos_d_info_status | gauge | dnode 状态，标签 value 表示状态， ready 表示正常， offline 表示下线， unknown 表示未知。 | | taos_d_info_status | gauge | dnode 状态，标签 value 表示状态， ready 表示正常， offline 表示下线， unknown 表示未知。 |
@ -361,6 +380,7 @@ taos_cluster_info_first_ep_dnode_id{cluster_id="554014120921134497"} 1
#### 数据目录 #### 数据目录
##### 监控信息支持的标签 ##### 监控信息支持的标签
- `cluster_id`： 集群 id - `cluster_id`： 集群 id
- `dnode_ep`： dnode 端点 - `dnode_ep`： dnode 端点
- `dnode_id`：dnode id - `dnode_id`：dnode id
@ -368,6 +388,7 @@ taos_cluster_info_first_ep_dnode_id{cluster_id="554014120921134497"} 1
- `data_dir_level`：数据目录级别 - `data_dir_level`：数据目录级别
##### 相关指标及其含义 ##### 相关指标及其含义
| 指标名称 | 类型 | 含义 | | 指标名称 | 类型 | 含义 |
| --------------------------------- | ----- | -------------------- | | --------------------------------- | ----- | -------------------- |
| taos_taosd_dnodes_data_dirs_avail | gauge | 可用空间（单位 Byte) | | taos_taosd_dnodes_data_dirs_avail | gauge | 可用空间（单位 Byte) |
@ -377,12 +398,14 @@ taos_cluster_info_first_ep_dnode_id{cluster_id="554014120921134497"} 1
#### 日志目录 #### 日志目录
##### 监控信息支持的标签 ##### 监控信息支持的标签
- `cluster_id`： 集群 id - `cluster_id`： 集群 id
- `dnode_ep`： dnode 端点 - `dnode_ep`： dnode 端点
- `dnode_id`：dnode id - `dnode_id`：dnode id
- `log_dir_name`：日志目录名 - `log_dir_name`：日志目录名
##### 相关指标及其含义 ##### 相关指标及其含义
| 指标名称 | 类型 | 含义 | | 指标名称 | 类型 | 含义 |
| -------------------------------- | ----- | -------------------- | | -------------------------------- | ----- | -------------------- |
| taos_taosd_dnodes_log_dirs_avail | gauge | 可用空间（单位 Byte) | | taos_taosd_dnodes_log_dirs_avail | gauge | 可用空间（单位 Byte) |
@ -392,11 +415,13 @@ taos_cluster_info_first_ep_dnode_id{cluster_id="554014120921134497"} 1
#### 日志数量 #### 日志数量
##### 监控信息支持的标签 ##### 监控信息支持的标签
- `cluster_id`： 集群 id - `cluster_id`： 集群 id
- `dnode_ep`： dnode 端点 - `dnode_ep`： dnode 端点
- `dnode_id`：dnode id - `dnode_id`：dnode id
##### 相关指标及其含义 ##### 相关指标及其含义
| 指标名称 | 类型 | 含义 | | 指标名称 | 类型 | 含义 |
| ---------------------- | ------- | ------------ | | ---------------------- | ------- | ------------ |
| taos_log_summary_debug | counter | 调试日志数量 | | taos_log_summary_debug | counter | 调试日志数量 |
@ -404,14 +429,15 @@ taos_cluster_info_first_ep_dnode_id{cluster_id="554014120921134497"} 1
| taos_log_summary_info | counter | 信息日志数量 | | taos_log_summary_info | counter | 信息日志数量 |
| taos_log_summary_trace | counter | 跟踪日志数量 | | taos_log_summary_trace | counter | 跟踪日志数量 |
#### taosadapter #### taosadapter
##### 监控信息支持的标签 ##### 监控信息支持的标签
- `endpoint`：端点 - `endpoint`：端点
- `req_type`：请求类型，0 表示 rest，1 表示 websocket - `req_type`：请求类型，0 表示 rest，1 表示 websocket
##### 相关指标及其含义 ##### 相关指标及其含义
| 指标名称 | 类型 | 含义 | | 指标名称 | 类型 | 含义 |
| -------------------------------------- | ------- | -------------------- | | -------------------------------------- | ------- | -------------------- |
| taos_adapter_requests_fail | counter | 失败的请求数 | | taos_adapter_requests_fail | counter | 失败的请求数 |
@ -433,9 +459,11 @@ taos_cluster_info_first_ep_dnode_id{cluster_id="554014120921134497"} 1
#### taoskeeper #### taoskeeper
##### 监控信息支持的标签 ##### 监控信息支持的标签
- `identify`： 节点 endpoint - `identify`： 节点 endpoint
##### 相关指标及其含义 ##### 相关指标及其含义
| 指标名称 | 类型 | 含义 | | 指标名称 | 类型 | 含义 |
| ----------------------- | ----- | ------------------------------------- | | ----------------------- | ----- | ------------------------------------- |
| taos_keeper_monitor_cpu | gauge | taoskeeper CPU 使用率（取值范围 0~1） | | taos_keeper_monitor_cpu | gauge | taoskeeper CPU 使用率（取值范围 0~1） |
@ -444,6 +472,7 @@ taos_cluster_info_first_ep_dnode_id{cluster_id="554014120921134497"} 1
#### 其他 taosd 集群监控项 #### 其他 taosd 集群监控项
##### taos_m_info_role ##### taos_m_info_role
- **标签**: - **标签**:
- `cluster_id`: 集群 id - `cluster_id`: 集群 id
- `mnode_ep`: mnode 端点 - `mnode_ep`: mnode 端点
@ -453,6 +482,7 @@ taos_cluster_info_first_ep_dnode_id{cluster_id="554014120921134497"} 1
- **含义**: mnode 角色 - **含义**: mnode 角色
##### taos_taos_sql_req_count ##### taos_taos_sql_req_count
- **标签**: - **标签**:
- `cluster_id`: 集群 id - `cluster_id`: 集群 id
- `result`: 请求结果（取值范围： Success, Failed） - `result`: 请求结果（取值范围： Success, Failed）
@ -462,6 +492,7 @@ taos_cluster_info_first_ep_dnode_id{cluster_id="554014120921134497"} 1
- **含义**: SQL 请求数量 - **含义**: SQL 请求数量
##### taos_taosd_sql_req_count ##### taos_taosd_sql_req_count
- **标签**: - **标签**:
- `cluster_id`: 集群 id - `cluster_id`: 集群 id
- `dnode_ep`: dnode 端点 - `dnode_ep`: dnode 端点
@ -474,6 +505,7 @@ taos_cluster_info_first_ep_dnode_id{cluster_id="554014120921134497"} 1
- **含义**: SQL 请求数量 - **含义**: SQL 请求数量
##### taos_taosd_vgroups_info_status ##### taos_taosd_vgroups_info_status
- **标签**: - **标签**:
- `cluster_id`: 集群 id - `cluster_id`: 集群 id
- `database_name`: 数据库名称 - `database_name`: 数据库名称
@ -482,6 +514,7 @@ taos_cluster_info_first_ep_dnode_id{cluster_id="554014120921134497"} 1
- **含义**: 虚拟组状态。 0 为 unsynced，表示没有leader选出；1 为 ready。 - **含义**: 虚拟组状态。 0 为 unsynced，表示没有leader选出；1 为 ready。
##### taos_taosd_vgroups_info_tables_num ##### taos_taosd_vgroups_info_tables_num
- **标签**: - **标签**:
- `cluster_id`: 集群 id - `cluster_id`: 集群 id
- `database_name`: 数据库名称 - `database_name`: 数据库名称
@ -490,6 +523,7 @@ taos_cluster_info_first_ep_dnode_id{cluster_id="554014120921134497"} 1
- **含义**: 虚拟组表数量 - **含义**: 虚拟组表数量
##### taos_taosd_vnodes_info_role ##### taos_taosd_vnodes_info_role
- **标签**: - **标签**:
- `cluster_id`: 集群 id - `cluster_id`: 集群 id
- `database_name`: 数据库名称 - `database_name`: 数据库名称
@ -499,7 +533,6 @@ taos_cluster_info_first_ep_dnode_id{cluster_id="554014120921134497"} 1
- **类型**: gauge - **类型**: gauge
- **含义**: 虚拟节点角色 - **含义**: 虚拟节点角色
### 抽取配置 ### 抽取配置
Prometheus 提供了 `scrape_configs` 配置如何从 endpoint 抽取监控数据，通常只需要修改 `static_configs` 中的 targets 配置为 taoskeeper 的 endpoint 地址，更多配置信息请参考 [Prometheus 配置文档](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config)。 Prometheus 提供了 `scrape_configs` 配置如何从 endpoint 抽取监控数据，通常只需要修改 `static_configs` 中的 targets 配置为 taoskeeper 的 endpoint 地址，更多配置信息请参考 [Prometheus 配置文档](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config)。
@ -521,8 +554,6 @@ scrape_configs:
在 Grafana Dashboard 菜单点击 `import`，dashboard ID 填写 `18587`，点击 `Load` 按钮即可导入 `TaosKeeper Prometheus Dashboard for 3.x` dashboard。 在 Grafana Dashboard 菜单点击 `import`，dashboard ID 填写 `18587`，点击 `Load` 按钮即可导入 `TaosKeeper Prometheus Dashboard for 3.x` dashboard。
## taosKeeper 监控指标 ## taosKeeper 监控指标
taosKeeper 也会将自己采集的监控数据写入监控数据库，默认是 `log` 库，可以在 taoskeeper 配置文件中修改。 taosKeeper 也会将自己采集的监控数据写入监控数据库，默认是 `log` 库，可以在 taoskeeper 配置文件中修改。

3
docs/zh/14-reference/03-taos-sql/14-stream.md
View File

@ -249,8 +249,11 @@ T = 最新事件时间 - DELETE_MARK
- [percentile](../function/#percentile) - [percentile](../function/#percentile)
- [top](../function/#top) - [top](../function/#top)
- [bottom](../function/#bottom) - [bottom](../function/#bottom)
- [elapsed](../function/#elapsed)
- [interp](../function/#interp)
- [derivative](../function/#derivative) - [derivative](../function/#derivative)
- [irate](../function/#irate) - [irate](../function/#irate)
- [twa](../function/#twa)
- [histogram](../function/#histogram) - [histogram](../function/#histogram)
- [diff](../function/#diff) - [diff](../function/#diff)
- [statecount](../function/#statecount) - [statecount](../function/#statecount)

391
include/common/ttokendef.h
View File

@ -16,396 +16,7 @@
#ifndef _TD_COMMON_TOKEN_H_ #ifndef _TD_COMMON_TOKEN_H_
#define _TD_COMMON_TOKEN_H_ #define _TD_COMMON_TOKEN_H_
#define TK_OR 1 #include "ttokenauto.h"
#define TK_AND 2
#define TK_UNION 3
#define TK_ALL 4
#define TK_MINUS 5
#define TK_EXCEPT 6
#define TK_INTERSECT 7
#define TK_NK_BITAND 8
#define TK_NK_BITOR 9
#define TK_NK_LSHIFT 10
#define TK_NK_RSHIFT 11
#define TK_NK_PLUS 12
#define TK_NK_MINUS 13
#define TK_NK_STAR 14
#define TK_NK_SLASH 15
#define TK_NK_REM 16
#define TK_NK_CONCAT 17
#define TK_CREATE 18
#define TK_ACCOUNT 19
#define TK_NK_ID 20
#define TK_PASS 21
#define TK_NK_STRING 22
#define TK_ALTER 23
#define TK_PPS 24
#define TK_TSERIES 25
#define TK_STORAGE 26
#define TK_STREAMS 27
#define TK_QTIME 28
#define TK_DBS 29
#define TK_USERS 30
#define TK_CONNS 31
#define TK_STATE 32
#define TK_NK_COMMA 33
#define TK_HOST 34
#define TK_IS_IMPORT 35
#define TK_NK_INTEGER 36
#define TK_CREATEDB 37
#define TK_USER 38
#define TK_ENABLE 39
#define TK_SYSINFO 40
#define TK_ADD 41
#define TK_DROP 42
#define TK_GRANT 43
#define TK_ON 44
#define TK_TO 45
#define TK_REVOKE 46
#define TK_FROM 47
#define TK_SUBSCRIBE 48
#define TK_READ 49
#define TK_WRITE 50
#define TK_NK_DOT 51
#define TK_WITH 52
#define TK_ENCRYPT_KEY 53
#define TK_ANODE 54
#define TK_UPDATE 55
#define TK_ANODES 56
#define TK_DNODE 57
#define TK_PORT 58
#define TK_DNODES 59
#define TK_RESTORE 60
#define TK_NK_IPTOKEN 61
#define TK_FORCE 62
#define TK_UNSAFE 63
#define TK_CLUSTER 64
#define TK_LOCAL 65
#define TK_QNODE 66
#define TK_BNODE 67
#define TK_SNODE 68
#define TK_MNODE 69
#define TK_VNODE 70
#define TK_DATABASE 71
#define TK_USE 72
#define TK_FLUSH 73
#define TK_TRIM 74
#define TK_S3MIGRATE 75
#define TK_COMPACT 76
#define TK_IF 77
#define TK_NOT 78
#define TK_EXISTS 79
#define TK_BUFFER 80
#define TK_CACHEMODEL 81
#define TK_CACHESIZE 82
#define TK_COMP 83
#define TK_DURATION 84
#define TK_NK_VARIABLE 85
#define TK_MAXROWS 86
#define TK_MINROWS 87
#define TK_KEEP 88
#define TK_PAGES 89
#define TK_PAGESIZE 90
#define TK_TSDB_PAGESIZE 91
#define TK_PRECISION 92
#define TK_REPLICA 93
#define TK_VGROUPS 94
#define TK_SINGLE_STABLE 95
#define TK_RETENTIONS 96
#define TK_SCHEMALESS 97
#define TK_WAL_LEVEL 98
#define TK_WAL_FSYNC_PERIOD 99
#define TK_WAL_RETENTION_PERIOD 100
#define TK_WAL_RETENTION_SIZE 101
#define TK_WAL_ROLL_PERIOD 102
#define TK_WAL_SEGMENT_SIZE 103
#define TK_STT_TRIGGER 104
#define TK_TABLE_PREFIX 105
#define TK_TABLE_SUFFIX 106
#define TK_S3_CHUNKPAGES 107
#define TK_S3_KEEPLOCAL 108
#define TK_S3_COMPACT 109
#define TK_KEEP_TIME_OFFSET 110
#define TK_ENCRYPT_ALGORITHM 111
#define TK_NK_COLON 112
#define TK_BWLIMIT 113
#define TK_START 114
#define TK_TIMESTAMP 115
#define TK_END 116
#define TK_TABLE 117
#define TK_NK_LP 118
#define TK_NK_RP 119
#define TK_USING 120
#define TK_FILE 121
#define TK_STABLE 122
#define TK_COLUMN 123
#define TK_MODIFY 124
#define TK_RENAME 125
#define TK_TAG 126
#define TK_SET 127
#define TK_NK_EQ 128
#define TK_TAGS 129
#define TK_BOOL 130
#define TK_TINYINT 131
#define TK_SMALLINT 132
#define TK_INT 133
#define TK_INTEGER 134
#define TK_BIGINT 135
#define TK_FLOAT 136
#define TK_DOUBLE 137
#define TK_BINARY 138
#define TK_NCHAR 139
#define TK_UNSIGNED 140
#define TK_JSON 141
#define TK_VARCHAR 142
#define TK_MEDIUMBLOB 143
#define TK_BLOB 144
#define TK_VARBINARY 145
#define TK_GEOMETRY 146
#define TK_DECIMAL 147
#define TK_COMMENT 148
#define TK_MAX_DELAY 149
#define TK_WATERMARK 150
#define TK_ROLLUP 151
#define TK_TTL 152
#define TK_SMA 153
#define TK_DELETE_MARK 154
#define TK_FIRST 155
#define TK_LAST 156
#define TK_SHOW 157
#define TK_FULL 158
#define TK_PRIVILEGES 159
#define TK_DATABASES 160
#define TK_TABLES 161
#define TK_STABLES 162
#define TK_MNODES 163
#define TK_QNODES 164
#define TK_ARBGROUPS 165
#define TK_FUNCTIONS 166
#define TK_INDEXES 167
#define TK_ACCOUNTS 168
#define TK_APPS 169
#define TK_CONNECTIONS 170
#define TK_LICENCES 171
#define TK_GRANTS 172
#define TK_LOGS 173
#define TK_MACHINES 174
#define TK_ENCRYPTIONS 175
#define TK_QUERIES 176
#define TK_SCORES 177
#define TK_TOPICS 178
#define TK_VARIABLES 179
#define TK_BNODES 180
#define TK_SNODES 181
#define TK_TRANSACTIONS 182
#define TK_DISTRIBUTED 183
#define TK_CONSUMERS 184
#define TK_SUBSCRIPTIONS 185
#define TK_VNODES 186
#define TK_ALIVE 187
#define TK_VIEWS 188
#define TK_VIEW 189
#define TK_COMPACTS 190
#define TK_NORMAL 191
#define TK_CHILD 192
#define TK_LIKE 193
#define TK_TBNAME 194
#define TK_QTAGS 195
#define TK_AS 196
#define TK_SYSTEM 197
#define TK_TSMA 198
#define TK_INTERVAL 199
#define TK_RECURSIVE 200
#define TK_TSMAS 201
#define TK_FUNCTION 202
#define TK_INDEX 203
#define TK_COUNT 204
#define TK_LAST_ROW 205
#define TK_META 206
#define TK_ONLY 207
#define TK_TOPIC 208
#define TK_CONSUMER 209
#define TK_GROUP 210
#define TK_DESC 211
#define TK_DESCRIBE 212
#define TK_RESET 213
#define TK_QUERY 214
#define TK_CACHE 215
#define TK_EXPLAIN 216
#define TK_ANALYZE 217
#define TK_VERBOSE 218
#define TK_NK_BOOL 219
#define TK_RATIO 220
#define TK_NK_FLOAT 221
#define TK_OUTPUTTYPE 222
#define TK_AGGREGATE 223
#define TK_BUFSIZE 224
#define TK_LANGUAGE 225
#define TK_REPLACE 226
#define TK_STREAM 227
#define TK_INTO 228
#define TK_PAUSE 229
#define TK_RESUME 230
#define TK_PRIMARY 231
#define TK_KEY 232
#define TK_TRIGGER 233
#define TK_AT_ONCE 234
#define TK_WINDOW_CLOSE 235
#define TK_FORCE_WINDOW_CLOSE 236
#define TK_IGNORE 237
#define TK_EXPIRED 238
#define TK_FILL_HISTORY 239
#define TK_SUBTABLE 240
#define TK_UNTREATED 241
#define TK_KILL 242
#define TK_CONNECTION 243
#define TK_TRANSACTION 244
#define TK_BALANCE 245
#define TK_VGROUP 246
#define TK_LEADER 247
#define TK_MERGE 248
#define TK_REDISTRIBUTE 249
#define TK_SPLIT 250
#define TK_DELETE 251
#define TK_INSERT 252
#define TK_NK_BIN 253
#define TK_NK_HEX 254
#define TK_NULL 255
#define TK_NK_QUESTION 256
#define TK_NK_ALIAS 257
#define TK_NK_ARROW 258
#define TK_ROWTS 259
#define TK_QSTART 260
#define TK_QEND 261
#define TK_QDURATION 262
#define TK_WSTART 263
#define TK_WEND 264
#define TK_WDURATION 265
#define TK_IROWTS 266
#define TK_ISFILLED 267
#define TK_FLOW 268
#define TK_FHIGH 269
#define TK_FROWTS 270
#define TK_CAST 271
#define TK_POSITION 272
#define TK_IN 273
#define TK_FOR 274
#define TK_NOW 275
#define TK_TODAY 276
#define TK_RAND 277
#define TK_SUBSTR 278
#define TK_SUBSTRING 279
#define TK_BOTH 280
#define TK_TRAILING 281
#define TK_LEADING 282
#define TK_TIMEZONE 283
#define TK_CLIENT_VERSION 284
#define TK_SERVER_VERSION 285
#define TK_SERVER_STATUS 286
#define TK_CURRENT_USER 287
#define TK_PI 288
#define TK_CASE 289
#define TK_WHEN 290
#define TK_THEN 291
#define TK_ELSE 292
#define TK_BETWEEN 293
#define TK_IS 294
#define TK_NK_LT 295
#define TK_NK_GT 296
#define TK_NK_LE 297
#define TK_NK_GE 298
#define TK_NK_NE 299
#define TK_MATCH 300
#define TK_NMATCH 301
#define TK_CONTAINS 302
#define TK_JOIN 303
#define TK_INNER 304
#define TK_LEFT 305
#define TK_RIGHT 306
#define TK_OUTER 307
#define TK_SEMI 308
#define TK_ANTI 309
#define TK_ASOF 310
#define TK_WINDOW 311
#define TK_WINDOW_OFFSET 312
#define TK_JLIMIT 313
#define TK_SELECT 314
#define TK_NK_HINT 315
#define TK_DISTINCT 316
#define TK_WHERE 317
#define TK_PARTITION 318
#define TK_BY 319
#define TK_SESSION 320
#define TK_STATE_WINDOW 321
#define TK_EVENT_WINDOW 322
#define TK_COUNT_WINDOW 323
#define TK_ANOMALY_WINDOW 324
#define TK_SLIDING 325
#define TK_FILL 326
#define TK_VALUE 327
#define TK_VALUE_F 328
#define TK_NONE 329
#define TK_PREV 330
#define TK_NULL_F 331
#define TK_LINEAR 332
#define TK_NEXT 333
#define TK_HAVING 334
#define TK_RANGE 335
#define TK_EVERY 336
#define TK_ORDER 337
#define TK_SLIMIT 338
#define TK_SOFFSET 339
#define TK_LIMIT 340
#define TK_OFFSET 341
#define TK_ASC 342
#define TK_NULLS 343
#define TK_ABORT 344
#define TK_AFTER 345
#define TK_ATTACH 346
#define TK_BEFORE 347
#define TK_BEGIN 348
#define TK_BITAND 349
#define TK_BITNOT 350
#define TK_BITOR 351
#define TK_BLOCKS 352
#define TK_CHANGE 353
#define TK_COMMA 354
#define TK_CONCAT 355
#define TK_CONFLICT 356
#define TK_COPY 357
#define TK_DEFERRED 358
#define TK_DELIMITERS 359
#define TK_DETACH 360
#define TK_DIVIDE 361
#define TK_DOT 362
#define TK_EACH 363
#define TK_FAIL 364
#define TK_GLOB 365
#define TK_ID 366
#define TK_IMMEDIATE 367
#define TK_IMPORT 368
#define TK_INITIALLY 369
#define TK_INSTEAD 370
#define TK_ISNULL 371
#define TK_MODULES 372
#define TK_NK_BITNOT 373
#define TK_NK_SEMI 374
#define TK_NOTNULL 375
#define TK_OF 376
#define TK_PLUS 377
#define TK_PRIVILEGE 378
#define TK_RAISE 379
#define TK_RESTRICT 380
#define TK_ROW 381
#define TK_STAR 382
#define TK_STATEMENT 383
#define TK_STRICT 384
#define TK_STRING 385
#define TK_TIMES 386
#define TK_VALUES 387
#define TK_VARIABLE 388
#define TK_WAL 389
#define TK_NK_SPACE 600 #define TK_NK_SPACE 600
#define TK_NK_COMMENT 601 #define TK_NK_COMMENT 601

2
source/common/CMakeLists.txt
View File

@ -6,6 +6,8 @@ endif()
add_library(common STATIC ${COMMON_SRC}) add_library(common STATIC ${COMMON_SRC})
add_dependencies(common lemon_sql)
if(DEFINED GRANT_CFG_INCLUDE_DIR) if(DEFINED GRANT_CFG_INCLUDE_DIR)
add_definitions(-DGRANTS_CFG) add_definitions(-DGRANTS_CFG)
endif() endif()

19
source/libs/executor/src/scanoperator.c
View File

@ -2096,6 +2096,7 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr
if (pSrcBlock->info.rows == 0) { if (pSrcBlock->info.rows == 0) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SSHashObj* pScanRange = tSimpleHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
SExecTaskInfo* pTaskInfo = pInfo->pStreamScanOp->pTaskInfo; SExecTaskInfo* pTaskInfo = pInfo->pStreamScanOp->pTaskInfo;
SColumnInfoData* pStartTsCol = taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pStartTsCol = taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX);
TSKEY* startData = (TSKEY*)pStartTsCol->pData; TSKEY* startData = (TSKEY*)pStartTsCol->pData;
@ -2157,6 +2158,14 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr
qError("generate session scan range failed. rang start:%" PRIx64 ", end:%" PRIx64, startData[i], endData[i]); qError("generate session scan range failed. rang start:%" PRIx64 ", end:%" PRIx64, startData[i], endData[i]);
continue; continue;
} }
SSessionKey checkKey = {.groupId = groupId, .win.skey = startWin.win.skey, .win.ekey = endWin.win.ekey};
if (tSimpleHashGet(pScanRange, &checkKey, sizeof(SSessionKey)) != NULL) {
continue;
}
code = tSimpleHashPut(pScanRange, &checkKey, sizeof(SSessionKey), NULL, 0);
QUERY_CHECK_CODE(code, lino, _end);
code = colDataSetVal(pDestStartCol, i, (const char*)&startWin.win.skey, false); code = colDataSetVal(pDestStartCol, i, (const char*)&startWin.win.skey, false);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
@ -2173,6 +2182,7 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr
} }
_end: _end:
tSimpleHashCleanup(pScanRange);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
} }
@ -2431,6 +2441,7 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS
if (pSrcBlock->info.rows == 0) { if (pSrcBlock->info.rows == 0) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SSHashObj* pScanRange = tSimpleHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
SExecTaskInfo* pTaskInfo = pInfo->pStreamScanOp->pTaskInfo; SExecTaskInfo* pTaskInfo = pInfo->pStreamScanOp->pTaskInfo;
SColumnInfoData* pSrcStartTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pSrcStartTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX);
SColumnInfoData* pSrcEndTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pSrcEndTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX);
@ -2491,6 +2502,13 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS
code = colDataSetVal(pDeUidCol, pDestBlock->info.rows, (const char*)(&srcUid), false); code = colDataSetVal(pDeUidCol, pDestBlock->info.rows, (const char*)(&srcUid), false);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
SSessionKey checkKey = {.groupId = groupId, .win = win};
if (tSimpleHashGet(pScanRange, &checkKey, sizeof(SSessionKey)) != NULL) {
continue;
}
code = tSimpleHashPut(pScanRange, &checkKey, sizeof(SSessionKey), NULL, 0);
QUERY_CHECK_CODE(code, lino, _end);
code = colDataSetVal(pStartTsCol, pDestBlock->info.rows, (const char*)(&win.skey), false); code = colDataSetVal(pStartTsCol, pDestBlock->info.rows, (const char*)(&win.skey), false);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
@ -2506,6 +2524,7 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS
} }
_end: _end:
tSimpleHashCleanup(pScanRange);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
} }

20
source/libs/parser/CMakeLists.txt
View File

@ -4,7 +4,27 @@ IF(TD_ENTERPRISE)
LIST(APPEND PARSER_SRC ${TD_ENTERPRISE_DIR}/src/plugins/view/src/parserView.c) LIST(APPEND PARSER_SRC ${TD_ENTERPRISE_DIR}/src/plugins/view/src/parserView.c)
ENDIF() ENDIF()
add_custom_command(
OUTPUT ${TD_SOURCE_DIR}/source/libs/parser/src/sql.c ${TD_SOURCE_DIR}/include/common/ttokenauto.h
COMMAND echo "Running lemon process in ${TD_SOURCE_DIR}/source/libs/parser/inc"
COMMAND ${TD_CONTRIB_DIR}/lemon/lemon sql.y || true
COMMAND echo "copy sql.c from ${TD_SOURCE_DIR}/source/libs/parser/inc/sql.c to ${TD_SOURCE_DIR}/source/libs/parser/src/"
COMMAND mv ${TD_SOURCE_DIR}/source/libs/parser/inc/sql.c ${TD_SOURCE_DIR}/source/libs/parser/src/sql.c
COMMAND mv ${TD_SOURCE_DIR}/source/libs/parser/inc/sql.h ${TD_SOURCE_DIR}/include/common/ttokenauto.h
COMMAND echo "lemon process completed."
DEPENDS ${TD_SOURCE_DIR}/source/libs/parser/inc/sql.y
WORKING_DIRECTORY ${TD_SOURCE_DIR}/source/libs/parser/inc
COMMENT "Generating sql.c using lemon"
)
add_custom_target(lemon_sql ALL
DEPENDS ${TD_SOURCE_DIR}/source/libs/parser/src/sql.c ${TD_SOURCE_DIR}/include/common/ttokenauto.h
)
list(APPEND PARSER_SRC ${TD_SOURCE_DIR}/source/libs/parser/src/sql.c)
add_library(parser STATIC ${PARSER_SRC}) add_library(parser STATIC ${PARSER_SRC})
add_dependencies(parser lemon_sql)
target_include_directories( target_include_directories(
parser parser
PUBLIC "${TD_SOURCE_DIR}/include/libs/parser" PUBLIC "${TD_SOURCE_DIR}/include/libs/parser"

8545
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff