xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into feat/TS-4994-3.0

This commit is contained in:
Hongze Cheng 2024-11-11 10:31:01 +08:00
parent def6f69819 539478f1ec
commit 7d9c7918cb
90 changed files with 1732 additions and 650 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docs/examples/JDBC/SpringJdbcTemplate/pom.xml
View File

@ -22,19 +22,19 @@
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
<version>5.2.8.RELEASE</version>
<version>5.3.39</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jdbc</artifactId>
<version>5.1.9.RELEASE</version>
<version>5.3.39</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
<version>5.1.9.RELEASE</version>
<version>5.3.39</version>
</dependency>
<dependency>
@ -47,7 +47,7 @@
<dependency>
<groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId>
<version>3.0.0</version>
<version>3.4.0</version>
</dependency>
</dependencies>

4
docs/examples/JDBC/springbootdemo/pom.xml
View File

@ -5,7 +5,7 @@
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.2.1.RELEASE</version>
<version>2.6.15</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.taosdata.example</groupId>
@ -65,6 +65,8 @@
<artifactId>spring-boot-starter-aop</artifactId>
</dependency>
<dependency>
<groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId>

3
docs/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java
View File

@ -3,9 +3,10 @@ package com.taosdata.example.springbootdemo;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.data.jdbc.JdbcRepositoriesAutoConfiguration;
@MapperScan(basePackages = {"com.taosdata.example.springbootdemo"})
@SpringBootApplication
@SpringBootApplication(exclude = {JdbcRepositoriesAutoConfiguration.class})
public class SpringbootdemoApplication {
public static void main(String[] args) {

2
docs/examples/JDBC/springbootdemo/src/main/resources/application.properties
View File

@ -15,6 +15,8 @@ spring.datasource.druid.max-wait=30000
spring.datasource.druid.validation-query=select SERVER_VERSION();
spring.aop.auto=true
spring.aop.proxy-target-class=true
spring.jooq.sql-dialect=
#mybatis
mybatis.mapper-locations=classpath:mapper/*.xml
logging.level.com.taosdata.jdbc.springbootdemo.dao=debug

3
docs/examples/JDBC/taosdemo/pom.xml
View File

@ -10,7 +10,7 @@
<description>Demo project for TDengine</description>
<properties>
<spring.version>5.3.27</spring.version>
<spring.version>5.3.39</spring.version>
</properties>
<dependencies>
@ -130,6 +130,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.13.0</version>
<configuration>
<source>8</source>
<target>8</target>

2
docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/QueryService.java
View File

@ -37,7 +37,7 @@ public class QueryService {
stmt.execute("use " + dbName);
ResultSet rs = stmt.executeQuery("show stables");
while (rs.next()) {
String name = rs.getString("name");
String name = rs.getString("stable_name");
sqls.add("select count(*) from " + dbName + "." + name);
sqls.add("select first(*) from " + dbName + "." + name);
sqls.add("select last(*) from " + dbName + "." + name);

18
docs/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/DatabaseServiceTest.java
View File

@ -1,10 +1,14 @@
package com.taosdata.taosdemo.service;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import org.junit.BeforeClass;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
public class DatabaseServiceTest {
private DatabaseService service;
private static DatabaseService service;
@Test
public void testCreateDatabase1() {
@ -20,4 +24,16 @@ public class DatabaseServiceTest {
public void useDatabase() {
service.useDatabase("test");
}
@BeforeClass
public static void beforeClass() throws ClassNotFoundException {
Class.forName("com.taosdata.jdbc.TSDBDriver");
HikariConfig config = new HikariConfig();
config.setJdbcUrl("jdbc:TAOS://127.0.0.1:6030/?charset=UTF-8&locale=en_US.UTF-8&timezone=UTC-8");
config.setUsername("root");
config.setPassword("taosdata");
HikariDataSource dataSource = new HikariDataSource(config);
service = new DatabaseService(dataSource);
}
}

6
docs/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/QueryServiceTest.java
View File

@ -15,7 +15,7 @@ public class QueryServiceTest {
@Test
public void generateSuperTableQueries() {
String[] sqls = queryService.generateSuperTableQueries("restful_test");
String[] sqls = queryService.generateSuperTableQueries("test");
for (String sql : sqls) {
System.out.println(sql);
}
@ -23,8 +23,8 @@ public class QueryServiceTest {
@Test
public void querySuperTable() {
String[] sqls = queryService.generateSuperTableQueries("restful_test");
queryService.querySuperTable(sqls, 1000, 10, 10);
String[] sqls = queryService.generateSuperTableQueries("test");
queryService.querySuperTable(sqls, 100, 3, 3);
}
@BeforeClass

16
docs/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/SuperTableServiceTest.java
View File

@ -3,6 +3,9 @@ package com.taosdata.taosdemo.service;
import com.taosdata.taosdemo.domain.FieldMeta;
import com.taosdata.taosdemo.domain.SuperTableMeta;
import com.taosdata.taosdemo.domain.TagMeta;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import org.junit.BeforeClass;
import org.junit.Test;
import java.util.ArrayList;
@ -10,7 +13,7 @@ import java.util.List;
public class SuperTableServiceTest {
private SuperTableService service;
private static SuperTableService service;
@Test
public void testCreate() {
@ -29,4 +32,15 @@ public class SuperTableServiceTest {
service.create(superTableMeta);
}
@BeforeClass
public static void beforeClass() throws ClassNotFoundException {
Class.forName("com.taosdata.jdbc.TSDBDriver");
HikariConfig config = new HikariConfig();
config.setJdbcUrl("jdbc:TAOS://127.0.0.1:6030/?charset=UTF-8&locale=en_US.UTF-8&timezone=UTC-8");
config.setUsername("root");
config.setPassword("taosdata");
HikariDataSource dataSource = new HikariDataSource(config);
service = new SuperTableService(dataSource);
}
}

6
docs/examples/java/src/main/java/com/taos/example/DruidDemo.java
View File

@ -1,4 +1,4 @@
package com.taosdata.example;
package com.taos.example;
import com.alibaba.druid.pool.DruidDataSource;
@ -8,11 +8,11 @@ import java.sql.Statement;
public class DruidDemo {
// ANCHOR: connection_pool
public static void main(String[] args) throws Exception {
String url = "jdbc:TAOS://127.0.0.1:6030/log";
String url = "jdbc:TAOS-WS://127.0.0.1:6041/log";
DruidDataSource dataSource = new DruidDataSource();
// jdbc properties
dataSource.setDriverClassName("com.taosdata.jdbc.TSDBDriver");
dataSource.setDriverClassName("com.taosdata.jdbc.ws.WebSocketDriver");
dataSource.setUrl(url);
dataSource.setUsername("root");
dataSource.setPassword("taosdata");

5
docs/examples/java/src/main/java/com/taos/example/GeometryDemo.java
View File

@ -144,8 +144,9 @@ public class GeometryDemo {
private void executeQuery(String sql) {
long start = System.currentTimeMillis();
try (Statement statement = connection.createStatement()) {
ResultSet resultSet = statement.executeQuery(sql);
try (Statement statement = connection.createStatement();
ResultSet resultSet = statement.executeQuery(sql)) {
long end = System.currentTimeMillis();
printSql(sql, true, (end - start));

4
docs/examples/java/src/main/java/com/taos/example/HikariDemo.java
View File

@ -1,4 +1,4 @@
package com.taosdata.example;
package com.taos.example;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
@ -11,7 +11,7 @@ public class HikariDemo {
public static void main(String[] args) throws Exception {
HikariConfig config = new HikariConfig();
// jdbc properties
config.setJdbcUrl("jdbc:TAOS://127.0.0.1:6030/log");
config.setJdbcUrl("jdbc:TAOS-WS://127.0.0.1:6041/log");
config.setUsername("root");
config.setPassword("taosdata");
// connection pool configurations

1
docs/examples/java/src/main/java/com/taos/example/TelnetLineProtocolExample.java
View File

@ -39,6 +39,7 @@ public class TelnetLineProtocolExample {
createDatabase(conn);
SchemalessWriter writer = new SchemalessWriter(conn);
writer.write(lines, SchemalessProtocolType.TELNET, SchemalessTimestampType.NOT_CONFIGURED);
writer.close();
}
}

4
docs/zh/06-advanced/06-data-analysis/03-anomaly-detection.md
View File

@ -1,6 +1,6 @@
---
title: "Anomaly-detection"
sidebar_label: "Anomaly-detection"
title: "异常检测算法"
sidebar_label: "异常检测算法"
---
本节讲述异常检测算法模型的使用方法。

20
docs/zh/06-advanced/06-data-analysis/addins.md
View File

@ -1,13 +1,27 @@
---
title: "addins"
sidebar_label: "addins"
title: "算法开发者指南"
sidebar_label: "算法开发者指南"
---
本节说明如何将自己开发的预测算法和异常检测算法整合到 TDengine 分析平台，并能够通过 SQL 语句进行调用。
## 目录结构
![数据分析功能架构图](./pic/dir.png)
```bash
.
├── cfg
├── model
│   └── ac_detection
├── release
├── script
└── taosanalytics
├── algo
│   ├── ad
│   └── fc
├── misc
└── test
```
|目录|说明|
|---|---|

23
docs/zh/06-advanced/06-data-analysis/analysis-preprocess.md Normal file
View File

@ -0,0 +1,23 @@
---
title: "数据分析预处理"
sidebar_label: "数据分析预处理"
---
## 时序数据分析功能
### 白噪声检查
分析平台提供的 Restful 服务要求输入的时间序列不能是白噪声时间序列（White Noise Data, WND）和随机数序列 , 因此针对所有数据均默认进行白噪声检查。当前白噪声检查采用通行的 `Ljung-Box` 检验，`Ljung-Box` 统计量检查过程需要遍历整个输入序列并进行计算。
如果用户能够明确输入序列一定不是白噪声序列，那么可以通过输入参数，指定预测之前忽略该检查，从而节省分析过程的 CPU 计算资源。
同时支持独立地针对输入序列进行白噪声检测（该检测功能暂不独立对外开放）。
### 数据重采样和时间戳对齐
分析平台支持将输入数据进行重采样预处理，从而确保输出结果按照用户指定的等间隔进行处理。处理过程分为两种类别：
- 数据时间戳对齐。由于真实数据可能并非严格按照查询指定的时间戳输入。此时分析平台会自动将数据的时间间隔按照指定的时间间隔进行对齐。例如输入时间序列 [11, 22, 29, 41]，用户指定时间间隔为 10，该序列将被对齐重整为以下序列 [10, 20, 30, 40]。
- 数据时间重采样。用户输入时间序列的采样频率超过了输出结果的频率，例如输入时间序列的采样频率是 5，输出结果的频率是 10，输入时间序列 [0， 5， 10， 15， 20， 25， 30] 将被重采用为间隔 为 10 的序列 [0, 10, 20，30]，[5, 15, 25] 处的数据将被丢弃。
需要注意的是，数据输入平台不支持缺失数据补齐后进行的预测分析，如果输入时间序列数据 [11, 22, 29, 49]，并且用户要求的时间间隔为 10，重整对齐后的序列是 [10, 20, 30, 50] 那么该序列进行预测分析将返回错误。

183
docs/zh/06-advanced/06-data-analysis/index.md
View File

@ -1,190 +1,21 @@
---
sidebar_label: 数据分析
title: 数据分析功能
sidebar_label: TDgpt
title: TDgpt
---
## 概述
ANode（Analysis Node）是 TDengine 提供数据分析功能的扩展组件，通过 Restful 接口提供分析服务，拓展 TDengine 的功能，支持时间序列高级分析。
ANode 是无状态的数据分析节点，集群中可以存在多个 ANode 节点，相互之间没有关联。将 ANode 注册到 TDengine 集群以后，通过 SQL 语句即可调用并完成时序分析任务。
下图是数据分析的技术架构示意图。
TDgpt 是 TDengine Enterprise 中针对时序数据提供高级分析功能的企业级组件，能够独立于 TDengine 主进程部署和运行，不消耗和占用 TDengine 主进程的资源，通过内置接口向 TDengine 提供运行时动态扩展的高级时序数据分析功能。TDgpt 具有服务无状态、功能易扩展、快速弹性部署、应用轻量化、高安全性等特点。
TDgpt 运行在部署于 TDengine 集群中的 Analysis Node (ANode)中。每个 TDengine 集群中可以部署一个或若干个 ANode 节点，不同的 ANode 节点之间不相关，无同步或协同的要求。ANode 注册到 TDengine 集群以后，就可以通过内部接口提供服务。TDgpt 提供的高级时序数据分析服务可分为时序数据异常检测和时序数据预测分析两个类别。
![数据分析功能架构图](./pic/data-analysis.png)
如下是数据分析的技术架构示意图。
## 安装部署
### 环境准备
ANode 要求节点上准备有 Python 3.10 及以上版本，以及相应的 Python 包自动安装组件 Pip，同时请确保能够正常连接互联网。
<img src="./pic/data-analysis.png" width="560" alt="TDgpt架构图" />
### 安装及卸载
使用专门的 ANode 安装包 TDengine-enterprise-anode-1.x.x.tar.gz 进行 ANode 的安装部署工作，安装过程与 TDengine 的安装流程一致。
```bash
tar -xzvf TDengine-enterprise-anode-1.0.0.tar.gz
cd TDengine-enterprise-anode-1.0.0
sudo ./install.sh
```
卸载 ANode，执行命令 `rmtaosanode` 即可。
### 其他
为了避免 ANode 安装后影响目标节点现有的 Python 库。 ANode 使用 Python 虚拟环境运行，安装后的默认 Python 目录处于 `/var/lib/taos/taosanode/venv/`。为了避免反复安装虚拟环境带来的开销，卸载 ANode 并不会自动删除该虚拟环境，如果您确认不需要 Python 的虚拟环境，可以手动删除。
## 启动及停止服务
安装 ANode 以后，可以使用 `systemctl` 来管理 ANode 的服务。使用如下命令可以启动/停止/检查状态。
```bash
systemctl start taosanoded
systemctl stop taosanoded
systemctl status taosanoded
```
## 目录及配置说明
|目录/文件|说明|
|---------------|------|
|/usr/local/taos/taosanode/bin|可执行文件目录|
|/usr/local/taos/taosanode/resource|资源文件目录，链接到文件夹 /var/lib/taos/taosanode/resource/|
|/usr/local/taos/taosanode/lib|库文件目录|
|/var/lib/taos/taosanode/model/|模型文件目录，链接到文件夹 /var/lib/taos/taosanode/model|
|/var/log/taos/taosanode/|日志文件目录|
|/etc/taos/taosanode.ini|配置文件|
### 配置说明
Anode 提供的 RestFul 服务使用 uWSGI 驱动，因此 ANode 和 uWSGI 的配置信息存放在同一个配置文件中，具体如下：
```ini
[uwsgi]
# charset
env = LC_ALL = en_US.UTF-8
# ip:port
http = 127.0.0.1:6050
# the local unix socket file than communicate to Nginx
#socket = 127.0.0.1:8001
#socket-timeout = 10
# base directory
chdir = /usr/local/taos/taosanode/lib
# initialize python file
wsgi-file = /usr/local/taos/taosanode/lib/taos/app.py
# call module of uWSGI
callable = app
# auto remove unix Socket and pid file when stopping
vacuum = true
# socket exec model
#chmod-socket = 664
# uWSGI pid
uid = root
# uWSGI gid
gid = root
# main process
master = true
# the number of worker processes
processes = 2
# pid file
pidfile = /usr/local/taos/taosanode/taosanode.pid
# enable threads
enable-threads = true
# the number of threads for each process
threads = 4
# memory useage report
memory-report = true
# smooth restart
reload-mercy = 10
# conflict with systemctl, so do NOT uncomment this
# daemonize = /var/log/taos/taosanode/taosanode.log
# log directory
logto = /var/log/taos/taosanode/taosanode.log
# wWSGI monitor port
stats = 127.0.0.1:8387
# python virtual environment directory
virtualenv = /usr/local/taos/taosanode/venv/
[taosanode]
# default app log file
app-log = /var/log/taos/taosanode/taosanode.app.log
# model storage directory
model-dir = /usr/local/taos/taosanode/model/
# default log level
log-level = DEBUG
# draw the query results
draw-result = 0
```
**提示**
请勿设置 `daemonize` 参数，该参数会导致 uWSGI 与 systemctl 冲突，从而无法正常启动。
通过注册指令语句，将 ANode 注册到 MNode 中就加入到 TDengine 集群，查询会按需向其请求数据分析服务。请求服务通过 VNode 直接向 ANode 发起，用户则可以通过 SQL 语句直接调用 ANode 提供的服务。
## ANode 基本操作
### 管理 ANode
#### 创建 ANode
```sql
CREATE ANODE {node_url}
```
node_url 是提供服务的 ANode 的 IP 和 PORT, 例如：`create anode 'http://localhost:6050'`。启动 ANode 以后如果不注册到 TDengine 集群中，则无法提供正常的服务。不建议 ANode 注册到两个或多个集群中。
#### 查看 ANode
列出集群中所有的数据分析节点，包括其 `FQDN`, `PORT`, `STATUS`。
```sql
SHOW ANODES;
```
#### 查看提供的时序数据分析服务
```SQL
SHOW ANODES FULL;
```
#### 强制刷新集群中的分析算法缓存
```SQL
UPDATE ANODE {node_id}
UPDATE ALL ANODES
```
#### 删除 ANode
```sql
DROP ANODE {anode_id}
```
删除 ANode 只是将 ANode 从 TDengine 集群中删除，管理 ANode 的启停仍然需要使用`systemctl`命令。
### 时序数据分析功能
#### 白噪声检查
分析平台提供的 Restful 服务要求输入的时间序列不能是白噪声时间序列（White Noise Data, WND）和随机数序列 , 因此针对所有数据均默认进行白噪声检查。当前白噪声检查采用通行的 `Ljung-Box` 检验，`Ljung-Box` 统计量检查过程需要遍历整个输入序列并进行计算。
如果用户能够明确输入序列一定不是白噪声序列，那么可以通过输入参数，指定预测之前忽略该检查，从而节省分析过程的 CPU 计算资源。
同时支持独立地针对输入序列进行白噪声检测（该检测功能暂不独立对外开放）。
#### 数据重采样和时间戳对齐
分析平台支持将输入数据进行重采样预处理，从而确保输出结果按照用户指定的等间隔进行处理。处理过程分为两种类别：
- 数据时间戳对齐。由于真实数据可能并非严格按照查询指定的时间戳输入。此时分析平台会自动将数据的时间间隔按照指定的时间间隔进行对齐。例如输入时间序列 [11, 22, 29, 41]，用户指定时间间隔为 10，该序列将被对齐重整为以下序列 [10, 20, 30, 40]。
- 数据时间重采样。用户输入时间序列的采样频率超过了输出结果的频率，例如输入时间序列的采样频率是 5，输出结果的频率是 10，输入时间序列 [0， 5， 10， 15， 20， 25， 30] 将被重采用为间隔 为 10 的序列 [0, 10, 20，30]，[5, 15, 25] 处的数据将被丢弃。
需要注意的是，数据输入平台不支持缺失数据补齐后进行的预测分析，如果输入时间序列数据 [11, 22, 29, 49]，并且用户要求的时间间隔为 10，重整对齐后的序列是 [10, 20, 30, 50] 那么该序列进行预测分析将返回错误。
#### 时序数据异常检测

161
docs/zh/06-advanced/06-data-analysis/management.md Normal file
View File

@ -0,0 +1,161 @@
---
title: "安装部署"
sidebar_label: "安装部署"
---
## 安装部署
### 环境准备
ANode 要求节点上准备有 Python 3.10 及以上版本，以及相应的 Python 包自动安装组件 Pip，同时请确保能够正常连接互联网。
### 安装及卸载
使用专门的 ANode 安装包 TDengine-enterprise-anode-1.x.x.tar.gz 进行 ANode 的安装部署工作，安装过程与 TDengine 的安装流程一致。
```bash
tar -xzvf TDengine-enterprise-anode-1.0.0.tar.gz
cd TDengine-enterprise-anode-1.0.0
sudo ./install.sh
```
卸载 ANode，执行命令 `rmtaosanode` 即可。
### 其他
为了避免 ANode 安装后影响目标节点现有的 Python 库。 ANode 使用 Python 虚拟环境运行，安装后的默认 Python 目录处于 `/var/lib/taos/taosanode/venv/`。为了避免反复安装虚拟环境带来的开销，卸载 ANode 并不会自动删除该虚拟环境，如果您确认不需要 Python 的虚拟环境，可以手动删除。
## 启动及停止服务
安装 ANode 以后，可以使用 `systemctl` 来管理 ANode 的服务。使用如下命令可以启动/停止/检查状态。
```bash
systemctl start taosanoded
systemctl stop taosanoded
systemctl status taosanoded
```
## 目录及配置说明
|目录/文件|说明|
|---------------|------|
|/usr/local/taos/taosanode/bin|可执行文件目录|
|/usr/local/taos/taosanode/resource|资源文件目录，链接到文件夹 /var/lib/taos/taosanode/resource/|
|/usr/local/taos/taosanode/lib|库文件目录|
|/var/lib/taos/taosanode/model/|模型文件目录，链接到文件夹 /var/lib/taos/taosanode/model|
|/var/log/taos/taosanode/|日志文件目录|
|/etc/taos/taosanode.ini|配置文件|
### 配置说明
Anode 提供的 RestFul 服务使用 uWSGI 驱动，因此 ANode 和 uWSGI 的配置信息存放在同一个配置文件中，具体如下：
```ini
[uwsgi]
# charset
env = LC_ALL = en_US.UTF-8
# ip:port
http = 127.0.0.1:6050
# the local unix socket file than communicate to Nginx
#socket = 127.0.0.1:8001
#socket-timeout = 10
# base directory
chdir = /usr/local/taos/taosanode/lib
# initialize python file
wsgi-file = /usr/local/taos/taosanode/lib/taos/app.py
# call module of uWSGI
callable = app
# auto remove unix Socket and pid file when stopping
vacuum = true
# socket exec model
#chmod-socket = 664
# uWSGI pid
uid = root
# uWSGI gid
gid = root
# main process
master = true
# the number of worker processes
processes = 2
# pid file
pidfile = /usr/local/taos/taosanode/taosanode.pid
# enable threads
enable-threads = true
# the number of threads for each process
threads = 4
# memory useage report
memory-report = true
# smooth restart
reload-mercy = 10
# conflict with systemctl, so do NOT uncomment this
# daemonize = /var/log/taos/taosanode/taosanode.log
# log directory
logto = /var/log/taos/taosanode/taosanode.log
# wWSGI monitor port
stats = 127.0.0.1:8387
# python virtual environment directory
virtualenv = /usr/local/taos/taosanode/venv/
[taosanode]
# default app log file
app-log = /var/log/taos/taosanode/taosanode.app.log
# model storage directory
model-dir = /usr/local/taos/taosanode/model/
# default log level
log-level = DEBUG
# draw the query results
draw-result = 0
```
**提示**
请勿设置 `daemonize` 参数，该参数会导致 uWSGI 与 systemctl 冲突，从而无法正常启动。
## ANode 基本操作
#### 创建 ANode
```sql
CREATE ANODE {node_url}
```
node_url 是提供服务的 ANode 的 IP 和 PORT, 例如：`create anode 'http://localhost:6050'`。启动 ANode 以后如果不注册到 TDengine 集群中，则无法提供正常的服务。不建议 ANode 注册到两个或多个集群中。
#### 查看 ANode
列出集群中所有的数据分析节点，包括其 `FQDN`, `PORT`, `STATUS`。
```sql
SHOW ANODES;
```
#### 查看提供的时序数据分析服务
```SQL
SHOW ANODES FULL;
```
#### 强制刷新集群中的分析算法缓存
```SQL
UPDATE ANODE {node_id}
UPDATE ALL ANODES
```
#### 删除 ANode
```sql
DROP ANODE {anode_id}
```
删除 ANode 只是将 ANode 从 TDengine 集群中删除，管理 ANode 的启停仍然需要使用`systemctl`命令。

BIN
docs/zh/06-advanced/06-data-analysis/pic/data-analysis.png Normal file → Executable file
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 49 KiB

After

Width:  |  Height:  |  Size: 58 KiB

BIN
docs/zh/06-advanced/06-data-analysis/pic/dir.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 7.1 KiB

16
docs/zh/08-operation/16-security.md
View File

@ -26,6 +26,22 @@ SHOW USERS;
```sql
ALTER USER TEST DROP HOST HOST_NAME1
```
说明
- 开源版和企业版本都能添加成功，且可以查询到，但是开源版本不会对 IP 做任何限制。
- create user u_write pass 'taosdata1' host 'iprange1','iprange2', 可以一次添加多个 iprange, 服务端会做去重，去重的逻辑是需要 iprange 完全一样
- 默认会把 127.0.0.1 添加到白名单列表，且在白名单列表可以查询
- 集群的节点 IP 集合会自动添加到白名单列表，但是查询不到。
- taosadaper 和 taosd 不在一个机器的时候，需要把 taosadaper IP 手动添加到 taosd 白名单列表中
- 集群情况下，各个节点 enableWhiteList 成一样，或者全为 false,或者全为 true, 要不然集群无法启动
- 白名单变更生效时间 1s，不超过 2s, 每次变更对收发性能有些微影响（多一次判断，可以忽略），变更完之后、影响忽略不计, 变更过程中对集群没有影响，对正在访问客户端也没有影响（假设这些客户端的 IP 包含在 white list 内）
- 如果添加两个 ip range, 192.168.1.1/16(假设为 A), 192.168.1.1/24(假设为 B), 严格来说，A 包含了 B，但是考虑情况太复杂，并不会对 A 和 B 做合并
- 要删除的时候，必须严格匹配。 也就是如果添加的是 192.168.1.1/24, 要删除也是 192.168.1.1/24
- 只有 root 才有权限对其他用户增删 ip white list
- 兼容之前的版本，但是不支持从当前版本回退到之前版本
- x.x.x.x/32 和 x.x.x.x 属于同一个 iprange, 显示为 x.x.x.x
- 如果客户端拿到的 0.0.0.0/0, 说明没有开启白名单
- 如果白名单发生了改变， 客户端会在 heartbeat 里检测到。
- 针对一个 user, 添加的 IP 个数上限是 2048
## 审计日志

3
include/common/tcommon.h
View File

@ -155,6 +155,7 @@ typedef enum EStreamType {
STREAM_MID_RETRIEVE,
STREAM_PARTITION_DELETE_DATA,
STREAM_GET_RESULT,
STREAM_DROP_CHILD_TABLE,
} EStreamType;
#pragma pack(push, 1)
@ -401,6 +402,8 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol);
#define TSMA_RES_STB_EXTRA_COLUMN_NUM 4 // 3 columns: _wstart, _wend, _wduration, 1 tag: tbname
static inline bool isTsmaResSTb(const char* stbName) {
static bool showTsmaTables = false;
if (showTsmaTables) return false;
const char* pos = strstr(stbName, TSMA_RES_STB_POSTFIX);
if (pos && strlen(stbName) == (pos - stbName) + strlen(TSMA_RES_STB_POSTFIX)) {
return true;

1
include/common/tglobal.h
View File

@ -188,7 +188,6 @@ extern int32_t tsMaxRetryWaitTime;
extern bool tsUseAdapter;
extern int32_t tsMetaCacheMaxSize;
extern int32_t tsSlowLogThreshold;
extern int32_t tsSlowLogThresholdTest;
extern char tsSlowLogExceptDb[];
extern int32_t tsSlowLogScope;
extern int32_t tsSlowLogMaxLen;

12
include/common/tmsg.h
View File

@ -677,7 +677,7 @@ typedef struct {
int32_t tsSlowLogThreshold;
int32_t tsSlowLogMaxLen;
int32_t tsSlowLogScope;
int32_t tsSlowLogThresholdTest;
int32_t tsSlowLogThresholdTest; //Obsolete
char tsSlowLogExceptDb[TSDB_DB_NAME_LEN];
} SMonitorParas;
@ -2309,6 +2309,7 @@ typedef struct {
typedef struct {
SExplainRsp rsp;
uint64_t qId;
uint64_t cId;
uint64_t tId;
int64_t rId;
int32_t eId;
@ -2662,6 +2663,7 @@ typedef struct SSubQueryMsg {
SMsgHead header;
uint64_t sId;
uint64_t queryId;
uint64_t clientId;
uint64_t taskId;
int64_t refId;
int32_t execId;
@ -2691,6 +2693,7 @@ typedef struct {
SMsgHead header;
uint64_t sId;
uint64_t queryId;
uint64_t clientId;
uint64_t taskId;
int32_t execId;
} SQueryContinueReq;
@ -2725,6 +2728,7 @@ typedef struct {
SMsgHead header;
uint64_t sId;
uint64_t queryId;
uint64_t clientId;
uint64_t taskId;
int32_t execId;
SOperatorParam* pOpParam;
@ -2740,6 +2744,7 @@ typedef struct {
typedef struct {
uint64_t queryId;
uint64_t clientId;
uint64_t taskId;
int64_t refId;
int32_t execId;
@ -2786,6 +2791,7 @@ typedef struct {
SMsgHead header;
uint64_t sId;
uint64_t queryId;
uint64_t clientId;
uint64_t taskId;
int64_t refId;
int32_t execId;
@ -2799,6 +2805,7 @@ typedef struct {
SMsgHead header;
uint64_t sId;
uint64_t queryId;
uint64_t clientId;
uint64_t taskId;
int64_t refId;
int32_t execId;
@ -2815,6 +2822,7 @@ typedef struct {
SMsgHead header;
uint64_t sId;
uint64_t queryId;
uint64_t clientId;
uint64_t taskId;
int64_t refId;
int32_t execId;
@ -3222,6 +3230,7 @@ int tDecodeSVCreateTbBatchRsp(SDecoder* pCoder, SVCreateTbBatchRsp* pRsp);
typedef struct {
char* name;
uint64_t suid; // for tmq in wal format
int64_t uid;
int8_t igNotExists;
} SVDropTbReq;
@ -4263,6 +4272,7 @@ typedef struct {
SMsgHead header;
uint64_t sId;
uint64_t queryId;
uint64_t clientId;
uint64_t taskId;
uint32_t sqlLen;
uint32_t phyLen;

3
include/common/ttime.h
View File

@ -62,7 +62,8 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) {
int64_t factor = (precision == TSDB_TIME_PRECISION_MILLI) ? 1000
: (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000
: 1000000000;
time_t t = taosTime(NULL);
time_t t;
(void) taosTime(&t);
struct tm tm;
(void) taosLocalTime(&t, &tm, NULL, 0);
tm.tm_hour = 0;

2
include/libs/executor/executor.h
View File

@ -31,7 +31,7 @@ typedef void* DataSinkHandle;
struct SRpcMsg;
struct SSubplan;
typedef int32_t (*localFetchFp)(void*, uint64_t, uint64_t, uint64_t, int64_t, int32_t, void**, SArray*);
typedef int32_t (*localFetchFp)(void*, uint64_t, uint64_t, uint64_t, uint64_t, int64_t, int32_t, void**, SArray*);
typedef struct {
void* handle;

1
include/libs/executor/storageapi.h
View File

@ -336,6 +336,7 @@ typedef struct SStateStore {
int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname);
int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache,
int32_t* pWinCode);
int32_t (*streamStateDeleteParName)(SStreamState* pState, int64_t groupId);
int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen,
int32_t* pWinCode);

1
include/libs/nodes/plannodes.h
View File

@ -624,6 +624,7 @@ typedef struct SAggPhysiNode {
typedef struct SDownstreamSourceNode {
ENodeType type;
SQueryNodeAddr addr;
uint64_t clientId;
uint64_t taskId;
uint64_t schedId;
int32_t execId;

8
include/libs/qworker/qworker.h
View File

@ -105,11 +105,11 @@ void qWorkerDestroy(void **qWorkerMgmt);
int32_t qWorkerGetStat(SReadHandle *handle, void *qWorkerMgmt, SQWorkerStat *pStat);
int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId,
SQWMsg *qwMsg, SArray *explainRes);
int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t cId, uint64_t tId, int64_t rId,
int32_t eId, SQWMsg *qwMsg, SArray *explainRes);
int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId,
void **pRsp, SArray *explainRes);
int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t cId, uint64_t tId, int64_t rId,
int32_t eId, void **pRsp, SArray *explainRes);
int32_t qWorkerDbgEnableDebug(char *option);

3
include/libs/scheduler/scheduler.h
View File

@ -83,6 +83,9 @@ void schedulerStopQueryHb(void* pTrans);
int32_t schedulerUpdatePolicy(int32_t policy);
int32_t schedulerEnableReSchedule(bool enableResche);
int32_t initClientId(void);
uint64_t getClientId(void);
/**
* Cancel query job
* @param pJob

1
include/libs/stream/streamState.h
View File

@ -116,6 +116,7 @@ void streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname);
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache, int32_t* pWinCode);
int32_t streamStateDeleteParName(SStreamState* pState, int64_t groupId);
// group id
int32_t streamStateGroupPut(SStreamState* pState, int64_t groupId, void* value, int32_t vLen);

2
include/libs/stream/tstream.h
View File

@ -462,7 +462,7 @@ struct SStreamTask {
struct SStreamMeta* pMeta;
SSHashObj* pNameMap;
void* pBackend;
int8_t subtableWithoutMd5;
int8_t subtableWithoutMd5; // only for tsma stream tasks
char reserve[256];
char* backendPath;
};

1
include/libs/wal/wal.h
View File

@ -138,6 +138,7 @@ typedef struct {
int8_t scanMeta;
int8_t deleteMsg;
int8_t enableRef;
int8_t scanDropCtb;
} SWalFilterCond;
// todo hide this struct

2
include/os/osTime.h
View File

@ -93,7 +93,7 @@ static FORCE_INLINE int64_t taosGetMonoTimestampMs() {
char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm);
struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf, int32_t bufSize);
struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst);
time_t taosTime(time_t *t);
int32_t taosTime(time_t *t);
time_t taosMktime(struct tm *timep);
int64_t user_mktime64(const uint32_t year, const uint32_t mon, const uint32_t day, const uint32_t hour,
const uint32_t min, const uint32_t sec, int64_t time_zone);

4
source/client/src/clientEnv.c
View File

@ -294,8 +294,7 @@ static void deregisterRequest(SRequestObj *pRequest) {
}
}
if ((duration >= pTscObj->pAppInfo->serverCfg.monitorParas.tsSlowLogThreshold * 1000000UL ||
duration >= pTscObj->pAppInfo->serverCfg.monitorParas.tsSlowLogThresholdTest * 1000000UL) &&
if ((duration >= pTscObj->pAppInfo->serverCfg.monitorParas.tsSlowLogThreshold * 1000000UL) &&
checkSlowLogExceptDb(pRequest, pTscObj->pAppInfo->serverCfg.monitorParas.tsSlowLogExceptDb)) {
(void)atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1);
if (pTscObj->pAppInfo->serverCfg.monitorParas.tsSlowLogScope & reqType) {
@ -983,6 +982,7 @@ void taos_init_imp(void) {
SCatalogCfg cfg = {.maxDBCacheNum = 100, .maxTblCacheNum = 100};
ENV_ERR_RET(catalogInit(&cfg), "failed to init catalog");
ENV_ERR_RET(schedulerInit(), "failed to init scheduler");
ENV_ERR_RET(initClientId(), "failed to init clientId");
tscDebug("starting to initialize TAOS driver");

4
source/common/src/tcol.c
View File

@ -166,6 +166,7 @@ const char* columnCompressStr(uint16_t type) {
}
uint8_t columnLevelVal(const char* level) {
if (level == NULL) return TSDB_COLVAL_LEVEL_NOCHANGE;
uint8_t l = TSDB_COLVAL_LEVEL_MEDIUM;
if (0 == strcmp(level, "h") || 0 == strcmp(level, TSDB_COLUMN_LEVEL_HIGH)) {
l = TSDB_COLVAL_LEVEL_HIGH;
@ -180,6 +181,7 @@ uint8_t columnLevelVal(const char* level) {
}
uint16_t columnCompressVal(const char* compress) {
if (compress == NULL) return TSDB_COLVAL_COMPRESS_NOCHANGE;
uint16_t c = TSDB_COLVAL_COMPRESS_LZ4;
if (0 == strcmp(compress, TSDB_COLUMN_COMPRESS_LZ4)) {
c = TSDB_COLVAL_COMPRESS_LZ4;
@ -200,6 +202,7 @@ uint16_t columnCompressVal(const char* compress) {
}
uint8_t columnEncodeVal(const char* encode) {
if (encode == NULL) return TSDB_COLVAL_ENCODE_NOCHANGE;
uint8_t e = TSDB_COLVAL_ENCODE_SIMPLE8B;
if (0 == strcmp(encode, TSDB_COLUMN_ENCODE_SIMPLE8B)) {
e = TSDB_COLVAL_ENCODE_SIMPLE8B;
@ -311,6 +314,7 @@ void setColLevel(uint32_t* compress, uint8_t level) {
int32_t setColCompressByOption(uint8_t type, uint8_t encode, uint16_t compressType, uint8_t level, bool check,
uint32_t* compress) {
if(compress == NULL) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
if (check && !validColEncode(type, encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
setColEncode(compress, encode);

6
source/common/src/tglobal.c
View File

@ -185,7 +185,6 @@ int32_t tsMaxRetryWaitTime = 10000;
bool tsUseAdapter = false;
int32_t tsMetaCacheMaxSize = -1; // MB
int32_t tsSlowLogThreshold = 10; // seconds
int32_t tsSlowLogThresholdTest = INT32_MAX; // seconds
char tsSlowLogExceptDb[TSDB_DB_NAME_LEN] = ""; // seconds
int32_t tsSlowLogScope = SLOW_LOG_TYPE_QUERY;
char *tsSlowLogScopeString = "query";
@ -766,7 +765,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 86400, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "slowLogThresholdTest", tsSlowLogThresholdTest, 0, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 1, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "slowLogMaxLen", tsSlowLogMaxLen, 1, 16384, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "slowLogScope", tsSlowLogScopeString, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
@ -1450,9 +1448,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "slowLogExceptDb");
tstrncpy(tsSlowLogExceptDb, pItem->str, TSDB_DB_NAME_LEN);
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "slowLogThresholdTest");
tsSlowLogThresholdTest = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "slowLogThreshold");
tsSlowLogThreshold = pItem->i32;
@ -2024,7 +2019,6 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) {
{"monitor", &tsEnableMonitor},
{"monitorInterval", &tsMonitorInterval},
{"slowLogThreshold", &tsSlowLogThreshold},
{"slowLogThresholdTest", &tsSlowLogThresholdTest},
{"slowLogMaxLen", &tsSlowLogMaxLen},
{"mndSdbWriteDelta", &tsMndSdbWriteDelta},

46
source/common/src/tmsg.c
View File

@ -76,7 +76,7 @@ static int32_t tSerializeSMonitorParas(SEncoder *encoder, const SMonitorParas *p
TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogScope));
TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogMaxLen));
TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogThreshold));
TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogThresholdTest));
TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogThresholdTest)); //Obsolete
TAOS_CHECK_RETURN(tEncodeCStr(encoder, pMonitorParas->tsSlowLogExceptDb));
return 0;
}
@ -87,7 +87,7 @@ static int32_t tDeserializeSMonitorParas(SDecoder *decoder, SMonitorParas *pMoni
TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogScope));
TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogMaxLen));
TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogThreshold));
TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogThresholdTest));
TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogThresholdTest)); //Obsolete
TAOS_CHECK_RETURN(tDecodeCStrTo(decoder, pMonitorParas->tsSlowLogExceptDb));
return 0;
}
@ -8752,6 +8752,7 @@ int32_t tSerializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq) {
TAOS_CHECK_EXIT(tEncodeCStrWithLen(&encoder, pReq->sql, pReq->sqlLen));
TAOS_CHECK_EXIT(tEncodeU32(&encoder, pReq->msgLen));
TAOS_CHECK_EXIT(tEncodeBinary(&encoder, (uint8_t *)pReq->msg, pReq->msgLen));
TAOS_CHECK_EXIT(tEncodeU64(&encoder, pReq->clientId));
tEndEncode(&encoder);
@ -8800,6 +8801,11 @@ int32_t tDeserializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq)
TAOS_CHECK_EXIT(tDecodeCStrAlloc(&decoder, &pReq->sql));
TAOS_CHECK_EXIT(tDecodeU32(&decoder, &pReq->msgLen));
TAOS_CHECK_EXIT(tDecodeBinaryAlloc(&decoder, (void **)&pReq->msg, NULL));
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pReq->clientId));
} else {
pReq->clientId = 0;
}
tEndDecode(&decoder);
@ -8929,6 +8935,7 @@ int32_t tSerializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq) {
} else {
TAOS_CHECK_EXIT(tEncodeI32(&encoder, 0));
}
TAOS_CHECK_EXIT(tEncodeU64(&encoder, pReq->clientId));
tEndEncode(&encoder);
@ -8978,6 +8985,11 @@ int32_t tDeserializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq)
}
TAOS_CHECK_EXIT(tDeserializeSOperatorParam(&decoder, pReq->pOpParam));
}
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pReq->clientId));
} else {
pReq->clientId = 0;
}
tEndDecode(&decoder);
@ -9090,6 +9102,7 @@ int32_t tSerializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq) {
TAOS_CHECK_EXIT(tEncodeU64(&encoder, pReq->taskId));
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->refId));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->execId));
TAOS_CHECK_EXIT(tEncodeU64(&encoder, pReq->clientId));
tEndEncode(&encoder);
@ -9130,6 +9143,11 @@ int32_t tDeserializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq)
TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pReq->taskId));
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->refId));
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->execId));
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pReq->clientId));
} else {
pReq->clientId = 0;
}
tEndDecode(&decoder);
@ -9158,6 +9176,7 @@ int32_t tSerializeSTaskNotifyReq(void *buf, int32_t bufLen, STaskNotifyReq *pReq
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->refId));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->execId));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->type));
TAOS_CHECK_EXIT(tEncodeU64(&encoder, pReq->clientId));
tEndEncode(&encoder);
@ -9199,6 +9218,11 @@ int32_t tDeserializeSTaskNotifyReq(void *buf, int32_t bufLen, STaskNotifyReq *pR
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->refId));
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->execId));
TAOS_CHECK_EXIT(tDecodeI32(&decoder, (int32_t *)&pReq->type));
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pReq->clientId));
} else {
pReq->clientId = 0;
}
tEndDecode(&decoder);
@ -9388,6 +9412,10 @@ int32_t tSerializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *pR
TAOS_CHECK_EXIT(tEncodeI32(&encoder, status->execId));
TAOS_CHECK_EXIT(tEncodeI8(&encoder, status->status));
}
for (int32_t i = 0; i < num; ++i) {
STaskStatus *status = taosArrayGet(pRsp->taskStatus, i);
TAOS_CHECK_EXIT(tEncodeU64(&encoder, status->clientId));
}
} else {
TAOS_CHECK_EXIT(tEncodeI32(&encoder, 0));
}
@ -9431,6 +9459,12 @@ int32_t tDeserializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *
TAOS_CHECK_EXIT(terrno);
}
}
if (!tDecodeIsEnd(&decoder)) {
for (int32_t i = 0; i < num; ++i) {
STaskStatus *status = taosArrayGet(pRsp->taskStatus, i);
TAOS_CHECK_EXIT(tDecodeU64(&decoder, &status->clientId));
}
}
} else {
pRsp->taskStatus = NULL;
}
@ -9595,6 +9629,7 @@ int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) {
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->sql));
TAOS_CHECK_EXIT(tEncodeBinary(&encoder, pReq->msg, pReq->phyLen));
TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->source));
TAOS_CHECK_EXIT(tEncodeU64(&encoder, pReq->clientId));
tEndEncode(&encoder);
_exit:
@ -9643,6 +9678,11 @@ int32_t tDeserializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) {
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->source));
}
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pReq->clientId));
} else {
pReq->clientId = 0;
}
tEndDecode(&decoder);
_exit:
@ -10312,6 +10352,7 @@ static int32_t tEncodeSVDropTbReq(SEncoder *pCoder, const SVDropTbReq *pReq) {
TAOS_CHECK_RETURN(tStartEncode(pCoder));
TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pReq->name));
TAOS_CHECK_RETURN(tEncodeU64(pCoder, pReq->suid));
TAOS_CHECK_RETURN(tEncodeI64(pCoder, pReq->uid));
TAOS_CHECK_RETURN(tEncodeI8(pCoder, pReq->igNotExists));
tEndEncode(pCoder);
@ -10322,6 +10363,7 @@ static int32_t tDecodeSVDropTbReq(SDecoder *pCoder, SVDropTbReq *pReq) {
TAOS_CHECK_RETURN(tStartDecode(pCoder));
TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pReq->name));
TAOS_CHECK_RETURN(tDecodeU64(pCoder, &pReq->suid));
TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pReq->uid));
TAOS_CHECK_RETURN(tDecodeI8(pCoder, &pReq->igNotExists));
tEndDecode(pCoder);

2
source/common/src/ttime.c
View File

@ -30,7 +30,7 @@ static int64_t m_deltaUtc = 0;
void deltaToUtcInitOnce() {
struct tm tm = {0};
if (taosStrpTime("1970-01-01 00:00:00", (const char*)("%Y-%m-%d %H:%M:%S"), &tm) != 0) {
if (taosStrpTime("1970-01-01 00:00:00", (const char*)("%Y-%m-%d %H:%M:%S"), &tm) == NULL) {
uError("failed to parse time string");
}
m_deltaUtc = (int64_t)taosMktime(&tm);

1
source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
View File

@ -195,7 +195,6 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
req.clusterCfg.monitorParas.tsSlowLogScope = tsSlowLogScope;
req.clusterCfg.monitorParas.tsSlowLogMaxLen = tsSlowLogMaxLen;
req.clusterCfg.monitorParas.tsSlowLogThreshold = tsSlowLogThreshold;
req.clusterCfg.monitorParas.tsSlowLogThresholdTest = tsSlowLogThresholdTest;
tstrncpy(req.clusterCfg.monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb, TSDB_DB_NAME_LEN);
char timestr[32] = "1970-01-01 00:00:00.00";
if (taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0) != 0) {

1
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -482,7 +482,6 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, SDnodeObj *pDnode, const S
CHECK_MONITOR_PARA(tsEnableMonitor, DND_REASON_STATUS_MONITOR_SWITCH_NOT_MATCH);
CHECK_MONITOR_PARA(tsMonitorInterval, DND_REASON_STATUS_MONITOR_INTERVAL_NOT_MATCH);
CHECK_MONITOR_PARA(tsSlowLogThreshold, DND_REASON_STATUS_MONITOR_SLOW_LOG_THRESHOLD_NOT_MATCH);
CHECK_MONITOR_PARA(tsSlowLogThresholdTest, DND_REASON_STATUS_MONITOR_NOT_MATCH);
CHECK_MONITOR_PARA(tsSlowLogMaxLen, DND_REASON_STATUS_MONITOR_SLOW_LOG_SQL_MAX_LEN_NOT_MATCH);
CHECK_MONITOR_PARA(tsSlowLogScope, DND_REASON_STATUS_MONITOR_SLOW_LOG_SCOPE_NOT_MATCH);

2
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -304,7 +304,6 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
connectRsp.monitorParas.tsSlowLogScope = tsSlowLogScope;
connectRsp.monitorParas.tsSlowLogMaxLen = tsSlowLogMaxLen;
connectRsp.monitorParas.tsSlowLogThreshold = tsSlowLogThreshold;
connectRsp.monitorParas.tsSlowLogThresholdTest = tsSlowLogThresholdTest;
connectRsp.enableAuditDelete = tsEnableAuditDelete;
tstrncpy(connectRsp.monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb, TSDB_DB_NAME_LEN);
connectRsp.whiteListVer = pUser->ipWhiteListVer;
@ -706,7 +705,6 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) {
batchRsp.monitorParas.tsEnableMonitor = tsEnableMonitor;
batchRsp.monitorParas.tsMonitorInterval = tsMonitorInterval;
batchRsp.monitorParas.tsSlowLogThreshold = tsSlowLogThreshold;
batchRsp.monitorParas.tsSlowLogThresholdTest = tsSlowLogThresholdTest;
tstrncpy(batchRsp.monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb, TSDB_DB_NAME_LEN);
batchRsp.monitorParas.tsSlowLogMaxLen = tsSlowLogMaxLen;
batchRsp.monitorParas.tsSlowLogScope = tsSlowLogScope;

292
source/dnode/mnode/impl/src/mndStb.c
View File

@ -4063,8 +4063,8 @@ static int32_t mndProcessDropStbReqFromMNode(SRpcMsg *pReq) {
}
typedef struct SVDropTbVgReqs {
SVDropTbBatchReq req;
SVgroupInfo info;
SArray *pBatchReqs;
SVgroupInfo info;
} SVDropTbVgReqs;
typedef struct SMDropTbDbInfo {
@ -4086,45 +4086,21 @@ typedef struct SMDropTbTsmaInfos {
} SMDropTbTsmaInfos;
typedef struct SMndDropTbsWithTsmaCtx {
SHashObj *pTsmaMap; // <suid, SMDropTbTsmaInfos>
SHashObj *pDbMap; // <dbuid, SMDropTbDbInfo>
SHashObj *pVgMap; // <vgId, SVDropTbVgReqs>
SArray *pResTbNames; // SArray<char*>
SHashObj *pVgMap; // <vgId, SVDropTbVgReqs>
} SMndDropTbsWithTsmaCtx;
static int32_t mndDropTbAddTsmaResTbsForSingleVg(SMnode *pMnode, SMndDropTbsWithTsmaCtx *pCtx, SArray *pTbs,
static int32_t mndDropTbForSingleVg(SMnode *pMnode, SMndDropTbsWithTsmaCtx *pCtx, SArray *pTbs,
int32_t vgId);
static void destroySVDropTbBatchReqs(void *p);
static void mndDestroyDropTbsWithTsmaCtx(SMndDropTbsWithTsmaCtx *p) {
if (!p) return;
if (p->pDbMap) {
void *pIter = taosHashIterate(p->pDbMap, NULL);
while (pIter) {
SMDropTbDbInfo *pInfo = pIter;
taosArrayDestroy(pInfo->dbVgInfos);
pIter = taosHashIterate(p->pDbMap, pIter);
}
taosHashCleanup(p->pDbMap);
}
if (p->pResTbNames) {
taosArrayDestroyP(p->pResTbNames, taosMemoryFree);
}
if (p->pTsmaMap) {
void *pIter = taosHashIterate(p->pTsmaMap, NULL);
while (pIter) {
SMDropTbTsmaInfos *pInfos = pIter;
taosArrayDestroy(pInfos->pTsmaInfos);
pIter = taosHashIterate(p->pTsmaMap, pIter);
}
taosHashCleanup(p->pTsmaMap);
}
if (p->pVgMap) {
void *pIter = taosHashIterate(p->pVgMap, NULL);
while (pIter) {
SVDropTbVgReqs *pReqs = pIter;
taosArrayDestroy(pReqs->req.pArray);
taosArrayDestroyEx(pReqs->pBatchReqs, destroySVDropTbBatchReqs);
pIter = taosHashIterate(p->pVgMap, pIter);
}
taosHashCleanup(p->pVgMap);
@ -4136,24 +4112,13 @@ static int32_t mndInitDropTbsWithTsmaCtx(SMndDropTbsWithTsmaCtx **ppCtx) {
int32_t code = 0;
SMndDropTbsWithTsmaCtx *pCtx = taosMemoryCalloc(1, sizeof(SMndDropTbsWithTsmaCtx));
if (!pCtx) return terrno;
pCtx->pTsmaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
if (!pCtx->pTsmaMap) {
code = terrno;
goto _end;
}
pCtx->pDbMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
if (!pCtx->pDbMap) {
code = terrno;
goto _end;
}
pCtx->pResTbNames = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
pCtx->pVgMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
if (!pCtx->pVgMap) {
code = terrno;
goto _end;
}
*ppCtx = pCtx;
_end:
if (code) mndDestroyDropTbsWithTsmaCtx(pCtx);
@ -4192,16 +4157,43 @@ static void *mndBuildVDropTbsReq(SMnode *pMnode, const SVgroupInfo *pVgInfo, con
}
static int32_t mndSetDropTbsRedoActions(SMnode *pMnode, STrans *pTrans, const SVDropTbVgReqs *pVgReqs, void *pCont,
int32_t contLen) {
int32_t contLen, tmsg_t msgType) {
STransAction action = {0};
action.epSet = pVgReqs->info.epSet;
action.pCont = pCont;
action.contLen = contLen;
action.msgType = TDMT_VND_DROP_TABLE;
action.msgType = msgType;
action.acceptableCode = TSDB_CODE_TDB_TABLE_NOT_EXIST;
return mndTransAppendRedoAction(pTrans, &action);
}
static int32_t mndBuildDropTbRedoActions(SMnode *pMnode, STrans *pTrans, SHashObj *pVgMap, tmsg_t msgType) {
int32_t code = 0;
void *pIter = taosHashIterate(pVgMap, NULL);
while (pIter) {
const SVDropTbVgReqs *pVgReqs = pIter;
int32_t len = 0;
for (int32_t i = 0; i < taosArrayGetSize(pVgReqs->pBatchReqs) && code == TSDB_CODE_SUCCESS; ++i) {
SVDropTbBatchReq *pBatchReq = taosArrayGet(pVgReqs->pBatchReqs, i);
void *p = mndBuildVDropTbsReq(pMnode, &pVgReqs->info, pBatchReq, &len);
if (!p) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
break;
}
if ((code = mndSetDropTbsRedoActions(pMnode, pTrans, pVgReqs, p, len, msgType)) != 0) {
break;
}
}
if (TSDB_CODE_SUCCESS != code) {
taosHashCancelIterate(pVgMap, pIter);
break;
}
pIter = taosHashIterate(pVgMap, pIter);
}
return code;
}
static int32_t mndCreateDropTbsTxnPrepare(SRpcMsg *pRsp, SMndDropTbsWithTsmaCtx *pCtx) {
int32_t code = 0;
SMnode *pMnode = pRsp->info.node;
@ -4216,23 +4208,7 @@ static int32_t mndCreateDropTbsTxnPrepare(SRpcMsg *pRsp, SMndDropTbsWithTsmaCtx
TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
void *pIter = taosHashIterate(pCtx->pVgMap, NULL);
while (pIter) {
const SVDropTbVgReqs *pVgReqs = pIter;
int32_t len = 0;
void *p = mndBuildVDropTbsReq(pMnode, &pVgReqs->info, &pVgReqs->req, &len);
if (!p) {
taosHashCancelIterate(pCtx->pVgMap, pIter);
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
if ((code = mndSetDropTbsRedoActions(pMnode, pTrans, pVgReqs, p, len)) != 0) {
taosHashCancelIterate(pCtx->pVgMap, pIter);
goto _OVER;
}
pIter = taosHashIterate(pCtx->pVgMap, pIter);
}
if ((code = mndBuildDropTbRedoActions(pMnode, pTrans, pCtx->pVgMap, TDMT_VND_DROP_TABLE)) != 0) goto _OVER;
if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER;
_OVER:
@ -4257,10 +4233,11 @@ static int32_t mndProcessDropTbWithTsma(SRpcMsg *pReq) {
if (code) goto _OVER;
for (int32_t i = 0; i < dropReq.pVgReqs->size; ++i) {
SMDropTbReqsOnSingleVg *pReq = taosArrayGet(dropReq.pVgReqs, i);
code = mndDropTbAddTsmaResTbsForSingleVg(pMnode, pCtx, pReq->pTbs, pReq->vgInfo.vgId);
code = mndDropTbForSingleVg(pMnode, pCtx, pReq->pTbs, pReq->vgInfo.vgId);
if (code) goto _OVER;
}
if (mndCreateDropTbsTxnPrepare(pReq, pCtx) == 0) {
code = mndCreateDropTbsTxnPrepare(pReq, pCtx);
if (code == 0) {
code = TSDB_CODE_ACTION_IN_PROGRESS;
}
_OVER:
@ -4269,87 +4246,58 @@ _OVER:
TAOS_RETURN(code);
}
static int32_t createDropTbBatchReq(const SVDropTbReq *pReq, SVDropTbBatchReq *pBatchReq) {
pBatchReq->nReqs = 1;
pBatchReq->pArray = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVDropTbReq));
if (!pBatchReq->pArray) return terrno;
if (taosArrayPush(pBatchReq->pArray, pReq) == NULL) {
taosArrayDestroy(pBatchReq->pArray);
pBatchReq->pArray = NULL;
return terrno;
}
return TSDB_CODE_SUCCESS;
}
static void destroySVDropTbBatchReqs(void *p) {
SVDropTbBatchReq *pReq = p;
taosArrayDestroy(pReq->pArray);
pReq->pArray = NULL;
}
static int32_t mndDropTbAdd(SMnode *pMnode, SHashObj *pVgHashMap, const SVgroupInfo *pVgInfo, char *name, tb_uid_t suid,
bool ignoreNotExists) {
SVDropTbReq req = {.name = name, .suid = suid, .igNotExists = ignoreNotExists};
SVDropTbReq req = {.name = name, .suid = suid, .igNotExists = ignoreNotExists, .uid = 0};
SVDropTbVgReqs *pReqs = taosHashGet(pVgHashMap, &pVgInfo->vgId, sizeof(pVgInfo->vgId));
SVDropTbVgReqs reqs = {0};
if (pReqs == NULL) {
reqs.info = *pVgInfo;
reqs.req.pArray = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVDropTbReq));
if (reqs.req.pArray == NULL) {
SVDropTbVgReqs *pVgReqs = taosHashGet(pVgHashMap, &pVgInfo->vgId, sizeof(pVgInfo->vgId));
SVDropTbVgReqs vgReqs = {0};
if (pVgReqs == NULL) {
vgReqs.info = *pVgInfo;
vgReqs.pBatchReqs = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVDropTbBatchReq));
if (!vgReqs.pBatchReqs) return terrno;
SVDropTbBatchReq batchReq = {0};
int32_t code = createDropTbBatchReq(&req, &batchReq);
if (TSDB_CODE_SUCCESS != code) return code;
if (taosArrayPush(vgReqs.pBatchReqs, &batchReq) == NULL) {
taosArrayDestroy(batchReq.pArray);
return terrno;
}
if (taosArrayPush(reqs.req.pArray, &req) == NULL) {
return terrno;
}
if (taosHashPut(pVgHashMap, &pVgInfo->vgId, sizeof(pVgInfo->vgId), &reqs, sizeof(reqs)) != 0) {
if (taosHashPut(pVgHashMap, &pVgInfo->vgId, sizeof(pVgInfo->vgId), &vgReqs, sizeof(vgReqs)) != 0) {
taosArrayDestroyEx(vgReqs.pBatchReqs, destroySVDropTbBatchReqs);
return terrno;
}
} else {
if (taosArrayPush(pReqs->req.pArray, &req) == NULL) {
SVDropTbBatchReq batchReq = {0};
int32_t code = createDropTbBatchReq(&req, &batchReq);
if (TSDB_CODE_SUCCESS != code) return code;
if (taosArrayPush(pVgReqs->pBatchReqs, &batchReq) == NULL) {
taosArrayDestroy(batchReq.pArray);
return terrno;
}
}
return 0;
}
int vgInfoCmp(const void *lp, const void *rp) {
SVgroupInfo *pLeft = (SVgroupInfo *)lp;
SVgroupInfo *pRight = (SVgroupInfo *)rp;
if (pLeft->hashBegin < pRight->hashBegin) {
return -1;
} else if (pLeft->hashBegin > pRight->hashBegin) {
return 1;
}
return 0;
}
static int32_t mndGetDbVgInfoForTsma(SMnode *pMnode, const char *dbname, SMDropTbTsmaInfo *pInfo) {
int32_t code = 0;
SDbObj *pDb = mndAcquireDb(pMnode, dbname);
if (!pDb) {
code = TSDB_CODE_MND_DB_NOT_EXIST;
goto _end;
}
pInfo->dbInfo.dbVgInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo));
if (!pInfo->dbInfo.dbVgInfos) {
code = terrno;
goto _end;
}
mndBuildDBVgroupInfo(pDb, pMnode, pInfo->dbInfo.dbVgInfos);
taosArraySort(pInfo->dbInfo.dbVgInfos, vgInfoCmp);
pInfo->dbInfo.hashPrefix = pDb->cfg.hashPrefix;
pInfo->dbInfo.hashSuffix = pDb->cfg.hashSuffix;
pInfo->dbInfo.hashMethod = pDb->cfg.hashMethod;
_end:
if (pDb) mndReleaseDb(pMnode, pDb);
if (code && pInfo->dbInfo.dbVgInfos) {
taosArrayDestroy(pInfo->dbInfo.dbVgInfos);
pInfo->dbInfo.dbVgInfos = NULL;
}
TAOS_RETURN(code);
}
int32_t vgHashValCmp(const void *lp, const void *rp) {
uint32_t *key = (uint32_t *)lp;
SVgroupInfo *pVg = (SVgroupInfo *)rp;
if (*key < pVg->hashBegin) {
return -1;
} else if (*key > pVg->hashEnd) {
return 1;
}
return 0;
}
static int32_t mndDropTbAddTsmaResTbsForSingleVg(SMnode *pMnode, SMndDropTbsWithTsmaCtx *pCtx, SArray *pTbs,
static int32_t mndDropTbForSingleVg(SMnode *pMnode, SMndDropTbsWithTsmaCtx *pCtx, SArray *pTbs,
int32_t vgId) {
int32_t code = 0;
@ -4365,88 +4313,9 @@ static int32_t mndDropTbAddTsmaResTbsForSingleVg(SMnode *pMnode, SMndDropTbsWith
vgInfo.epSet = mndGetVgroupEpset(pMnode, pVgObj);
mndReleaseVgroup(pMnode, pVgObj);
// get all stb uids
for (int32_t i = 0; i < pTbs->size; ++i) {
const SVDropTbReq *pTb = taosArrayGet(pTbs, i);
if (taosHashGet(pCtx->pTsmaMap, &pTb->suid, sizeof(pTb->suid))) {
} else {
SMDropTbTsmaInfos infos = {0};
infos.pTsmaInfos = taosArrayInit(2, sizeof(SMDropTbTsmaInfo));
if (!infos.pTsmaInfos) {
code = terrno;
goto _end;
}
if (taosHashPut(pCtx->pTsmaMap, &pTb->suid, sizeof(pTb->suid), &infos, sizeof(infos)) != 0) {
code = terrno;
goto _end;
}
}
}
void *pIter = NULL;
SSmaObj *pSma = NULL;
char buf[TSDB_TABLE_FNAME_LEN] = {0};
// get used tsmas and it's dbs
while (1) {
pIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter, (void **)&pSma);
if (!pIter) break;
SMDropTbTsmaInfos *pInfos = taosHashGet(pCtx->pTsmaMap, &pSma->stbUid, sizeof(pSma->stbUid));
if (pInfos) {
SMDropTbTsmaInfo info = {0};
int32_t len = sprintf(buf, "%s", pSma->name);
sprintf(info.tsmaResTbDbFName, "%s", pSma->db);
snprintf(info.tsmaResTbNamePrefix, TSDB_TABLE_FNAME_LEN, "%s", buf);
SMDropTbDbInfo *pDbInfo = taosHashGet(pCtx->pDbMap, pSma->db, TSDB_DB_FNAME_LEN);
info.suid = pSma->dstTbUid;
if (!pDbInfo) {
code = mndGetDbVgInfoForTsma(pMnode, pSma->db, &info);
if (code != TSDB_CODE_SUCCESS) {
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pSma);
goto _end;
}
if (taosHashPut(pCtx->pDbMap, pSma->db, TSDB_DB_FNAME_LEN, &info.dbInfo, sizeof(SMDropTbDbInfo)) != 0) {
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pSma);
goto _end;
}
} else {
info.dbInfo = *pDbInfo;
}
if (taosArrayPush(pInfos->pTsmaInfos, &info) == NULL) {
code = terrno;
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pSma);
goto _end;
}
}
sdbRelease(pMnode->pSdb, pSma);
}
// generate vg req map
for (int32_t i = 0; i < pTbs->size; ++i) {
SVDropTbReq *pTb = taosArrayGet(pTbs, i);
TAOS_CHECK_GOTO(mndDropTbAdd(pMnode, pCtx->pVgMap, &vgInfo, pTb->name, pTb->suid, pTb->igNotExists), NULL, _end);
SMDropTbTsmaInfos *pInfos = taosHashGet(pCtx->pTsmaMap, &pTb->suid, sizeof(pTb->suid));
SArray *pVgInfos = NULL;
char buf[TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN + 1];
char resTbFullName[TSDB_TABLE_FNAME_LEN + 1] = {0};
for (int32_t j = 0; j < pInfos->pTsmaInfos->size; ++j) {
SMDropTbTsmaInfo *pInfo = taosArrayGet(pInfos->pTsmaInfos, j);
int32_t len = sprintf(buf, "%s_%s", pInfo->tsmaResTbNamePrefix, pTb->name);
len = taosCreateMD5Hash(buf, len);
len = snprintf(resTbFullName, TSDB_TABLE_FNAME_LEN + 1, "%s.%s", pInfo->tsmaResTbDbFName, buf);
uint32_t hashVal = taosGetTbHashVal(resTbFullName, len, pInfo->dbInfo.hashMethod, pInfo->dbInfo.hashPrefix,
pInfo->dbInfo.hashSuffix);
const SVgroupInfo *pVgInfo = taosArraySearch(pInfo->dbInfo.dbVgInfos, &hashVal, vgHashValCmp, TD_EQ);
void *p = taosStrdup(resTbFullName + strlen(pInfo->tsmaResTbDbFName) + TSDB_NAME_DELIMITER_LEN);
if (taosArrayPush(pCtx->pResTbNames, &p) == NULL) {
code = terrno;
goto _end;
}
TAOS_CHECK_GOTO(mndDropTbAdd(pMnode, pCtx->pVgMap, pVgInfo, p, pInfo->suid, true), NULL, _end);
}
}
_end:
return code;
@ -4474,9 +4343,10 @@ static int32_t mndProcessFetchTtlExpiredTbs(SRpcMsg *pRsp) {
code = mndInitDropTbsWithTsmaCtx(&pCtx);
if (code) goto _end;
code = mndDropTbAddTsmaResTbsForSingleVg(pMnode, pCtx, rsp.pExpiredTbs, rsp.vgId);
code = mndDropTbForSingleVg(pMnode, pCtx, rsp.pExpiredTbs, rsp.vgId);
if (code) goto _end;
if (mndCreateDropTbsTxnPrepare(pRsp, pCtx) == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
code = mndCreateDropTbsTxnPrepare(pRsp, pCtx);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
_end:
if (pCtx) mndDestroyDropTbsWithTsmaCtx(pCtx);
tDecoderClear(&decoder);

1
source/dnode/snode/src/snodeInitApi.c
View File

@ -31,6 +31,7 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStatePutParName = streamStatePutParName;
pStore->streamStateGetParName = streamStateGetParName;
pStore->streamStateDeleteParName = streamStateDeleteParName;
pStore->streamStateAddIfNotExist = streamStateAddIfNotExist;
pStore->streamStateReleaseBuf = streamStateReleaseBuf;

3
source/dnode/vnode/src/inc/tq.h
View File

@ -146,7 +146,7 @@ int32_t tqBuildFName(char** data, const char* path, char* name);
int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name);
// tq util
int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type);
int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type, EStreamType blockType);
int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg);
int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId,
int32_t type, int64_t sver, int64_t ever);
@ -158,6 +158,7 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
int32_t buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock,
SArray* pTagArray, bool newSubTableRule, SVCreateTbReq** pReq);
int32_t tqExtractDropCtbDataBlock(const void* data, int32_t len, int64_t ver, void** pRefBlock, int32_t type);
#define TQ_ERR_GO_TO_END(c) \
do { \

2
source/dnode/vnode/src/sma/smaRollup.c
View File

@ -1551,7 +1551,7 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA
_resume_delete:
version = RSMA_EXEC_MSG_VER(msg);
if ((code = tqExtractDelDataBlock(RSMA_EXEC_MSG_BODY(msg), RSMA_EXEC_MSG_LEN(msg), version,
&packData.pDataBlock, 1))) {
&packData.pDataBlock, 1, STREAM_DELETE_DATA))) {
taosFreeQitem(msg);
TAOS_CHECK_EXIT(code);
}

3
source/dnode/vnode/src/tq/tq.c
View File

@ -758,7 +758,8 @@ int32_t tqBuildStreamTask(void* pTqObj, SStreamTask* pTask, int64_t nextProcessV
}
if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
SWalFilterCond cond = {.deleteMsg = 1}; // delete msg also extract from wal files
bool scanDropCtb = pTask->subtableWithoutMd5 ? true : false;
SWalFilterCond cond = {.deleteMsg = 1, .scanDropCtb = scanDropCtb}; // delete msg also extract from wal files
pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, &cond, pTask->id.taskId);
if (pTask->exec.pWalReader == NULL) {
tqError("vgId:%d failed init wal reader, code:%s", vgId, tstrerror(terrno));

18
source/dnode/vnode/src/tq/tqRead.c
View File

@ -366,8 +366,8 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con
} else if (pCont->msgType == TDMT_VND_DELETE) {
void* pBody = POINTER_SHIFT(pCont->body, sizeof(SMsgHead));
int32_t len = pCont->bodyLen - sizeof(SMsgHead);
code = tqExtractDelDataBlock(pBody, len, ver, (void**)pItem, 0);
EStreamType blockType = STREAM_DELETE_DATA;
code = tqExtractDelDataBlock(pBody, len, ver, (void**)pItem, 0, blockType);
if (code == TSDB_CODE_SUCCESS) {
if (*pItem == NULL) {
tqDebug("s-task:%s empty delete msg, discard it, len:%d, ver:%" PRId64, id, len, ver);
@ -382,6 +382,20 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con
return code;
}
} else if (pCont->msgType == TDMT_VND_DROP_TABLE && pReader->cond.scanDropCtb) {
void* pBody = POINTER_SHIFT(pCont->body, sizeof(SMsgHead));
int32_t len = pCont->bodyLen - sizeof(SMsgHead);
code = tqExtractDropCtbDataBlock(pBody, len, ver, (void**)pItem, 0);
if (TSDB_CODE_SUCCESS == code) {
if (!*pItem) {
continue;
} else {
tqDebug("s-task:%s drop ctb msg extract from WAL, len:%d, ver:%"PRId64, id, len, ver);
}
} else {
terrno = code;
return code;
}
} else {
tqError("s-task:%s invalid msg type:%d, ver:%" PRId64, id, pCont->msgType, ver);
return TSDB_CODE_STREAM_INTERNAL_ERROR;

137
source/dnode/vnode/src/tq/tqSink.c
View File

@ -53,6 +53,7 @@ static int32_t checkTagSchema(SStreamTask* pTask, SVnode* pVnode);
static void reubuildAndSendMultiResBlock(SStreamTask* pTask, const SArray* pBlocks, SVnode* pVnode, int64_t earlyTs);
static int32_t handleResultBlockMsg(SStreamTask* pTask, SSDataBlock* pDataBlock, int32_t index, SVnode* pVnode,
int64_t earlyTs);
static int32_t doWaitForDstTableDropped(SVnode* pVnode, SStreamTask* pTask, const char* dstTableName);
int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq,
const char* pIdStr, bool newSubTableRule) {
@ -138,7 +139,7 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p
return 0;
}
static int32_t encodeCreateChildTableForRPC(SVCreateTbBatchReq* pReqs, int32_t vgId, void** pBuf, int32_t* contLen) {
static int32_t encodeCreateChildTableForRPC(void* pReqs, int32_t vgId, void** pBuf, int32_t* contLen) {
int32_t ret = 0;
tEncodeSize(tEncodeSVCreateTbBatchReq, pReqs, *contLen, ret);
@ -170,17 +171,50 @@ end:
return ret;
}
static int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) {
static int32_t encodeDropChildTableForRPC(void* pReqs, int32_t vgId, void** ppBuf, int32_t *contLen) {
int32_t code = 0;
SEncoder ec = {0};
tEncodeSize(tEncodeSVDropTbBatchReq, pReqs, *contLen, code);
if (code < 0) {
code = TSDB_CODE_INVALID_MSG;
goto end;
}
*contLen += sizeof(SMsgHead);
*ppBuf = rpcMallocCont(*contLen);
if (!*ppBuf) {
code = terrno;
goto end;
}
((SMsgHead*)(*ppBuf))->vgId = vgId;
((SMsgHead*)(*ppBuf))->contLen = htonl(*contLen);
tEncoderInit(&ec, POINTER_SHIFT(*ppBuf, sizeof(SMsgHead)), (*contLen) - sizeof(SMsgHead));
code = tEncodeSVDropTbBatchReq(&ec, pReqs);
tEncoderClear(&ec);
if (code < 0) {
rpcFreeCont(*ppBuf);
*ppBuf = NULL;
*contLen = 0;
code = TSDB_CODE_INVALID_MSG;
goto end;
}
end:
return code;
}
static int32_t tqPutReqToQueue(SVnode* pVnode, void* pReqs, int32_t(*encoder)(void* pReqs, int32_t vgId, void** ppBuf, int32_t *contLen), tmsg_t msgType) {
void* buf = NULL;
int32_t tlen = 0;
int32_t code = encodeCreateChildTableForRPC(pReqs, TD_VID(pVnode), &buf, &tlen);
int32_t code = encoder(pReqs, TD_VID(pVnode), &buf, &tlen);
if (code) {
tqError("vgId:%d failed to encode create table msg, create table failed, code:%s", TD_VID(pVnode), tstrerror(code));
return code;
}
SRpcMsg msg = {.msgType = TDMT_VND_CREATE_TABLE, .pCont = buf, .contLen = tlen};
SRpcMsg msg = {.msgType = msgType, .pCont = buf, .contLen = tlen};
code = tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg);
if (code) {
tqError("failed to put into write-queue since %s", terrstr());
@ -388,7 +422,7 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
}
reqs.nReqs = taosArrayGetSize(reqs.pArray);
code = tqPutReqToQueue(pVnode, &reqs);
code = tqPutReqToQueue(pVnode, &reqs, encodeCreateChildTableForRPC, TDMT_VND_CREATE_TABLE);
if (code != TSDB_CODE_SUCCESS) {
tqError("s-task:%s failed to send create table msg", id);
}
@ -399,6 +433,61 @@ _end:
return code;
}
static int32_t doBuildAndSendDropTableMsg(SVnode* pVnode, char* pStbFullname, SSDataBlock* pDataBlock,
SStreamTask* pTask, int64_t suid) {
int32_t lino = 0;
int32_t code = 0;
int32_t rows = pDataBlock->info.rows;
const char* id = pTask->id.idStr;
SVDropTbBatchReq batchReq = {0};
SVDropTbReq req = {0};
if (rows <= 0 || rows > 1 || pTask->subtableWithoutMd5 == 0) return TSDB_CODE_SUCCESS;
batchReq.pArray = taosArrayInit(rows, sizeof(SVDropTbReq));
if (!batchReq.pArray) return terrno;
batchReq.nReqs = rows;
req.suid = suid;
req.igNotExists = true;
SColumnInfoData* pTbNameCol = taosArrayGet(pDataBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
char tbName[TSDB_TABLE_NAME_LEN + 1] = {0};
int32_t i = 0;
void* pData = colDataGetVarData(pTbNameCol, i);
memcpy(tbName, varDataVal(pData), varDataLen(pData));
tbName[varDataLen(pData) + 1] = 0;
req.name = tbName;
if (taosArrayPush(batchReq.pArray, &req) == NULL) {
TSDB_CHECK_CODE(terrno, lino, _exit);
}
SMetaReader mr = {0};
metaReaderDoInit(&mr, pVnode->pMeta, META_READER_LOCK);
code = metaGetTableEntryByName(&mr, tbName);
if (TSDB_CODE_SUCCESS == code && isValidDstChildTable(&mr, TD_VID(pVnode), tbName, pTask->outputInfo.tbSink.stbUid)) {
STableSinkInfo* pTableSinkInfo = NULL;
bool alreadyCached = doGetSinkTableInfoFromCache(pTask->outputInfo.tbSink.pTbInfo, pDataBlock->info.id.groupId, &pTableSinkInfo);
if (alreadyCached) {
pTableSinkInfo->uid = mr.me.uid;
}
}
metaReaderClear(&mr);
tqDebug("s-task:%s build drop %d table(s) msg", id, rows);
code = tqPutReqToQueue(pVnode, &batchReq, encodeDropChildTableForRPC, TDMT_VND_DROP_TABLE);
TSDB_CHECK_CODE(code, lino, _exit);
code = doWaitForDstTableDropped(pVnode, pTask, tbName);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (batchReq.pArray) {
taosArrayDestroy(batchReq.pArray);
}
return code;
}
int32_t doBuildAndSendSubmitMsg(SVnode* pVnode, SStreamTask* pTask, SSubmitReq2* pReq, int32_t numOfBlocks) {
const char* id = pTask->id.idStr;
int32_t vgId = TD_VID(pVnode);
@ -807,6 +896,40 @@ int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkI
return TSDB_CODE_SUCCESS;
}
static int32_t doWaitForDstTableDropped(SVnode* pVnode, SStreamTask* pTask, const char* dstTableName) {
int32_t vgId = TD_VID(pVnode);
int64_t suid = pTask->outputInfo.tbSink.stbUid;
const char* id = pTask->id.idStr;
while (1) {
if (streamTaskShouldStop(pTask)) {
tqDebug("s-task:%s task will stop, quit from waiting for table:%s drop", id, dstTableName);
return TSDB_CODE_STREAM_EXEC_CANCELLED;
}
SMetaReader mr = {0};
metaReaderDoInit(&mr, pVnode->pMeta, META_READER_LOCK);
int32_t code = metaGetTableEntryByName(&mr, dstTableName);
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
metaReaderClear(&mr);
break;
} else if (TSDB_CODE_SUCCESS == code) {
if (isValidDstChildTable(&mr, vgId, dstTableName, suid)) {
metaReaderClear(&mr);
taosMsleep(100);
tqDebug("s-task:%s wait 100ms for table:%s drop", id, dstTableName);
} else {
metaReaderClear(&mr);
break;
}
} else {
tqError("s-task:%s failed to wait for table:%s drop", id, dstTableName);
metaReaderClear(&mr);
return terrno;
}
}
return TSDB_CODE_SUCCESS;
}
int32_t doCreateSinkTableInfo(const char* pDstTableName, STableSinkInfo** pInfo) {
int32_t nameLen = strlen(pDstTableName);
(*pInfo) = taosMemoryCalloc(1, sizeof(STableSinkInfo) + nameLen + 1);
@ -1032,7 +1155,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
}
bool onlySubmitData = hasOnlySubmitData(pBlocks, numOfBlocks);
if (!onlySubmitData) {
if (!onlySubmitData || pTask->subtableWithoutMd5 == 1) {
tqDebug("vgId:%d, s-task:%s write %d stream resBlock(s) into table, has delete block, submit one-by-one", vgId, id,
numOfBlocks);
@ -1052,6 +1175,8 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
code = doBuildAndSendCreateTableMsg(pVnode, stbFullName, pDataBlock, pTask, suid);
} else if (pDataBlock->info.type == STREAM_CHECKPOINT) {
continue;
} else if (pDataBlock->info.type == STREAM_DROP_CHILD_TABLE && pTask->subtableWithoutMd5) {
code = doBuildAndSendDropTableMsg(pVnode, stbFullName, pDataBlock, pTask, suid);
} else {
code = handleResultBlockMsg(pTask, pDataBlock, i, pVnode, earlyTs);
}

46
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -572,7 +572,7 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp*
return 0;
}
int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type) {
int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type, EStreamType blockType) {
int32_t code = 0;
int32_t line = 0;
SDecoder* pCoder = &(SDecoder){0};
@ -593,7 +593,7 @@ int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void*
}
SSDataBlock* pDelBlock = NULL;
code = createSpecialDataBlock(STREAM_DELETE_DATA, &pDelBlock);
code = createSpecialDataBlock(blockType, &pDelBlock);
TSDB_CHECK_CODE(code, line, END);
code = blockDataEnsureCapacity(pDelBlock, numOfTables);
@ -751,3 +751,45 @@ int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, b
return TSDB_CODE_SUCCESS;
}
int32_t tqExtractDropCtbDataBlock(const void* data, int32_t len, int64_t ver, void** pRefBlock, int32_t type) {
int32_t code = 0;
int32_t lino = 0;
SDecoder dc = {0};
SVDropTbBatchReq batchReq = {0};
tDecoderInit(&dc, (uint8_t*)data, len);
code = tDecodeSVDropTbBatchReq(&dc, &batchReq);
TSDB_CHECK_CODE(code, lino, _exit);
if (batchReq.nReqs <= 0) goto _exit;
SSDataBlock* pBlock = NULL;
code = createSpecialDataBlock(STREAM_DROP_CHILD_TABLE, &pBlock);
TSDB_CHECK_CODE(code, lino, _exit);
code = blockDataEnsureCapacity(pBlock, batchReq.nReqs);
TSDB_CHECK_CODE(code, lino, _exit);
pBlock->info.rows = batchReq.nReqs;
pBlock->info.version = ver;
for (int32_t i = 0; i < batchReq.nReqs; ++i) {
SVDropTbReq* pReq = batchReq.pReqs + i;
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
TSDB_CHECK_NULL(pCol, code, lino, _exit, terrno);
code = colDataSetVal(pCol, i, (const char* )&pReq->uid, false);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0, pRefBlock);
TSDB_CHECK_CODE(code, lino, _exit);
((SStreamRefDataBlock*)(*pRefBlock))->type = STREAM_INPUT__REF_DATA_BLOCK;
((SStreamRefDataBlock*)(*pRefBlock))->pBlock = pBlock;
_exit:
tDecoderClear(&dc);
if (TSDB_CODE_SUCCESS != code) {
tqError("faled to extract drop ctb data block, line:%d code:%s", lino, tstrerror(code));
blockDataCleanup(pBlock);
taosMemoryFree(pBlock);
}
return code;
}

1
source/dnode/vnode/src/vnd/vnodeInitApi.c
View File

@ -147,6 +147,7 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStatePutParName = streamStatePutParName;
pStore->streamStateGetParName = streamStateGetParName;
pStore->streamStateDeleteParName = streamStateDeleteParName;
pStore->streamStateAddIfNotExist = streamStateAddIfNotExist;
pStore->streamStateReleaseBuf = streamStateReleaseBuf;

62
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -50,6 +50,8 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq,
static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessConfigChangeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessArbCheckSyncReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessDropTSmaCtbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp,
SRpcMsg *pOriginRpc);
static int32_t vnodePreCheckAssignedLogSyncd(SVnode *pVnode, char *member0Token, char *member1Token);
static int32_t vnodeCheckAssignedLogSyncd(SVnode *pVnode, char *member0Token, char *member1Token);
@ -481,6 +483,61 @@ static int32_t vnodePreProcessArbCheckSyncMsg(SVnode *pVnode, SRpcMsg *pMsg) {
return code;
}
int32_t vnodePreProcessDropTbMsg(SVnode* pVnode, SRpcMsg* pMsg) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t size = 0;
SDecoder dc = {0};
SEncoder ec = {0};
SVDropTbBatchReq receivedBatchReqs = {0};
SVDropTbBatchReq sentBatchReqs = {0};
tDecoderInit(&dc, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead));
code = tDecodeSVDropTbBatchReq(&dc, &receivedBatchReqs);
if (code < 0) {
terrno = code;
TSDB_CHECK_CODE(code, lino, _exit);
}
sentBatchReqs.pArray = taosArrayInit(receivedBatchReqs.nReqs, sizeof(SVDropTbReq));
if (!sentBatchReqs.pArray) {
code = terrno;
goto _exit;
}
for (int32_t i = 0; i < receivedBatchReqs.nReqs; ++i) {
SVDropTbReq* pReq = receivedBatchReqs.pReqs + i;
tb_uid_t uid = metaGetTableEntryUidByName(pVnode->pMeta, pReq->name);
if (uid == 0) {
vWarn("vgId:%d, preprocess drop ctb: %s not found", TD_VID(pVnode), pReq->name);
continue;
}
pReq->uid = uid;
vDebug("vgId:%d %s for: %s, uid: %"PRId64, TD_VID(pVnode), __func__, pReq->name, pReq->uid);
if (taosArrayPush(sentBatchReqs.pArray, pReq) == NULL) {
code = terrno;
goto _exit;
}
}
sentBatchReqs.nReqs = sentBatchReqs.pArray->size;
tEncodeSize(tEncodeSVDropTbBatchReq, &sentBatchReqs, size, code);
tEncoderInit(&ec, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), size);
code = tEncodeSVDropTbBatchReq(&ec, &sentBatchReqs);
tEncoderClear(&ec);
if (code != TSDB_CODE_SUCCESS) {
vError("vgId:%d %s failed to encode drop tb batch req: %s", TD_VID(pVnode), __func__, tstrerror(code));
TSDB_CHECK_CODE(code, lino, _exit);
}
_exit:
tDecoderClear(&dc);
if (sentBatchReqs.pArray) {
taosArrayDestroy(sentBatchReqs.pArray);
}
return code;
}
int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
int32_t code = 0;
@ -507,6 +564,9 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
case TDMT_VND_ARB_CHECK_SYNC: {
code = vnodePreProcessArbCheckSyncMsg(pVnode, pMsg);
} break;
case TDMT_VND_DROP_TABLE: {
code = vnodePreProcessDropTbMsg(pVnode, pMsg);
} break;
default:
break;
}
@ -1110,7 +1170,6 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
STbUidStore *pStore = NULL;
SArray *tbUids = NULL;
SArray *tbNames = NULL;
pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP;
pRsp->code = TSDB_CODE_SUCCESS;
pRsp->pCont = NULL;
@ -2512,3 +2571,4 @@ _OVER:
int32_t vnodeAsyncCompact(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { return 0; }
int32_t tsdbAsyncCompact(STsdb *tsdb, const STimeWindow *tw, bool sync) { return 0; }
#endif

18
source/libs/command/src/command.c
View File

@ -35,6 +35,9 @@
extern SConfig* tsCfg;
static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRetrieveTableRsp** pRsp) {
if (NULL == pBlock || NULL == pRsp) {
return TSDB_CODE_INVALID_PARA;
}
size_t dataEncodeBufSize = blockGetEncodeSize(pBlock);
size_t rspSize = sizeof(SRetrieveTableRsp) + dataEncodeBufSize + PAYLOAD_PREFIX_LEN;
*pRsp = taosMemoryCalloc(1, rspSize);
@ -216,6 +219,9 @@ static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock,
static int32_t execDescribe(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp, int8_t biMode) {
SDescribeStmt* pDesc = (SDescribeStmt*)pStmt;
if (NULL == pDesc || NULL == pDesc->pMeta) {
return TSDB_CODE_INVALID_PARA;
}
int32_t numOfRows = TABLE_TOTAL_COL_NUM(pDesc->pMeta);
SSDataBlock* pBlock = NULL;
@ -505,7 +511,7 @@ static int32_t buildCreateViewResultDataBlock(SSDataBlock** pOutput) {
return code;
}
void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) {
static void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) {
for (int32_t i = 0; i < pCfg->numOfColumns; ++i) {
SSchema* pSchema = pCfg->pSchemas + i;
#define LTYPE_LEN (32 + 60) // 60 byte for compress info
@ -539,7 +545,7 @@ void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) {
}
}
void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) {
static void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) {
for (int32_t i = 0; i < pCfg->numOfTags; ++i) {
SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i;
char type[32];
@ -558,7 +564,7 @@ void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) {
}
}
void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) {
static void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) {
for (int32_t i = 0; i < pCfg->numOfTags; ++i) {
SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i;
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
@ -566,7 +572,7 @@ void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) {
}
}
int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
static int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
int32_t code = TSDB_CODE_SUCCESS;
SArray* pTagVals = NULL;
STag* pTag = (STag*)pCfg->pTags;
@ -643,7 +649,7 @@ _exit:
return code;
}
void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg* pCfg) {
static void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg* pCfg) {
if (pCfg->commentLen > 0) {
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
" COMMENT '%s'", pCfg->pComment);
@ -997,7 +1003,7 @@ static int32_t createSelectResultDataBlock(SNodeList* pProjects, SSDataBlock** p
return code;
}
int32_t buildSelectResultDataBlock(SNodeList* pProjects, SSDataBlock* pBlock) {
static int32_t buildSelectResultDataBlock(SNodeList* pProjects, SSDataBlock* pBlock) {
QRY_ERR_RET(blockDataEnsureCapacity(pBlock, 1));
int32_t index = 0;

32
source/libs/command/src/explain.c
View File

@ -30,8 +30,8 @@ char *gJoinTypeStr[JOIN_TYPE_MAX_VALUE][JOIN_STYPE_MAX_VALUE] = {
/*FULL*/ {"Full Join", "Full Join", NULL, NULL, NULL, NULL},
};
int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pRes);
int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, bool singleChannel);
static int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pRes);
static int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, bool singleChannel);
char *qExplainGetDynQryCtrlType(EDynQueryType type) {
switch (type) {
@ -118,7 +118,7 @@ void qExplainFreeCtx(SExplainCtx *pCtx) {
taosMemoryFree(pCtx);
}
int32_t qExplainInitCtx(SExplainCtx **pCtx, SHashObj *groupHash, bool verbose, double ratio, EExplainMode mode) {
static int32_t qExplainInitCtx(SExplainCtx **pCtx, SHashObj *groupHash, bool verbose, double ratio, EExplainMode mode) {
int32_t code = 0;
SExplainCtx *ctx = taosMemoryCalloc(1, sizeof(SExplainCtx));
if (NULL == ctx) {
@ -158,7 +158,7 @@ _return:
QRY_RET(code);
}
int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNodeList **pChildren) {
static int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNodeList **pChildren) {
int32_t tlen = 0;
SNodeList *pPhysiChildren = pNode->pChildren;
@ -180,7 +180,7 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo
return TSDB_CODE_SUCCESS;
}
int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, SExplainGroup *group) {
static int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, SExplainGroup *group) {
*pExecInfo = taosArrayInit(group->nodeNum, sizeof(SExplainExecInfo));
if (NULL == (*pExecInfo)) {
qError("taosArrayInit %d explainExecInfo failed", group->nodeNum);
@ -217,7 +217,7 @@ int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, S
return TSDB_CODE_SUCCESS;
}
int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pResNode) {
static int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pResNode) {
if (NULL == pNode) {
*pResNode = NULL;
qError("physical node is NULL");
@ -250,7 +250,7 @@ _return:
QRY_RET(code);
}
int32_t qExplainBufAppendExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) {
static int32_t qExplainBufAppendExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) {
int32_t tlen = *len;
int32_t nodeNum = taosArrayGetSize(pExecInfo);
SExplainExecInfo maxExecInfo = {0};
@ -275,7 +275,7 @@ int32_t qExplainBufAppendExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) {
return TSDB_CODE_SUCCESS;
}
int32_t qExplainBufAppendVerboseExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) {
static int32_t qExplainBufAppendVerboseExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) {
int32_t tlen = 0;
bool gotVerbose = false;
int32_t nodeNum = taosArrayGetSize(pExecInfo);
@ -297,7 +297,7 @@ int32_t qExplainBufAppendVerboseExecInfo(SArray *pExecInfo, char *tbuf, int32_t
return TSDB_CODE_SUCCESS;
}
int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t level) {
static int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t level) {
SQueryExplainRowInfo row = {0};
row.buf = taosMemoryMalloc(len);
if (NULL == row.buf) {
@ -362,7 +362,7 @@ static char* qExplainGetScanDataLoad(STableScanPhysiNode* pScan) {
return "unknown";
}
int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) {
static int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) {
int32_t tlen = 0;
bool isVerboseLine = false;
char *tbuf = ctx->tbuf;
@ -1900,7 +1900,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
return TSDB_CODE_SUCCESS;
}
int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) {
static int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) {
if (NULL == pResNode) {
qError("explain res node is NULL");
QRY_ERR_RET(TSDB_CODE_APP_ERROR);
@ -1915,7 +1915,7 @@ int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32
return TSDB_CODE_SUCCESS;
}
int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, bool singleChannel) {
static int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, bool singleChannel) {
SExplainResNode *node = NULL;
int32_t code = 0;
SExplainCtx *ctx = (SExplainCtx *)pCtx;
@ -1940,7 +1940,7 @@ _return:
QRY_RET(code);
}
int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
static int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
int32_t code = 0;
SSDataBlock *pBlock = NULL;
SExplainCtx *pCtx = (SExplainCtx *)ctx;
@ -1997,7 +1997,7 @@ _return:
QRY_RET(code);
}
int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) {
static int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) {
int32_t code = 0;
SNodeListNode *plans = NULL;
int32_t taskNum = 0;
@ -2080,7 +2080,7 @@ _return:
QRY_RET(code);
}
int32_t qExplainAppendPlanRows(SExplainCtx *pCtx) {
static int32_t qExplainAppendPlanRows(SExplainCtx *pCtx) {
if (EXPLAIN_MODE_ANALYZE != pCtx->mode) {
return TSDB_CODE_SUCCESS;
}
@ -2103,7 +2103,7 @@ int32_t qExplainAppendPlanRows(SExplainCtx *pCtx) {
return TSDB_CODE_SUCCESS;
}
int32_t qExplainGenerateRsp(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) {
static int32_t qExplainGenerateRsp(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) {
QRY_ERR_RET(qExplainAppendGroupResRows(pCtx, pCtx->rootGroupId, 0, false));
QRY_ERR_RET(qExplainAppendPlanRows(pCtx));
QRY_ERR_RET(qExplainGetRspFromCtx(pCtx, pRsp));

67
source/libs/executor/src/exchangeoperator.c
View File

@ -121,10 +121,10 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
}
} else {
pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64
", totalRows:%" PRIu64 ", try next %d/%" PRIzu,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows,
pExchangeInfo->loadInfo.totalRows, i + 1, totalSources);
qDebug("%s vgId:%d, clientId:0x%" PRIx64 " taskId:0x%" PRIx64
" execId:%d index:%d completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", try next %d/%" PRIzu,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->clientId, pSource->taskId, pSource->execId, i,
pDataInfo->totalRows, pExchangeInfo->loadInfo.totalRows, i + 1, totalSources);
taosMemoryFreeClear(pDataInfo->pRsp);
}
break;
@ -141,17 +141,17 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
if (pRsp->completed == 1) {
pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64
qDebug("%s fetch msg rsp from vgId:%d, clientId:0x%" PRIx64 " taskId:0x%" PRIx64
" execId:%d index:%d completed, blocks:%d, numOfRows:%" PRId64 ", rowsOfSource:%" PRIu64
", totalRows:%" PRIu64 ", total:%.2f Kb, try next %d/%" PRIzu,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks,
pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, i + 1,
totalSources);
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->clientId, pSource->taskId, pSource->execId, i,
pRsp->numOfBlocks, pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows,
pLoadInfo->totalSize / 1024.0, i + 1, totalSources);
} else {
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d blocks:%d, numOfRows:%" PRId64
", totalRows:%" PRIu64 ", total:%.2f Kb",
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRsp->numOfBlocks,
pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0);
qDebug("%s fetch msg rsp from vgId:%d, clientId:0x%" PRIx64 " taskId:0x%" PRIx64
" execId:%d blocks:%d, numOfRows:%" PRId64 ", totalRows:%" PRIu64 ", total:%.2f Kb",
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->clientId, pSource->taskId, pSource->execId,
pRsp->numOfBlocks, pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0);
}
taosMemoryFreeClear(pDataInfo->pRsp);
@ -640,9 +640,9 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas
if (pSource->localExec) {
SDataBuf pBuf = {0};
int32_t code =
(*pTaskInfo->localFetch.fp)(pTaskInfo->localFetch.handle, pSource->schedId, pTaskInfo->id.queryId,
pSource->taskId, 0, pSource->execId, &pBuf.pData, pTaskInfo->localFetch.explainRes);
int32_t code = (*pTaskInfo->localFetch.fp)(pTaskInfo->localFetch.handle, pSource->schedId, pTaskInfo->id.queryId,
pSource->clientId, pSource->taskId, 0, pSource->execId, &pBuf.pData,
pTaskInfo->localFetch.explainRes);
code = loadRemoteDataCallback(pWrapper, &pBuf, code);
QUERY_CHECK_CODE(code, lino, _end);
taosMemoryFree(pWrapper);
@ -650,6 +650,7 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas
SResFetchReq req = {0};
req.header.vgId = pSource->addr.nodeId;
req.sId = pSource->schedId;
req.clientId = pSource->clientId;
req.taskId = pSource->taskId;
req.queryId = pTaskInfo->id.queryId;
req.execId = pSource->execId;
@ -691,9 +692,10 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas
freeOperatorParam(req.pOpParam, OP_GET_PARAM);
qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %p, %d/%" PRIzu,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId,
pSource->execId, pExchangeInfo, sourceIndex, totalSources);
qDebug("%s build fetch msg and send to vgId:%d, ep:%s, clientId:0x%" PRIx64 " taskId:0x%" PRIx64
", execId:%d, %p, %d/%" PRIzu,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->clientId,
pSource->taskId, pSource->execId, pExchangeInfo, sourceIndex, totalSources);
// send the fetch remote task result reques
SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
@ -974,8 +976,9 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) {
}
if (pDataInfo->code != TSDB_CODE_SUCCESS) {
qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo),
pSource->addr.nodeId, pSource->taskId, pSource->execId, tstrerror(pDataInfo->code));
qError("%s vgId:%d, clientId:0x%" PRIx64 " taskID:0x%" PRIx64 " execId:%d error happens, code:%s",
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->clientId, pSource->taskId, pSource->execId,
tstrerror(pDataInfo->code));
pOperator->pTaskInfo->code = pDataInfo->code;
return pOperator->pTaskInfo->code;
}
@ -984,10 +987,10 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) {
SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo;
if (pRsp->numOfRows == 0) {
qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64
", totalRows:%" PRIu64 " try next",
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pExchangeInfo->current + 1,
pDataInfo->totalRows, pLoadInfo->totalRows);
qDebug("%s vgId:%d, clientId:0x%" PRIx64 " taskID:0x%" PRIx64
" execId:%d %d of total completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 " try next",
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->clientId, pSource->taskId, pSource->execId,
pExchangeInfo->current + 1, pDataInfo->totalRows, pLoadInfo->totalRows);
pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
pExchangeInfo->current += 1;
@ -1002,19 +1005,19 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) {
SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp;
if (pRsp->completed == 1) {
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64
qDebug("%s fetch msg rsp from vgId:%d, clientId:0x%" PRIx64 " taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64
", rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows,
pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1,
totalSources);
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->clientId, pSource->taskId, pSource->execId,
pRetrieveRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize,
pExchangeInfo->current + 1, totalSources);
pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
pExchangeInfo->current += 1;
} else {
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64 ", totalRows:%" PRIu64
", totalBytes:%" PRIu64,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows,
pLoadInfo->totalRows, pLoadInfo->totalSize);
qDebug("%s fetch msg rsp from vgId:%d, clientId:0x%" PRIx64 " taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64
", totalRows:%" PRIu64 ", totalBytes:%" PRIu64,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->clientId, pSource->taskId, pSource->execId,
pRetrieveRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize);
}
updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, startTs, pOperator);

15
source/libs/executor/src/executorInt.c
View File

@ -1083,18 +1083,13 @@ void cleanupBasicInfo(SOptrBasicInfo* pInfo) {
bool groupbyTbname(SNodeList* pGroupList) {
bool bytbname = false;
if (LIST_LENGTH(pGroupList) == 1) {
SNode* p = nodesListGetNode(pGroupList, 0);
if (!p) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));
return false;
}
if (p->type == QUERY_NODE_FUNCTION) {
// partition by tbname/group by tbname
bytbname = (strcmp(((struct SFunctionNode*)p)->functionName, "tbname") == 0);
SNode*pNode = NULL;
FOREACH(pNode, pGroupList) {
if (pNode->type == QUERY_NODE_FUNCTION) {
bytbname = (strcmp(((struct SFunctionNode*)pNode)->functionName, "tbname") == 0);
break;
}
}
return bytbname;
}

1
source/libs/executor/src/groupoperator.c
View File

@ -1326,7 +1326,6 @@ int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTag
int32_t winCode = TSDB_CODE_SUCCESS;
code = pAPI->streamStateGetParName(pState, groupId, &pValue, true, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
if (winCode != TSDB_CODE_SUCCESS) {
SSDataBlock* pTmpBlock = NULL;
code = blockCopyOneRow(pSrcBlock, rowId, &pTmpBlock);

59
source/libs/executor/src/scanoperator.c
View File

@ -289,6 +289,7 @@ static int32_t doSetTagColumnData(STableScanBase* pTableScanInfo, SSDataBlock* p
pTaskInfo, &pTableScanInfo->metaCache);
// ignore the table not exists error, since this table may have been dropped during the scan procedure.
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
if (pTaskInfo->streamInfo.pState) blockDataCleanup(pBlock);
code = 0;
}
}
@ -3038,10 +3039,6 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes,
pBlockInfo->rows, pTaskInfo, &pTableScanInfo->base.metaCache);
// ignore the table not exists error, since this table may have been dropped during the scan procedure.
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
code = 0;
}
if (code) {
blockDataFreeRes((SSDataBlock*)pBlock);
QUERY_CHECK_CODE(code, lino, _end);
@ -3535,6 +3532,46 @@ static int32_t copyGetResultBlock(SSDataBlock* dest, TSKEY start, TSKEY end) {
return appendDataToSpecialBlock(dest, &start, &end, NULL, NULL, NULL);
}
static int32_t deletePartName(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t *deleteNum) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
for (int32_t i = 0; i < pBlock->info.rows; i++) {
// uid is the same as gid
SColumnInfoData* pGpIdCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
SColumnInfoData* pTbnameCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
int64_t* gpIdCol = (int64_t*)pGpIdCol->pData;
void* pParName = NULL;
int32_t winCode = 0;
code = pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, gpIdCol[i],
&pParName, false, &winCode);
if (TSDB_CODE_SUCCESS == code && winCode != 0) {
qDebug("delete stream part Name for:%"PRId64 " not found", gpIdCol[i]);
colDataSetNULL(pTbnameCol, i);
continue;
}
(*deleteNum)++;
QUERY_CHECK_CODE(code, lino, _end);
char varTbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE + 1] = {0};
varDataSetLen(varTbName, strlen(pParName));
int64_t len = tsnprintf(varTbName + VARSTR_HEADER_SIZE, TSDB_TABLE_NAME_LEN + 1, "%s", pParName);
code = colDataSetVal(pTbnameCol, i, varTbName, false);
qDebug("delete stream part for:%"PRId64 " res tb: %s", gpIdCol[i], (char*)pParName);
pInfo->stateStore.streamStateFreeVal(pParName);
QUERY_CHECK_CODE(code, lino, _end);
code = pInfo->stateStore.streamStateDeleteParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, gpIdCol[i]);
QUERY_CHECK_CODE(code, lino, _end);
pBlock->info.id.groupId = gpIdCol[i];
// currently, only one valid row in pBlock
memcpy(pBlock->info.parTbName, varTbName + VARSTR_HEADER_SIZE, TSDB_TABLE_NAME_LEN + 1);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
static int32_t doStreamScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
// NOTE: this operator does never check if current status is done or not
int32_t code = TSDB_CODE_SUCCESS;
@ -3774,6 +3811,12 @@ FETCH_NEXT_BLOCK:
prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex, NULL);
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
} break;
case STREAM_DROP_CHILD_TABLE: {
int32_t deleteNum = 0;
code = deletePartName(pInfo, pBlock, &deleteNum);
QUERY_CHECK_CODE(code, lino, _end);
if (deleteNum == 0) goto FETCH_NEXT_BLOCK;
} break;
case STREAM_CHECKPOINT: {
qError("stream check point error. msg type: STREAM_INPUT__DATA_BLOCK");
} break;
@ -3915,7 +3958,13 @@ FETCH_NEXT_BLOCK:
}
code = setBlockIntoRes(pInfo, pRes, &pStreamInfo->fillHistoryWindow, false);
QUERY_CHECK_CODE(code, lino, _end);
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
pInfo->pRes->info.rows = 0;
code = TSDB_CODE_SUCCESS;
} else {
QUERY_CHECK_CODE(code, lino, _end);
}
if (pInfo->pRes->info.rows == 0) {
continue;
}

2
source/libs/executor/src/streamtimewindowoperator.c
View File

@ -5215,7 +5215,7 @@ static int32_t doStreamIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock** p
code = getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap);
QUERY_CHECK_CODE(code, lino, _end);
continue;
} else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {
} else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE || pBlock->info.type == STREAM_DROP_CHILD_TABLE) {
printDataBlock(pBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
(*ppRes) = pBlock;
return code;

8
source/libs/function/src/builtins.c
View File

@ -188,7 +188,11 @@ static int32_t countTrailingSpaces(const SValueNode* pVal, bool isLtrim) {
static int32_t addTimezoneParam(SNodeList* pList) {
char buf[TD_TIME_STR_LEN] = {0};
time_t t = taosTime(NULL);
time_t t;
int32_t code = taosTime(&t);
if (code != 0) {
return code;
}
struct tm tmInfo;
if (taosLocalTime(&t, &tmInfo, buf, sizeof(buf)) != NULL) {
(void)strftime(buf, sizeof(buf), "%z", &tmInfo);
@ -196,7 +200,7 @@ static int32_t addTimezoneParam(SNodeList* pList) {
int32_t len = (int32_t)strlen(buf);
SValueNode* pVal = NULL;
int32_t code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pVal);
code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pVal);
if (pVal == NULL) {
return code;
}

4
source/libs/function/src/tudf.c
View File

@ -64,6 +64,10 @@ static void udfWatchUdfd(void *args);
void udfUdfdExit(uv_process_t *process, int64_t exitStatus, int32_t termSignal) {
fnInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal);
SUdfdData *pData = process->data;
if(pData == NULL) {
fnError("udfd process data is NULL");
return;
}
if (exitStatus == 0 && termSignal == 0 || atomic_load_32(&pData->stopCalled)) {
fnInfo("udfd process exit due to SIGINT or dnode-mgmt called stop");
} else {

2
source/libs/function/src/udfd.c
View File

@ -1507,7 +1507,7 @@ static void removeListeningPipe() {
int err = uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL);
uv_fs_req_cleanup(&req);
if(err) {
fnError("remove listening pipe %s failed, reason:%s, lino:%d", global.listenPipeName, uv_strerror(err), __LINE__);
fnInfo("remove listening pipe %s : %s, lino:%d", global.listenPipeName, uv_strerror(err), __LINE__);
}
}

1
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -851,6 +851,7 @@ static int32_t slotDescCopy(const SSlotDescNode* pSrc, SSlotDescNode* pDst) {
static int32_t downstreamSourceCopy(const SDownstreamSourceNode* pSrc, SDownstreamSourceNode* pDst) {
COPY_OBJECT_FIELD(addr, sizeof(SQueryNodeAddr));
COPY_SCALAR_FIELD(clientId);
COPY_SCALAR_FIELD(taskId);
COPY_SCALAR_FIELD(schedId);
COPY_SCALAR_FIELD(execId);

7
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -5261,6 +5261,7 @@ static int32_t jsonToColumnDefNode(const SJson* pJson, void* pObj) {
}
static const char* jkDownstreamSourceAddr = "Addr";
static const char* jkDownstreamSourceClientId = "ClientId";
static const char* jkDownstreamSourceTaskId = "TaskId";
static const char* jkDownstreamSourceSchedId = "SchedId";
static const char* jkDownstreamSourceExecId = "ExecId";
@ -5270,6 +5271,9 @@ static int32_t downstreamSourceNodeToJson(const void* pObj, SJson* pJson) {
const SDownstreamSourceNode* pNode = (const SDownstreamSourceNode*)pObj;
int32_t code = tjsonAddObject(pJson, jkDownstreamSourceAddr, queryNodeAddrToJson, &pNode->addr);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDownstreamSourceClientId, pNode->clientId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDownstreamSourceTaskId, pNode->taskId);
}
@ -5290,6 +5294,9 @@ static int32_t jsonToDownstreamSourceNode(const SJson* pJson, void* pObj) {
SDownstreamSourceNode* pNode = (SDownstreamSourceNode*)pObj;
int32_t code = tjsonToObject(pJson, jkDownstreamSourceAddr, jsonToQueryNodeAddr, &pNode->addr);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetUBigIntValue(pJson, jkDownstreamSourceClientId, &pNode->clientId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetUBigIntValue(pJson, jkDownstreamSourceTaskId, &pNode->taskId);
}

6
source/libs/nodes/src/nodesMsgFuncs.c
View File

@ -1769,6 +1769,9 @@ static int32_t downstreamSourceNodeInlineToMsg(const void* pObj, STlvEncoder* pE
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeValueI32(pEncoder, pNode->fetchMsgType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeValueU64(pEncoder, pNode->clientId);
}
return code;
}
@ -1793,6 +1796,9 @@ static int32_t msgToDownstreamSourceNodeInlineToMsg(STlvDecoder* pDecoder, void*
if (TSDB_CODE_SUCCESS == code) {
code = tlvDecodeValueI32(pDecoder, &pNode->fetchMsgType);
}
if (TSDB_CODE_SUCCESS == code && !tlvDecodeEnd(pDecoder)) {
code = tlvDecodeValueU64(pDecoder, &pNode->clientId);
}
return code;
}

2
source/libs/parser/src/parInsertSql.c
View File

@ -246,7 +246,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, E
return code;
}
static int parseTimestampOrInterval(const char** end, SToken* pToken, int16_t timePrec, int64_t* ts, int64_t* interval,
static int32_t parseTimestampOrInterval(const char** end, SToken* pToken, int16_t timePrec, int64_t* ts, int64_t* interval,
SMsgBuf* pMsgBuf, bool* isTs) {
if (pToken->type == TK_NOW) {
*isTs = true;

3
source/libs/parser/src/parser.c
View File

@ -433,9 +433,6 @@ int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx
nodesDestroyNode(pQuery->pRoot);
pQuery->pRoot = NULL;
code = nodesCloneNode(pQuery->pPrepareRoot, &pQuery->pRoot);
if (NULL == pQuery->pRoot) {
code = code;
}
}
if (TSDB_CODE_SUCCESS == code) {
rewriteExprAlias(pQuery->pRoot);

27
source/libs/planner/src/planLogicCreater.c
View File

@ -1534,21 +1534,20 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
if (TSDB_CODE_SUCCESS == code) {
pSort->pSortKeys = NULL;
code = nodesCloneList(pSelect->pOrderByList, &pSort->pSortKeys);
if (NULL == pSort->pSortKeys) {
code = code;
}
SNode* pNode = NULL;
SOrderByExprNode* firstSortKey = (SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0);
if (isPrimaryKeySort(pSelect->pOrderByList)) pSort->node.outputTsOrder = firstSortKey->order;
if (firstSortKey->pExpr->type == QUERY_NODE_COLUMN) {
SColumnNode* pCol = (SColumnNode*)firstSortKey->pExpr;
int16_t projIdx = 1;
FOREACH(pNode, pSelect->pProjectionList) {
SExprNode* pExpr = (SExprNode*)pNode;
if (0 == strcmp(pCol->node.aliasName, pExpr->aliasName)) {
pCol->projIdx = projIdx; break;
if (NULL != pSort->pSortKeys) {
SNode* pNode = NULL;
SOrderByExprNode* firstSortKey = (SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0);
if (isPrimaryKeySort(pSelect->pOrderByList)) pSort->node.outputTsOrder = firstSortKey->order;
if (firstSortKey->pExpr->type == QUERY_NODE_COLUMN) {
SColumnNode* pCol = (SColumnNode*)firstSortKey->pExpr;
int16_t projIdx = 1;
FOREACH(pNode, pSelect->pProjectionList) {
SExprNode* pExpr = (SExprNode*)pNode;
if (0 == strcmp(pCol->node.aliasName, pExpr->aliasName)) {
pCol->projIdx = projIdx; break;
}
projIdx++;
}
projIdx++;
}
}
}

6
source/libs/planner/src/planSpliter.c
View File

@ -836,11 +836,9 @@ static int32_t stbSplSplitSessionForStream(SSplitContext* pCxt, SStableSplitInfo
nodesDestroyNode(pMergeWin->pTsEnd);
pMergeWin->pTsEnd = NULL;
code = nodesCloneNode(nodesListGetNode(pPartWin->node.pTargets, index), &pMergeWin->pTsEnd);
if (NULL == pMergeWin->pTsEnd) {
code = code;
}
}
code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartWindow);
if (TSDB_CODE_SUCCESS == code)
code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartWindow);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren,

64
source/libs/qworker/inc/qwInt.h
View File

@ -215,8 +215,8 @@ typedef struct SQWorkerMgmt {
#define QW_CTX_NOT_EXISTS_ERR_CODE(mgmt) \
(atomic_load_8(&(mgmt)->nodeStopped) ? TSDB_CODE_VND_STOPPED : TSDB_CODE_QRY_TASK_CTX_NOT_EXIST)
#define QW_FPARAMS_DEF SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId
#define QW_IDS() sId, qId, tId, rId, eId
#define QW_FPARAMS_DEF SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t cId, uint64_t tId, int64_t rId, int32_t eId
#define QW_IDS() sId, qId, cId, tId, rId, eId
#define QW_FPARAMS() mgmt, QW_IDS()
#define QW_STAT_INC(_item, _n) (void)atomic_add_fetch_64(&(_item), _n)
@ -257,18 +257,20 @@ typedef struct SQWorkerMgmt {
#define QW_FETCH_RUNNING(ctx) ((ctx)->inFetch)
#define QW_QUERY_NOT_STARTED(ctx) (QW_GET_PHASE(ctx) == -1)
#define QW_SET_QTID(id, qId, tId, eId) \
do { \
*(uint64_t *)(id) = (qId); \
*(uint64_t *)((char *)(id) + sizeof(qId)) = (tId); \
*(int32_t *)((char *)(id) + sizeof(qId) + sizeof(tId)) = (eId); \
#define QW_SET_QTID(id, qId, cId, tId, eId) \
do { \
*(uint64_t *)(id) = (qId); \
*(uint64_t *)((char *)(id) + sizeof(qId)) = (cId); \
*(uint64_t *)((char *)(id) + sizeof(qId) + sizeof(cId)) = (tId); \
*(int32_t *)((char *)(id) + sizeof(qId) + sizeof(cId) + sizeof(tId)) = (eId); \
} while (0)
#define QW_GET_QTID(id, qId, tId, eId) \
do { \
(qId) = *(uint64_t *)(id); \
(tId) = *(uint64_t *)((char *)(id) + sizeof(qId)); \
(eId) = *(int32_t *)((char *)(id) + sizeof(qId) + sizeof(tId)); \
#define QW_GET_QTID(id, qId, cId, tId, eId) \
do { \
(qId) = *(uint64_t *)(id); \
(cId) = *(uint64_t *)((char *)(id) + sizeof(qId)); \
(tId) = *(uint64_t *)((char *)(id) + sizeof(qId) + sizeof(cId)); \
(eId) = *(int32_t *)((char *)(id) + sizeof(qId) + sizeof(cId) + sizeof(tId)); \
} while (0)
#define QW_ERR_RET(c) \
@ -310,25 +312,31 @@ typedef struct SQWorkerMgmt {
#define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%" PRIx64 " " param, mgmt, sId, __VA_ARGS__)
#define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%" PRIx64 " " param, mgmt, sId, __VA_ARGS__)
#define QW_TASK_ELOG(param, ...) qError("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId, __VA_ARGS__)
#define QW_TASK_WLOG(param, ...) qWarn("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId, __VA_ARGS__)
#define QW_TASK_DLOG(param, ...) qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId, __VA_ARGS__)
#define QW_TASK_ELOG(param, ...) \
qError("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId, __VA_ARGS__)
#define QW_TASK_WLOG(param, ...) \
qWarn("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId, __VA_ARGS__)
#define QW_TASK_DLOG(param, ...) \
qDebug("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId, __VA_ARGS__)
#define QW_TASK_DLOGL(param, ...) \
qDebugL("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId, __VA_ARGS__)
qDebugL("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId, __VA_ARGS__)
#define QW_TASK_ELOG_E(param) qError("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId)
#define QW_TASK_WLOG_E(param) qWarn("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId)
#define QW_TASK_DLOG_E(param) qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId)
#define QW_TASK_ELOG_E(param) \
qError("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId)
#define QW_TASK_WLOG_E(param) \
qWarn("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId)
#define QW_TASK_DLOG_E(param) \
qDebug("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId)
#define QW_SCH_TASK_ELOG(param, ...) \
qError("QW:%p SID:0x%" PRIx64 ",qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, qId, tId, eId, \
__VA_ARGS__)
#define QW_SCH_TASK_WLOG(param, ...) \
qWarn("QW:%p SID:0x%" PRIx64 ",qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, qId, tId, eId, \
__VA_ARGS__)
#define QW_SCH_TASK_DLOG(param, ...) \
qDebug("QW:%p SID:0x%" PRIx64 ",qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, qId, tId, eId, \
__VA_ARGS__)
#define QW_SCH_TASK_ELOG(param, ...) \
qError("QW:%p SID:0x%" PRIx64 ",qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, \
qId, cId, tId, eId, __VA_ARGS__)
#define QW_SCH_TASK_WLOG(param, ...) \
qWarn("QW:%p SID:0x%" PRIx64 ",qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, qId, \
cId, tId, eId, __VA_ARGS__)
#define QW_SCH_TASK_DLOG(param, ...) \
qDebug("QW:%p SID:0x%" PRIx64 ",qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, \
qId, cId, tId, eId, __VA_ARGS__)
#define QW_LOCK_DEBUG(...) \
do { \

8
source/libs/qworker/src/qwDbg.c
View File

@ -96,14 +96,14 @@ void qwDbgDumpSchInfo(SQWorker *mgmt, SQWSchStatus *sch, int32_t i) {
int32_t taskNum = taosHashGetSize(sch->tasksHash);
QW_DLOG("***The %dth scheduler status, hbBrokenTs:%" PRId64 ",taskNum:%d", i, sch->hbBrokenTs, taskNum);
uint64_t qId, tId;
uint64_t qId, cId, tId;
int32_t eId;
SQWTaskStatus *pTask = NULL;
void *pIter = taosHashIterate(sch->tasksHash, NULL);
while (pIter) {
pTask = (SQWTaskStatus *)pIter;
void *key = taosHashGetKey(pIter, NULL);
QW_GET_QTID(key, qId, tId, eId);
QW_GET_QTID(key, qId, cId, tId, eId);
QW_TASK_DLOG("job refId:%" PRIx64 ", code:%x, task status:%d", pTask->refId, pTask->code, pTask->status);
@ -118,13 +118,13 @@ void qwDbgDumpTasksInfo(SQWorker *mgmt) {
int32_t i = 0;
SQWTaskCtx *ctx = NULL;
uint64_t qId, tId;
uint64_t qId, cId, tId;
int32_t eId;
void *pIter = taosHashIterate(mgmt->ctxHash, NULL);
while (pIter) {
ctx = (SQWTaskCtx *)pIter;
void *key = taosHashGetKey(pIter, NULL);
QW_GET_QTID(key, qId, tId, eId);
QW_GET_QTID(key, qId, cId, tId, eId);
QW_TASK_DLOG("%p lock:%x, phase:%d, type:%d, explain:%d, needFetch:%d, localExec:%d, queryMsgType:%d, "
"sId:%" PRId64 ", level:%d, queryGotData:%d, queryRsped:%d, queryEnd:%d, queryContinue:%d, queryInQueue:%d, "

13
source/libs/qworker/src/qwMsg.c
View File

@ -233,6 +233,7 @@ int32_t qwBuildAndSendDropMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) {
qMsg.header.contLen = 0;
qMsg.sId = sId;
qMsg.queryId = qId;
qMsg.clientId = cId;
qMsg.taskId = tId;
qMsg.refId = rId;
qMsg.execId = eId;
@ -284,6 +285,7 @@ int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) {
req->header.vgId = mgmt->nodeId;
req->sId = sId;
req->queryId = qId;
req->clientId = cId;
req->taskId = tId;
req->execId = eId;
@ -312,6 +314,7 @@ int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) {
qMsg.header.contLen = 0;
qMsg.sId = sId;
qMsg.queryId = qId;
qMsg.clientId = cId;
qMsg.taskId = tId;
qMsg.refId = rId;
qMsg.execId = eId;
@ -416,6 +419,7 @@ int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg, bool chkGran
uint64_t sId = msg.sId;
uint64_t qId = msg.queryId;
uint64_t cId = msg.clientId;
uint64_t tId = msg.taskId;
int64_t rId = msg.refId;
int32_t eId = msg.execId;
@ -447,6 +451,7 @@ int32_t qWorkerAbortPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg) {
uint64_t sId = msg.sId;
uint64_t qId = msg.queryId;
uint64_t cId = msg.clientId;
uint64_t tId = msg.taskId;
int64_t rId = msg.refId;
int32_t eId = msg.execId;
@ -479,6 +484,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int
uint64_t sId = msg.sId;
uint64_t qId = msg.queryId;
uint64_t cId = msg.clientId;
uint64_t tId = msg.taskId;
int64_t rId = msg.refId;
int32_t eId = msg.execId;
@ -524,6 +530,7 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in
uint64_t sId = msg->sId;
uint64_t qId = msg->queryId;
uint64_t cId = msg->clientId;
uint64_t tId = msg->taskId;
int64_t rId = 0;
int32_t eId = msg->execId;
@ -557,6 +564,7 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int
uint64_t sId = req.sId;
uint64_t qId = req.queryId;
uint64_t cId = req.clientId;
uint64_t tId = req.taskId;
int64_t rId = 0;
int32_t eId = req.execId;
@ -604,12 +612,14 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in
msg->sId = be64toh(msg->sId);
msg->queryId = be64toh(msg->queryId);
msg->clientId = be64toh(msg->clientId);
msg->taskId = be64toh(msg->taskId);
msg->refId = be64toh(msg->refId);
msg->execId = ntohl(msg->execId);
uint64_t sId = msg->sId;
uint64_t qId = msg->queryId;
uint64_t cId = msg->clientId;
uint64_t tId = msg->taskId;
int64_t rId = msg->refId;
int32_t eId = msg->execId;
@ -646,6 +656,7 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int6
uint64_t sId = msg.sId;
uint64_t qId = msg.queryId;
uint64_t cId = msg.clientId;
uint64_t tId = msg.taskId;
int64_t rId = msg.refId;
int32_t eId = msg.execId;
@ -684,6 +695,7 @@ int32_t qWorkerProcessNotifyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in
uint64_t sId = msg.sId;
uint64_t qId = msg.queryId;
uint64_t cId = msg.clientId;
uint64_t tId = msg.taskId;
int64_t rId = msg.refId;
int32_t eId = msg.execId;
@ -753,6 +765,7 @@ int32_t qWorkerProcessDeleteMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, SD
uint64_t sId = req.sId;
uint64_t qId = req.queryId;
uint64_t cId = req.clientId;
uint64_t tId = req.taskId;
int64_t rId = 0;
int32_t eId = -1;

37
source/libs/qworker/src/qwUtil.c
View File

@ -137,8 +137,8 @@ int32_t qwAcquireScheduler(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQWSchS
void qwReleaseScheduler(int32_t rwType, SQWorker *mgmt) { QW_UNLOCK(rwType, &mgmt->schLock); }
int32_t qwAcquireTaskStatus(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus *sch, SQWTaskStatus **task) {
char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0};
QW_SET_QTID(id, qId, tId, eId);
char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0};
QW_SET_QTID(id, qId, cId, tId, eId);
QW_LOCK(rwType, &sch->tasksLock);
*task = taosHashGet(sch->tasksHash, id, sizeof(id));
@ -153,8 +153,8 @@ int32_t qwAcquireTaskStatus(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus *sch, S
int32_t qwAddTaskStatusImpl(QW_FPARAMS_DEF, SQWSchStatus *sch, int32_t rwType, int32_t status, SQWTaskStatus **task) {
int32_t code = 0;
char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0};
QW_SET_QTID(id, qId, tId, eId);
char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0};
QW_SET_QTID(id, qId, cId, tId, eId);
SQWTaskStatus ntask = {0};
ntask.status = status;
@ -209,8 +209,8 @@ int32_t qwAddAcquireTaskStatus(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus *sch
void qwReleaseTaskStatus(int32_t rwType, SQWSchStatus *sch) { QW_UNLOCK(rwType, &sch->tasksLock); }
int32_t qwAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) {
char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0};
QW_SET_QTID(id, qId, tId, eId);
char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0};
QW_SET_QTID(id, qId, cId, tId, eId);
*ctx = taosHashAcquire(mgmt->ctxHash, id, sizeof(id));
if (NULL == (*ctx)) {
@ -222,8 +222,8 @@ int32_t qwAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) {
}
int32_t qwGetTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) {
char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0};
QW_SET_QTID(id, qId, tId, eId);
char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0};
QW_SET_QTID(id, qId, cId, tId, eId);
*ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id));
if (NULL == (*ctx)) {
@ -235,8 +235,8 @@ int32_t qwGetTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) {
}
int32_t qwAddTaskCtxImpl(QW_FPARAMS_DEF, bool acquire, SQWTaskCtx **ctx) {
char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0};
QW_SET_QTID(id, qId, tId, eId);
char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0};
QW_SET_QTID(id, qId, cId, tId, eId);
SQWTaskCtx nctx = {0};
@ -347,6 +347,7 @@ int32_t qwSendExplainResponse(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
(void)memcpy(pExec, taosArrayGet(execInfoList, 0), localRsp.rsp.numOfPlans * sizeof(SExplainExecInfo));
localRsp.rsp.subplanInfo = pExec;
localRsp.qId = qId;
localRsp.cId = cId;
localRsp.tId = tId;
localRsp.rId = rId;
localRsp.eId = eId;
@ -376,8 +377,8 @@ _return:
int32_t qwDropTaskCtx(QW_FPARAMS_DEF) {
char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0};
QW_SET_QTID(id, qId, tId, eId);
char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0};
QW_SET_QTID(id, qId, cId, tId, eId);
SQWTaskCtx octx;
SQWTaskCtx *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id));
@ -411,8 +412,8 @@ int32_t qwDropTaskStatus(QW_FPARAMS_DEF) {
SQWTaskStatus *task = NULL;
int32_t code = 0;
char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0};
QW_SET_QTID(id, qId, tId, eId);
char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0};
QW_SET_QTID(id, qId, cId, tId, eId);
if (qwAcquireScheduler(mgmt, sId, QW_WRITE, &sch)) {
QW_TASK_WLOG_E("scheduler does not exist");
@ -465,8 +466,8 @@ _return:
int32_t qwHandleDynamicTaskEnd(QW_FPARAMS_DEF) {
char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0};
QW_SET_QTID(id, qId, tId, eId);
char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0};
QW_SET_QTID(id, qId, cId, tId, eId);
SQWTaskCtx octx;
SQWTaskCtx *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id));
@ -588,14 +589,14 @@ void qwDestroyImpl(void *pMgmt) {
mgmt->hbTimer = NULL;
taosTmrCleanUp(mgmt->timer);
uint64_t qId, tId;
uint64_t qId, cId, tId;
int32_t eId;
void *pIter = taosHashIterate(mgmt->ctxHash, NULL);
while (pIter) {
SQWTaskCtx *ctx = (SQWTaskCtx *)pIter;
void *key = taosHashGetKey(pIter, NULL);
QW_GET_QTID(key, qId, tId, eId);
QW_GET_QTID(key, qId, cId, tId, eId);
qwFreeTaskCtx(ctx);
QW_TASK_DLOG_E("task ctx freed");

14
source/libs/qworker/src/qworker.c
View File

@ -19,7 +19,7 @@ SQWorkerMgmt gQwMgmt = {
};
void qwStopAllTasks(SQWorker *mgmt) {
uint64_t qId, tId, sId;
uint64_t qId, cId, tId, sId;
int32_t eId;
int64_t rId = 0;
int32_t code = TSDB_CODE_SUCCESS;
@ -28,7 +28,7 @@ void qwStopAllTasks(SQWorker *mgmt) {
while (pIter) {
SQWTaskCtx *ctx = (SQWTaskCtx *)pIter;
void *key = taosHashGetKey(pIter, NULL);
QW_GET_QTID(key, qId, tId, eId);
QW_GET_QTID(key, qId, cId, tId, eId);
QW_LOCK(QW_WRITE, &ctx->lock);
@ -288,7 +288,7 @@ int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo)
// TODO GET EXECUTOR API TO GET MORE INFO
QW_GET_QTID(key, status.queryId, status.taskId, status.execId);
QW_GET_QTID(key, status.queryId, status.clientId, status.taskId, status.execId);
status.status = taskStatus->status;
status.refId = taskStatus->refId;
@ -1473,8 +1473,8 @@ int32_t qWorkerGetStat(SReadHandle *handle, void *qWorkerMgmt, SQWorkerStat *pSt
return TSDB_CODE_SUCCESS;
}
int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId,
SQWMsg *qwMsg, SArray *explainRes) {
int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t cId, uint64_t tId, int64_t rId,
int32_t eId, SQWMsg *qwMsg, SArray *explainRes) {
SQWorker *mgmt = (SQWorker *)pMgmt;
int32_t code = 0;
SQWTaskCtx *ctx = NULL;
@ -1538,8 +1538,8 @@ _return:
QW_RET(code);
}
int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId,
void **pRsp, SArray *explainRes) {
int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t cId, uint64_t tId, int64_t rId,
int32_t eId, void **pRsp, SArray *explainRes) {
SQWorker *mgmt = (SQWorker *)pMgmt;
int32_t code = 0;
int32_t dataLen = 0;

38
source/libs/scheduler/inc/schInt.h
View File

@ -142,8 +142,9 @@ typedef struct SSchedulerCfg {
} SSchedulerCfg;
typedef struct SSchedulerMgmt {
uint64_t taskId; // sequential taksId
uint64_t sId; // schedulerId
uint64_t clientId; // unique clientId
uint64_t taskId; // sequential taksId
uint64_t sId; // schedulerId
SSchedulerCfg cfg;
bool exit;
int32_t jobRef;
@ -163,6 +164,7 @@ typedef struct SSchTaskCallbackParam {
SSchCallbackParamHeader head;
uint64_t queryId;
int64_t refId;
uint64_t clientId;
uint64_t taskId;
int32_t execId;
void *pTrans;
@ -222,6 +224,7 @@ typedef struct SSchTimerParam {
} SSchTimerParam;
typedef struct SSchTask {
uint64_t clientId; // current client id
uint64_t taskId; // task id
SRWLatch lock; // task reentrant lock
int32_t maxExecTimes; // task max exec times
@ -329,6 +332,7 @@ extern SSchedulerMgmt schMgmt;
#define SCH_LOCK_TASK(_task) SCH_LOCK(SCH_WRITE, &(_task)->lock)
#define SCH_UNLOCK_TASK(_task) SCH_UNLOCK(SCH_WRITE, &(_task)->lock)
#define SCH_CLIENT_ID(_task) ((_task) ? (_task)->clientId : -1)
#define SCH_TASK_ID(_task) ((_task) ? (_task)->taskId : -1)
#define SCH_TASK_EID(_task) ((_task) ? (_task)->execId : -1)
@ -449,21 +453,21 @@ extern SSchedulerMgmt schMgmt;
#define SCH_JOB_ELOG(param, ...) qError("qid:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__)
#define SCH_JOB_DLOG(param, ...) qDebug("qid:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__)
#define SCH_TASK_ELOG(param, ...) \
qError("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \
__VA_ARGS__)
#define SCH_TASK_DLOG(param, ...) \
qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \
__VA_ARGS__)
#define SCH_TASK_TLOG(param, ...) \
qTrace("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \
__VA_ARGS__)
#define SCH_TASK_DLOGL(param, ...) \
qDebugL("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \
__VA_ARGS__)
#define SCH_TASK_WLOG(param, ...) \
qWarn("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \
__VA_ARGS__)
#define SCH_TASK_ELOG(param, ...) \
qError("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_CLIENT_ID(pTask), \
SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), __VA_ARGS__)
#define SCH_TASK_DLOG(param, ...) \
qDebug("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_CLIENT_ID(pTask), \
SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), __VA_ARGS__)
#define SCH_TASK_TLOG(param, ...) \
qTrace("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_CLIENT_ID(pTask), \
SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), __VA_ARGS__)
#define SCH_TASK_DLOGL(param, ...) \
qDebugL("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_CLIENT_ID(pTask), \
SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), __VA_ARGS__)
#define SCH_TASK_WLOG(param, ...) \
qWarn("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_CLIENT_ID(pTask), \
SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), __VA_ARGS__)
#define SCH_SET_ERRNO(_err) \
do { \

14
source/libs/scheduler/src/schRemote.c
View File

@ -500,8 +500,8 @@ _return:
int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) {
SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param;
qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId,
code);
qDebug("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId,
pParam->clientId, pParam->taskId, code);
// called if drop task rsp received code
(void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error
@ -517,8 +517,8 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) {
int32_t schHandleNotifyCallback(void *param, SDataBuf *pMsg, int32_t code) {
SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param;
qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " task notify rsp received, code:0x%x", pParam->queryId, pParam->taskId,
code);
qDebug("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 " task notify rsp received, code:0x%x", pParam->queryId,
pParam->clientId, pParam->taskId, code);
if (pMsg) {
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
@ -595,6 +595,7 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bo
param->queryId = pJob->queryId;
param->refId = pJob->refId;
param->clientId = SCH_CLIENT_ID(pTask);
param->taskId = SCH_TASK_ID(pTask);
param->pTrans = pJob->conn.pTrans;
param->execId = pTask->execId;
@ -1138,6 +1139,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
req.header.vgId = addr->nodeId;
req.sId = schMgmt.sId;
req.queryId = pJob->queryId;
req.clientId = pTask->clientId;
req.taskId = pTask->taskId;
req.phyLen = pTask->msgLen;
req.sqlLen = strlen(pJob->sql);
@ -1171,6 +1173,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
qMsg.header.contLen = 0;
qMsg.sId = schMgmt.sId;
qMsg.queryId = pJob->queryId;
qMsg.clientId = pTask->clientId;
qMsg.taskId = pTask->taskId;
qMsg.refId = pJob->refId;
qMsg.execId = pTask->execId;
@ -1226,6 +1229,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
req.header.vgId = addr->nodeId;
req.sId = schMgmt.sId;
req.queryId = pJob->queryId;
req.clientId = pTask->clientId;
req.taskId = pTask->taskId;
req.execId = pTask->execId;
@ -1253,6 +1257,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
qMsg.header.contLen = 0;
qMsg.sId = schMgmt.sId;
qMsg.queryId = pJob->queryId;
qMsg.clientId = pTask->clientId;
qMsg.taskId = pTask->taskId;
qMsg.refId = pJob->refId;
qMsg.execId = *(int32_t*)param;
@ -1310,6 +1315,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
qMsg.header.contLen = 0;
qMsg.sId = schMgmt.sId;
qMsg.queryId = pJob->queryId;
qMsg.clientId = pTask->clientId;
qMsg.taskId = pTask->taskId;
qMsg.refId = pJob->refId;
qMsg.execId = pTask->execId;

25
source/libs/scheduler/src/schTask.c
View File

@ -66,6 +66,7 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *
pTask->execId = -1;
pTask->failedExecId = -2;
pTask->timeoutUsec = SCH_DEFAULT_TASK_TIMEOUT_USEC;
pTask->clientId = getClientId();
pTask->taskId = schGenTaskId();
schInitTaskRetryTimes(pJob, pTask, pLevel);
@ -305,6 +306,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) {
SCH_LOCK(SCH_WRITE, &parent->planLock);
SDownstreamSourceNode source = {
.type = QUERY_NODE_DOWNSTREAM_SOURCE,
.clientId = pTask->clientId,
.taskId = pTask->taskId,
.schedId = schMgmt.sId,
.execId = pTask->execId,
@ -996,8 +998,8 @@ int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList) {
int32_t code = 0;
qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s", pStatus->queryId, pStatus->taskId,
pStatus->execId, jobTaskStatusStr(pStatus->status));
qDebug("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s", pStatus->queryId,
pStatus->clientId, pStatus->taskId, pStatus->execId, jobTaskStatusStr(pStatus->status));
if (schProcessOnCbBegin(&pJob, &pTask, pStatus->queryId, pStatus->refId, pStatus->taskId)) {
continue;
@ -1043,13 +1045,14 @@ int32_t schHandleExplainRes(SArray *pExplainRes) {
continue;
}
qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg", localRsp->qId, localRsp->tId);
qDebug("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg",
localRsp->qId, localRsp->cId, localRsp->tId);
pJob = NULL;
(void)schAcquireJob(localRsp->rId, &pJob);
if (NULL == pJob) {
qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, localRsp->qId,
localRsp->tId, localRsp->rId);
qWarn("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64,
localRsp->qId, localRsp->cId, localRsp->tId, localRsp->rId);
SCH_ERR_JRET(TSDB_CODE_QRY_JOB_NOT_EXIST);
}
@ -1068,8 +1071,8 @@ int32_t schHandleExplainRes(SArray *pExplainRes) {
(void)schReleaseJob(pJob->refId);
qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", localRsp->qId,
localRsp->tId, code);
qDebug("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x",
localRsp->qId, localRsp->cId, localRsp->tId, code);
SCH_ERR_JRET(code);
@ -1147,8 +1150,8 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) {
}
}
SCH_ERR_JRET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId,
pTask->execId, &qwMsg, explainRes));
SCH_ERR_JRET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->clientId, pTask->taskId,
pJob->refId, pTask->execId, &qwMsg, explainRes));
if (SCH_IS_EXPLAIN_JOB(pJob)) {
SCH_ERR_RET(schHandleExplainRes(explainRes));
@ -1407,8 +1410,8 @@ int32_t schExecLocalFetch(SSchJob *pJob, SSchTask *pTask) {
}
}
SCH_ERR_JRET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId,
pTask->execId, &pRsp, explainRes));
SCH_ERR_JRET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->clientId, pTask->taskId,
pJob->refId, pTask->execId, &pRsp, explainRes));
if (SCH_IS_EXPLAIN_JOB(pJob)) {
SCH_ERR_RET(schHandleExplainRes(explainRes));

12
source/libs/scheduler/src/schUtil.c
View File

@ -293,6 +293,18 @@ void schCloseJobRef(void) {
}
}
int32_t initClientId(void) {
int32_t code = taosGetSystemUUIDU64(&schMgmt.clientId);
if (code != TSDB_CODE_SUCCESS) {
qError("failed to generate clientId since %s", tstrerror(code));
SCH_ERR_RET(code);
}
qInfo("initialize");
return TSDB_CODE_SUCCESS;
}
uint64_t getClientId(void) { return schMgmt.clientId; }
uint64_t schGenTaskId(void) { return atomic_add_fetch_64(&schMgmt.taskId, 1); }
#ifdef BUILD_NO_CALL

1
source/libs/stream/inc/streamBackendRocksdb.h
View File

@ -223,6 +223,7 @@ int32_t streamStateParTagGetKVByCur_rocksdb(SStreamStateCur* pCur, int64_t* pGro
// parname cf
int32_t streamStatePutParName_rocksdb(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]);
int32_t streamStateGetParName_rocksdb(SStreamState* pState, int64_t groupId, void** pVal);
int32_t streamStateDeleteParName_rocksdb(SStreamState* pState, int64_t groupId);
void streamStateDestroy_rocksdb(SStreamState* pState, bool remove);

6
source/libs/stream/src/streamBackendRocksdb.c
View File

@ -4432,6 +4432,12 @@ int32_t streamStateGetParName_rocksdb(SStreamState* pState, int64_t groupId, voi
return code;
}
int32_t streamStateDeleteParName_rocksdb(SStreamState* pState, int64_t groupId) {
int code = 0;
STREAM_STATE_DEL_ROCKSDB(pState, "parname", &groupId);
return code;
}
int32_t streamDefaultPut_rocksdb(SStreamState* pState, const void* key, void* pVal, int32_t pVLen) {
int code = 0;
STREAM_STATE_PUT_ROCKSDB(pState, "default", key, pVal, pVLen);

6
source/libs/stream/src/streamQueue.c
View File

@ -166,6 +166,8 @@ const char* streamQueueItemGetTypeStr(int32_t type) {
return "checkpoint-trigger";
case STREAM_INPUT__TRANS_STATE:
return "trans-state";
case STREAM_INPUT__REF_DATA_BLOCK:
return "ref-block";
default:
return "datablock";
}
@ -211,7 +213,7 @@ EExtractDataCode streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueIte
// do not merge blocks for sink node and check point data block
int8_t type = qItem->type;
if (type == STREAM_INPUT__CHECKPOINT || type == STREAM_INPUT__CHECKPOINT_TRIGGER ||
type == STREAM_INPUT__TRANS_STATE) {
type == STREAM_INPUT__TRANS_STATE || type == STREAM_INPUT__REF_DATA_BLOCK) {
const char* p = streamQueueItemGetTypeStr(type);
if (*pInput == NULL) {
@ -504,4 +506,4 @@ void streamTaskPutbackToken(STokenBucket* pBucket) {
// size in KB
void streamTaskConsumeQuota(STokenBucket* pBucket, int32_t bytes) { pBucket->quotaRemain -= SIZE_IN_MiB(bytes); }
void streamTaskInputFail(SStreamTask* pTask) { atomic_store_8(&pTask->inputq.status, TASK_INPUT_STATUS__FAILED); }
void streamTaskInputFail(SStreamTask* pTask) { atomic_store_8(&pTask->inputq.status, TASK_INPUT_STATUS__FAILED); }

12
source/libs/stream/src/streamState.c
View File

@ -525,6 +525,18 @@ _end:
return code;
}
int32_t streamStateDeleteParName(SStreamState* pState, int64_t groupId) {
int32_t code = tSimpleHashRemove(pState->parNameMap, &groupId, sizeof(int64_t));
if (TSDB_CODE_SUCCESS != code) {
qWarn("failed to remove parname from cache, code:%d", code);
}
code = streamStateDeleteParName_rocksdb(pState, groupId);
if (TSDB_CODE_SUCCESS != code) {
qWarn("failed to remove parname from rocksdb, code:%d", code);
}
return TSDB_CODE_SUCCESS;
}
void streamStateDestroy(SStreamState* pState, bool remove) {
streamFileStateDestroy(pState->pFileState);
// streamStateDestroy_rocksdb(pState, remove);

2
source/libs/wal/src/walRead.c
View File

@ -89,6 +89,8 @@ int32_t walNextValidMsg(SWalReader *pReader) {
if (type == TDMT_VND_SUBMIT || ((type == TDMT_VND_DELETE) && (pReader->cond.deleteMsg == 1)) ||
(IS_META_MSG(type) && pReader->cond.scanMeta)) {
TAOS_RETURN(walFetchBody(pReader));
} else if (type == TDMT_VND_DROP_TABLE && pReader->cond.scanDropCtb) {
TAOS_RETURN(walFetchBody(pReader));
} else {
TAOS_CHECK_RETURN(walSkipFetchBody(pReader));

13
source/os/src/osTime.c
View File

@ -81,6 +81,7 @@ static const char *am_pm[2] = {"AM", "PM"};
#endif
char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) {
if (!buf || !fmt || !tm) return NULL;
#ifdef WINDOWS
char c;
const char *bp;
@ -345,6 +346,9 @@ char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) {
}
int32_t taosGetTimeOfDay(struct timeval *tv) {
if (tv == NULL) {
return TSDB_CODE_INVALID_PARA;
}
int32_t code = 0;
#ifdef WINDOWS
LARGE_INTEGER t;
@ -365,12 +369,15 @@ int32_t taosGetTimeOfDay(struct timeval *tv) {
#endif
}
time_t taosTime(time_t *t) {
int32_t taosTime(time_t *t) {
if (t == NULL) {
return TSDB_CODE_INVALID_PARA;
}
time_t r = time(t);
if (r == (time_t)-1) {
terrno = TAOS_SYSTEM_ERROR(errno);
return TAOS_SYSTEM_ERROR(errno);
}
return r;
return 0;
}
/*

29
source/util/src/tlog.c
View File

@ -154,16 +154,26 @@ static int32_t taosStartLog() {
return 0;
}
static void getDay(char *buf, int32_t bufSize) {
time_t t = taosTime(NULL);
static int32_t getDay(char *buf, int32_t bufSize) {
time_t t;
int32_t code = taosTime(&t);
if(code != 0) {
return code;
}
struct tm tmInfo;
if (taosLocalTime(&t, &tmInfo, buf, bufSize) != NULL) {
TAOS_UNUSED(strftime(buf, bufSize, "%Y-%m-%d", &tmInfo));
}
return 0;
}
static int64_t getTimestampToday() {
time_t t = taosTime(NULL);
time_t t;
int32_t code = taosTime(&t);
if (code != 0) {
uError("failed to get time, reason:%s", tstrerror(code));
return 0;
}
struct tm tm;
if (taosLocalTime(&t, &tm, NULL, 0) == NULL) {
return 0;
@ -203,7 +213,11 @@ int32_t taosInitSlowLog() {
char name[PATH_MAX + TD_TIME_STR_LEN] = {0};
char day[TD_TIME_STR_LEN] = {0};
getDay(day, sizeof(day));
int32_t code = getDay(day, sizeof(day));
if (code != 0) {
(void)printf("failed to get day, reason:%s\n", tstrerror(code));
return code;
}
(void)snprintf(name, PATH_MAX + TD_TIME_STR_LEN, "%s.%s", tsLogObj.slowLogName, day);
tsLogObj.timestampToday = getTimestampToday();
@ -434,7 +448,12 @@ static void taosOpenNewSlowLogFile() {
atomic_store_32(&tsLogObj.slowHandle->lock, 0);
char day[TD_TIME_STR_LEN] = {0};
getDay(day, sizeof(day));
int32_t code = getDay(day, sizeof(day));
if (code != 0) {
uError("failed to get day, reason:%s", tstrerror(code));
(void)taosThreadMutexUnlock(&tsLogObj.logMutex);
return;
}
TdFilePtr pFile = NULL;
char name[PATH_MAX + TD_TIME_STR_LEN] = {0};
(void)snprintf(name, PATH_MAX + TD_TIME_STR_LEN, "%s.%s", tsLogObj.slowLogName, day);

1
tests/parallel_test/cases.task
View File

@ -438,6 +438,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show_tag_index.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/information_schema.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/grant.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -R
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/and_or_for_byte.py

406
tests/script/api/sameReqidTest.c Normal file
View File

@ -0,0 +1,406 @@
// sample code to verify multiple queries with the same reqid
// to compile: gcc -o sameReqdiTest sameReqidTest.c -ltaos
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include "taos.h"
#define NUM_ROUNDS 10
#define CONST_REQ_ID 12345
#define TEST_DB "test"
#define CHECK_CONDITION(condition, prompt, errstr) \
do { \
if (!(condition)) { \
printf("\033[31m[%s:%d] failed to " prompt ", reason: %s\033[0m\n", __func__, __LINE__, errstr); \
exit(EXIT_FAILURE); \
} \
} while (0)
#define CHECK_RES(res, prompt) CHECK_CONDITION(taos_errno(res) == 0, prompt, taos_errstr(res))
#define CHECK_CODE(code, prompt) CHECK_CONDITION(code == 0, prompt, taos_errstr(NULL))
static TAOS* getNewConnection() {
const char* host = "127.0.0.1";
const char* user = "root";
const char* passwd = "taosdata";
TAOS* taos = NULL;
taos_options(TSDB_OPTION_TIMEZONE, "GMT-8");
taos = taos_connect(host, user, passwd, "", 0);
CHECK_CONDITION(taos != NULL, "connect to db", taos_errstr(NULL));
return taos;
}
static void prepareData(TAOS* taos) {
TAOS_RES* res = NULL;
int32_t code = 0;
res = taos_query(taos, "create database if not exists " TEST_DB " precision 'ns'");
CHECK_RES(res, "create database");
taos_free_result(res);
usleep(100000);
code = taos_select_db(taos, TEST_DB);
CHECK_CODE(code, "switch to database");
res = taos_query(taos, "create table if not exists meters(ts timestamp, a int) tags(area int)");
CHECK_RES(res, "create stable meters");
taos_free_result(res);
res = taos_query(taos, "create table if not exists t0 using meters tags(0)");
CHECK_RES(res, "create table t0");
taos_free_result(res);
res = taos_query(taos, "create table if not exists t1 using meters tags(1)");
CHECK_RES(res, "create table t1");
taos_free_result(res);
res = taos_query(taos, "create table if not exists t2 using meters tags(2)");
CHECK_RES(res, "create table t2");
taos_free_result(res);
res = taos_query(taos, "create table if not exists t3 using meters tags(3)");
CHECK_RES(res, "create table t3");
taos_free_result(res);
res = taos_query(taos, "create table if not exists t4 using meters tags(4)");
CHECK_RES(res, "create table t4");
taos_free_result(res);
res = taos_query(taos, "create table if not exists t5 using meters tags(5)");
CHECK_RES(res, "create table t5");
taos_free_result(res);
res = taos_query(taos, "create table if not exists t6 using meters tags(6)");
CHECK_RES(res, "create table t6");
taos_free_result(res);
res = taos_query(taos, "create table if not exists t7 using meters tags(7)");
CHECK_RES(res, "create table t7");
taos_free_result(res);
res = taos_query(taos, "create table if not exists t8 using meters tags(8)");
CHECK_RES(res, "create table t8");
taos_free_result(res);
res = taos_query(taos, "create table if not exists t9 using meters tags(9)");
CHECK_RES(res, "create table t9");
taos_free_result(res);
res = taos_query(taos,
"insert into t0 values('2020-01-01 00:00:00.000', 0)"
" ('2020-01-01 00:01:00.000', 0)"
" ('2020-01-01 00:02:00.000', 0)"
" t1 values('2020-01-01 00:00:00.000', 1)"
" ('2020-01-01 00:01:00.000', 1)"
" ('2020-01-01 00:02:00.000', 1)"
" ('2020-01-01 00:03:00.000', 1)"
" t2 values('2020-01-01 00:00:00.000', 2)"
" ('2020-01-01 00:01:00.000', 2)"
" ('2020-01-01 00:01:01.000', 2)"
" ('2020-01-01 00:01:02.000', 2)"
" t3 values('2020-01-01 00:01:02.000', 3)"
" t4 values('2020-01-01 00:01:02.000', 4)"
" t5 values('2020-01-01 00:01:02.000', 5)"
" t6 values('2020-01-01 00:01:02.000', 6)"
" t7 values('2020-01-01 00:01:02.000', 7)"
" t8 values('2020-01-01 00:01:02.000', 8)"
" t9 values('2020-01-01 00:01:02.000', 9)");
CHECK_RES(res, "insert into meters");
CHECK_CONDITION(taos_affected_rows(res), "insert into meters", "insufficient count");
taos_free_result(res);
res = taos_query(
taos,
"create table if not exists m1 (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 "
"double, bin binary(40), blob nchar(10))");
CHECK_RES(res, "create table m1");
taos_free_result(res);
usleep(1000000);
}
static void verifySchemaLess(TAOS* taos) {
TAOS_RES* res = NULL;
char* lines[] = {
"st,t1=3i64,t2=4f64,t3=L\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000",
"st,t1=4i64,t3=L\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64 1626006833640000000",
"st,t2=5f64,t3=L\"ste\" c1=4i64,c2=true,c3=L\"iam\" 1626056811823316532",
"st,t1=4i64,t2=5f64,t3=L\"t4\" c1=3i64,c3=L\"passitagain\",c2=true,c4=5f64 1626006833642000000",
"st,t2=5f64,t3=L\"ste2\" c3=L\"iamszhou\",c2=false 1626056811843316532",
"st,t2=5f64,t3=L\"ste2\" c3=L\"iamszhou\",c2=false,c5=5f64,c6=7u64,c7=32i32,c8=88.88f32 1626056812843316532",
"st,t1=4i64,t3=L\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64,c6=7u64 "
"1626006933640000000",
"st,t1=4i64,t3=L\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64,c6=7u64 "
"1626006933640000000",
"st,t1=4i64,t3=L\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 "
"1626006933641000000"};
res = taos_schemaless_insert_with_reqid(taos, lines, sizeof(lines) / sizeof(char*), TSDB_SML_LINE_PROTOCOL,
TSDB_SML_TIMESTAMP_NANO_SECONDS, CONST_REQ_ID);
CHECK_RES(res, "insert schema-less data");
printf("successfully inserted %d rows\n", taos_affected_rows(res));
taos_free_result(res);
}
static int32_t printResult(TAOS_RES* res, int32_t nlimit) {
TAOS_ROW row = NULL;
TAOS_FIELD* fields = NULL;
int32_t numFields = 0;
int32_t nRows = 0;
numFields = taos_num_fields(res);
fields = taos_fetch_fields(res);
while ((nlimit-- > 0) && (row = taos_fetch_row(res))) {
char temp[256] = {0};
taos_print_row(temp, row, fields, numFields);
puts(temp);
nRows++;
}
return nRows;
}
static void verifyQuery(TAOS* taos) {
TAOS_RES* res = NULL;
res = taos_query_with_reqid(taos, "select * from meters", CONST_REQ_ID);
CHECK_RES(res, "select from meters");
printResult(res, INT32_MAX);
taos_free_result(res);
res = taos_query_with_reqid(taos, "select * from t0", CONST_REQ_ID);
CHECK_RES(res, "select from t0");
printResult(res, INT32_MAX);
taos_free_result(res);
res = taos_query_with_reqid(taos, "select * from t1", CONST_REQ_ID);
CHECK_RES(res, "select from t1");
printResult(res, INT32_MAX);
taos_free_result(res);
res = taos_query_with_reqid(taos, "select * from t2", CONST_REQ_ID);
CHECK_RES(res, "select from t2");
printResult(res, INT32_MAX);
taos_free_result(res);
res = taos_query_with_reqid(taos, "select * from t3", CONST_REQ_ID);
CHECK_RES(res, "select from t3");
printResult(res, INT32_MAX);
taos_free_result(res);
printf("succeed to read from meters\n");
}
void retrieveCallback(void* param, TAOS_RES* res, int32_t nrows) {
if (nrows == 0) {
taos_free_result(res);
} else {
printResult(res, nrows);
taos_fetch_rows_a(res, retrieveCallback, param);
}
}
void selectCallback(void* param, TAOS_RES* res, int32_t code) {
CHECK_CODE(code, "read async from table");
taos_fetch_rows_a(res, retrieveCallback, param);
}
static void verifyQueryAsync(TAOS* taos) {
taos_query_a_with_reqid(taos, "select *from meters", selectCallback, NULL, CONST_REQ_ID);
taos_query_a_with_reqid(taos, "select *from t0", selectCallback, NULL, CONST_REQ_ID);
taos_query_a_with_reqid(taos, "select *from t1", selectCallback, NULL, CONST_REQ_ID);
taos_query_a_with_reqid(taos, "select *from t2", selectCallback, NULL, CONST_REQ_ID);
taos_query_a_with_reqid(taos, "select *from t3", selectCallback, NULL, CONST_REQ_ID);
sleep(1);
}
void veriryStmt(TAOS* taos) {
// insert 10 records
struct {
int64_t ts[10];
int8_t b[10];
int8_t v1[10];
int16_t v2[10];
int32_t v4[10];
int64_t v8[10];
float f4[10];
double f8[10];
char bin[10][40];
char blob[10][80];
} v;
int32_t* t8_len = malloc(sizeof(int32_t) * 10);
int32_t* t16_len = malloc(sizeof(int32_t) * 10);
int32_t* t32_len = malloc(sizeof(int32_t) * 10);
int32_t* t64_len = malloc(sizeof(int32_t) * 10);
int32_t* float_len = malloc(sizeof(int32_t) * 10);
int32_t* double_len = malloc(sizeof(int32_t) * 10);
int32_t* bin_len = malloc(sizeof(int32_t) * 10);
int32_t* blob_len = malloc(sizeof(int32_t) * 10);
TAOS_STMT* stmt = taos_stmt_init_with_reqid(taos, CONST_REQ_ID);
TAOS_MULTI_BIND params[10];
char is_null[10] = {0};
params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP;
params[0].buffer_length = sizeof(v.ts[0]);
params[0].buffer = v.ts;
params[0].length = t64_len;
params[0].is_null = is_null;
params[0].num = 10;
params[1].buffer_type = TSDB_DATA_TYPE_BOOL;
params[1].buffer_length = sizeof(v.b[0]);
params[1].buffer = v.b;
params[1].length = t8_len;
params[1].is_null = is_null;
params[1].num = 10;
params[2].buffer_type = TSDB_DATA_TYPE_TINYINT;
params[2].buffer_length = sizeof(v.v1[0]);
params[2].buffer = v.v1;
params[2].length = t8_len;
params[2].is_null = is_null;
params[2].num = 10;
params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT;
params[3].buffer_length = sizeof(v.v2[0]);
params[3].buffer = v.v2;
params[3].length = t16_len;
params[3].is_null = is_null;
params[3].num = 10;
params[4].buffer_type = TSDB_DATA_TYPE_INT;
params[4].buffer_length = sizeof(v.v4[0]);
params[4].buffer = v.v4;
params[4].length = t32_len;
params[4].is_null = is_null;
params[4].num = 10;
params[5].buffer_type = TSDB_DATA_TYPE_BIGINT;
params[5].buffer_length = sizeof(v.v8[0]);
params[5].buffer = v.v8;
params[5].length = t64_len;
params[5].is_null = is_null;
params[5].num = 10;
params[6].buffer_type = TSDB_DATA_TYPE_FLOAT;
params[6].buffer_length = sizeof(v.f4[0]);
params[6].buffer = v.f4;
params[6].length = float_len;
params[6].is_null = is_null;
params[6].num = 10;
params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE;
params[7].buffer_length = sizeof(v.f8[0]);
params[7].buffer = v.f8;
params[7].length = double_len;
params[7].is_null = is_null;
params[7].num = 10;
params[8].buffer_type = TSDB_DATA_TYPE_BINARY;
params[8].buffer_length = sizeof(v.bin[0]);
params[8].buffer = v.bin;
params[8].length = bin_len;
params[8].is_null = is_null;
params[8].num = 10;
params[9].buffer_type = TSDB_DATA_TYPE_NCHAR;
params[9].buffer_length = sizeof(v.blob[0]);
params[9].buffer = v.blob;
params[9].length = blob_len;
params[9].is_null = is_null;
params[9].num = 10;
int32_t code = taos_stmt_prepare(
stmt, "insert into ? (ts, b, v1, v2, v4, v8, f4, f8, bin, blob) values(?,?,?,?,?,?,?,?,?,?)", 0);
CHECK_CODE(code, "taos_stmt_prepare");
code = taos_stmt_set_tbname(stmt, "m1");
CHECK_CODE(code, "taos_stmt_set_tbname");
int64_t ts = 1591060628000000000;
for (int i = 0; i < 10; ++i) {
v.ts[i] = ts;
ts += 1000000;
is_null[i] = 0;
v.b[i] = (int8_t)i % 2;
v.v1[i] = (int8_t)i;
v.v2[i] = (int16_t)(i * 2);
v.v4[i] = (int32_t)(i * 4);
v.v8[i] = (int64_t)(i * 8);
v.f4[i] = (float)(i * 40);
v.f8[i] = (double)(i * 80);
for (int j = 0; j < sizeof(v.bin[0]); ++j) {
v.bin[i][j] = (char)(i + '0');
}
strcpy(v.blob[i], "一二三四五六七八九十");
t8_len[i] = sizeof(int8_t);
t16_len[i] = sizeof(int16_t);
t32_len[i] = sizeof(int32_t);
t64_len[i] = sizeof(int64_t);
float_len[i] = sizeof(float);
double_len[i] = sizeof(double);
bin_len[i] = sizeof(v.bin[0]);
blob_len[i] = (int32_t)strlen(v.blob[i]);
}
code = taos_stmt_bind_param_batch(stmt, params);
CHECK_CODE(code, "taos_stmt_bind_param_batch");
code = taos_stmt_add_batch(stmt);
CHECK_CODE(code, "taos_stmt_add_batch");
code = taos_stmt_execute(stmt);
CHECK_CODE(code, "taos_stmt_execute");
taos_stmt_close(stmt);
free(t8_len);
free(t16_len);
free(t32_len);
free(t64_len);
free(float_len);
free(double_len);
free(bin_len);
free(blob_len);
}
int main(int argc, char* argv[]) {
TAOS* taos = NULL;
int32_t code = 0;
taos = getNewConnection();
taos_select_db(taos, TEST_DB);
CHECK_CODE(code, "switch to database");
printf("************ prepare data *************\n");
prepareData(taos);
for (int32_t i = 0; i < NUM_ROUNDS; ++i) {
printf("************ verify schema-less *************\n");
verifySchemaLess(taos);
printf("************ verify query *************\n");
verifyQuery(taos);
printf("********* verify async query **********\n");
verifyQueryAsync(taos);
printf("********* verify stmt query **********\n");
veriryStmt(taos);
printf("done\n");
}
taos_close(taos);
taos_cleanup();
return 0;
}

222
tests/system-test/0-others/grant.py Normal file
View File

@ -0,0 +1,222 @@
from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE
import taos
import sys
import time
import os
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
from util.dnodes import TDDnodes
from util.dnodes import TDDnode
import time
import socket
import subprocess
class MyDnodes(TDDnodes):
def __init__(self ,dnodes_lists):
super(MyDnodes,self).__init__()
self.dnodes = dnodes_lists # dnode must be TDDnode instance
if platform.system().lower() == 'windows':
self.simDeployed = True
else:
self.simDeployed = False
class TDTestCase:
noConn = True
def getTDinternalPath():
path_parts = os.getcwd().split(os.sep)
try:
tdinternal_index = path_parts.index("TDinternal")
except ValueError:
raise ValueError("The specified directory 'TDinternal' was not found in the path.")
return os.sep.join(path_parts[:tdinternal_index + 1])
def init(self, conn, logSql, replicaVar=1):
tdLog.debug(f"start to excute {__file__}")
self.TDDnodes = None
self.depoly_cluster(5)
self.master_dnode = self.TDDnodes.dnodes[0]
self.host=self.master_dnode.cfgDict["fqdn"]
conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir)
tdSql.init(conn1.cursor(), True)
self.TDinternal = TDTestCase.getTDinternalPath()
self.workPath = os.path.join(self.TDinternal, "debug", "build", "bin")
tdLog.info(self.workPath)
def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__))
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
if ("taosd" in files or "taosd.exe" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root) - len("/build/bin")]
break
return buildPath
def depoly_cluster(self ,dnodes_nums):
testCluster = False
valgrind = 0
hostname = socket.gethostname()
dnodes = []
start_port = 6030
for num in range(1, dnodes_nums+1):
dnode = TDDnode(num)
dnode.addExtraCfg("firstEp", f"{hostname}:{start_port}")
dnode.addExtraCfg("fqdn", f"{hostname}")
dnode.addExtraCfg("serverPort", f"{start_port + (num-1)*100}")
dnode.addExtraCfg("monitorFqdn", hostname)
dnode.addExtraCfg("monitorPort", 7043)
dnodes.append(dnode)
self.TDDnodes = MyDnodes(dnodes)
self.TDDnodes.init("")
self.TDDnodes.setTestCluster(testCluster)
self.TDDnodes.setValgrind(valgrind)
self.TDDnodes.setAsan(tdDnodes.getAsan())
self.TDDnodes.stopAll()
for dnode in self.TDDnodes.dnodes:
self.TDDnodes.deploy(dnode.index,{})
for dnode in self.TDDnodes.dnodes:
self.TDDnodes.starttaosd(dnode.index)
# create cluster
for dnode in self.TDDnodes.dnodes[1:]:
# print(dnode.cfgDict)
dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"]
dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0]
dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1]
cmd = f"{self.getBuildPath()}/build/bin/taos -h {dnode_first_host} -P {dnode_first_port} -s \"create dnode \\\"{dnode_id}\\\"\""
print(cmd)
os.system(cmd)
time.sleep(2)
tdLog.info(" create cluster done! ")
def s0_five_dnode_one_mnode(self):
tdSql.query("select * from information_schema.ins_dnodes;")
tdSql.checkData(0,1,'%s:6030'%self.host)
tdSql.checkData(4,1,'%s:6430'%self.host)
tdSql.checkData(0,4,'ready')
tdSql.checkData(4,4,'ready')
tdSql.query("select * from information_schema.ins_mnodes;")
tdSql.checkData(0,1,'%s:6030'%self.host)
tdSql.checkData(0,2,'leader')
tdSql.checkData(0,3,'ready')
tdSql.error("create mnode on dnode 1;")
tdSql.error("drop mnode on dnode 1;")
tdSql.execute("create database if not exists audit");
tdSql.execute("use audit");
tdSql.execute("create table operations(ts timestamp, c0 int primary key,c1 bigint,c2 int,c3 float,c4 double) tags(t0 bigint unsigned)");
tdSql.execute("create table t_operations_abc using operations tags(1)");
tdSql.execute("drop database if exists db")
tdSql.execute("create database if not exists db replica 1")
tdSql.execute("use db")
tdSql.execute("create table stb0(ts timestamp, c0 int primary key,c1 bigint,c2 int,c3 float,c4 double) tags(t0 bigint unsigned)");
tdSql.execute("create table ctb0 using stb0 tags(0)");
tdSql.execute("create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, count(c2) c2 , sum(c3) c3 , max(c4) c4 from stb0 interval(10s)");
tdSql.execute("create topic topic_stb_column as select ts, c3 from stb0");
tdSql.execute("create topic topic_stb_all as select ts, c1, c2, c3 from stb0");
tdSql.execute("create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb0");
tdSql.execute("create view view1 as select * from stb0");
def getConnection(self, dnode):
host = dnode.cfgDict["fqdn"]
port = dnode.cfgDict["serverPort"]
config_dir = dnode.cfgDir
return taos.connect(host=host, port=int(port), config=config_dir)
def s1_check_alive(self):
# check cluster alive
tdLog.printNoPrefix("======== test cluster alive: ")
tdSql.checkDataLoop(0, 0, 1, "show cluster alive;", 20, 0.5)
tdSql.query("show db.alive;")
tdSql.checkData(0, 0, 1)
def s2_check_show_grants_ungranted(self):
tdLog.printNoPrefix("======== test show grants ungranted: ")
self.infoPath = os.path.join(self.workPath, ".clusterInfo")
infoFile = open(self.infoPath, "w")
try:
tdSql.query(f'select create_time,expire_time,version from information_schema.ins_cluster;')
tdSql.checkEqual(len(tdSql.queryResult), 1)
infoFile.write(";".join(map(str, tdSql.queryResult[0])) + "\n")
tdSql.query(f'show cluster machines;')
tdSql.checkEqual(len(tdSql.queryResult), 1)
infoFile.write(";".join(map(str,tdSql.queryResult[0])) + "\n")
tdSql.query(f'show grants;')
tdSql.checkEqual(len(tdSql.queryResult), 1)
infoFile.write(";".join(map(str,tdSql.queryResult[0])) + "\n")
tdSql.query(f'show grants full;')
tdSql.checkEqual(len(tdSql.queryResult), 31)
if infoFile:
infoFile.flush()
files_and_dirs = os.listdir(f'{self.workPath}')
print(f"files_and_dirs: {files_and_dirs}")
process = subprocess.Popen(f'{self.workPath}{os.sep}grantTest', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, error = process.communicate()
output = output.decode(encoding="utf-8")
error = error.decode(encoding="utf-8")
print(f"code: {process.returncode}")
print(f"error:\n{error}")
tdSql.checkEqual(process.returncode, 0)
tdSql.checkEqual(error, "")
lines = output.splitlines()
for line in lines:
if line.startswith("code:"):
fields = line.split(":")
tdSql.error(f"{fields[2]}", int(fields[1]), fields[3])
except Exception as e:
if os.path.exists(self.infoPath):
os.remove(self.infoPath)
raise Exception(repr(e))
finally:
if infoFile:
infoFile.close()
def s3_check_show_grants_granted(self):
tdLog.printNoPrefix("======== test show grants granted: ")
try:
process = subprocess.Popen(f'{self.workPath}{os.sep}grantTest 1', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, error = process.communicate()
output = output.decode(encoding="utf-8")
error = error.decode(encoding="utf-8")
print(f"code: {process.returncode}")
print(f"error:\n{error}")
print(f"output:\n{output}")
tdSql.checkEqual(process.returncode, 0)
except Exception as e:
raise Exception(repr(e))
finally:
if os.path.exists(self.infoPath):
os.remove(self.infoPath)
def run(self):
# print(self.master_dnode.cfgDict)
# keep the order of following steps
self.s0_five_dnode_one_mnode()
self.s1_check_alive()
self.s2_check_show_grants_ungranted()
self.s3_check_show_grants_granted()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

1
tests/system-test/0-others/information_schema.py
View File

@ -299,6 +299,7 @@ class TDTestCase:
'oracle':'Oracle',
'mssql':'SqlServer',
'mongodb':'MongoDB',
'csv':'CSV',
}
tdSql.execute('drop database if exists db2')

42
tests/system-test/2-query/tsma.py
View File

@ -604,7 +604,7 @@ class TSMATestSQLGenerator:
class TDTestCase:
updatecfgDict = {'asynclog': 0, 'ttlUnit': 1, 'ttlPushInterval': 5, 'ratioOfVnodeStreamThrea': 4, 'maxTsmaNum': 3}
updatecfgDict = {'asynclog': 0, 'ttlUnit': 1, 'ttlPushInterval': 5, 'ratioOfVnodeStreamThrea': 4, 'maxTsmaNum': 3, 'debugFlag': 143}
def __init__(self):
self.vgroups = 4
@ -804,8 +804,8 @@ class TDTestCase:
self.tsma_tester.check_sql(ctx.sql, ctx)
def test_query_with_tsma(self):
self.create_tsma('tsma1', 'test', 'meters', ['avg(c1)', 'avg(c2)'], '5m')
self.create_tsma('tsma2', 'test', 'meters', ['avg(c1)', 'avg(c2)'], '30m')
self.create_tsma('tsma1', 'test', 'meters', ['avg(c1)', 'avg(c2)', 'count(ts)'], '5m')
self.create_tsma('tsma2', 'test', 'meters', ['avg(c1)', 'avg(c2)', 'count(ts)'], '30m')
self.create_tsma('tsma5', 'test', 'norm_tb', ['avg(c1)', 'avg(c2)'], '10m')
self.test_query_with_tsma_interval()
@ -1237,7 +1237,41 @@ class TDTestCase:
clust_dnode_nums = len(cluster_dnode_list)
if clust_dnode_nums > 1:
self.test_redistribute_vgroups()
tdSql.execute("drop tsma test.tsma5")
for _ in range(4):
self.test_td_32519()
def test_td_32519(self):
self.create_recursive_tsma('tsma1', 'tsma_r', 'test', '1h', 'meters', ['avg(c1)', 'avg(c2)', 'count(ts)'])
tdSql.execute('INSERT INTO test.t1 VALUES("2024-10-24 11:45:00", 1,1,1,1,1,1,1, "a", "a")', queryTimes=1)
tdSql.execute('INSERT INTO test.t1 VALUES("2024-10-24 11:55:00", 2,1,1,1,1,1,1, "a", "a")', queryTimes=1)
tdSql.execute('DROP TABLE test.t1', queryTimes=1)
self.wait_query_err('desc test.`404e15422d96c8b5de9603c2296681b1`', 10, -2147473917)
self.wait_query_err('desc test.`82b56f091c4346369da0af777c3e580d`', 10, -2147473917)
self.wait_query_err('desc test.`163b7c69922cf6d83a98bfa44e52dade`', 10, -2147473917)
tdSql.execute('CREATE TABLE test.t1 USING test.meters TAGS(1, "a", "b", 1,1,1)')
tdSql.execute('INSERT INTO test.t1 VALUES("2024-10-24 11:59:00", 3,1,1,1,1,1,1, "a", "a")', queryTimes=1)
tdSql.execute('INSERT INTO test.t1 VALUES("2024-10-24 12:10:00", 4,1,1,1,1,1,1, "a", "a")', queryTimes=1)
tdSql.execute('INSERT INTO test.t1 VALUES("2024-10-24 12:20:00", 5,1,1,1,1,1,1, "a", "a")', queryTimes=1)
tdSql.execute('FLUSH DATABASE test', queryTimes=1)
tdSql.query('SELECT * FROM test.t1', queryTimes=1)
tdSql.checkRows(3)
sql = 'SELECT * FROM test.`404e15422d96c8b5de9603c2296681b1`'
self.wait_query(sql, 3, 20) ## tsma1 output ctb for t1
tdSql.query(sql, queryTimes=1)
tdSql.checkData(0,1, 1)
tdSql.checkData(1,1, 1)
tdSql.checkData(2,1, 1)
#sql = 'select * from test.`82b56f091c4346369da0af777c3e580d`'
#self.wait_query(sql, 2, 10) ## tsma2 output ctb for t1
#tdSql.query(sql, queryTimes=1)
#tdSql.checkData(0, 1, 1)
#tdSql.checkData(1, 1, 2)
sql = 'select * from test.`163b7c69922cf6d83a98bfa44e52dade`'
self.wait_query(sql, 2, 20) ## tsma_r output ctb for t1
tdSql.checkData(0, 1, 1)
self.drop_tsma('tsma_r', 'test')
def test_create_tsma(self):
function_name = sys._getframe().f_code.co_name
tdLog.debug(f'-----{function_name}------')