diff --git a/docs/examples/JDBC/SpringJdbcTemplate/pom.xml b/docs/examples/JDBC/SpringJdbcTemplate/pom.xml index 6e4941b4f1..34719dc135 100644 --- a/docs/examples/JDBC/SpringJdbcTemplate/pom.xml +++ b/docs/examples/JDBC/SpringJdbcTemplate/pom.xml @@ -22,19 +22,19 @@ org.springframework spring-context - 5.2.8.RELEASE + 5.3.39 org.springframework spring-jdbc - 5.1.9.RELEASE + 5.3.39 org.springframework spring-test - 5.1.9.RELEASE + 5.3.39 @@ -47,7 +47,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.0.0 + 3.4.0 diff --git a/docs/examples/JDBC/springbootdemo/pom.xml b/docs/examples/JDBC/springbootdemo/pom.xml index ee15f6013e..ba75cdcec3 100644 --- a/docs/examples/JDBC/springbootdemo/pom.xml +++ b/docs/examples/JDBC/springbootdemo/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 2.2.1.RELEASE + 2.6.15 com.taosdata.example @@ -65,6 +65,8 @@ spring-boot-starter-aop + + com.taosdata.jdbc taos-jdbcdriver diff --git a/docs/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java b/docs/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java index 53edaa5796..df7aa32158 100644 --- a/docs/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java +++ b/docs/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java @@ -3,9 +3,10 @@ package com.taosdata.example.springbootdemo; import org.mybatis.spring.annotation.MapperScan; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.data.jdbc.JdbcRepositoriesAutoConfiguration; @MapperScan(basePackages = {"com.taosdata.example.springbootdemo"}) -@SpringBootApplication +@SpringBootApplication(exclude = {JdbcRepositoriesAutoConfiguration.class}) public class SpringbootdemoApplication { public static void main(String[] args) { diff --git a/docs/examples/JDBC/springbootdemo/src/main/resources/application.properties b/docs/examples/JDBC/springbootdemo/src/main/resources/application.properties index 00a06a5098..2b231f403b 100644 --- a/docs/examples/JDBC/springbootdemo/src/main/resources/application.properties +++ b/docs/examples/JDBC/springbootdemo/src/main/resources/application.properties @@ -15,6 +15,8 @@ spring.datasource.druid.max-wait=30000 spring.datasource.druid.validation-query=select SERVER_VERSION(); spring.aop.auto=true spring.aop.proxy-target-class=true + +spring.jooq.sql-dialect= #mybatis mybatis.mapper-locations=classpath:mapper/*.xml logging.level.com.taosdata.jdbc.springbootdemo.dao=debug diff --git a/docs/examples/JDBC/taosdemo/pom.xml b/docs/examples/JDBC/taosdemo/pom.xml index ab5912aa9e..c36973947b 100644 --- a/docs/examples/JDBC/taosdemo/pom.xml +++ b/docs/examples/JDBC/taosdemo/pom.xml @@ -10,7 +10,7 @@ Demo project for TDengine - 5.3.27 + 5.3.39 @@ -130,6 +130,7 @@ org.apache.maven.plugins maven-compiler-plugin + 3.13.0 8 8 diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/QueryService.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/QueryService.java index ab0a1125d2..33e8845d12 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/QueryService.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/QueryService.java @@ -37,7 +37,7 @@ public class QueryService { stmt.execute("use " + dbName); ResultSet rs = stmt.executeQuery("show stables"); while (rs.next()) { - String name = rs.getString("name"); + String name = rs.getString("stable_name"); sqls.add("select count(*) from " + dbName + "." + name); sqls.add("select first(*) from " + dbName + "." + name); sqls.add("select last(*) from " + dbName + "." + name); diff --git a/docs/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/DatabaseServiceTest.java b/docs/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/DatabaseServiceTest.java index 621ba7df5d..e8c6432e38 100644 --- a/docs/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/DatabaseServiceTest.java +++ b/docs/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/DatabaseServiceTest.java @@ -1,10 +1,14 @@ package com.taosdata.taosdemo.service; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.junit.BeforeClass; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; public class DatabaseServiceTest { - private DatabaseService service; + + private static DatabaseService service; @Test public void testCreateDatabase1() { @@ -20,4 +24,16 @@ public class DatabaseServiceTest { public void useDatabase() { service.useDatabase("test"); } + + @BeforeClass + public static void beforeClass() throws ClassNotFoundException { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + HikariConfig config = new HikariConfig(); + config.setJdbcUrl("jdbc:TAOS://127.0.0.1:6030/?charset=UTF-8&locale=en_US.UTF-8&timezone=UTC-8"); + config.setUsername("root"); + config.setPassword("taosdata"); + HikariDataSource dataSource = new HikariDataSource(config); + service = new DatabaseService(dataSource); + } + } \ No newline at end of file diff --git a/docs/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/QueryServiceTest.java b/docs/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/QueryServiceTest.java index f2ad25710c..989aa094f3 100644 --- a/docs/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/QueryServiceTest.java +++ b/docs/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/QueryServiceTest.java @@ -15,7 +15,7 @@ public class QueryServiceTest { @Test public void generateSuperTableQueries() { - String[] sqls = queryService.generateSuperTableQueries("restful_test"); + String[] sqls = queryService.generateSuperTableQueries("test"); for (String sql : sqls) { System.out.println(sql); } @@ -23,8 +23,8 @@ public class QueryServiceTest { @Test public void querySuperTable() { - String[] sqls = queryService.generateSuperTableQueries("restful_test"); - queryService.querySuperTable(sqls, 1000, 10, 10); + String[] sqls = queryService.generateSuperTableQueries("test"); + queryService.querySuperTable(sqls, 100, 3, 3); } @BeforeClass diff --git a/docs/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/SuperTableServiceTest.java b/docs/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/SuperTableServiceTest.java index 33e52af1ea..4edba8c518 100644 --- a/docs/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/SuperTableServiceTest.java +++ b/docs/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/SuperTableServiceTest.java @@ -3,6 +3,9 @@ package com.taosdata.taosdemo.service; import com.taosdata.taosdemo.domain.FieldMeta; import com.taosdata.taosdemo.domain.SuperTableMeta; import com.taosdata.taosdemo.domain.TagMeta; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.junit.BeforeClass; import org.junit.Test; import java.util.ArrayList; @@ -10,7 +13,7 @@ import java.util.List; public class SuperTableServiceTest { - private SuperTableService service; + private static SuperTableService service; @Test public void testCreate() { @@ -29,4 +32,15 @@ public class SuperTableServiceTest { service.create(superTableMeta); } + @BeforeClass + public static void beforeClass() throws ClassNotFoundException { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + HikariConfig config = new HikariConfig(); + config.setJdbcUrl("jdbc:TAOS://127.0.0.1:6030/?charset=UTF-8&locale=en_US.UTF-8&timezone=UTC-8"); + config.setUsername("root"); + config.setPassword("taosdata"); + HikariDataSource dataSource = new HikariDataSource(config); + service = new SuperTableService(dataSource); + } + } \ No newline at end of file diff --git a/docs/examples/java/src/main/java/com/taos/example/DruidDemo.java b/docs/examples/java/src/main/java/com/taos/example/DruidDemo.java index a366efd419..8fbf33ef6d 100644 --- a/docs/examples/java/src/main/java/com/taos/example/DruidDemo.java +++ b/docs/examples/java/src/main/java/com/taos/example/DruidDemo.java @@ -1,4 +1,4 @@ -package com.taosdata.example; +package com.taos.example; import com.alibaba.druid.pool.DruidDataSource; @@ -8,11 +8,11 @@ import java.sql.Statement; public class DruidDemo { // ANCHOR: connection_pool public static void main(String[] args) throws Exception { - String url = "jdbc:TAOS://127.0.0.1:6030/log"; + String url = "jdbc:TAOS-WS://127.0.0.1:6041/log"; DruidDataSource dataSource = new DruidDataSource(); // jdbc properties - dataSource.setDriverClassName("com.taosdata.jdbc.TSDBDriver"); + dataSource.setDriverClassName("com.taosdata.jdbc.ws.WebSocketDriver"); dataSource.setUrl(url); dataSource.setUsername("root"); dataSource.setPassword("taosdata"); diff --git a/docs/examples/java/src/main/java/com/taos/example/GeometryDemo.java b/docs/examples/java/src/main/java/com/taos/example/GeometryDemo.java index 036125e7ea..4045a96642 100644 --- a/docs/examples/java/src/main/java/com/taos/example/GeometryDemo.java +++ b/docs/examples/java/src/main/java/com/taos/example/GeometryDemo.java @@ -144,8 +144,9 @@ public class GeometryDemo { private void executeQuery(String sql) { long start = System.currentTimeMillis(); - try (Statement statement = connection.createStatement()) { - ResultSet resultSet = statement.executeQuery(sql); + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + long end = System.currentTimeMillis(); printSql(sql, true, (end - start)); diff --git a/docs/examples/java/src/main/java/com/taos/example/HikariDemo.java b/docs/examples/java/src/main/java/com/taos/example/HikariDemo.java index 50b20fdb0c..e7a90276d7 100644 --- a/docs/examples/java/src/main/java/com/taos/example/HikariDemo.java +++ b/docs/examples/java/src/main/java/com/taos/example/HikariDemo.java @@ -1,4 +1,4 @@ -package com.taosdata.example; +package com.taos.example; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; @@ -11,7 +11,7 @@ public class HikariDemo { public static void main(String[] args) throws Exception { HikariConfig config = new HikariConfig(); // jdbc properties - config.setJdbcUrl("jdbc:TAOS://127.0.0.1:6030/log"); + config.setJdbcUrl("jdbc:TAOS-WS://127.0.0.1:6041/log"); config.setUsername("root"); config.setPassword("taosdata"); // connection pool configurations diff --git a/docs/examples/java/src/main/java/com/taos/example/TelnetLineProtocolExample.java b/docs/examples/java/src/main/java/com/taos/example/TelnetLineProtocolExample.java index 4c9368288d..11b234c4e0 100644 --- a/docs/examples/java/src/main/java/com/taos/example/TelnetLineProtocolExample.java +++ b/docs/examples/java/src/main/java/com/taos/example/TelnetLineProtocolExample.java @@ -39,6 +39,7 @@ public class TelnetLineProtocolExample { createDatabase(conn); SchemalessWriter writer = new SchemalessWriter(conn); writer.write(lines, SchemalessProtocolType.TELNET, SchemalessTimestampType.NOT_CONFIGURED); + writer.close(); } } diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index 7486b3b043..c47831dde3 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -228,4 +228,35 @@ PAUSE STREAM [IF EXISTS] stream_name; RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; ``` -没有指定 IF EXISTS，如果该 stream 不存在，则报错。如果存在，则恢复流计算。指定了 IF EXISTS，如果 stream 不存在，则返回成功。如果存在，则恢复流计算。如果指定 IGNORE UNTREATED，则恢复流计算时，忽略流计算暂停期间写入的数据。 \ No newline at end of file +没有指定 IF EXISTS，如果该 stream 不存在，则报错。如果存在，则恢复流计算。指定了 IF EXISTS，如果 stream 不存在，则返回成功。如果存在，则恢复流计算。如果指定 IGNORE UNTREATED，则恢复流计算时，忽略流计算暂停期间写入的数据。 + +### 流计算升级故障恢复 + +升级 TDengine 后，如果流计算不兼容，需要删除流计算，然后重新创建流计算。步骤如下： + +1.修改 taos.cfg，添加 disableStream 1 + +2.重启 taosd。如果启动失败，修改 stream 目录的名称，避免 taosd 启动的时候尝试加载 stream 目录下的流计算数据信息。不使用删除操作避免误操作导致的风险。需要修改的文件夹：$dataDir/vnode/vnode*/tq/stream，$dataDir 指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似 vnode1 、vnode2...vnode* 的目录，全部需要修改里面的 tq/stream 目录的名字，改为 tq/stream.bk + +3.启动 taos + +```sql +drop stream xxxx; ---- xxx 指stream name +flush database stream_source_db; ---- 流计算读取数据的超级表所在的 database +flush database stream_dest_db; ---- 流计算写入数据的超级表所在的 database +``` + +举例： + +```sql +create stream streams1 into test1.streamst as select _wstart, count(a) c1 from test.st interval(1s) ; +drop database streams1; +flush database test; +flush database test1; +``` + +4.关闭 taosd + +5.修改 taos.cfg，去掉 disableStream 1，或将 disableStream 改为 0 + +6.启动 taosd \ No newline at end of file diff --git a/docs/zh/06-advanced/06-data-analysis/03-anomaly-detection.md b/docs/zh/06-advanced/06-data-analysis/03-anomaly-detection.md index bdfa455ae3..1eae6088ce 100644 --- a/docs/zh/06-advanced/06-data-analysis/03-anomaly-detection.md +++ b/docs/zh/06-advanced/06-data-analysis/03-anomaly-detection.md @@ -1,6 +1,6 @@ --- -title: "Anomaly-detection" -sidebar_label: "Anomaly-detection" +title: "异常检测算法" +sidebar_label: "异常检测算法" --- 本节讲述异常检测算法模型的使用方法。 diff --git a/docs/zh/06-advanced/06-data-analysis/addins.md b/docs/zh/06-advanced/06-data-analysis/addins.md index c0b8921718..de24eb15a6 100644 --- a/docs/zh/06-advanced/06-data-analysis/addins.md +++ b/docs/zh/06-advanced/06-data-analysis/addins.md @@ -1,13 +1,27 @@ --- -title: "addins" -sidebar_label: "addins" +title: "算法开发者指南" +sidebar_label: "算法开发者指南" --- 本节说明如何将自己开发的预测算法和异常检测算法整合到 TDengine 分析平台，并能够通过 SQL 语句进行调用。 ## 目录结构 -![数据分析功能架构图](./pic/dir.png) +```bash +. +├── cfg +├── model +│   └── ac_detection +├── release +├── script +└── taosanalytics + ├── algo + │   ├── ad + │   └── fc + ├── misc + └── test + +``` |目录|说明| |---|---| diff --git a/docs/zh/06-advanced/06-data-analysis/analysis-preprocess.md b/docs/zh/06-advanced/06-data-analysis/analysis-preprocess.md new file mode 100644 index 0000000000..4d65b247c2 --- /dev/null +++ b/docs/zh/06-advanced/06-data-analysis/analysis-preprocess.md @@ -0,0 +1,23 @@ +--- +title: "数据分析预处理" +sidebar_label: "数据分析预处理" +--- + +## 时序数据分析功能 + +### 白噪声检查 + +分析平台提供的 Restful 服务要求输入的时间序列不能是白噪声时间序列（White Noise Data, WND）和随机数序列 , 因此针对所有数据均默认进行白噪声检查。当前白噪声检查采用通行的 `Ljung-Box` 检验，`Ljung-Box` 统计量检查过程需要遍历整个输入序列并进行计算。 +如果用户能够明确输入序列一定不是白噪声序列，那么可以通过输入参数，指定预测之前忽略该检查，从而节省分析过程的 CPU 计算资源。 +同时支持独立地针对输入序列进行白噪声检测（该检测功能暂不独立对外开放）。 + + +### 数据重采样和时间戳对齐 + +分析平台支持将输入数据进行重采样预处理，从而确保输出结果按照用户指定的等间隔进行处理。处理过程分为两种类别： + +- 数据时间戳对齐。由于真实数据可能并非严格按照查询指定的时间戳输入。此时分析平台会自动将数据的时间间隔按照指定的时间间隔进行对齐。例如输入时间序列 [11, 22, 29, 41]，用户指定时间间隔为 10，该序列将被对齐重整为以下序列 [10, 20, 30, 40]。 +- 数据时间重采样。用户输入时间序列的采样频率超过了输出结果的频率，例如输入时间序列的采样频率是 5，输出结果的频率是 10，输入时间序列 [0， 5， 10， 15， 20， 25， 30] 将被重采用为间隔 为 10 的序列 [0, 10, 20，30]，[5, 15, 25] 处的数据将被丢弃。 + +需要注意的是，数据输入平台不支持缺失数据补齐后进行的预测分析，如果输入时间序列数据 [11, 22, 29, 49]，并且用户要求的时间间隔为 10，重整对齐后的序列是 [10, 20, 30, 50] 那么该序列进行预测分析将返回错误。 + diff --git a/docs/zh/06-advanced/06-data-analysis/index.md b/docs/zh/06-advanced/06-data-analysis/index.md index 2cbea1caba..c04b0b47bd 100644 --- a/docs/zh/06-advanced/06-data-analysis/index.md +++ b/docs/zh/06-advanced/06-data-analysis/index.md @@ -1,190 +1,21 @@ --- -sidebar_label: 数据分析 -title: 数据分析功能 +sidebar_label: TDgpt +title: TDgpt --- ## 概述 -ANode（Analysis Node）是 TDengine 提供数据分析功能的扩展组件，通过 Restful 接口提供分析服务，拓展 TDengine 的功能，支持时间序列高级分析。 -ANode 是无状态的数据分析节点，集群中可以存在多个 ANode 节点，相互之间没有关联。将 ANode 注册到 TDengine 集群以后，通过 SQL 语句即可调用并完成时序分析任务。 -下图是数据分析的技术架构示意图。 +TDgpt 是 TDengine Enterprise 中针对时序数据提供高级分析功能的企业级组件，能够独立于 TDengine 主进程部署和运行，不消耗和占用 TDengine 主进程的资源，通过内置接口向 TDengine 提供运行时动态扩展的高级时序数据分析功能。TDgpt 具有服务无状态、功能易扩展、快速弹性部署、应用轻量化、高安全性等特点。 +TDgpt 运行在部署于 TDengine 集群中的 Analysis Node (ANode)中。每个 TDengine 集群中可以部署一个或若干个 ANode 节点，不同的 ANode 节点之间不相关，无同步或协同的要求。ANode 注册到 TDengine 集群以后，就可以通过内部接口提供服务。TDgpt 提供的高级时序数据分析服务可分为时序数据异常检测和时序数据预测分析两个类别。 -![数据分析功能架构图](./pic/data-analysis.png) +如下是数据分析的技术架构示意图。 -## 安装部署 -### 环境准备 -ANode 要求节点上准备有 Python 3.10 及以上版本，以及相应的 Python 包自动安装组件 Pip，同时请确保能够正常连接互联网。 +TDgpt架构图 -### 安装及卸载 -使用专门的 ANode 安装包 TDengine-enterprise-anode-1.x.x.tar.gz 进行 ANode 的安装部署工作，安装过程与 TDengine 的安装流程一致。 - -```bash -tar -xzvf TDengine-enterprise-anode-1.0.0.tar.gz -cd TDengine-enterprise-anode-1.0.0 -sudo ./install.sh -``` - -卸载 ANode，执行命令 `rmtaosanode` 即可。 - -### 其他 -为了避免 ANode 安装后影响目标节点现有的 Python 库。 ANode 使用 Python 虚拟环境运行，安装后的默认 Python 目录处于 `/var/lib/taos/taosanode/venv/`。为了避免反复安装虚拟环境带来的开销，卸载 ANode 并不会自动删除该虚拟环境，如果您确认不需要 Python 的虚拟环境，可以手动删除。 - -## 启动及停止服务 -安装 ANode 以后，可以使用 `systemctl` 来管理 ANode 的服务。使用如下命令可以启动/停止/检查状态。 - -```bash -systemctl start taosanoded -systemctl stop taosanoded -systemctl status taosanoded -``` - -## 目录及配置说明 -|目录/文件|说明| -|---------------|------| -|/usr/local/taos/taosanode/bin|可执行文件目录| -|/usr/local/taos/taosanode/resource|资源文件目录，链接到文件夹 /var/lib/taos/taosanode/resource/| -|/usr/local/taos/taosanode/lib|库文件目录| -|/var/lib/taos/taosanode/model/|模型文件目录，链接到文件夹 /var/lib/taos/taosanode/model| -|/var/log/taos/taosanode/|日志文件目录| -|/etc/taos/taosanode.ini|配置文件| - -### 配置说明 - -Anode 提供的 RestFul 服务使用 uWSGI 驱动，因此 ANode 和 uWSGI 的配置信息存放在同一个配置文件中，具体如下： - -```ini -[uwsgi] -# charset -env = LC_ALL = en_US.UTF-8 - -# ip:port -http = 127.0.0.1:6050 - -# the local unix socket file than communicate to Nginx -#socket = 127.0.0.1:8001 -#socket-timeout = 10 - -# base directory -chdir = /usr/local/taos/taosanode/lib - -# initialize python file -wsgi-file = /usr/local/taos/taosanode/lib/taos/app.py - -# call module of uWSGI -callable = app - -# auto remove unix Socket and pid file when stopping -vacuum = true - -# socket exec model -#chmod-socket = 664 - -# uWSGI pid -uid = root - -# uWSGI gid -gid = root - -# main process -master = true - -# the number of worker processes -processes = 2 - -# pid file -pidfile = /usr/local/taos/taosanode/taosanode.pid - -# enable threads -enable-threads = true - -# the number of threads for each process -threads = 4 - -# memory useage report -memory-report = true - -# smooth restart -reload-mercy = 10 - -# conflict with systemctl, so do NOT uncomment this -# daemonize = /var/log/taos/taosanode/taosanode.log - -# log directory -logto = /var/log/taos/taosanode/taosanode.log - -# wWSGI monitor port -stats = 127.0.0.1:8387 - -# python virtual environment directory -virtualenv = /usr/local/taos/taosanode/venv/ - -[taosanode] -# default app log file -app-log = /var/log/taos/taosanode/taosanode.app.log - -# model storage directory -model-dir = /usr/local/taos/taosanode/model/ - -# default log level -log-level = DEBUG - -# draw the query results -draw-result = 0 -``` - -**提示** -请勿设置 `daemonize` 参数，该参数会导致 uWSGI 与 systemctl 冲突，从而无法正常启动。 +通过注册指令语句，将 ANode 注册到 MNode 中就加入到 TDengine 集群，查询会按需向其请求数据分析服务。请求服务通过 VNode 直接向 ANode 发起，用户则可以通过 SQL 语句直接调用 ANode 提供的服务。 -## ANode 基本操作 -### 管理 ANode -#### 创建 ANode -```sql -CREATE ANODE {node_url} -``` -node_url 是提供服务的 ANode 的 IP 和 PORT, 例如：`create anode 'http://localhost:6050'`。启动 ANode 以后如果不注册到 TDengine 集群中，则无法提供正常的服务。不建议 ANode 注册到两个或多个集群中。 -#### 查看 ANode -列出集群中所有的数据分析节点，包括其 `FQDN`, `PORT`, `STATUS`。 -```sql -SHOW ANODES; -``` - -#### 查看提供的时序数据分析服务 - -```SQL -SHOW ANODES FULL; -``` - -#### 强制刷新集群中的分析算法缓存 -```SQL -UPDATE ANODE {node_id} -UPDATE ALL ANODES -``` - -#### 删除 ANode -```sql -DROP ANODE {anode_id} -``` -删除 ANode 只是将 ANode 从 TDengine 集群中删除，管理 ANode 的启停仍然需要使用`systemctl`命令。 - -### 时序数据分析功能 - -#### 白噪声检查 - -分析平台提供的 Restful 服务要求输入的时间序列不能是白噪声时间序列（White Noise Data, WND）和随机数序列 , 因此针对所有数据均默认进行白噪声检查。当前白噪声检查采用通行的 `Ljung-Box` 检验，`Ljung-Box` 统计量检查过程需要遍历整个输入序列并进行计算。 -如果用户能够明确输入序列一定不是白噪声序列，那么可以通过输入参数，指定预测之前忽略该检查，从而节省分析过程的 CPU 计算资源。 -同时支持独立地针对输入序列进行白噪声检测（该检测功能暂不独立对外开放）。 - - -#### 数据重采样和时间戳对齐 - -分析平台支持将输入数据进行重采样预处理，从而确保输出结果按照用户指定的等间隔进行处理。处理过程分为两种类别： - -- 数据时间戳对齐。由于真实数据可能并非严格按照查询指定的时间戳输入。此时分析平台会自动将数据的时间间隔按照指定的时间间隔进行对齐。例如输入时间序列 [11, 22, 29, 41]，用户指定时间间隔为 10，该序列将被对齐重整为以下序列 [10, 20, 30, 40]。 -- 数据时间重采样。用户输入时间序列的采样频率超过了输出结果的频率，例如输入时间序列的采样频率是 5，输出结果的频率是 10，输入时间序列 [0， 5， 10， 15， 20， 25， 30] 将被重采用为间隔 为 10 的序列 [0, 10, 20，30]，[5, 15, 25] 处的数据将被丢弃。 - -需要注意的是，数据输入平台不支持缺失数据补齐后进行的预测分析，如果输入时间序列数据 [11, 22, 29, 49]，并且用户要求的时间间隔为 10，重整对齐后的序列是 [10, 20, 30, 50] 那么该序列进行预测分析将返回错误。 #### 时序数据异常检测 diff --git a/docs/zh/06-advanced/06-data-analysis/management.md b/docs/zh/06-advanced/06-data-analysis/management.md new file mode 100644 index 0000000000..3f814f68f0 --- /dev/null +++ b/docs/zh/06-advanced/06-data-analysis/management.md @@ -0,0 +1,161 @@ +--- +title: "安装部署" +sidebar_label: "安装部署" +--- + +## 安装部署 +### 环境准备 +ANode 要求节点上准备有 Python 3.10 及以上版本，以及相应的 Python 包自动安装组件 Pip，同时请确保能够正常连接互联网。 + +### 安装及卸载 +使用专门的 ANode 安装包 TDengine-enterprise-anode-1.x.x.tar.gz 进行 ANode 的安装部署工作，安装过程与 TDengine 的安装流程一致。 + +```bash +tar -xzvf TDengine-enterprise-anode-1.0.0.tar.gz +cd TDengine-enterprise-anode-1.0.0 +sudo ./install.sh +``` + +卸载 ANode，执行命令 `rmtaosanode` 即可。 + +### 其他 +为了避免 ANode 安装后影响目标节点现有的 Python 库。 ANode 使用 Python 虚拟环境运行，安装后的默认 Python 目录处于 `/var/lib/taos/taosanode/venv/`。为了避免反复安装虚拟环境带来的开销，卸载 ANode 并不会自动删除该虚拟环境，如果您确认不需要 Python 的虚拟环境，可以手动删除。 + +## 启动及停止服务 +安装 ANode 以后，可以使用 `systemctl` 来管理 ANode 的服务。使用如下命令可以启动/停止/检查状态。 + +```bash +systemctl start taosanoded +systemctl stop taosanoded +systemctl status taosanoded +``` + +## 目录及配置说明 +|目录/文件|说明| +|---------------|------| +|/usr/local/taos/taosanode/bin|可执行文件目录| +|/usr/local/taos/taosanode/resource|资源文件目录，链接到文件夹 /var/lib/taos/taosanode/resource/| +|/usr/local/taos/taosanode/lib|库文件目录| +|/var/lib/taos/taosanode/model/|模型文件目录，链接到文件夹 /var/lib/taos/taosanode/model| +|/var/log/taos/taosanode/|日志文件目录| +|/etc/taos/taosanode.ini|配置文件| + +### 配置说明 + +Anode 提供的 RestFul 服务使用 uWSGI 驱动，因此 ANode 和 uWSGI 的配置信息存放在同一个配置文件中，具体如下： + +```ini +[uwsgi] +# charset +env = LC_ALL = en_US.UTF-8 + +# ip:port +http = 127.0.0.1:6050 + +# the local unix socket file than communicate to Nginx +#socket = 127.0.0.1:8001 +#socket-timeout = 10 + +# base directory +chdir = /usr/local/taos/taosanode/lib + +# initialize python file +wsgi-file = /usr/local/taos/taosanode/lib/taos/app.py + +# call module of uWSGI +callable = app + +# auto remove unix Socket and pid file when stopping +vacuum = true + +# socket exec model +#chmod-socket = 664 + +# uWSGI pid +uid = root + +# uWSGI gid +gid = root + +# main process +master = true + +# the number of worker processes +processes = 2 + +# pid file +pidfile = /usr/local/taos/taosanode/taosanode.pid + +# enable threads +enable-threads = true + +# the number of threads for each process +threads = 4 + +# memory useage report +memory-report = true + +# smooth restart +reload-mercy = 10 + +# conflict with systemctl, so do NOT uncomment this +# daemonize = /var/log/taos/taosanode/taosanode.log + +# log directory +logto = /var/log/taos/taosanode/taosanode.log + +# wWSGI monitor port +stats = 127.0.0.1:8387 + +# python virtual environment directory +virtualenv = /usr/local/taos/taosanode/venv/ + +[taosanode] +# default app log file +app-log = /var/log/taos/taosanode/taosanode.app.log + +# model storage directory +model-dir = /usr/local/taos/taosanode/model/ + +# default log level +log-level = DEBUG + +# draw the query results +draw-result = 0 +``` + +**提示** +请勿设置 `daemonize` 参数，该参数会导致 uWSGI 与 systemctl 冲突，从而无法正常启动。 + + + +## ANode 基本操作 +#### 创建 ANode +```sql +CREATE ANODE {node_url} +``` +node_url 是提供服务的 ANode 的 IP 和 PORT, 例如：`create anode 'http://localhost:6050'`。启动 ANode 以后如果不注册到 TDengine 集群中，则无法提供正常的服务。不建议 ANode 注册到两个或多个集群中。 + +#### 查看 ANode +列出集群中所有的数据分析节点，包括其 `FQDN`, `PORT`, `STATUS`。 +```sql +SHOW ANODES; +``` + +#### 查看提供的时序数据分析服务 + +```SQL +SHOW ANODES FULL; +``` + +#### 强制刷新集群中的分析算法缓存 +```SQL +UPDATE ANODE {node_id} +UPDATE ALL ANODES +``` + +#### 删除 ANode +```sql +DROP ANODE {anode_id} +``` +删除 ANode 只是将 ANode 从 TDengine 集群中删除，管理 ANode 的启停仍然需要使用`systemctl`命令。 diff --git a/docs/zh/06-advanced/06-data-analysis/pic/data-analysis.png b/docs/zh/06-advanced/06-data-analysis/pic/data-analysis.png old mode 100644 new mode 100755 index 44fd82832f..8950cbd58f Binary files a/docs/zh/06-advanced/06-data-analysis/pic/data-analysis.png and b/docs/zh/06-advanced/06-data-analysis/pic/data-analysis.png differ diff --git a/docs/zh/06-advanced/06-data-analysis/pic/dir.png b/docs/zh/06-advanced/06-data-analysis/pic/dir.png deleted file mode 100644 index d5aafb4427..0000000000 Binary files a/docs/zh/06-advanced/06-data-analysis/pic/dir.png and /dev/null differ diff --git a/docs/zh/08-operation/16-security.md b/docs/zh/08-operation/16-security.md index 4f47a644f7..e3cd72d9dc 100644 --- a/docs/zh/08-operation/16-security.md +++ b/docs/zh/08-operation/16-security.md @@ -26,6 +26,22 @@ SHOW USERS; ```sql ALTER USER TEST DROP HOST HOST_NAME1 ``` +说明 +- 开源版和企业版本都能添加成功，且可以查询到，但是开源版本不会对 IP 做任何限制。 +- create user u_write pass 'taosdata1' host 'iprange1','iprange2', 可以一次添加多个 iprange, 服务端会做去重，去重的逻辑是需要 iprange 完全一样 +- 默认会把 127.0.0.1 添加到白名单列表，且在白名单列表可以查询 +- 集群的节点 IP 集合会自动添加到白名单列表，但是查询不到。 +- taosadaper 和 taosd 不在一个机器的时候，需要把 taosadaper IP 手动添加到 taosd 白名单列表中 +- 集群情况下，各个节点 enableWhiteList 成一样，或者全为 false,或者全为 true, 要不然集群无法启动 +- 白名单变更生效时间 1s，不超过 2s, 每次变更对收发性能有些微影响（多一次判断，可以忽略），变更完之后、影响忽略不计, 变更过程中对集群没有影响，对正在访问客户端也没有影响（假设这些客户端的 IP 包含在 white list 内） +- 如果添加两个 ip range, 192.168.1.1/16(假设为 A), 192.168.1.1/24(假设为 B), 严格来说，A 包含了 B，但是考虑情况太复杂，并不会对 A 和 B 做合并 +- 要删除的时候，必须严格匹配。 也就是如果添加的是 192.168.1.1/24, 要删除也是 192.168.1.1/24 +- 只有 root 才有权限对其他用户增删 ip white list +- 兼容之前的版本，但是不支持从当前版本回退到之前版本 +- x.x.x.x/32 和 x.x.x.x 属于同一个 iprange, 显示为 x.x.x.x +- 如果客户端拿到的 0.0.0.0/0, 说明没有开启白名单 +- 如果白名单发生了改变， 客户端会在 heartbeat 里检测到。 +- 针对一个 user, 添加的 IP 个数上限是 2048 ## 审计日志 diff --git a/docs/zh/14-reference/03-taos-sql/02-database.md b/docs/zh/14-reference/03-taos-sql/02-database.md index 24eca97952..91b39976a1 100644 --- a/docs/zh/14-reference/03-taos-sql/02-database.md +++ b/docs/zh/14-reference/03-taos-sql/02-database.md @@ -122,6 +122,7 @@ alter_database_option: { | KEEP value | WAL_RETENTION_PERIOD value | WAL_RETENTION_SIZE value + | MINROWS value } ``` diff --git a/include/common/tanal.h b/include/common/tanalytics.h similarity index 96% rename from include/common/tanal.h rename to include/common/tanalytics.h index 69d110d161..85eb963129 100644 --- a/include/common/tanal.h +++ b/include/common/tanalytics.h @@ -36,7 +36,7 @@ typedef struct { int32_t anode; int32_t urlLen; char *url; -} SAnalUrl; +} SAnalyticsUrl; typedef enum { ANAL_BUF_TYPE_JSON = 0, @@ -53,18 +53,18 @@ typedef struct { TdFilePtr filePtr; char fileName[TSDB_FILENAME_LEN + 10]; int64_t numOfRows; -} SAnalColBuf; +} SAnalyticsColBuf; typedef struct { EAnalBufType bufType; TdFilePtr filePtr; char fileName[TSDB_FILENAME_LEN]; int32_t numOfCols; - SAnalColBuf *pCols; + SAnalyticsColBuf *pCols; } SAnalBuf; -int32_t taosAnalInit(); -void taosAnalCleanup(); +int32_t taosAnalyticsInit(); +void taosAnalyticsCleanup(); SJson *taosAnalSendReqRetJson(const char *url, EAnalHttpType type, SAnalBuf *pBuf); int32_t taosAnalGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, int32_t urlLen); diff --git a/include/common/tcommon.h b/include/common/tcommon.h index ea764e6760..1d9a9bcc61 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -155,6 +155,7 @@ typedef enum EStreamType { STREAM_MID_RETRIEVE, STREAM_PARTITION_DELETE_DATA, STREAM_GET_RESULT, + STREAM_DROP_CHILD_TABLE, } EStreamType; #pragma pack(push, 1) @@ -401,6 +402,8 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol); #define TSMA_RES_STB_EXTRA_COLUMN_NUM 4 // 3 columns: _wstart, _wend, _wduration, 1 tag: tbname static inline bool isTsmaResSTb(const char* stbName) { + static bool showTsmaTables = false; + if (showTsmaTables) return false; const char* pos = strstr(stbName, TSMA_RES_STB_POSTFIX); if (pos && strlen(stbName) == (pos - stbName) + strlen(TSMA_RES_STB_POSTFIX)) { return true; diff --git a/include/common/tglobal.h b/include/common/tglobal.h index bf3fa716c6..e6c471eaf1 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -188,7 +188,6 @@ extern int32_t tsMaxRetryWaitTime; extern bool tsUseAdapter; extern int32_t tsMetaCacheMaxSize; extern int32_t tsSlowLogThreshold; -extern int32_t tsSlowLogThresholdTest; extern char tsSlowLogExceptDb[]; extern int32_t tsSlowLogScope; extern int32_t tsSlowLogMaxLen; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7ff70b243a..bdf333b635 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -676,7 +676,7 @@ typedef struct { int32_t tsSlowLogThreshold; int32_t tsSlowLogMaxLen; int32_t tsSlowLogScope; - int32_t tsSlowLogThresholdTest; + int32_t tsSlowLogThresholdTest; //Obsolete char tsSlowLogExceptDb[TSDB_DB_NAME_LEN]; } SMonitorParas; @@ -2307,6 +2307,7 @@ typedef struct { typedef struct { SExplainRsp rsp; uint64_t qId; + uint64_t cId; uint64_t tId; int64_t rId; int32_t eId; @@ -2660,6 +2661,7 @@ typedef struct SSubQueryMsg { SMsgHead header; uint64_t sId; uint64_t queryId; + uint64_t clientId; uint64_t taskId; int64_t refId; int32_t execId; @@ -2689,6 +2691,7 @@ typedef struct { SMsgHead header; uint64_t sId; uint64_t queryId; + uint64_t clientId; uint64_t taskId; int32_t execId; } SQueryContinueReq; @@ -2723,6 +2726,7 @@ typedef struct { SMsgHead header; uint64_t sId; uint64_t queryId; + uint64_t clientId; uint64_t taskId; int32_t execId; SOperatorParam* pOpParam; @@ -2738,6 +2742,7 @@ typedef struct { typedef struct { uint64_t queryId; + uint64_t clientId; uint64_t taskId; int64_t refId; int32_t execId; @@ -2784,6 +2789,7 @@ typedef struct { SMsgHead header; uint64_t sId; uint64_t queryId; + uint64_t clientId; uint64_t taskId; int64_t refId; int32_t execId; @@ -2797,6 +2803,7 @@ typedef struct { SMsgHead header; uint64_t sId; uint64_t queryId; + uint64_t clientId; uint64_t taskId; int64_t refId; int32_t execId; @@ -2813,6 +2820,7 @@ typedef struct { SMsgHead header; uint64_t sId; uint64_t queryId; + uint64_t clientId; uint64_t taskId; int64_t refId; int32_t execId; @@ -3220,6 +3228,7 @@ int tDecodeSVCreateTbBatchRsp(SDecoder* pCoder, SVCreateTbBatchRsp* pRsp); typedef struct { char* name; uint64_t suid; // for tmq in wal format + int64_t uid; int8_t igNotExists; } SVDropTbReq; @@ -4261,6 +4270,7 @@ typedef struct { SMsgHead header; uint64_t sId; uint64_t queryId; + uint64_t clientId; uint64_t taskId; uint32_t sqlLen; uint32_t phyLen; diff --git a/include/common/ttime.h b/include/common/ttime.h index 65bb763b1f..1ffcc29eca 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -62,7 +62,8 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) { int64_t factor = (precision == TSDB_TIME_PRECISION_MILLI) ? 1000 : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000 : 1000000000; - time_t t = taosTime(NULL); + time_t t; + (void) taosTime(&t); struct tm tm; (void) taosLocalTime(&t, &tm, NULL, 0); tm.tm_hour = 0; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index d955a7b3b9..82cb899cb6 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -31,7 +31,7 @@ typedef void* DataSinkHandle; struct SRpcMsg; struct SSubplan; -typedef int32_t (*localFetchFp)(void*, uint64_t, uint64_t, uint64_t, int64_t, int32_t, void**, SArray*); +typedef int32_t (*localFetchFp)(void*, uint64_t, uint64_t, uint64_t, uint64_t, int64_t, int32_t, void**, SArray*); typedef struct { void* handle; diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index db0d6339c8..feb7bcc25e 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -336,6 +336,7 @@ typedef struct SStateStore { int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname); int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache, int32_t* pWinCode); + int32_t (*streamStateDeleteParName)(SStreamState* pState, int64_t groupId); int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index cfd9c1a422..48852e5552 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -624,6 +624,7 @@ typedef struct SAggPhysiNode { typedef struct SDownstreamSourceNode { ENodeType type; SQueryNodeAddr addr; + uint64_t clientId; uint64_t taskId; uint64_t schedId; int32_t execId; diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index 83daf0376c..cb4e359727 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -105,11 +105,11 @@ void qWorkerDestroy(void **qWorkerMgmt); int32_t qWorkerGetStat(SReadHandle *handle, void *qWorkerMgmt, SQWorkerStat *pStat); -int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId, - SQWMsg *qwMsg, SArray *explainRes); +int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t cId, uint64_t tId, int64_t rId, + int32_t eId, SQWMsg *qwMsg, SArray *explainRes); -int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId, - void **pRsp, SArray *explainRes); +int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t cId, uint64_t tId, int64_t rId, + int32_t eId, void **pRsp, SArray *explainRes); int32_t qWorkerDbgEnableDebug(char *option); diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index af8deff1a0..2988ffc4b1 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -83,6 +83,9 @@ void schedulerStopQueryHb(void* pTrans); int32_t schedulerUpdatePolicy(int32_t policy); int32_t schedulerEnableReSchedule(bool enableResche); +int32_t initClientId(void); +uint64_t getClientId(void); + /** * Cancel query job * @param pJob diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index a50451c3eb..2179547352 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -116,6 +116,7 @@ void streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname); int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache, int32_t* pWinCode); +int32_t streamStateDeleteParName(SStreamState* pState, int64_t groupId); // group id int32_t streamStateGroupPut(SStreamState* pState, int64_t groupId, void* value, int32_t vLen); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index de10d6844e..2cf791c8da 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -462,7 +462,7 @@ struct SStreamTask { struct SStreamMeta* pMeta; SSHashObj* pNameMap; void* pBackend; - int8_t subtableWithoutMd5; + int8_t subtableWithoutMd5; // only for tsma stream tasks char reserve[256]; char* backendPath; }; diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index f95b3f20ca..999adc2eff 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -138,6 +138,7 @@ typedef struct { int8_t scanMeta; int8_t deleteMsg; int8_t enableRef; + int8_t scanDropCtb; } SWalFilterCond; // todo hide this struct diff --git a/include/os/osTime.h b/include/os/osTime.h index 5d74146e9c..7a65efe28d 100644 --- a/include/os/osTime.h +++ b/include/os/osTime.h @@ -93,7 +93,7 @@ static FORCE_INLINE int64_t taosGetMonoTimestampMs() { char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm); struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf, int32_t bufSize); struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst); -time_t taosTime(time_t *t); +int32_t taosTime(time_t *t); time_t taosMktime(struct tm *timep); int64_t user_mktime64(const uint32_t year, const uint32_t mon, const uint32_t day, const uint32_t hour, const uint32_t min, const uint32_t sec, int64_t time_zone); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 9a8b39b84c..2c811495fd 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -208,6 +208,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_TSC_COMPRESS_PARAM_ERROR TAOS_DEF_ERROR_CODE(0, 0X0233) #define TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR TAOS_DEF_ERROR_CODE(0, 0X0234) #define TSDB_CODE_TSC_FAIL_GENERATE_JSON TAOS_DEF_ERROR_CODE(0, 0X0235) +#define TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR TAOS_DEF_ERROR_CODE(0, 0X0236) #define TSDB_CODE_TSC_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0X02FF) // mnode-common diff --git a/packaging/delete_ref_lock.py b/packaging/delete_ref_lock.py new file mode 100644 index 0000000000..cf0e4cdd05 --- /dev/null +++ b/packaging/delete_ref_lock.py @@ -0,0 +1,59 @@ +import subprocess +import re + +# 执行 git fetch 命令并捕获输出 +def git_fetch(): + result = subprocess.run(['git', 'fetch'], capture_output=True, text=True) + return result + +# 解析分支名称 +def parse_branch_name_type1(error_output): + # 使用正则表达式匹配 'is at' 前的分支名称 + match = re.search(r"error: cannot lock ref '(refs/remotes/origin/[^']+)': is at", error_output) + if match: + return match.group(1) + return None + +# 解析第二种错误中的分支名称 +def parse_branch_name_type2(error_output): + # 使用正则表达式匹配 'exists' 前的第一个引号内的分支名称 + match = re.search(r"'(refs/remotes/origin/[^']+)' exists;", error_output) + if match: + return match.group(1) + return None + +# 执行 git update-ref -d 命令 +def git_update_ref(branch_name): + if branch_name: + subprocess.run(['git', 'update-ref', '-d', f'{branch_name}'], check=True) + +# 解析错误类型并执行相应的修复操作 +def handle_error(error_output): + # 错误类型1：本地引用的提交ID与远程不一致 + if "is at" in error_output and "but expected" in error_output: + branch_name = parse_branch_name_type1(error_output) + if branch_name: + print(f"Detected error type 1, attempting to delete ref for branch: {branch_name}") + git_update_ref(branch_name) + else: + print("Error parsing branch name for type 1.") + # 错误类型2：尝试创建新的远程引用时，本地已经存在同名的引用 + elif "exists; cannot create" in error_output: + branch_name = parse_branch_name_type2(error_output) + if branch_name: + print(f"Detected error type 2, attempting to delete ref for branch: {branch_name}") + git_update_ref(branch_name) + else: + print("Error parsing branch name for type 2.") + +# 主函数 +def main(): + fetch_result = git_fetch() + if fetch_result.returncode != 0: # 如果 git fetch 命令失败 + error_output = fetch_result.stderr + handle_error(error_output) + else: + print("Git fetch successful.") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 1b8fa2fb70..0874433e94 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -145,7 +145,14 @@ function kill_taosd() { function install_main_path() { #create install main dir and all sub dir - ${csudo}rm -rf ${install_main_dir} || : + ${csudo}rm -rf ${install_main_dir}/cfg || : + ${csudo}rm -rf ${install_main_dir}/bin || : + ${csudo}rm -rf ${install_main_dir}/driver || : + ${csudo}rm -rf ${install_main_dir}/examples || : + ${csudo}rm -rf ${install_main_dir}/include || : + ${csudo}rm -rf ${install_main_dir}/share || : + ${csudo}rm -rf ${install_main_dir}/log || : + ${csudo}mkdir -p ${install_main_dir} ${csudo}mkdir -p ${install_main_dir}/cfg ${csudo}mkdir -p ${install_main_dir}/bin diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index c56a627ec7..b0be3a4d3b 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -294,8 +294,7 @@ static void deregisterRequest(SRequestObj *pRequest) { } } - if ((duration >= pTscObj->pAppInfo->serverCfg.monitorParas.tsSlowLogThreshold * 1000000UL || - duration >= pTscObj->pAppInfo->serverCfg.monitorParas.tsSlowLogThresholdTest * 1000000UL) && + if ((duration >= pTscObj->pAppInfo->serverCfg.monitorParas.tsSlowLogThreshold * 1000000UL) && checkSlowLogExceptDb(pRequest, pTscObj->pAppInfo->serverCfg.monitorParas.tsSlowLogExceptDb)) { (void)atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1); if (pTscObj->pAppInfo->serverCfg.monitorParas.tsSlowLogScope & reqType) { @@ -983,6 +982,7 @@ void taos_init_imp(void) { SCatalogCfg cfg = {.maxDBCacheNum = 100, .maxTblCacheNum = 100}; ENV_ERR_RET(catalogInit(&cfg), "failed to init catalog"); ENV_ERR_RET(schedulerInit(), "failed to init scheduler"); + ENV_ERR_RET(initClientId(), "failed to init clientId"); tscDebug("starting to initialize TAOS driver"); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 4a719373f2..9f6be8e45c 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -1803,7 +1803,7 @@ int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { if (bind->num > 1) { tscError("invalid bind number %d for %s", bind->num, __FUNCTION__); - terrno = TSDB_CODE_INVALID_PARA; + terrno = TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR; return terrno; } @@ -1819,7 +1819,7 @@ int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { if (bind->num <= 0 || bind->num > INT16_MAX) { tscError("invalid bind num %d", bind->num); - terrno = TSDB_CODE_INVALID_PARA; + terrno = TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR; return terrno; } @@ -1831,7 +1831,7 @@ int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { } if (0 == insert && bind->num > 1) { tscError("only one row data allowed for query"); - terrno = TSDB_CODE_INVALID_PARA; + terrno = TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR; return terrno; } @@ -1859,7 +1859,7 @@ int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, in } if (0 == insert && bind->num > 1) { tscError("only one row data allowed for query"); - terrno = TSDB_CODE_INVALID_PARA; + terrno = TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR; return terrno; } @@ -2019,7 +2019,7 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col if (bind->num <= 0 || bind->num > INT16_MAX) { tscError("invalid bind num %d", bind->num); - terrno = TSDB_CODE_INVALID_PARA; + terrno = TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR; return terrno; } @@ -2027,7 +2027,7 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col (void)stmtIsInsert2(stmt, &insert); if (0 == insert && bind->num > 1) { tscError("only one row data allowed for query"); - terrno = TSDB_CODE_INVALID_PARA; + terrno = TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR; return terrno; } diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index 054b5af2b9..7ca3086871 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -29,6 +29,12 @@ TARGET_LINK_LIBRARIES( # PUBLIC os util common transport monitor parser catalog scheduler function gtest taos_static qcom executor #) +ADD_EXECUTABLE(userOperTest ../../../tests/script/api/passwdTest.c) +TARGET_LINK_LIBRARIES( + userOperTest + PUBLIC taos +) + TARGET_INCLUDE_DIRECTORIES( clientTest PUBLIC "${TD_SOURCE_DIR}/include/client/" @@ -69,3 +75,8 @@ add_test( # NAME clientMonitorTest # COMMAND clientMonitorTest # ) + +add_test( + NAME userOperTest + COMMAND userOperTest +) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index dbf91aac69..4993ece7c1 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -437,6 +437,7 @@ static const SSysDbTableSchema userGrantsLogsSchema[] = { {.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "machine", .bytes = TSDB_GRANT_LOG_COL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "active_info", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; static const SSysDbTableSchema userMachinesSchema[] = { diff --git a/source/common/src/tcol.c b/source/common/src/tcol.c index 923aab12ca..a23385aba0 100644 --- a/source/common/src/tcol.c +++ b/source/common/src/tcol.c @@ -166,6 +166,7 @@ const char* columnCompressStr(uint16_t type) { } uint8_t columnLevelVal(const char* level) { + if (level == NULL) return TSDB_COLVAL_LEVEL_NOCHANGE; uint8_t l = TSDB_COLVAL_LEVEL_MEDIUM; if (0 == strcmp(level, "h") || 0 == strcmp(level, TSDB_COLUMN_LEVEL_HIGH)) { l = TSDB_COLVAL_LEVEL_HIGH; @@ -180,6 +181,7 @@ uint8_t columnLevelVal(const char* level) { } uint16_t columnCompressVal(const char* compress) { + if (compress == NULL) return TSDB_COLVAL_COMPRESS_NOCHANGE; uint16_t c = TSDB_COLVAL_COMPRESS_LZ4; if (0 == strcmp(compress, TSDB_COLUMN_COMPRESS_LZ4)) { c = TSDB_COLVAL_COMPRESS_LZ4; @@ -200,6 +202,7 @@ uint16_t columnCompressVal(const char* compress) { } uint8_t columnEncodeVal(const char* encode) { + if (encode == NULL) return TSDB_COLVAL_ENCODE_NOCHANGE; uint8_t e = TSDB_COLVAL_ENCODE_SIMPLE8B; if (0 == strcmp(encode, TSDB_COLUMN_ENCODE_SIMPLE8B)) { e = TSDB_COLVAL_ENCODE_SIMPLE8B; @@ -311,6 +314,7 @@ void setColLevel(uint32_t* compress, uint8_t level) { int32_t setColCompressByOption(uint8_t type, uint8_t encode, uint16_t compressType, uint8_t level, bool check, uint32_t* compress) { + if(compress == NULL) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; if (check && !validColEncode(type, encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; setColEncode(compress, encode); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 9c72e3b498..da7b8b14a5 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -185,7 +185,6 @@ int32_t tsMaxRetryWaitTime = 10000; bool tsUseAdapter = false; int32_t tsMetaCacheMaxSize = -1; // MB int32_t tsSlowLogThreshold = 10; // seconds -int32_t tsSlowLogThresholdTest = INT32_MAX; // seconds char tsSlowLogExceptDb[TSDB_DB_NAME_LEN] = ""; // seconds int32_t tsSlowLogScope = SLOW_LOG_TYPE_QUERY; char *tsSlowLogScopeString = "query"; @@ -766,7 +765,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 86400, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "slowLogThresholdTest", tsSlowLogThresholdTest, 0, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 1, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "slowLogMaxLen", tsSlowLogMaxLen, 1, 16384, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "slowLogScope", tsSlowLogScopeString, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); @@ -1450,9 +1448,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "slowLogExceptDb"); tstrncpy(tsSlowLogExceptDb, pItem->str, TSDB_DB_NAME_LEN); - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "slowLogThresholdTest"); - tsSlowLogThresholdTest = pItem->i32; - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "slowLogThreshold"); tsSlowLogThreshold = pItem->i32; @@ -2024,7 +2019,6 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"monitor", &tsEnableMonitor}, {"monitorInterval", &tsMonitorInterval}, {"slowLogThreshold", &tsSlowLogThreshold}, - {"slowLogThresholdTest", &tsSlowLogThresholdTest}, {"slowLogMaxLen", &tsSlowLogMaxLen}, {"mndSdbWriteDelta", &tsMndSdbWriteDelta}, diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 458badc764..ca5af34e15 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -40,7 +40,7 @@ #define TD_MSG_RANGE_CODE_ #include "tmsgdef.h" -#include "tanal.h" +#include "tanalytics.h" #include "tcol.h" #include "tlog.h" @@ -76,7 +76,7 @@ static int32_t tSerializeSMonitorParas(SEncoder *encoder, const SMonitorParas *p TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogScope)); TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogMaxLen)); TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogThreshold)); - TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogThresholdTest)); + TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogThresholdTest)); //Obsolete TAOS_CHECK_RETURN(tEncodeCStr(encoder, pMonitorParas->tsSlowLogExceptDb)); return 0; } @@ -87,7 +87,7 @@ static int32_t tDeserializeSMonitorParas(SDecoder *decoder, SMonitorParas *pMoni TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogScope)); TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogMaxLen)); TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogThreshold)); - TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogThresholdTest)); + TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogThresholdTest)); //Obsolete TAOS_CHECK_RETURN(tDecodeCStrTo(decoder, pMonitorParas->tsSlowLogExceptDb)); return 0; } @@ -2166,7 +2166,7 @@ int32_t tSerializeRetrieveAnalAlgoRsp(void *buf, int32_t bufLen, SRetrieveAnalAl int32_t numOfAlgos = 0; void *pIter = taosHashIterate(pRsp->hash, NULL); while (pIter != NULL) { - SAnalUrl *pUrl = pIter; + SAnalyticsUrl *pUrl = pIter; size_t nameLen = 0; const char *name = taosHashGetKey(pIter, &nameLen); if (nameLen > 0 && nameLen <= TSDB_ANAL_ALGO_KEY_LEN && pUrl->urlLen > 0) { @@ -2181,7 +2181,7 @@ int32_t tSerializeRetrieveAnalAlgoRsp(void *buf, int32_t bufLen, SRetrieveAnalAl pIter = taosHashIterate(pRsp->hash, NULL); while (pIter != NULL) { - SAnalUrl *pUrl = pIter; + SAnalyticsUrl *pUrl = pIter; size_t nameLen = 0; const char *name = taosHashGetKey(pIter, &nameLen); if (nameLen > 0 && pUrl->urlLen > 0) { @@ -2225,7 +2225,7 @@ int32_t tDeserializeRetrieveAnalAlgoRsp(void *buf, int32_t bufLen, SRetrieveAnal int32_t nameLen; int32_t type; char name[TSDB_ANAL_ALGO_KEY_LEN]; - SAnalUrl url = {0}; + SAnalyticsUrl url = {0}; TAOS_CHECK_EXIT(tStartDecode(&decoder)); TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pRsp->ver)); @@ -2245,7 +2245,7 @@ int32_t tDeserializeRetrieveAnalAlgoRsp(void *buf, int32_t bufLen, SRetrieveAnal TAOS_CHECK_EXIT(tDecodeBinaryAlloc(&decoder, (void **)&url.url, NULL) < 0); } - TAOS_CHECK_EXIT(taosHashPut(pRsp->hash, name, nameLen, &url, sizeof(SAnalUrl))); + TAOS_CHECK_EXIT(taosHashPut(pRsp->hash, name, nameLen, &url, sizeof(SAnalyticsUrl))); } tEndDecode(&decoder); @@ -2258,7 +2258,7 @@ _exit: void tFreeRetrieveAnalAlgoRsp(SRetrieveAnalAlgoRsp *pRsp) { void *pIter = taosHashIterate(pRsp->hash, NULL); while (pIter != NULL) { - SAnalUrl *pUrl = (SAnalUrl *)pIter; + SAnalyticsUrl *pUrl = (SAnalyticsUrl *)pIter; taosMemoryFree(pUrl->url); pIter = taosHashIterate(pRsp->hash, pIter); } @@ -8717,6 +8717,7 @@ int32_t tSerializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq) { TAOS_CHECK_EXIT(tEncodeCStrWithLen(&encoder, pReq->sql, pReq->sqlLen)); TAOS_CHECK_EXIT(tEncodeU32(&encoder, pReq->msgLen)); TAOS_CHECK_EXIT(tEncodeBinary(&encoder, (uint8_t *)pReq->msg, pReq->msgLen)); + TAOS_CHECK_EXIT(tEncodeU64(&encoder, pReq->clientId)); tEndEncode(&encoder); @@ -8765,6 +8766,11 @@ int32_t tDeserializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq) TAOS_CHECK_EXIT(tDecodeCStrAlloc(&decoder, &pReq->sql)); TAOS_CHECK_EXIT(tDecodeU32(&decoder, &pReq->msgLen)); TAOS_CHECK_EXIT(tDecodeBinaryAlloc(&decoder, (void **)&pReq->msg, NULL)); + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pReq->clientId)); + } else { + pReq->clientId = 0; + } tEndDecode(&decoder); @@ -8894,6 +8900,7 @@ int32_t tSerializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq) { } else { TAOS_CHECK_EXIT(tEncodeI32(&encoder, 0)); } + TAOS_CHECK_EXIT(tEncodeU64(&encoder, pReq->clientId)); tEndEncode(&encoder); @@ -8943,6 +8950,11 @@ int32_t tDeserializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq) } TAOS_CHECK_EXIT(tDeserializeSOperatorParam(&decoder, pReq->pOpParam)); } + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pReq->clientId)); + } else { + pReq->clientId = 0; + } tEndDecode(&decoder); @@ -9055,6 +9067,7 @@ int32_t tSerializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq) { TAOS_CHECK_EXIT(tEncodeU64(&encoder, pReq->taskId)); TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->refId)); TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->execId)); + TAOS_CHECK_EXIT(tEncodeU64(&encoder, pReq->clientId)); tEndEncode(&encoder); @@ -9095,6 +9108,11 @@ int32_t tDeserializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq) TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pReq->taskId)); TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->refId)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->execId)); + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pReq->clientId)); + } else { + pReq->clientId = 0; + } tEndDecode(&decoder); @@ -9123,6 +9141,7 @@ int32_t tSerializeSTaskNotifyReq(void *buf, int32_t bufLen, STaskNotifyReq *pReq TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->refId)); TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->execId)); TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->type)); + TAOS_CHECK_EXIT(tEncodeU64(&encoder, pReq->clientId)); tEndEncode(&encoder); @@ -9164,6 +9183,11 @@ int32_t tDeserializeSTaskNotifyReq(void *buf, int32_t bufLen, STaskNotifyReq *pR TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->refId)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->execId)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, (int32_t *)&pReq->type)); + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pReq->clientId)); + } else { + pReq->clientId = 0; + } tEndDecode(&decoder); @@ -9353,6 +9377,10 @@ int32_t tSerializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *pR TAOS_CHECK_EXIT(tEncodeI32(&encoder, status->execId)); TAOS_CHECK_EXIT(tEncodeI8(&encoder, status->status)); } + for (int32_t i = 0; i < num; ++i) { + STaskStatus *status = taosArrayGet(pRsp->taskStatus, i); + TAOS_CHECK_EXIT(tEncodeU64(&encoder, status->clientId)); + } } else { TAOS_CHECK_EXIT(tEncodeI32(&encoder, 0)); } @@ -9396,6 +9424,12 @@ int32_t tDeserializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp * TAOS_CHECK_EXIT(terrno); } } + if (!tDecodeIsEnd(&decoder)) { + for (int32_t i = 0; i < num; ++i) { + STaskStatus *status = taosArrayGet(pRsp->taskStatus, i); + TAOS_CHECK_EXIT(tDecodeU64(&decoder, &status->clientId)); + } + } } else { pRsp->taskStatus = NULL; } @@ -9560,6 +9594,7 @@ int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) { TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->sql)); TAOS_CHECK_EXIT(tEncodeBinary(&encoder, pReq->msg, pReq->phyLen)); TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->source)); + TAOS_CHECK_EXIT(tEncodeU64(&encoder, pReq->clientId)); tEndEncode(&encoder); _exit: @@ -9608,6 +9643,11 @@ int32_t tDeserializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) { if (!tDecodeIsEnd(&decoder)) { TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->source)); } + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pReq->clientId)); + } else { + pReq->clientId = 0; + } tEndDecode(&decoder); _exit: @@ -10277,6 +10317,7 @@ static int32_t tEncodeSVDropTbReq(SEncoder *pCoder, const SVDropTbReq *pReq) { TAOS_CHECK_RETURN(tStartEncode(pCoder)); TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pReq->name)); TAOS_CHECK_RETURN(tEncodeU64(pCoder, pReq->suid)); + TAOS_CHECK_RETURN(tEncodeI64(pCoder, pReq->uid)); TAOS_CHECK_RETURN(tEncodeI8(pCoder, pReq->igNotExists)); tEndEncode(pCoder); @@ -10287,6 +10328,7 @@ static int32_t tDecodeSVDropTbReq(SDecoder *pCoder, SVDropTbReq *pReq) { TAOS_CHECK_RETURN(tStartDecode(pCoder)); TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pReq->name)); TAOS_CHECK_RETURN(tDecodeU64(pCoder, &pReq->suid)); + TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pReq->uid)); TAOS_CHECK_RETURN(tDecodeI8(pCoder, &pReq->igNotExists)); tEndDecode(pCoder); diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 75624593d9..ecdb3de9a2 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -30,7 +30,7 @@ static int64_t m_deltaUtc = 0; void deltaToUtcInitOnce() { struct tm tm = {0}; - if (taosStrpTime("1970-01-01 00:00:00", (const char*)("%Y-%m-%d %H:%M:%S"), &tm) != 0) { + if (taosStrpTime("1970-01-01 00:00:00", (const char*)("%Y-%m-%d %H:%M:%S"), &tm) == NULL) { uError("failed to parse time string"); } m_deltaUtc = (int64_t)taosMktime(&tm); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index d6b792ca74..c01fdcc85b 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -18,7 +18,7 @@ #include "dmInt.h" #include "monitor.h" #include "systable.h" -#include "tanal.h" +#include "tanalytics.h" #include "tchecksum.h" extern SConfig *tsCfg; @@ -195,7 +195,6 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { req.clusterCfg.monitorParas.tsSlowLogScope = tsSlowLogScope; req.clusterCfg.monitorParas.tsSlowLogMaxLen = tsSlowLogMaxLen; req.clusterCfg.monitorParas.tsSlowLogThreshold = tsSlowLogThreshold; - req.clusterCfg.monitorParas.tsSlowLogThresholdTest = tsSlowLogThresholdTest; tstrncpy(req.clusterCfg.monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb, TSDB_DB_NAME_LEN); char timestr[32] = "1970-01-01 00:00:00.00"; if (taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0) != 0) { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index 04b4e9101c..fb7d891c67 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "dmInt.h" #include "libs/function/tudf.h" -#include "tanal.h" +#include "tanalytics.h" static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) { int32_t code = 0; @@ -85,7 +85,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { dError("failed to start udfd since %s", tstrerror(code)); } - if ((code = taosAnalInit()) != 0) { + if ((code = taosAnalyticsInit()) != 0) { dError("failed to init analysis env since %s", tstrerror(code)); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 694cc52d64..6d4ebe424a 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -21,7 +21,7 @@ #include "tgrant.h" #include "tcompare.h" #include "tcs.h" -#include "tanal.h" +#include "tanalytics.h" // clang-format on #define DM_INIT_AUDIT() \ @@ -209,7 +209,7 @@ void dmCleanup() { dError("failed to close udfc"); } udfStopUdfd(); - taosAnalCleanup(); + taosAnalyticsCleanup(); taosStopCacheRefreshWorker(); (void)dmDiskClose(); DestroyRegexCache(); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 5a276de251..61543e619e 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "dmMgmt.h" #include "qworker.h" -#include "tanal.h" +#include "tanalytics.h" #include "tversion.h" static inline void dmSendRsp(SRpcMsg *pMsg) { diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index 8a390948ae..ad36d8c8ae 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -18,7 +18,7 @@ if(TD_ENTERPRISE) endif() if(${BUILD_WITH_ANALYSIS}) - add_definitions(-DUSE_ANAL) + add_definitions(-DUSE_ANALYTICS) endif() endif() diff --git a/source/dnode/mnode/impl/src/mndAnode.c b/source/dnode/mnode/impl/src/mndAnode.c index 17e3e84c81..87bfe9f7af 100644 --- a/source/dnode/mnode/impl/src/mndAnode.c +++ b/source/dnode/mnode/impl/src/mndAnode.c @@ -21,10 +21,10 @@ #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" -#include "tanal.h" +#include "tanalytics.h" #include "tjson.h" -#ifdef USE_ANAL +#ifdef USE_ANALYTICS #define TSDB_ANODE_VER_NUMBER 1 #define TSDB_ANODE_RESERVE_SIZE 64 @@ -806,7 +806,7 @@ static int32_t mndProcessAnalAlgoReq(SRpcMsg *pReq) { SSdb *pSdb = pMnode->pSdb; int32_t code = -1; SAnodeObj *pObj = NULL; - SAnalUrl url; + SAnalyticsUrl url; int32_t nameLen; char name[TSDB_ANAL_ALGO_KEY_LEN]; SRetrieveAnalAlgoReq req = {0}; @@ -838,7 +838,7 @@ static int32_t mndProcessAnalAlgoReq(SRpcMsg *pReq) { SAnodeAlgo *algo = taosArrayGet(algos, a); nameLen = 1 + tsnprintf(name, sizeof(name) - 1, "%d:%s", url.type, algo->name); - SAnalUrl *pOldUrl = taosHashAcquire(rsp.hash, name, nameLen); + SAnalyticsUrl *pOldUrl = taosHashAcquire(rsp.hash, name, nameLen); if (pOldUrl == NULL || (pOldUrl != NULL && pOldUrl->anode < url.anode)) { if (pOldUrl != NULL) { taosMemoryFreeClear(pOldUrl->url); @@ -855,7 +855,7 @@ static int32_t mndProcessAnalAlgoReq(SRpcMsg *pReq) { url.urlLen = 1 + tsnprintf(url.url, TSDB_ANAL_ANODE_URL_LEN + TSDB_ANAL_ALGO_TYPE_LEN, "%s/%s", pAnode->url, taosAnalAlgoUrlStr(url.type)); - if (taosHashPut(rsp.hash, name, nameLen, &url, sizeof(SAnalUrl)) != 0) { + if (taosHashPut(rsp.hash, name, nameLen, &url, sizeof(SAnalyticsUrl)) != 0) { taosMemoryFree(url.url); sdbRelease(pSdb, pAnode); goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 24ae8382f9..406128e232 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -482,7 +482,6 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, SDnodeObj *pDnode, const S CHECK_MONITOR_PARA(tsEnableMonitor, DND_REASON_STATUS_MONITOR_SWITCH_NOT_MATCH); CHECK_MONITOR_PARA(tsMonitorInterval, DND_REASON_STATUS_MONITOR_INTERVAL_NOT_MATCH); CHECK_MONITOR_PARA(tsSlowLogThreshold, DND_REASON_STATUS_MONITOR_SLOW_LOG_THRESHOLD_NOT_MATCH); - CHECK_MONITOR_PARA(tsSlowLogThresholdTest, DND_REASON_STATUS_MONITOR_NOT_MATCH); CHECK_MONITOR_PARA(tsSlowLogMaxLen, DND_REASON_STATUS_MONITOR_SLOW_LOG_SQL_MAX_LEN_NOT_MATCH); CHECK_MONITOR_PARA(tsSlowLogScope, DND_REASON_STATUS_MONITOR_SLOW_LOG_SCOPE_NOT_MATCH); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 21aba8df10..fd02367f6d 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -304,7 +304,6 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { connectRsp.monitorParas.tsSlowLogScope = tsSlowLogScope; connectRsp.monitorParas.tsSlowLogMaxLen = tsSlowLogMaxLen; connectRsp.monitorParas.tsSlowLogThreshold = tsSlowLogThreshold; - connectRsp.monitorParas.tsSlowLogThresholdTest = tsSlowLogThresholdTest; connectRsp.enableAuditDelete = tsEnableAuditDelete; tstrncpy(connectRsp.monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb, TSDB_DB_NAME_LEN); connectRsp.whiteListVer = pUser->ipWhiteListVer; @@ -706,7 +705,6 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) { batchRsp.monitorParas.tsEnableMonitor = tsEnableMonitor; batchRsp.monitorParas.tsMonitorInterval = tsMonitorInterval; batchRsp.monitorParas.tsSlowLogThreshold = tsSlowLogThreshold; - batchRsp.monitorParas.tsSlowLogThresholdTest = tsSlowLogThresholdTest; tstrncpy(batchRsp.monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb, TSDB_DB_NAME_LEN); batchRsp.monitorParas.tsSlowLogMaxLen = tsSlowLogMaxLen; batchRsp.monitorParas.tsSlowLogScope = tsSlowLogScope; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 3725d3a3fc..eb6c326d1e 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -4063,8 +4063,8 @@ static int32_t mndProcessDropStbReqFromMNode(SRpcMsg *pReq) { } typedef struct SVDropTbVgReqs { - SVDropTbBatchReq req; - SVgroupInfo info; + SArray *pBatchReqs; + SVgroupInfo info; } SVDropTbVgReqs; typedef struct SMDropTbDbInfo { @@ -4086,45 +4086,21 @@ typedef struct SMDropTbTsmaInfos { } SMDropTbTsmaInfos; typedef struct SMndDropTbsWithTsmaCtx { - SHashObj *pTsmaMap; // - SHashObj *pDbMap; // - SHashObj *pVgMap; // - SArray *pResTbNames; // SArray + SHashObj *pVgMap; // } SMndDropTbsWithTsmaCtx; -static int32_t mndDropTbAddTsmaResTbsForSingleVg(SMnode *pMnode, SMndDropTbsWithTsmaCtx *pCtx, SArray *pTbs, +static int32_t mndDropTbForSingleVg(SMnode *pMnode, SMndDropTbsWithTsmaCtx *pCtx, SArray *pTbs, int32_t vgId); +static void destroySVDropTbBatchReqs(void *p); static void mndDestroyDropTbsWithTsmaCtx(SMndDropTbsWithTsmaCtx *p) { if (!p) return; - if (p->pDbMap) { - void *pIter = taosHashIterate(p->pDbMap, NULL); - while (pIter) { - SMDropTbDbInfo *pInfo = pIter; - taosArrayDestroy(pInfo->dbVgInfos); - pIter = taosHashIterate(p->pDbMap, pIter); - } - taosHashCleanup(p->pDbMap); - } - if (p->pResTbNames) { - taosArrayDestroyP(p->pResTbNames, taosMemoryFree); - } - if (p->pTsmaMap) { - void *pIter = taosHashIterate(p->pTsmaMap, NULL); - while (pIter) { - SMDropTbTsmaInfos *pInfos = pIter; - taosArrayDestroy(pInfos->pTsmaInfos); - pIter = taosHashIterate(p->pTsmaMap, pIter); - } - taosHashCleanup(p->pTsmaMap); - } - if (p->pVgMap) { void *pIter = taosHashIterate(p->pVgMap, NULL); while (pIter) { SVDropTbVgReqs *pReqs = pIter; - taosArrayDestroy(pReqs->req.pArray); + taosArrayDestroyEx(pReqs->pBatchReqs, destroySVDropTbBatchReqs); pIter = taosHashIterate(p->pVgMap, pIter); } taosHashCleanup(p->pVgMap); @@ -4136,24 +4112,13 @@ static int32_t mndInitDropTbsWithTsmaCtx(SMndDropTbsWithTsmaCtx **ppCtx) { int32_t code = 0; SMndDropTbsWithTsmaCtx *pCtx = taosMemoryCalloc(1, sizeof(SMndDropTbsWithTsmaCtx)); if (!pCtx) return terrno; - pCtx->pTsmaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - if (!pCtx->pTsmaMap) { - code = terrno; - goto _end; - } - - pCtx->pDbMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - if (!pCtx->pDbMap) { - code = terrno; - goto _end; - } - pCtx->pResTbNames = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); pCtx->pVgMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); if (!pCtx->pVgMap) { code = terrno; goto _end; } + *ppCtx = pCtx; _end: if (code) mndDestroyDropTbsWithTsmaCtx(pCtx); @@ -4192,16 +4157,43 @@ static void *mndBuildVDropTbsReq(SMnode *pMnode, const SVgroupInfo *pVgInfo, con } static int32_t mndSetDropTbsRedoActions(SMnode *pMnode, STrans *pTrans, const SVDropTbVgReqs *pVgReqs, void *pCont, - int32_t contLen) { + int32_t contLen, tmsg_t msgType) { STransAction action = {0}; action.epSet = pVgReqs->info.epSet; action.pCont = pCont; action.contLen = contLen; - action.msgType = TDMT_VND_DROP_TABLE; + action.msgType = msgType; action.acceptableCode = TSDB_CODE_TDB_TABLE_NOT_EXIST; return mndTransAppendRedoAction(pTrans, &action); } +static int32_t mndBuildDropTbRedoActions(SMnode *pMnode, STrans *pTrans, SHashObj *pVgMap, tmsg_t msgType) { + int32_t code = 0; + void *pIter = taosHashIterate(pVgMap, NULL); + while (pIter) { + const SVDropTbVgReqs *pVgReqs = pIter; + int32_t len = 0; + for (int32_t i = 0; i < taosArrayGetSize(pVgReqs->pBatchReqs) && code == TSDB_CODE_SUCCESS; ++i) { + SVDropTbBatchReq *pBatchReq = taosArrayGet(pVgReqs->pBatchReqs, i); + void *p = mndBuildVDropTbsReq(pMnode, &pVgReqs->info, pBatchReq, &len); + if (!p) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + break; + } + if ((code = mndSetDropTbsRedoActions(pMnode, pTrans, pVgReqs, p, len, msgType)) != 0) { + break; + } + } + if (TSDB_CODE_SUCCESS != code) { + taosHashCancelIterate(pVgMap, pIter); + break; + } + pIter = taosHashIterate(pVgMap, pIter); + } + return code; +} + static int32_t mndCreateDropTbsTxnPrepare(SRpcMsg *pRsp, SMndDropTbsWithTsmaCtx *pCtx) { int32_t code = 0; SMnode *pMnode = pRsp->info.node; @@ -4216,23 +4208,7 @@ static int32_t mndCreateDropTbsTxnPrepare(SRpcMsg *pRsp, SMndDropTbsWithTsmaCtx TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER); - void *pIter = taosHashIterate(pCtx->pVgMap, NULL); - while (pIter) { - const SVDropTbVgReqs *pVgReqs = pIter; - int32_t len = 0; - void *p = mndBuildVDropTbsReq(pMnode, &pVgReqs->info, &pVgReqs->req, &len); - if (!p) { - taosHashCancelIterate(pCtx->pVgMap, pIter); - code = TSDB_CODE_MND_RETURN_VALUE_NULL; - if (terrno != 0) code = terrno; - goto _OVER; - } - if ((code = mndSetDropTbsRedoActions(pMnode, pTrans, pVgReqs, p, len)) != 0) { - taosHashCancelIterate(pCtx->pVgMap, pIter); - goto _OVER; - } - pIter = taosHashIterate(pCtx->pVgMap, pIter); - } + if ((code = mndBuildDropTbRedoActions(pMnode, pTrans, pCtx->pVgMap, TDMT_VND_DROP_TABLE)) != 0) goto _OVER; if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER; _OVER: @@ -4257,10 +4233,11 @@ static int32_t mndProcessDropTbWithTsma(SRpcMsg *pReq) { if (code) goto _OVER; for (int32_t i = 0; i < dropReq.pVgReqs->size; ++i) { SMDropTbReqsOnSingleVg *pReq = taosArrayGet(dropReq.pVgReqs, i); - code = mndDropTbAddTsmaResTbsForSingleVg(pMnode, pCtx, pReq->pTbs, pReq->vgInfo.vgId); + code = mndDropTbForSingleVg(pMnode, pCtx, pReq->pTbs, pReq->vgInfo.vgId); if (code) goto _OVER; } - if (mndCreateDropTbsTxnPrepare(pReq, pCtx) == 0) { + code = mndCreateDropTbsTxnPrepare(pReq, pCtx); + if (code == 0) { code = TSDB_CODE_ACTION_IN_PROGRESS; } _OVER: @@ -4269,87 +4246,58 @@ _OVER: TAOS_RETURN(code); } +static int32_t createDropTbBatchReq(const SVDropTbReq *pReq, SVDropTbBatchReq *pBatchReq) { + pBatchReq->nReqs = 1; + pBatchReq->pArray = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVDropTbReq)); + if (!pBatchReq->pArray) return terrno; + if (taosArrayPush(pBatchReq->pArray, pReq) == NULL) { + taosArrayDestroy(pBatchReq->pArray); + pBatchReq->pArray = NULL; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + +static void destroySVDropTbBatchReqs(void *p) { + SVDropTbBatchReq *pReq = p; + taosArrayDestroy(pReq->pArray); + pReq->pArray = NULL; +} + static int32_t mndDropTbAdd(SMnode *pMnode, SHashObj *pVgHashMap, const SVgroupInfo *pVgInfo, char *name, tb_uid_t suid, bool ignoreNotExists) { - SVDropTbReq req = {.name = name, .suid = suid, .igNotExists = ignoreNotExists}; + SVDropTbReq req = {.name = name, .suid = suid, .igNotExists = ignoreNotExists, .uid = 0}; - SVDropTbVgReqs *pReqs = taosHashGet(pVgHashMap, &pVgInfo->vgId, sizeof(pVgInfo->vgId)); - SVDropTbVgReqs reqs = {0}; - if (pReqs == NULL) { - reqs.info = *pVgInfo; - reqs.req.pArray = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVDropTbReq)); - if (reqs.req.pArray == NULL) { + SVDropTbVgReqs *pVgReqs = taosHashGet(pVgHashMap, &pVgInfo->vgId, sizeof(pVgInfo->vgId)); + SVDropTbVgReqs vgReqs = {0}; + if (pVgReqs == NULL) { + vgReqs.info = *pVgInfo; + vgReqs.pBatchReqs = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVDropTbBatchReq)); + if (!vgReqs.pBatchReqs) return terrno; + SVDropTbBatchReq batchReq = {0}; + int32_t code = createDropTbBatchReq(&req, &batchReq); + if (TSDB_CODE_SUCCESS != code) return code; + if (taosArrayPush(vgReqs.pBatchReqs, &batchReq) == NULL) { + taosArrayDestroy(batchReq.pArray); return terrno; } - if (taosArrayPush(reqs.req.pArray, &req) == NULL) { - return terrno; - } - if (taosHashPut(pVgHashMap, &pVgInfo->vgId, sizeof(pVgInfo->vgId), &reqs, sizeof(reqs)) != 0) { + if (taosHashPut(pVgHashMap, &pVgInfo->vgId, sizeof(pVgInfo->vgId), &vgReqs, sizeof(vgReqs)) != 0) { + taosArrayDestroyEx(vgReqs.pBatchReqs, destroySVDropTbBatchReqs); return terrno; } } else { - if (taosArrayPush(pReqs->req.pArray, &req) == NULL) { + SVDropTbBatchReq batchReq = {0}; + int32_t code = createDropTbBatchReq(&req, &batchReq); + if (TSDB_CODE_SUCCESS != code) return code; + if (taosArrayPush(pVgReqs->pBatchReqs, &batchReq) == NULL) { + taosArrayDestroy(batchReq.pArray); return terrno; } } return 0; } -int vgInfoCmp(const void *lp, const void *rp) { - SVgroupInfo *pLeft = (SVgroupInfo *)lp; - SVgroupInfo *pRight = (SVgroupInfo *)rp; - if (pLeft->hashBegin < pRight->hashBegin) { - return -1; - } else if (pLeft->hashBegin > pRight->hashBegin) { - return 1; - } - - return 0; -} - -static int32_t mndGetDbVgInfoForTsma(SMnode *pMnode, const char *dbname, SMDropTbTsmaInfo *pInfo) { - int32_t code = 0; - SDbObj *pDb = mndAcquireDb(pMnode, dbname); - if (!pDb) { - code = TSDB_CODE_MND_DB_NOT_EXIST; - goto _end; - } - - pInfo->dbInfo.dbVgInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); - if (!pInfo->dbInfo.dbVgInfos) { - code = terrno; - goto _end; - } - mndBuildDBVgroupInfo(pDb, pMnode, pInfo->dbInfo.dbVgInfos); - taosArraySort(pInfo->dbInfo.dbVgInfos, vgInfoCmp); - - pInfo->dbInfo.hashPrefix = pDb->cfg.hashPrefix; - pInfo->dbInfo.hashSuffix = pDb->cfg.hashSuffix; - pInfo->dbInfo.hashMethod = pDb->cfg.hashMethod; - -_end: - if (pDb) mndReleaseDb(pMnode, pDb); - if (code && pInfo->dbInfo.dbVgInfos) { - taosArrayDestroy(pInfo->dbInfo.dbVgInfos); - pInfo->dbInfo.dbVgInfos = NULL; - } - TAOS_RETURN(code); -} - -int32_t vgHashValCmp(const void *lp, const void *rp) { - uint32_t *key = (uint32_t *)lp; - SVgroupInfo *pVg = (SVgroupInfo *)rp; - - if (*key < pVg->hashBegin) { - return -1; - } else if (*key > pVg->hashEnd) { - return 1; - } - - return 0; -} - -static int32_t mndDropTbAddTsmaResTbsForSingleVg(SMnode *pMnode, SMndDropTbsWithTsmaCtx *pCtx, SArray *pTbs, +static int32_t mndDropTbForSingleVg(SMnode *pMnode, SMndDropTbsWithTsmaCtx *pCtx, SArray *pTbs, int32_t vgId) { int32_t code = 0; @@ -4365,88 +4313,9 @@ static int32_t mndDropTbAddTsmaResTbsForSingleVg(SMnode *pMnode, SMndDropTbsWith vgInfo.epSet = mndGetVgroupEpset(pMnode, pVgObj); mndReleaseVgroup(pMnode, pVgObj); - // get all stb uids - for (int32_t i = 0; i < pTbs->size; ++i) { - const SVDropTbReq *pTb = taosArrayGet(pTbs, i); - if (taosHashGet(pCtx->pTsmaMap, &pTb->suid, sizeof(pTb->suid))) { - } else { - SMDropTbTsmaInfos infos = {0}; - infos.pTsmaInfos = taosArrayInit(2, sizeof(SMDropTbTsmaInfo)); - if (!infos.pTsmaInfos) { - code = terrno; - goto _end; - } - if (taosHashPut(pCtx->pTsmaMap, &pTb->suid, sizeof(pTb->suid), &infos, sizeof(infos)) != 0) { - code = terrno; - goto _end; - } - } - } - - void *pIter = NULL; - SSmaObj *pSma = NULL; - char buf[TSDB_TABLE_FNAME_LEN] = {0}; - // get used tsmas and it's dbs - while (1) { - pIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter, (void **)&pSma); - if (!pIter) break; - SMDropTbTsmaInfos *pInfos = taosHashGet(pCtx->pTsmaMap, &pSma->stbUid, sizeof(pSma->stbUid)); - if (pInfos) { - SMDropTbTsmaInfo info = {0}; - int32_t len = sprintf(buf, "%s", pSma->name); - sprintf(info.tsmaResTbDbFName, "%s", pSma->db); - snprintf(info.tsmaResTbNamePrefix, TSDB_TABLE_FNAME_LEN, "%s", buf); - SMDropTbDbInfo *pDbInfo = taosHashGet(pCtx->pDbMap, pSma->db, TSDB_DB_FNAME_LEN); - info.suid = pSma->dstTbUid; - if (!pDbInfo) { - code = mndGetDbVgInfoForTsma(pMnode, pSma->db, &info); - if (code != TSDB_CODE_SUCCESS) { - sdbCancelFetch(pMnode->pSdb, pIter); - sdbRelease(pMnode->pSdb, pSma); - goto _end; - } - if (taosHashPut(pCtx->pDbMap, pSma->db, TSDB_DB_FNAME_LEN, &info.dbInfo, sizeof(SMDropTbDbInfo)) != 0) { - sdbCancelFetch(pMnode->pSdb, pIter); - sdbRelease(pMnode->pSdb, pSma); - goto _end; - } - } else { - info.dbInfo = *pDbInfo; - } - if (taosArrayPush(pInfos->pTsmaInfos, &info) == NULL) { - code = terrno; - sdbCancelFetch(pMnode->pSdb, pIter); - sdbRelease(pMnode->pSdb, pSma); - goto _end; - } - } - sdbRelease(pMnode->pSdb, pSma); - } - - // generate vg req map for (int32_t i = 0; i < pTbs->size; ++i) { SVDropTbReq *pTb = taosArrayGet(pTbs, i); TAOS_CHECK_GOTO(mndDropTbAdd(pMnode, pCtx->pVgMap, &vgInfo, pTb->name, pTb->suid, pTb->igNotExists), NULL, _end); - - SMDropTbTsmaInfos *pInfos = taosHashGet(pCtx->pTsmaMap, &pTb->suid, sizeof(pTb->suid)); - SArray *pVgInfos = NULL; - char buf[TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN + 1]; - char resTbFullName[TSDB_TABLE_FNAME_LEN + 1] = {0}; - for (int32_t j = 0; j < pInfos->pTsmaInfos->size; ++j) { - SMDropTbTsmaInfo *pInfo = taosArrayGet(pInfos->pTsmaInfos, j); - int32_t len = sprintf(buf, "%s_%s", pInfo->tsmaResTbNamePrefix, pTb->name); - len = taosCreateMD5Hash(buf, len); - len = snprintf(resTbFullName, TSDB_TABLE_FNAME_LEN + 1, "%s.%s", pInfo->tsmaResTbDbFName, buf); - uint32_t hashVal = taosGetTbHashVal(resTbFullName, len, pInfo->dbInfo.hashMethod, pInfo->dbInfo.hashPrefix, - pInfo->dbInfo.hashSuffix); - const SVgroupInfo *pVgInfo = taosArraySearch(pInfo->dbInfo.dbVgInfos, &hashVal, vgHashValCmp, TD_EQ); - void *p = taosStrdup(resTbFullName + strlen(pInfo->tsmaResTbDbFName) + TSDB_NAME_DELIMITER_LEN); - if (taosArrayPush(pCtx->pResTbNames, &p) == NULL) { - code = terrno; - goto _end; - } - TAOS_CHECK_GOTO(mndDropTbAdd(pMnode, pCtx->pVgMap, pVgInfo, p, pInfo->suid, true), NULL, _end); - } } _end: return code; @@ -4474,9 +4343,10 @@ static int32_t mndProcessFetchTtlExpiredTbs(SRpcMsg *pRsp) { code = mndInitDropTbsWithTsmaCtx(&pCtx); if (code) goto _end; - code = mndDropTbAddTsmaResTbsForSingleVg(pMnode, pCtx, rsp.pExpiredTbs, rsp.vgId); + code = mndDropTbForSingleVg(pMnode, pCtx, rsp.pExpiredTbs, rsp.vgId); if (code) goto _end; - if (mndCreateDropTbsTxnPrepare(pRsp, pCtx) == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + code = mndCreateDropTbsTxnPrepare(pRsp, pCtx); + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _end: if (pCtx) mndDestroyDropTbsWithTsmaCtx(pCtx); tDecoderClear(&decoder); diff --git a/source/dnode/snode/src/snodeInitApi.c b/source/dnode/snode/src/snodeInitApi.c index 680a2fd83c..4fe4333534 100644 --- a/source/dnode/snode/src/snodeInitApi.c +++ b/source/dnode/snode/src/snodeInitApi.c @@ -31,6 +31,7 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->streamStatePutParName = streamStatePutParName; pStore->streamStateGetParName = streamStateGetParName; + pStore->streamStateDeleteParName = streamStateDeleteParName; pStore->streamStateAddIfNotExist = streamStateAddIfNotExist; pStore->streamStateReleaseBuf = streamStateReleaseBuf; diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 653b47ff14..3c40100f9d 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -146,7 +146,7 @@ int32_t tqBuildFName(char** data, const char* path, char* name); int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name); // tq util -int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type); +int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type, EStreamType blockType); int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg); int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, int32_t type, int64_t sver, int64_t ever); @@ -158,6 +158,7 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c int32_t buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock, SArray* pTagArray, bool newSubTableRule, SVCreateTbReq** pReq); +int32_t tqExtractDropCtbDataBlock(const void* data, int32_t len, int64_t ver, void** pRefBlock, int32_t type); #define TQ_ERR_GO_TO_END(c) \ do { \ diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 80c04a3276..bbc58004d9 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -1551,7 +1551,7 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA _resume_delete: version = RSMA_EXEC_MSG_VER(msg); if ((code = tqExtractDelDataBlock(RSMA_EXEC_MSG_BODY(msg), RSMA_EXEC_MSG_LEN(msg), version, - &packData.pDataBlock, 1))) { + &packData.pDataBlock, 1, STREAM_DELETE_DATA))) { taosFreeQitem(msg); TAOS_CHECK_EXIT(code); } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index bd78f62cae..6195899566 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -758,7 +758,8 @@ int32_t tqBuildStreamTask(void* pTqObj, SStreamTask* pTask, int64_t nextProcessV } if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { - SWalFilterCond cond = {.deleteMsg = 1}; // delete msg also extract from wal files + bool scanDropCtb = pTask->subtableWithoutMd5 ? true : false; + SWalFilterCond cond = {.deleteMsg = 1, .scanDropCtb = scanDropCtb}; // delete msg also extract from wal files pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, &cond, pTask->id.taskId); if (pTask->exec.pWalReader == NULL) { tqError("vgId:%d failed init wal reader, code:%s", vgId, tstrerror(terrno)); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 95955e579f..d924e97ae3 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -366,8 +366,8 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con } else if (pCont->msgType == TDMT_VND_DELETE) { void* pBody = POINTER_SHIFT(pCont->body, sizeof(SMsgHead)); int32_t len = pCont->bodyLen - sizeof(SMsgHead); - - code = tqExtractDelDataBlock(pBody, len, ver, (void**)pItem, 0); + EStreamType blockType = STREAM_DELETE_DATA; + code = tqExtractDelDataBlock(pBody, len, ver, (void**)pItem, 0, blockType); if (code == TSDB_CODE_SUCCESS) { if (*pItem == NULL) { tqDebug("s-task:%s empty delete msg, discard it, len:%d, ver:%" PRId64, id, len, ver); @@ -382,6 +382,20 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con return code; } + } else if (pCont->msgType == TDMT_VND_DROP_TABLE && pReader->cond.scanDropCtb) { + void* pBody = POINTER_SHIFT(pCont->body, sizeof(SMsgHead)); + int32_t len = pCont->bodyLen - sizeof(SMsgHead); + code = tqExtractDropCtbDataBlock(pBody, len, ver, (void**)pItem, 0); + if (TSDB_CODE_SUCCESS == code) { + if (!*pItem) { + continue; + } else { + tqDebug("s-task:%s drop ctb msg extract from WAL, len:%d, ver:%"PRId64, id, len, ver); + } + } else { + terrno = code; + return code; + } } else { tqError("s-task:%s invalid msg type:%d, ver:%" PRId64, id, pCont->msgType, ver); return TSDB_CODE_STREAM_INTERNAL_ERROR; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index be41f7e99e..3f4ff7f3d9 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -53,6 +53,7 @@ static int32_t checkTagSchema(SStreamTask* pTask, SVnode* pVnode); static void reubuildAndSendMultiResBlock(SStreamTask* pTask, const SArray* pBlocks, SVnode* pVnode, int64_t earlyTs); static int32_t handleResultBlockMsg(SStreamTask* pTask, SSDataBlock* pDataBlock, int32_t index, SVnode* pVnode, int64_t earlyTs); +static int32_t doWaitForDstTableDropped(SVnode* pVnode, SStreamTask* pTask, const char* dstTableName); int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq, const char* pIdStr, bool newSubTableRule) { @@ -138,7 +139,7 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p return 0; } -static int32_t encodeCreateChildTableForRPC(SVCreateTbBatchReq* pReqs, int32_t vgId, void** pBuf, int32_t* contLen) { +static int32_t encodeCreateChildTableForRPC(void* pReqs, int32_t vgId, void** pBuf, int32_t* contLen) { int32_t ret = 0; tEncodeSize(tEncodeSVCreateTbBatchReq, pReqs, *contLen, ret); @@ -170,17 +171,50 @@ end: return ret; } -static int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) { +static int32_t encodeDropChildTableForRPC(void* pReqs, int32_t vgId, void** ppBuf, int32_t *contLen) { + int32_t code = 0; + SEncoder ec = {0}; + tEncodeSize(tEncodeSVDropTbBatchReq, pReqs, *contLen, code); + if (code < 0) { + code = TSDB_CODE_INVALID_MSG; + goto end; + } + *contLen += sizeof(SMsgHead); + *ppBuf = rpcMallocCont(*contLen); + + if (!*ppBuf) { + code = terrno; + goto end; + } + + ((SMsgHead*)(*ppBuf))->vgId = vgId; + ((SMsgHead*)(*ppBuf))->contLen = htonl(*contLen); + + tEncoderInit(&ec, POINTER_SHIFT(*ppBuf, sizeof(SMsgHead)), (*contLen) - sizeof(SMsgHead)); + code = tEncodeSVDropTbBatchReq(&ec, pReqs); + tEncoderClear(&ec); + if (code < 0) { + rpcFreeCont(*ppBuf); + *ppBuf = NULL; + *contLen = 0; + code = TSDB_CODE_INVALID_MSG; + goto end; + } +end: + return code; +} + +static int32_t tqPutReqToQueue(SVnode* pVnode, void* pReqs, int32_t(*encoder)(void* pReqs, int32_t vgId, void** ppBuf, int32_t *contLen), tmsg_t msgType) { void* buf = NULL; int32_t tlen = 0; - int32_t code = encodeCreateChildTableForRPC(pReqs, TD_VID(pVnode), &buf, &tlen); + int32_t code = encoder(pReqs, TD_VID(pVnode), &buf, &tlen); if (code) { tqError("vgId:%d failed to encode create table msg, create table failed, code:%s", TD_VID(pVnode), tstrerror(code)); return code; } - SRpcMsg msg = {.msgType = TDMT_VND_CREATE_TABLE, .pCont = buf, .contLen = tlen}; + SRpcMsg msg = {.msgType = msgType, .pCont = buf, .contLen = tlen}; code = tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg); if (code) { tqError("failed to put into write-queue since %s", terrstr()); @@ -388,7 +422,7 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S } reqs.nReqs = taosArrayGetSize(reqs.pArray); - code = tqPutReqToQueue(pVnode, &reqs); + code = tqPutReqToQueue(pVnode, &reqs, encodeCreateChildTableForRPC, TDMT_VND_CREATE_TABLE); if (code != TSDB_CODE_SUCCESS) { tqError("s-task:%s failed to send create table msg", id); } @@ -399,6 +433,61 @@ _end: return code; } +static int32_t doBuildAndSendDropTableMsg(SVnode* pVnode, char* pStbFullname, SSDataBlock* pDataBlock, + SStreamTask* pTask, int64_t suid) { + int32_t lino = 0; + int32_t code = 0; + int32_t rows = pDataBlock->info.rows; + const char* id = pTask->id.idStr; + SVDropTbBatchReq batchReq = {0}; + SVDropTbReq req = {0}; + + if (rows <= 0 || rows > 1 || pTask->subtableWithoutMd5 == 0) return TSDB_CODE_SUCCESS; + + batchReq.pArray = taosArrayInit(rows, sizeof(SVDropTbReq)); + if (!batchReq.pArray) return terrno; + batchReq.nReqs = rows; + req.suid = suid; + req.igNotExists = true; + + SColumnInfoData* pTbNameCol = taosArrayGet(pDataBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); + char tbName[TSDB_TABLE_NAME_LEN + 1] = {0}; + int32_t i = 0; + void* pData = colDataGetVarData(pTbNameCol, i); + memcpy(tbName, varDataVal(pData), varDataLen(pData)); + tbName[varDataLen(pData) + 1] = 0; + req.name = tbName; + if (taosArrayPush(batchReq.pArray, &req) == NULL) { + TSDB_CHECK_CODE(terrno, lino, _exit); + } + + SMetaReader mr = {0}; + metaReaderDoInit(&mr, pVnode->pMeta, META_READER_LOCK); + + code = metaGetTableEntryByName(&mr, tbName); + if (TSDB_CODE_SUCCESS == code && isValidDstChildTable(&mr, TD_VID(pVnode), tbName, pTask->outputInfo.tbSink.stbUid)) { + STableSinkInfo* pTableSinkInfo = NULL; + bool alreadyCached = doGetSinkTableInfoFromCache(pTask->outputInfo.tbSink.pTbInfo, pDataBlock->info.id.groupId, &pTableSinkInfo); + if (alreadyCached) { + pTableSinkInfo->uid = mr.me.uid; + } + } + metaReaderClear(&mr); + tqDebug("s-task:%s build drop %d table(s) msg", id, rows); + code = tqPutReqToQueue(pVnode, &batchReq, encodeDropChildTableForRPC, TDMT_VND_DROP_TABLE); + TSDB_CHECK_CODE(code, lino, _exit); + + + code = doWaitForDstTableDropped(pVnode, pTask, tbName); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (batchReq.pArray) { + taosArrayDestroy(batchReq.pArray); + } + return code; +} + int32_t doBuildAndSendSubmitMsg(SVnode* pVnode, SStreamTask* pTask, SSubmitReq2* pReq, int32_t numOfBlocks) { const char* id = pTask->id.idStr; int32_t vgId = TD_VID(pVnode); @@ -807,6 +896,40 @@ int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkI return TSDB_CODE_SUCCESS; } +static int32_t doWaitForDstTableDropped(SVnode* pVnode, SStreamTask* pTask, const char* dstTableName) { + int32_t vgId = TD_VID(pVnode); + int64_t suid = pTask->outputInfo.tbSink.stbUid; + const char* id = pTask->id.idStr; + + while (1) { + if (streamTaskShouldStop(pTask)) { + tqDebug("s-task:%s task will stop, quit from waiting for table:%s drop", id, dstTableName); + return TSDB_CODE_STREAM_EXEC_CANCELLED; + } + SMetaReader mr = {0}; + metaReaderDoInit(&mr, pVnode->pMeta, META_READER_LOCK); + int32_t code = metaGetTableEntryByName(&mr, dstTableName); + if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + metaReaderClear(&mr); + break; + } else if (TSDB_CODE_SUCCESS == code) { + if (isValidDstChildTable(&mr, vgId, dstTableName, suid)) { + metaReaderClear(&mr); + taosMsleep(100); + tqDebug("s-task:%s wait 100ms for table:%s drop", id, dstTableName); + } else { + metaReaderClear(&mr); + break; + } + } else { + tqError("s-task:%s failed to wait for table:%s drop", id, dstTableName); + metaReaderClear(&mr); + return terrno; + } + } + return TSDB_CODE_SUCCESS; +} + int32_t doCreateSinkTableInfo(const char* pDstTableName, STableSinkInfo** pInfo) { int32_t nameLen = strlen(pDstTableName); (*pInfo) = taosMemoryCalloc(1, sizeof(STableSinkInfo) + nameLen + 1); @@ -1032,7 +1155,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { } bool onlySubmitData = hasOnlySubmitData(pBlocks, numOfBlocks); - if (!onlySubmitData) { + if (!onlySubmitData || pTask->subtableWithoutMd5 == 1) { tqDebug("vgId:%d, s-task:%s write %d stream resBlock(s) into table, has delete block, submit one-by-one", vgId, id, numOfBlocks); @@ -1052,6 +1175,8 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { code = doBuildAndSendCreateTableMsg(pVnode, stbFullName, pDataBlock, pTask, suid); } else if (pDataBlock->info.type == STREAM_CHECKPOINT) { continue; + } else if (pDataBlock->info.type == STREAM_DROP_CHILD_TABLE && pTask->subtableWithoutMd5) { + code = doBuildAndSendDropTableMsg(pVnode, stbFullName, pDataBlock, pTask, suid); } else { code = handleResultBlockMsg(pTask, pDataBlock, i, pVnode, earlyTs); } diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index e066938fc0..a92049e5f3 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -572,7 +572,7 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* return 0; } -int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type) { +int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type, EStreamType blockType) { int32_t code = 0; int32_t line = 0; SDecoder* pCoder = &(SDecoder){0}; @@ -593,7 +593,7 @@ int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void* } SSDataBlock* pDelBlock = NULL; - code = createSpecialDataBlock(STREAM_DELETE_DATA, &pDelBlock); + code = createSpecialDataBlock(blockType, &pDelBlock); TSDB_CHECK_CODE(code, line, END); code = blockDataEnsureCapacity(pDelBlock, numOfTables); @@ -751,3 +751,45 @@ int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, b return TSDB_CODE_SUCCESS; } + +int32_t tqExtractDropCtbDataBlock(const void* data, int32_t len, int64_t ver, void** pRefBlock, int32_t type) { + int32_t code = 0; + int32_t lino = 0; + SDecoder dc = {0}; + SVDropTbBatchReq batchReq = {0}; + tDecoderInit(&dc, (uint8_t*)data, len); + code = tDecodeSVDropTbBatchReq(&dc, &batchReq); + TSDB_CHECK_CODE(code, lino, _exit); + if (batchReq.nReqs <= 0) goto _exit; + + SSDataBlock* pBlock = NULL; + code = createSpecialDataBlock(STREAM_DROP_CHILD_TABLE, &pBlock); + TSDB_CHECK_CODE(code, lino, _exit); + + code = blockDataEnsureCapacity(pBlock, batchReq.nReqs); + TSDB_CHECK_CODE(code, lino, _exit); + + pBlock->info.rows = batchReq.nReqs; + pBlock->info.version = ver; + for (int32_t i = 0; i < batchReq.nReqs; ++i) { + SVDropTbReq* pReq = batchReq.pReqs + i; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); + TSDB_CHECK_NULL(pCol, code, lino, _exit, terrno); + code = colDataSetVal(pCol, i, (const char* )&pReq->uid, false); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0, pRefBlock); + TSDB_CHECK_CODE(code, lino, _exit); + ((SStreamRefDataBlock*)(*pRefBlock))->type = STREAM_INPUT__REF_DATA_BLOCK; + ((SStreamRefDataBlock*)(*pRefBlock))->pBlock = pBlock; + +_exit: + tDecoderClear(&dc); + if (TSDB_CODE_SUCCESS != code) { + tqError("faled to extract drop ctb data block, line:%d code:%s", lino, tstrerror(code)); + blockDataCleanup(pBlock); + taosMemoryFree(pBlock); + } + return code; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index d508d75922..c7626dcf36 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -613,6 +613,16 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 singleTableLastTs = pColVal->rowKey.ts; } + if (p->colVal.value.type != pColVal->colVal.value.type) { + // check for type/cid mismatch + tsdbError("last cache type mismatch, uid:%" PRIu64 + ", schema-type:%d, slotId:%d, cache-type:%d, cache-col:%d", + uid, p->colVal.value.type, slotIds[k], pColVal->colVal.value.type, pColVal->colVal.cid); + taosArrayClearEx(pRow, tsdbCacheFreeSLastColItem); + code = TSDB_CODE_INVALID_PARA; + goto _end; + } + if (!IS_VAR_DATA_TYPE(pColVal->colVal.value.type)) { p->colVal = pColVal->colVal; } else { diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index d688d1323d..0ac0ee1b8f 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -147,6 +147,7 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->streamStatePutParName = streamStatePutParName; pStore->streamStateGetParName = streamStateGetParName; + pStore->streamStateDeleteParName = streamStateDeleteParName; pStore->streamStateAddIfNotExist = streamStateAddIfNotExist; pStore->streamStateReleaseBuf = streamStateReleaseBuf; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index dd13c975cf..6702b8b588 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -50,6 +50,8 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessConfigChangeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessArbCheckSyncReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDropTSmaCtbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, + SRpcMsg *pOriginRpc); static int32_t vnodePreCheckAssignedLogSyncd(SVnode *pVnode, char *member0Token, char *member1Token); static int32_t vnodeCheckAssignedLogSyncd(SVnode *pVnode, char *member0Token, char *member1Token); @@ -481,6 +483,61 @@ static int32_t vnodePreProcessArbCheckSyncMsg(SVnode *pVnode, SRpcMsg *pMsg) { return code; } +int32_t vnodePreProcessDropTbMsg(SVnode* pVnode, SRpcMsg* pMsg) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t size = 0; + SDecoder dc = {0}; + SEncoder ec = {0}; + SVDropTbBatchReq receivedBatchReqs = {0}; + SVDropTbBatchReq sentBatchReqs = {0}; + + tDecoderInit(&dc, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead)); + + code = tDecodeSVDropTbBatchReq(&dc, &receivedBatchReqs); + if (code < 0) { + terrno = code; + TSDB_CHECK_CODE(code, lino, _exit); + } + sentBatchReqs.pArray = taosArrayInit(receivedBatchReqs.nReqs, sizeof(SVDropTbReq)); + if (!sentBatchReqs.pArray) { + code = terrno; + goto _exit; + } + + for (int32_t i = 0; i < receivedBatchReqs.nReqs; ++i) { + SVDropTbReq* pReq = receivedBatchReqs.pReqs + i; + tb_uid_t uid = metaGetTableEntryUidByName(pVnode->pMeta, pReq->name); + if (uid == 0) { + vWarn("vgId:%d, preprocess drop ctb: %s not found", TD_VID(pVnode), pReq->name); + continue; + } + pReq->uid = uid; + vDebug("vgId:%d %s for: %s, uid: %"PRId64, TD_VID(pVnode), __func__, pReq->name, pReq->uid); + if (taosArrayPush(sentBatchReqs.pArray, pReq) == NULL) { + code = terrno; + goto _exit; + } + } + sentBatchReqs.nReqs = sentBatchReqs.pArray->size; + + tEncodeSize(tEncodeSVDropTbBatchReq, &sentBatchReqs, size, code); + tEncoderInit(&ec, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), size); + code = tEncodeSVDropTbBatchReq(&ec, &sentBatchReqs); + tEncoderClear(&ec); + if (code != TSDB_CODE_SUCCESS) { + vError("vgId:%d %s failed to encode drop tb batch req: %s", TD_VID(pVnode), __func__, tstrerror(code)); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + tDecoderClear(&dc); + if (sentBatchReqs.pArray) { + taosArrayDestroy(sentBatchReqs.pArray); + } + return code; +} + int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t code = 0; @@ -507,6 +564,9 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { case TDMT_VND_ARB_CHECK_SYNC: { code = vnodePreProcessArbCheckSyncMsg(pVnode, pMsg); } break; + case TDMT_VND_DROP_TABLE: { + code = vnodePreProcessDropTbMsg(pVnode, pMsg); + } break; default: break; } @@ -1110,7 +1170,6 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, STbUidStore *pStore = NULL; SArray *tbUids = NULL; SArray *tbNames = NULL; - pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP; pRsp->code = TSDB_CODE_SUCCESS; pRsp->pCont = NULL; @@ -2512,3 +2571,4 @@ _OVER: int32_t vnodeAsyncCompact(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { return 0; } int32_t tsdbAsyncCompact(STsdb *tsdb, const STimeWindow *tw, bool sync) { return 0; } #endif + diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 6272ac7049..5afdf87afb 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -35,6 +35,9 @@ extern SConfig* tsCfg; static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRetrieveTableRsp** pRsp) { + if (NULL == pBlock || NULL == pRsp) { + return TSDB_CODE_INVALID_PARA; + } size_t dataEncodeBufSize = blockGetEncodeSize(pBlock); size_t rspSize = sizeof(SRetrieveTableRsp) + dataEncodeBufSize + PAYLOAD_PREFIX_LEN; *pRsp = taosMemoryCalloc(1, rspSize); @@ -216,6 +219,9 @@ static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, static int32_t execDescribe(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp, int8_t biMode) { SDescribeStmt* pDesc = (SDescribeStmt*)pStmt; + if (NULL == pDesc || NULL == pDesc->pMeta) { + return TSDB_CODE_INVALID_PARA; + } int32_t numOfRows = TABLE_TOTAL_COL_NUM(pDesc->pMeta); SSDataBlock* pBlock = NULL; @@ -505,7 +511,7 @@ static int32_t buildCreateViewResultDataBlock(SSDataBlock** pOutput) { return code; } -void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { +static void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { for (int32_t i = 0; i < pCfg->numOfColumns; ++i) { SSchema* pSchema = pCfg->pSchemas + i; #define LTYPE_LEN (32 + 60) // 60 byte for compress info @@ -539,7 +545,7 @@ void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { } } -void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) { +static void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) { for (int32_t i = 0; i < pCfg->numOfTags; ++i) { SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; char type[32]; @@ -558,7 +564,7 @@ void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) { } } -void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) { +static void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) { for (int32_t i = 0; i < pCfg->numOfTags; ++i) { SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), @@ -566,7 +572,7 @@ void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) { } } -int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { +static int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { int32_t code = TSDB_CODE_SUCCESS; SArray* pTagVals = NULL; STag* pTag = (STag*)pCfg->pTags; @@ -643,7 +649,7 @@ _exit: return code; } -void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg* pCfg) { +static void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg* pCfg) { if (pCfg->commentLen > 0) { *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), " COMMENT '%s'", pCfg->pComment); @@ -997,7 +1003,7 @@ static int32_t createSelectResultDataBlock(SNodeList* pProjects, SSDataBlock** p return code; } -int32_t buildSelectResultDataBlock(SNodeList* pProjects, SSDataBlock* pBlock) { +static int32_t buildSelectResultDataBlock(SNodeList* pProjects, SSDataBlock* pBlock) { QRY_ERR_RET(blockDataEnsureCapacity(pBlock, 1)); int32_t index = 0; diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 42c214fac7..3ab739334d 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -30,8 +30,8 @@ char *gJoinTypeStr[JOIN_TYPE_MAX_VALUE][JOIN_STYPE_MAX_VALUE] = { /*FULL*/ {"Full Join", "Full Join", NULL, NULL, NULL, NULL}, }; -int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pRes); -int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, bool singleChannel); +static int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pRes); +static int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, bool singleChannel); char *qExplainGetDynQryCtrlType(EDynQueryType type) { switch (type) { @@ -118,7 +118,7 @@ void qExplainFreeCtx(SExplainCtx *pCtx) { taosMemoryFree(pCtx); } -int32_t qExplainInitCtx(SExplainCtx **pCtx, SHashObj *groupHash, bool verbose, double ratio, EExplainMode mode) { +static int32_t qExplainInitCtx(SExplainCtx **pCtx, SHashObj *groupHash, bool verbose, double ratio, EExplainMode mode) { int32_t code = 0; SExplainCtx *ctx = taosMemoryCalloc(1, sizeof(SExplainCtx)); if (NULL == ctx) { @@ -158,7 +158,7 @@ _return: QRY_RET(code); } -int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNodeList **pChildren) { +static int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNodeList **pChildren) { int32_t tlen = 0; SNodeList *pPhysiChildren = pNode->pChildren; @@ -180,7 +180,7 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo return TSDB_CODE_SUCCESS; } -int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, SExplainGroup *group) { +static int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, SExplainGroup *group) { *pExecInfo = taosArrayInit(group->nodeNum, sizeof(SExplainExecInfo)); if (NULL == (*pExecInfo)) { qError("taosArrayInit %d explainExecInfo failed", group->nodeNum); @@ -217,7 +217,7 @@ int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, S return TSDB_CODE_SUCCESS; } -int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pResNode) { +static int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pResNode) { if (NULL == pNode) { *pResNode = NULL; qError("physical node is NULL"); @@ -250,7 +250,7 @@ _return: QRY_RET(code); } -int32_t qExplainBufAppendExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) { +static int32_t qExplainBufAppendExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) { int32_t tlen = *len; int32_t nodeNum = taosArrayGetSize(pExecInfo); SExplainExecInfo maxExecInfo = {0}; @@ -275,7 +275,7 @@ int32_t qExplainBufAppendExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) { return TSDB_CODE_SUCCESS; } -int32_t qExplainBufAppendVerboseExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) { +static int32_t qExplainBufAppendVerboseExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) { int32_t tlen = 0; bool gotVerbose = false; int32_t nodeNum = taosArrayGetSize(pExecInfo); @@ -297,7 +297,7 @@ int32_t qExplainBufAppendVerboseExecInfo(SArray *pExecInfo, char *tbuf, int32_t return TSDB_CODE_SUCCESS; } -int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t level) { +static int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t level) { SQueryExplainRowInfo row = {0}; row.buf = taosMemoryMalloc(len); if (NULL == row.buf) { @@ -362,7 +362,7 @@ static char* qExplainGetScanDataLoad(STableScanPhysiNode* pScan) { return "unknown"; } -int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { +static int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { int32_t tlen = 0; bool isVerboseLine = false; char *tbuf = ctx->tbuf; @@ -1900,7 +1900,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i return TSDB_CODE_SUCCESS; } -int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { +static int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { if (NULL == pResNode) { qError("explain res node is NULL"); QRY_ERR_RET(TSDB_CODE_APP_ERROR); @@ -1915,7 +1915,7 @@ int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32 return TSDB_CODE_SUCCESS; } -int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, bool singleChannel) { +static int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, bool singleChannel) { SExplainResNode *node = NULL; int32_t code = 0; SExplainCtx *ctx = (SExplainCtx *)pCtx; @@ -1940,7 +1940,7 @@ _return: QRY_RET(code); } -int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { +static int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { int32_t code = 0; SSDataBlock *pBlock = NULL; SExplainCtx *pCtx = (SExplainCtx *)ctx; @@ -1997,7 +1997,7 @@ _return: QRY_RET(code); } -int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { +static int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { int32_t code = 0; SNodeListNode *plans = NULL; int32_t taskNum = 0; @@ -2080,7 +2080,7 @@ _return: QRY_RET(code); } -int32_t qExplainAppendPlanRows(SExplainCtx *pCtx) { +static int32_t qExplainAppendPlanRows(SExplainCtx *pCtx) { if (EXPLAIN_MODE_ANALYZE != pCtx->mode) { return TSDB_CODE_SUCCESS; } @@ -2103,7 +2103,7 @@ int32_t qExplainAppendPlanRows(SExplainCtx *pCtx) { return TSDB_CODE_SUCCESS; } -int32_t qExplainGenerateRsp(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) { +static int32_t qExplainGenerateRsp(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) { QRY_ERR_RET(qExplainAppendGroupResRows(pCtx, pCtx->rootGroupId, 0, false)); QRY_ERR_RET(qExplainAppendPlanRows(pCtx)); QRY_ERR_RET(qExplainGetRspFromCtx(pCtx, pRsp)); diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 014b538375..9a49076b6b 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -7,7 +7,7 @@ if(${TD_DARWIN}) endif(${TD_DARWIN}) if(${BUILD_WITH_ANALYSIS}) - add_definitions(-DUSE_ANAL) + add_definitions(-DUSE_ANALYTICS) endif() target_link_libraries(executor diff --git a/source/libs/executor/src/anomalywindowoperator.c b/source/libs/executor/src/anomalywindowoperator.c index d03e527c2b..94cc5d9129 100644 --- a/source/libs/executor/src/anomalywindowoperator.c +++ b/source/libs/executor/src/anomalywindowoperator.c @@ -19,14 +19,14 @@ #include "functionMgt.h" #include "operator.h" #include "querytask.h" -#include "tanal.h" +#include "tanalytics.h" #include "tcommon.h" #include "tcompare.h" #include "tdatablock.h" #include "tjson.h" #include "ttime.h" -#ifdef USE_ANAL +#ifdef USE_ANALYTICS typedef struct { SArray* blocks; // SSDataBlock* @@ -55,7 +55,7 @@ typedef struct { static void anomalyDestroyOperatorInfo(void* param); static int32_t anomalyAggregateNext(SOperatorInfo* pOperator, SSDataBlock** ppRes); -static void anomalyAggregateBlocks(SOperatorInfo* pOperator); +static int32_t anomalyAggregateBlocks(SOperatorInfo* pOperator); static int32_t anomalyCacheBlock(SAnomalyWindowOperatorInfo* pInfo, SSDataBlock* pBlock); int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo, @@ -78,6 +78,7 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p code = TSDB_CODE_ANAL_ALGO_NOT_FOUND; goto _error; } + if (taosAnalGetAlgoUrl(pInfo->algoName, ANAL_ALGO_TYPE_ANOMALY_DETECT, pInfo->algoUrl, sizeof(pInfo->algoUrl)) != 0) { qError("failed to get anomaly_window algorithm url from %s", pInfo->algoName); code = TSDB_CODE_ANAL_ALGO_NOT_LOAD; @@ -198,7 +199,9 @@ static int32_t anomalyAggregateNext(SOperatorInfo* pOperator, SSDataBlock** ppRe QUERY_CHECK_CODE(code, lino, _end); } else { qDebug("group:%" PRId64 ", read finish for new group coming, blocks:%d", pSupp->groupId, numOfBlocks); - anomalyAggregateBlocks(pOperator); + code = anomalyAggregateBlocks(pOperator); + QUERY_CHECK_CODE(code, lino, _end); + pSupp->groupId = pBlock->info.id.groupId; numOfBlocks = 1; pSupp->cachedRows = pBlock->info.rows; @@ -217,7 +220,7 @@ static int32_t anomalyAggregateNext(SOperatorInfo* pOperator, SSDataBlock** ppRe if (numOfBlocks > 0) { qDebug("group:%" PRId64 ", read finish, blocks:%d", pInfo->anomalySup.groupId, numOfBlocks); - anomalyAggregateBlocks(pOperator); + code = anomalyAggregateBlocks(pOperator); } int64_t cost = taosGetTimestampUs() - st; @@ -229,6 +232,7 @@ _end: pTaskInfo->code = code; T_LONG_JMP(pTaskInfo->env, code); } + (*ppRes) = (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes; return code; } @@ -338,8 +342,8 @@ static int32_t anomalyAnalysisWindow(SOperatorInfo* pOperator) { SAnalBuf analBuf = {.bufType = ANAL_BUF_TYPE_JSON}; char dataBuf[64] = {0}; int32_t code = 0; + int64_t ts = 0; - int64_t ts = 0; // int64_t ts = taosGetTimestampMs(); snprintf(analBuf.fileName, sizeof(analBuf.fileName), "%s/tdengine-anomaly-%" PRId64 "-%" PRId64, tsTempDir, ts, pSupp->groupId); @@ -431,6 +435,7 @@ _OVER: if (code != 0) { qError("failed to analysis window since %s", tstrerror(code)); } + taosAnalBufDestroy(&analBuf); if (pJson != NULL) tjsonDelete(pJson); return code; @@ -473,7 +478,7 @@ static int32_t anomalyBuildResult(SOperatorInfo* pOperator) { return code; } -static void anomalyAggregateBlocks(SOperatorInfo* pOperator) { +static int32_t anomalyAggregateBlocks(SOperatorInfo* pOperator) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SAnomalyWindowOperatorInfo* pInfo = pOperator->info; @@ -623,6 +628,8 @@ _OVER: pSupp->curWin.ekey = 0; pSupp->curWin.skey = 0; pSupp->curWinIndex = 0; + + return code; } #else diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 042fcf0120..7222f2d297 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -121,10 +121,10 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn } } else { pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; - qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 ", try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows, - pExchangeInfo->loadInfo.totalRows, i + 1, totalSources); + qDebug("%s vgId:%d, clientId:0x%" PRIx64 " taskId:0x%" PRIx64 + " execId:%d index:%d completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", try next %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->clientId, pSource->taskId, pSource->execId, i, + pDataInfo->totalRows, pExchangeInfo->loadInfo.totalRows, i + 1, totalSources); taosMemoryFreeClear(pDataInfo->pRsp); } break; @@ -141,17 +141,17 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn if (pRsp->completed == 1) { pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 + qDebug("%s fetch msg rsp from vgId:%d, clientId:0x%" PRIx64 " taskId:0x%" PRIx64 " execId:%d index:%d completed, blocks:%d, numOfRows:%" PRId64 ", rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", total:%.2f Kb, try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks, - pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, i + 1, - totalSources); + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->clientId, pSource->taskId, pSource->execId, i, + pRsp->numOfBlocks, pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, + pLoadInfo->totalSize / 1024.0, i + 1, totalSources); } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d blocks:%d, numOfRows:%" PRId64 - ", totalRows:%" PRIu64 ", total:%.2f Kb", - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRsp->numOfBlocks, - pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0); + qDebug("%s fetch msg rsp from vgId:%d, clientId:0x%" PRIx64 " taskId:0x%" PRIx64 + " execId:%d blocks:%d, numOfRows:%" PRId64 ", totalRows:%" PRIu64 ", total:%.2f Kb", + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->clientId, pSource->taskId, pSource->execId, + pRsp->numOfBlocks, pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0); } taosMemoryFreeClear(pDataInfo->pRsp); @@ -640,9 +640,9 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas if (pSource->localExec) { SDataBuf pBuf = {0}; - int32_t code = - (*pTaskInfo->localFetch.fp)(pTaskInfo->localFetch.handle, pSource->schedId, pTaskInfo->id.queryId, - pSource->taskId, 0, pSource->execId, &pBuf.pData, pTaskInfo->localFetch.explainRes); + int32_t code = (*pTaskInfo->localFetch.fp)(pTaskInfo->localFetch.handle, pSource->schedId, pTaskInfo->id.queryId, + pSource->clientId, pSource->taskId, 0, pSource->execId, &pBuf.pData, + pTaskInfo->localFetch.explainRes); code = loadRemoteDataCallback(pWrapper, &pBuf, code); QUERY_CHECK_CODE(code, lino, _end); taosMemoryFree(pWrapper); @@ -650,6 +650,7 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas SResFetchReq req = {0}; req.header.vgId = pSource->addr.nodeId; req.sId = pSource->schedId; + req.clientId = pSource->clientId; req.taskId = pSource->taskId; req.queryId = pTaskInfo->id.queryId; req.execId = pSource->execId; @@ -691,9 +692,10 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas freeOperatorParam(req.pOpParam, OP_GET_PARAM); - qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %p, %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, - pSource->execId, pExchangeInfo, sourceIndex, totalSources); + qDebug("%s build fetch msg and send to vgId:%d, ep:%s, clientId:0x%" PRIx64 " taskId:0x%" PRIx64 + ", execId:%d, %p, %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->clientId, + pSource->taskId, pSource->execId, pExchangeInfo, sourceIndex, totalSources); // send the fetch remote task result reques SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); @@ -974,8 +976,9 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { } if (pDataInfo->code != TSDB_CODE_SUCCESS) { - qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo), - pSource->addr.nodeId, pSource->taskId, pSource->execId, tstrerror(pDataInfo->code)); + qError("%s vgId:%d, clientId:0x%" PRIx64 " taskID:0x%" PRIx64 " execId:%d error happens, code:%s", + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->clientId, pSource->taskId, pSource->execId, + tstrerror(pDataInfo->code)); pOperator->pTaskInfo->code = pDataInfo->code; return pOperator->pTaskInfo->code; } @@ -984,10 +987,10 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; if (pRsp->numOfRows == 0) { - qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 " try next", - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pExchangeInfo->current + 1, - pDataInfo->totalRows, pLoadInfo->totalRows); + qDebug("%s vgId:%d, clientId:0x%" PRIx64 " taskID:0x%" PRIx64 + " execId:%d %d of total completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 " try next", + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->clientId, pSource->taskId, pSource->execId, + pExchangeInfo->current + 1, pDataInfo->totalRows, pLoadInfo->totalRows); pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; pExchangeInfo->current += 1; @@ -1002,19 +1005,19 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; if (pRsp->completed == 1) { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64 + qDebug("%s fetch msg rsp from vgId:%d, clientId:0x%" PRIx64 " taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64 ", rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, - pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, - totalSources); + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->clientId, pSource->taskId, pSource->execId, + pRetrieveRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, + pExchangeInfo->current + 1, totalSources); pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; pExchangeInfo->current += 1; } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64 ", totalRows:%" PRIu64 - ", totalBytes:%" PRIu64, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, - pLoadInfo->totalRows, pLoadInfo->totalSize); + qDebug("%s fetch msg rsp from vgId:%d, clientId:0x%" PRIx64 " taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64 + ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->clientId, pSource->taskId, pSource->execId, + pRetrieveRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize); } updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, startTs, pOperator); diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 1b823bf69d..af8e01be5e 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -1083,18 +1083,13 @@ void cleanupBasicInfo(SOptrBasicInfo* pInfo) { bool groupbyTbname(SNodeList* pGroupList) { bool bytbname = false; - if (LIST_LENGTH(pGroupList) == 1) { - SNode* p = nodesListGetNode(pGroupList, 0); - if (!p) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); - return false; - } - if (p->type == QUERY_NODE_FUNCTION) { - // partition by tbname/group by tbname - bytbname = (strcmp(((struct SFunctionNode*)p)->functionName, "tbname") == 0); + SNode*pNode = NULL; + FOREACH(pNode, pGroupList) { + if (pNode->type == QUERY_NODE_FUNCTION) { + bytbname = (strcmp(((struct SFunctionNode*)pNode)->functionName, "tbname") == 0); + break; } } - return bytbname; } diff --git a/source/libs/executor/src/forecastoperator.c b/source/libs/executor/src/forecastoperator.c index 0afa933ee8..20dc9e28ba 100644 --- a/source/libs/executor/src/forecastoperator.c +++ b/source/libs/executor/src/forecastoperator.c @@ -19,14 +19,14 @@ #include "operator.h" #include "querytask.h" #include "storageapi.h" -#include "tanal.h" +#include "tanalytics.h" #include "tcommon.h" #include "tcompare.h" #include "tdatablock.h" #include "tfill.h" #include "ttime.h" -#ifdef USE_ANAL +#ifdef USE_ANALYTICS typedef struct { char algoName[TSDB_ANAL_ALGO_NAME_LEN]; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index fec35c3371..d6e3d26267 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -1326,7 +1326,6 @@ int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTag int32_t winCode = TSDB_CODE_SUCCESS; code = pAPI->streamStateGetParName(pState, groupId, &pValue, true, &winCode); QUERY_CHECK_CODE(code, lino, _end); - if (winCode != TSDB_CODE_SUCCESS) { SSDataBlock* pTmpBlock = NULL; code = blockCopyOneRow(pSrcBlock, rowId, &pTmpBlock); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 5b5d5c5d11..84dde6a579 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -289,6 +289,7 @@ static int32_t doSetTagColumnData(STableScanBase* pTableScanInfo, SSDataBlock* p pTaskInfo, &pTableScanInfo->metaCache); // ignore the table not exists error, since this table may have been dropped during the scan procedure. if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + if (pTaskInfo->streamInfo.pState) blockDataCleanup(pBlock); code = 0; } } @@ -3038,10 +3039,6 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes, pBlockInfo->rows, pTaskInfo, &pTableScanInfo->base.metaCache); // ignore the table not exists error, since this table may have been dropped during the scan procedure. - if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { - code = 0; - } - if (code) { blockDataFreeRes((SSDataBlock*)pBlock); QUERY_CHECK_CODE(code, lino, _end); @@ -3535,6 +3532,46 @@ static int32_t copyGetResultBlock(SSDataBlock* dest, TSKEY start, TSKEY end) { return appendDataToSpecialBlock(dest, &start, &end, NULL, NULL, NULL); } +static int32_t deletePartName(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t *deleteNum) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + for (int32_t i = 0; i < pBlock->info.rows; i++) { + // uid is the same as gid + SColumnInfoData* pGpIdCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); + SColumnInfoData* pTbnameCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); + int64_t* gpIdCol = (int64_t*)pGpIdCol->pData; + void* pParName = NULL; + int32_t winCode = 0; + code = pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, gpIdCol[i], + &pParName, false, &winCode); + if (TSDB_CODE_SUCCESS == code && winCode != 0) { + qDebug("delete stream part Name for:%"PRId64 " not found", gpIdCol[i]); + colDataSetNULL(pTbnameCol, i); + continue; + } + (*deleteNum)++; + QUERY_CHECK_CODE(code, lino, _end); + char varTbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE + 1] = {0}; + varDataSetLen(varTbName, strlen(pParName)); + int64_t len = tsnprintf(varTbName + VARSTR_HEADER_SIZE, TSDB_TABLE_NAME_LEN + 1, "%s", pParName); + code = colDataSetVal(pTbnameCol, i, varTbName, false); + qDebug("delete stream part for:%"PRId64 " res tb: %s", gpIdCol[i], (char*)pParName); + pInfo->stateStore.streamStateFreeVal(pParName); + QUERY_CHECK_CODE(code, lino, _end); + code = pInfo->stateStore.streamStateDeleteParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, gpIdCol[i]); + QUERY_CHECK_CODE(code, lino, _end); + pBlock->info.id.groupId = gpIdCol[i]; + // currently, only one valid row in pBlock + memcpy(pBlock->info.parTbName, varTbName + VARSTR_HEADER_SIZE, TSDB_TABLE_NAME_LEN + 1); + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; +} + static int32_t doStreamScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { // NOTE: this operator does never check if current status is done or not int32_t code = TSDB_CODE_SUCCESS; @@ -3774,6 +3811,12 @@ FETCH_NEXT_BLOCK: prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex, NULL); pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; } break; + case STREAM_DROP_CHILD_TABLE: { + int32_t deleteNum = 0; + code = deletePartName(pInfo, pBlock, &deleteNum); + QUERY_CHECK_CODE(code, lino, _end); + if (deleteNum == 0) goto FETCH_NEXT_BLOCK; + } break; case STREAM_CHECKPOINT: { qError("stream check point error. msg type: STREAM_INPUT__DATA_BLOCK"); } break; @@ -3915,7 +3958,13 @@ FETCH_NEXT_BLOCK: } code = setBlockIntoRes(pInfo, pRes, &pStreamInfo->fillHistoryWindow, false); - QUERY_CHECK_CODE(code, lino, _end); + if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + pInfo->pRes->info.rows = 0; + code = TSDB_CODE_SUCCESS; + } else { + QUERY_CHECK_CODE(code, lino, _end); + } + if (pInfo->pRes->info.rows == 0) { continue; } diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 8fd00e9313..2e906d2ba6 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -5215,7 +5215,7 @@ static int32_t doStreamIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock** p code = getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); QUERY_CHECK_CODE(code, lino, _end); continue; - } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE || pBlock->info.type == STREAM_DROP_CHILD_TABLE) { printDataBlock(pBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); (*ppRes) = pBlock; return code; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 552933dcad..5ce15a32b2 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -19,7 +19,7 @@ #include "geomFunc.h" #include "querynodes.h" #include "scalar.h" -#include "tanal.h" +#include "tanalytics.h" #include "taoserror.h" #include "ttime.h" @@ -188,7 +188,11 @@ static int32_t countTrailingSpaces(const SValueNode* pVal, bool isLtrim) { static int32_t addTimezoneParam(SNodeList* pList) { char buf[TD_TIME_STR_LEN] = {0}; - time_t t = taosTime(NULL); + time_t t; + int32_t code = taosTime(&t); + if (code != 0) { + return code; + } struct tm tmInfo; if (taosLocalTime(&t, &tmInfo, buf, sizeof(buf)) != NULL) { (void)strftime(buf, sizeof(buf), "%z", &tmInfo); @@ -196,7 +200,7 @@ static int32_t addTimezoneParam(SNodeList* pList) { int32_t len = (int32_t)strlen(buf); SValueNode* pVal = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pVal); + code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pVal); if (pVal == NULL) { return code; } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 0aad1501ce..acdac7cbc3 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -19,7 +19,7 @@ #include "functionResInfoInt.h" #include "query.h" #include "querynodes.h" -#include "tanal.h" +#include "tanalytics.h" #include "tcompare.h" #include "tdatablock.h" #include "tdigest.h" diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index a8198a804d..0e5b3ddbdb 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -64,6 +64,10 @@ static void udfWatchUdfd(void *args); void udfUdfdExit(uv_process_t *process, int64_t exitStatus, int32_t termSignal) { fnInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal); SUdfdData *pData = process->data; + if(pData == NULL) { + fnError("udfd process data is NULL"); + return; + } if (exitStatus == 0 && termSignal == 0 || atomic_load_32(&pData->stopCalled)) { fnInfo("udfd process exit due to SIGINT or dnode-mgmt called stop"); } else { diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 6eef99e1f8..e3d533186d 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -1507,7 +1507,7 @@ static void removeListeningPipe() { int err = uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL); uv_fs_req_cleanup(&req); if(err) { - fnError("remove listening pipe %s failed, reason:%s, lino:%d", global.listenPipeName, uv_strerror(err), __LINE__); + fnInfo("remove listening pipe %s : %s, lino:%d", global.listenPipeName, uv_strerror(err), __LINE__); } } diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 1a5785190b..ba87912670 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -851,6 +851,7 @@ static int32_t slotDescCopy(const SSlotDescNode* pSrc, SSlotDescNode* pDst) { static int32_t downstreamSourceCopy(const SDownstreamSourceNode* pSrc, SDownstreamSourceNode* pDst) { COPY_OBJECT_FIELD(addr, sizeof(SQueryNodeAddr)); + COPY_SCALAR_FIELD(clientId); COPY_SCALAR_FIELD(taskId); COPY_SCALAR_FIELD(schedId); COPY_SCALAR_FIELD(execId); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 3275cfd838..f7f858db78 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -5259,6 +5259,7 @@ static int32_t jsonToColumnDefNode(const SJson* pJson, void* pObj) { } static const char* jkDownstreamSourceAddr = "Addr"; +static const char* jkDownstreamSourceClientId = "ClientId"; static const char* jkDownstreamSourceTaskId = "TaskId"; static const char* jkDownstreamSourceSchedId = "SchedId"; static const char* jkDownstreamSourceExecId = "ExecId"; @@ -5268,6 +5269,9 @@ static int32_t downstreamSourceNodeToJson(const void* pObj, SJson* pJson) { const SDownstreamSourceNode* pNode = (const SDownstreamSourceNode*)pObj; int32_t code = tjsonAddObject(pJson, jkDownstreamSourceAddr, queryNodeAddrToJson, &pNode->addr); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDownstreamSourceClientId, pNode->clientId); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkDownstreamSourceTaskId, pNode->taskId); } @@ -5288,6 +5292,9 @@ static int32_t jsonToDownstreamSourceNode(const SJson* pJson, void* pObj) { SDownstreamSourceNode* pNode = (SDownstreamSourceNode*)pObj; int32_t code = tjsonToObject(pJson, jkDownstreamSourceAddr, jsonToQueryNodeAddr, &pNode->addr); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUBigIntValue(pJson, jkDownstreamSourceClientId, &pNode->clientId); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetUBigIntValue(pJson, jkDownstreamSourceTaskId, &pNode->taskId); } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 28d0b9fbd4..bf3ea66e47 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -1769,6 +1769,9 @@ static int32_t downstreamSourceNodeInlineToMsg(const void* pObj, STlvEncoder* pE if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeValueI32(pEncoder, pNode->fetchMsgType); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueU64(pEncoder, pNode->clientId); + } return code; } @@ -1793,6 +1796,9 @@ static int32_t msgToDownstreamSourceNodeInlineToMsg(STlvDecoder* pDecoder, void* if (TSDB_CODE_SUCCESS == code) { code = tlvDecodeValueI32(pDecoder, &pNode->fetchMsgType); } + if (TSDB_CODE_SUCCESS == code && !tlvDecodeEnd(pDecoder)) { + code = tlvDecodeValueU64(pDecoder, &pNode->clientId); + } return code; } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 4b91f01a8c..750621bf66 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -246,7 +246,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, E return code; } -static int parseTimestampOrInterval(const char** end, SToken* pToken, int16_t timePrec, int64_t* ts, int64_t* interval, +static int32_t parseTimestampOrInterval(const char** end, SToken* pToken, int16_t timePrec, int64_t* ts, int64_t* interval, SMsgBuf* pMsgBuf, bool* isTs) { if (pToken->type == TK_NOW) { *isTs = true; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 02295b34da..99c03c412c 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -24,7 +24,7 @@ #include "parUtil.h" #include "scalar.h" #include "systable.h" -#include "tanal.h" +#include "tanalytics.h" #include "tcol.h" #include "tglobal.h" #include "ttime.h" diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 8ac1acb1a2..c2714659ec 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -433,9 +433,6 @@ int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx nodesDestroyNode(pQuery->pRoot); pQuery->pRoot = NULL; code = nodesCloneNode(pQuery->pPrepareRoot, &pQuery->pRoot); - if (NULL == pQuery->pRoot) { - code = code; - } } if (TSDB_CODE_SUCCESS == code) { rewriteExprAlias(pQuery->pRoot); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 34c83acee8..09a4b9c593 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1534,21 +1534,20 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect if (TSDB_CODE_SUCCESS == code) { pSort->pSortKeys = NULL; code = nodesCloneList(pSelect->pOrderByList, &pSort->pSortKeys); - if (NULL == pSort->pSortKeys) { - code = code; - } - SNode* pNode = NULL; - SOrderByExprNode* firstSortKey = (SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0); - if (isPrimaryKeySort(pSelect->pOrderByList)) pSort->node.outputTsOrder = firstSortKey->order; - if (firstSortKey->pExpr->type == QUERY_NODE_COLUMN) { - SColumnNode* pCol = (SColumnNode*)firstSortKey->pExpr; - int16_t projIdx = 1; - FOREACH(pNode, pSelect->pProjectionList) { - SExprNode* pExpr = (SExprNode*)pNode; - if (0 == strcmp(pCol->node.aliasName, pExpr->aliasName)) { - pCol->projIdx = projIdx; break; + if (NULL != pSort->pSortKeys) { + SNode* pNode = NULL; + SOrderByExprNode* firstSortKey = (SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0); + if (isPrimaryKeySort(pSelect->pOrderByList)) pSort->node.outputTsOrder = firstSortKey->order; + if (firstSortKey->pExpr->type == QUERY_NODE_COLUMN) { + SColumnNode* pCol = (SColumnNode*)firstSortKey->pExpr; + int16_t projIdx = 1; + FOREACH(pNode, pSelect->pProjectionList) { + SExprNode* pExpr = (SExprNode*)pNode; + if (0 == strcmp(pCol->node.aliasName, pExpr->aliasName)) { + pCol->projIdx = projIdx; break; + } + projIdx++; } - projIdx++; } } } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index e0e42087f3..e960c0ff5d 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -836,11 +836,9 @@ static int32_t stbSplSplitSessionForStream(SSplitContext* pCxt, SStableSplitInfo nodesDestroyNode(pMergeWin->pTsEnd); pMergeWin->pTsEnd = NULL; code = nodesCloneNode(nodesListGetNode(pPartWin->node.pTargets, index), &pMergeWin->pTsEnd); - if (NULL == pMergeWin->pTsEnd) { - code = code; - } } - code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartWindow); + if (TSDB_CODE_SUCCESS == code) + code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartWindow); } if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 7a902bdd66..708c285aea 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -215,8 +215,8 @@ typedef struct SQWorkerMgmt { #define QW_CTX_NOT_EXISTS_ERR_CODE(mgmt) \ (atomic_load_8(&(mgmt)->nodeStopped) ? TSDB_CODE_VND_STOPPED : TSDB_CODE_QRY_TASK_CTX_NOT_EXIST) -#define QW_FPARAMS_DEF SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId -#define QW_IDS() sId, qId, tId, rId, eId +#define QW_FPARAMS_DEF SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t cId, uint64_t tId, int64_t rId, int32_t eId +#define QW_IDS() sId, qId, cId, tId, rId, eId #define QW_FPARAMS() mgmt, QW_IDS() #define QW_STAT_INC(_item, _n) (void)atomic_add_fetch_64(&(_item), _n) @@ -257,18 +257,20 @@ typedef struct SQWorkerMgmt { #define QW_FETCH_RUNNING(ctx) ((ctx)->inFetch) #define QW_QUERY_NOT_STARTED(ctx) (QW_GET_PHASE(ctx) == -1) -#define QW_SET_QTID(id, qId, tId, eId) \ - do { \ - *(uint64_t *)(id) = (qId); \ - *(uint64_t *)((char *)(id) + sizeof(qId)) = (tId); \ - *(int32_t *)((char *)(id) + sizeof(qId) + sizeof(tId)) = (eId); \ +#define QW_SET_QTID(id, qId, cId, tId, eId) \ + do { \ + *(uint64_t *)(id) = (qId); \ + *(uint64_t *)((char *)(id) + sizeof(qId)) = (cId); \ + *(uint64_t *)((char *)(id) + sizeof(qId) + sizeof(cId)) = (tId); \ + *(int32_t *)((char *)(id) + sizeof(qId) + sizeof(cId) + sizeof(tId)) = (eId); \ } while (0) -#define QW_GET_QTID(id, qId, tId, eId) \ - do { \ - (qId) = *(uint64_t *)(id); \ - (tId) = *(uint64_t *)((char *)(id) + sizeof(qId)); \ - (eId) = *(int32_t *)((char *)(id) + sizeof(qId) + sizeof(tId)); \ +#define QW_GET_QTID(id, qId, cId, tId, eId) \ + do { \ + (qId) = *(uint64_t *)(id); \ + (cId) = *(uint64_t *)((char *)(id) + sizeof(qId)); \ + (tId) = *(uint64_t *)((char *)(id) + sizeof(qId) + sizeof(cId)); \ + (eId) = *(int32_t *)((char *)(id) + sizeof(qId) + sizeof(cId) + sizeof(tId)); \ } while (0) #define QW_ERR_RET(c) \ @@ -310,25 +312,31 @@ typedef struct SQWorkerMgmt { #define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%" PRIx64 " " param, mgmt, sId, __VA_ARGS__) #define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%" PRIx64 " " param, mgmt, sId, __VA_ARGS__) -#define QW_TASK_ELOG(param, ...) qError("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId, __VA_ARGS__) -#define QW_TASK_WLOG(param, ...) qWarn("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId, __VA_ARGS__) -#define QW_TASK_DLOG(param, ...) qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId, __VA_ARGS__) +#define QW_TASK_ELOG(param, ...) \ + qError("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId, __VA_ARGS__) +#define QW_TASK_WLOG(param, ...) \ + qWarn("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId, __VA_ARGS__) +#define QW_TASK_DLOG(param, ...) \ + qDebug("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId, __VA_ARGS__) #define QW_TASK_DLOGL(param, ...) \ - qDebugL("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId, __VA_ARGS__) + qDebugL("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId, __VA_ARGS__) -#define QW_TASK_ELOG_E(param) qError("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId) -#define QW_TASK_WLOG_E(param) qWarn("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId) -#define QW_TASK_DLOG_E(param) qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId) +#define QW_TASK_ELOG_E(param) \ + qError("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId) +#define QW_TASK_WLOG_E(param) \ + qWarn("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId) +#define QW_TASK_DLOG_E(param) \ + qDebug("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId) -#define QW_SCH_TASK_ELOG(param, ...) \ - qError("QW:%p SID:0x%" PRIx64 ",qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, qId, tId, eId, \ - __VA_ARGS__) -#define QW_SCH_TASK_WLOG(param, ...) \ - qWarn("QW:%p SID:0x%" PRIx64 ",qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, qId, tId, eId, \ - __VA_ARGS__) -#define QW_SCH_TASK_DLOG(param, ...) \ - qDebug("QW:%p SID:0x%" PRIx64 ",qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, qId, tId, eId, \ - __VA_ARGS__) +#define QW_SCH_TASK_ELOG(param, ...) \ + qError("QW:%p SID:0x%" PRIx64 ",qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, \ + qId, cId, tId, eId, __VA_ARGS__) +#define QW_SCH_TASK_WLOG(param, ...) \ + qWarn("QW:%p SID:0x%" PRIx64 ",qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, qId, \ + cId, tId, eId, __VA_ARGS__) +#define QW_SCH_TASK_DLOG(param, ...) \ + qDebug("QW:%p SID:0x%" PRIx64 ",qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, \ + qId, cId, tId, eId, __VA_ARGS__) #define QW_LOCK_DEBUG(...) \ do { \ diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index d3b8d36b25..897080df3e 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -96,14 +96,14 @@ void qwDbgDumpSchInfo(SQWorker *mgmt, SQWSchStatus *sch, int32_t i) { int32_t taskNum = taosHashGetSize(sch->tasksHash); QW_DLOG("***The %dth scheduler status, hbBrokenTs:%" PRId64 ",taskNum:%d", i, sch->hbBrokenTs, taskNum); - uint64_t qId, tId; + uint64_t qId, cId, tId; int32_t eId; SQWTaskStatus *pTask = NULL; void *pIter = taosHashIterate(sch->tasksHash, NULL); while (pIter) { pTask = (SQWTaskStatus *)pIter; void *key = taosHashGetKey(pIter, NULL); - QW_GET_QTID(key, qId, tId, eId); + QW_GET_QTID(key, qId, cId, tId, eId); QW_TASK_DLOG("job refId:%" PRIx64 ", code:%x, task status:%d", pTask->refId, pTask->code, pTask->status); @@ -118,13 +118,13 @@ void qwDbgDumpTasksInfo(SQWorker *mgmt) { int32_t i = 0; SQWTaskCtx *ctx = NULL; - uint64_t qId, tId; + uint64_t qId, cId, tId; int32_t eId; void *pIter = taosHashIterate(mgmt->ctxHash, NULL); while (pIter) { ctx = (SQWTaskCtx *)pIter; void *key = taosHashGetKey(pIter, NULL); - QW_GET_QTID(key, qId, tId, eId); + QW_GET_QTID(key, qId, cId, tId, eId); QW_TASK_DLOG("%p lock:%x, phase:%d, type:%d, explain:%d, needFetch:%d, localExec:%d, queryMsgType:%d, " "sId:%" PRId64 ", level:%d, queryGotData:%d, queryRsped:%d, queryEnd:%d, queryContinue:%d, queryInQueue:%d, " diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 20b81bfc14..7dbad90cc0 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -233,6 +233,7 @@ int32_t qwBuildAndSendDropMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { qMsg.header.contLen = 0; qMsg.sId = sId; qMsg.queryId = qId; + qMsg.clientId = cId; qMsg.taskId = tId; qMsg.refId = rId; qMsg.execId = eId; @@ -284,6 +285,7 @@ int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { req->header.vgId = mgmt->nodeId; req->sId = sId; req->queryId = qId; + req->clientId = cId; req->taskId = tId; req->execId = eId; @@ -312,6 +314,7 @@ int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { qMsg.header.contLen = 0; qMsg.sId = sId; qMsg.queryId = qId; + qMsg.clientId = cId; qMsg.taskId = tId; qMsg.refId = rId; qMsg.execId = eId; @@ -416,6 +419,7 @@ int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg, bool chkGran uint64_t sId = msg.sId; uint64_t qId = msg.queryId; + uint64_t cId = msg.clientId; uint64_t tId = msg.taskId; int64_t rId = msg.refId; int32_t eId = msg.execId; @@ -447,6 +451,7 @@ int32_t qWorkerAbortPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg) { uint64_t sId = msg.sId; uint64_t qId = msg.queryId; + uint64_t cId = msg.clientId; uint64_t tId = msg.taskId; int64_t rId = msg.refId; int32_t eId = msg.execId; @@ -479,6 +484,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int uint64_t sId = msg.sId; uint64_t qId = msg.queryId; + uint64_t cId = msg.clientId; uint64_t tId = msg.taskId; int64_t rId = msg.refId; int32_t eId = msg.execId; @@ -524,6 +530,7 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in uint64_t sId = msg->sId; uint64_t qId = msg->queryId; + uint64_t cId = msg->clientId; uint64_t tId = msg->taskId; int64_t rId = 0; int32_t eId = msg->execId; @@ -557,6 +564,7 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int uint64_t sId = req.sId; uint64_t qId = req.queryId; + uint64_t cId = req.clientId; uint64_t tId = req.taskId; int64_t rId = 0; int32_t eId = req.execId; @@ -604,12 +612,14 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in msg->sId = be64toh(msg->sId); msg->queryId = be64toh(msg->queryId); + msg->clientId = be64toh(msg->clientId); msg->taskId = be64toh(msg->taskId); msg->refId = be64toh(msg->refId); msg->execId = ntohl(msg->execId); uint64_t sId = msg->sId; uint64_t qId = msg->queryId; + uint64_t cId = msg->clientId; uint64_t tId = msg->taskId; int64_t rId = msg->refId; int32_t eId = msg->execId; @@ -646,6 +656,7 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int6 uint64_t sId = msg.sId; uint64_t qId = msg.queryId; + uint64_t cId = msg.clientId; uint64_t tId = msg.taskId; int64_t rId = msg.refId; int32_t eId = msg.execId; @@ -684,6 +695,7 @@ int32_t qWorkerProcessNotifyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in uint64_t sId = msg.sId; uint64_t qId = msg.queryId; + uint64_t cId = msg.clientId; uint64_t tId = msg.taskId; int64_t rId = msg.refId; int32_t eId = msg.execId; @@ -753,6 +765,7 @@ int32_t qWorkerProcessDeleteMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, SD uint64_t sId = req.sId; uint64_t qId = req.queryId; + uint64_t cId = req.clientId; uint64_t tId = req.taskId; int64_t rId = 0; int32_t eId = -1; diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index ef07a42629..917579deb0 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -137,8 +137,8 @@ int32_t qwAcquireScheduler(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQWSchS void qwReleaseScheduler(int32_t rwType, SQWorker *mgmt) { QW_UNLOCK(rwType, &mgmt->schLock); } int32_t qwAcquireTaskStatus(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus *sch, SQWTaskStatus **task) { - char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0}; - QW_SET_QTID(id, qId, tId, eId); + char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0}; + QW_SET_QTID(id, qId, cId, tId, eId); QW_LOCK(rwType, &sch->tasksLock); *task = taosHashGet(sch->tasksHash, id, sizeof(id)); @@ -153,8 +153,8 @@ int32_t qwAcquireTaskStatus(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus *sch, S int32_t qwAddTaskStatusImpl(QW_FPARAMS_DEF, SQWSchStatus *sch, int32_t rwType, int32_t status, SQWTaskStatus **task) { int32_t code = 0; - char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0}; - QW_SET_QTID(id, qId, tId, eId); + char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0}; + QW_SET_QTID(id, qId, cId, tId, eId); SQWTaskStatus ntask = {0}; ntask.status = status; @@ -209,8 +209,8 @@ int32_t qwAddAcquireTaskStatus(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus *sch void qwReleaseTaskStatus(int32_t rwType, SQWSchStatus *sch) { QW_UNLOCK(rwType, &sch->tasksLock); } int32_t qwAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { - char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0}; - QW_SET_QTID(id, qId, tId, eId); + char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0}; + QW_SET_QTID(id, qId, cId, tId, eId); *ctx = taosHashAcquire(mgmt->ctxHash, id, sizeof(id)); if (NULL == (*ctx)) { @@ -222,8 +222,8 @@ int32_t qwAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { } int32_t qwGetTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { - char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0}; - QW_SET_QTID(id, qId, tId, eId); + char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0}; + QW_SET_QTID(id, qId, cId, tId, eId); *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); if (NULL == (*ctx)) { @@ -235,8 +235,8 @@ int32_t qwGetTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { } int32_t qwAddTaskCtxImpl(QW_FPARAMS_DEF, bool acquire, SQWTaskCtx **ctx) { - char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0}; - QW_SET_QTID(id, qId, tId, eId); + char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0}; + QW_SET_QTID(id, qId, cId, tId, eId); SQWTaskCtx nctx = {0}; @@ -347,6 +347,7 @@ int32_t qwSendExplainResponse(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { (void)memcpy(pExec, taosArrayGet(execInfoList, 0), localRsp.rsp.numOfPlans * sizeof(SExplainExecInfo)); localRsp.rsp.subplanInfo = pExec; localRsp.qId = qId; + localRsp.cId = cId; localRsp.tId = tId; localRsp.rId = rId; localRsp.eId = eId; @@ -376,8 +377,8 @@ _return: int32_t qwDropTaskCtx(QW_FPARAMS_DEF) { - char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0}; - QW_SET_QTID(id, qId, tId, eId); + char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0}; + QW_SET_QTID(id, qId, cId, tId, eId); SQWTaskCtx octx; SQWTaskCtx *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); @@ -411,8 +412,8 @@ int32_t qwDropTaskStatus(QW_FPARAMS_DEF) { SQWTaskStatus *task = NULL; int32_t code = 0; - char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0}; - QW_SET_QTID(id, qId, tId, eId); + char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0}; + QW_SET_QTID(id, qId, cId, tId, eId); if (qwAcquireScheduler(mgmt, sId, QW_WRITE, &sch)) { QW_TASK_WLOG_E("scheduler does not exist"); @@ -465,8 +466,8 @@ _return: int32_t qwHandleDynamicTaskEnd(QW_FPARAMS_DEF) { - char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0}; - QW_SET_QTID(id, qId, tId, eId); + char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0}; + QW_SET_QTID(id, qId, cId, tId, eId); SQWTaskCtx octx; SQWTaskCtx *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); @@ -588,14 +589,14 @@ void qwDestroyImpl(void *pMgmt) { mgmt->hbTimer = NULL; taosTmrCleanUp(mgmt->timer); - uint64_t qId, tId; + uint64_t qId, cId, tId; int32_t eId; void *pIter = taosHashIterate(mgmt->ctxHash, NULL); while (pIter) { SQWTaskCtx *ctx = (SQWTaskCtx *)pIter; void *key = taosHashGetKey(pIter, NULL); - QW_GET_QTID(key, qId, tId, eId); + QW_GET_QTID(key, qId, cId, tId, eId); qwFreeTaskCtx(ctx); QW_TASK_DLOG_E("task ctx freed"); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 9b96c1e519..13e1d0e231 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -19,7 +19,7 @@ SQWorkerMgmt gQwMgmt = { }; void qwStopAllTasks(SQWorker *mgmt) { - uint64_t qId, tId, sId; + uint64_t qId, cId, tId, sId; int32_t eId; int64_t rId = 0; int32_t code = TSDB_CODE_SUCCESS; @@ -28,7 +28,7 @@ void qwStopAllTasks(SQWorker *mgmt) { while (pIter) { SQWTaskCtx *ctx = (SQWTaskCtx *)pIter; void *key = taosHashGetKey(pIter, NULL); - QW_GET_QTID(key, qId, tId, eId); + QW_GET_QTID(key, qId, cId, tId, eId); QW_LOCK(QW_WRITE, &ctx->lock); @@ -288,7 +288,7 @@ int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) // TODO GET EXECUTOR API TO GET MORE INFO - QW_GET_QTID(key, status.queryId, status.taskId, status.execId); + QW_GET_QTID(key, status.queryId, status.clientId, status.taskId, status.execId); status.status = taskStatus->status; status.refId = taskStatus->refId; @@ -1473,8 +1473,8 @@ int32_t qWorkerGetStat(SReadHandle *handle, void *qWorkerMgmt, SQWorkerStat *pSt return TSDB_CODE_SUCCESS; } -int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId, - SQWMsg *qwMsg, SArray *explainRes) { +int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t cId, uint64_t tId, int64_t rId, + int32_t eId, SQWMsg *qwMsg, SArray *explainRes) { SQWorker *mgmt = (SQWorker *)pMgmt; int32_t code = 0; SQWTaskCtx *ctx = NULL; @@ -1538,8 +1538,8 @@ _return: QW_RET(code); } -int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId, - void **pRsp, SArray *explainRes) { +int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t cId, uint64_t tId, int64_t rId, + int32_t eId, void **pRsp, SArray *explainRes) { SQWorker *mgmt = (SQWorker *)pMgmt; int32_t code = 0; int32_t dataLen = 0; diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 96b9d2da8d..6a910453f0 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -142,8 +142,9 @@ typedef struct SSchedulerCfg { } SSchedulerCfg; typedef struct SSchedulerMgmt { - uint64_t taskId; // sequential taksId - uint64_t sId; // schedulerId + uint64_t clientId; // unique clientId + uint64_t taskId; // sequential taksId + uint64_t sId; // schedulerId SSchedulerCfg cfg; bool exit; int32_t jobRef; @@ -163,6 +164,7 @@ typedef struct SSchTaskCallbackParam { SSchCallbackParamHeader head; uint64_t queryId; int64_t refId; + uint64_t clientId; uint64_t taskId; int32_t execId; void *pTrans; @@ -222,6 +224,7 @@ typedef struct SSchTimerParam { } SSchTimerParam; typedef struct SSchTask { + uint64_t clientId; // current client id uint64_t taskId; // task id SRWLatch lock; // task reentrant lock int32_t maxExecTimes; // task max exec times @@ -329,6 +332,7 @@ extern SSchedulerMgmt schMgmt; #define SCH_LOCK_TASK(_task) SCH_LOCK(SCH_WRITE, &(_task)->lock) #define SCH_UNLOCK_TASK(_task) SCH_UNLOCK(SCH_WRITE, &(_task)->lock) +#define SCH_CLIENT_ID(_task) ((_task) ? (_task)->clientId : -1) #define SCH_TASK_ID(_task) ((_task) ? (_task)->taskId : -1) #define SCH_TASK_EID(_task) ((_task) ? (_task)->execId : -1) @@ -449,21 +453,21 @@ extern SSchedulerMgmt schMgmt; #define SCH_JOB_ELOG(param, ...) qError("qid:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) #define SCH_JOB_DLOG(param, ...) qDebug("qid:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) -#define SCH_TASK_ELOG(param, ...) \ - qError("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ - __VA_ARGS__) -#define SCH_TASK_DLOG(param, ...) \ - qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ - __VA_ARGS__) -#define SCH_TASK_TLOG(param, ...) \ - qTrace("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ - __VA_ARGS__) -#define SCH_TASK_DLOGL(param, ...) \ - qDebugL("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ - __VA_ARGS__) -#define SCH_TASK_WLOG(param, ...) \ - qWarn("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ - __VA_ARGS__) +#define SCH_TASK_ELOG(param, ...) \ + qError("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_CLIENT_ID(pTask), \ + SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), __VA_ARGS__) +#define SCH_TASK_DLOG(param, ...) \ + qDebug("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_CLIENT_ID(pTask), \ + SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), __VA_ARGS__) +#define SCH_TASK_TLOG(param, ...) \ + qTrace("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_CLIENT_ID(pTask), \ + SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), __VA_ARGS__) +#define SCH_TASK_DLOGL(param, ...) \ + qDebugL("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_CLIENT_ID(pTask), \ + SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), __VA_ARGS__) +#define SCH_TASK_WLOG(param, ...) \ + qWarn("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_CLIENT_ID(pTask), \ + SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), __VA_ARGS__) #define SCH_SET_ERRNO(_err) \ do { \ diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index b15a6a09d3..eefb32f783 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -500,8 +500,8 @@ _return: int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, - code); + qDebug("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, + pParam->clientId, pParam->taskId, code); // called if drop task rsp received code (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error @@ -517,8 +517,8 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { int32_t schHandleNotifyCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " task notify rsp received, code:0x%x", pParam->queryId, pParam->taskId, - code); + qDebug("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 " task notify rsp received, code:0x%x", pParam->queryId, + pParam->clientId, pParam->taskId, code); if (pMsg) { taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); @@ -595,6 +595,7 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bo param->queryId = pJob->queryId; param->refId = pJob->refId; + param->clientId = SCH_CLIENT_ID(pTask); param->taskId = SCH_TASK_ID(pTask); param->pTrans = pJob->conn.pTrans; param->execId = pTask->execId; @@ -1138,6 +1139,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, req.header.vgId = addr->nodeId; req.sId = schMgmt.sId; req.queryId = pJob->queryId; + req.clientId = pTask->clientId; req.taskId = pTask->taskId; req.phyLen = pTask->msgLen; req.sqlLen = strlen(pJob->sql); @@ -1171,6 +1173,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, qMsg.header.contLen = 0; qMsg.sId = schMgmt.sId; qMsg.queryId = pJob->queryId; + qMsg.clientId = pTask->clientId; qMsg.taskId = pTask->taskId; qMsg.refId = pJob->refId; qMsg.execId = pTask->execId; @@ -1226,6 +1229,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, req.header.vgId = addr->nodeId; req.sId = schMgmt.sId; req.queryId = pJob->queryId; + req.clientId = pTask->clientId; req.taskId = pTask->taskId; req.execId = pTask->execId; @@ -1253,6 +1257,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, qMsg.header.contLen = 0; qMsg.sId = schMgmt.sId; qMsg.queryId = pJob->queryId; + qMsg.clientId = pTask->clientId; qMsg.taskId = pTask->taskId; qMsg.refId = pJob->refId; qMsg.execId = *(int32_t*)param; @@ -1310,6 +1315,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, qMsg.header.contLen = 0; qMsg.sId = schMgmt.sId; qMsg.queryId = pJob->queryId; + qMsg.clientId = pTask->clientId; qMsg.taskId = pTask->taskId; qMsg.refId = pJob->refId; qMsg.execId = pTask->execId; diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index fe24633c12..9be0e3fc40 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -66,6 +66,7 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * pTask->execId = -1; pTask->failedExecId = -2; pTask->timeoutUsec = SCH_DEFAULT_TASK_TIMEOUT_USEC; + pTask->clientId = getClientId(); pTask->taskId = schGenTaskId(); schInitTaskRetryTimes(pJob, pTask, pLevel); @@ -305,6 +306,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SCH_LOCK(SCH_WRITE, &parent->planLock); SDownstreamSourceNode source = { .type = QUERY_NODE_DOWNSTREAM_SOURCE, + .clientId = pTask->clientId, .taskId = pTask->taskId, .schedId = schMgmt.sId, .execId = pTask->execId, @@ -996,8 +998,8 @@ int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList) { int32_t code = 0; - qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s", pStatus->queryId, pStatus->taskId, - pStatus->execId, jobTaskStatusStr(pStatus->status)); + qDebug("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s", pStatus->queryId, + pStatus->clientId, pStatus->taskId, pStatus->execId, jobTaskStatusStr(pStatus->status)); if (schProcessOnCbBegin(&pJob, &pTask, pStatus->queryId, pStatus->refId, pStatus->taskId)) { continue; @@ -1043,13 +1045,14 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { continue; } - qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg", localRsp->qId, localRsp->tId); + qDebug("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg", + localRsp->qId, localRsp->cId, localRsp->tId); pJob = NULL; (void)schAcquireJob(localRsp->rId, &pJob); if (NULL == pJob) { - qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, localRsp->qId, - localRsp->tId, localRsp->rId); + qWarn("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, + localRsp->qId, localRsp->cId, localRsp->tId, localRsp->rId); SCH_ERR_JRET(TSDB_CODE_QRY_JOB_NOT_EXIST); } @@ -1068,8 +1071,8 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { (void)schReleaseJob(pJob->refId); - qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", localRsp->qId, - localRsp->tId, code); + qDebug("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", + localRsp->qId, localRsp->cId, localRsp->tId, code); SCH_ERR_JRET(code); @@ -1147,8 +1150,8 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { } } - SCH_ERR_JRET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, - pTask->execId, &qwMsg, explainRes)); + SCH_ERR_JRET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->clientId, pTask->taskId, + pJob->refId, pTask->execId, &qwMsg, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); @@ -1407,8 +1410,8 @@ int32_t schExecLocalFetch(SSchJob *pJob, SSchTask *pTask) { } } - SCH_ERR_JRET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, - pTask->execId, &pRsp, explainRes)); + SCH_ERR_JRET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->clientId, pTask->taskId, + pJob->refId, pTask->execId, &pRsp, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 4697de6f28..ac34099417 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -293,6 +293,18 @@ void schCloseJobRef(void) { } } +int32_t initClientId(void) { + int32_t code = taosGetSystemUUIDU64(&schMgmt.clientId); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to generate clientId since %s", tstrerror(code)); + SCH_ERR_RET(code); + } + qInfo("initialize"); + return TSDB_CODE_SUCCESS; +} + +uint64_t getClientId(void) { return schMgmt.clientId; } + uint64_t schGenTaskId(void) { return atomic_add_fetch_64(&schMgmt.taskId, 1); } #ifdef BUILD_NO_CALL diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index d313acc61d..6a10b21c53 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -223,6 +223,7 @@ int32_t streamStateParTagGetKVByCur_rocksdb(SStreamStateCur* pCur, int64_t* pGro // parname cf int32_t streamStatePutParName_rocksdb(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]); int32_t streamStateGetParName_rocksdb(SStreamState* pState, int64_t groupId, void** pVal); +int32_t streamStateDeleteParName_rocksdb(SStreamState* pState, int64_t groupId); void streamStateDestroy_rocksdb(SStreamState* pState, bool remove); diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 09f4e95376..65746b3100 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -4432,6 +4432,12 @@ int32_t streamStateGetParName_rocksdb(SStreamState* pState, int64_t groupId, voi return code; } +int32_t streamStateDeleteParName_rocksdb(SStreamState* pState, int64_t groupId) { + int code = 0; + STREAM_STATE_DEL_ROCKSDB(pState, "parname", &groupId); + return code; +} + int32_t streamDefaultPut_rocksdb(SStreamState* pState, const void* key, void* pVal, int32_t pVLen) { int code = 0; STREAM_STATE_PUT_ROCKSDB(pState, "default", key, pVal, pVLen); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index db6c841f5e..86f305df60 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -884,6 +884,14 @@ static int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask, void* param) { tstrerror(code)); } } + + // let's kill the query procedure within stream, to end it ASAP. + if (pTask->info.taskLevel != TASK_LEVEL__SINK && pTask->exec.pExecutor != NULL) { + code = qKillTask(pTask->exec.pExecutor, TSDB_CODE_SUCCESS); + if (code != TSDB_CODE_SUCCESS) { + stError("s-task:%s failed to kill task related query handle, code:%s", pTask->id.idStr, tstrerror(code)); + } + } return code; } diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 20c3e5a6b9..401aa7530d 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -166,6 +166,8 @@ const char* streamQueueItemGetTypeStr(int32_t type) { return "checkpoint-trigger"; case STREAM_INPUT__TRANS_STATE: return "trans-state"; + case STREAM_INPUT__REF_DATA_BLOCK: + return "ref-block"; default: return "datablock"; } @@ -211,7 +213,7 @@ EExtractDataCode streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueIte // do not merge blocks for sink node and check point data block int8_t type = qItem->type; if (type == STREAM_INPUT__CHECKPOINT || type == STREAM_INPUT__CHECKPOINT_TRIGGER || - type == STREAM_INPUT__TRANS_STATE) { + type == STREAM_INPUT__TRANS_STATE || type == STREAM_INPUT__REF_DATA_BLOCK) { const char* p = streamQueueItemGetTypeStr(type); if (*pInput == NULL) { @@ -504,4 +506,4 @@ void streamTaskPutbackToken(STokenBucket* pBucket) { // size in KB void streamTaskConsumeQuota(STokenBucket* pBucket, int32_t bytes) { pBucket->quotaRemain -= SIZE_IN_MiB(bytes); } -void streamTaskInputFail(SStreamTask* pTask) { atomic_store_8(&pTask->inputq.status, TASK_INPUT_STATUS__FAILED); } \ No newline at end of file +void streamTaskInputFail(SStreamTask* pTask) { atomic_store_8(&pTask->inputq.status, TASK_INPUT_STATUS__FAILED); } diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 794fc346bf..5461b5899b 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -525,6 +525,18 @@ _end: return code; } +int32_t streamStateDeleteParName(SStreamState* pState, int64_t groupId) { + int32_t code = tSimpleHashRemove(pState->parNameMap, &groupId, sizeof(int64_t)); + if (TSDB_CODE_SUCCESS != code) { + qWarn("failed to remove parname from cache, code:%d", code); + } + code = streamStateDeleteParName_rocksdb(pState, groupId); + if (TSDB_CODE_SUCCESS != code) { + qWarn("failed to remove parname from rocksdb, code:%d", code); + } + return TSDB_CODE_SUCCESS; +} + void streamStateDestroy(SStreamState* pState, bool remove) { streamFileStateDestroy(pState->pFileState); // streamStateDestroy_rocksdb(pState, remove); diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index c3a2742aa2..f995c48688 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -500,7 +500,9 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even STaskStateTrans* pTrans = pSM->pActiveTrans; if (pTrans == NULL) { ETaskStatus s = pSM->current.state; - + // when trying to finish current event successfully, another event with high priorities, such as dropping/stop, has + // interrupted this procedure, and changed the status after freeing the activeTrans, resulting in the failure of + // processing of current event. if (s != TASK_STATUS__DROPPING && s != TASK_STATUS__PAUSE && s != TASK_STATUS__STOP && s != TASK_STATUS__UNINIT && s != TASK_STATUS__READY) { stError("s-task:%s invalid task status:%s on handling event:%s success", id, pSM->current.name, diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 5f9811ab80..2aeffc6395 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -620,7 +620,7 @@ int32_t cliHandleState_mayCreateAhandle(SCliConn* conn, STransMsgHead* pHead, ST int32_t code = 0; int64_t qId = taosHton64(pHead->qid); if (qId == 0) { - return 0; + return TSDB_CODE_RPC_NO_STATE; } STransCtx* pCtx = taosHashGet(conn->pQTable, &qId, sizeof(qId)); @@ -1608,6 +1608,7 @@ static int32_t cliDoConn(SCliThrd* pThrd, SCliConn* conn) { ret = uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, cliConnCb); if (ret != 0) { tError("failed connect to %s since %s", conn->dstAddr, uv_err_name(ret)); + cliMayUpdateFqdnCache(pThrd->fqdn2ipCache, conn->dstAddr); TAOS_CHECK_GOTO(TSDB_CODE_THIRDPARTY_ERROR, &lino, _exception1); } @@ -1699,6 +1700,7 @@ void cliConnCb(uv_connect_t* req, int status) { if (status != 0) { tDebug("%s conn %p failed to connect to %s since %s", CONN_GET_INST_LABEL(pConn), pConn, pConn->dstAddr, uv_strerror(status)); + cliMayUpdateFqdnCache(pThrd->fqdn2ipCache, pConn->dstAddr); TAOS_UNUSED(transUnrefCliHandle(pConn)); return; } @@ -1850,7 +1852,7 @@ static FORCE_INLINE int32_t cliUpdateFqdnCache(SHashObj* cache, char* fqdn) { size_t len = strlen(fqdn); uint32_t* v = taosHashGet(cache, fqdn, len); if (addr != *v) { - char old[TD_IP_LEN] = {0}, new[TD_IP_LEN] = {0}; + char old[TSDB_FQDN_LEN] = {0}, new[TSDB_FQDN_LEN] = {0}; tinet_ntoa(old, *v); tinet_ntoa(new, addr); tWarn("update ip of fqdn:%s, old: %s, new: %s", fqdn, old, new); @@ -1870,7 +1872,7 @@ static void cliMayUpdateFqdnCache(SHashObj* cache, char* dst) { if (dst[i] == ':') break; } if (i > 0) { - char fqdn[TSDB_FQDN_LEN + 1] = {0}; + char fqdn[TSDB_FQDN_LEN] = {0}; memcpy(fqdn, dst, i); TAOS_UNUSED(cliUpdateFqdnCache(cache, fqdn)); } @@ -2917,6 +2919,7 @@ bool cliMayRetry(SCliConn* pConn, SCliReq* pReq, STransMsg* pResp) { noDelay = cliResetEpset(pCtx, pResp, false); transFreeMsg(pResp->pCont); } + pResp->pCont = NULL; if (code != TSDB_CODE_RPC_BROKEN_LINK && code != TSDB_CODE_RPC_NETWORK_UNAVAIL && code != TSDB_CODE_SUCCESS) { // save one internal code pCtx->retryCode = code; diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index da26ddae3a..6d52c8d6cb 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -424,6 +424,9 @@ static void printFileSet(int32_t vgId, SArray* fileSet, const char* str) { int32_t walCheckAndRepairMeta(SWal* pWal) { // load log files, get first/snapshot/last version info + if (pWal->cfg.level == TAOS_WAL_SKIP) { + return TSDB_CODE_SUCCESS; + } int32_t code = 0; const char* logPattern = "^[0-9]+.log$"; const char* idxPattern = "^[0-9]+.idx$"; diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 610adfb0e1..da5e1f47e9 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -89,6 +89,8 @@ int32_t walNextValidMsg(SWalReader *pReader) { if (type == TDMT_VND_SUBMIT || ((type == TDMT_VND_DELETE) && (pReader->cond.deleteMsg == 1)) || (IS_META_MSG(type) && pReader->cond.scanMeta)) { TAOS_RETURN(walFetchBody(pReader)); + } else if (type == TDMT_VND_DROP_TABLE && pReader->cond.scanDropCtb) { + TAOS_RETURN(walFetchBody(pReader)); } else { TAOS_CHECK_RETURN(walSkipFetchBody(pReader)); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 1c53baf360..1a9652b3bb 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -294,8 +294,11 @@ int32_t walRollback(SWal *pWal, int64_t ver) { static int32_t walRollImpl(SWal *pWal) { int32_t code = 0, lino = 0; + if (pWal->cfg.level == TAOS_WAL_SKIP && pWal->pIdxFile != NULL && pWal->pLogFile != NULL) { + TAOS_RETURN(TSDB_CODE_SUCCESS); + } if (pWal->pIdxFile != NULL) { - if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pIdxFile)) != 0) { + if ((code = taosFsyncFile(pWal->pIdxFile)) != 0) { TAOS_CHECK_GOTO(terrno, &lino, _exit); } code = taosCloseFile(&pWal->pIdxFile); @@ -305,7 +308,7 @@ static int32_t walRollImpl(SWal *pWal) { } if (pWal->pLogFile != NULL) { - if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pLogFile)) != 0) { + if ((code = taosFsyncFile(pWal->pLogFile)) != 0) { TAOS_CHECK_GOTO(terrno, &lino, _exit); } code = taosCloseFile(&pWal->pLogFile); diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index a0285f1363..3e6fab116f 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -455,3 +455,59 @@ TEST_F(WalRetentionEnv, repairMeta1) { } walCloseReader(pRead); } + +class WalSkipLevel : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(NULL); + ASSERT(code == 0); + } + + static void TearDownTestCase() { walCleanUp(); } + + void walResetEnv() { + TearDown(); + taosRemoveDir(pathName); + SetUp(); + } + + void SetUp() override { + SWalCfg cfg; + cfg.rollPeriod = -1; + cfg.segSize = -1; + cfg.committed =-1; + cfg.retentionPeriod = -1; + cfg.retentionSize = 0; + cfg.rollPeriod = 0; + cfg.vgId = 1; + cfg.level = TAOS_WAL_SKIP; + pWal = walOpen(pathName, &cfg); + ASSERT(pWal != NULL); + } + + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + + SWal* pWal = NULL; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; +}; + +TEST_F(WalSkipLevel, restart) { + walResetEnv(); + int code; + + int i; + for (i = 0; i < 100; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); + ASSERT_EQ(code, 0); + } + + TearDown(); + + SetUp(); +} \ No newline at end of file diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index d4d9936154..60339fc646 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -81,6 +81,7 @@ static const char *am_pm[2] = {"AM", "PM"}; #endif char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) { + if (!buf || !fmt || !tm) return NULL; #ifdef WINDOWS char c; const char *bp; @@ -345,6 +346,9 @@ char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) { } int32_t taosGetTimeOfDay(struct timeval *tv) { + if (tv == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; #ifdef WINDOWS LARGE_INTEGER t; @@ -365,12 +369,15 @@ int32_t taosGetTimeOfDay(struct timeval *tv) { #endif } -time_t taosTime(time_t *t) { +int32_t taosTime(time_t *t) { + if (t == NULL) { + return TSDB_CODE_INVALID_PARA; + } time_t r = time(t); if (r == (time_t)-1) { - terrno = TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(errno); } - return r; + return 0; } /* diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index 7f5955f3dd..2633bb3268 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -18,7 +18,7 @@ else() endif(${ASSERT_NOT_CORE}) if(${BUILD_WITH_ANALYSIS}) - add_definitions(-DUSE_ANAL) + add_definitions(-DUSE_ANALYTICS) endif() target_include_directories( diff --git a/source/util/src/tanal.c b/source/util/src/tanalytics.c similarity index 97% rename from source/util/src/tanal.c rename to source/util/src/tanalytics.c index 92eee28ba8..99d91700a2 100644 --- a/source/util/src/tanal.c +++ b/source/util/src/tanalytics.c @@ -14,18 +14,17 @@ */ #define _DEFAULT_SOURCE -#include "tanal.h" -#include "tmsg.h" +#include "tanalytics.h" #include "ttypes.h" #include "tutil.h" -#ifdef USE_ANAL +#ifdef USE_ANALYTICS #include #define ANAL_ALGO_SPLIT "," typedef struct { int64_t ver; - SHashObj *hash; // algoname:algotype -> SAnalUrl + SHashObj *hash; // algoname:algotype -> SAnalyticsUrl TdThreadMutex lock; } SAlgoMgmt; @@ -69,7 +68,7 @@ EAnalAlgoType taosAnalAlgoInt(const char *name) { return ANAL_ALGO_TYPE_END; } -int32_t taosAnalInit() { +int32_t taosAnalyticsInit() { if (curl_global_init(CURL_GLOBAL_ALL) != 0) { uError("failed to init curl"); return -1; @@ -94,14 +93,14 @@ int32_t taosAnalInit() { static void taosAnalFreeHash(SHashObj *hash) { void *pIter = taosHashIterate(hash, NULL); while (pIter != NULL) { - SAnalUrl *pUrl = (SAnalUrl *)pIter; + SAnalyticsUrl *pUrl = (SAnalyticsUrl *)pIter; taosMemoryFree(pUrl->url); pIter = taosHashIterate(hash, pIter); } taosHashCleanup(hash); } -void taosAnalCleanup() { +void taosAnalyticsCleanup() { curl_global_cleanup(); if (taosThreadMutexDestroy(&tsAlgos.lock) != 0) { uError("failed to destroy anal lock"); @@ -167,8 +166,10 @@ int32_t taosAnalGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, char name[TSDB_ANAL_ALGO_KEY_LEN] = {0}; int32_t nameLen = 1 + tsnprintf(name, sizeof(name) - 1, "%d:%s", type, algoName); + char *unused = strntolower(name, name, nameLen); + if (taosThreadMutexLock(&tsAlgos.lock) == 0) { - SAnalUrl *pUrl = taosHashAcquire(tsAlgos.hash, name, nameLen); + SAnalyticsUrl *pUrl = taosHashAcquire(tsAlgos.hash, name, nameLen); if (pUrl != NULL) { tstrncpy(url, pUrl->url, urlLen); uDebug("algo:%s, type:%s, url:%s", algoName, taosAnalAlgoStr(type), url); @@ -178,6 +179,7 @@ int32_t taosAnalGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, code = terrno; uError("algo:%s, type:%s, url not found", algoName, taosAnalAlgoStr(type)); } + if (taosThreadMutexUnlock(&tsAlgos.lock) != 0) { uError("failed to unlock hash"); return TSDB_CODE_OUT_OF_MEMORY; @@ -403,7 +405,7 @@ static int32_t tsosAnalJsonBufOpen(SAnalBuf *pBuf, int32_t numOfCols) { return terrno; } - pBuf->pCols = taosMemoryCalloc(numOfCols, sizeof(SAnalColBuf)); + pBuf->pCols = taosMemoryCalloc(numOfCols, sizeof(SAnalyticsColBuf)); if (pBuf->pCols == NULL) return TSDB_CODE_OUT_OF_MEMORY; pBuf->numOfCols = numOfCols; @@ -412,7 +414,7 @@ static int32_t tsosAnalJsonBufOpen(SAnalBuf *pBuf, int32_t numOfCols) { } for (int32_t i = 0; i < numOfCols; ++i) { - SAnalColBuf *pCol = &pBuf->pCols[i]; + SAnalyticsColBuf *pCol = &pBuf->pCols[i]; snprintf(pCol->fileName, sizeof(pCol->fileName), "%s-c%d", pBuf->fileName, i); pCol->filePtr = taosOpenFile(pCol->fileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); @@ -546,7 +548,7 @@ static int32_t taosAnalJsonBufWriteDataEnd(SAnalBuf *pBuf) { if (pBuf->bufType == ANAL_BUF_TYPE_JSON_COL) { for (int32_t i = 0; i < pBuf->numOfCols; ++i) { - SAnalColBuf *pCol = &pBuf->pCols[i]; + SAnalyticsColBuf *pCol = &pBuf->pCols[i]; code = taosFsyncFile(pCol->filePtr); if (code != 0) return code; @@ -588,7 +590,7 @@ int32_t taosAnalJsonBufClose(SAnalBuf *pBuf) { if (pBuf->bufType == ANAL_BUF_TYPE_JSON_COL) { for (int32_t i = 0; i < pBuf->numOfCols; ++i) { - SAnalColBuf *pCol = &pBuf->pCols[i]; + SAnalyticsColBuf *pCol = &pBuf->pCols[i]; if (pCol->filePtr != NULL) { code = taosFsyncFile(pCol->filePtr); if (code != 0) return code; @@ -610,7 +612,7 @@ void taosAnalBufDestroy(SAnalBuf *pBuf) { if (pBuf->bufType == ANAL_BUF_TYPE_JSON_COL) { for (int32_t i = 0; i < pBuf->numOfCols; ++i) { - SAnalColBuf *pCol = &pBuf->pCols[i]; + SAnalyticsColBuf *pCol = &pBuf->pCols[i]; if (pCol->fileName[0] != 0) { if (pCol->filePtr != NULL) (void)taosCloseFile(&pCol->filePtr); if (taosRemoveFile(pCol->fileName) != 0) { @@ -726,8 +728,8 @@ static int32_t taosAnalBufGetCont(SAnalBuf *pBuf, char **ppCont, int64_t *pContL #else -int32_t taosAnalInit() { return 0; } -void taosAnalCleanup() {} +int32_t taosAnalyticsInit() { return 0; } +void taosAnalyticsCleanup() {} SJson *taosAnalSendReqRetJson(const char *url, EAnalHttpType type, SAnalBuf *pBuf) { return NULL; } int32_t taosAnalGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, int32_t urlLen) { return 0; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index d660edd0b8..0d8a85155a 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -164,6 +164,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_ENCODE_PARAM_NULL, "Not found compress pa TAOS_DEFINE_ERROR(TSDB_CODE_TSC_COMPRESS_PARAM_ERROR, "Invalid compress param") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR, "Invalid compress level param") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_FAIL_GENERATE_JSON, "failed to generate JSON") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR, "bind number out of range or not match") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INTERNAL_ERROR, "Internal error") @@ -360,8 +361,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_ANODE_TOO_MANY_ALGO, "Anode too many algori TAOS_DEFINE_ERROR(TSDB_CODE_MND_ANODE_TOO_LONG_ALGO_NAME, "Anode too long algorithm name") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ANODE_TOO_MANY_ALGO_TYPE, "Anode too many algorithm type") -TAOS_DEFINE_ERROR(TSDB_CODE_ANAL_URL_RSP_IS_NULL, "Analysis url response is NULL") -TAOS_DEFINE_ERROR(TSDB_CODE_ANAL_URL_CANT_ACCESS, "Analysis url can't access") +TAOS_DEFINE_ERROR(TSDB_CODE_ANAL_URL_RSP_IS_NULL, "Analysis service response is NULL") +TAOS_DEFINE_ERROR(TSDB_CODE_ANAL_URL_CANT_ACCESS, "Analysis service can't access") TAOS_DEFINE_ERROR(TSDB_CODE_ANAL_ALGO_NOT_FOUND, "Analysis algorithm not found") TAOS_DEFINE_ERROR(TSDB_CODE_ANAL_ALGO_NOT_LOAD, "Analysis algorithm not loaded") TAOS_DEFINE_ERROR(TSDB_CODE_ANAL_BUF_INVALID_TYPE, "Analysis invalid buffer type") diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 76d0139521..f70b145dbc 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -154,16 +154,26 @@ static int32_t taosStartLog() { return 0; } -static void getDay(char *buf, int32_t bufSize) { - time_t t = taosTime(NULL); +static int32_t getDay(char *buf, int32_t bufSize) { + time_t t; + int32_t code = taosTime(&t); + if(code != 0) { + return code; + } struct tm tmInfo; if (taosLocalTime(&t, &tmInfo, buf, bufSize) != NULL) { TAOS_UNUSED(strftime(buf, bufSize, "%Y-%m-%d", &tmInfo)); } + return 0; } static int64_t getTimestampToday() { - time_t t = taosTime(NULL); + time_t t; + int32_t code = taosTime(&t); + if (code != 0) { + uError("failed to get time, reason:%s", tstrerror(code)); + return 0; + } struct tm tm; if (taosLocalTime(&t, &tm, NULL, 0) == NULL) { return 0; @@ -203,7 +213,11 @@ int32_t taosInitSlowLog() { char name[PATH_MAX + TD_TIME_STR_LEN] = {0}; char day[TD_TIME_STR_LEN] = {0}; - getDay(day, sizeof(day)); + int32_t code = getDay(day, sizeof(day)); + if (code != 0) { + (void)printf("failed to get day, reason:%s\n", tstrerror(code)); + return code; + } (void)snprintf(name, PATH_MAX + TD_TIME_STR_LEN, "%s.%s", tsLogObj.slowLogName, day); tsLogObj.timestampToday = getTimestampToday(); @@ -434,7 +448,12 @@ static void taosOpenNewSlowLogFile() { atomic_store_32(&tsLogObj.slowHandle->lock, 0); char day[TD_TIME_STR_LEN] = {0}; - getDay(day, sizeof(day)); + int32_t code = getDay(day, sizeof(day)); + if (code != 0) { + uError("failed to get day, reason:%s", tstrerror(code)); + (void)taosThreadMutexUnlock(&tsLogObj.logMutex); + return; + } TdFilePtr pFile = NULL; char name[PATH_MAX + TD_TIME_STR_LEN] = {0}; (void)snprintf(name, PATH_MAX + TD_TIME_STR_LEN, "%s.%s", tsLogObj.slowLogName, day); diff --git a/source/util/test/errorCodeTable.ini b/source/util/test/errorCodeTable.ini index 33c9d77c5e..e837954a0b 100644 --- a/source/util/test/errorCodeTable.ini +++ b/source/util/test/errorCodeTable.ini @@ -97,6 +97,7 @@ TSDB_CODE_TSC_ENCODE_PARAM_ERROR = 0x80000231 TSDB_CODE_TSC_ENCODE_PARAM_NULL = 0x80000232 TSDB_CODE_TSC_COMPRESS_PARAM_ERROR = 0x80000233 TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR = 0x80000234 +TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR = 0x80000236 TSDB_CODE_TSC_INTERNAL_ERROR = 0x800002FF TSDB_CODE_MND_REQ_REJECTED = 0x80000300 TSDB_CODE_MND_NO_RIGHTS = 0x80000303 diff --git a/tests/army/user/test_passwd.py b/tests/army/user/test_passwd.py new file mode 100644 index 0000000000..dfec175824 --- /dev/null +++ b/tests/army/user/test_passwd.py @@ -0,0 +1,55 @@ +import os +import platform +import subprocess +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame.epath import * +from frame import * + +class TDTestCase(TBase): + def apiPath(self): + apiPath = None + currentFilePath = os.path.dirname(os.path.realpath(__file__)) + if (os.sep.join(["community", "tests"]) in currentFilePath): + testFilePath = currentFilePath[:currentFilePath.find(os.sep.join(["community", "tests"]))] + else: + testFilePath = currentFilePath[:currentFilePath.find(os.sep.join(["TDengine", "tests"]))] + + for root, dirs, files in os.walk(testFilePath): + if ("passwdTest.c" in files): + apiPath = root + break + return apiPath + + def run(self): + apiPath = self.apiPath() + tdLog.info(f"api path: {apiPath}") + if platform.system().lower() == 'linux': + p = subprocess.Popen(f"cd {apiPath} && make", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + if 0 != p.returncode: + tdLog.exit("Test script passwdTest.c make failed") + + p = subprocess.Popen(f"ls {apiPath}", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + tdLog.info(f"test files: {out}") + if apiPath: + test_file_cmd = os.sep.join([apiPath, "passwdTest localhost"]) + try: + p = subprocess.Popen(test_file_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + if 0 != p.returncode: + tdLog.exit("Failed to run passwd test with output: %s \n error: %s" % (out, err)) + else: + tdLog.info(out) + tdLog.success(f"{__file__} successfully executed") + except Exception as e: + tdLog.exit(f"Failed to execute {__file__} with error: {e}") + else: + tdLog.exit("passwdTest.c not found") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 151358aec3..9ad6378326 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -44,6 +44,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f storage/compressBasic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f grant/grantBugs.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/queryBugs.py -N 3 +,,n,army,python3 ./test.py -f user/test_passwd.py ,,y,army,./pytest.sh python3 ./test.py -f tmq/tmqBugs.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/fill/fill_compare_asc_desc.py ,,y,army,./pytest.sh python3 ./test.py -f query/last/test_last.py @@ -51,6 +52,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f query/sys/tb_perf_queries_exist_test.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/test_having.py ,,n,army,python3 ./test.py -f tmq/drop_lost_comsumers.py + # # system test # diff --git a/tests/script/api/makefile b/tests/script/api/makefile index 9c2bb6be3d..ce5980b37a 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -13,7 +13,7 @@ all: $(TARGET) exe: gcc $(CFLAGS) ./batchprepare.c -o $(ROOT)batchprepare $(LFLAGS) - gcc $(CFLAGS) ./stmt2-test.c -o $(ROOT)stmt2-test $(LFLAGS) + # gcc $(CFLAGS) ./stmt2-test.c -o $(ROOT)stmt2-test $(LFLAGS) gcc $(CFLAGS) ./stopquery.c -o $(ROOT)stopquery $(LFLAGS) gcc $(CFLAGS) ./dbTableRoute.c -o $(ROOT)dbTableRoute $(LFLAGS) gcc $(CFLAGS) ./insertSameTs.c -o $(ROOT)insertSameTs $(LFLAGS) @@ -22,11 +22,11 @@ exe: gcc $(CFLAGS) ./insert_stb.c -o $(ROOT)insert_stb $(LFLAGS) gcc $(CFLAGS) ./tmqViewTest.c -o $(ROOT)tmqViewTest $(LFLAGS) gcc $(CFLAGS) ./stmtQuery.c -o $(ROOT)stmtQuery $(LFLAGS) - gcc $(CFLAGS) ./stmt.c -o $(ROOT)stmt $(LFLAGS) - gcc $(CFLAGS) ./stmt2.c -o $(ROOT)stmt2 $(LFLAGS) - gcc $(CFLAGS) ./stmt2-example.c -o $(ROOT)stmt2-example $(LFLAGS) - gcc $(CFLAGS) ./stmt2-get-fields.c -o $(ROOT)stmt2-get-fields $(LFLAGS) - gcc $(CFLAGS) ./stmt2-nohole.c -o $(ROOT)stmt2-nohole $(LFLAGS) + # gcc $(CFLAGS) ./stmt.c -o $(ROOT)stmt $(LFLAGS) + # gcc $(CFLAGS) ./stmt2.c -o $(ROOT)stmt2 $(LFLAGS) + # gcc $(CFLAGS) ./stmt2-example.c -o $(ROOT)stmt2-example $(LFLAGS) + # gcc $(CFLAGS) ./stmt2-get-fields.c -o $(ROOT)stmt2-get-fields $(LFLAGS) + # gcc $(CFLAGS) ./stmt2-nohole.c -o $(ROOT)stmt2-nohole $(LFLAGS) gcc $(CFLAGS) ./stmt-crash.c -o $(ROOT)stmt-crash $(LFLAGS) clean: diff --git a/tests/script/api/makefile_win64.mak b/tests/script/api/makefile_win64.mak new file mode 100644 index 0000000000..50a2447a06 --- /dev/null +++ b/tests/script/api/makefile_win64.mak @@ -0,0 +1,20 @@ +# Makefile.mak for win64 + +TARGET = passwdTest.exe +CC = cl +CFLAGS = /W4 /EHsc /I"C:\TDengine\include" /DWINDOWS +LDFLAGS = /link /LIBPATH:"C:\TDengine\driver" taos.lib + +SRCS = passwdTest.c +OBJS = $(SRCS:.c=.obj) + +all: $(TARGET) + +$(TARGET): $(OBJS) + $(CC) $(OBJS) $(LDFLAGS) + +.c.obj: + $(CC) $(CFLAGS) /c $< + +clean: + del $(OBJS) $(TARGET) \ No newline at end of file diff --git a/tests/script/api/passwdTest.c b/tests/script/api/passwdTest.c index 928525750e..259d3bec8e 100644 --- a/tests/script/api/passwdTest.c +++ b/tests/script/api/passwdTest.c @@ -20,12 +20,27 @@ * passwdTest.c * - Run the test case in clear TDengine environment with default root passwd 'taosdata' */ +#ifdef WINDOWS +#include +#include +#include +#ifndef PRId64 +#define PRId64 "I64d" +#endif + +#ifndef PRIu64 +#define PRIu64 "I64u" +#endif + +#else #include +#include +#endif + #include #include #include -#include #include "taos.h" // TAOS header file #define nDup 1 @@ -50,6 +65,16 @@ void sysInfoTest(TAOS *taos, const char *host, char *qstr); void userDroppedTest(TAOS *taos, const char *host, char *qstr); void clearTestEnv(TAOS *taos, const char *host, char *qstr); +void taosMsleep(int64_t ms) { + if (ms < 0) return; +#ifdef WINDOWS + Sleep(ms); +#else + usleep(ms * 1000); +#endif +} + + int nPassVerNotified = 0; int nUserDropped = 0; TAOS *taosu[nRoot] = {0}; @@ -59,7 +84,8 @@ void __taos_notify_cb(void *param, void *ext, int type) { switch (type) { case TAOS_NOTIFY_PASSVER: { ++nPassVerNotified; - printf("%s:%d type:%d user:%s passVer:%d\n", __func__, __LINE__, type, param ? (char *)param : "NULL", *(int *)ext); + printf("%s:%d type:%d user:%s passVer:%d\n", __func__, __LINE__, type, param ? (char *)param : "NULL", + *(int *)ext); break; } case TAOS_NOTIFY_USER_DROPPED: { @@ -191,11 +217,11 @@ static int printResult(TAOS_RES *res, char *output) { printRow(temp, row, fields, numFields); puts(temp); } + return 0; } int main(int argc, char *argv[]) { char qstr[1024]; - // connect to server if (argc < 2) { printf("please input server-ip \n"); @@ -215,6 +241,7 @@ int main(int argc, char *argv[]) { taos_close(taos); taos_cleanup(); + exit(EXIT_SUCCESS); } void createUsers(TAOS *taos, const char *host, char *qstr) { @@ -234,6 +261,7 @@ void createUsers(TAOS *taos, const char *host, char *qstr) { if (code != 0) { fprintf(stderr, "failed to run: taos_set_notify_cb(TAOS_NOTIFY_PASSVER) for user:%s since %d\n", users[i], code); + exit(EXIT_FAILURE); } else { fprintf(stderr, "success to run: taos_set_notify_cb(TAOS_NOTIFY_PASSVER) for user:%s\n", users[i]); } @@ -260,6 +288,7 @@ void passVerTestMulti(const char *host, char *qstr) { if (code != 0) { fprintf(stderr, "failed to run: taos_set_notify_cb since %d\n", code); + exit(EXIT_FAILURE); } else { fprintf(stderr, "success to run: taos_set_notify_cb\n"); } @@ -283,26 +312,25 @@ void passVerTestMulti(const char *host, char *qstr) { printf("%s:%d [%d] second(s) elasped, passVer notification received:%d, total:%d\n", __func__, __LINE__, i, nPassVerNotified, nConn); if (nPassVerNotified >= nConn) break; - sleep(1); + taosMsleep(1000); } // close the taos_conn for (int i = 0; i < nRoot; ++i) { taos_close(taos[i]); printf("%s:%d close taos[%d]\n", __func__, __LINE__, i); - // sleep(1); + // taosMsleep(1000); } for (int i = 0; i < nUser; ++i) { taos_close(taosu[i]); printf("%s:%d close taosu[%d]\n", __func__, __LINE__, i); - // sleep(1); + // taosMsleep(1000); } fprintf(stderr, "######## %s #########\n", __func__); if (nPassVerNotified == nConn) { - fprintf(stderr, ">>> succeed to get passVer notification since nNotify %d == nConn %d\n", nPassVerNotified, - nConn); + fprintf(stderr, ">>> succeed to get passVer notification since nNotify %d == nConn %d\n", nPassVerNotified, nConn); } else { fprintf(stderr, ">>> failed to get passVer notification since nNotify %d != nConn %d\n", nPassVerNotified, nConn); exit(1); @@ -337,7 +365,7 @@ void sysInfoTest(TAOS *taosRoot, const char *host, char *qstr) { TAOS_RES *res = NULL; int32_t nRep = 0; -_REP: +_REP: fprintf(stderr, "######## %s loop:%d #########\n", __func__, nRep); res = taos_query(taos[0], qstr); if (taos_errno(res) != 0) { @@ -356,7 +384,7 @@ _REP: fprintf(stderr, "%s:%d sleep 2 seconds to wait HB take effect\n", __func__, __LINE__); for (int i = 1; i <= 2; ++i) { - sleep(1); + taosMsleep(1000); } res = taos_query(taos[0], qstr); @@ -372,10 +400,10 @@ _REP: queryDB(taosRoot, "alter user user0 sysinfo 1"); fprintf(stderr, "%s:%d sleep 2 seconds to wait HB take effect\n", __func__, __LINE__); for (int i = 1; i <= 2; ++i) { - sleep(1); + taosMsleep(1000); } - if(++nRep < 5) { + if (++nRep < 5) { goto _REP; } @@ -390,7 +418,7 @@ _REP: fprintf(stderr, "######## %s #########\n", __func__); } static bool isDropUser = true; -void userDroppedTest(TAOS *taos, const char *host, char *qstr) { +void userDroppedTest(TAOS *taos, const char *host, char *qstr) { // users int nTestUsers = nUser; int nLoop = 0; @@ -408,6 +436,7 @@ _loop: if (code != 0) { fprintf(stderr, "failed to run: taos_set_notify_cb:%d for user:%s since %d\n", TAOS_NOTIFY_USER_DROPPED, users[i], code); + exit(EXIT_FAILURE); } else { fprintf(stderr, "success to run: taos_set_notify_cb:%d for user:%s\n", TAOS_NOTIFY_USER_DROPPED, users[i]); } @@ -426,7 +455,7 @@ _loop: printf("%s:%d [%d] second(s) elasped, user dropped notification received:%d, total:%d\n", __func__, __LINE__, i, nUserDropped, nConn); if (nUserDropped >= nConn) break; - sleep(1); + taosMsleep(1000); } for (int i = 0; i < nTestUsers; ++i) { diff --git a/tests/script/api/sameReqidTest.c b/tests/script/api/sameReqidTest.c new file mode 100644 index 0000000000..7507619886 --- /dev/null +++ b/tests/script/api/sameReqidTest.c @@ -0,0 +1,406 @@ +// sample code to verify multiple queries with the same reqid +// to compile: gcc -o sameReqdiTest sameReqidTest.c -ltaos + +#include +#include +#include +#include +#include "taos.h" + +#define NUM_ROUNDS 10 +#define CONST_REQ_ID 12345 +#define TEST_DB "test" + +#define CHECK_CONDITION(condition, prompt, errstr) \ + do { \ + if (!(condition)) { \ + printf("\033[31m[%s:%d] failed to " prompt ", reason: %s\033[0m\n", __func__, __LINE__, errstr); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#define CHECK_RES(res, prompt) CHECK_CONDITION(taos_errno(res) == 0, prompt, taos_errstr(res)) +#define CHECK_CODE(code, prompt) CHECK_CONDITION(code == 0, prompt, taos_errstr(NULL)) + +static TAOS* getNewConnection() { + const char* host = "127.0.0.1"; + const char* user = "root"; + const char* passwd = "taosdata"; + TAOS* taos = NULL; + + taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); + taos = taos_connect(host, user, passwd, "", 0); + CHECK_CONDITION(taos != NULL, "connect to db", taos_errstr(NULL)); + return taos; +} + +static void prepareData(TAOS* taos) { + TAOS_RES* res = NULL; + int32_t code = 0; + + res = taos_query(taos, "create database if not exists " TEST_DB " precision 'ns'"); + CHECK_RES(res, "create database"); + taos_free_result(res); + usleep(100000); + + code = taos_select_db(taos, TEST_DB); + CHECK_CODE(code, "switch to database"); + + res = taos_query(taos, "create table if not exists meters(ts timestamp, a int) tags(area int)"); + CHECK_RES(res, "create stable meters"); + taos_free_result(res); + + res = taos_query(taos, "create table if not exists t0 using meters tags(0)"); + CHECK_RES(res, "create table t0"); + taos_free_result(res); + + res = taos_query(taos, "create table if not exists t1 using meters tags(1)"); + CHECK_RES(res, "create table t1"); + taos_free_result(res); + + res = taos_query(taos, "create table if not exists t2 using meters tags(2)"); + CHECK_RES(res, "create table t2"); + taos_free_result(res); + + res = taos_query(taos, "create table if not exists t3 using meters tags(3)"); + CHECK_RES(res, "create table t3"); + taos_free_result(res); + + res = taos_query(taos, "create table if not exists t4 using meters tags(4)"); + CHECK_RES(res, "create table t4"); + taos_free_result(res); + + res = taos_query(taos, "create table if not exists t5 using meters tags(5)"); + CHECK_RES(res, "create table t5"); + taos_free_result(res); + + res = taos_query(taos, "create table if not exists t6 using meters tags(6)"); + CHECK_RES(res, "create table t6"); + taos_free_result(res); + + res = taos_query(taos, "create table if not exists t7 using meters tags(7)"); + CHECK_RES(res, "create table t7"); + taos_free_result(res); + + res = taos_query(taos, "create table if not exists t8 using meters tags(8)"); + CHECK_RES(res, "create table t8"); + taos_free_result(res); + + res = taos_query(taos, "create table if not exists t9 using meters tags(9)"); + CHECK_RES(res, "create table t9"); + taos_free_result(res); + + res = taos_query(taos, + "insert into t0 values('2020-01-01 00:00:00.000', 0)" + " ('2020-01-01 00:01:00.000', 0)" + " ('2020-01-01 00:02:00.000', 0)" + " t1 values('2020-01-01 00:00:00.000', 1)" + " ('2020-01-01 00:01:00.000', 1)" + " ('2020-01-01 00:02:00.000', 1)" + " ('2020-01-01 00:03:00.000', 1)" + " t2 values('2020-01-01 00:00:00.000', 2)" + " ('2020-01-01 00:01:00.000', 2)" + " ('2020-01-01 00:01:01.000', 2)" + " ('2020-01-01 00:01:02.000', 2)" + " t3 values('2020-01-01 00:01:02.000', 3)" + " t4 values('2020-01-01 00:01:02.000', 4)" + " t5 values('2020-01-01 00:01:02.000', 5)" + " t6 values('2020-01-01 00:01:02.000', 6)" + " t7 values('2020-01-01 00:01:02.000', 7)" + " t8 values('2020-01-01 00:01:02.000', 8)" + " t9 values('2020-01-01 00:01:02.000', 9)"); + CHECK_RES(res, "insert into meters"); + CHECK_CONDITION(taos_affected_rows(res), "insert into meters", "insufficient count"); + taos_free_result(res); + + res = taos_query( + taos, + "create table if not exists m1 (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 " + "double, bin binary(40), blob nchar(10))"); + CHECK_RES(res, "create table m1"); + taos_free_result(res); + + usleep(1000000); +} + +static void verifySchemaLess(TAOS* taos) { + TAOS_RES* res = NULL; + char* lines[] = { + "st,t1=3i64,t2=4f64,t3=L\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000", + "st,t1=4i64,t3=L\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64 1626006833640000000", + "st,t2=5f64,t3=L\"ste\" c1=4i64,c2=true,c3=L\"iam\" 1626056811823316532", + "st,t1=4i64,t2=5f64,t3=L\"t4\" c1=3i64,c3=L\"passitagain\",c2=true,c4=5f64 1626006833642000000", + "st,t2=5f64,t3=L\"ste2\" c3=L\"iamszhou\",c2=false 1626056811843316532", + "st,t2=5f64,t3=L\"ste2\" c3=L\"iamszhou\",c2=false,c5=5f64,c6=7u64,c7=32i32,c8=88.88f32 1626056812843316532", + "st,t1=4i64,t3=L\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64,c6=7u64 " + "1626006933640000000", + "st,t1=4i64,t3=L\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64,c6=7u64 " + "1626006933640000000", + "st,t1=4i64,t3=L\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 " + "1626006933641000000"}; + + res = taos_schemaless_insert_with_reqid(taos, lines, sizeof(lines) / sizeof(char*), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS, CONST_REQ_ID); + CHECK_RES(res, "insert schema-less data"); + printf("successfully inserted %d rows\n", taos_affected_rows(res)); + taos_free_result(res); +} + +static int32_t printResult(TAOS_RES* res, int32_t nlimit) { + TAOS_ROW row = NULL; + TAOS_FIELD* fields = NULL; + int32_t numFields = 0; + int32_t nRows = 0; + + numFields = taos_num_fields(res); + fields = taos_fetch_fields(res); + while ((nlimit-- > 0) && (row = taos_fetch_row(res))) { + char temp[256] = {0}; + taos_print_row(temp, row, fields, numFields); + puts(temp); + nRows++; + } + return nRows; +} + +static void verifyQuery(TAOS* taos) { + TAOS_RES* res = NULL; + + res = taos_query_with_reqid(taos, "select * from meters", CONST_REQ_ID); + CHECK_RES(res, "select from meters"); + printResult(res, INT32_MAX); + taos_free_result(res); + + res = taos_query_with_reqid(taos, "select * from t0", CONST_REQ_ID); + CHECK_RES(res, "select from t0"); + printResult(res, INT32_MAX); + taos_free_result(res); + + res = taos_query_with_reqid(taos, "select * from t1", CONST_REQ_ID); + CHECK_RES(res, "select from t1"); + printResult(res, INT32_MAX); + taos_free_result(res); + + res = taos_query_with_reqid(taos, "select * from t2", CONST_REQ_ID); + CHECK_RES(res, "select from t2"); + printResult(res, INT32_MAX); + taos_free_result(res); + + res = taos_query_with_reqid(taos, "select * from t3", CONST_REQ_ID); + CHECK_RES(res, "select from t3"); + printResult(res, INT32_MAX); + taos_free_result(res); + + printf("succeed to read from meters\n"); +} + +void retrieveCallback(void* param, TAOS_RES* res, int32_t nrows) { + if (nrows == 0) { + taos_free_result(res); + } else { + printResult(res, nrows); + taos_fetch_rows_a(res, retrieveCallback, param); + } +} + +void selectCallback(void* param, TAOS_RES* res, int32_t code) { + CHECK_CODE(code, "read async from table"); + taos_fetch_rows_a(res, retrieveCallback, param); +} + +static void verifyQueryAsync(TAOS* taos) { + taos_query_a_with_reqid(taos, "select *from meters", selectCallback, NULL, CONST_REQ_ID); + taos_query_a_with_reqid(taos, "select *from t0", selectCallback, NULL, CONST_REQ_ID); + taos_query_a_with_reqid(taos, "select *from t1", selectCallback, NULL, CONST_REQ_ID); + taos_query_a_with_reqid(taos, "select *from t2", selectCallback, NULL, CONST_REQ_ID); + taos_query_a_with_reqid(taos, "select *from t3", selectCallback, NULL, CONST_REQ_ID); + + sleep(1); +} + +void veriryStmt(TAOS* taos) { + // insert 10 records + struct { + int64_t ts[10]; + int8_t b[10]; + int8_t v1[10]; + int16_t v2[10]; + int32_t v4[10]; + int64_t v8[10]; + float f4[10]; + double f8[10]; + char bin[10][40]; + char blob[10][80]; + } v; + + int32_t* t8_len = malloc(sizeof(int32_t) * 10); + int32_t* t16_len = malloc(sizeof(int32_t) * 10); + int32_t* t32_len = malloc(sizeof(int32_t) * 10); + int32_t* t64_len = malloc(sizeof(int32_t) * 10); + int32_t* float_len = malloc(sizeof(int32_t) * 10); + int32_t* double_len = malloc(sizeof(int32_t) * 10); + int32_t* bin_len = malloc(sizeof(int32_t) * 10); + int32_t* blob_len = malloc(sizeof(int32_t) * 10); + + TAOS_STMT* stmt = taos_stmt_init_with_reqid(taos, CONST_REQ_ID); + TAOS_MULTI_BIND params[10]; + char is_null[10] = {0}; + + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(v.ts[0]); + params[0].buffer = v.ts; + params[0].length = t64_len; + params[0].is_null = is_null; + params[0].num = 10; + + params[1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[1].buffer_length = sizeof(v.b[0]); + params[1].buffer = v.b; + params[1].length = t8_len; + params[1].is_null = is_null; + params[1].num = 10; + + params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[2].buffer_length = sizeof(v.v1[0]); + params[2].buffer = v.v1; + params[2].length = t8_len; + params[2].is_null = is_null; + params[2].num = 10; + + params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[3].buffer_length = sizeof(v.v2[0]); + params[3].buffer = v.v2; + params[3].length = t16_len; + params[3].is_null = is_null; + params[3].num = 10; + + params[4].buffer_type = TSDB_DATA_TYPE_INT; + params[4].buffer_length = sizeof(v.v4[0]); + params[4].buffer = v.v4; + params[4].length = t32_len; + params[4].is_null = is_null; + params[4].num = 10; + + params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[5].buffer_length = sizeof(v.v8[0]); + params[5].buffer = v.v8; + params[5].length = t64_len; + params[5].is_null = is_null; + params[5].num = 10; + + params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[6].buffer_length = sizeof(v.f4[0]); + params[6].buffer = v.f4; + params[6].length = float_len; + params[6].is_null = is_null; + params[6].num = 10; + + params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[7].buffer_length = sizeof(v.f8[0]); + params[7].buffer = v.f8; + params[7].length = double_len; + params[7].is_null = is_null; + params[7].num = 10; + + params[8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[8].buffer_length = sizeof(v.bin[0]); + params[8].buffer = v.bin; + params[8].length = bin_len; + params[8].is_null = is_null; + params[8].num = 10; + + params[9].buffer_type = TSDB_DATA_TYPE_NCHAR; + params[9].buffer_length = sizeof(v.blob[0]); + params[9].buffer = v.blob; + params[9].length = blob_len; + params[9].is_null = is_null; + params[9].num = 10; + + int32_t code = taos_stmt_prepare( + stmt, "insert into ? (ts, b, v1, v2, v4, v8, f4, f8, bin, blob) values(?,?,?,?,?,?,?,?,?,?)", 0); + CHECK_CODE(code, "taos_stmt_prepare"); + + code = taos_stmt_set_tbname(stmt, "m1"); + CHECK_CODE(code, "taos_stmt_set_tbname"); + + int64_t ts = 1591060628000000000; + for (int i = 0; i < 10; ++i) { + v.ts[i] = ts; + ts += 1000000; + is_null[i] = 0; + + v.b[i] = (int8_t)i % 2; + v.v1[i] = (int8_t)i; + v.v2[i] = (int16_t)(i * 2); + v.v4[i] = (int32_t)(i * 4); + v.v8[i] = (int64_t)(i * 8); + v.f4[i] = (float)(i * 40); + v.f8[i] = (double)(i * 80); + for (int j = 0; j < sizeof(v.bin[0]); ++j) { + v.bin[i][j] = (char)(i + '0'); + } + strcpy(v.blob[i], "一二三四五六七八九十"); + + t8_len[i] = sizeof(int8_t); + t16_len[i] = sizeof(int16_t); + t32_len[i] = sizeof(int32_t); + t64_len[i] = sizeof(int64_t); + float_len[i] = sizeof(float); + double_len[i] = sizeof(double); + bin_len[i] = sizeof(v.bin[0]); + blob_len[i] = (int32_t)strlen(v.blob[i]); + } + + code = taos_stmt_bind_param_batch(stmt, params); + CHECK_CODE(code, "taos_stmt_bind_param_batch"); + + code = taos_stmt_add_batch(stmt); + CHECK_CODE(code, "taos_stmt_add_batch"); + + code = taos_stmt_execute(stmt); + CHECK_CODE(code, "taos_stmt_execute"); + + taos_stmt_close(stmt); + + free(t8_len); + free(t16_len); + free(t32_len); + free(t64_len); + free(float_len); + free(double_len); + free(bin_len); + free(blob_len); +} + +int main(int argc, char* argv[]) { + TAOS* taos = NULL; + int32_t code = 0; + + taos = getNewConnection(); + taos_select_db(taos, TEST_DB); + CHECK_CODE(code, "switch to database"); + + printf("************ prepare data *************\n"); + prepareData(taos); + + for (int32_t i = 0; i < NUM_ROUNDS; ++i) { + printf("************ verify schema-less *************\n"); + verifySchemaLess(taos); + + printf("************ verify query *************\n"); + verifyQuery(taos); + + printf("********* verify async query **********\n"); + verifyQueryAsync(taos); + + printf("********* verify stmt query **********\n"); + veriryStmt(taos); + + printf("done\n"); + } + + taos_close(taos); + taos_cleanup(); + + return 0; +} diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index 01e416bb26..aa548d4e59 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -222,7 +222,7 @@ class TDTestCase: tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'") tdLog.info(len(tdSql.queryResult)) - tdSql.checkEqual(True, len(tdSql.queryResult) in range(280, 281)) + tdSql.checkEqual(True, len(tdSql.queryResult) in range(281, 282)) tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'") tdSql.checkEqual(56, len(tdSql.queryResult)) diff --git a/tests/system-test/2-query/tsma.py b/tests/system-test/2-query/tsma.py index a1638ae4cb..1c688d568c 100644 --- a/tests/system-test/2-query/tsma.py +++ b/tests/system-test/2-query/tsma.py @@ -604,7 +604,7 @@ class TSMATestSQLGenerator: class TDTestCase: - updatecfgDict = {'asynclog': 0, 'ttlUnit': 1, 'ttlPushInterval': 5, 'ratioOfVnodeStreamThrea': 4, 'maxTsmaNum': 3} + updatecfgDict = {'asynclog': 0, 'ttlUnit': 1, 'ttlPushInterval': 5, 'ratioOfVnodeStreamThrea': 4, 'maxTsmaNum': 3, 'debugFlag': 143} def __init__(self): self.vgroups = 4 @@ -804,8 +804,8 @@ class TDTestCase: self.tsma_tester.check_sql(ctx.sql, ctx) def test_query_with_tsma(self): - self.create_tsma('tsma1', 'test', 'meters', ['avg(c1)', 'avg(c2)'], '5m') - self.create_tsma('tsma2', 'test', 'meters', ['avg(c1)', 'avg(c2)'], '30m') + self.create_tsma('tsma1', 'test', 'meters', ['avg(c1)', 'avg(c2)', 'count(ts)'], '5m') + self.create_tsma('tsma2', 'test', 'meters', ['avg(c1)', 'avg(c2)', 'count(ts)'], '30m') self.create_tsma('tsma5', 'test', 'norm_tb', ['avg(c1)', 'avg(c2)'], '10m') self.test_query_with_tsma_interval() @@ -1237,7 +1237,41 @@ class TDTestCase: clust_dnode_nums = len(cluster_dnode_list) if clust_dnode_nums > 1: self.test_redistribute_vgroups() - + tdSql.execute("drop tsma test.tsma5") + for _ in range(4): + self.test_td_32519() + + def test_td_32519(self): + self.create_recursive_tsma('tsma1', 'tsma_r', 'test', '1h', 'meters', ['avg(c1)', 'avg(c2)', 'count(ts)']) + tdSql.execute('INSERT INTO test.t1 VALUES("2024-10-24 11:45:00", 1,1,1,1,1,1,1, "a", "a")', queryTimes=1) + tdSql.execute('INSERT INTO test.t1 VALUES("2024-10-24 11:55:00", 2,1,1,1,1,1,1, "a", "a")', queryTimes=1) + tdSql.execute('DROP TABLE test.t1', queryTimes=1) + self.wait_query_err('desc test.`404e15422d96c8b5de9603c2296681b1`', 10, -2147473917) + self.wait_query_err('desc test.`82b56f091c4346369da0af777c3e580d`', 10, -2147473917) + self.wait_query_err('desc test.`163b7c69922cf6d83a98bfa44e52dade`', 10, -2147473917) + tdSql.execute('CREATE TABLE test.t1 USING test.meters TAGS(1, "a", "b", 1,1,1)') + tdSql.execute('INSERT INTO test.t1 VALUES("2024-10-24 11:59:00", 3,1,1,1,1,1,1, "a", "a")', queryTimes=1) + tdSql.execute('INSERT INTO test.t1 VALUES("2024-10-24 12:10:00", 4,1,1,1,1,1,1, "a", "a")', queryTimes=1) + tdSql.execute('INSERT INTO test.t1 VALUES("2024-10-24 12:20:00", 5,1,1,1,1,1,1, "a", "a")', queryTimes=1) + tdSql.execute('FLUSH DATABASE test', queryTimes=1) + tdSql.query('SELECT * FROM test.t1', queryTimes=1) + tdSql.checkRows(3) + sql = 'SELECT * FROM test.`404e15422d96c8b5de9603c2296681b1`' + self.wait_query(sql, 3, 20) ## tsma1 output ctb for t1 + tdSql.query(sql, queryTimes=1) + tdSql.checkData(0,1, 1) + tdSql.checkData(1,1, 1) + tdSql.checkData(2,1, 1) + #sql = 'select * from test.`82b56f091c4346369da0af777c3e580d`' + #self.wait_query(sql, 2, 10) ## tsma2 output ctb for t1 + #tdSql.query(sql, queryTimes=1) + #tdSql.checkData(0, 1, 1) + #tdSql.checkData(1, 1, 2) + sql = 'select * from test.`163b7c69922cf6d83a98bfa44e52dade`' + self.wait_query(sql, 2, 20) ## tsma_r output ctb for t1 + tdSql.checkData(0, 1, 1) + self.drop_tsma('tsma_r', 'test') + def test_create_tsma(self): function_name = sys._getframe().f_code.co_name tdLog.debug(f'-----{function_name}------') diff --git a/tools/keeper/go.mod b/tools/keeper/go.mod index f520ceb774..f8edf2709b 100644 --- a/tools/keeper/go.mod +++ b/tools/keeper/go.mod @@ -21,38 +21,39 @@ require ( require ( github.com/beorn7/perks v1.0.1 // indirect - github.com/bytedance/sonic v1.9.1 // indirect + github.com/bytedance/sonic v1.11.2 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect + github.com/chenzhuoyu/iasm v0.9.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect - github.com/gabriel-vasile/mimetype v1.4.2 // indirect - github.com/gin-contrib/cors v1.3.1 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gin-contrib/cors v1.6.0 // indirect github.com/gin-contrib/gzip v0.0.3 // indirect github.com/gin-contrib/pprof v1.3.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.14.0 // indirect + github.com/go-playground/validator/v10 v10.19.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect - github.com/leodido/go-urn v1.2.4 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/lestrrat-go/strftime v1.0.6 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.6 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect @@ -66,14 +67,14 @@ require ( github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.11 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.9.0 // indirect - golang.org/x/net v0.10.0 // indirect + golang.org/x/arch v0.7.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/sys v0.24.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.66.4 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/tools/keeper/go.sum b/tools/keeper/go.sum index 9c7721c4d7..8f6e9bd13a 100644 --- a/tools/keeper/go.sum +++ b/tools/keeper/go.sum @@ -52,15 +52,20 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= -github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= +github.com/bytedance/sonic v1.11.2 h1:ywfwo0a/3j9HR8wsYGWsIWl2mvRsI950HyoxiBERw5A= +github.com/bytedance/sonic v1.11.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0= +github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA= +github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= +github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0= +github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -88,10 +93,11 @@ github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= -github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= -github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= -github.com/gin-contrib/cors v1.3.1 h1:doAsuITavI4IOcd0Y19U4B+O0dNWihRyX//nn4sEmgA= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/gin-contrib/cors v1.3.1/go.mod h1:jjEJ4268OPZUcU7k9Pm653S7lXUGcqMADzFA61xsmDk= +github.com/gin-contrib/cors v1.6.0 h1:0Z7D/bVhE6ja07lI8CTjTonp6SB07o8bNuFyRbsBUQg= +github.com/gin-contrib/cors v1.6.0/go.mod h1:cI+h6iOAyxKRtUtC6iF/Si1KSFvGm/gK+kshxlCi8ro= github.com/gin-contrib/gzip v0.0.3 h1:etUaeesHhEORpZMp18zoOhepboiWnFtXrBZxszWUn4k= github.com/gin-contrib/gzip v0.0.3/go.mod h1:YxxswVZIqOvcHEQpsSn+QF5guQtO1dCfy0shBPy4jFc= github.com/gin-contrib/pprof v1.3.0 h1:G9eK6HnbkSqDZBYbzG4wrjCsA4e+cvYAHUZw6W+W9K0= @@ -127,8 +133,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= -github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= -github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4= +github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= @@ -235,8 +241,9 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -250,8 +257,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= -github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8= github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ= @@ -262,8 +269,8 @@ github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamh github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -290,8 +297,8 @@ github.com/panjf2000/ants/v2 v2.4.6 h1:drmj9mcygn2gawZ155dRbo+NfXEfAssjZNU1qoIb4 github.com/panjf2000/ants/v2 v2.4.6/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= -github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= +github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -330,7 +337,7 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/shirou/gopsutil/v3 v3.22.4 h1:srAQaiX6jX/cYL6q29aE0m8lOskT9CurZ9N61YR3yoI= github.com/shirou/gopsutil/v3 v3.22.4/go.mod h1:D01hZJ4pVHPpCTZ3m3T2+wDF2YAGfd+H4ifUguaQzHM= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= @@ -363,8 +370,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI= @@ -387,8 +393,8 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= -github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/xdg/scram v1.0.3/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -404,8 +410,8 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= -golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= +golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -418,8 +424,8 @@ golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -489,8 +495,8 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -565,7 +571,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -577,8 +583,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -725,8 +731,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -758,6 +764,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=