diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 4ba38879eb..14d3888e16 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -5,7 +5,7 @@ node { } file_zh_changed = '' file_en_changed = '' -file_no_doc_changed = '' +file_no_doc_changed = '1' def abortPreviousBuilds() { def currentJobName = env.JOB_NAME def currentBuildNumber = env.BUILD_NUMBER.toInteger() @@ -656,4 +656,4 @@ pipeline { ) } } -} \ No newline at end of file +} diff --git a/docs/en/14-reference/03-taos-sql/31-compress.md b/docs/en/14-reference/03-taos-sql/31-compress.md index 39abfe69bd..f726c8bbe6 100644 --- a/docs/en/14-reference/03-taos-sql/31-compress.md +++ b/docs/en/14-reference/03-taos-sql/31-compress.md @@ -30,11 +30,12 @@ In this article, it specifically refers to the level within the secondary compre | Data Type | Optional Encoding Algorithm | Default Encoding Algorithm | Optional Compression Algorithm|Default Compression Algorithm| Default Compression Level| | :-----------:|:----------:|:-------:|:-------:|:----------:|:----:| -| tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium| +| int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium| +| tinyint/untinyint/smallint/usmallint | simple8b| simple8b | lz4/zlib/zstd/xz| zlib| medium| | bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium| |float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|lz4| medium| -|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| medium| -|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| medium| +|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| zstd| +|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| zstd| ## SQL diff --git a/docs/examples/node/websocketexample/all_type_stmt.js b/docs/examples/node/websocketexample/all_type_stmt.js old mode 100644 new mode 100755 index f095bee090..2297923e75 --- a/docs/examples/node/websocketexample/all_type_stmt.js +++ b/docs/examples/node/websocketexample/all_type_stmt.js @@ -95,8 +95,8 @@ async function all_type_example() { tagParams.setBoolean([true]); tagParams.setVarchar(["hello"]); tagParams.setNchar(["stmt"]); - tagParams.setGeometry([geometryData]); tagParams.setVarBinary([vbData]); + tagParams.setGeometry([geometryData]); await stmt.setTags(tagParams); @@ -108,8 +108,8 @@ async function all_type_example() { bindParams.setBoolean([true]); bindParams.setVarchar(["hello"]); bindParams.setNchar(["stmt"]); - bindParams.setGeometry([geometryData]); bindParams.setVarBinary([vbData]); + bindParams.setGeometry([geometryData]); await stmt.bind(bindParams); await stmt.batch(); diff --git a/docs/zh/06-advanced/05-data-in/05-opcua.md b/docs/zh/06-advanced/05-data-in/05-opcua.md index 5795528d01..5123dacd1b 100644 --- a/docs/zh/06-advanced/05-data-in/05-opcua.md +++ b/docs/zh/06-advanced/05-data-in/05-opcua.md @@ -150,7 +150,7 @@ CSV 文件中的每个 Row 配置一个 OPC 数据点位。Row 的规则如下 #### 5.2. 选择数据点位 -可以通过配置 **根节点ID**、**命名空间**、**正则匹配** 等条件，对点位进行筛选。 +可以通过配置 **根节点ID**、**命名空间**、**节点ID**、**节点名称** 等条件，对点位进行筛选。 通过配置 **超级表名**、**表名称**，指定数据要写入的超级表、子表。 diff --git a/docs/zh/06-advanced/05-data-in/06-opcda.md b/docs/zh/06-advanced/05-data-in/06-opcda.md index 7da5b89fe6..32ac1c1f8a 100644 --- a/docs/zh/06-advanced/05-data-in/06-opcda.md +++ b/docs/zh/06-advanced/05-data-in/06-opcda.md @@ -126,7 +126,7 @@ CSV 文件中的每个 Row 配置一个 OPC 数据点位。Row 的规则如下 #### 4.2. 选择数据点位 -可以通过配置 **根节点ID** 和 **正则匹配** 作为过滤条件，对点位进行筛选。 +可以通过配置 **根节点ID**、**节点ID**、**节点名称** 作为过滤条件，对点位进行筛选。 通过配置 **超级表名**、**表名称**，指定数据要写入的超级表、子表。 diff --git a/docs/zh/06-advanced/05-data-in/pic/opcda-06-point.png b/docs/zh/06-advanced/05-data-in/pic/opcda-06-point.png index 9ab69fb386..b47463dfbb 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/opcda-06-point.png and b/docs/zh/06-advanced/05-data-in/pic/opcda-06-point.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/opcua-06-point.png b/docs/zh/06-advanced/05-data-in/pic/opcua-06-point.png index 783adae3be..18f01e8885 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/opcua-06-point.png and b/docs/zh/06-advanced/05-data-in/pic/opcua-06-point.png differ diff --git a/docs/zh/06-advanced/06-TDgpt/02-management.md b/docs/zh/06-advanced/06-TDgpt/02-management.md index b8261797f9..9aaa123299 100644 --- a/docs/zh/06-advanced/06-TDgpt/02-management.md +++ b/docs/zh/06-advanced/06-TDgpt/02-management.md @@ -4,10 +4,12 @@ sidebar_label: "安装部署" --- ### 环境准备 -为了使用 TDgpt 的高级时序数据分析功能功能，需要在 TDengine 集群中安装部署 AI node（Anode）。Anode 可以运行在 Linux/Windows/Mac 等操作系统之上。请确保安装部署 Anode之前，系统中已经具备 3.10 及以上版本的Python环境，以及相应的 Python 包自动安装组件 Pip，否则无法正常安装 Anode。 +使用 TDgpt 的高级时序数据分析功能需要在 TDengine 集群中安装部署 AI node（Anode）。Anode 可以运行在 Linux/Windows/MacOS 等平台上，同时需要 3.10 或以上版本的 Python 环境支持。 +> 部署 Anode 需要 TDengine Enterprise 3.3.4.3 及以后版本，请首先确认搭配 Anode 使用的 TDengine 能够支持 Anode。 ### 安装及卸载 -不同操作系统上安装及部署操作有细微的差异，主要是安装/卸载操作、安装路径、Anode服务的启停等几个方面。下面将以 Linux 系统为例，说明安装部署的整个流程。使用 Linux 环境下的安装包 TDengine-enterprise-anode-1.x.x.tar.gz 可进行 Anode 的安装部署工作，使用如下命令： +不同操作系统上安装及部署 Anode 有一些差异，主要是卸载操作、安装路径、服务启停等方面。本文以 Linux 系统为例，说明安装部署的流程。 +使用 Linux 环境下的安装包 TDengine-enterprise-anode-1.x.x.tar.gz 可进行 Anode 的安装部署工作，命令如下： ```bash tar -xzvf TDengine-enterprise-anode-1.0.0.tar.gz @@ -15,11 +17,11 @@ cd TDengine-enterprise-anode-1.0.0 sudo ./install.sh ``` -在安装完成 Anode 之后，执行命令 `rmtaosanode` 即可已经安装的 Anode。 -Anode 使用 Python 虚拟环境运行，避免影响安装环境中现有的 Python 库。安装后的默认 Python 虚拟环境目录位于 `/var/lib/taos/taosanode/venv/`。为了避免反复安装虚拟环境带来的开销，卸载 Anode 执行的命令 `rmtaosanode` 并不会自动删除该虚拟环境，如果您确认不需要 Python 的虚拟环境，手动删除即可。 +对于已经安装的 Anode，执行命令 `rmtaosanode` 即可完成卸载。 +为了避免影响系统已有的 Python 环境，Anode 使用虚拟环境运行。安装 Anode 会在目录 `/var/lib/taos/taosanode/venv/` 中创建默认的 Python 虚拟环境，Anode 运行所需要的库均安装在该目录下。为了避免反复安装虚拟环境带来的开销，卸载命令 `rmtaosanode` 并不会自动删除该虚拟环境，如果您确认不再需要 Python 的虚拟环境，手动删除该目录即可。 ### 启停服务 -在 Linux 系统中，安装 Anode 以后可以使用 `systemd` 来管理 Anode 服务。使用如下命令可以启动/停止/检查状态。 +在 Linux 系统中，安装 Anode 以后会自动创建 `taosanoded` 服务。可以使用 `systemd` 来管理 Anode 服务，使用如下命令启动/停止/检查 Anode。 ```bash systemctl start taosanoded @@ -28,6 +30,8 @@ systemctl status taosanoded ``` ### 目录及配置说明 +安装完成后，Anode 主体目录结构如下： + |目录/文件|说明| |---------------|------| |/usr/local/taos/taosanode/bin|可执行文件目录| @@ -39,13 +43,14 @@ systemctl status taosanoded #### 配置说明 -Anode 提供的服务使用 uWSGI 驱动，因此 Anode 和 uWSGI 的配置信息共同存放在相同的配置文件 `taosanode.ini`，该配置文件默认位于 `/etc/taos/`目录下，其具体内容及说明如下： +Anode 的服务需要使用 uWSGI 驱动驱动运行，因此 Anode 和 uWSGI 的配置信息共同存放在相同的配置文件 `taosanode.ini` 中，该配置文件默认位于 `/etc/taos/` 目录下。 +具体内容及配置项说明如下： ```ini [uwsgi] -# Anode HTTP service ip:port -http = 127.0.0.1:6050 +# Anode RESTful service ip:port +http = 127.0.0.1:6090 # base directory for Anode python files， do NOT modified this chdir = /usr/local/taos/taosanode/lib @@ -81,8 +86,8 @@ log-level = DEBUG ``` **提示** -请勿设置 `daemonize` 参数，该参数会导致 uWSGI 与 systemctl 冲突，从而无法正常启动。 -上面的示例配置文件 `taosanode.ini` 只包含了使用 Anode 提供服务的基础配置参数，对于 uWSGI 的其他配置参数设置及其含义和说明请参考 [uWSGIS官方文档](https://uwsgi-docs-zh.readthedocs.io/zh-cn/latest/Options.html)。 +请勿设置 `daemonize` 参数，该参数会导致 uWSGI 与 systemctl 冲突，从而导致 Anode 无法正常启动。 +上面的示例配置文件 `taosanode.ini` 只包含了使用 Anode 提供服务的基础配置参数，对于 uWSGI 的其他配置参数的设置及其说明请参考 [uWSGIS官方文档](https://uwsgi-docs-zh.readthedocs.io/zh-cn/latest/Options.html)。 Anode 运行配置主要是以下： - app-log: Anode 服务运行产生的日志，用户可以调整其到需要的位置 @@ -91,14 +96,15 @@ Anode 运行配置主要是以下： ### Anode 基本操作 +对于 Anode 的管理，用户需要通过 TDengine 的命令行接口 taos 进行。因此下述介绍的管理命令都需要先打开 taos, 连接到 TDengine 运行实例。 #### 创建 Anode ```sql CREATE ANODE {node_url} ``` -node_url 是提供服务的 Anode 的 IP 和 PORT, 例如：`create anode 'http://localhost:6050'`。启动 Anode 以后如果不注册到 TDengine 集群中，则无法提供正常的服务。不建议 Anode 注册到两个或多个集群中。 +node_url 是提供服务的 Anode 的 IP 和 PORT组成的字符串, 例如：`create anode '127.0.0.1:6090'`。Anode 启动后还需要注册到 TDengine 集群中才能提供服务。不建议将 Anode 同时注册到两个集群中。 #### 查看 Anode -列出集群中所有的数据分析节点，包括其 `FQDN`, `PORT`, `STATUS`。 +列出集群中所有的数据分析节点，包括其 `FQDN`, `PORT`, `STATUS`等属性。 ```sql SHOW ANODES; ``` @@ -111,7 +117,7 @@ SHOW ANODES FULL; #### 刷新集群中的分析算法缓存 ```SQL -UPDATE ANODE {node_id} +UPDATE ANODE {anode_id} UPDATE ALL ANODES ``` @@ -119,4 +125,4 @@ UPDATE ALL ANODES ```sql DROP ANODE {anode_id} ``` -删除 Anode 只是将 Anode 从 TDengine 集群中删除，管理 Anode 的启停仍然需要使用`systemctl`命令。 +删除 Anode 只是将 Anode 从 TDengine 集群中删除，管理 Anode 的启停仍然需要使用 `systemctl` 命令。卸载 Anode 则需要使用上面提到的 `rmtaosanode` 命令。 diff --git a/docs/zh/06-advanced/06-TDgpt/03-preprocess.md b/docs/zh/06-advanced/06-TDgpt/03-preprocess.md index 77ddbb49a4..9efd2bdf11 100644 --- a/docs/zh/06-advanced/06-TDgpt/03-preprocess.md +++ b/docs/zh/06-advanced/06-TDgpt/03-preprocess.md @@ -7,27 +7,43 @@ import activity from './pic/activity.png'; import wndata from './pic/white-noise-data.png' ### 分析流程 -在针对时序数据进行高级分析之前，首先进行数据的白噪声检查（White Noise Data check, WND）。整体的流程如下图所示。 +时序数据分析之前需要有预处理的过程，为减轻分析算法的负担，TDgpt 在将时序数据发给具体分析算法进行分析时，已经对数据做了预处理，整体的流程如下图所示。 预处理流程 -- 对于时间序列数据预测分析，首先进行白噪声检查，不是白噪声数据，进行数据重采样和时间戳对齐的预处理，预处理完成后进行数据预测分析。 -- 对于时间序列异常检测，首先进行白噪声检查，检查通过以后无后续的处理流程，直接进行异常检测分析。 +TDgpt 首先对输入数据进行白噪声检查（White Noise Data check）, 检查通过以后针对预测分析，还要进行输入（历史）数据的重采样和时间戳对齐处理（异常检测跳过数据重采样和时间戳对齐步骤）。 +预处理完成以后，再进行预测或异常检测操作。预处理过程部署于预测或异常检测处理逻辑的一部分。 ### 白噪声检查 white-noise-data -白噪声时序数据可以简单地认为是随机数构成的时序数据序列（如上图所示），随机数的时间序列没有分析的价值，因此会直接返回空结果。白噪声检查采用 `Ljung-Box` 检验，`Ljung-Box` 统计量的计算过程需遍历整个输入序列。如果用户能够明确输入序列一定不是白噪声序列，那么可以通过增加参数 `wncheck=0` 要求分析平台忽略白噪声输入时间序列检查，从而节省计算资源。 +白噪声时序数据可以简单地认为是随机数构成的时间数据序列（如上图所示的正态分布随机数序列），随机数构成的时间序列没有分析的价值，因此会直接返回。白噪声检查采用经典的 `Ljung-Box` 统计量检验，计算 `Ljung-Box` 统计量需遍历整个输入时间序列。如果用户能够明确输入序列一定不是白噪声序列，那么可以在参数列表中增加参数 `wncheck=0` 强制要求分析平台忽略白噪声检查，从而节省计算资源。 TDgpt 暂不提供独立的时间序列白噪声检测功能。 -### 数据重采样和时间戳对齐 +### 重采样和时间戳对齐 -对于输入的时间序列数据，在对齐进行预测分析之前需要进行必要的预处理流程。预处理解决以下两个方面的问题： +对于进行预测分析的时间序列数据，在进行预测分析前需要进行必要的预处理。预处理主要解决以下两个问题： -- 真实时间序列数据时间戳未对齐。由于数据生成的原因或者网关给时间序列数据赋值时间戳并不能保证按照严格的时间间隔赋值，此时 分析平台会自动将输入数据按照用户指定的采样频率对时间戳进行对齐处理。例如输入时间序列 [11, 22, 29, 41]，用户指定时间间隔为 10，该时间序列的时间戳将被自动重整为以下时间戳序列 [10, 20, 30, 40]。 -- 数据时间重采样。用户输入时间序列的采样频率超过了输出结果的频率，例如输入时间序列的采样频率是 5，输出结果的频率是 10，输入时间序列 [0， 5， 10， 15， 20， 25， 30] 将被重采用为间隔 为 10 的序列 [0, 10, 20，30]，[5, 15, 25] 处的数据将被丢弃。 +- 真实时间序列数据时间戳未对齐。由于数据生成设备的原因或网关赋值时间戳的时候并不能保证按照严格的时间间隔赋值，时间序列数据并不能保证是严格按照采样频率对齐。例如采样频率为 1Hz 的一个时间序列数据序列，其时间戳序列如下： -需要注意的是，数据输入平台不支持缺失数据补齐后进行的预测分析，如果输入时间序列数据 [11, 22, 29, 49]，并且用户要求的时间间隔为 10，重整对齐后的序列是 [10, 20, 30, 50] 那么该序列进行预测分析将返回错误。 + > ['20:12:21.143', '20:12:22.187', '20:12:23.032', '20:12:24.384', '20:12:25.033'] + + 预测返回的时间序列时间戳会严格对齐，例如返回后续的两个预测结果的时间戳，其时间一定如下：['20:12:26.000', '20:12:27.000']。因此上述的输入时间戳序列要进行时间戳对齐，变换成为如下时间戳序列 + + > ['20:12:21.000', '20:12:22.000', '20:12:23.000', '20:12:24.000', '20:12:25.000'] + + +- 数据时间重采样。用户输入时间序列的采样频率超过了输出结果的频率，例如输入时间序列的采样时间间隔是 5 sec，但是要求输出预测结果的采样时间间隔是 10sec + + > ['20:12:20.000', '20:12:25.000', '20:12:30.000', '20:12:35.000', '20:12:40.000'] + + 重采样为采样间隔为 10sec 的时间戳序列 + + > ['20:12:20.000', '20:12:30.000', '20:12:40.000'] + + 然后将其作为预测分析的输入， ['20:12:25.000', '20:12:35.000'] 数据被丢弃。 + +需要注意的是，预处理过程不支持缺失数据补齐操作，如果输入时间序列数据 ['20:12:10.113', '20:12:21.393', '20:12:29.143', '20:12:51.330']，并且要求的采样时间间隔为 10sec，重整对齐后的时间戳序列是 ['20:12:10.000', '20:12:20.000', '20:12:30.000', '20:12:50.000'] 那么对该序列进行预测分析将返回错误。 diff --git a/docs/zh/06-advanced/06-TDgpt/04-forecast/02-arima.md b/docs/zh/06-advanced/06-TDgpt/04-forecast/02-arima.md index 0b5a80ad71..469f557984 100644 --- a/docs/zh/06-advanced/06-TDgpt/04-forecast/02-arima.md +++ b/docs/zh/06-advanced/06-TDgpt/04-forecast/02-arima.md @@ -3,14 +3,14 @@ title: "ARIMA" sidebar_label: "ARIMA" --- -本节讲述 ARIMA 算法模型的使用方法。 +本节说明 ARIMA 算法模型的使用方法。 ## 功能概述 -ARIMA 即自回归移动平均模型（Autoregressive Integrated Moving Average, ARIMA），也记作 ARIMA(p,d,q)，是统计模型中最常见的一种用来进行时间序列预测的模型。 +ARIMA：Autoregressive Integrated Moving Average，即自回归移动平均模型，记作 ARIMA(p,d,q)，是统计模型中最常见的一种用来进行时间序列预测的模型。 ARIMA 模型是一种自回归模型，只需要自变量即可预测后续的值。ARIMA 模型要求时间序列**平稳**，或经过差分处理后平稳，如果是不平稳的数据，**无法**获得正确的结果。 ->平稳的时间序列：其性质不随观测时间的变化而变化。具有趋势或季节性的时间序列不是平稳时间序列——趋势和季节性使得时间序列在不同时段呈现不同性质。 +> 平稳的时间序列：其性质不随观测时间的变化而变化。具有趋势或季节性的时间序列不是平稳时间序列——趋势和季节性使得时间序列在不同时段呈现不同性质。 以下参数可以动态输入，控制预测过程中生成合适的 ARIMA 模型。 @@ -38,6 +38,11 @@ ARIMA 模型是一种自回归模型，只需要自变量即可预测后续的 FORECAST(i32, "algo=arima,alpha=95,period=10,start_p=1,max_p=5,start_q=1,max_q=5") ``` +完整的调用SQL语句如下： +```SQL +SELECT _frowts, FORECAST(i32, "algo=arima,alpha=95,period=10,start_p=1,max_p=5,start_q=1,max_q=5") from foo +``` + ```json5 { "rows": fc_rows, // 返回结果的行数 diff --git a/docs/zh/06-advanced/06-TDgpt/04-forecast/03-holtwinters.md b/docs/zh/06-advanced/06-TDgpt/04-forecast/03-holtwinters.md index 38662ca2b3..7e92a8ae1a 100644 --- a/docs/zh/06-advanced/06-TDgpt/04-forecast/03-holtwinters.md +++ b/docs/zh/06-advanced/06-TDgpt/04-forecast/03-holtwinters.md @@ -23,11 +23,16 @@ HoltWinters 有两种不同的季节性组成部分，当季节变化在该时 参数 `trend` 和 `seasonal`的均可以选择 `add` （加法模型）或 `mul`（乘法模型）。 ### 示例及结果 -针对 i32 列进行数据预测，输入列 i32 每 10 个点是一个周期，趋势采用乘法模型，季节采用乘法模型 +针对 i32 列进行数据预测，输入列 i32 每 10 个点是一个周期，趋势参数采用乘法模型，季节参数采用乘法模型 ``` FORECAST(i32, "algo=holtwinters,period=10,trend=mul,seasonal=mul") ``` +完整的调用SQL语句如下： +```SQL +SELECT _frowts, FORECAST(i32, "algo=holtwinters, peroid=10,trend=mul,seasonal=mul") from foo +``` + ```json5 { "rows": rows, // 返回结果的行数 diff --git a/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md b/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md index 8cc9cb5b6a..c7388ab9c0 100644 --- a/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md +++ b/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md @@ -3,9 +3,30 @@ title: 预测算法 description: 预测算法 --- -时序数据预测处理以持续一个时间段的时序数据作为输入，预测接下来一个连续时间区间内时间序列数据分布及运行的趋势。用户可以指定输出的（预测）时间序列数据点的数量，因此其输出的结果行数不确定。为此，我们引入了 `FORECAST` 函数提供预测服务。基础数据（用于预测的历史时间序列数据）是该函数的输入，预测结果是该函数的输出。用户可以通过 `FORECAST` 函数调用 Anode 提供的预测算法提供的服务。 +时序数据预测处理以持续一个时间段的时序数据作为输入，预测接下来一个连续时间区间内时间序列数据趋势。用户可以指定输出的（预测）时间序列数据点的数量，因此其输出的结果行数不确定。为此，TDengine 使用新 SQL 函数 `FORECAST` 提供时序数据预测服务。基础数据（用于预测的历史时间序列数据）是该函数的输入，预测结果是该函数的输出。用户可以通过 `FORECAST` 函数调用 Anode 提供的预测算法提供的服务。 -##### 语法 +在后续章节中，使用时序数据表`foo`作为示例，介绍预测和异常检测算法的使用方式，`foo` 表的模式如下： + +|列名称|类型|说明| +|---|---|---| +|ts| timestamp| 主时间戳列| +|i32| int32| 4字节整数，设备测量值 metric| + +```bash +taos> select * from foo; + ts | k | +======================================== + 2020-01-01 00:00:12.681 | 13 | + 2020-01-01 00:00:13.727 | 14 | + 2020-01-01 00:00:14.378 | 8 | + 2020-01-01 00:00:15.774 | 10 | + 2020-01-01 00:00:16.170 | 16 | + 2020-01-01 00:00:17.558 | 26 | + 2020-01-01 00:00:18.938 | 32 | + 2020-01-01 00:00:19.308 | 27 | +``` + +### 语法 ```SQL FORECAST(column_expr, option_expr) @@ -23,7 +44,7 @@ algo=expr1 1. `column_expr`：预测的时序数据列。与异常检测相同，只支持数值类型列输入。 2. `options`：异常检测函数的参数，使用规则与 anomaly_window 相同。预测支持 `conf`, `every`, `rows`, `start`, `rows` 几个控制参数，其含义如下： -**参数说明** +### 参数说明 |参数|含义|默认值| |---|---|---| @@ -31,7 +52,7 @@ algo=expr1 |wncheck|白噪声（white noise data）检查|默认值为 1，0 表示不进行检查| |conf|预测数据的置信区间范围 ，取值范围 [0, 100]|95| |every|预测数据的采样间隔|输入数据的采样间隔| -|start|预测结果的开始时间戳|输入数据最后一个时间戳加上一个采样时间段| +|start|预测结果的开始时间戳|输入数据最后一个时间戳加上一个采样间隔时间区间| |rows|预测结果的记录数|10| 1. 预测查询结果新增三个伪列，具体如下：`_FROWTS`：预测结果的时间戳、`_FLOW`：置信区间下界、`_FHIGH`：置信区间上界, 对于没有置信区间的预测算法，其置信区间同预测结果 @@ -39,34 +60,34 @@ algo=expr1 3. `EVERY`：可以与输入数据的采样频率不同。采样频率只能低于或等于输入数据采样频率，不能**高于**输入数据的采样频率。 4. 对于某些不需要计算置信区间的算法，即使指定了置信区间，返回的结果中其上下界退化成为一个点。 -**示例** +### 示例 ```SQL --- 使用 arima 算法进行预测，预测结果是 10 条记录（默认值），数据进行白噪声检查，默认置信区间 95%. SELECT _flow, _fhigh, _frowts, FORECAST(i32, "algo=arima") FROM foo; ---- 使用 arima 算法进行预测，输入数据的是周期数据，每 10 个采样点是一个周期。返回置信区间是 95%. -SELECT _flow, _fhigh, _frowts, FORECAST(i32, "algo=arima,alpha=95,period=10") +--- 使用 arima 算法进行预测，输入数据的是周期数据，每 10 个采样点是一个周期，返回置信区间是95%的上下边界，同时忽略白噪声检查 +SELECT _flow, _fhigh, _frowts, FORECAST(i32, "algo=arima,alpha=95,period=10,wncheck=0") FROM foo; ``` ``` taos> select _flow, _fhigh, _frowts, forecast(i32) from foo; _flow | _fhigh | _frowts | forecast(i32) | ======================================================================================== - 10.5286684 | 41.8038254 | 2020-01-01 00:01:35.001 | 26 | - -21.9861946 | 83.3938904 | 2020-01-01 00:01:36.001 | 30 | - -78.5686035 | 144.6729126 | 2020-01-01 00:01:37.001 | 33 | - -154.9797363 | 230.3057709 | 2020-01-01 00:01:38.001 | 37 | - -253.9852905 | 337.6083984 | 2020-01-01 00:01:39.001 | 41 | - -375.7857971 | 466.4594727 | 2020-01-01 00:01:40.001 | 45 | - -514.8043823 | 622.4426270 | 2020-01-01 00:01:41.001 | 53 | - -680.6343994 | 796.2861328 | 2020-01-01 00:01:42.001 | 57 | - -868.4956665 | 992.8603516 | 2020-01-01 00:01:43.001 | 62 | - -1076.1566162 | 1214.4498291 | 2020-01-01 00:01:44.001 | 69 | + 10.5286684 | 41.8038254 | 2020-01-01 00:01:35.000 | 26 | + -21.9861946 | 83.3938904 | 2020-01-01 00:01:36.000 | 30 | + -78.5686035 | 144.6729126 | 2020-01-01 00:01:37.000 | 33 | + -154.9797363 | 230.3057709 | 2020-01-01 00:01:38.000 | 37 | + -253.9852905 | 337.6083984 | 2020-01-01 00:01:39.000 | 41 | + -375.7857971 | 466.4594727 | 2020-01-01 00:01:40.000 | 45 | + -514.8043823 | 622.4426270 | 2020-01-01 00:01:41.000 | 53 | + -680.6343994 | 796.2861328 | 2020-01-01 00:01:42.000 | 57 | + -868.4956665 | 992.8603516 | 2020-01-01 00:01:43.000 | 62 | + -1076.1566162 | 1214.4498291 | 2020-01-01 00:01:44.000 | 69 | ``` -**可用预测算法** -- arima -- holtwinters +## 内置预测算法 +- [arima](./02-arima.md) +- [holtwinters](./03-holtwinters.md) diff --git a/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/02-anomaly-detection.md b/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/02-anomaly-detection.md deleted file mode 100644 index 511a9cef11..0000000000 --- a/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/02-anomaly-detection.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -title: "异常检测算法" -sidebar_label: "异常检测算法" ---- - -本节讲述异常检测算法模型的使用方法。 - -## 概述 -分析平台提供了 6 种异常检查模型，6 种异常检查模型分为 3 个类别，分别属于基于统计的异常检测模型、基于数据密度的检测模型、基于深度学习的异常检测模型。在不指定异常检测使用的方法的情况下，默认调用 iqr 的方法进行计算。 - - -### 统计学异常检测方法 - -- k-sigma[1]: 即 ***68–95–99.7 rule*** 。***k***值默认为 3，即序列均值的 3 倍标准差范围为边界，超过边界的是异常值。KSigma 要求数据整体上服从正态分布，如果一个点偏离均值 K 倍标准差，则该点被视为异常点. - -|参数|说明|是否必选|默认值| -|---|---|---|---| -|k|标准差倍数|选填|3| - - -- IQR[2]：四分位距 (Interquartile range, IQR) 是一种衡量变异性的方法. 四分位数将一个按等级排序的数据集划分为四个相等的部分。即 Q1（第 1 个四分位数）、Q2（第 2 个四分位数）和 Q3（第 3 个四分位数）。IQR 定义为 $Q3–Q1$，位于 $Q3+1.5$。无输入参数。 - -- Grubbs[3]: 又称为 Grubbs' test，即最大标准残差测试。Grubbs 通常用作检验最大值、最小值偏离均值的程度是否为异常，该单变量数据集遵循近似标准正态分布。非正态分布数据集不能使用该方法。无输入参数。 - -- SHESD[4]： 带有季节性的 ESD 检测算法。ESD 可以检测时间序列数据的多异常点。需要指定异常点比例的上界***k***，最差的情况是至多 49.9%。数据集的异常比例一般不超过 5% - -|参数|说明|是否必选|默认值| -|---|---|---|---| -|k|异常点在输入数据集中占比，范围是 $1\le K \le 49.9$ |选填|5| - - -### 基于数据密度的检测方法 -LOF[5]: 局部离群因子（LOF，又叫局部异常因子）算法是 Breunig 于 2000 年提出的一种基于密度的局部离群点检测算法，该方法适用于不同类簇密度分散情况迥异的数据。根据数据点周围的数据密集情况，首先计算每个数据点的一个局部可达密度，然后通过局部可达密度进一步计算得到每个数据点的一个离群因子，该离群因子即标识了一个数据点的离群程度，因子值越大，表示离群程度越高，因子值越小，表示离群程度越低。最后，输出离群程度最大的 $top(n)$ 个点。 - - -### 基于自编码器的检测方法 -使用自动编码器的异常检测模型。可以对具有周期性的数据具有较好的检测结果。但是使用该模型需要针对输入的时序数据进行训练，同时将训练完成的模型部署到服务目录中，才能够运行与使用。 - - -### 参考文献 -1. [https://en.wikipedia.org/wiki/68–95–99.7 rule](https://en.wikipedia.org/wiki/68%E2%80%9395%E2%80%9399.7_rule) -2. https://en.wikipedia.org/wiki/Interquartile_range -3. Adikaram, K. K. L. B.; Hussein, M. A.; Effenberger, M.; Becker, T. (2015-01-14). "Data Transformation Technique to Improve the Outlier Detection Power of Grubbs's Test for Data Expected to Follow Linear Relation". Journal of Applied Mathematics. 2015: 1–9. doi:10.1155/2015/708948. -4. Hochenbaum, O. S. Vallis, and A. Kejariwal. 2017. Automatic Anomaly Detection in the Cloud Via Statistical Learning. arXiv preprint arXiv:1704.07706 (2017). -5. Breunig, M. M.; Kriegel, H.-P.; Ng, R. T.; Sander, J. (2000). LOF: Identifying Density-based Local Outliers (PDF). Proceedings of the 2000 ACM SIGMOD International Conference on Management of Data. SIGMOD. pp. 93–104. doi:10.1145/335191.335388. ISBN 1-58113-217-4. - diff --git a/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/02-statistics-approach.md b/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/02-statistics-approach.md new file mode 100644 index 0000000000..d0d6815c25 --- /dev/null +++ b/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/02-statistics-approach.md @@ -0,0 +1,57 @@ +--- +title: "统计学算法" +sidebar_label: "统计学算法" +--- + +- k-sigma[1]: 即 ***68–95–99.7 rule*** 。***k***值默认为 3，即序列均值的 3 倍标准差范围为边界，超过边界的是异常值。KSigma 要求数据整体上服从正态分布，如果一个点偏离均值 K 倍标准差，则该点被视为异常点. + +|参数|说明|是否必选|默认值| +|---|---|---|---| +|k|标准差倍数|选填|3| + +```SQL +--- 指定调用的算法为ksigma, 参数 k 为 2 +SELECT _WSTART, COUNT(*) +FROM foo +ANOMALY_WINDOW(foo.i32, "algo=ksigma,k=2") +``` + +- IQR[2]：Interquartile range(IQR)，四分位距是一种衡量变异性的方法。四分位数将一个按等级排序的数据集划分为四个相等的部分。即 Q1（第 1 个四分位数）、Q2（第 2 个四分位数）和 Q3（第 3 个四分位数）。 $IQR=Q3-Q1$，对于 $v$, $Q1-(1.5 \times IQR) \le v \le Q3+(1.5 \times IQR)$ 是正常值，范围之外的是异常值。无输入参数。 + +```SQL +--- 指定调用的算法为 iqr, 无参数 +SELECT _WSTART, COUNT(*) +FROM foo +ANOMALY_WINDOW(foo.i32, "algo=iqr") +``` + +- Grubbs[3]: Grubbs' test，即最大标准残差测试。Grubbs 通常用作检验最大值、最小值偏离均值的程度是否为异常，要求单变量数据集遵循近似标准正态分布。非正态分布数据集不能使用该方法。无输入参数。 + +```SQL +--- 指定调用的算法为 grubbs, 无参数 +SELECT _WSTART, COUNT(*) +FROM foo +ANOMALY_WINDOW(foo.i32, "algo=grubbs") +``` + +- SHESD[4]： 带有季节性的 ESD 检测算法。ESD 可以检测时间序列数据的多异常点。需要指定异常检测方向（'pos' / 'neg' / 'both'），异常值比例的上界***max_anoms***，最差的情况是至多 49.9%。数据集的异常比例一般不超过 5% + +|参数|说明|是否必选|默认值| +|---|---|---|---| +|direction|异常检测方向类型('pos' / 'neg' / 'both')|否|"both"| +|max_anoms|异常值比例 $0 < K \le 49.9$|否|0.05| +|period|一个周期包含的数据点|否|0| + + +```SQL +--- 指定调用的算法为 shesd, 参数 direction 为 both，异常值比例 5% +SELECT _WSTART, COUNT(*) +FROM foo +ANOMALY_WINDOW(foo.i32, "algo=shesd,direction=both,anoms=0.05") +``` + +### 参考文献 +1. [https://en.wikipedia.org/wiki/68–95–99.7 rule](https://en.wikipedia.org/wiki/68%E2%80%9395%E2%80%9399.7_rule) +2. https://en.wikipedia.org/wiki/Interquartile_range +3. Adikaram, K. K. L. B.; Hussein, M. A.; Effenberger, M.; Becker, T. (2015-01-14). "Data Transformation Technique to Improve the Outlier Detection Power of Grubbs's Test for Data Expected to Follow Linear Relation". Journal of Applied Mathematics. 2015: 1–9. doi:10.1155/2015/708948. +4. Hochenbaum, O. S. Vallis, and A. Kejariwal. 2017. Automatic Anomaly Detection in the Cloud Via Statistical Learning. arXiv preprint arXiv:1704.07706 (2017). diff --git a/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/03-data-density.md b/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/03-data-density.md new file mode 100644 index 0000000000..7c0998c917 --- /dev/null +++ b/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/03-data-density.md @@ -0,0 +1,20 @@ +--- +title: "数据密度算法" +sidebar_label: "数据密度算法" +--- + +### 基于数据密度的检测方法 +LOF[1]: Local Outlier Factor(LOF)，局部离群因子/局部异常因子， +是 Breunig 在 2000 年提出的一种基于密度的局部离群点检测算法，该方法适用于不同类簇密度分散情况迥异的数据。根据数据点周围的数据密集情况，首先计算每个数据点的一个局部可达密度，然后通过局部可达密度进一步计算得到每个数据点的一个离群因子， +该离群因子即标识了一个数据点的离群程度，因子值越大，表示离群程度越高，因子值越小，表示离群程度越低。最后，输出离群程度最大的 $topK$ 个点。 + +```SQL +--- 指定调用的算法为LOF，即可调用该算法 +SELECT count(*) +FROM foo +ANOMALY_WINDOW(foo.i32, "algo=lof") +``` + +### 参考文献 + +1. Breunig, M. M.; Kriegel, H.-P.; Ng, R. T.; Sander, J. (2000). LOF: Identifying Density-based Local Outliers (PDF). Proceedings of the 2000 ACM SIGMOD International Conference on Management of Data. SIGMOD. pp. 93–104. doi:10.1145/335191.335388. ISBN 1-58113-217-4. diff --git a/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/04-machine-learning.md b/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/04-machine-learning.md new file mode 100644 index 0000000000..d72b8e70a9 --- /dev/null +++ b/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/04-machine-learning.md @@ -0,0 +1,17 @@ +--- +title: "机器学习算法" +sidebar_label: "机器学习算法" +--- + +Autoencoder[1]: TDgpt 内置使用自编码器（Autoencoder）的异常检测算法，对周期性的时间序列数据具有较好的检测结果。使用该模型需要针对输入时序数据进行预训练，同时将训练完成的模型保存在到服务目录 `ad_autoencoder` 中，然后在 SQL 语句中指定调用该算法模型即可使用。 + +```SQL +--- 在 options 中增加 model 的名称，ad_autoencoder_foo， 针对 foo 数据集（表）训练的采用自编码器的异常检测模型进行异常检测 +SELECT COUNT(*), _WSTART +FROM foo +ANOMALY_DETECTION(col1, 'algo=encoder, model=ad_autoencoder_foo'); +``` + +### 参考文献 + +1. https://en.wikipedia.org/wiki/Autoencoder diff --git a/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/index.md b/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/index.md index c831b63668..632492ce72 100644 --- a/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/index.md +++ b/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/index.md @@ -5,12 +5,13 @@ description: 异常检测算法 import ad from '../pic/anomaly-detection.png'; -时序数据异常检测，在TDengine 查询处理中以异常窗口的形式服务。因此，可以将异常检测获得的窗口视为一种特殊的**事件窗口**，区别在于异常窗口的触发条件和结束条件不是用户指定，而是检测算法自动识别。因此，可以应用在事件窗口上的函数均可应用在异常窗口中。由于异常检测结果是一个时间窗口，因此调用异常检测的方式也与使用事件窗口的方式相同，在 `WHERE` 子句中使用 `ANOMALY_WINDOW` 关键词即可调用时序数据异常检测服务，同时窗口伪列（`_WSTART`, `_WEND`, `_WDURATION`）也能够像其他窗口函数一样使用。例如： +TDengine 中定义了异常（状态）窗口来提供异常检测服务。异常窗口可以视为一种特殊的**事件窗口（Event Window）**，即异常检测算法确定的连续异常时间序列数据所在的时间窗口。与普通事件窗口区别在于——时间窗口的起始时间和结束时间均是分析算法识别确定，不是用户给定的表达式进行判定。因此，在 `WHERE` 子句中使用 `ANOMALY_WINDOW` 关键词即可调用时序数据异常检测服务，同时窗口伪列（`_WSTART`, `_WEND`, `_WDURATION`）也能够像其他时间窗口一样用于描述异常窗口的起始时间(`_WSTART`)、结束时间(`_WEND`)、持续时间(`_WDURATION`)。例如： ```SQL -SELECT _wstart, _wend, SUM(i32) +--- 使用异常检测算法 IQR 对输入列 col_val 进行异常检测。同时输出异常窗口的起始时间、结束时间、以及异常窗口内 col 列的和。 +SELECT _wstart, _wend, SUM(col) FROM foo -ANOMALY_WINDOW(i32, "algo=iqr"); +ANOMALY_WINDOW(col_val, "algo=iqr"); ``` 如下图所示，Anode 将返回时序数据异常窗口 $[10:51:30, 10:53:40]$ @@ -36,18 +37,14 @@ algo=expr1 3. 异常检测的结果可以作为外层查询的子查询输入，在 `SELECT` 子句中使用的聚合函数或标量函数与其他类型的窗口查询相同。 4. 输入数据默认进行白噪声检查，如果输入数据是白噪声，将不会有任何（异常）窗口信息返回。 -**参数说明** +### 参数说明 |参数|含义|默认值| |---|---|---| |algo|异常检测调用的算法|iqr| -|wncheck|对输入数据列是否进行白噪声检查|取值为 0 或者 1，默认值为 1，表示进行白噪声检查| +|wncheck|对输入数据列是否进行白噪声检查，取值为0或1|1| -异常检测的返回结果以窗口形式呈现，因此窗口查询相关的伪列在这种场景下仍然可用。可用的伪列如下： -1. `_WSTART`： 异常窗口开始时间戳 -2. `_WEND`：异常窗口结束时间戳 -3. `_WDURATION`：异常窗口持续时间 -**示例** +### 示例 ```SQL --- 使用 iqr 算法进行异常检测，检测列 i32 列。 SELECT _wstart, _wend, SUM(i32) @@ -58,10 +55,8 @@ ANOMALY_WINDOW(i32, "algo=iqr"); SELECT _wstart, _wend, SUM(i32) FROM foo ANOMALY_WINDOW(i32, "algo=ksigma,k=2"); -``` -``` -taos> SELECT _wstart, _wend, count(*) FROM ai.atb ANOMAYL_WINDOW(i32); +taos> SELECT _wstart, _wend, count(*) FROM foo ANOMAYL_WINDOW(i32); _wstart | _wend | count(*) | ==================================================================== 2020-01-01 00:00:16.000 | 2020-01-01 00:00:17.000 | 2 | @@ -69,10 +64,6 @@ Query OK, 1 row(s) in set (0.028946s) ``` -**可用异常检测算法** -- iqr -- ksigma -- grubbs -- lof -- shesd -- tac +### 内置异常检测算法 +分析平台内置了6个异常检查模型，分为3个类别，分别是[基于统计学的算法](./02-statistics-approach.md)、[基于数据密度的算法](./03-data-density.md)、以及[基于机器学习的算法](./04-machine-learning.md)。在不指定异常检测使用的方法的情况下，默认调用 IQR 进行异常检测。 + diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md b/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md index d375ad44b8..954076c8fd 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md @@ -10,8 +10,8 @@ sidebar_label: "预测算法" `execute` 方法执行完成后的返回一个如下字典对象， 预测返回结果如下： ```python return { - "mse": mse, # 预测算法的拟合数据最小均方误差(minimum squared error) - "res": res # 结果数组 [时间戳数组, 预测结果数组, 预测结果执行区间下界数组，预测结果执行区间上界数组] + "mse": mse, # 预测算法的拟合数据最小均方误差(minimum squared error) + "res": res # 结果数组 [时间戳数组, 预测结果数组, 预测结果执行区间下界数组，预测结果执行区间上界数组] } ``` @@ -77,14 +77,8 @@ class _MyForecastService(AbstractForecastService): """该算法无需任何输入参数，直接重载父类该函数，不处理算法参数设置逻辑""" pass ``` -将该文件保存在 `./taosanalytics/algo/ad/` 目录下，然后重启 taosanode 服务。然后就可以通过 SQL 语句调用该检测算法。 -```SQL ---- 对 col 列进行异常检测，通过指定 algo 参数为 myad 来调用新添加的异常检测类 -SELECT COUNT(*) FROM foo ANOMALY_DETECTION(col, 'algo=myad') -``` - -将该文件保存在 `./taosanalytics/algo/fc/` 目录下，然后重启 taosanode 服务。通过执行 `SHOW ANODES FULL` 能够看到新加入的算法，通过 SQL 语句调用该预测算法。 +将该文件保存在 `./taosanalytics/algo/fc/` 目录下，然后重启 taosanode 服务。在 TDengine 命令行接口中执行 `SHOW ANODES FULL` 能够看到新加入的算法。应用就可以通过 SQL 语句调用该预测算法。 ```SQL --- 对 col 列进行异常检测，通过指定 algo 参数为 myfc 来调用新添加的预测类 @@ -92,6 +86,7 @@ SELECT _flow, _fhigh, _frowts, FORECAST(col_name, "algo=myfc") FROM foo; ``` +如果是第一次启动该 Anode, 请按照 [TDgpt 安装部署](../../management/) 里的步骤先将该 Anode 添加到 TDengine 系统中。 ### 单元测试 diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md b/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md index 8068931653..dc0a534706 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md @@ -48,13 +48,13 @@ class _MyAnomalyDetectionService(AbstractAnomalyDetectionService): pass ``` -将该文件保存在 `./taosanalytics/algo/ad/` 目录下，然后重启 taosanode 服务。然后就可以通过 SQL 语句调用该检测算法。 +将该文件保存在 `./taosanalytics/algo/ad/` 目录下，然后重启 taosanode 服务。在 TDengine 命令行接口 taos 中执行 `SHOW ANODES FULL` 就能够看到新加入的算法，然后应用就可以通过 SQL 语句调用该检测算法。 ```SQL --- 对 col 列进行异常检测，通过指定 algo 参数为 myad 来调用新添加的异常检测类 SELECT COUNT(*) FROM foo ANOMALY_DETECTION(col, 'algo=myad') ``` - +如果是第一次启动该 Anode, 请按照 [TDgpt 安装部署](../../management/) 里的步骤先将该 Anode 添加到 TDengine 系统中。 ### 单元测试 diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/index.md b/docs/zh/06-advanced/06-TDgpt/06-dev/index.md index 6ef9e67a20..b7f048cefc 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/index.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/index.md @@ -2,14 +2,20 @@ title: "算法开发者指南" sidebar_label: "算法开发者指南" --- -TDgpt 是一个可扩展的时序数据高级分析平台，用户仅按照简易的步骤就能将新分析算法添加到分析平台中。将开发完成的算法代码文件放入对应的目录文件夹，然后重启 Anode 即可完成扩展升级。Anode 启动后会自动加载特定目录的分析算法。用户可以直接使用 SQL 语句调用添加到 TDgpt 系统中的分析算法。得益于 TDgpt 与 taosd 的松散耦合关系，分析平台升级对 taosd 完全没有影响。应用系统也不需要做任何更改就能够完成分析功能和分析算法的升级。 +TDgpt 是一个可扩展的时序数据高级分析平台，用户遵循简易的步骤就能将自己开发的分析算法添加到分析平台, 各种应用就可以通过SQL语句直接调用, 让高级分析算法的使用门槛降到几乎为零。目前 TDpgt 平台只支持使用 Python 语言开发的分析算法。 +Anode 采用类动态加载模式，在启动的时候扫描特定目录内满足约定条件的所有代码文件，并将其加载到系统中。因此，开发者只需要遵循以下几步就能完成新算法的添加工作： +1. 开发完成符合要求的分析算法类 +2. 将代码文件放入对应目录，然后重启 Anode +3. 使用SQL命令"CREATE ANODE"，将 Anode 添加到 TDengine + +此时就完成了新算法的添加工作，之后应用就可以直接使用SQL语句调用新算法。得益于 TDgpt 与 TDengine主进程 `taosd` 的松散耦合，Anode算法升级对 `taosd` 完全没有影响。应用系统只需要调整对应的SQL语句调用新（升级的）算法，就能够快速完成分析功能和分析算法的升级。 -这种方式能够按需扩展新分析算法，极大地拓展了 TDgpt 适应的范围，用户可以将契合业务场景开发的（预测、异常检测）分析算法嵌入到 TDgpt，并通过 SQL 语句进行调用。在不更改或更改非常少的应用系统代码的前提下，就能够快速完成分析功能的平滑升级。 +这种方式能够按需扩展分析算法，极大地拓展 TDgpt 的适应范围，用户可以按需将更契合业务场景的、更准确的（预测、异常检测）分析算法动态嵌入到 TDgpt，并通过 SQL 语句进行调用。在基本不用更改应用系统代码的前提下，就能够快速完成分析功能的平滑升级。 -本节说明如何将预测算法和异常检测算法添加到 TDengine 分析平台。 +以下内容将说明如何将分析算法添加到 Anode 中并能够通过SQL语句进行调用。 ## 目录结构 -首先需要了解TDgpt的目录结构。其主体目录结构如下图： +Anode的主要目录结构如下图所示 ```bash . @@ -42,7 +48,7 @@ TDgpt 是一个可扩展的时序数据高级分析平台，用户仅按照简 ### 类命名规范 -由于算法采用自动加载，因此其只识别按照特定命名方式的类。算法类的名称需要以下划线开始，以 Service 结尾。例如：`_KsigmaService` 是 KSigma 异常检测算法类。 +Anode采用算法自动加载模式，因此只识别符合命名约定的 Python 类。需要加载的算法类名称需要以下划线 `_` 开始并以 `Service` 结尾。例如：`_KsigmaService` 是 KSigma 异常检测算法类。 ### 类继承约定 @@ -50,33 +56,27 @@ TDgpt 是一个可扩展的时序数据高级分析平台，用户仅按照简 - 预测算法需要从 `AbstractForecastService` 继承，同样需要实现其核心抽象方法 `execute` ### 类属性初始化 -每个算法实现的类需要静态初始化两个类属性，分别是： +实现的类需要初始化以下两个类属性： -- `name`：触发调用的关键词，全小写英文字母。该名称也是通过 `SHOW` 命令查看可用分析算法是显示的名称。 -- `desc`：算法的描述信息 +- `name`：识别该算法的关键词，全小写英文字母。通过 `SHOW` 命令查看可用算法显示的名称即为该名称。 +- `desc`：算法的基础描述信息 ```SQL ---- algo 后面的参数 algo_name 即为类名称 `name` -SELECT COUNT(*) FROM foo ANOMALY_DETECTION(col_name, 'algo=algo_name') -``` - -## 添加具有训练模型的分析算法 - -某些深度学习的分析算法需要使用输入时间序列数据进行训练，然后生成针对训练数据集的分析模型。这种情况下，同一个分析算法对应不同的输入数据集有不同的分析模型。 -这种类型的分析算法要添加到 TDgpt 中，首先需要在 `model` 目录中建立目录，将采用该算法针对不同的输入时间序列数据生成的训练模型均保存在该目录下。如下图所示，针对不同的数据集，采用自编码器训练的数据异常检测算法生成的模型均保存在该目录下。为了确保模型能够正常读取加载，要求存储的模型使用`joblib`库进行序列化保存。 -采用训练-保存模型的方式可以一次训练，多次调用的优势。避免动态训练调用所带来的反复训练开销。 - -调用已经保存的模型，需要首先调用`set_params`方法，并在参数中指定调用模型的名称 `{"model": "ad_encoder_keras"}` 即可调用该模型进行计算。调用方式如下： - -```python -def test_autoencoder_ad(self): - # 获取特定的算法对象 - # ... - - # 指定调用的模型，该模型是之前针对该数据集进行训练获得 - s.set_params({"model": "ad_encoder_keras"}) - - # 执行检查动作，并返回结果 - r = s.execute() +--- algo 后面的参数 name 即为类属性 `name` +SELECT COUNT(*) FROM foo ANOMALY_DETECTION(col_name, 'algo=name') ``` +## 添加具有模型的分析算法 + +基于统计学的分析算法可以直接针对输入时间序列数据进行分析，但是某些深度学习算法对于输入数据需要较长的时间训练，并且生成相应的模型。这种情况下，同一个分析算法对应不同的输入数据集有不同的分析模型。 +将具有模型的分析算法添加到 Anode 中，首先需要在 `model` 目录中建立该算法对应的目录（目录名称可自拟），将采用该算法针对不同的输入时间序列数据生成的训练模型均需要保存在该目录下，同时目录名称要在分析算法中确定，以便能够固定加载该目录下的分析模型。为了确保模型能够正常读取加载，存储的模型使用`joblib`库进行序列化保存。 + +下面以自编码器（Autoencoder）为例，说明如何添加要预先训练的模型进行异常检测。 +首先我们在`model`目录中创建一个目录 -- `ad_detection`，该目录将用来保存所有使用自编码器训练的模型。然后，我们使用自编码器对 foo 表的时间序列数据进行训练，得到模型 ad_autoencoder_foo，使用 `joblib`序列化以后保存在`ad_detection` 目录中。 + +使用 SQL 调用已经保存的模型，需要在调用参数中指定模型名称``model=ad_autoencoder_foo`，而 `algo=encoder` 是确定调用的自编码器生成的模型（这里的`encoder`说明调用的是自编码器算法模型，该名称是添加算法的时候在代码中定义）以便能够调用该模型。 + +```SQL +--- 在 options 中增加 model 的名称，ad_autoencoder_foo， 针对 foo 数据集（表）训练的采用自编码器的异常检测模型进行异常检测 +SELECT COUNT(*), _WSTART FROM foo ANOMALY_DETECTION(col1, 'algo=encoder, model=ad_autoencoder_foo'); +``` diff --git a/docs/zh/06-advanced/06-TDgpt/index.md b/docs/zh/06-advanced/06-TDgpt/index.md index 96b654b068..3f650b196b 100644 --- a/docs/zh/06-advanced/06-TDgpt/index.md +++ b/docs/zh/06-advanced/06-TDgpt/index.md @@ -6,17 +6,20 @@ title: TDgpt import TDgpt from './pic/data-analysis.png'; -TDgpt 是 TDengine Enterprise 中针对时序数据提供高级分析功能的企业级组件，能够独立于 TDengine 主进程部署和运行，不消耗和占用 TDengine 主进程的资源，通过内置接口向 TDengine 提供运行时动态扩展的高级时序数据分析功能。TDgpt 具有服务无状态、功能易扩展、快速弹性部署、应用轻量化、高安全性等特点。 -TDgpt 运行在部署于 TDengine 集群中的 AI Node (Anode)中。每个 TDengine 集群中可以部署一个或若干个 Anode 节点，不同的 Anode 节点之间不相关，无同步或协同的要求。Anode 注册到 TDengine 集群以后，就可以通过内部接口提供服务。TDgpt 提供的高级时序数据分析服务可分为时序数据异常检测和时序数据预测分析两个类别。 +TDgpt 是 TDengine Enterprise 中针对时序数据提供高级分析功能的企业级组件，通过内置接口向 TDengine 提供运行时动态扩展的时序数据分析服务。TDgpt 能够独立于 TDengine 主进程部署和运行，因此可避免消耗占用 TDengine 集群的主进程资源。 +TDgpt 具有服务无状态、功能易扩展、快速弹性部署、应用轻量化、高安全性等优势。 +TDgpt 运行在集群中的 AI Node (Anode)中，集群中可以部署若干个 Anode 节点，不同的 Anode 节点之间无同步依赖或协同的要求。Anode 注册到 TDengine 集群以后，立即就可以提供服务。TDgpt 提供的高级时序数据分析服务可分为时序数据异常检测和时序数据预测分析两大类。 下图是部署 TDgpt 的 TDengine 集群示意图。 TDgpt架构图 -通过注册指令将 Anode 注册到 Mnode 中以后，就加入到 TDengine 集群，并可被查询引擎动态调用执行。在查询处理过程中，查询引擎根据生成的物理执行计划，**按需**向 Anode 请求高级时序数据分析服务。用户可通过SQL语句与 Anode 节点交互，并使用其提供的全部分析服务。需要注意的是 Anode 不直接接受用户的数据分析请求。同时 Anode 提供高效的动态注册机制，其注册和卸载过程完全不影响 TDengine 集群的服务，只影响提供对应的查询服务能力。 +在查询处理过程中，Vnode中运行的查询引擎会根据查询处理物理执行计划，按需向 Anode 请求高级时序数据分析服务。因此用户可通过 SQL 语句与 Anode 节点交互并使用其提供的全部分析服务。需要注意的是 Anode 不直接接受用户的数据分析请求。同时 Anode 具备分析算法动态注册机制，其算法扩展过程完全不影响 TDengine 集群的服务，仅在非常小的（秒级）时间窗口内影响涉及高级分析的查询服务。 -TDgpt 提供的高级数据分析功能分为时序数据异常检测和时序数据预测。 -- 时序数据异常检测的结果采用异常窗口的形式提供，即分析系统自动将算法检测到的连续异常数据以时间窗口的形式返回，其使用方式与 TDengine 中其他类型的时间窗口（例如状态窗口、事件窗口）类似。特别地，可以将异常数据窗口视作为一种特殊的**事件窗口（Event Window）**，因此状态窗口可使用的所有查询操作均可应用在异常窗口上。 -- 时序数据预测是基于输入的时间序列数据，使用指定（或默认）预测算法给出输入时序数据后续时间序列的**预测**观测值数据。因此，不同于异常检测是以窗口的形式存在，时序数据预测在 TDengine 中是一个（不确定输出）函数。 +目前 TDgpt 提供如下的高级分析服务： +- 时序数据异常检测。TDengine 中定义了新的时间窗口——异常（状态）窗口——来提供异常检测服务。异常窗口可以视为一种特殊的**事件窗口（Event Window）**，即异常检测算法确定的连续异常时间序列数据所在的时间窗口。与普通事件窗口区别在于——时间窗口的起始时间和结束时间均是分析算法确定，不是用户指定的表达式判定。异常窗口使用方式与其他类型的时间窗口（例如状态窗口、会话窗口等）类似。因此时间窗口内可使用的查询操作均可应用在异常窗口上。 +- 时序数据预测。定义了一个新函数`FORECAST`，基于输入的（历史）时间序列数据调用指定（或默认）预测算法给出输入时序数据后续时间序列的**预测**数据。 + +TDgpt 还为算法开发者提供了一 SDK。任何开发者只需要按照[算法开发者指南](./dev)的步骤，就可以将自己独有的时序数据预测或时序数据异常检测算法无缝集成到 TDgpt, 这样 TDengine 用户就可以通过一条 SQL 获得时序数据预测结果或是异常窗口了, 大幅降低了用户使用新的时序数据分析算法的门槛，而且让 TDengine 成为一开放的系统。 diff --git a/docs/zh/14-reference/01-components/01-taosd.md b/docs/zh/14-reference/01-components/01-taosd.md index 64ae69528b..101058c2a8 100644 --- a/docs/zh/14-reference/01-components/01-taosd.md +++ b/docs/zh/14-reference/01-components/01-taosd.md @@ -294,6 +294,7 @@ charset 的有效值是 UTF-8。 |checkpointBackupDir | |内部参数，用于恢复 snode 数据| |enableAuditDelete | |内部参数，用于测试审计功能| |slowLogThresholdTest| |内部参数，用于测试慢日志| +|bypassFlag |3.3.4.5 后|内部参数，用于短路测试，0：正常写入，1：写入消息在 taos 客户端发送 RPC 消息前返回，2：写入消息在 taosd 服务端收到 RPC 消息后返回，4：写入消息在 taosd 服务端写入内存缓存前返回，8：写入消息在 taosd 服务端数据落盘前返回；默认值 0| ### 压缩参数 |参数名称|支持版本|参数含义| diff --git a/docs/zh/14-reference/01-components/02-taosc.md b/docs/zh/14-reference/01-components/02-taosc.md index 09653ae3ef..631f457391 100755 --- a/docs/zh/14-reference/01-components/02-taosc.md +++ b/docs/zh/14-reference/01-components/02-taosc.md @@ -97,6 +97,7 @@ TDengine 客户端驱动提供了应用编程所需要的全部 API，并且在 |safetyCheckLevel |3.3.3.0 后|内部参数，用于随机失败测试| |simdEnable |3.3.4.3 后|内部参数，用于测试 SIMD 加速| |AVX512Enable |3.3.4.3 后|内部参数，用于测试 AVX512 加速| +|bypassFlag |3.3.4.5 后|内部参数，用于短路测试，0：正常写入，1：写入消息在 taos 客户端发送 RPC 消息前返回，2：写入消息在 taosd 服务端收到 RPC 消息后返回，4：写入消息在 taosd 服务端写入内存缓存前返回，8：写入消息在 taosd 服务端数据落盘前返回；缺省值：0| ### SHELL 相关 |参数名称|支持版本|参数含义| diff --git a/docs/zh/14-reference/03-taos-sql/02-database.md b/docs/zh/14-reference/03-taos-sql/02-database.md index 8ba04f686c..1ccc5071f0 100644 --- a/docs/zh/14-reference/03-taos-sql/02-database.md +++ b/docs/zh/14-reference/03-taos-sql/02-database.md @@ -64,7 +64,7 @@ database_option: { - DURATION：数据文件存储数据的时间跨度。可以使用加单位的表示形式，如 DURATION 100h、DURATION 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。不加时间单位时默认单位为天，如 DURATION 50 表示 50 天。 - MAXROWS：文件块中记录的最大条数，默认为 4096 条。 - MINROWS：文件块中记录的最小条数，默认为 100 条。 -- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于3倍的 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据从而释放存储空间。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 \<= keep 1 \<= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。 +- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于 3 倍的 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据从而释放存储空间。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 \<= keep 1 \<= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。了解更多，请点击 [关于主键时间戳](https://docs.taosdata.com/reference/taos-sql/insert/) - KEEP_TIME_OFFSET：自 3.2.0.0 版本生效。删除或迁移保存时间超过 KEEP 值的数据的延迟执行时间，默认值为 0 (小时)。在数据文件保存时间超过 KEEP 后，删除或迁移操作不会立即执行，而会额外等待本参数指定的时间间隔，以实现与业务高峰期错开的目的。 - STT_TRIGGER：表示落盘文件触发文件合并的个数。开源版本固定为 1，企业版本可设置范围为 1 到 16。对于少表高频写入场景，此参数建议使用默认配置；而对于多表低频写入场景，此参数建议配置较大的值。 - SINGLE_STABLE：表示此数据库中是否只可以创建一个超级表，用于超级表列非常多的情况。 @@ -80,6 +80,7 @@ database_option: { - WAL_FSYNC_PERIOD：当 WAL_LEVEL 参数设置为 2 时，用于设置落盘的周期。默认为 3000，单位毫秒。最小为 0，表示每次写入立即落盘；最大为 180000，即三分钟。 - WAL_RETENTION_PERIOD: 为了数据订阅消费，需要 WAL 日志文件额外保留的最大时长策略。WAL 日志清理，不受订阅客户端消费状态影响。单位为 s。默认为 3600，表示在 WAL 保留最近 3600 秒的数据，请根据数据订阅的需要修改这个参数为适当值。 - WAL_RETENTION_SIZE：为了数据订阅消费，需要 WAL 日志文件额外保留的最大累计大小策略。单位为 KB。默认为 0，表示累计大小无上限。 + ### 创建数据库示例 ```sql @@ -90,7 +91,7 @@ create database if not exists db vgroups 10 buffer 10 ### 使用数据库 -``` +```sql USE db_name; ``` @@ -98,7 +99,7 @@ USE db_name; ## 删除数据库 -``` +```sql DROP DATABASE [IF EXISTS] db_name ``` @@ -128,7 +129,7 @@ alter_database_option: { } ``` -### 修改 CACHESIZE +### 修改 CACHESIZE 修改数据库参数的命令使用简单，难的是如何确定是否需要修改以及如何修改。本小节描述如何判断数据库的 cachesize 是否够用。 @@ -157,13 +158,13 @@ alter_database_option: { ### 查看系统中的所有数据库 -``` +```sql SHOW DATABASES; ``` ### 显示一个数据库的创建语句 -``` +```sql SHOW CREATE DATABASE db_name \G; ``` diff --git a/docs/zh/14-reference/03-taos-sql/05-insert.md b/docs/zh/14-reference/03-taos-sql/05-insert.md index 40f8e95006..ccf24e882c 100644 --- a/docs/zh/14-reference/03-taos-sql/05-insert.md +++ b/docs/zh/14-reference/03-taos-sql/05-insert.md @@ -5,9 +5,11 @@ description: 写入数据的详细语法 --- ## 写入语法 + 写入记录支持两种语法, 正常语法和超级表语法. 正常语法下, 紧跟INSERT INTO后名的表名是子表名或者普通表名. 超级表语法下, 紧跟INSERT INTO后名的表名是超级表名 ### 正常语法 + ```sql INSERT INTO tb_name @@ -22,7 +24,9 @@ INSERT INTO INSERT INTO tb_name [(field1_name, ...)] subquery ``` + ### 超级表语法 + ```sql INSERT INTO stb1_name [(field1_name, ...)] @@ -32,16 +36,18 @@ INSERT INTO ...]; ``` -**关于时间戳** +#### 关于主键时间戳 -1. TDengine 要求插入的数据必须要有时间戳，插入数据的时间戳要注意以下几点： +TDengine 要求插入的数据必须要有时间戳，插入数据的时间戳要注意以下几点： -2. 时间戳不同的格式语法会有不同的精度影响。字符串格式的时间戳写法不受所在 DATABASE 的时间精度设置影响；而长整形格式的时间戳写法会受到所在 DATABASE 的时间精度设置影响。例如，时间戳"2021-07-13 16:16:48"的 UNIX 秒数为 1626164208。则其在毫秒精度下需要写作 1626164208000，在微秒精度设置下就需要写为 1626164208000000，纳秒精度设置下需要写为 1626164208000000000。 +1. 时间戳不同的格式语法会有不同的精度影响。字符串格式的时间戳写法不受所在 DATABASE 的时间精度设置影响；而长整形格式的时间戳写法会受到所在 DATABASE 的时间精度设置影响。例如，时间戳"2021-07-13 16:16:48"的 UNIX 秒数为 1626164208。则其在毫秒精度下需要写作 1626164208000，在微秒精度设置下就需要写为 1626164208000000，纳秒精度设置下需要写为 1626164208000000000。 -3. 一次插入多行数据时，不要把首列的时间戳的值都写 NOW。否则会导致语句中的多条记录使用相同的时间戳，于是就可能出现相互覆盖以致这些数据行无法全部被正确保存。其原因在于，NOW 函数在执行中会被解析为所在 SQL 语句的客户端执行时间，出现在同一语句中的多个 NOW 标记也就会被替换为完全相同的时间戳取值。 - 允许插入的最老记录的时间戳，是相对于当前服务器时间，减去配置的 KEEP 值（数据保留的天数, 可以在创建数据库时指定，缺省值是 3650 天）。允许插入的最新记录的时间戳，取决于数据库的 PRECISION 值（时间戳精度, 可以在创建数据库时指定, ms 表示毫秒，us 表示微秒，ns 表示纳秒，默认毫秒）：如果是毫秒或微秒, 取值为 1970 年 1 月 1 日 00:00:00.000 UTC 加上 1000 年, 即 2970 年 1 月 1 日 00:00:00.000 UTC; 如果是纳秒, 取值为 1970 年 1 月 1 日 00:00:00.000000000 UTC 加上 292 年, 即 2262 年 1 月 1 日 00:00:00.000000000 UTC。 +2. 一次插入多行数据时，不要把首列的时间戳的值都写 NOW。否则会导致语句中的多条记录使用相同的时间戳，于是就可能出现相互覆盖以致这些数据行无法全部被正确保存。其原因在于，NOW 函数在执行中会被解析为所在 SQL 语句的客户端执行时间，出现在同一语句中的多个 NOW 标记也就会被替换为完全相同的时间戳取值。 -**语法说明** +3. 允许插入的最大时间戳为当前时间加上 100 年, 比如当前时间为`2024-11-11 12:00:00`，则允许插入的最大时间戳为`2124-11-11 12:00:00`。允许插入的最小时间戳取决于数据库的 KEEP 设置。企业版支持三级存储，可以设置多个 KEEP 时间，如下图所示，如果数据库的 KEEP 配置为`100h,100d,3650d`，则允许的最小时间戳为当前时间减去 3650 天。那么时间戳在`[Now - 100h, Now + 100y)`内的会保存在一级存储，时间戳在`[Now - 100d, Now - 100h)`内的会保存在二级存储，时间戳在`[Now - 3650d, Now - 100d)`内的会保存在三级存储。社区版不支持多级存储功能，只能配置一个 KEEP 值，如果配置多个，则取其最大者。如果时间戳不在有效时间范围内，TDengine 将返回错误“Timestamp out of range"。 +![Keep timerange 示意图](./pic/database-keep.jpg) + +#### 语法说明 1. 可以指定要插入值的列，对于未指定的列数据库将自动填充为 NULL。 @@ -56,22 +62,24 @@ INSERT INTO ```sql INSERT INTO d1001 USING meters TAGS('Beijing.Chaoyang', 2) VALUES('a'); ``` + 6. 对于向多个子表插入数据的情况，依然会有部分数据写入失败，部分数据写入成功的情况。这是因为多个子表可能分布在不同的 VNODE 上，客户端将 INSERT 语句完整解析后，将数据发往各个涉及的 VNODE 上，每个 VNODE 独立进行写入操作。如果某个 VNODE 因为某些原因（比如网络问题或磁盘故障）导致写入失败，并不会影响其他 VNODE 节点的写入。 7. 主键列值必须指定且不能为 NULL。 -**正常语法说明** +#### 正常语法说明 1. USING 子句是自动建表语法。如果用户在写数据时并不确定某个表是否存在，此时可以在写入数据时使用自动建表语法来创建不存在的表，若该表已存在则不会建立新表。自动建表时，要求必须以超级表为模板，并写明数据表的 TAGS 取值。可以只是指定部分 TAGS 列的取值，未被指定的 TAGS 列将置为 NULL。 2. 可以使用 `INSERT ... subquery` 语句将 TDengine 中的数据插入到指定表中。subquery 可以是任意的查询语句。此语法只能用于子表和普通表，且不支持自动建表。 -**超级表语法说明** +#### 超级表语法说明 1. 在 field_name 列表中必须指定 tbname 列，否则报错. tbname列是子表名, 类型是字符串. 其中字符不用转义, 不能包含点‘.‘ 2. 在 field_name 列表中支持标签列，当子表已经存在时，指定标签值并不会触发标签值的修改；当子表不存在时会使用所指定的标签值建立子表. 如果没有指定任何标签列，则把所有标签列的值设置为NULL 3. 不支持参数绑定写入 + ## 插入一条记录 指定已经创建好的数据子表的表名，并通过 VALUES 关键字提供一行或多行数据，即可向数据库写入这些数据。例如，执行如下语句可以写入一行记录： @@ -154,15 +162,18 @@ INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/c INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/csvfile_21001.csv' d21002 USING meters (groupId) TAGS (2) FILE '/tmp/csvfile_21002.csv'; ``` + ## 向超级表插入数据并自动创建子表 自动建表, 表名通过 tbname 列指定 + ```sql INSERT INTO meters(tbname, location, groupId, ts, current, voltage, phase) VALUES ('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32) ('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33) ('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33) ``` + ## 通过 CSV 文件向超级表插入数据并自动创建子表 根据 csv 文件内容，为 超级表创建子表，并填充相应 column 与 tag diff --git a/docs/zh/14-reference/03-taos-sql/32-compress.md b/docs/zh/14-reference/03-taos-sql/32-compress.md index 0f2b260832..51650c9123 100644 --- a/docs/zh/14-reference/03-taos-sql/32-compress.md +++ b/docs/zh/14-reference/03-taos-sql/32-compress.md @@ -31,11 +31,12 @@ description: 可配置压缩算法 | 数据类型 | 可选编码算法 | 编码算法默认值 | 可选压缩算法|压缩算法默认值| 压缩等级默认值| | :-----------:|:----------:|:-------:|:-------:|:----------:|:----:| -| tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium| -| bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium| +| int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium| +| tinyint/untinyint/smallint/usmallint | simple8b| simple8b | lz4/zlib/zstd/xz| zlib | medium| +| bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium| |float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|lz4| medium| -|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| medium| -|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| medium| +|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| zstd| medium| +|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| zstd| medium| ## SQL 语法 diff --git a/docs/zh/14-reference/03-taos-sql/pic/database-keep.jpg b/docs/zh/14-reference/03-taos-sql/pic/database-keep.jpg new file mode 100644 index 0000000000..248a9041d3 Binary files /dev/null and b/docs/zh/14-reference/03-taos-sql/pic/database-keep.jpg differ diff --git a/docs/zh/26-tdinternal/01-arch.md b/docs/zh/26-tdinternal/01-arch.md index 8aa69e45d5..7091ca9661 100644 --- a/docs/zh/26-tdinternal/01-arch.md +++ b/docs/zh/26-tdinternal/01-arch.md @@ -293,6 +293,14 @@ TDengine 采纳了一种独特的时间驱动缓存管理策略，亦称为写 此外，考虑到物联网数据的特点，用户通常最关注的是数据的实时性，即最新产生的数据。TDengine 很好地利用了这一特点，优先将最新到达的（即当前状态）数据存储在缓存中。具体而言，TDengine 会将最新到达的数据直接存入缓存，以便快速响应用户对最新一条或多条数据的查询和分析需求，从而在整体上提高数据库查询的响应速度。从这个角度来看，通过合理设置数据库参数，TDengine 完全可以作为数据缓存来使用，这样就无须再部署 Redis 或其他额外的缓存系统。这种做法不仅有效简化了系统架构，还有助于降低运维成本。需要注意的是，一旦 TDengine 重启，缓存中的数据将被清除，所有先前缓存的数据都会被批量写入硬盘，而不会像专业的 Key-Value 缓存系统那样自动将之前缓存的数据重新加载回缓存。 +### last/last_row 缓存 + +在时序数据的场景中，查询表的最后一条记录（last_row）或最后一条非 NULL 记录（last）是一个常见的需求。为了提高 TDengine 对这种查询的响应速度，TSDB 为每张表的 last 和 last_row 数据提供了 LRU 缓存。LRU 缓存采用延迟加载策略，当首次查询某张表的 last 或 last_row 时，缓存模块会去内存池和磁盘文件加载数据，处理后放入LRU 缓存，并返回给查询模块继续处理；当有新的数据插入或删除时，如果缓存需要更新，会进行相应的更新操作；如果缓存中没有当前被写入表的数据，则直接跳过，无需其它操作。 + +此外在缓存配置更新的时候，也会更新缓存数据。比如，缓存功能默认是关闭的，用户使用命令开启缓存功能之后，就会在首次查询时加载数据；当关闭缓存开关时，会释放之前的缓存区。当查询某一个子表的 last 或 last_row 数据时，如果缓存中没有，则从内存池和磁盘文件加载对应的 last 或 last_row 数据到缓存中；当查询某一个超级表的 last 或 last_row 数据时，这个超级表对应的所有子表都需要加载到缓存中。 + +通过数据库参数 cachemodel 可以配置某一个数据库的缓存参数，默认值为 "none"，表示不开启缓存，另外三个值为 "last_row"，"last_value"，"both"；分别是开启 last_row 缓存，开启 last 缓存，和两个同时开启。缓存当前所使用的内存数量，可在通过 show vgroups; 命令，在 cacheload 列中进行查看，单位为字节。 + ### 持久化存储 TDengine 采用了一种数据驱动的策略来实现缓存数据的持久化存储。当 vnode 中的缓存数据积累到一定量时，为了避免阻塞后续数据的写入，TDengine 会启动落盘线程，将这些缓存数据写入持久化存储设备。在此过程中，TDengine 会创建新的数据库日志文件用于数据落盘，并在落盘成功后删除旧的日志文件，以防止日志文件无限制增长。 diff --git a/include/libs/stream/streamMsg.h b/include/common/streamMsg.h similarity index 85% rename from include/libs/stream/streamMsg.h rename to include/common/streamMsg.h index 0ceaa93a72..3db92ba58d 100644 --- a/include/libs/stream/streamMsg.h +++ b/include/common/streamMsg.h @@ -17,12 +17,23 @@ #define TDENGINE_STREAMMSG_H #include "tmsg.h" -#include "trpc.h" +//#include "trpc.h" #ifdef __cplusplus extern "C" { #endif +typedef struct SStreamRetrieveReq SStreamRetrieveReq; +typedef struct SStreamDispatchReq SStreamDispatchReq; +typedef struct STokenBucket STokenBucket; +typedef struct SMetaHbInfo SMetaHbInfo; + +typedef struct SNodeUpdateInfo { + int32_t nodeId; + SEpSet prevEp; + SEpSet newEp; +} SNodeUpdateInfo; + typedef struct SStreamUpstreamEpInfo { int32_t nodeId; int32_t childId; @@ -170,8 +181,8 @@ typedef struct SStreamHbMsg { SArray* pUpdateNodes; // SArray, needs update the epsets in stream tasks for those nodes. } SStreamHbMsg; -int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pRsp); -int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pRsp); +int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pReq); +int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq); void tCleanupStreamHbMsg(SStreamHbMsg* pMsg); typedef struct { @@ -179,6 +190,9 @@ typedef struct { int32_t msgId; } SMStreamHbRspMsg; +int32_t tEncodeStreamHbRsp(SEncoder* pEncoder, const SMStreamHbRspMsg* pRsp); +int32_t tDecodeStreamHbRsp(SDecoder* pDecoder, SMStreamHbRspMsg* pRsp); + typedef struct SRetrieveChkptTriggerReq { SMsgHead head; int64_t streamId; @@ -189,6 +203,9 @@ typedef struct SRetrieveChkptTriggerReq { int64_t downstreamTaskId; } SRetrieveChkptTriggerReq; +int32_t tEncodeRetrieveChkptTriggerReq(SEncoder* pEncoder, const SRetrieveChkptTriggerReq* pReq); +int32_t tDecodeRetrieveChkptTriggerReq(SDecoder* pDecoder, SRetrieveChkptTriggerReq* pReq); + typedef struct SCheckpointTriggerRsp { int64_t streamId; int64_t checkpointId; @@ -198,6 +215,9 @@ typedef struct SCheckpointTriggerRsp { int32_t rspCode; } SCheckpointTriggerRsp; +int32_t tEncodeCheckpointTriggerRsp(SEncoder* pEncoder, const SCheckpointTriggerRsp* pRsp); +int32_t tDecodeCheckpointTriggerRsp(SDecoder* pDecoder, SCheckpointTriggerRsp* pRsp); + typedef struct SCheckpointReport { int64_t streamId; int32_t taskId; @@ -222,7 +242,7 @@ typedef struct SRestoreCheckpointInfo { int32_t nodeId; } SRestoreCheckpointInfo; -int32_t tEncodeRestoreCheckpointInfo (SEncoder* pEncoder, const SRestoreCheckpointInfo* pReq); +int32_t tEncodeRestoreCheckpointInfo(SEncoder* pEncoder, const SRestoreCheckpointInfo* pReq); int32_t tDecodeRestoreCheckpointInfo(SDecoder* pDecoder, SRestoreCheckpointInfo* pReq); typedef struct { @@ -232,10 +252,8 @@ typedef struct { int32_t reqType; } SStreamTaskRunReq; -typedef struct SCheckpointConsensusEntry { - SRestoreCheckpointInfo req; - int64_t ts; -} SCheckpointConsensusEntry; +int32_t tEncodeStreamTaskRunReq(SEncoder* pEncoder, const SStreamTaskRunReq* pReq); +int32_t tDecodeStreamTaskRunReq(SDecoder* pDecoder, SStreamTaskRunReq* pReq); #ifdef __cplusplus } diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 19f3e222d1..f899fc5589 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -57,9 +57,9 @@ const static uint8_t BIT2_MAP[4] = {0b11111100, 0b11110011, 0b11001111, 0b001111 #define ONE ((uint8_t)1) #define THREE ((uint8_t)3) #define DIV_8(i) ((i) >> 3) -#define MOD_8(i) ((i)&7) +#define MOD_8(i) ((i) & 7) #define DIV_4(i) ((i) >> 2) -#define MOD_4(i) ((i)&3) +#define MOD_4(i) ((i) & 3) #define MOD_4_TIME_2(i) (MOD_4(i) << 1) #define BIT1_SIZE(n) (DIV_8((n)-1) + 1) #define BIT2_SIZE(n) (DIV_4((n)-1) + 1) @@ -173,6 +173,8 @@ typedef struct { } SColDataCompressInfo; typedef void *(*xMallocFn)(void *, int32_t); +typedef int32_t (*checkWKBGeometryFn)(const unsigned char *geoWKB, size_t nGeom); +typedef int32_t (*initGeosFn)(); void tColDataDestroy(void *ph); void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t cflag); @@ -191,7 +193,8 @@ int32_t tColDataCompress(SColData *colData, SColDataCompressInfo *info, SBuffer int32_t tColDataDecompress(void *input, SColDataCompressInfo *info, SColData *colData, SBuffer *assist); // for stmt bind -int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen); +int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen, initGeosFn igeos, + checkWKBGeometryFn cgeos); int32_t tColDataSortMerge(SArray **arr); // for raw block @@ -378,7 +381,8 @@ int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted, SArray *rowArray); // stmt2 binding -int32_t tColDataAddValueByBind2(SColData *pColData, TAOS_STMT2_BIND *pBind, int32_t buffMaxLen); +int32_t tColDataAddValueByBind2(SColData *pColData, TAOS_STMT2_BIND *pBind, int32_t buffMaxLen, initGeosFn igeos, + checkWKBGeometryFn cgeos); typedef struct { int32_t columnId; diff --git a/include/common/tglobal.h b/include/common/tglobal.h index e6c471eaf1..5125c1caef 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -67,6 +67,7 @@ extern int64_t tsTickPerHour[3]; extern int32_t tsCountAlwaysReturnValue; extern float tsSelectivityRatio; extern int32_t tsTagFilterResCacheSize; +extern int32_t tsBypassFlag; // queue & threads extern int32_t tsNumOfRpcThreads; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index acf1d5e182..2294cf6f73 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3798,7 +3798,14 @@ typedef struct { SMsgHead head; int64_t streamId; int32_t taskId; -} SVPauseStreamTaskReq, SVResetStreamTaskReq; +} SVPauseStreamTaskReq; + +typedef struct { + SMsgHead head; + int64_t streamId; + int32_t taskId; + int64_t chkptId; +} SVResetStreamTaskReq; typedef struct { char name[TSDB_STREAM_FNAME_LEN]; diff --git a/include/libs/geometry/geosWrapper.h b/include/libs/geometry/geosWrapper.h index a5bc0cec17..d27d300b82 100644 --- a/include/libs/geometry/geosWrapper.h +++ b/include/libs/geometry/geosWrapper.h @@ -35,6 +35,7 @@ int32_t doGeomFromText(const char *inputWKT, unsigned char **outputGeom, size_t int32_t initCtxAsText(); int32_t doAsText(const unsigned char *inputGeom, size_t size, char **outputWKT); +int32_t checkWKB(const unsigned char *wkb, size_t size); int32_t initCtxRelationFunc(); int32_t doIntersects(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, @@ -47,11 +48,12 @@ int32_t doCovers(const GEOSGeometry *geom1, const GEOSPreparedGeometry *prepared bool swapped, char *res); int32_t doContains(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, bool swapped, char *res); -int32_t doContainsProperly(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, - bool swapped, char *res); +int32_t doContainsProperly(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, + const GEOSGeometry *geom2, bool swapped, char *res); -int32_t readGeometry(const unsigned char *input, GEOSGeometry **outputGeom, const GEOSPreparedGeometry **outputPreparedGeom); -void destroyGeometry(GEOSGeometry **geom, const GEOSPreparedGeometry **preparedGeom); +int32_t readGeometry(const unsigned char *input, GEOSGeometry **outputGeom, + const GEOSPreparedGeometry **outputPreparedGeom); +void destroyGeometry(GEOSGeometry **geom, const GEOSPreparedGeometry **preparedGeom); #ifdef __cplusplus } diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 2cf791c8da..6b8e9f12a6 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -319,11 +319,6 @@ typedef struct SSTaskBasicInfo { SInterval interval; } SSTaskBasicInfo; -typedef struct SStreamRetrieveReq SStreamRetrieveReq; -typedef struct SStreamDispatchReq SStreamDispatchReq; -typedef struct STokenBucket STokenBucket; -typedef struct SMetaHbInfo SMetaHbInfo; - typedef struct SDispatchMsgInfo { SStreamDispatchReq* pData; // current dispatch data @@ -626,11 +621,11 @@ typedef struct STaskStatusEntry { STaskCkptInfo checkpointInfo; } STaskStatusEntry; -typedef struct SNodeUpdateInfo { - int32_t nodeId; - SEpSet prevEp; - SEpSet newEp; -} SNodeUpdateInfo; +//typedef struct SNodeUpdateInfo { +// int32_t nodeId; +// SEpSet prevEp; +// SEpSet newEp; +//} SNodeUpdateInfo; typedef struct SStreamTaskState { ETaskStatus state; @@ -643,6 +638,11 @@ typedef struct SCheckpointConsensusInfo { int64_t streamId; } SCheckpointConsensusInfo; +typedef struct SCheckpointConsensusEntry { + SRestoreCheckpointInfo req; + int64_t ts; +} SCheckpointConsensusEntry; + void streamSetupScheduleTrigger(SStreamTask* pTask); // dispatch related @@ -718,6 +718,7 @@ int32_t streamTaskInitTriggerDispatchInfo(SStreamTask* pTask); void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId); int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId, int32_t downstreamNodeId, SRpcHandleInfo* pInfo, int32_t code); +void streamTaskSetFailedCheckpointId(SStreamTask* pTask, int64_t failedId); int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue); int32_t streamQueueGetNumOfUnAccessedItems(const SStreamQueue* pQueue); diff --git a/include/util/tdef.h b/include/util/tdef.h index 4e1fb21838..0e2f7ed8a6 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -620,6 +620,16 @@ enum { enum { RAND_ERR_MEMORY = 1, RAND_ERR_FILE = 2, RAND_ERR_NETWORK = 4 }; +/** + * RB: return before + * RA: return after + * NR: not return, skip and go on following steps + */ +#define TSDB_BYPASS_RB_RPC_SEND_SUBMIT 0x01u +#define TSDB_BYPASS_RA_RPC_RECV_SUBMIT 0x02u +#define TSDB_BYPASS_RB_TSDB_WRITE_MEM 0x04u +#define TSDB_BYPASS_RB_TSDB_COMMIT 0x08u + #define DEFAULT_HANDLE 0 #define MNODE_HANDLE 1 #define QNODE_HANDLE -1 diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index f10eb6a611..39380a0644 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -1,4 +1,7 @@ aux_source_directory(src COMMON_SRC) +aux_source_directory(src/msg COMMON_MSG_SRC) + +LIST(APPEND COMMON_SRC ${COMMON_MSG_SRC}) if(TD_ENTERPRISE) LIST(APPEND COMMON_SRC ${TD_ENTERPRISE_DIR}/src/plugins/common/src/tglobal.c) diff --git a/source/libs/stream/src/streamMsg.c b/source/common/src/msg/streamMsg.c similarity index 75% rename from source/libs/stream/src/streamMsg.c rename to source/common/src/msg/streamMsg.c index 193daa0cc4..c92ab52ac1 100644 --- a/source/libs/stream/src/streamMsg.c +++ b/source/common/src/msg/streamMsg.c @@ -15,8 +15,48 @@ #include "streamMsg.h" #include "os.h" -#include "tstream.h" -#include "streamInt.h" +#include "tcommon.h" + +typedef struct STaskId { + int64_t streamId; + int64_t taskId; +} STaskId; + +typedef struct STaskCkptInfo { + int64_t latestId; // saved checkpoint id + int64_t latestVer; // saved checkpoint ver + int64_t latestTime; // latest checkpoint time + int64_t latestSize; // latest checkpoint size + int8_t remoteBackup; // latest checkpoint backup done + int64_t activeId; // current active checkpoint id + int32_t activeTransId; // checkpoint trans id + int8_t failed; // denote if the checkpoint is failed or not + int8_t consensusChkptId; // required the consensus-checkpointId + int64_t consensusTs; // +} STaskCkptInfo; + +typedef struct STaskStatusEntry { + STaskId id; + int32_t status; + int32_t statusLastDuration; // to record the last duration of current status + int64_t stage; + int32_t nodeId; + SVersionRange verRange; // start/end version in WAL, only valid for source task + int64_t processedVer; // only valid for source task + double inputQUsed; // in MiB + double inputRate; + double procsThroughput; // duration between one element put into input queue and being processed. + double procsTotal; // duration between one element put into input queue and being processed. + double outputThroughput; // the size of dispatched result blocks in bytes + double outputTotal; // the size of dispatched result blocks in bytes + double sinkQuota; // existed quota size for sink task + double sinkDataSize; // sink to dst data size + int64_t startTime; + int64_t startCheckpointId; + int64_t startCheckpointVer; + int64_t hTaskId; + STaskCkptInfo checkpointInfo; +} STaskStatusEntry; int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamUpstreamEpInfo* pInfo) { TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pInfo->taskId)); @@ -289,7 +329,7 @@ int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* p TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->totalLen)); if (taosArrayGetSize(pReq->data) != pReq->blockNum || taosArrayGetSize(pReq->dataLen) != pReq->blockNum) { - stError("invalid dispatch req msg"); + uError("invalid dispatch req msg"); TAOS_CHECK_EXIT(TSDB_CODE_INVALID_MSG); } @@ -605,173 +645,92 @@ void tCleanupStreamHbMsg(SStreamHbMsg* pMsg) { pMsg->numOfTasks = -1; } -int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { +int32_t tEncodeStreamHbRsp(SEncoder* pEncoder, const SMStreamHbRspMsg* pRsp) { int32_t code = 0; int32_t lino; TAOS_CHECK_EXIT(tStartEncode(pEncoder)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->ver)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->id.streamId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->id.taskId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->info.trigger)); - TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->info.taskLevel)); - TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->outputInfo.type)); - TAOS_CHECK_EXIT(tEncodeI16(pEncoder, pTask->msgInfo.msgType)); - - TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->status.taskStatus)); - TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->status.schedStatus)); - - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->info.selfChildId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->info.nodeId)); - TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pTask->info.epSet)); - TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pTask->info.mnodeEpset)); - - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->chkInfo.checkpointId)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->chkInfo.checkpointVer)); - TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->info.fillHistory)); - - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->hTaskInfo.id.streamId)); - int32_t taskId = pTask->hTaskInfo.id.taskId; - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, taskId)); - - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->streamTaskId.streamId)); - taskId = pTask->streamTaskId.taskId; - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, taskId)); - - TAOS_CHECK_EXIT(tEncodeU64(pEncoder, pTask->dataRange.range.minVer)); - TAOS_CHECK_EXIT(tEncodeU64(pEncoder, pTask->dataRange.range.maxVer)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->dataRange.window.skey)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->dataRange.window.ekey)); - - int32_t epSz = taosArrayGetSize(pTask->upstreamInfo.pList); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, epSz)); - for (int32_t i = 0; i < epSz; i++) { - SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); - TAOS_CHECK_EXIT(tEncodeStreamEpInfo(pEncoder, pInfo)); - } - - if (pTask->info.taskLevel != TASK_LEVEL__SINK) { - TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, pTask->exec.qmsg)); - } - - if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->outputInfo.tbSink.stbUid)); - TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, pTask->outputInfo.tbSink.stbFullName)); - TAOS_CHECK_EXIT(tEncodeSSchemaWrapper(pEncoder, pTask->outputInfo.tbSink.pSchemaWrapper)); - } else if (pTask->outputInfo.type == TASK_OUTPUT__SMA) { - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->outputInfo.smaSink.smaId)); - } else if (pTask->outputInfo.type == TASK_OUTPUT__FETCH) { - TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->outputInfo.fetchSink.reserved)); - } else if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->outputInfo.fixedDispatcher.taskId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->outputInfo.fixedDispatcher.nodeId)); - TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pTask->outputInfo.fixedDispatcher.epSet)); - } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { - TAOS_CHECK_EXIT(tSerializeSUseDbRspImp(pEncoder, &pTask->outputInfo.shuffleDispatcher.dbInfo)); - TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, pTask->outputInfo.shuffleDispatcher.stbFullName)); - } - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->info.delaySchedParam)); - TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->subtableWithoutMd5)); - TAOS_CHECK_EXIT(tEncodeCStrWithLen(pEncoder, pTask->reserve, sizeof(pTask->reserve) - 1)); - + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->msgId)); tEndEncode(pEncoder); + _exit: return code; } -int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { - int32_t taskId = 0; +int32_t tDecodeStreamHbRsp(SDecoder* pDecoder, SMStreamHbRspMsg* pRsp) { int32_t code = 0; int32_t lino; TAOS_CHECK_EXIT(tStartDecode(pDecoder)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->ver)); - if (pTask->ver <= SSTREAM_TASK_INCOMPATIBLE_VER || pTask->ver > SSTREAM_TASK_VER) { - TAOS_CHECK_EXIT(TSDB_CODE_INVALID_MSG); - } + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->msgId)); + tEndDecode(pDecoder); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->id.streamId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->id.taskId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->info.trigger)); - TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->info.taskLevel)); - TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->outputInfo.type)); - TAOS_CHECK_EXIT(tDecodeI16(pDecoder, &pTask->msgInfo.msgType)); +_exit: + return code; +} - TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->status.taskStatus)); - TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->status.schedStatus)); +int32_t tEncodeRetrieveChkptTriggerReq(SEncoder* pEncoder, const SRetrieveChkptTriggerReq* pReq) { + int32_t code = 0; + int32_t lino; - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->info.selfChildId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->info.nodeId)); - TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &pTask->info.epSet)); - TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &pTask->info.mnodeEpset)); + TAOS_CHECK_EXIT(tStartEncode(pEncoder)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->streamId)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->checkpointId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->upstreamNodeId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->upstreamTaskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->downstreamNodeId)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->downstreamTaskId)); + tEndEncode(pEncoder); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->chkInfo.checkpointId)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->chkInfo.checkpointVer)); - TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->info.fillHistory)); +_exit: + return code; +} - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->hTaskInfo.id.streamId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &taskId)); - pTask->hTaskInfo.id.taskId = taskId; +int32_t tDecodeRetrieveChkptTriggerReq(SDecoder* pDecoder, SRetrieveChkptTriggerReq* pReq) { + int32_t code = 0; + int32_t lino; - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->streamTaskId.streamId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &taskId)); - pTask->streamTaskId.taskId = taskId; + TAOS_CHECK_EXIT(tStartDecode(pDecoder)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->streamId)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->checkpointId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->upstreamNodeId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->upstreamTaskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->downstreamNodeId)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->downstreamTaskId)); + tEndDecode(pDecoder); - TAOS_CHECK_EXIT(tDecodeU64(pDecoder, (uint64_t*)&pTask->dataRange.range.minVer)); - TAOS_CHECK_EXIT(tDecodeU64(pDecoder, (uint64_t*)&pTask->dataRange.range.maxVer)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->dataRange.window.skey)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->dataRange.window.ekey)); +_exit: + return code; +} - int32_t epSz = -1; - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &epSz) < 0); +int32_t tEncodeCheckpointTriggerRsp(SEncoder* pEncoder, const SCheckpointTriggerRsp* pRsp) { + int32_t code = 0; + int32_t lino; - if ((pTask->upstreamInfo.pList = taosArrayInit(epSz, POINTER_BYTES)) == NULL) { - TAOS_CHECK_EXIT(terrno); - } - for (int32_t i = 0; i < epSz; i++) { - SStreamUpstreamEpInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamUpstreamEpInfo)); - if (pInfo == NULL) { - TAOS_CHECK_EXIT(terrno); - } - if ((code = tDecodeStreamEpInfo(pDecoder, pInfo)) < 0) { - taosMemoryFreeClear(pInfo); - goto _exit; - } - if (taosArrayPush(pTask->upstreamInfo.pList, &pInfo) == NULL) { - TAOS_CHECK_EXIT(terrno); - } - } + TAOS_CHECK_EXIT(tStartEncode(pEncoder)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pRsp->streamId)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pRsp->checkpointId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->upstreamTaskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->taskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->transId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->rspCode)); + tEndEncode(pEncoder); - if (pTask->info.taskLevel != TASK_LEVEL__SINK) { - TAOS_CHECK_EXIT(tDecodeCStrAlloc(pDecoder, &pTask->exec.qmsg)); - } +_exit: + return code; +} - if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->outputInfo.tbSink.stbUid)); - TAOS_CHECK_EXIT(tDecodeCStrTo(pDecoder, pTask->outputInfo.tbSink.stbFullName)); - pTask->outputInfo.tbSink.pSchemaWrapper = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); - if (pTask->outputInfo.tbSink.pSchemaWrapper == NULL) { - TAOS_CHECK_EXIT(terrno); - } - TAOS_CHECK_EXIT(tDecodeSSchemaWrapper(pDecoder, pTask->outputInfo.tbSink.pSchemaWrapper)); - } else if (pTask->outputInfo.type == TASK_OUTPUT__SMA) { - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->outputInfo.smaSink.smaId)); - } else if (pTask->outputInfo.type == TASK_OUTPUT__FETCH) { - TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->outputInfo.fetchSink.reserved)); - } else if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->outputInfo.fixedDispatcher.taskId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->outputInfo.fixedDispatcher.nodeId)); - TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &pTask->outputInfo.fixedDispatcher.epSet)); - } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { - TAOS_CHECK_EXIT(tDeserializeSUseDbRspImp(pDecoder, &pTask->outputInfo.shuffleDispatcher.dbInfo)); - TAOS_CHECK_EXIT(tDecodeCStrTo(pDecoder, pTask->outputInfo.shuffleDispatcher.stbFullName)); - } - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->info.delaySchedParam)); - if (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER) { - TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->subtableWithoutMd5)); - } - TAOS_CHECK_EXIT(tDecodeCStrTo(pDecoder, pTask->reserve)); +int32_t tDecodeCheckpointTriggerRsp(SDecoder* pDecoder, SCheckpointTriggerRsp* pRsp) { + int32_t code = 0; + int32_t lino; + TAOS_CHECK_EXIT(tStartDecode(pDecoder)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pRsp->streamId)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pRsp->checkpointId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->upstreamTaskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->taskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->transId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->rspCode)); tEndDecode(pDecoder); _exit: @@ -830,11 +789,7 @@ int32_t tEncodeRestoreCheckpointInfo(SEncoder* pEncoder, const SRestoreCheckpoin tEndEncode(pEncoder); _exit: - if (code) { - return code; - } else { - return pEncoder->pos; - } + return code; } int32_t tDecodeRestoreCheckpointInfo(SDecoder* pDecoder, SRestoreCheckpointInfo* pReq) { @@ -853,3 +808,31 @@ int32_t tDecodeRestoreCheckpointInfo(SDecoder* pDecoder, SRestoreCheckpointInfo* _exit: return code; } + +int32_t tEncodeStreamTaskRunReq (SEncoder* pEncoder, const SStreamTaskRunReq* pReq) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartEncode(pEncoder)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->streamId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->taskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->reqType)); + tEndEncode(pEncoder); + +_exit: + return code; +} + +int32_t tDecodeStreamTaskRunReq(SDecoder* pDecoder, SStreamTaskRunReq* pReq) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartDecode(pDecoder)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->streamId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->taskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->reqType)); + tEndDecode(pDecoder); + +_exit: + return code; +} \ No newline at end of file diff --git a/source/common/src/tmsg.c b/source/common/src/msg/tmsg.c similarity index 100% rename from source/common/src/tmsg.c rename to source/common/src/msg/tmsg.c diff --git a/source/common/src/tcol.c b/source/common/src/tcol.c index a23385aba0..55a4b21208 100644 --- a/source/common/src/tcol.c +++ b/source/common/src/tcol.c @@ -81,26 +81,42 @@ const char* getDefaultEncodeStr(uint8_t type) { return columnEncodeStr(getDefaul uint16_t getDefaultCompress(uint8_t type) { switch (type) { case TSDB_DATA_TYPE_NULL: + return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_BOOL: + return TSDB_COLVAL_COMPRESS_ZSTD; case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: + return TSDB_COLVAL_COMPRESS_ZLIB; case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_BIGINT: + return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: + return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_VARCHAR: // TSDB_DATA_TYPE_BINARY + return TSDB_COLVAL_COMPRESS_ZSTD; case TSDB_DATA_TYPE_TIMESTAMP: + return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_NCHAR: + return TSDB_COLVAL_COMPRESS_ZSTD; case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_USMALLINT: + return TSDB_COLVAL_COMPRESS_ZLIB; case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UBIGINT: + return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_JSON: + return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_VARBINARY: + return TSDB_COLVAL_COMPRESS_ZSTD; case TSDB_DATA_TYPE_DECIMAL: + return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_BLOB: + return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_MEDIUMBLOB: + return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_GEOMETRY: + return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_MAX: return TSDB_COLVAL_COMPRESS_LZ4; default: diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index e580ad33bd..a38842735c 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -3036,7 +3036,8 @@ _exit: return code; } -int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen) { +int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen, initGeosFn igeos, + checkWKBGeometryFn cgeos) { int32_t code = 0; if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) { @@ -3046,6 +3047,12 @@ int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32 } if (IS_VAR_DATA_TYPE(pColData->type)) { // var-length data type + if (pColData->type == TSDB_DATA_TYPE_GEOMETRY) { + code = igeos(); + if (code) { + return code; + } + } for (int32_t i = 0; i < pBind->num; ++i) { if (pBind->is_null && pBind->is_null[i]) { if (pColData->cflag & COL_IS_KEY) { @@ -3055,9 +3062,12 @@ int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32 code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); if (code) goto _exit; } else if (pBind->length[i] > buffMaxLen) { - uError("var data length too big, len:%d, max:%d", pBind->length[i], buffMaxLen); - return TSDB_CODE_INVALID_PARA; + return TSDB_CODE_PAR_VALUE_TOO_LONG; } else { + if (pColData->type == TSDB_DATA_TYPE_GEOMETRY) { + code = cgeos((char *)pBind->buffer + pBind->buffer_length * i, (size_t)pBind->length[i]); + if (code) goto _exit; + } code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE]( pColData, (uint8_t *)pBind->buffer + pBind->buffer_length * i, pBind->length[i]); } @@ -3108,7 +3118,8 @@ _exit: return code; } -int32_t tColDataAddValueByBind2(SColData *pColData, TAOS_STMT2_BIND *pBind, int32_t buffMaxLen) { +int32_t tColDataAddValueByBind2(SColData *pColData, TAOS_STMT2_BIND *pBind, int32_t buffMaxLen, initGeosFn igeos, + checkWKBGeometryFn cgeos) { int32_t code = 0; if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) { @@ -3118,6 +3129,13 @@ int32_t tColDataAddValueByBind2(SColData *pColData, TAOS_STMT2_BIND *pBind, int3 } if (IS_VAR_DATA_TYPE(pColData->type)) { // var-length data type + if (pColData->type == TSDB_DATA_TYPE_GEOMETRY) { + code = igeos(); + if (code) { + return code; + } + } + uint8_t *buf = pBind->buffer; for (int32_t i = 0; i < pBind->num; ++i) { if (pBind->is_null && pBind->is_null[i]) { @@ -3133,9 +3151,12 @@ int32_t tColDataAddValueByBind2(SColData *pColData, TAOS_STMT2_BIND *pBind, int3 if (code) goto _exit; } } else if (pBind->length[i] > buffMaxLen) { - uError("var data length too big, len:%d, max:%d", pBind->length[i], buffMaxLen); - return TSDB_CODE_INVALID_PARA; + return TSDB_CODE_PAR_VALUE_TOO_LONG; } else { + if (pColData->type == TSDB_DATA_TYPE_GEOMETRY) { + code = cgeos(buf, pBind->length[i]); + if (code) goto _exit; + } code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, buf, pBind->length[i]); buf += pBind->length[i]; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 0133428c53..93c86a2dcd 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -217,6 +217,8 @@ float tsSelectivityRatio = 1.0; int32_t tsTagFilterResCacheSize = 1024 * 10; char tsTagFilterCache = 0; +int32_t tsBypassFlag = 0; + // the maximum allowed query buffer size during query processing for each data node. // -1 no limit (default) // 0 no query allowed, queries are disabled @@ -612,6 +614,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { cfgAddInt64(pCfg, "randErrorDivisor", tsRandErrDivisor, 1, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorScope", tsRandErrScope, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "safetyCheckLevel", tsSafetyCheckLevel, 0, 5, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "bypassFlag", tsBypassFlag, 0, INT32_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 1, TSDB_MAX_RPC_THREADS); @@ -1303,6 +1306,10 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "safetyCheckLevel"); tsSafetyCheckLevel = pItem->i32; + + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "bypassFlag"); + tsBypassFlag = pItem->i32; + TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -2046,7 +2053,8 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"supportVnodes", &tsNumOfSupportVnodes}, {"experimental", &tsExperimental}, {"maxTsmaNum", &tsMaxTsmaNum}, - {"safetyCheckLevel", &tsSafetyCheckLevel}}; + {"safetyCheckLevel", &tsSafetyCheckLevel}, + {"bypassFlag", &tsBypassFlag}}; if ((code = taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true)) != TSDB_CODE_SUCCESS) { code = taosCfgSetOption(options, tListLen(options), pItem, false); @@ -2302,7 +2310,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { {"multiResultFunctionStarReturnTags", &tsMultiResultFunctionStarReturnTags}, {"maxTsmaCalcDelay", &tsMaxTsmaCalcDelay}, {"tsmaDataDeleteMark", &tsmaDataDeleteMark}, - {"safetyCheckLevel", &tsSafetyCheckLevel}}; + {"safetyCheckLevel", &tsSafetyCheckLevel}, + {"bypassFlag", &tsBypassFlag}}; if ((code = taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true)) != TSDB_CODE_SUCCESS) { code = taosCfgSetOption(options, tListLen(options), pItem, false); diff --git a/source/common/test/CMakeLists.txt b/source/common/test/CMakeLists.txt index 2fe3ef652d..bb12612273 100644 --- a/source/common/test/CMakeLists.txt +++ b/source/common/test/CMakeLists.txt @@ -46,7 +46,7 @@ if (${TD_LINUX}) target_sources(tmsgTest PRIVATE "tmsgTest.cpp" - "../src/tmsg.c" + "../src/msg/tmsg.c" ) target_include_directories(tmsgTest PUBLIC "${TD_SOURCE_DIR}/include/common/") target_link_libraries(tmsgTest PUBLIC os util gtest gtest_main) diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 61543e619e..5f396a520a 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -214,8 +214,6 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { } else if ((pRpc->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pRpc->code == TSDB_CODE_RPC_BROKEN_LINK) && (!IsReq(pRpc)) && (pRpc->pCont == NULL)) { dGError("msg:%p, type:%s pCont is NULL, err: %s", pRpc, TMSG_INFO(pRpc->msgType), tstrerror(pRpc->code)); - code = pRpc->code; - goto _OVER; } if (pHandle->defaultNtype == NODE_END) { diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index c9155f536c..b519d8509a 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -56,6 +56,7 @@ typedef struct SStreamTransMgmt { typedef struct SStreamTaskResetMsg { int64_t streamId; int32_t transId; + int64_t checkpointId; } SStreamTaskResetMsg; typedef struct SChkptReportInfo { @@ -142,9 +143,9 @@ int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pSt int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList); -int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); +int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream, int64_t chkptId); int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); -int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream); +int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream, int64_t chkptId); int32_t mndStreamSetChkptIdAction(SMnode *pMnode, STrans *pTrans, SStreamTask* pTask, int64_t checkpointId, int64_t ts); int32_t mndStreamSetRestartAction(SMnode* pMnode, STrans *pTrans, SStreamObj* pStream); int32_t mndStreamSetCheckpointAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, int64_t checkpointId, diff --git a/source/dnode/mnode/impl/src/mndStreamErrorInjection.c b/source/dnode/mnode/impl/src/mndStreamErrorInjection.c new file mode 100644 index 0000000000..c68416369d --- /dev/null +++ b/source/dnode/mnode/impl/src/mndStreamErrorInjection.c @@ -0,0 +1,72 @@ +#include "mndTrans.h" + +uint32_t seed = 0; + +static SRpcMsg createRpcMsg(STransAction* pAction, int64_t traceId, int64_t signature) { + SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen, .info.ahandle = (void *)signature}; + rpcMsg.pCont = rpcMallocCont(pAction->contLen); + if (rpcMsg.pCont == NULL) { + return rpcMsg; + } + + rpcMsg.info.traceId.rootId = traceId; + rpcMsg.info.notFreeAhandle = 1; + + memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen); + return rpcMsg; +} + +void streamTransRandomErrorGen(STransAction *pAction, STrans *pTrans, int64_t signature) { + if ((pAction->msgType == TDMT_STREAM_TASK_UPDATE_CHKPT && pAction->id > 2) || + (pAction->msgType == TDMT_STREAM_CONSEN_CHKPT) || + (pAction->msgType == TDMT_VND_STREAM_CHECK_POINT_SOURCE && pAction->id > 2)) { + if (seed == 0) { + seed = taosGetTimestampSec(); + } + + uint32_t v = taosRandR(&seed); + int32_t choseItem = v % 5; + + if (choseItem == 0) { + // 1. one of update-checkpoint not send, restart and send it again + taosMsleep(5000); + if (pAction->msgType == TDMT_STREAM_TASK_UPDATE_CHKPT) { + mError( + "***sleep 5s and core dump, following tasks will not recv update-checkpoint info, so the checkpoint will " + "rollback***"); + exit(-1); + } else if (pAction->msgType == TDMT_STREAM_CONSEN_CHKPT) { // pAction->msgType == TDMT_STREAM_CONSEN_CHKPT + mError( + "***sleep 5s and core dump, following tasks will not recv consen-checkpoint info, so the tasks will " + "not started***"); + } else { // pAction->msgType == TDMT_VND_STREAM_CHECK_POINT_SOURCE + mError( + "***sleep 5s and core dump, following tasks will not recv checkpoint-source info, so the tasks will " + "started after restart***"); + exit(-1); + } + } else if (choseItem == 1) { + // 2. repeat send update chkpt msg + mError("***repeat send update-checkpoint/consensus/checkpoint trans msg 3times to vnode***"); + + mError("***repeat 1***"); + SRpcMsg rpcMsg1 = createRpcMsg(pAction, pTrans->mTraceId, signature); + int32_t code = tmsgSendReq(&pAction->epSet, &rpcMsg1); + + mError("***repeat 2***"); + SRpcMsg rpcMsg2 = createRpcMsg(pAction, pTrans->mTraceId, signature); + code = tmsgSendReq(&pAction->epSet, &rpcMsg2); + + mError("***repeat 3***"); + SRpcMsg rpcMsg3 = createRpcMsg(pAction, pTrans->mTraceId, signature); + code = tmsgSendReq(&pAction->epSet, &rpcMsg3); + } else if (choseItem == 2) { + // 3. sleep 40s and then send msg + mError("***idle for 30s, and then send msg***"); + taosMsleep(30000); + } else { + // do nothing + // mInfo("no error triggered"); + } + } +} diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 941956ae2b..4b3db28aa1 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -24,7 +24,7 @@ typedef struct SFailedCheckpointInfo { static int32_t mndStreamSendUpdateChkptInfoMsg(SMnode *pMnode); static int32_t mndSendDropOrphanTasksMsg(SMnode *pMnode, SArray *pList); -static int32_t mndSendResetFromCheckpointMsg(SMnode *pMnode, int64_t streamId, int32_t transId); +static int32_t mndSendResetFromCheckpointMsg(SMnode *pMnode, int64_t streamId, int32_t transId, int64_t checkpointId); static void updateStageInfo(STaskStatusEntry *pTaskEntry, int64_t stage); static void addIntoFailedChkptList(SArray *pList, const SFailedCheckpointInfo *pInfo); static int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList); @@ -68,7 +68,7 @@ void addIntoFailedChkptList(SArray *pList, const SFailedCheckpointInfo *pInfo) { } } -int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { +int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream, int64_t chkptId) { STrans *pTrans = NULL; int32_t code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_RESET_NAME, " reset from failed checkpoint", &pTrans); @@ -84,7 +84,7 @@ int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { return code; } - code = mndStreamSetResetTaskAction(pMnode, pTrans, pStream); + code = mndStreamSetResetTaskAction(pMnode, pTrans, pStream, chkptId); if (code) { sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); @@ -115,7 +115,7 @@ int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { return code; } -int32_t mndSendResetFromCheckpointMsg(SMnode *pMnode, int64_t streamId, int32_t transId) { +int32_t mndSendResetFromCheckpointMsg(SMnode *pMnode, int64_t streamId, int32_t transId, int64_t checkpointId) { int32_t size = sizeof(SStreamTaskResetMsg); int32_t num = taosArrayGetSize(execInfo.pKilledChkptTrans); @@ -135,8 +135,9 @@ int32_t mndSendResetFromCheckpointMsg(SMnode *pMnode, int64_t streamId, int32_t taosArrayRemove(execInfo.pKilledChkptTrans, 0); // remove this first, append new reset trans in the tail } - SStreamTaskResetMsg p = {.streamId = streamId, .transId = transId}; + SStreamTaskResetMsg p = {.streamId = streamId, .transId = transId, .checkpointId = checkpointId}; + // let's remember that this trans had been killed already void *px = taosArrayPush(execInfo.pKilledChkptTrans, &p); if (px == NULL) { mError("failed to push reset-msg trans:%d into the killed chkpt trans list, size:%d", transId, num - 1); @@ -150,6 +151,7 @@ int32_t mndSendResetFromCheckpointMsg(SMnode *pMnode, int64_t streamId, int32_t pReq->streamId = streamId; pReq->transId = transId; + pReq->checkpointId = checkpointId; SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_TASK_RESET, .pCont = pReq, .contLen = size}; int32_t code = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); @@ -234,7 +236,7 @@ int32_t mndProcessResetStatusReq(SRpcMsg *pReq) { } else { mDebug("stream:%s (0x%" PRIx64 ") reset checkpoint procedure, transId:%d, create reset trans", pStream->name, pStream->uid, pMsg->transId); - code = mndCreateStreamResetStatusTrans(pMnode, pStream); + code = mndCreateStreamResetStatusTrans(pMnode, pStream, pMsg->checkpointId); } } @@ -379,9 +381,10 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { } if ((pEntry->lastHbMsgId == req.msgId) && (pEntry->lastHbMsgTs == req.ts)) { - mError("vgId:%d HbMsgId:%d already handled, bh msg discard", pEntry->nodeId, req.msgId); + mError("vgId:%d HbMsgId:%d already handled, bh msg discard, and send HbRsp", pEntry->nodeId, req.msgId); - terrno = TSDB_CODE_INVALID_MSG; + // return directly and after the vnode to continue to send the next HbMsg. + terrno = TSDB_CODE_SUCCESS; doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId); streamMutexUnlock(&execInfo.lock); @@ -495,10 +498,11 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { continue; } - mInfo("checkpointId:%" PRId64 " transId:%d failed, issue task-reset trans to reset all tasks status", - pInfo->checkpointId, pInfo->transId); + mInfo("stream:0x%" PRIx64 " checkpointId:%" PRId64 + " transId:%d failed issue task-reset trans to reset all tasks status", + pInfo->streamUid, pInfo->checkpointId, pInfo->transId); - code = mndSendResetFromCheckpointMsg(pMnode, pInfo->streamUid, pInfo->transId); + code = mndSendResetFromCheckpointMsg(pMnode, pInfo->streamUid, pInfo->transId, pInfo->checkpointId); if (code) { mError("failed to create reset task trans, code:%s", tstrerror(code)); } @@ -549,12 +553,37 @@ void cleanupAfterProcessHbMsg(SStreamHbMsg *pReq, SArray *pFailedChkptList, SArr } void doSendHbMsgRsp(int32_t code, SRpcHandleInfo *pRpcInfo, int32_t vgId, int32_t msgId) { - SRpcMsg rsp = {.code = code, .info = *pRpcInfo, .contLen = sizeof(SMStreamHbRspMsg)}; - rsp.pCont = rpcMallocCont(rsp.contLen); + int32_t ret = 0; + int32_t tlen = 0; + void *buf = NULL; - SMStreamHbRspMsg *pMsg = rsp.pCont; - pMsg->head.vgId = htonl(vgId); - pMsg->msgId = msgId; + const SMStreamHbRspMsg msg = {.msgId = msgId}; + + tEncodeSize(tEncodeStreamHbRsp, &msg, tlen, ret); + if (ret < 0) { + mError("encode stream hb msg rsp failed, code:%s", tstrerror(code)); + } + + buf = rpcMallocCont(tlen + sizeof(SMsgHead)); + if (buf == NULL) { + mError("encode stream hb msg rsp failed, code:%s", tstrerror(terrno)); + return; + } + + ((SMStreamHbRspMsg*)buf)->head.vgId = htonl(vgId); + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + + SEncoder encoder; + tEncoderInit(&encoder, abuf, tlen); + if ((code = tEncodeStreamHbRsp(&encoder, &msg)) < 0) { + rpcFreeCont(buf); + tEncoderClear(&encoder); + mError("encode stream hb msg rsp failed, code:%s", tstrerror(code)); + return; + } + tEncoderClear(&encoder); + + SRpcMsg rsp = {.code = code, .info = *pRpcInfo, .contLen = tlen + sizeof(SMsgHead), .pCont = buf}; tmsgSendRsp(&rsp); pRpcInfo->handle = NULL; // disable auto rsp diff --git a/source/dnode/mnode/impl/src/mndStreamTransAct.c b/source/dnode/mnode/impl/src/mndStreamTransAct.c index 139ea4f147..5ccb626609 100644 --- a/source/dnode/mnode/impl/src/mndStreamTransAct.c +++ b/source/dnode/mnode/impl/src/mndStreamTransAct.c @@ -295,7 +295,7 @@ static int32_t doSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamTas return code; } -static int32_t doSetResetAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) { +static int32_t doSetResetAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, int64_t chkptId) { SVResetStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResetStreamTaskReq)); if (pReq == NULL) { mError("failed to malloc in reset stream, size:%" PRIzu ", code:%s", sizeof(SVResetStreamTaskReq), @@ -306,6 +306,7 @@ static int32_t doSetResetAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa pReq->head.vgId = htonl(pTask->info.nodeId); pReq->taskId = pTask->id.taskId; pReq->streamId = pTask->id.streamId; + pReq->chkptId = chkptId; SEpSet epset = {0}; bool hasEpset = false; @@ -544,7 +545,7 @@ int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray* p return 0; } -int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { +int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream, int64_t chkptId) { SStreamTaskIter *pIter = NULL; taosWLockLatch(&pStream->lock); @@ -564,7 +565,7 @@ int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj * return code; } - code = doSetResetAction(pMnode, pTrans, pTask); + code = doSetResetAction(pMnode, pTrans, pTask, chkptId); if (code != TSDB_CODE_SUCCESS) { destroyStreamTaskIter(pIter); taosWUnLockLatch(&pStream->lock); @@ -606,7 +607,7 @@ int32_t mndStreamSetChkptIdAction(SMnode *pMnode, STrans *pTrans, SStreamTask* p tEncoderInit(&encoder, abuf, tlen); code = tEncodeRestoreCheckpointInfo(&encoder, &req); tEncoderClear(&encoder); - if (code == -1) { + if (code < 0) { taosMemoryFree(pBuf); return code; } diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index f9b7644af4..615c383f07 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -1521,74 +1521,4 @@ int32_t mndCheckForSnode(SMnode *pMnode, SDbObj *pSrcDb) { mError("snode not existed when trying to create stream in db with multiple replica"); return TSDB_CODE_SNODE_NOT_DEPLOYED; } -} - -uint32_t seed = 0; -static SRpcMsg createRpcMsg(STransAction* pAction, int64_t traceId, int64_t signature) { - SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen, .info.ahandle = (void *)signature}; - rpcMsg.pCont = rpcMallocCont(pAction->contLen); - if (rpcMsg.pCont == NULL) { - return rpcMsg; - } - - rpcMsg.info.traceId.rootId = traceId; - rpcMsg.info.notFreeAhandle = 1; - - memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen); - return rpcMsg; -} - -void streamTransRandomErrorGen(STransAction *pAction, STrans *pTrans, int64_t signature) { - if ((pAction->msgType == TDMT_STREAM_TASK_UPDATE_CHKPT && pAction->id > 2) || - (pAction->msgType == TDMT_STREAM_CONSEN_CHKPT) || - (pAction->msgType == TDMT_VND_STREAM_CHECK_POINT_SOURCE && pAction->id > 2)) { - if (seed == 0) { - seed = taosGetTimestampSec(); - } - - uint32_t v = taosRandR(&seed); - int32_t choseItem = v % 5; - - if (choseItem == 0) { - // 1. one of update-checkpoint not send, restart and send it again - taosMsleep(5000); - if (pAction->msgType == TDMT_STREAM_TASK_UPDATE_CHKPT) { - mError( - "***sleep 5s and core dump, following tasks will not recv update-checkpoint info, so the checkpoint will " - "rollback***"); - exit(-1); - } else if (pAction->msgType == TDMT_STREAM_CONSEN_CHKPT) { // pAction->msgType == TDMT_STREAM_CONSEN_CHKPT - mError( - "***sleep 5s and core dump, following tasks will not recv consen-checkpoint info, so the tasks will " - "not started***"); - } else { // pAction->msgType == TDMT_VND_STREAM_CHECK_POINT_SOURCE - mError( - "***sleep 5s and core dump, following tasks will not recv checkpoint-source info, so the tasks will " - "started after restart***"); - exit(-1); - } - } else if (choseItem == 1) { - // 2. repeat send update chkpt msg - mError("***repeat send update-checkpoint/consensus/checkpoint trans msg 3times to vnode***"); - - mError("***repeat 1***"); - SRpcMsg rpcMsg1 = createRpcMsg(pAction, pTrans->mTraceId, signature); - int32_t code = tmsgSendReq(&pAction->epSet, &rpcMsg1); - - mError("***repeat 2***"); - SRpcMsg rpcMsg2 = createRpcMsg(pAction, pTrans->mTraceId, signature); - code = tmsgSendReq(&pAction->epSet, &rpcMsg2); - - mError("***repeat 3***"); - SRpcMsg rpcMsg3 = createRpcMsg(pAction, pTrans->mTraceId, signature); - code = tmsgSendReq(&pAction->epSet, &rpcMsg3); - } else if (choseItem == 2) { - // 3. sleep 40s and then send msg - mError("***idle for 30s, and then send msg***"); - taosMsleep(30000); - } else { - // do nothing - // mInfo("no error triggered"); - } - } } \ No newline at end of file diff --git a/source/dnode/mnode/impl/test/stream/stream.cpp b/source/dnode/mnode/impl/test/stream/stream.cpp index d508cf7390..45bc4c2ce2 100644 --- a/source/dnode/mnode/impl/test/stream/stream.cpp +++ b/source/dnode/mnode/impl/test/stream/stream.cpp @@ -246,7 +246,7 @@ TEST_F(StreamTest, kill_checkpoint_trans) { px = taosArrayPush(pStream->tasks, &pLevel); ASSERT(px != NULL); - code = mndCreateStreamResetStatusTrans(pMnode, pStream); + code = mndCreateStreamResetStatusTrans(pMnode, pStream, 1); ASSERT(code != 0); tFreeStreamObj(pStream); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6195899566..a234777441 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1009,21 +1009,34 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { } int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { - SStreamTaskRunReq* pReq = pMsg->pCont; + int32_t code = 0; + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + SDecoder decoder; + + SStreamTaskRunReq req = {0}; + tDecoderInit(&decoder, (uint8_t*)msg, len); + if ((code = tDecodeStreamTaskRunReq(&decoder, &req)) < 0) { + tqError("vgId:%d failed to decode task run req, code:%s", pTq->pStreamMeta->vgId, tstrerror(code)); + tDecoderClear(&decoder); + return TSDB_CODE_SUCCESS; + } + + tDecoderClear(&decoder); // extracted submit data from wal files for all tasks - if (pReq->reqType == STREAM_EXEC_T_EXTRACT_WAL_DATA) { + if (req.reqType == STREAM_EXEC_T_EXTRACT_WAL_DATA) { return tqScanWal(pTq); } - int32_t code = tqStreamTaskProcessRunReq(pTq->pStreamMeta, pMsg, vnodeIsRoleLeader(pTq->pVnode)); + code = tqStreamTaskProcessRunReq(pTq->pStreamMeta, pMsg, vnodeIsRoleLeader(pTq->pVnode)); if (code) { tqError("vgId:%d failed to create task run req, code:%s", TD_VID(pTq->pVnode), tstrerror(code)); return code; } // let's continue scan data in the wal files - if (pReq->reqType >= 0 || pReq->reqType == STREAM_EXEC_T_RESUME_TASK) { + if (req.reqType >= 0 || req.reqType == STREAM_EXEC_T_RESUME_TASK) { code = tqScanWalAsync(pTq, false); // it's ok to failed if (code) { tqError("vgId:%d failed to start scan wal file, code:%s", pTq->pStreamMeta->vgId, tstrerror(code)); @@ -1297,7 +1310,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg) { int32_t vgId = TD_VID(pTq->pVnode); - SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*)pMsg->pCont; + SStreamCheckpointReadyMsg* pReq = (SStreamCheckpointReadyMsg*)pMsg->pCont; if (!vnodeIsRoleLeader(pTq->pVnode)) { tqError("vgId:%d not leader, ignore the retrieve checkpoint-trigger msg from 0x%x", vgId, (int32_t)pReq->downstreamTaskId); @@ -1318,10 +1331,23 @@ int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessTaskRetrieveTriggerReq(STQ* pTq, SRpcMsg* pMsg) { int32_t vgId = TD_VID(pTq->pVnode); - SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*)pMsg->pCont; if (!vnodeIsRoleLeader(pTq->pVnode)) { - tqError("vgId:%d not leader, ignore the retrieve checkpoint-trigger msg from 0x%x", vgId, - (int32_t)pReq->downstreamTaskId); + SRetrieveChkptTriggerReq req = {0}; + + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + SDecoder decoder = {0}; + + tDecoderInit(&decoder, (uint8_t*)msg, len); + if (tDecodeRetrieveChkptTriggerReq(&decoder, &req) < 0) { + tDecoderClear(&decoder); + tqError("vgId:%d invalid retrieve checkpoint-trigger req received", vgId); + return TSDB_CODE_INVALID_MSG; + } + tDecoderClear(&decoder); + + tqError("vgId:%d not leader, ignore the retrieve checkpoint-trigger msg from s-task:0x%" PRId64, vgId, + req.downstreamTaskId); return TSDB_CODE_STREAM_NOT_LEADER; } diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 326e8d4ada..3f67503454 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -828,14 +828,25 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { } int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader) { - SStreamTaskRunReq* pReq = pMsg->pCont; + int32_t code = 0; + int32_t vgId = pMeta->vgId; + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + SDecoder decoder; - int32_t type = pReq->reqType; - int32_t vgId = pMeta->vgId; - int32_t code = 0; + SStreamTaskRunReq req = {0}; + tDecoderInit(&decoder, (uint8_t*)msg, len); + if ((code = tDecodeStreamTaskRunReq(&decoder, &req)) < 0) { + tqError("vgId:%d failed to decode task run req, code:%s", pMeta->vgId, tstrerror(code)); + tDecoderClear(&decoder); + return TSDB_CODE_SUCCESS; + } + tDecoderClear(&decoder); + + int32_t type = req.reqType; if (type == STREAM_EXEC_T_START_ONE_TASK) { - code = streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId); + code = streamMetaStartOneTask(pMeta, req.streamId, req.taskId); return 0; } else if (type == STREAM_EXEC_T_START_ALL_TASKS) { code = streamMetaStartAllTasks(pMeta); @@ -847,11 +858,11 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead code = streamMetaStopAllTasks(pMeta); return 0; } else if (type == STREAM_EXEC_T_ADD_FAILED_TASK) { - code = streamMetaAddFailedTask(pMeta, pReq->streamId, pReq->taskId); + code = streamMetaAddFailedTask(pMeta, req.streamId, req.taskId); return code; } else if (type == STREAM_EXEC_T_RESUME_TASK) { // task resume to run after idle for a while SStreamTask* pTask = NULL; - code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); + code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); if (pTask != NULL && (code == 0)) { char* pStatus = NULL; @@ -873,7 +884,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead } SStreamTask* pTask = NULL; - code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); + code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); if ((pTask != NULL) && (code == 0)) { // even in halt status, the data in inputQ must be processed char* p = NULL; if (streamTaskReadyToRun(pTask, &p)) { @@ -890,7 +901,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead return 0; } else { // NOTE: pTask->status.schedStatus is not updated since it is not be handled by the run exec. // todo add one function to handle this - tqError("vgId:%d failed to found s-task, taskId:0x%x may have been dropped", vgId, pReq->taskId); + tqError("vgId:%d failed to found s-task, taskId:0x%x may have been dropped", vgId, req.taskId); return code; } } @@ -939,7 +950,7 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) { } int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) { - SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg; + SVResetStreamTaskReq* pReq = (SVResetStreamTaskReq*)pMsg; SStreamTask* pTask = NULL; int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); @@ -954,17 +965,13 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) { streamMutexLock(&pTask->lock); streamTaskClearCheckInfo(pTask, true); + streamTaskSetFailedCheckpointId(pTask, pReq->chkptId); + // clear flag set during do checkpoint, and open inputQ for all upstream tasks SStreamTaskState pState = streamTaskGetStatus(pTask); if (pState.state == TASK_STATUS__CK) { - int32_t tranId = 0; - int64_t activeChkId = 0; - streamTaskGetActiveCheckpointInfo(pTask, &tranId, &activeChkId); - - tqDebug("s-task:%s reset task status from checkpoint, current checkpointingId:%" PRId64 ", transId:%d", - pTask->id.idStr, activeChkId, tranId); - streamTaskSetStatusReady(pTask); + tqDebug("s-task:%s reset checkpoint status to ready", pTask->id.idStr); } else if (pState.state == TASK_STATUS__UNINIT) { // tqDebug("s-task:%s start task by checking downstream tasks", pTask->id.idStr); // tqStreamTaskRestoreCheckpoint(pMeta, pTask->id.streamId, pTask->id.taskId); @@ -980,25 +987,36 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) { } int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { - SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*)pMsg->pCont; + SRetrieveChkptTriggerReq req = {0}; + SStreamTask* pTask = NULL; + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + SDecoder decoder = {0}; - SStreamTask* pTask = NULL; - int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId, &pTask); + tDecoderInit(&decoder, (uint8_t*)msg, len); + if (tDecodeRetrieveChkptTriggerReq(&decoder, &req) < 0) { + tDecoderClear(&decoder); + tqError("vgId:%d invalid retrieve checkpoint-trigger req received", pMeta->vgId); + return TSDB_CODE_INVALID_MSG; + } + tDecoderClear(&decoder); + + int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId, &pTask); if (pTask == NULL || (code != 0)) { - tqError("vgId:%d process retrieve checkpoint trigger, checkpointId:%" PRId64 + tqError("vgId:%d process retrieve checkpoint-trigger, checkpointId:%" PRId64 " from s-task:0x%x, failed to acquire task:0x%x, it may have been dropped already", - pMeta->vgId, pReq->checkpointId, (int32_t)pReq->downstreamTaskId, pReq->upstreamTaskId); + pMeta->vgId, req.checkpointId, (int32_t)req.downstreamTaskId, req.upstreamTaskId); return TSDB_CODE_STREAM_TASK_NOT_EXIST; } tqDebug("s-task:0x%x recv retrieve checkpoint-trigger msg from downstream s-task:0x%x, checkpointId:%" PRId64, - pReq->upstreamTaskId, (int32_t)pReq->downstreamTaskId, pReq->checkpointId); + req.upstreamTaskId, (int32_t)req.downstreamTaskId, req.checkpointId); if (pTask->status.downstreamReady != 1) { tqError("s-task:%s not ready for checkpoint-trigger retrieve from 0x%x, since downstream not ready", - pTask->id.idStr, (int32_t)pReq->downstreamTaskId); + pTask->id.idStr, (int32_t)req.downstreamTaskId); - code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, + code = streamTaskSendCheckpointTriggerMsg(pTask, req.downstreamTaskId, req.downstreamNodeId, &pMsg->info, TSDB_CODE_STREAM_TASK_IVLD_STATUS); streamMetaReleaseTask(pMeta, pTask); return code; @@ -1010,19 +1028,19 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) int64_t checkpointId = 0; streamTaskGetActiveCheckpointInfo(pTask, &transId, &checkpointId); - if (checkpointId != pReq->checkpointId) { + if (checkpointId != req.checkpointId) { tqError("s-task:%s invalid checkpoint-trigger retrieve msg from 0x%" PRIx64 ", current checkpointId:%" PRId64 " req:%" PRId64, - pTask->id.idStr, pReq->downstreamTaskId, checkpointId, pReq->checkpointId); + pTask->id.idStr, req.downstreamTaskId, checkpointId, req.checkpointId); streamMetaReleaseTask(pMeta, pTask); return TSDB_CODE_INVALID_MSG; } - if (streamTaskAlreadySendTrigger(pTask, pReq->downstreamNodeId)) { + if (streamTaskAlreadySendTrigger(pTask, req.downstreamNodeId)) { // re-send the lost checkpoint-trigger msg to downstream task tqDebug("s-task:%s re-send checkpoint-trigger to:0x%x, checkpointId:%" PRId64 ", transId:%d", pTask->id.idStr, - (int32_t)pReq->downstreamTaskId, checkpointId, transId); - code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, + (int32_t)req.downstreamTaskId, checkpointId, transId); + code = streamTaskSendCheckpointTriggerMsg(pTask, req.downstreamTaskId, req.downstreamNodeId, &pMsg->info, TSDB_CODE_SUCCESS); } else { // not send checkpoint-trigger yet, wait int32_t recv = 0, total = 0; @@ -1036,7 +1054,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) "sending checkpoint-source/trigger", pTask->id.idStr, recv, total); } - code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, + code = streamTaskSendCheckpointTriggerMsg(pTask, req.downstreamTaskId, req.downstreamNodeId, &pMsg->info, TSDB_CODE_ACTION_IN_PROGRESS); } } else { // upstream not recv the checkpoint-source/trigger till now @@ -1048,7 +1066,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) "s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all " "upstream sending checkpoint-source/trigger", pTask->id.idStr); - code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, + code = streamTaskSendCheckpointTriggerMsg(pTask, req.downstreamTaskId, req.downstreamNodeId, &pMsg->info, TSDB_CODE_ACTION_IN_PROGRESS); } @@ -1057,23 +1075,34 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) } int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { - SCheckpointTriggerRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + SCheckpointTriggerRsp rsp = {0}; + SStreamTask* pTask = NULL; + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + SDecoder decoder = {0}; - SStreamTask* pTask = NULL; - int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId, &pTask); + tDecoderInit(&decoder, (uint8_t*)msg, len); + if (tDecodeCheckpointTriggerRsp(&decoder, &rsp) < 0) { + tDecoderClear(&decoder); + tqError("vgId:%d invalid retrieve checkpoint-trigger rsp received", pMeta->vgId); + return TSDB_CODE_INVALID_MSG; + } + tDecoderClear(&decoder); + + int32_t code = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.taskId, &pTask); if (pTask == NULL || (code != 0)) { tqError( "vgId:%d process retrieve checkpoint-trigger, failed to acquire task:0x%x, it may have been dropped already", - pMeta->vgId, pRsp->taskId); + pMeta->vgId, rsp.taskId); return code; } tqDebug( - "s-task:%s recv re-send checkpoint-trigger msg from through retrieve/rsp channel, upstream:0x%x, " - "checkpointId:%" PRId64 ", transId:%d", - pTask->id.idStr, pRsp->upstreamTaskId, pRsp->checkpointId, pRsp->transId); + "s-task:%s recv re-send checkpoint-trigger msg through retrieve/rsp channel, upstream:0x%x, checkpointId:%" PRId64 + ", transId:%d", + pTask->id.idStr, rsp.upstreamTaskId, rsp.checkpointId, rsp.transId); - code = streamTaskProcessCheckpointTriggerRsp(pTask, pRsp); + code = streamTaskProcessCheckpointTriggerRsp(pTask, &rsp); streamMetaReleaseTask(pMeta, pTask); return code; } @@ -1203,7 +1232,23 @@ int32_t doProcessDummyRspMsg(SStreamMeta* UNUSED_PARAM(pMeta), SRpcMsg* pMsg) { } int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { - return streamProcessHeartbeatRsp(pMeta, pMsg->pCont); + SMStreamHbRspMsg rsp = {0}; + int32_t code = 0; + SDecoder decoder; + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + + tDecoderInit(&decoder, (uint8_t*)msg, len); + code = tDecodeStreamHbRsp(&decoder, &rsp); + if (code < 0) { + terrno = TSDB_CODE_INVALID_MSG; + tDecoderClear(&decoder); + tqError("vgId:%d failed to parse hb rsp msg, code:%s", pMeta->vgId, tstrerror(terrno)); + return terrno; + } + + tDecoderClear(&decoder); + return streamProcessHeartbeatRsp(pMeta, &rsp); } int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); } @@ -1237,7 +1282,7 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { SRestoreCheckpointInfo req = {0}; tDecoderInit(&decoder, (uint8_t*)msg, len); - if (tDecodeRestoreCheckpointInfo(&decoder, &req) < 0) { + if ((code = tDecodeRestoreCheckpointInfo(&decoder, &req)) < 0) { tqError("vgId:%d failed to decode set consensus checkpointId req, code:%s", vgId, tstrerror(code)); tDecoderClear(&decoder); return TSDB_CODE_SUCCESS; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 95c5daf842..e3c75760c8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -667,7 +667,7 @@ int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info) { int64_t nRow = imem->nRow; int64_t nDel = imem->nDel; - if (nRow == 0 && nDel == 0) { + if ((nRow == 0 && nDel == 0) || (tsBypassFlag & TSDB_BYPASS_RB_TSDB_COMMIT)) { (void)taosThreadMutexLock(&tsdb->mutex); tsdb->imem = NULL; (void)taosThreadMutexUnlock(&tsdb->mutex); diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index eb22335311..5b26d17519 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -122,6 +122,10 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitTbData *pSubmi tb_uid_t suid = pSubmitTbData->suid; tb_uid_t uid = pSubmitTbData->uid; + if (tsBypassFlag & TSDB_BYPASS_RB_TSDB_WRITE_MEM) { + goto _err; + } + // create/get STbData to op code = tsdbGetOrCreateTbData(pMemTable, suid, uid, &pTbData); if (code) { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 6702b8b588..16c5e026d1 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -362,6 +362,10 @@ static int32_t vnodePreProcessSubmitMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t code = 0; int32_t lino = 0; + if (tsBypassFlag & TSDB_BYPASS_RA_RPC_RECV_SUBMIT) { + return TSDB_CODE_MSG_PREPROCESSED; + } + SDecoder *pCoder = &(SDecoder){0}; if (taosHton64(((SSubmitReq2Msg *)pMsg->pCont)->version) != 1) { diff --git a/source/libs/geometry/src/geosWrapper.c b/source/libs/geometry/src/geosWrapper.c index 13c5f7208e..8789762a85 100644 --- a/source/libs/geometry/src/geosWrapper.c +++ b/source/libs/geometry/src/geosWrapper.c @@ -63,7 +63,7 @@ int32_t initCtxMakePoint() { int32_t doMakePoint(double x, double y, unsigned char **outputGeom, size_t *size) { int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = NULL; - + TAOS_CHECK_RETURN(getThreadLocalGeosCtx(&geosCtx)); GEOSGeometry *geom = NULL; @@ -170,7 +170,7 @@ static int32_t initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData int32_t initCtxGeomFromText() { int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = NULL; - + TAOS_CHECK_RETURN(getThreadLocalGeosCtx(&geosCtx)); if (geosCtx->handle == NULL) { @@ -208,7 +208,7 @@ int32_t initCtxGeomFromText() { int32_t doGeomFromText(const char *inputWKT, unsigned char **outputGeom, size_t *size) { int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = NULL; - + TAOS_CHECK_RETURN(getThreadLocalGeosCtx(&geosCtx)); GEOSGeometry *geom = NULL; @@ -245,7 +245,7 @@ _exit: int32_t initCtxAsText() { int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = NULL; - + TAOS_CHECK_RETURN(getThreadLocalGeosCtx(&geosCtx)); if (geosCtx->handle == NULL) { @@ -283,11 +283,11 @@ int32_t initCtxAsText() { int32_t doAsText(const unsigned char *inputGeom, size_t size, char **outputWKT) { int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = NULL; - + TAOS_CHECK_RETURN(getThreadLocalGeosCtx(&geosCtx)); - GEOSGeometry *geom = NULL; - char *wkt = NULL; + GEOSGeometry *geom = NULL; + char *wkt = NULL; geom = GEOSWKBReader_read_r(geosCtx->handle, geosCtx->WKBReader, inputGeom, size); if (geom == NULL) { @@ -313,10 +313,35 @@ _exit: return code; } +int32_t checkWKB(const unsigned char *wkb, size_t size) { + int32_t code = TSDB_CODE_SUCCESS; + GEOSGeometry *geom = NULL; + SGeosContext *geosCtx = NULL; + + TAOS_CHECK_RETURN(getThreadLocalGeosCtx(&geosCtx)); + + geom = GEOSWKBReader_read_r(geosCtx->handle, geosCtx->WKBReader, wkb, size); + if (geom == NULL) { + return TSDB_CODE_FUNC_FUNTION_PARA_VALUE; + } + + if (!GEOSisValid_r(geosCtx->handle, geom)) { + code = TSDB_CODE_FUNC_FUNTION_PARA_VALUE; + goto _exit; + } + +_exit: + if (geom) { + GEOSGeom_destroy_r(geosCtx->handle, geom); + geom = NULL; + } + return code; +} + int32_t initCtxRelationFunc() { int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = NULL; - + TAOS_CHECK_RETURN(getThreadLocalGeosCtx(&geosCtx)); if (geosCtx->handle == NULL) { @@ -343,7 +368,7 @@ int32_t doGeosRelation(const GEOSGeometry *geom1, const GEOSPreparedGeometry *pr _geosPreparedRelationFunc_t preparedRelationFn, _geosPreparedRelationFunc_t swappedPreparedRelationFn) { SGeosContext *geosCtx = NULL; - + TAOS_CHECK_RETURN(getThreadLocalGeosCtx(&geosCtx)); if (!preparedGeom1) { diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 0979028e6d..bedd4fa540 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "geosWrapper.h" #include "os.h" #include "parInsertUtil.h" #include "parInt.h" @@ -192,6 +193,12 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch // strcpy(val.colName, pTagSchema->name); if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || pTagSchema->type == TSDB_DATA_TYPE_VARBINARY || pTagSchema->type == TSDB_DATA_TYPE_GEOMETRY) { + if (pTagSchema->type == TSDB_DATA_TYPE_GEOMETRY) { + if (initCtxAsText() || checkWKB(bind[c].buffer, colLen)) { + code = buildSyntaxErrMsg(&pBuf, "invalid geometry tag", bind[c].buffer); + goto end; + } + } val.pData = (uint8_t*)bind[c].buffer; val.nData = colLen; } else if (pTagSchema->type == TSDB_DATA_TYPE_NCHAR) { @@ -409,7 +416,8 @@ int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, c } code = tColDataAddValueByBind(pCol, pBind, - IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE : -1); + IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE : -1, + initCtxAsText, checkWKB); if (code) { goto _return; } @@ -461,7 +469,8 @@ int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bi } code = tColDataAddValueByBind(pCol, pBind, - IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE : -1); + IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE : -1, + initCtxAsText, checkWKB); qDebug("stmt col %d bind %d rows data", colIdx, rowNum); @@ -544,6 +553,12 @@ int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const c // strcpy(val.colName, pTagSchema->name); if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || pTagSchema->type == TSDB_DATA_TYPE_VARBINARY || pTagSchema->type == TSDB_DATA_TYPE_GEOMETRY) { + if (pTagSchema->type == TSDB_DATA_TYPE_GEOMETRY) { + if (initCtxAsText() || checkWKB(bind[c].buffer, colLen)) { + code = buildSyntaxErrMsg(&pBuf, "invalid geometry tag", bind[c].buffer); + goto end; + } + } val.pData = (uint8_t*)bind[c].buffer; val.nData = colLen; } else if (pTagSchema->type == TSDB_DATA_TYPE_NCHAR) { @@ -824,7 +839,8 @@ int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, } code = tColDataAddValueByBind2(pCol, pBind, - IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE : -1); + IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE : -1, + initCtxAsText, checkWKB); if (code) { goto _return; } @@ -876,7 +892,8 @@ int32_t qBindStmtSingleColValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* b } code = tColDataAddValueByBind2(pCol, pBind, - IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE : -1); + IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE : -1, + initCtxAsText, checkWKB); qDebug("stmt col %d bind %d rows data", colIdx, rowNum); diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index c2714659ec..e2135bfd63 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -154,6 +154,9 @@ static int32_t parseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, SParseMetaCa } static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { + if (!pParam || IS_NULL_TYPE(pParam->buffer_type)) { + return TSDB_CODE_APP_ERROR; + } if (IS_VAR_DATA_TYPE(pVal->node.resType.type)) { taosMemoryFreeClear(pVal->datum.p); } @@ -441,6 +444,9 @@ int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx } static int32_t setValueByBindParam2(SValueNode* pVal, TAOS_STMT2_BIND* pParam) { + if (!pParam || IS_NULL_TYPE(pParam->buffer_type)) { + return TSDB_CODE_APP_ERROR; + } if (IS_VAR_DATA_TYPE(pVal->node.resType.type)) { taosMemoryFreeClear(pVal->datum.p); } diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index eefb32f783..3321fdb4b5 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1345,30 +1345,19 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } -#if 1 - SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)}; - code = schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, (uint32_t)msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL)); - msg = NULL; - SCH_ERR_JRET(code); - - if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY) { - SCH_ERR_RET(schAppendTaskExecNode(pJob, pTask, addr, pTask->execId)); - } -#else - if (TDMT_VND_SUBMIT != msgType) { + if ((tsBypassFlag & TSDB_BYPASS_RB_RPC_SEND_SUBMIT) && (TDMT_VND_SUBMIT == msgType)) { + taosMemoryFree(msg); + SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); + } else { SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)}; - code = schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL)); + code = schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, (uint32_t)msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL)); msg = NULL; SCH_ERR_JRET(code); if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY) { SCH_ERR_RET(schAppendTaskExecNode(pJob, pTask, addr, pTask->execId)); } - } else { - taosMemoryFree(msg); - SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); } -#endif return TSDB_CODE_SUCCESS; diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 863bc76c79..427733e9ec 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -192,7 +192,6 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask); int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask); int32_t streamTaskSendCheckpointReq(SStreamTask* pTask); -void streamTaskSetFailedCheckpointId(SStreamTask* pTask); int32_t streamTaskGetNumOfDownstream(const SStreamTask* pTask); int32_t streamTaskGetNumOfUpstream(const SStreamTask* pTask); int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate, const char*); @@ -245,6 +244,9 @@ int32_t streamCreateSinkResTrigger(SStreamTrigger** pTrigger); int32_t streamCreateForcewindowTrigger(SStreamTrigger** pTrigger, int32_t trigger, SInterval* pInterval, STimeWindow* pLatestWindow, const char* id); +// inject stream errors +void chkptFailedByRetrieveReqToSource(SStreamTask* pTask, int64_t checkpointId); + #ifdef __cplusplus } #endif diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 2280b7f06f..d8ddd0fd02 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -161,33 +161,52 @@ int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTri int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId, int32_t downstreamNodeId, SRpcHandleInfo* pRpcInfo, int32_t code) { - int32_t size = sizeof(SMsgHead) + sizeof(SCheckpointTriggerRsp); - void* pBuf = rpcMallocCont(size); - if (pBuf == NULL) { + int32_t ret = 0; + int32_t tlen = 0; + void* buf = NULL; + SEncoder encoder; + + SCheckpointTriggerRsp req = {.streamId = pTask->id.streamId, + .upstreamTaskId = pTask->id.taskId, + .taskId = dstTaskId, + .rspCode = code}; + + if (code == TSDB_CODE_SUCCESS) { + req.checkpointId = pTask->chkInfo.pActiveInfo->activeId; + req.transId = pTask->chkInfo.pActiveInfo->transId; + } else { + req.checkpointId = -1; + req.transId = -1; + } + + tEncodeSize(tEncodeCheckpointTriggerRsp, &req, tlen, ret); + if (ret < 0) { + stError("s-task:%s encode checkpoint-trigger rsp msg failed, code:%s", pTask->id.idStr, tstrerror(code)); + return ret; + } + + buf = rpcMallocCont(tlen + sizeof(SMsgHead)); + if (buf == NULL) { + stError("s-task:%s malloc chkpt-trigger rsp failed for task:0x%x, since out of memory", pTask->id.idStr, dstTaskId); return terrno; } - SCheckpointTriggerRsp* pRsp = POINTER_SHIFT(pBuf, sizeof(SMsgHead)); + ((SMsgHead*)buf)->vgId = htonl(downstreamNodeId); + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - ((SMsgHead*)pBuf)->vgId = htonl(downstreamNodeId); - - pRsp->streamId = pTask->id.streamId; - pRsp->upstreamTaskId = pTask->id.taskId; - pRsp->taskId = dstTaskId; - pRsp->rspCode = code; - - if (code == TSDB_CODE_SUCCESS) { - pRsp->checkpointId = pTask->chkInfo.pActiveInfo->activeId; - pRsp->transId = pTask->chkInfo.pActiveInfo->transId; - } else { - pRsp->checkpointId = -1; - pRsp->transId = -1; + tEncoderInit(&encoder, abuf, tlen); + if ((ret = tEncodeCheckpointTriggerRsp(&encoder, &req)) < 0) { + rpcFreeCont(buf); + tEncoderClear(&encoder); + stError("encode checkpoint-trigger rsp failed, code:%s", tstrerror(code)); + return ret; } + tEncoderClear(&encoder); - SRpcMsg rspMsg = {.code = 0, .pCont = pBuf, .contLen = size, .info = *pRpcInfo}; + SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = tlen + sizeof(SMsgHead), .info = *pRpcInfo}; tmsgSendRsp(&rspMsg); - return 0; + return ret; } int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStreamTask* pTask) { @@ -222,14 +241,14 @@ static int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t check stError("s-task:%s vgId:%d current checkpointId:%" PRId64 " recv expired checkpoint-trigger block, checkpointId:%" PRId64 " transId:%d, discard", id, vgId, pTask->chkInfo.checkpointId, checkpointId, transId); - return TSDB_CODE_STREAM_TASK_NOT_EXIST; + return TSDB_CODE_STREAM_INVLD_CHKPT; } if (pActiveInfo->failedId >= checkpointId) { stError("s-task:%s vgId:%d checkpointId:%" PRId64 " transId:%d, has been marked failed, failedId:%" PRId64 " discard the checkpoint-trigger block", id, vgId, checkpointId, transId, pActiveInfo->failedId); - return TSDB_CODE_STREAM_TASK_NOT_EXIST; + return TSDB_CODE_STREAM_INVLD_CHKPT; } if (pTask->chkInfo.checkpointId == checkpointId) { @@ -282,7 +301,7 @@ static int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t check } if (p->upstreamTaskId == pBlock->srcTaskId) { - stWarn("s-task:%s repeatly recv checkpoint-source msg from task:0x%x vgId:%d, checkpointId:%" PRId64 + stWarn("s-task:%s repeatly recv checkpoint-trigger msg from task:0x%x vgId:%d, checkpointId:%" PRId64 ", prev recvTs:%" PRId64 " discard", pTask->id.idStr, p->upstreamTaskId, p->upstreamNodeId, p->checkpointId, p->recvTs); return TSDB_CODE_STREAM_INVLD_CHKPT; @@ -319,7 +338,6 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock if (taskLevel != TASK_LEVEL__SOURCE) { // the checkpoint-trigger is discard, open the inputQ for upstream tasks streamTaskOpenUpstreamInput(pTask, pBlock->srcTaskId); } - streamFreeQitem((SStreamQueueItem*)pBlock); return code; } @@ -333,6 +351,11 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock pActiveInfo->activeId = checkpointId; pActiveInfo->transId = transId; + if (pTask->chkInfo.startTs == 0) { + pTask->chkInfo.startTs = taosGetTimestampMs(); + pTask->execInfo.checkpoint += 1; + } + code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT); if (code != TSDB_CODE_SUCCESS) { stError("s-task:%s handle checkpoint-trigger block failed, code:%s", id, tstrerror(code)); @@ -380,6 +403,10 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock return code; } +#if 0 + chkptFailedByRetrieveReqToSource(pTask, checkpointId); +#endif + if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH) { stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId); code = continueDispatchCheckpointTriggerBlock(pBlock, pTask); // todo handle this failure @@ -389,11 +416,6 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock streamFreeQitem((SStreamQueueItem*)pBlock); } } else if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) { - if (pTask->chkInfo.startTs == 0) { - pTask->chkInfo.startTs = taosGetTimestampMs(); - pTask->execInfo.checkpoint += 1; - } - // todo: handle this // update the child Id for downstream tasks code = streamAddCheckpointReadyMsg(pTask, pBlock->srcTaskId, pTask->info.selfChildId, checkpointId); @@ -569,7 +591,7 @@ void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) { } streamMutexUnlock(&pInfo->lock); - stDebug("s-task:%s clear active checkpointInfo, failed checkpointId:%" PRId64 ", current checkpointId:%" PRId64, + stDebug("s-task:%s clear active checkpointInfo, failed checkpointId:%" PRId64 ", latest checkpointId:%" PRId64, pTask->id.idStr, pInfo->failedId, pTask->chkInfo.checkpointId); } @@ -689,15 +711,22 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV return TSDB_CODE_SUCCESS; } -void streamTaskSetFailedCheckpointId(SStreamTask* pTask) { +void streamTaskSetFailedCheckpointId(SStreamTask* pTask, int64_t failedId) { struct SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; - if (pInfo->activeId <= 0) { - stWarn("s-task:%s checkpoint-info is cleared now, not set the failed checkpoint info", pTask->id.idStr); + if (failedId <= 0) { + stWarn("s-task:%s failedId is 0, not update the failed checkpoint info, current failedId:%" PRId64 + " activeId:%" PRId64, + pTask->id.idStr, pInfo->failedId, pInfo->activeId); } else { - pInfo->failedId = pInfo->activeId; - stDebug("s-task:%s mark and set the failed checkpointId:%" PRId64 " (transId:%d)", pTask->id.idStr, pInfo->activeId, - pInfo->transId); + if (failedId <= pInfo->failedId) { + stDebug("s-task:%s failedId:%" PRId64 " not update to:%" PRId64, pTask->id.idStr, pInfo->failedId, failedId); + } else { + stDebug("s-task:%s mark and set the failed checkpointId:%" PRId64 " (transId:%d) activeId:%" PRId64 + " prev failedId:%" PRId64, + pTask->id.idStr, failedId, pInfo->transId, pInfo->activeId, pInfo->failedId); + pInfo->failedId = failedId; + } } } @@ -705,7 +734,7 @@ void streamTaskSetCheckpointFailed(SStreamTask* pTask) { streamMutexLock(&pTask->lock); ETaskStatus status = streamTaskGetStatus(pTask).state; if (status == TASK_STATUS__CK) { - streamTaskSetFailedCheckpointId(pTask); + streamTaskSetFailedCheckpointId(pTask, pTask->chkInfo.pActiveInfo->activeId); } streamMutexUnlock(&pTask->lock); } @@ -883,8 +912,9 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { code = streamSendChkptReportMsg(pTask, &pTask->chkInfo, dropRelHTask); } } else { // clear the checkpoint info if failed + // set failed checkpoint id before clear the checkpoint info streamMutexLock(&pTask->lock); - streamTaskSetFailedCheckpointId(pTask); // set failed checkpoint id before clear the checkpoint info + streamTaskSetFailedCheckpointId(pTask, ckId); streamMutexUnlock(&pTask->lock); code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_CHECKPOINT_DONE); @@ -1108,23 +1138,43 @@ int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList) { return TSDB_CODE_INVALID_PARA; } - SRetrieveChkptTriggerReq* pReq = rpcMallocCont(sizeof(SRetrieveChkptTriggerReq)); - if (pReq == NULL) { - code = terrno; - stError("vgId:%d failed to create msg to retrieve trigger msg for task:%s exec, code:out of memory", vgId, pId); + int32_t ret = 0; + int32_t tlen = 0; + void* buf = NULL; + SRpcMsg rpcMsg = {0}; + SEncoder encoder; + + SRetrieveChkptTriggerReq req = {.streamId = pTask->id.streamId, + .downstreamTaskId = pTask->id.taskId, + .downstreamNodeId = vgId, + .upstreamTaskId = pUpstreamTask->taskId, + .upstreamNodeId = pUpstreamTask->nodeId, + .checkpointId = checkpointId}; + + tEncodeSize(tEncodeRetrieveChkptTriggerReq, &req, tlen, ret); + if (ret < 0) { + stError("encode retrieve checkpoint-trigger msg failed, code:%s", tstrerror(code)); + } + + buf = rpcMallocCont(tlen + sizeof(SMsgHead)); + if (buf == NULL) { + stError("vgId:%d failed to create retrieve checkpoint-trigger msg for task:%s exec, code:out of memory", vgId, pId); continue; } - pReq->head.vgId = htonl(pUpstreamTask->nodeId); - pReq->streamId = pTask->id.streamId; - pReq->downstreamTaskId = pTask->id.taskId; - pReq->downstreamNodeId = vgId; - pReq->upstreamTaskId = pUpstreamTask->taskId; - pReq->upstreamNodeId = pUpstreamTask->nodeId; - pReq->checkpointId = checkpointId; + ((SRetrieveChkptTriggerReq*)buf)->head.vgId = htonl(pUpstreamTask->nodeId); + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - SRpcMsg rpcMsg = {0}; - initRpcMsg(&rpcMsg, TDMT_STREAM_RETRIEVE_TRIGGER, pReq, sizeof(SRetrieveChkptTriggerReq)); + tEncoderInit(&encoder, abuf, tlen); + if ((code = tEncodeRetrieveChkptTriggerReq(&encoder, &req)) < 0) { + rpcFreeCont(buf); + tEncoderClear(&encoder); + stError("encode retrieve checkpoint-trigger req failed, code:%s", tstrerror(code)); + continue; + } + tEncoderClear(&encoder); + + initRpcMsg(&rpcMsg, TDMT_STREAM_RETRIEVE_TRIGGER, buf, tlen + sizeof(SMsgHead)); code = tmsgSendReq(&pUpstreamTask->epSet, &rpcMsg); if (code == TSDB_CODE_SUCCESS) { diff --git a/source/libs/stream/src/streamErrorInjection.c b/source/libs/stream/src/streamErrorInjection.c new file mode 100644 index 0000000000..515845ba2b --- /dev/null +++ b/source/libs/stream/src/streamErrorInjection.c @@ -0,0 +1,17 @@ +#include "streamInt.h" + +/** + * pre-request: checkpoint interval should be 60s + * @param pTask + * @param checkpointId + */ +void chkptFailedByRetrieveReqToSource(SStreamTask* pTask, int64_t checkpointId) { + streamMutexLock(&pTask->lock); + + // set current checkpoint failed immediately, set failed checkpoint id before clear the checkpoint info + streamTaskSetFailedCheckpointId(pTask, checkpointId); + streamMutexUnlock(&pTask->lock); + + // the checkpoint interval should be 60s, and the next checkpoint req should be issued by mnode + taosMsleep(65*1000); +} \ No newline at end of file diff --git a/source/libs/stream/src/streamSched.c b/source/libs/stream/src/streamSched.c index 8c79abfd02..9e131fd526 100644 --- a/source/libs/stream/src/streamSched.c +++ b/source/libs/stream/src/streamSched.c @@ -83,13 +83,37 @@ int32_t streamTrySchedExec(SStreamTask* pTask) { } int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId, int32_t execType) { - SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); - if (pRunReq == NULL) { + int32_t code = 0; + int32_t tlen = 0; + + SStreamTaskRunReq req = {.streamId = streamId, .taskId = taskId, .reqType = execType}; + + tEncodeSize(tEncodeStreamTaskRunReq, &req, tlen, code); + if (code < 0) { + stError("s-task:0x%" PRIx64 " vgId:%d encode stream task run req failed, code:%s", streamId, vgId, tstrerror(code)); + return code; + } + + void* buf = rpcMallocCont(tlen + sizeof(SMsgHead)); + if (buf == NULL) { stError("vgId:%d failed to create msg to start stream task:0x%x exec, type:%d, code:%s", vgId, taskId, execType, tstrerror(terrno)); return terrno; } + ((SMsgHead*)buf)->vgId = vgId; + char* bufx = POINTER_SHIFT(buf, sizeof(SMsgHead)); + + SEncoder encoder; + tEncoderInit(&encoder, (uint8_t*)bufx, tlen); + if ((code = tEncodeStreamTaskRunReq(&encoder, &req)) < 0) { + rpcFreeCont(buf); + tEncoderClear(&encoder); + stError("s-task:0x%x vgId:%d encode run task msg failed, code:%s", taskId, vgId, tstrerror(code)); + return code; + } + tEncoderClear(&encoder); + if (streamId != 0) { stDebug("vgId:%d create msg to for task:0x%x, exec type:%d, %s", vgId, taskId, execType, streamTaskGetExecType(execType)); @@ -97,13 +121,8 @@ int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int3 stDebug("vgId:%d create msg to exec, type:%d, %s", vgId, execType, streamTaskGetExecType(execType)); } - pRunReq->head.vgId = vgId; - pRunReq->streamId = streamId; - pRunReq->taskId = taskId; - pRunReq->reqType = execType; - - SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; - int32_t code = tmsgPutToQueue(pMsgCb, STREAM_QUEUE, &msg); + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = buf, .contLen = tlen + sizeof(SMsgHead)}; + code = tmsgPutToQueue(pMsgCb, STREAM_QUEUE, &msg); if (code) { stError("vgId:%d failed to put msg into stream queue, code:%s, %x", vgId, tstrerror(code), taskId); } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index a044859b80..f46228fd47 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -22,6 +22,7 @@ #include "tstream.h" #include "ttimer.h" #include "wal.h" +#include "streamMsg.h" static void streamTaskDestroyUpstreamInfo(SUpstreamInfo* pUpstreamInfo); static int32_t streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool* pUpdated); @@ -1246,13 +1247,13 @@ void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo) { taosMemoryFree(pInfo); } -//NOTE: clear the checkpoint id, and keep the failed id +// NOTE: clear the checkpoint id, and keep the failed id +// failedId for a task will increase as the checkpoint I.D. increases. void streamTaskClearActiveInfo(SActiveCheckpointInfo* pInfo) { pInfo->activeId = 0; pInfo->transId = 0; pInfo->allUpstreamTriggerRecv = 0; pInfo->dispatchTrigger = false; -// pInfo->failedId = 0; taosArrayClear(pInfo->pDispatchTriggerList); taosArrayClear(pInfo->pCheckpointReadyRecvList); @@ -1303,4 +1304,178 @@ void streamTaskFreeRefId(int64_t* pRefId) { } metaRefMgtRemove(pRefId); +} + + +int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartEncode(pEncoder)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->ver)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->id.streamId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->id.taskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->info.trigger)); + TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->info.taskLevel)); + TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->outputInfo.type)); + TAOS_CHECK_EXIT(tEncodeI16(pEncoder, pTask->msgInfo.msgType)); + + TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->status.taskStatus)); + TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->status.schedStatus)); + + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->info.selfChildId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->info.nodeId)); + TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pTask->info.epSet)); + TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pTask->info.mnodeEpset)); + + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->chkInfo.checkpointId)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->chkInfo.checkpointVer)); + TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->info.fillHistory)); + + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->hTaskInfo.id.streamId)); + int32_t taskId = pTask->hTaskInfo.id.taskId; + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, taskId)); + + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->streamTaskId.streamId)); + taskId = pTask->streamTaskId.taskId; + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, taskId)); + + TAOS_CHECK_EXIT(tEncodeU64(pEncoder, pTask->dataRange.range.minVer)); + TAOS_CHECK_EXIT(tEncodeU64(pEncoder, pTask->dataRange.range.maxVer)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->dataRange.window.skey)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->dataRange.window.ekey)); + + int32_t epSz = taosArrayGetSize(pTask->upstreamInfo.pList); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, epSz)); + for (int32_t i = 0; i < epSz; i++) { + SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); + TAOS_CHECK_EXIT(tEncodeStreamEpInfo(pEncoder, pInfo)); + } + + if (pTask->info.taskLevel != TASK_LEVEL__SINK) { + TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, pTask->exec.qmsg)); + } + + if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->outputInfo.tbSink.stbUid)); + TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, pTask->outputInfo.tbSink.stbFullName)); + TAOS_CHECK_EXIT(tEncodeSSchemaWrapper(pEncoder, pTask->outputInfo.tbSink.pSchemaWrapper)); + } else if (pTask->outputInfo.type == TASK_OUTPUT__SMA) { + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->outputInfo.smaSink.smaId)); + } else if (pTask->outputInfo.type == TASK_OUTPUT__FETCH) { + TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->outputInfo.fetchSink.reserved)); + } else if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->outputInfo.fixedDispatcher.taskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->outputInfo.fixedDispatcher.nodeId)); + TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pTask->outputInfo.fixedDispatcher.epSet)); + } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { + TAOS_CHECK_EXIT(tSerializeSUseDbRspImp(pEncoder, &pTask->outputInfo.shuffleDispatcher.dbInfo)); + TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, pTask->outputInfo.shuffleDispatcher.stbFullName)); + } + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->info.delaySchedParam)); + TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->subtableWithoutMd5)); + TAOS_CHECK_EXIT(tEncodeCStrWithLen(pEncoder, pTask->reserve, sizeof(pTask->reserve) - 1)); + + tEndEncode(pEncoder); +_exit: + return code; +} + +int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { + int32_t taskId = 0; + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartDecode(pDecoder)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->ver)); + if (pTask->ver <= SSTREAM_TASK_INCOMPATIBLE_VER || pTask->ver > SSTREAM_TASK_VER) { + TAOS_CHECK_EXIT(TSDB_CODE_INVALID_MSG); + } + + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->id.streamId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->id.taskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->info.trigger)); + TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->info.taskLevel)); + TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->outputInfo.type)); + TAOS_CHECK_EXIT(tDecodeI16(pDecoder, &pTask->msgInfo.msgType)); + + TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->status.taskStatus)); + TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->status.schedStatus)); + + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->info.selfChildId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->info.nodeId)); + TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &pTask->info.epSet)); + TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &pTask->info.mnodeEpset)); + + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->chkInfo.checkpointId)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->chkInfo.checkpointVer)); + TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->info.fillHistory)); + + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->hTaskInfo.id.streamId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &taskId)); + pTask->hTaskInfo.id.taskId = taskId; + + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->streamTaskId.streamId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &taskId)); + pTask->streamTaskId.taskId = taskId; + + TAOS_CHECK_EXIT(tDecodeU64(pDecoder, (uint64_t*)&pTask->dataRange.range.minVer)); + TAOS_CHECK_EXIT(tDecodeU64(pDecoder, (uint64_t*)&pTask->dataRange.range.maxVer)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->dataRange.window.skey)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->dataRange.window.ekey)); + + int32_t epSz = -1; + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &epSz) < 0); + + if ((pTask->upstreamInfo.pList = taosArrayInit(epSz, POINTER_BYTES)) == NULL) { + TAOS_CHECK_EXIT(terrno); + } + for (int32_t i = 0; i < epSz; i++) { + SStreamUpstreamEpInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamUpstreamEpInfo)); + if (pInfo == NULL) { + TAOS_CHECK_EXIT(terrno); + } + if ((code = tDecodeStreamEpInfo(pDecoder, pInfo)) < 0) { + taosMemoryFreeClear(pInfo); + goto _exit; + } + if (taosArrayPush(pTask->upstreamInfo.pList, &pInfo) == NULL) { + TAOS_CHECK_EXIT(terrno); + } + } + + if (pTask->info.taskLevel != TASK_LEVEL__SINK) { + TAOS_CHECK_EXIT(tDecodeCStrAlloc(pDecoder, &pTask->exec.qmsg)); + } + + if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->outputInfo.tbSink.stbUid)); + TAOS_CHECK_EXIT(tDecodeCStrTo(pDecoder, pTask->outputInfo.tbSink.stbFullName)); + pTask->outputInfo.tbSink.pSchemaWrapper = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); + if (pTask->outputInfo.tbSink.pSchemaWrapper == NULL) { + TAOS_CHECK_EXIT(terrno); + } + TAOS_CHECK_EXIT(tDecodeSSchemaWrapper(pDecoder, pTask->outputInfo.tbSink.pSchemaWrapper)); + } else if (pTask->outputInfo.type == TASK_OUTPUT__SMA) { + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->outputInfo.smaSink.smaId)); + } else if (pTask->outputInfo.type == TASK_OUTPUT__FETCH) { + TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->outputInfo.fetchSink.reserved)); + } else if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->outputInfo.fixedDispatcher.taskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->outputInfo.fixedDispatcher.nodeId)); + TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &pTask->outputInfo.fixedDispatcher.epSet)); + } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { + TAOS_CHECK_EXIT(tDeserializeSUseDbRspImp(pDecoder, &pTask->outputInfo.shuffleDispatcher.dbInfo)); + TAOS_CHECK_EXIT(tDecodeCStrTo(pDecoder, pTask->outputInfo.shuffleDispatcher.stbFullName)); + } + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->info.delaySchedParam)); + if (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER) { + TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->subtableWithoutMd5)); + } + TAOS_CHECK_EXIT(tDecodeCStrTo(pDecoder, pTask->reserve)); + + tEndDecode(pDecoder); + +_exit: + return code; } \ No newline at end of file diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 0b653ddbe9..b19d1184a7 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -234,6 +234,10 @@ struct SSyncNode { bool isStart; + // statis + int32_t sendCount; + int32_t recvCount; + int32_t slowCount; }; // open/close -------------- diff --git a/source/libs/sync/inc/syncPipeline.h b/source/libs/sync/inc/syncPipeline.h index 427a3690f2..eeb24d2f16 100644 --- a/source/libs/sync/inc/syncPipeline.h +++ b/source/libs/sync/inc/syncPipeline.h @@ -39,6 +39,7 @@ typedef struct SSyncLogReplMgr { int64_t peerStartTime; int32_t retryBackoff; int32_t peerId; + int32_t sendCount; } SSyncLogReplMgr; typedef struct SSyncLogBufEntry { diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 0345880874..682d3f9e88 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -104,6 +104,11 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { goto _IGNORE; } + int32_t nRef = atomic_fetch_add_32(&ths->recvCount, 1); + if (nRef <= 0) { + sError("vgId:%d, recv count is %d", ths->vgId, nRef); + } + int32_t code = syncBuildAppendEntriesReply(&rpcRsp, ths->vgId); if (code != 0) { syncLogRecvAppendEntries(ths, pMsg, "build rsp error"); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 247b5624c3..66c49834d8 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -88,6 +88,22 @@ int32_t syncNodeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftI pMsg->destId = *destRaftId; TAOS_CHECK_RETURN(syncNodeSendMsgById(destRaftId, pSyncNode, pRpcMsg)); + int32_t nRef = 0; + if (pSyncNode != NULL) { + nRef = atomic_fetch_add_32(&pSyncNode->sendCount, 1); + if (nRef <= 0) { + sError("vgId:%d, send count is %d", pSyncNode->vgId, nRef); + } + } + + SSyncLogReplMgr* mgr = syncNodeGetLogReplMgr(pSyncNode, (SRaftId*)destRaftId); + if (mgr != NULL) { + nRef = atomic_fetch_add_32(&mgr->sendCount, 1); + if (nRef <= 0) { + sError("vgId:%d, send count is %d", pSyncNode->vgId, nRef); + } + } + TAOS_RETURN(TSDB_CODE_SUCCESS); } diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 9058b6ecef..3907bd5976 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -152,8 +152,9 @@ static void syncLogReplStates2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLe for (int32_t i = 0; i < pSyncNode->replicaNum; i++) { SSyncLogReplMgr* pMgr = pSyncNode->logReplMgrs[i]; if (pMgr == NULL) break; - len += tsnprintf(buf + len, bufLen - len, "%d:%d [%" PRId64 ", %" PRId64 ", %" PRId64 "]", i, pMgr->restored, + len += tsnprintf(buf + len, bufLen - len, "%d:%d [%" PRId64 ", %" PRId64 ", %" PRId64 "] ", i, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex); + len += tsnprintf(buf + len, bufLen - len, "%d", pMgr->sendCount); if (i + 1 < pSyncNode->replicaNum) { len += tsnprintf(buf + len, bufLen - len, "%s", ", "); } @@ -234,14 +235,15 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo ", elect-times:%d, as-leader-times:%d, as-assigned-leader-times:%d, cfg-ch-times:%d, hb-slow:%d, hbr-slow:%d, " "aq-items:%d, snaping:%" PRId64 ", replicas:%d, last-cfg:%" PRId64 ", chging:%d, restore:%d, quorum:%d, elect-lc-timer:%" PRId64 ", hb:%" PRId64 - ", buffer:%s, repl-mgrs:%s, members:%s, hb:%s, hb-reply:%s, arb-token:%s", + ", buffer:%s, repl-mgrs:%s, members:%s, hb:%s, hb-reply:%s, arb-token:%s, msg[sent:%d, recv:%d, slow-recev:%d]", pNode->vgId, eventLog, syncStr(pNode->state), currentTerm, pNode->commitIndex, pNode->assignedCommitIndex, appliedIndex, logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pNode->electNum, pNode->becomeLeaderNum, pNode->becomeAssignedLeaderNum, pNode->configChangeNum, pNode->hbSlowNum, pNode->hbrSlowNum, aqItems, pNode->snapshottingIndex, pNode->replicaNum, pNode->raftCfg.lastConfigIndex, pNode->changing, pNode->restoreFinish, syncNodeDynamicQuorum(pNode), pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, bufferStatesStr, - replMgrStatesStr, cfgStr, hbTimeStr, hbrTimeStr, pNode->arbToken); + replMgrStatesStr, cfgStr, hbTimeStr, hbrTimeStr, pNode->arbToken, pNode->sendCount, pNode->recvCount, + pNode->slowCount); } } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index c03d3418fa..9ad92590ba 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1679,7 +1679,7 @@ void cliConnCb(uv_connect_t* req, int status) { STUB_RAND_NETWORK_ERR(status); if (status != 0) { - tDebug("%s conn %p failed to connect to %s since %s", CONN_GET_INST_LABEL(pConn), pConn, pConn->dstAddr, + tError("%s conn %p failed to connect to %s since %s", CONN_GET_INST_LABEL(pConn), pConn, pConn->dstAddr, uv_strerror(status)); cliMayUpdateFqdnCache(pThrd->fqdn2ipCache, pConn->dstAddr); TAOS_UNUSED(transUnrefCliHandle(pConn)); @@ -1832,15 +1832,20 @@ static FORCE_INLINE int32_t cliUpdateFqdnCache(SHashObj* cache, char* fqdn) { if (code == 0) { size_t len = strlen(fqdn); uint32_t* v = taosHashGet(cache, fqdn, len); - if (addr != *v) { - char old[TSDB_FQDN_LEN] = {0}, new[TSDB_FQDN_LEN] = {0}; - tinet_ntoa(old, *v); - tinet_ntoa(new, addr); - tWarn("update ip of fqdn:%s, old: %s, new: %s", fqdn, old, new); - code = taosHashPut(cache, fqdn, strlen(fqdn), &addr, sizeof(addr)); + if (v != NULL) { + if (addr != *v) { + char old[TSDB_FQDN_LEN] = {0}, new[TSDB_FQDN_LEN] = {0}; + tinet_ntoa(old, *v); + tinet_ntoa(new, addr); + tWarn("update ip of fqdn:%s, old: %s, new: %s", fqdn, old, new); + code = taosHashPut(cache, fqdn, len, &addr, sizeof(addr)); + } + } else { + code = taosHashPut(cache, fqdn, len, &addr, sizeof(addr)); } } else { code = TSDB_CODE_RPC_FQDN_ERROR; // TSDB_CODE_RPC_INVALID_FQDN; + tWarn("failed to get ip from fqdn:%s since %s", fqdn, tstrerror(code)); } return code; } @@ -2933,10 +2938,8 @@ void cliMayResetRespCode(SCliReq* pReq, STransMsg* pResp) { // check whole vnodes is offline on this vgroup if (((pCtx->epSet != NULL) && pCtx->epsetRetryCnt >= pCtx->epSet->numOfEps) || pCtx->retryStep > 0) { - if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - pResp->code = TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED; - } else if (pResp->code == TSDB_CODE_RPC_BROKEN_LINK) { - pResp->code = TSDB_CODE_RPC_SOMENODE_BROKEN_LINK; + if (pResp->code == TSDB_CODE_RPC_BROKEN_LINK) { + pResp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; // TSDB_CODE_RPC_SOMENODE_BROKEN_LINK; } } } diff --git a/tests/army/alter/alterConfig.py b/tests/army/alter/alterConfig.py index f8c52551e3..ef8de0011b 100644 --- a/tests/army/alter/alterConfig.py +++ b/tests/army/alter/alterConfig.py @@ -100,6 +100,70 @@ class TDTestCase(TBase): tdSql.query('show dnodes') tdSql.checkData(0, 3, "64") + def checkKeyValue(self, res, key, value, ikey = 0, ival = 1): + result = False + for row in res: + if row[ikey] == key: + if row[ival] != value: + raise Exception(f"key:{key} value:{row[ival]} != {value}") + else: + tdLog.info(f"key:{key} value:{row[ival]} == {value}") + result = True + break + if not result: + raise Exception(f"key:{key} not found") + + def alterBypassFlag(self): + """Add test case for altering bypassFlag(TD-32907) + """ + tdSql.execute(f"drop database if exists db") + tdSql.execute(f"create database db") + tdSql.execute("use db") + self.checkKeyValue(tdSql.getResult("show local variables;"), "bypassFlag", "0") + self.checkKeyValue(tdSql.getResult("show dnode 1 variables like 'bypassFlag'"), "bypassFlag", "0", 1, 2) + tdSql.execute("alter local 'bypassFlag 1'") + self.checkKeyValue(tdSql.getResult("show local variables;"), "bypassFlag", "1") + self.checkKeyValue(tdSql.getResult("show dnode 1 variables like 'bypassFlag'"), "bypassFlag", "0", 1, 2) + tdSql.execute("create table stb0(ts timestamp, c0 int) tags(t0 int)") + tdSql.execute("create table ctb0 using stb0 tags(0)") + tdSql.execute("insert into ctb0 values(now, 1)") + tdSql.query("select * from stb0") + tdSql.checkRows(0) + tdSql.execute("alter local 'bypassFlag 0'") + tdSql.execute("alter all dnodes 'bypassFlag 2'") + self.checkKeyValue(tdSql.getResult("show local variables"), "bypassFlag", "0") + self.checkKeyValue(tdSql.getResult("show dnode 1 variables like 'bypassFlag'"), "bypassFlag", "2", 1, 2) + tdSql.execute("insert into ctb0 values(now, 2)") + tdSql.query("select * from stb0") + tdSql.checkRows(0) + tdSql.execute("alter all dnodes 'bypassFlag 4'") + self.checkKeyValue(tdSql.getResult("show dnode 1 variables like 'bypassFlag'"), "bypassFlag", "4", 1, 2) + tdSql.execute("insert into ctb0 values(now, 4)") + tdSql.execute("insert into ctb1 using stb0 tags(1) values(now, 10)") + tdSql.query("select * from stb0") + tdSql.checkRows(0) + tdSql.query("show db.tables") + tdSql.checkRows(2) + tdSql.execute("alter all dnodes 'bypassFlag 8'") + self.checkKeyValue(tdSql.getResult("show dnode 1 variables like 'bypassFlag'"), "bypassFlag", "8", 1, 2) + tdSql.execute("insert into ctb0 values(now, 8)") + tdSql.execute("insert into ctb1 values(now, 18)") + tdSql.query("select * from stb0") + tdSql.checkRows(2) + tdSql.execute("flush database db") + tdSql.query("select * from stb0") + tdSql.checkRows(0) + tdSql.execute("alter all dnodes 'bypassFlag 0'") + self.checkKeyValue(tdSql.getResult("show local variables"), "bypassFlag", "0") + self.checkKeyValue(tdSql.getResult("show dnode 1 variables like 'bypassFlag'"), "bypassFlag", "0", 1, 2) + tdSql.execute("insert into ctb0 values(now, 80)") + tdSql.execute("insert into ctb1 values(now, 180)") + tdSql.query("select * from stb0") + tdSql.checkRows(2) + tdSql.execute("flush database db") + tdSql.query("select * from stb0") + tdSql.checkRows(2) + # run def run(self): tdLog.debug(f"start to excute {__file__}") @@ -110,6 +174,8 @@ class TDTestCase(TBase): self.alterTtlConfig() # TS-5390 self.alterCachemodel() + # TD-32907 + self.alterBypassFlag() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/army/storage/compressBasic.py b/tests/army/storage/compressBasic.py index f24c4dd288..446cb920fb 100644 --- a/tests/army/storage/compressBasic.py +++ b/tests/army/storage/compressBasic.py @@ -33,7 +33,26 @@ class TDTestCase(TBase): "compressMsgSize" : "100", } # compress - compresses = ["lz4","tsz","zlib","zstd","disabled","xz"] + compresses = ["lz4","zlib","zstd","disabled","xz"] + + compressDefaultDict = {}; + compressDefaultDict["BOOL"] = "zstd" + compressDefaultDict["TINYINT"] = "zlib" + compressDefaultDict["SMALLINT"] = "zlib" + compressDefaultDict["INT"] = "lz4" + compressDefaultDict["BIGINT"] = "lz4" + compressDefaultDict["FLOAT"] = "lz4" + compressDefaultDict["DOUBLE"] = "lz4" + compressDefaultDict["VARCHAR"] = "zstd" + compressDefaultDict["TIMESTAMP"] = "lz4" + compressDefaultDict["NCHAR"] = "zstd" + compressDefaultDict["TINYINT UNSIGNED"] = "zlib" + compressDefaultDict["SMALLINT UNSIGNED"] = "zlib" + compressDefaultDict["INT UNSIGNED"] = "lz4" + compressDefaultDict["BIGINT UNSIGNED"] = "lz4" + compressDefaultDict["NCHAR"] = "zstd" + compressDefaultDict["BLOB"] = "lz4" + compressDefaultDict["VARBINARY"] = "zstd" # level levels = ["high","medium","low"] @@ -137,15 +156,20 @@ class TDTestCase(TBase): defEncodes = [ "delta-i","delta-i","simple8b","simple8b","simple8b","simple8b","simple8b","simple8b", "simple8b","simple8b","delta-d","delta-d","bit-packing", "disabled","disabled","disabled","disabled"] - + count = tdSql.getRows() for i in range(count): node = tdSql.getData(i, 3) if node == "TAG": break # check - tdSql.checkData(i, 4, defEncodes[i]) - tdSql.checkData(i, 5, self.defCompress) + tdLog.info(f"check default encode {tdSql.getData(i, 1)}") + #tdLog.info(f"check default encode compressDefaultDict[tdSql.getData(i, 2)]") + defaultValue = self.compressDefaultDict[tdSql.getData(i, 1)] + if defaultValue == None: + defaultValue = self.defCompress + tdLog.info(f"check default compress {tdSql.getData(i, 1)} {defaultValue}") + tdSql.checkData(i, 5, defaultValue) tdSql.checkData(i, 6, self.defLevel) # geometry encode is disabled @@ -185,10 +209,6 @@ class TDTestCase(TBase): comps.append(self.compresses[0]) # add lz4 for comp in comps: for i in range(self.colCnt - 1): - col = f"c{i}" - sql = f"alter table {tbname} modify column {col} COMPRESS '{comp}';" - tdSql.execute(sql, show=True) - self.checkDataDesc(tbname, i + 1, 5, comp) self.writeData(1000) # alter float(c9) double(c10) to tsz @@ -326,6 +346,7 @@ class TDTestCase(TBase): while offset < count: sql = f"select * from {tbname} limit {step} offset {offset}" + tdLog.info(sql) tdSql.query(sql) self.autoGen.dataCorrect(tdSql.res, tdSql.getRows(), step) offset += step diff --git a/tests/army/storage/s3/s3Basic.py b/tests/army/storage/s3/s3Basic.py index 273a6129e1..cefd4ef60d 100644 --- a/tests/army/storage/s3/s3Basic.py +++ b/tests/army/storage/s3/s3Basic.py @@ -47,7 +47,7 @@ for test: class TDTestCase(TBase): - index = eutil.cpuRand(20) + 1 + index = eutil.cpuRand(40) + 1 bucketName = f"ci-bucket{index}" updatecfgDict = { "supportVnodes":"1000", @@ -63,6 +63,10 @@ class TDTestCase(TBase): tdLog.info(f"assign bucketName is {bucketName}\n") maxFileSize = (128 + 10) * 1014 * 1024 # add 10M buffer + def exit(self, log): + self.dropDb(True) + tdLog.exit(log) + def insertData(self): tdLog.info(f"insert data.") # taosBenchmark run @@ -107,8 +111,8 @@ class TDTestCase(TBase): loop = 0 rets = [] overCnt = 0 - while loop < 200: - time.sleep(3) + while loop < 150: + time.sleep(2) # check upload to s3 rets = eos.runRetList(cmd) @@ -134,7 +138,7 @@ class TDTestCase(TBase): # check can pass if overCnt > 0: - tdLog.exit(f"s3 have {overCnt} files over size.") + self.exit(f"s3 have {overCnt} files over size.") def doAction(self): @@ -159,7 +163,7 @@ class TDTestCase(TBase): return True time.sleep(1) - tdLog.exit(f"stream count is not expect . expect = 100000 or 100001 real={count} . sql={sql}") + self.exit(f"stream count is not expect . expect = 100000 or 100001 real={count} . sql={sql}") def checkCreateDb(self, keepLocal, chunkSize, compact): diff --git a/tests/script/api/stmt2-geometry-test.c b/tests/script/api/stmt2-geometry-test.c new file mode 100644 index 0000000000..46fd9081ae --- /dev/null +++ b/tests/script/api/stmt2-geometry-test.c @@ -0,0 +1,107 @@ +#include +#include +#include +#include "taos.h" + +int8_t byteArray[21] = {0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, 0x40}; +int8_t worngArray[21] = {0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, 0x40}; + +void do_query(TAOS* taos, const char* sql) { + printf("[sql]%s\n", sql); + TAOS_RES* result = taos_query(taos, sql); + int code = taos_errno(result); + if (code) { + printf(" failed to query: %s, reason:%s\n", sql, taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); +} + +void execute_test(TAOS* taos, const char* tbname1, const char* tbname2, int8_t* tag2, int8_t* col2, + const char* case_desc, int size) { + // prepare stmt + TAOS_STMT2_OPTION option = {0, true, false, NULL, NULL}; + TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); + const char* sql; + if (tbname1 == "tb41") { + sql = "insert into db.? using db.stb2 tags(?, ?) values(?,?)"; + } else { + sql = "insert into db.? using db.stb tags(?, ?) values(?,?)"; + } + int code = taos_stmt2_prepare(stmt, sql, 0); + printf("\n%s\n insert into db.? using db.stb tags(?, ?) values(?,?)\n", case_desc); + if (code != 0) { + printf(" failed to execute taos_stmt2_prepare. error:%s\n", taos_stmt2_error(stmt)); + taos_stmt2_close(stmt); + return; + } + + // prepare data + int t1_val = 0; + int64_t ts = 1591060628000; + const char* tbname[2] = {tbname1, tbname2}; + int32_t length[5] = {sizeof(int), 2, sizeof(int64_t), size, 20, sizeof(col2)}; + + TAOS_STMT2_BIND tags[2][2] = { + {{TSDB_DATA_TYPE_INT, &t1_val, &length[0], NULL, 2}, {TSDB_DATA_TYPE_GEOMETRY, tag2, &length[3], NULL, 2}}, + {{TSDB_DATA_TYPE_INT, &t1_val, &length[0], NULL, 2}, {TSDB_DATA_TYPE_GEOMETRY, tag2, &length[3], NULL, 2}}}; + TAOS_STMT2_BIND params[2][2] = { + {{TSDB_DATA_TYPE_TIMESTAMP, &ts, &length[2], NULL, 1}, {TSDB_DATA_TYPE_GEOMETRY, col2, &length[3], NULL, 1}}, + {{TSDB_DATA_TYPE_TIMESTAMP, &ts, &length[2], NULL, 1}, {TSDB_DATA_TYPE_GEOMETRY, col2, &length[3], NULL, 1}}}; + TAOS_STMT2_BIND* tagv[2] = {&tags[0][0], &tags[1][0]}; + TAOS_STMT2_BIND* paramv[2] = {¶ms[0][0], ¶ms[1][0]}; + + TAOS_STMT2_BINDV bindv = {2, &tbname[0], &tagv[0], ¶mv[0]}; + code = taos_stmt2_bind_param(stmt, &bindv, -1); + if (code != 0) { + printf(" failed to bind param. error:%s\n", taos_stmt2_error(stmt)); + taos_stmt2_close(stmt); + return; + } + + if (taos_stmt2_exec(stmt, NULL)) { + printf(" failed to execute insert statement.error:%s\n", taos_stmt2_error(stmt)); + taos_stmt2_close(stmt); + return; + } + printf("[ok]\n"); + + taos_stmt2_close(stmt); +} + +void test1(TAOS* taos) { execute_test(taos, "tb11", "tb12", &byteArray[0], &byteArray[0], "[normal]case 1", 21); } + +void test2(TAOS* taos) { + execute_test(taos, "tb21", "tb22", &worngArray[0], &byteArray[0], "[wrong WKB tag]case 2", 21); +} + +void test3(TAOS* taos) { + execute_test(taos, "tb31", "tb32", "POLYGON((0 0, 4 0, 4 4, 0 4, 0 0))", "POLYGON((0 0, 4 0, 4 4, 0 4, 0 0))", + "[wrong WKT col]case 3", sizeof("POLYGON((0 0, 4 0, 4 4, 0 4, 0 0))")); +} + +void test4(TAOS* taos) { execute_test(taos, "tb41", "tb42", &byteArray[0], &byteArray[0], "[wrong size]case 4", 21); } + +int main() { + TAOS* taos = taos_connect("localhost", "root", "taosdata", "", 0); + if (!taos) { + printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); + exit(1); + } + // init test db & stb table + do_query(taos, "drop database if exists db"); + do_query(taos, "create database db"); + do_query(taos, "create table db.stb (ts timestamp, b geometry(100)) tags(t1 int, t2 geometry(100))"); + do_query(taos, "create table db.stb2 (ts timestamp, b geometry(100)) tags(t1 int, t2 geometry(10))"); + + test1(taos); + test2(taos); + test3(taos); + test4(taos); + + taos_close(taos); + taos_cleanup(); +} diff --git a/tests/script/tsim/compress/compress2.sim b/tests/script/tsim/compress/compress2.sim index 0af6f87de4..179317dfbb 100644 --- a/tests/script/tsim/compress/compress2.sim +++ b/tests/script/tsim/compress/compress2.sim @@ -38,7 +38,7 @@ sql alter table $tb modify column b level 'm' sql_error alter table $tb modify column b level 'l' # already exist -sql_error alter table $tb modify column b compress 'lz4' +sql alter table $tb modify column b compress 'lz4' sql alter table $tb modify column b compress 'xz' sql alter table $tb modify column b compress 'zstd' sql_error alter table $tb modify column b compress 'tsz' @@ -147,7 +147,7 @@ sql alter table $stb modify column b level 'm' sql_error alter table $stb modify column b level 'l' # already exist sql desc $stb -sql_error alter table $stb modify column b compress 'lz4' +sql alter table $stb modify column b compress 'lz4' sql alter table $stb modify column b compress 'xz' sql alter table $stb modify column b compress 'zstd' sql_error alter table $stb modify column b compress 'tsz' diff --git a/tests/script/tsim/db/basic1.sim b/tests/script/tsim/db/basic1.sim index 8eb6dce759..f3239957d3 100644 --- a/tests/script/tsim/db/basic1.sim +++ b/tests/script/tsim/db/basic1.sim @@ -53,6 +53,8 @@ if $rows != 5 then return -1 endi +sleep 500 + print =============== show vgroups2 sql show d2.vgroups if $rows != 2 then @@ -126,13 +128,14 @@ if $data12 != d2 then endi if $data13 != leader then + print expect leader , actual $13 return -1 endi -print $data14 -print $data15 +print $data14 , $data15 if $data16 != 1 then + print expect 1, acutal $data16 return -1 endi diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index cd70dd88f5..117f9fa2e1 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -65,12 +65,12 @@ static void msg_process(TAOS_RES* msg) { if (g_fp && strcmp(result, "") != 0) { // RES_TYPE__TMQ_BATCH_META if ((*(int8_t*)msg) == 5) { - cJSON* pJson = cJSON_Parse(result); - cJSON* pJsonArray = cJSON_GetObjectItem(pJson, "metas"); + cJSON* pJson = cJSON_Parse(result); + cJSON* pJsonArray = cJSON_GetObjectItem(pJson, "metas"); int32_t num = cJSON_GetArraySize(pJsonArray); for (int32_t i = 0; i < num; i++) { cJSON* pJsonItem = cJSON_GetArrayItem(pJsonArray, i); - char* itemStr = cJSON_PrintUnformatted(pJsonItem); + char* itemStr = cJSON_PrintUnformatted(pJsonItem); taosFprintfFile(g_fp, itemStr); tmq_free_json_meta(itemStr); taosFprintfFile(g_fp, "\n"); @@ -489,10 +489,11 @@ int buildStable(TAOS* pConn, TAOS_RES* pRes) { } taos_free_result(pRes); #else - pRes = taos_query(pConn, - "create stream meters_summary_s trigger at_once IGNORE EXPIRED 0 fill_history 1 into meters_summary as select " - "_wstart, max(current) as current, " - "groupid, location from meters partition by groupid, location interval(10m)"); + pRes = taos_query( + pConn, + "create stream meters_summary_s trigger at_once IGNORE EXPIRED 0 fill_history 1 into meters_summary as select " + "_wstart, max(current) as current, " + "groupid, location from meters partition by groupid, location interval(10m)"); if (taos_errno(pRes) != 0) { printf("failed to create super table meters_summary, reason:%s\n", taos_errstr(pRes)); return -1; @@ -632,8 +633,8 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "enable.auto.commit", "true"); tmq_conf_set(conf, "auto.offset.reset", "earliest"); tmq_conf_set(conf, "msg.consume.excluded", "1"); -// tmq_conf_set(conf, "session.timeout.ms", "1000000"); -// tmq_conf_set(conf, "max.poll.interval.ms", "20000"); + // tmq_conf_set(conf, "session.timeout.ms", "1000000"); + // tmq_conf_set(conf, "max.poll.interval.ms", "20000"); if (g_conf.snapShot) { tmq_conf_set(conf, "experimental.snapshot.enable", "true"); @@ -722,7 +723,7 @@ void initLogFile() { "\"level\":\"medium\"},{" "\"name\":\"groupid\",\"type\":4,\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\"," "\"level\":\"medium\"},{\"name\":" - "\"location\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"lz4\"," + "\"location\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"zstd\"," "\"level\":\"medium\"}],\"tags\":[{\"name\":\"group_id\"," "\"type\":14}" "]}", @@ -750,7 +751,7 @@ void initLogFile() { "\"level\":\"medium\"}" ",{" "\"name\":\"c3\",\"type\":8,\"length\":64,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":" - "\"lz4\",\"level\":\"medium\"},{" + "\"zstd\",\"level\":\"medium\"},{" "\"name\":\"c4\",\"type\":5,\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\",\"level\":" "\"medium\"}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":" "\"t3\"," @@ -772,7 +773,7 @@ void initLogFile() { "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"columns\":[{\"name\":\"ts\"," "\"type\":9," "\"isPrimarykey\":false,\"encode\":\"delta-i\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":\"c2\"," - "\"type\":10,\"length\":8,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"lz4\",\"level\":" + "\"type\":10,\"length\":8,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"zstd\",\"level\":" "\"medium\"},{\"name\":\"cc3\",\"type\":5," "\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\",\"level\":\"medium\"}],\"tags\":[]}", "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"jt\",\"columns\":[{\"name\":\"ts\"," @@ -794,7 +795,7 @@ void initLogFile() { "\"medium\"},{\"name\":\"c2\",\"type\":6,\"isPrimarykey\":" "false,\"encode\":\"delta-d\",\"compress\":\"lz4\",\"level\":\"medium\"},{" "\"name\":\"c3\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":" - "\"lz4\",\"level\":\"medium\"}]," + "\"zstd\",\"level\":\"medium\"}]," "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\"," "\"type\":" "1}]}", @@ -806,7 +807,7 @@ void initLogFile() { "\"name\":\"c2\",\"type\":6," "\"isPrimarykey\":false,\"encode\":\"delta-d\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":" "\"c3\"," - "\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"lz4\",\"level\":" + "\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"zstd\",\"level\":" "\"medium\"}]," "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\"," "\"type\":" @@ -863,7 +864,7 @@ void initLogFile() { "\"level\":\"medium\"},{\"name\":\"c2\",\"type\":6,\"isPrimarykey\":false,\"encode\":\"delta-d\"," "\"compress\":\"lz4\",\"level\":\"medium\"},{" "\"name\":\"c3\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":" - "\"lz4\",\"level\":\"medium\"}]," + "\"zstd\",\"level\":\"medium\"}]," "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\"," "\"type\":" "1}]}", @@ -883,7 +884,8 @@ void initLogFile() { "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\"," "\"colType\":5}", "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":6,\"colName\":\"c4\"}", - "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\",\"colType\":5}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\"," + "\"colType\":5}", "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":7,\"colName\":\"c3\"," "\"colType\":8,\"colLength\":64}", "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":1,\"colName\":\"t2\"," @@ -896,7 +898,7 @@ void initLogFile() { "9,\"isPrimarykey\":false,\"encode\":\"delta-i\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":" "\"c1\",\"type\":4,\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\",\"level\":" "\"medium\"},{\"name\":\"c2\",\"type\":10,\"length\":4," - "\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"lz4\",\"level\":\"medium\"}],\"tags\":[]}", + "\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"zstd\",\"level\":\"medium\"}],\"tags\":[]}", "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":5,\"colName\":\"c3\"," "\"colType\":5}", "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":7,\"colName\":\"c2\"," @@ -921,7 +923,7 @@ void initLogFile() { "{\"name\":\"c1\",\"type\":4,\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\",\"level\":" "\"medium\"},{\"name\":\"c2\",\"type\":6,\"isPrimarykey\":" "false,\"encode\":\"delta-d\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":\"c3\",\"type\":8," - "\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"lz4\",\"level\":\"medium\"}]," + "\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"zstd\",\"level\":\"medium\"}]," "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\"," "\"type\":1}]}", "{\"type\":\"drop\",\"tableType\":\"super\",\"tableName\":\"st1\"}", @@ -931,7 +933,7 @@ void initLogFile() { "\"level\":\"medium\"},{\"name\":\"c2\",\"type\":6,\"isPrimarykey\":" "false,\"encode\":\"delta-d\",\"compress\":\"lz4\",\"level\":\"medium\"},{" "\"name\":\"c3\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":" - "\"lz4\",\"level\":\"medium\"}]," + "\"zstd\",\"level\":\"medium\"}]," "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\"," "\"type\":" "1}]}", @@ -941,7 +943,7 @@ void initLogFile() { "\"c1\",\"type\":4,\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\",\"level\":" "\"medium\"},{\"name\":\"c2\",\"type\":6," "\"isPrimarykey\":false,\"encode\":\"delta-d\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":\"c3\"," - "\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"lz4\",\"level\":" + "\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"zstd\",\"level\":" "\"medium\"}]," "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\"," "\"type\":" @@ -985,7 +987,8 @@ void initLogFile() { "\"lz4\",\"level\":\"medium\"},{" "\"name\":\"groupid\",\"type\":4,\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\"," "\"level\":\"medium\"},{\"name\":" - "\"location\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"lz4\"," + "\"location\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":" + "\"zstd\"," "\"level\":\"medium\"}],\"tags\":[{\"name\":\"group_id\"," "\"type\":" "14}]}", @@ -1012,7 +1015,7 @@ void initLogFile() { "\"level\":\"medium\"},{\"name\":\"c2\",\"type\":6,\"isPrimarykey\":" "false,\"encode\":\"delta-d\",\"compress\":\"lz4\",\"level\":\"medium\"},{" "\"name\":\"c3\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":" - "\"lz4\",\"level\":\"medium\"}]," + "\"zstd\",\"level\":\"medium\"}]," "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\"," "\"type\":" "1}]}", @@ -1032,7 +1035,8 @@ void initLogFile() { "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\"," "\"colType\":5}", "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":6,\"colName\":\"c4\"}", - "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\",\"colType\":5}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\"," + "\"colType\":5}", "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":7,\"colName\":\"c3\"," "\"colType\":8,\"colLength\":64}", "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":1,\"colName\":\"t2\"," @@ -1044,7 +1048,7 @@ void initLogFile() { "9,\"isPrimarykey\":false,\"encode\":\"delta-i\",\"compress\":\"lz4\",\"level\":\"medium\"}" ",{\"name\":\"c1\",\"type\":4,\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\"," "\"level\":\"medium\"},{\"name\":\"c2\",\"type\":10,\"length\":4," - "\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"lz4\",\"level\":\"medium\"}],\"tags\":[]}", + "\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"zstd\",\"level\":\"medium\"}],\"tags\":[]}", "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":5,\"colName\":\"c3\"," "\"colType\":5}", "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":7,\"colName\":\"c2\"," @@ -1069,7 +1073,7 @@ void initLogFile() { "\"level\":\"medium\"},{\"name\":\"c2\",\"type\":6,\"isPrimarykey\":" "false,\"encode\":\"delta-d\",\"compress\":\"lz4\",\"level\":\"medium\"},{" "\"name\":\"c3\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":" - "\"lz4\",\"level\":\"medium\"}]," + "\"zstd\",\"level\":\"medium\"}]," "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\"," "\"type\":" "1}]}", @@ -1079,7 +1083,7 @@ void initLogFile() { "\"c1\",\"type\":4,\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\",\"level\":" "\"medium\"},{\"name\":\"c2\",\"type\":6," "\"isPrimarykey\":false,\"encode\":\"delta-d\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":\"c3\"," - "\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"lz4\",\"level\":" + "\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"zstd\",\"level\":" "\"medium\"}]," "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\"," "\"type\":"