Unable to

.github/pull_request_template.md

@@ -0,0 +1,5 @@
+# Pull Request Checklist
+
+- [ ] Is the user manual updated?
+- [ ] Are the test cases passed and automated?
+- [ ] Is there no significant decrease in test coverage?

Jenkinsfile2

@@ -5,7 +5,7 @@ node {
 }
 file_zh_changed = ''
 file_en_changed = ''
-file_no_doc_changed = ''
+file_no_doc_changed = '1'
 def abortPreviousBuilds() {
     def currentJobName = env.JOB_NAME
     def currentBuildNumber = env.BUILD_NUMBER.toInteger()
@@ -355,7 +355,7 @@ def pre_test_build_win() {
     bat '''
         cd %WIN_COMMUNITY_ROOT%/tests/ci
         pip3 install taospy==2.7.16 
-        pip3 install taos-ws-py==0.3.3
+        pip3 install taos-ws-py==0.3.5
         xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32
     '''
     return 1
@@ -656,4 +656,4 @@ pipeline {
             )
         }
     } 
-}
+}

docs/en/14-reference/03-taos-sql/31-compress.md

@@ -30,11 +30,12 @@ In this article, it specifically refers to the level within the secondary compre
 
 | Data Type |   Optional Encoding Algorithm      |  Default Encoding Algorithm  | Optional Compression Algorithm|Default Compression Algorithm| Default Compression Level|
 | :-----------:|:----------:|:-------:|:-------:|:----------:|:----:|
-|  tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium|
+|  int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium|
+|  tinyint/untinyint/smallint/usmallint | simple8b| simple8b | lz4/zlib/zstd/xz| zlib| medium|
 |   bigint/ubigint/timestamp   |  simple8b/delta-i    | delta-i |lz4/zlib/zstd/xz | lz4| medium|
 |float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|lz4| medium|
-|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| medium|
+|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| zstd|
-|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| medium|
+|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| zstd|
 
 ## SQL
 

docs/en/14-reference/05-connectors/30-python.mdx

@@ -41,12 +41,18 @@ We recommend using the latest version of `taospy`, regardless of the version of
 
 |Python Client Library Version|major changes|
 |:-------------------:|:----:|
+|2.7.16|add subscription configuration (session.timeout.ms, max.poll.interval.ms)|
+|2.7.15|added support for VARBINARY and GEOMETRY types|
+|2.7.14|fix known issues|
+|2.7.13|add TMQ synchronous submission offset interface|
 |2.7.12|1. added support for `varbinary` type (STMT does not yet support)<br/> 2. improved query performance (thanks to contributor [hadrianl](https://github.com/taosdata/taos-connector-python/pull/209))|
 |2.7.9|support for getting assignment and seek function on subscription|
 |2.7.8|add `execute_many` method|
 
 |Python Websocket Connection Version|major changes|
 |:----------------------------:|:-----:|
+|0.3.5|1. added support for VARBINARY and GEOMETRY types <br/> 2. Fix known issues|
+|0.3.2|1. optimize WebSocket SQL query and insertion performance <br/> 2. Fix known issues <br/> 3. Modify the readme and document|
 |0.2.9|bugs fixes|
 |0.2.5|1. support for getting assignment and seek function on subscription <br/> 2. support schemaless <br/> 3. support STMT|
 |0.2.4|support `unsubscribe` on subscription|

docs/en/14-reference/05-connectors/35-node.mdx

@@ -27,6 +27,8 @@ Node.js client library needs to be run with Node.js 14 or higher version.
 
 | Node.js connector version |    major changes     |   TDengine 版本    |
 | :-----------------------: | :------------------: | :----------------:|
+|        3.1.2              |  Optimized the data protocol and parsing, resulting in a significant improvement in performance |  3.2.0.0 or later |
+|        3.1.1              |  Optimized data transmission performance |  3.2.0.0 or later |
 |        3.1.0              |  new version, supports websocket |  3.2.0.0 or later |
 
 ## Supported features

docs/en/14-reference/12-config/index.md

@@ -773,7 +773,7 @@ lossyColumns     float|double
    02/22 10:49:27.607990 00002933 UTL  lossyColumns     float|double
 ```
 
-### ifAdtFse 
+### ifAdtFse
 
 | Attribute     | Description                         |
 | -------- | -------------------------------- |
@@ -898,4 +898,4 @@ lossyColumns     float|double
 | 53  |          udf           | Yes                    | Yes                          |                         |
 | 54  |     enableCoreFile     | Yes                    | Yes                          |                         |
 | 55  |    ttlChangeOnWrite    | No                     | Yes                          |                         |
-| 56  |     keepTimeOffset     | Yes                    | Yes(discarded since 3.2.0.0) |                         |
+| 56  |     keepTimeOffset     | Yes                    | Yes(discarded since 3.2.0.0) | see "KEEP_TIME_OFFSET"  |

docs/examples/node/package.json

@@ -4,6 +4,12 @@
   "main": "index.js",
   "license": "MIT",
   "dependencies": {
-    "@tdengine/websocket": "^3.1.1"
+    "@tdengine/websocket": "^3.1.2"
-  }
+  },
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "description": ""
 }

docs/examples/node/websocketexample/all_type_stmt.js

@@ -95,8 +95,8 @@ async function all_type_example() {
         tagParams.setBoolean([true]);
         tagParams.setVarchar(["hello"]);
         tagParams.setNchar(["stmt"]);
-        tagParams.setGeometry([geometryData]);
         tagParams.setVarBinary([vbData]);
+        tagParams.setGeometry([geometryData]);
         await stmt.setTags(tagParams);
 
 
@@ -108,8 +108,8 @@ async function all_type_example() {
         bindParams.setBoolean([true]);
         bindParams.setVarchar(["hello"]);
         bindParams.setNchar(["stmt"]);
-        bindParams.setGeometry([geometryData]);
         bindParams.setVarBinary([vbData]);
+        bindParams.setGeometry([geometryData]);
 
         await stmt.bind(bindParams);
         await stmt.batch();

docs/zh/06-advanced/03-stream.md

@@ -124,7 +124,7 @@ create stream if not exists count_history_s fill_history 1 into count_history as
 
 窗口关闭是由事件时间决定的，如事件流中断、或持续延迟，此时事件时间无法更新，可能导致无法得到最新的计算结果。
 
-因此，流计算提供了以事件时间结合处理时间计算的 MAX_DELAY 触发模式。MAX_DELAY 模式在窗口关闭时会立即触发计算。此外，当数据写入后，计算触发的时间超过 max delay 指定的时间，则立即触发计算。
+因此，流计算提供了以事件时间结合处理时间计算的 MAX_DELAY 触发模式：MAX_DELAY 模式在窗口关闭时会立即触发计算，它的单位可以自行指定，具体单位：a（毫秒）、s（秒）、m（分）、h（小时）、d（天）、w（周）。此外，当数据写入后，计算触发的时间超过 MAX_DELAY 指定的时间，则立即触发计算。
 
 ### 流计算的窗口关闭
 
@@ -259,4 +259,4 @@ flush database test1;
 
 5.修改 taos.cfg，去掉 disableStream 1，或将 disableStream 改为 0
 
-6.启动 taosd
+6.启动 taosd

docs/zh/06-advanced/05-data-in/05-opcua.md

@@ -150,7 +150,7 @@ CSV 文件中的每个 Row 配置一个 OPC 数据点位。Row 的规则如下
 
 #### 5.2. 选择数据点位
 
-可以通过配置 **根节点ID**、**命名空间**、**正则匹配** 等条件，对点位进行筛选。
+可以通过配置 **根节点ID**、**命名空间**、**节点ID**、**节点名称** 等条件，对点位进行筛选。
 
 通过配置 **超级表名**、**表名称**，指定数据要写入的超级表、子表。
 

docs/zh/06-advanced/05-data-in/06-opcda.md

@@ -126,7 +126,7 @@ CSV 文件中的每个 Row 配置一个 OPC 数据点位。Row 的规则如下
 
 #### 4.2. 选择数据点位
 
-可以通过配置 **根节点ID** 和 **正则匹配** 作为过滤条件，对点位进行筛选。
+可以通过配置 **根节点ID**、**节点ID**、**节点名称** 作为过滤条件，对点位进行筛选。
 
 通过配置 **超级表名**、**表名称**，指定数据要写入的超级表、子表。
 

docs/zh/06-advanced/06-TDgpt/02-management.md

@@ -5,6 +5,7 @@ sidebar_label: "安装部署"
 
 ### 环境准备
 使用 TDgpt 的高级时序数据分析功能需要在 TDengine 集群中安装部署 AI node（Anode）。Anode 可以运行在 Linux/Windows/MacOS 等平台上，同时需要 3.10 或以上版本的 Python 环境支持。
+> 部署 Anode 需要 TDengine Enterprise 3.3.4.3 及以后版本，请首先确认搭配 Anode 使用的 TDengine 能够支持 Anode。
 
 ### 安装及卸载
 不同操作系统上安装及部署 Anode 有一些差异，主要是卸载操作、安装路径、服务启停等方面。本文以 Linux 系统为例，说明安装部署的流程。
@@ -20,7 +21,7 @@ sudo ./install.sh
 为了避免影响系统已有的 Python 环境，Anode 使用虚拟环境运行。安装 Anode 会在目录 `/var/lib/taos/taosanode/venv/` 中创建默认的 Python 虚拟环境，Anode 运行所需要的库均安装在该目录下。为了避免反复安装虚拟环境带来的开销，卸载命令 `rmtaosanode` 并不会自动删除该虚拟环境，如果您确认不再需要 Python 的虚拟环境，手动删除该目录即可。
 
 ### 启停服务
-在 Linux 系统中，安装 Anode 以后会自动创建 `taosadnoded` 服务。可以使用 `systemd` 来管理 Anode 服务，使用如下命令启动/停止/检查 Anode。
+在 Linux 系统中，安装 Anode 以后会自动创建 `taosanoded` 服务。可以使用 `systemd` 来管理 Anode 服务，使用如下命令启动/停止/检查 Anode。
 
 ```bash
 systemctl start  taosanoded
@@ -48,8 +49,8 @@ Anode 的服务需要使用 uWSGI 驱动驱动运行，因此 Anode 和 uWSGI
 ```ini
 [uwsgi]
 
-# Anode HTTP service ip:port
+# Anode RESTful service ip:port
-http = 127.0.0.1:6050
+http = 127.0.0.1:6090
 
 # base directory for Anode python files， do NOT modified this
 chdir = /usr/local/taos/taosanode/lib
@@ -95,11 +96,12 @@ Anode 运行配置主要是以下：
 
 
 ### Anode 基本操作
+对于 Anode 的管理，用户需要通过 TDengine 的命令行接口 taos 进行。因此下述介绍的管理命令都需要先打开 taos, 连接到 TDengine 运行实例。 
 #### 创建 Anode
 ```sql 
 CREATE ANODE {node_url}
 ```
-node_url 是提供服务的 Anode 的 IP 和 PORT组成的字符串, 例如：`create anode 'http://localhost:6050'`。Anode 启动后还需要注册到 TDengine 集群中才能提供服务。不建议将 Anode 同时注册到两个集群中。
+node_url 是提供服务的 Anode 的 IP 和 PORT组成的字符串, 例如：`create anode '127.0.0.1:6090'`。Anode 启动后还需要注册到 TDengine 集群中才能提供服务。不建议将 Anode 同时注册到两个集群中。
 
 #### 查看 Anode
 列出集群中所有的数据分析节点，包括其 `FQDN`, `PORT`, `STATUS`等属性。
@@ -115,7 +117,7 @@ SHOW ANODES FULL;
 
 #### 刷新集群中的分析算法缓存
 ```SQL
-UPDATE ANODE {node_id}
+UPDATE ANODE {anode_id}
 UPDATE ALL ANODES
 ```
 

docs/zh/06-advanced/06-TDgpt/03-preprocess.md

@@ -7,12 +7,12 @@ import activity from './pic/activity.png';
 import wndata from './pic/white-noise-data.png'
 
 ### 分析流程
-时序数据分析之前需要有预处理的过程，整体的流程如下图所示。
+时序数据分析之前需要有预处理的过程，为减轻分析算法的负担，TDgpt 在将时序数据发给具体分析算法进行分析时，已经对数据做了预处理，整体的流程如下图所示。
 
 <img src={activity} width="560" alt="预处理流程" />
 
-首先对输入数据进行白噪声检查（White Noise Data check）, 检查通过以后针对预测分析，还需要进行输入（历史）数据的重采样和时间戳对齐处理操作，如果是异常检测则跳过数据重采样和时间戳对齐的预处理。
+TDgpt 首先对输入数据进行白噪声检查（White Noise Data check）, 检查通过以后针对预测分析，还要进行输入（历史）数据的重采样和时间戳对齐处理（异常检测跳过数据重采样和时间戳对齐步骤）。
-预处理完成以后，在进行后续的预测或异常检测操作。
+预处理完成以后，再进行预测或异常检测操作。预处理过程部署于预测或异常检测处理逻辑的一部分。
 
 ### 白噪声检查
 

docs/zh/06-advanced/06-TDgpt/04-forecast/02-arima.md

@@ -38,6 +38,11 @@ ARIMA 模型是一种自回归模型，只需要自变量即可预测后续的
 FORECAST(i32, "algo=arima,alpha=95,period=10,start_p=1,max_p=5,start_q=1,max_q=5")
 ```
 
+完整的调用SQL语句如下：
+```SQL
+SELECT _frowts, FORECAST(i32, "algo=arima,alpha=95,period=10,start_p=1,max_p=5,start_q=1,max_q=5") from foo
+```
+
 ```json5
 {
 "rows": fc_rows,  // 返回结果的行数

docs/zh/06-advanced/06-TDgpt/04-forecast/03-holtwinters.md

@@ -23,11 +23,16 @@ HoltWinters 有两种不同的季节性组成部分，当季节变化在该时
 参数 `trend` 和 `seasonal`的均可以选择 `add` （加法模型）或 `mul`（乘法模型）。
 
 ### 示例及结果
-针对 i32 列进行数据预测，输入列 i32 每 10 个点是一个周期，趋势采用乘法模型，季节采用乘法模型
+针对 i32 列进行数据预测，输入列 i32 每 10 个点是一个周期，趋势参数采用乘法模型，季节参数采用乘法模型
 ```
 FORECAST(i32, "algo=holtwinters,period=10,trend=mul,seasonal=mul")
 ```
 
+完整的调用SQL语句如下：
+```SQL
+SELECT _frowts, FORECAST(i32, "algo=holtwinters, peroid=10,trend=mul,seasonal=mul") from foo
+```
+
 ```json5
 {
 "rows": rows,         // 返回结果的行数

docs/zh/06-advanced/06-TDgpt/04-forecast/index.md

@@ -3,7 +3,28 @@ title: 预测算法
 description: 预测算法
 ---
 
-时序数据预测处理以持续一个时间段的时序数据作为输入，预测接下来一个连续时间区间内时间序列数据趋势。用户可以指定输出的（预测）时间序列数据点的数量，因此其输出的结果行数不确定。为此，TDengine 使用新函数 `FORECAST` 提供时序数据预测服务。基础数据（用于预测的历史时间序列数据）是该函数的输入，预测结果是该函数的输出。用户可以通过 `FORECAST` 函数调用 Anode 提供的预测算法提供的服务。
+时序数据预测处理以持续一个时间段的时序数据作为输入，预测接下来一个连续时间区间内时间序列数据趋势。用户可以指定输出的（预测）时间序列数据点的数量，因此其输出的结果行数不确定。为此，TDengine 使用新 SQL 函数 `FORECAST` 提供时序数据预测服务。基础数据（用于预测的历史时间序列数据）是该函数的输入，预测结果是该函数的输出。用户可以通过 `FORECAST` 函数调用 Anode 提供的预测算法提供的服务。
+
+在后续章节中，使用时序数据表`foo`作为示例，介绍预测和异常检测算法的使用方式，`foo` 表的模式如下：
+
+|列名称|类型|说明|
+|---|---|---|
+|ts| timestamp| 主时间戳列|
+|i32| int32| 4字节整数，设备测量值 metric|
+
+```bash
+taos> select * from foo;
+           ts            |      k      |
+========================================
+ 2020-01-01 00:00:12.681 |          13 |
+ 2020-01-01 00:00:13.727 |          14 |
+ 2020-01-01 00:00:14.378 |           8 |
+ 2020-01-01 00:00:15.774 |          10 |
+ 2020-01-01 00:00:16.170 |          16 |
+ 2020-01-01 00:00:17.558 |          26 |
+ 2020-01-01 00:00:18.938 |          32 |
+ 2020-01-01 00:00:19.308 |          27 |
+```
 
 ### 语法
 ```SQL
@@ -54,19 +75,19 @@ FROM foo;
 taos> select _flow, _fhigh, _frowts, forecast(i32) from foo;
         _flow         |        _fhigh        |       _frowts           | forecast(i32) |
 ========================================================================================
-           10.5286684 |           41.8038254 | 2020-01-01 00:01:35.001 |            26 |
+           10.5286684 |           41.8038254 | 2020-01-01 00:01:35.000 |            26 |
-          -21.9861946 |           83.3938904 | 2020-01-01 00:01:36.001 |            30 |
+          -21.9861946 |           83.3938904 | 2020-01-01 00:01:36.000 |            30 |
-          -78.5686035 |          144.6729126 | 2020-01-01 00:01:37.001 |            33 |
+          -78.5686035 |          144.6729126 | 2020-01-01 00:01:37.000 |            33 |
-         -154.9797363 |          230.3057709 | 2020-01-01 00:01:38.001 |            37 |
+         -154.9797363 |          230.3057709 | 2020-01-01 00:01:38.000 |            37 |
-         -253.9852905 |          337.6083984 | 2020-01-01 00:01:39.001 |            41 |
+         -253.9852905 |          337.6083984 | 2020-01-01 00:01:39.000 |            41 |
-         -375.7857971 |          466.4594727 | 2020-01-01 00:01:40.001 |            45 |
+         -375.7857971 |          466.4594727 | 2020-01-01 00:01:40.000 |            45 |
-         -514.8043823 |          622.4426270 | 2020-01-01 00:01:41.001 |            53 |
+         -514.8043823 |          622.4426270 | 2020-01-01 00:01:41.000 |            53 |
-         -680.6343994 |          796.2861328 | 2020-01-01 00:01:42.001 |            57 |
+         -680.6343994 |          796.2861328 | 2020-01-01 00:01:42.000 |            57 |
-         -868.4956665 |          992.8603516 | 2020-01-01 00:01:43.001 |            62 |
+         -868.4956665 |          992.8603516 | 2020-01-01 00:01:43.000 |            62 |
-        -1076.1566162 |         1214.4498291 | 2020-01-01 00:01:44.001 |            69 |
+        -1076.1566162 |         1214.4498291 | 2020-01-01 00:01:44.000 |            69 |
 ```
 
 
 ## 内置预测算法
-- arima
+- [arima](./02-arima.md)
-- holtwinters
+- [holtwinters](./03-holtwinters.md)

docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/02-statistics-approach.md

@@ -1,46 +1,57 @@
----
+---
-title: "检测算法"
+title: "统计学算法"
-sidebar_label: "检测算法"
+sidebar_label: "统计学算法"
----
+---
-
+
-本节介绍内置异常检测算法模型的定义和使用方法。
+- k-sigma<sup>[1]</sup>: 即 ***68–95–99.7 rule*** 。***k***值默认为 3，即序列均值的 3 倍标准差范围为边界，超过边界的是异常值。KSigma 要求数据整体上服从正态分布，如果一个点偏离均值 K 倍标准差，则该点被视为异常点.
-
+
-## 概述
+|参数|说明|是否必选|默认值|
-分析平台内置了6个异常检查模型，分为3个类别，分别是基于统计学的模型、基于数据密度的模型、以及基于深度学习的模型。在不指定异常检测使用的方法的情况下，默认调用 iqr 进行异常检测。
+|---|---|---|---|
-
+|k|标准差倍数|选填|3|
-
+
-### 统计学异常检测方法
+```SQL
-
+--- 指定调用的算法为ksigma, 参数 k 为 2
-- k-sigma<sup>[1]</sup>: 即 ***68–95–99.7 rule*** 。***k***值默认为 3，即序列均值的 3 倍标准差范围为边界，超过边界的是异常值。KSigma 要求数据整体上服从正态分布，如果一个点偏离均值 K 倍标准差，则该点被视为异常点.
+SELECT _WSTART, COUNT(*)
-  
+FROM foo
-|参数|说明|是否必选|默认值|
+ANOMALY_WINDOW(foo.i32, "algo=ksigma,k=2")
-|---|---|---|---|
+```
-|k|标准差倍数|选填|3|
+
-
+- IQR<sup>[2]</sup>：Interquartile range(IQR)，四分位距是一种衡量变异性的方法。四分位数将一个按等级排序的数据集划分为四个相等的部分。即 Q1（第 1 个四分位数）、Q2（第 2 个四分位数）和 Q3（第 3 个四分位数）。 $IQR=Q3-Q1$，对于 $v$, $Q1-(1.5 \times IQR) \le v \le Q3+(1.5 \times IQR)$ 是正常值，范围之外的是异常值。无输入参数。
-
+
-- IQR<sup>[2]</sup>：Interquartile range(IQR)，四分位距是一种衡量变异性的方法。四分位数将一个按等级排序的数据集划分为四个相等的部分。即 Q1（第 1 个四分位数）、Q2（第 2 个四分位数）和 Q3（第 3 个四分位数）。 $IQR=Q3-Q1$，对于 $v$, $Q1-(1.5 \times IQR) \le v \le Q3+(1.5 \times IQR)$ 是正常值，范围之外的是异常值。无输入参数。
+```SQL
-
+--- 指定调用的算法为 iqr, 无参数
-- Grubbs<sup>[3]</sup>: Grubbs' test，即最大标准残差测试。Grubbs 通常用作检验最大值、最小值偏离均值的程度是否为异常，要求单变量数据集遵循近似标准正态分布。非正态分布数据集不能使用该方法。无输入参数。
+SELECT _WSTART, COUNT(*)
-
+FROM foo
-- SHESD<sup>[4]</sup>： 带有季节性的 ESD 检测算法。ESD 可以检测时间序列数据的多异常点。需要指定异常点比例的上界***k***，最差的情况是至多 49.9%。数据集的异常比例一般不超过 5%
+ANOMALY_WINDOW(foo.i32, "algo=iqr")
-  
+```
-|参数|说明|是否必选|默认值|
+
-|---|---|---|---|
+- Grubbs<sup>[3]</sup>: Grubbs' test，即最大标准残差测试。Grubbs 通常用作检验最大值、最小值偏离均值的程度是否为异常，要求单变量数据集遵循近似标准正态分布。非正态分布数据集不能使用该方法。无输入参数。
-|k|异常点在输入数据集中占比 $1 \le K \le 49.9$ |选填|5|
+
-
+```SQL
-
+--- 指定调用的算法为 grubbs, 无参数
-### 基于数据密度的检测方法
+SELECT _WSTART, COUNT(*)
-LOF<sup>[5]</sup>: Local Outlier Factor(LOF)，局部离群因子/局部异常因子，是 Breunig 在 2000 年提出的一种基于密度的局部离群点检测算法，该方法适用于不同类簇密度分散情况迥异的数据。根据数据点周围的数据密集情况，首先计算每个数据点的一个局部可达密度，然后通过局部可达密度进一步计算得到每个数据点的一个离群因子，该离群因子即标识了一个数据点的离群程度，因子值越大，表示离群程度越高，因子值越小，表示离群程度越低。最后，输出离群程度最大的 $topK$ 个点。
+FROM foo
-
+ANOMALY_WINDOW(foo.i32, "algo=grubbs")
-
+```
-### 基于自编码器的检测方法
+
-使用自动编码器的异常检测模型。可以对具有周期性的数据具有较好的检测结果。但是使用该模型需要针对输入的时序数据进行训练，同时将训练完成的模型部署到服务目录中，才能够运行与使用。
+- SHESD<sup>[4]</sup>： 带有季节性的 ESD 检测算法。ESD 可以检测时间序列数据的多异常点。需要指定异常检测方向（'pos' / 'neg' / 'both'），异常值比例的上界***max_anoms***，最差的情况是至多 49.9%。数据集的异常比例一般不超过 5%
-
+
-
+|参数|说明|是否必选|默认值|
-### 参考文献
+|---|---|---|---|
-1. [https://en.wikipedia.org/wiki/68–95–99.7 rule](https://en.wikipedia.org/wiki/68%E2%80%9395%E2%80%9399.7_rule)
+|direction|异常检测方向类型('pos' / 'neg' / 'both')|否|"both"|
-2. https://en.wikipedia.org/wiki/Interquartile_range
+|max_anoms|异常值比例 $0 < K \le 49.9$|否|0.05|
-3. Adikaram, K. K. L. B.; Hussein, M. A.; Effenberger, M.; Becker, T. (2015-01-14). "Data Transformation Technique to Improve the Outlier Detection Power of Grubbs's Test for Data Expected to Follow Linear Relation". Journal of Applied Mathematics. 2015: 1–9. doi:10.1155/2015/708948.
+|period|一个周期包含的数据点|否|0|
-4. Hochenbaum, O. S. Vallis, and A. Kejariwal. 2017. Automatic Anomaly Detection in the Cloud Via Statistical Learning. arXiv preprint arXiv:1704.07706 (2017).
+
-5. Breunig, M. M.; Kriegel, H.-P.; Ng, R. T.; Sander, J. (2000). LOF: Identifying Density-based Local Outliers (PDF). Proceedings of the 2000 ACM SIGMOD International Conference on Management of Data. SIGMOD. pp. 93–104. doi:10.1145/335191.335388. ISBN 1-58113-217-4.
+
-
+```SQL
+--- 指定调用的算法为 shesd, 参数 direction 为 both，异常值比例 5%
+SELECT _WSTART, COUNT(*)
+FROM foo
+ANOMALY_WINDOW(foo.i32, "algo=shesd,direction=both,anoms=0.05")
+```
+
+### 参考文献
+1. [https://en.wikipedia.org/wiki/68–95–99.7 rule](https://en.wikipedia.org/wiki/68%E2%80%9395%E2%80%9399.7_rule)
+2. https://en.wikipedia.org/wiki/Interquartile_range
+3. Adikaram, K. K. L. B.; Hussein, M. A.; Effenberger, M.; Becker, T. (2015-01-14). "Data Transformation Technique to Improve the Outlier Detection Power of Grubbs's Test for Data Expected to Follow Linear Relation". Journal of Applied Mathematics. 2015: 1–9. doi:10.1155/2015/708948.
+4. Hochenbaum, O. S. Vallis, and A. Kejariwal. 2017. Automatic Anomaly Detection in the Cloud Via Statistical Learning. arXiv preprint arXiv:1704.07706 (2017).

docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/03-data-density.md

@@ -0,0 +1,20 @@
+---
+title: "数据密度算法"
+sidebar_label: "数据密度算法"
+---
+
+### 基于数据密度的检测方法
+LOF<sup>[1]</sup>: Local Outlier Factor(LOF)，局部离群因子/局部异常因子，
+是 Breunig 在 2000 年提出的一种基于密度的局部离群点检测算法，该方法适用于不同类簇密度分散情况迥异的数据。根据数据点周围的数据密集情况，首先计算每个数据点的一个局部可达密度，然后通过局部可达密度进一步计算得到每个数据点的一个离群因子，
+该离群因子即标识了一个数据点的离群程度，因子值越大，表示离群程度越高，因子值越小，表示离群程度越低。最后，输出离群程度最大的 $topK$ 个点。
+
+```SQL
+--- 指定调用的算法为LOF，即可调用该算法
+SELECT count(*)
+FROM foo
+ANOMALY_WINDOW(foo.i32, "algo=lof")
+```
+
+### 参考文献
+
+1. Breunig, M. M.; Kriegel, H.-P.; Ng, R. T.; Sander, J. (2000). LOF: Identifying Density-based Local Outliers (PDF). Proceedings of the 2000 ACM SIGMOD International Conference on Management of Data. SIGMOD. pp. 93–104. doi:10.1145/335191.335388. ISBN 1-58113-217-4.

docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/04-machine-learning.md

@@ -0,0 +1,17 @@
+---
+title: "机器学习算法"
+sidebar_label: "机器学习算法"
+---
+
+Autoencoder<sup>[1]</sup>: TDgpt 内置使用自编码器（Autoencoder）的异常检测算法，对周期性的时间序列数据具有较好的检测结果。使用该模型需要针对输入时序数据进行预训练，同时将训练完成的模型保存在到服务目录 `ad_autoencoder` 中，然后在 SQL 语句中指定调用该算法模型即可使用。
+
+```SQL
+--- 在 options 中增加 model 的名称，ad_autoencoder_foo， 针对 foo 数据集（表）训练的采用自编码器的异常检测模型进行异常检测
+SELECT COUNT(*), _WSTART
+FROM foo
+ANOMALY_DETECTION(col1, 'algo=encoder, model=ad_autoencoder_foo');
+```
+
+### 参考文献
+
+1. https://en.wikipedia.org/wiki/Autoencoder

docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/index.md

@@ -56,7 +56,7 @@ SELECT _wstart, _wend, SUM(i32)
 FROM foo
 ANOMALY_WINDOW(i32, "algo=ksigma,k=2");
 
-taos> SELECT _wstart, _wend, count(*) FROM ai.atb ANOMAYL_WINDOW(i32);
+taos> SELECT _wstart, _wend, count(*) FROM foo ANOMAYL_WINDOW(i32);
          _wstart         |          _wend          |   count(*)    |
 ====================================================================
  2020-01-01 00:00:16.000 | 2020-01-01 00:00:17.000 |             2 |
@@ -65,9 +65,5 @@ Query OK, 1 row(s) in set (0.028946s)
 
 
 ### 内置异常检测算法
-- iqr
+分析平台内置了6个异常检查模型，分为3个类别，分别是[基于统计学的算法](./02-statistics-approach.md)、[基于数据密度的算法](./03-data-density.md)、以及[基于机器学习的算法](./04-machine-learning.md)。在不指定异常检测使用的方法的情况下，默认调用 IQR 进行异常检测。
-- ksigma
+
-- grubbs
-- lof
-- shesd
-- tac

docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md

@@ -77,14 +77,8 @@ class _MyForecastService(AbstractForecastService):
 	"""该算法无需任何输入参数，直接重载父类该函数，不处理算法参数设置逻辑"""
         pass
 ```
-将该文件保存在 `./taosanalytics/algo/ad/` 目录下，然后重启 taosanode 服务。然后就可以通过 SQL 语句调用该检测算法。
 
-```SQL
+将该文件保存在 `./taosanalytics/algo/fc/` 目录下，然后重启 taosanode 服务。在 TDengine 命令行接口中执行 `SHOW ANODES FULL` 能够看到新加入的算法。应用就可以通过 SQL 语句调用该预测算法。
---- 对 col 列进行异常检测，通过指定 algo 参数为 myad 来调用新添加的异常检测类
-SELECT COUNT(*) FROM foo ANOMALY_DETECTION(col, 'algo=myad')
-```
-
-将该文件保存在 `./taosanalytics/algo/fc/` 目录下，然后重启 taosanode 服务。通过执行 `SHOW ANODES FULL` 能够看到新加入的算法，通过 SQL 语句调用该预测算法。
 
 ```SQL
 --- 对 col 列进行异常检测，通过指定 algo 参数为 myfc 来调用新添加的预测类
@@ -92,6 +86,7 @@ SELECT  _flow, _fhigh, _frowts, FORECAST(col_name, "algo=myfc")
 FROM foo;
 ```
 
+如果是第一次启动该 Anode, 请按照 [TDgpt 安装部署](../../management/) 里的步骤先将该 Anode 添加到 TDengine 系统中。
 
 ### 单元测试
 

docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md

@@ -48,13 +48,13 @@ class _MyAnomalyDetectionService(AbstractAnomalyDetectionService):
         pass
 ```
 
-将该文件保存在 `./taosanalytics/algo/ad/` 目录下，然后重启 taosanode 服务。然后就可以通过 SQL 语句调用该检测算法。
+将该文件保存在 `./taosanalytics/algo/ad/` 目录下，然后重启 taosanode 服务。在 TDengine 命令行接口 taos 中执行 `SHOW ANODES FULL` 就能够看到新加入的算法，然后应用就可以通过 SQL 语句调用该检测算法。
 
 ```SQL
 --- 对 col 列进行异常检测，通过指定 algo 参数为 myad 来调用新添加的异常检测类
 SELECT COUNT(*) FROM foo ANOMALY_DETECTION(col, 'algo=myad')
 ```
-
+如果是第一次启动该 Anode, 请按照 [TDgpt 安装部署](../../management/) 里的步骤先将该 Anode 添加到 TDengine 系统中。
 
 ### 单元测试
 

docs/zh/06-advanced/06-TDgpt/06-dev/index.md

@@ -2,13 +2,13 @@
 title: "算法开发者指南"
 sidebar_label: "算法开发者指南"
 ---
-TDgpt 是一个可扩展的时序数据高级分析平台，用户遵循简易的步骤就能将自己开发的分析算法添加到分析平台并通过SQL语句直接调用。目前 TDpgt 平台只支持使用 Python 语言开发的分析算法。
+TDgpt 是一个可扩展的时序数据高级分析平台，用户遵循简易的步骤就能将自己开发的分析算法添加到分析平台, 各种应用就可以通过SQL语句直接调用, 让高级分析算法的使用门槛降到几乎为零。目前 TDpgt 平台只支持使用 Python 语言开发的分析算法。
 Anode 采用类动态加载模式，在启动的时候扫描特定目录内满足约定条件的所有代码文件，并将其加载到系统中。因此，开发者只需要遵循以下几步就能完成新算法的添加工作：
 1. 开发完成符合要求的分析算法类
 2. 将代码文件放入对应目录，然后重启 Anode
-3. 使用SQL命令，向 Mnode 同步新的算法列表
+3. 使用SQL命令"CREATE ANODE"，将 Anode 添加到 TDengine
    
-此时就完成了新算法的添加工作，之后就可以直接使用SQL语句调用新算法。得益于 TDgpt 与 TDengine主进程 `taosd` 的松散耦合，Anode算法升级对 `taosd` 完全没有影响。应用系统只需要调整对应的SQL语句调用新（升级的）算法，就能够快速完成分析功能和分析算法的升级。
+此时就完成了新算法的添加工作，之后应用就可以直接使用SQL语句调用新算法。得益于 TDgpt 与 TDengine主进程 `taosd` 的松散耦合，Anode算法升级对 `taosd` 完全没有影响。应用系统只需要调整对应的SQL语句调用新（升级的）算法，就能够快速完成分析功能和分析算法的升级。
 
 这种方式能够按需扩展分析算法，极大地拓展 TDgpt 的适应范围，用户可以按需将更契合业务场景的、更准确的（预测、异常检测）分析算法动态嵌入到 TDgpt，并通过 SQL 语句进行调用。在基本不用更改应用系统代码的前提下，就能够快速完成分析功能的平滑升级。
 
@@ -68,20 +68,15 @@ SELECT COUNT(*) FROM foo ANOMALY_DETECTION(col_name, 'algo=name')
 
 ## 添加具有模型的分析算法
 
-基于统计学的分析算法可以直接针对输入时间序列数据进行分析，整体分析流程比较快捷，但是某些深度学习算法对于输入数据的训练需要较长的时间，并且形成相应的模型。这种情况下，同一个分析算法对应不同的输入数据集有不同的分析模型。
+基于统计学的分析算法可以直接针对输入时间序列数据进行分析，但是某些深度学习算法对于输入数据需要较长的时间训练，并且生成相应的模型。这种情况下，同一个分析算法对应不同的输入数据集有不同的分析模型。
-将具有模型的分析算法添加到 Anode 中，首先需要在 `model` 目录中建立该算法对应的目录（目录名称可自拟），将采用该算法针对不同的输入时间序列数据生成的训练模型均需要保存在该目录下，同时目录结构要在分析算法中确定，以便能够固定加载该目录下的分析模型。如下图所示，针对不同的数据集，采用自编码器（Autoencoder）训练的数据异常检测算法模型均保存在该目录下。为了确保模型能够正常读取加载，要求存储的模型使用`joblib`库进行序列化保存。
+将具有模型的分析算法添加到 Anode 中，首先需要在 `model` 目录中建立该算法对应的目录（目录名称可自拟），将采用该算法针对不同的输入时间序列数据生成的训练模型均需要保存在该目录下，同时目录名称要在分析算法中确定，以便能够固定加载该目录下的分析模型。为了确保模型能够正常读取加载，存储的模型使用`joblib`库进行序列化保存。
 
-调用已经保存的模型，需要首先调用`set_params`方法，并在参数中指定调用模型的名称 `{"model": "ad_encoder_keras"}` 即可调用该模型进行计算。调用方式如下：
+下面以自编码器（Autoencoder）为例，说明如何添加要预先训练的模型进行异常检测。
+首先我们在`model`目录中创建一个目录 -- `ad_detection`，该目录将用来保存所有使用自编码器训练的模型。然后，我们使用自编码器对 foo 表的时间序列数据进行训练，得到模型 ad_autoencoder_foo，使用 `joblib`序列化以后保存在`ad_detection` 目录中。
 
-```python
+使用 SQL 调用已经保存的模型，需要在调用参数中指定模型名称``model=ad_autoencoder_foo`，而 `algo=encoder` 是确定调用的自编码器生成的模型（这里的`encoder`说明调用的是自编码器算法模型，该名称是添加算法的时候在代码中定义）以便能够调用该模型。
-def test_autoencoder_ad(self):
-    # 获取特定的算法对象
-    # ...
-    
-    # 指定调用的模型，该模型是之前针对该数据集进行训练获得
-    s.set_params({"model": "ad_encoder_keras"})
 
-    # 执行检查动作，并返回结果
+```SQL
-    r = s.execute()
+--- 在 options 中增加 model 的名称，ad_autoencoder_foo， 针对 foo 数据集（表）训练的采用自编码器的异常检测模型进行异常检测
+SELECT COUNT(*), _WSTART FROM foo ANOMALY_DETECTION(col1, 'algo=encoder, model=ad_autoencoder_foo');
 ```
-

docs/zh/06-advanced/06-TDgpt/index.md

@@ -15,8 +15,11 @@ TDgpt 运行在集群中的 AI Node (Anode)中，集群中可以部署若干个
 
 在查询处理过程中，Vnode中运行的查询引擎会根据查询处理物理执行计划，按需向 Anode 请求高级时序数据分析服务。因此用户可通过 SQL 语句与 Anode 节点交互并使用其提供的全部分析服务。需要注意的是 Anode 不直接接受用户的数据分析请求。同时 Anode 具备分析算法动态注册机制，其算法扩展过程完全不影响 TDengine 集群的服务，仅在非常小的（秒级）时间窗口内影响涉及高级分析的查询服务。
 
+目前 TDgpt 提供如下的高级分析服务：
 - 时序数据异常检测。TDengine 中定义了新的时间窗口——异常（状态）窗口——来提供异常检测服务。异常窗口可以视为一种特殊的**事件窗口（Event Window）**，即异常检测算法确定的连续异常时间序列数据所在的时间窗口。与普通事件窗口区别在于——时间窗口的起始时间和结束时间均是分析算法确定，不是用户指定的表达式判定。异常窗口使用方式与其他类型的时间窗口（例如状态窗口、会话窗口等）类似。因此时间窗口内可使用的查询操作均可应用在异常窗口上。
 - 时序数据预测。定义了一个新函数`FORECAST`，基于输入的（历史）时间序列数据调用指定（或默认）预测算法给出输入时序数据后续时间序列的**预测**数据。
 
+TDgpt 还为算法开发者提供了一 SDK。任何开发者只需要按照[算法开发者指南](./dev)的步骤，就可以将自己独有的时序数据预测或时序数据异常检测算法无缝集成到 TDgpt, 这样 TDengine 用户就可以通过一条 SQL 获得时序数据预测结果或是异常窗口了, 大幅降低了用户使用新的时序数据分析算法的门槛，而且让 TDengine 成为一开放的系统。
+
 
 

docs/zh/14-reference/01-components/01-taosd.md

@@ -294,6 +294,7 @@ charset 的有效值是 UTF-8。
 |checkpointBackupDir |          |内部参数，用于恢复 snode 数据|
 |enableAuditDelete   |          |内部参数，用于测试审计功能|
 |slowLogThresholdTest|          |内部参数，用于测试慢日志|
+|bypassFlag          |3.3.4.5 后|内部参数，用于短路测试，0：正常写入，1：写入消息在 taos 客户端发送 RPC 消息前返回，2：写入消息在 taosd 服务端收到 RPC 消息后返回，4：写入消息在 taosd 服务端写入内存缓存前返回，8：写入消息在 taosd 服务端数据落盘前返回；默认值 0|
 
 ### 压缩参数
 |参数名称|支持版本|参数含义|

docs/zh/14-reference/01-components/02-taosc.md

@@ -97,6 +97,7 @@ TDengine 客户端驱动提供了应用编程所需要的全部 API，并且在
 |safetyCheckLevel |3.3.3.0 后|内部参数，用于随机失败测试|
 |simdEnable       |3.3.4.3 后|内部参数，用于测试 SIMD 加速|
 |AVX512Enable     |3.3.4.3 后|内部参数，用于测试 AVX512 加速|
+|bypassFlag       |3.3.4.5 后|内部参数，用于短路测试，0：正常写入，1：写入消息在 taos 客户端发送 RPC 消息前返回，2：写入消息在 taosd 服务端收到 RPC 消息后返回，4：写入消息在 taosd 服务端写入内存缓存前返回，8：写入消息在 taosd 服务端数据落盘前返回；缺省值：0|
 
 ### SHELL 相关
 |参数名称|支持版本|参数含义|

docs/zh/14-reference/03-taos-sql/02-database.md

@@ -8,10 +8,10 @@ description: "创建、删除数据库，查看、修改数据库参数"
 
 ```sql
 CREATE DATABASE [IF NOT EXISTS] db_name [database_options]
- 
+
 database_options:
     database_option ...
- 
+
 database_option: {
     VGROUPS value
   | PRECISION {'ms' | 'us' | 'ns'}
@@ -26,6 +26,7 @@ database_option: {
   | MAXROWS value
   | MINROWS value
   | KEEP value
+  | KEEP_TIME_OFFSET value
   | STT_TRIGGER value
   | SINGLE_STABLE {0 | 1}
   | TABLE_PREFIX value
@@ -63,8 +64,8 @@ database_option: {
 - DURATION：数据文件存储数据的时间跨度。可以使用加单位的表示形式，如 DURATION 100h、DURATION 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。不加时间单位时默认单位为天，如 DURATION 50 表示 50 天。
 - MAXROWS：文件块中记录的最大条数，默认为 4096 条。
 - MINROWS：文件块中记录的最小条数，默认为 100 条。
-- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于3倍的 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 \<= keep 1 \<= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。了解更多，请点击 [关于主键时间戳](https://docs.taosdata.com/reference/taos-sql/insert/#%E5%85%B3%E4%BA%8E%E4%B8%BB%E9%94%AE%E6%97%B6%E9%97%B4%E6%88%B3)。
+- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于 3 倍的 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据从而释放存储空间。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 \<= keep 1 \<= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。了解更多，请点击 [关于主键时间戳](https://docs.taosdata.com/reference/taos-sql/insert/)
-
+- KEEP_TIME_OFFSET：自 3.2.0.0 版本生效。删除或迁移保存时间超过 KEEP 值的数据的延迟执行时间，默认值为 0 (小时)。在数据文件保存时间超过 KEEP 后，删除或迁移操作不会立即执行，而会额外等待本参数指定的时间间隔，以实现与业务高峰期错开的目的。
 - STT_TRIGGER：表示落盘文件触发文件合并的个数。开源版本固定为 1，企业版本可设置范围为 1 到 16。对于少表高频写入场景，此参数建议使用默认配置；而对于多表低频写入场景，此参数建议配置较大的值。
 - SINGLE_STABLE：表示此数据库中是否只可以创建一个超级表，用于超级表列非常多的情况。
   - 0：表示可以创建多张超级表。

docs/zh/14-reference/03-taos-sql/32-compress.md

@@ -31,11 +31,12 @@ description: 可配置压缩算法
 
 | 数据类型 |   可选编码算法      |  编码算法默认值 | 可选压缩算法|压缩算法默认值| 压缩等级默认值|
 | :-----------:|:----------:|:-------:|:-------:|:----------:|:----:|
-|  tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium|
+|  int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium|
-|   bigint/ubigint/timestamp   |  simple8b/delta-i    | delta-i |lz4/zlib/zstd/xz | lz4| medium|
+|  tinyint/untinyint/smallint/usmallint | simple8b| simple8b | lz4/zlib/zstd/xz| zlib | medium|
+|  bigint/ubigint/timestamp   |  simple8b/delta-i    | delta-i |lz4/zlib/zstd/xz | lz4| medium|
 |float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|lz4| medium|
-|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| medium|
+|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| zstd| medium|
-|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| medium|
+|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| zstd| medium|
 
 ## SQL 语法
 

docs/zh/14-reference/05-connector/30-python.mdx

@@ -41,6 +41,7 @@ Python 连接器的源码托管在 [GitHub](https://github.com/taosdata/taos-con
 
 |Python Connector 版本|主要变化|
 |:-------------------:|:----:|
+|2.7.16|新增订阅配置 (session.timeout.ms, max.poll.interval.ms)|
 |2.7.15|新增 VARBINARY 和 GEOMETRY 类型支持|
 |2.7.14|修复已知问题|
 |2.7.13|新增 tmq 同步提交 offset 接口|
@@ -50,6 +51,7 @@ Python 连接器的源码托管在 [GitHub](https://github.com/taosdata/taos-con
 
 |Python WebSocket Connector 版本|主要变化|
 |:----------------------------:|:-----:|
+|0.3.5|新增 VARBINARY 和 GEOMETRY 类型支持，修复已知问题|
 |0.3.2|优化 WebSocket sql 查询和插入性能，修改 readme 和 文档，修复已知问题|
 |0.2.9|已知问题修复|
 |0.2.5|1. 数据订阅支持获取消费进度和重置消费进度 <br/> 2. 支持 schemaless <br/> 3. 支持 STMT|

docs/zh/14-reference/05-connector/35-node.mdx

@@ -26,6 +26,7 @@ Node.js 连接器目前仅支持 WebSocket 连接器， 其通过 taosAdapter
 
 | Node.js 连接器 版本 |          主要变化         |   TDengine 版本    |
 | :------------------:  | :----------------------: | :----------------: |
+|        3.1.2          | 对数据协议和解析进行了优化，性能得到大幅提升|  3.3.2.0 及更高版本 |
 |        3.1.1          | 优化了数据传输性能          |  3.3.2.0 及更高版本 |
 |        3.1.0          | 新版本发布，支持 WebSocket 连接 |  3.2.0.0 及更高版本 |
 

include/common/streamMsg.h

@@ -17,12 +17,23 @@
 #define TDENGINE_STREAMMSG_H
 
 #include "tmsg.h"
-#include "trpc.h"
+//#include "trpc.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+typedef struct SStreamRetrieveReq SStreamRetrieveReq;
+typedef struct SStreamDispatchReq SStreamDispatchReq;
+typedef struct STokenBucket       STokenBucket;
+typedef struct SMetaHbInfo        SMetaHbInfo;
+
+typedef struct SNodeUpdateInfo {
+  int32_t nodeId;
+  SEpSet  prevEp;
+  SEpSet  newEp;
+} SNodeUpdateInfo;
+
 typedef struct SStreamUpstreamEpInfo {
   int32_t nodeId;
   int32_t childId;
@@ -170,8 +181,8 @@ typedef struct SStreamHbMsg {
   SArray* pUpdateNodes;  // SArray<int32_t>, needs update the epsets in stream tasks for those nodes.
 } SStreamHbMsg;
 
-int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pRsp);
+int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pReq);
-int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pRsp);
+int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq);
 void    tCleanupStreamHbMsg(SStreamHbMsg* pMsg);
 
 typedef struct {
@@ -179,6 +190,9 @@ typedef struct {
   int32_t  msgId;
 } SMStreamHbRspMsg;
 
+int32_t tEncodeStreamHbRsp(SEncoder* pEncoder, const SMStreamHbRspMsg* pRsp);
+int32_t tDecodeStreamHbRsp(SDecoder* pDecoder, SMStreamHbRspMsg* pRsp);
+
 typedef struct SRetrieveChkptTriggerReq {
   SMsgHead head;
   int64_t  streamId;
@@ -189,6 +203,9 @@ typedef struct SRetrieveChkptTriggerReq {
   int64_t  downstreamTaskId;
 } SRetrieveChkptTriggerReq;
 
+int32_t tEncodeRetrieveChkptTriggerReq(SEncoder* pEncoder, const SRetrieveChkptTriggerReq* pReq);
+int32_t tDecodeRetrieveChkptTriggerReq(SDecoder* pDecoder, SRetrieveChkptTriggerReq* pReq);
+
 typedef struct SCheckpointTriggerRsp {
   int64_t streamId;
   int64_t checkpointId;
@@ -198,6 +215,9 @@ typedef struct SCheckpointTriggerRsp {
   int32_t rspCode;
 } SCheckpointTriggerRsp;
 
+int32_t tEncodeCheckpointTriggerRsp(SEncoder* pEncoder, const SCheckpointTriggerRsp* pRsp);
+int32_t tDecodeCheckpointTriggerRsp(SDecoder* pDecoder, SCheckpointTriggerRsp* pRsp);
+
 typedef struct SCheckpointReport {
   int64_t streamId;
   int32_t taskId;
@@ -222,7 +242,7 @@ typedef struct SRestoreCheckpointInfo {
   int32_t  nodeId;
 } SRestoreCheckpointInfo;
 
-int32_t tEncodeRestoreCheckpointInfo (SEncoder* pEncoder, const SRestoreCheckpointInfo* pReq);
+int32_t tEncodeRestoreCheckpointInfo(SEncoder* pEncoder, const SRestoreCheckpointInfo* pReq);
 int32_t tDecodeRestoreCheckpointInfo(SDecoder* pDecoder, SRestoreCheckpointInfo* pReq);
 
 typedef struct {
@@ -232,10 +252,8 @@ typedef struct {
   int32_t  reqType;
 } SStreamTaskRunReq;
 
-typedef struct SCheckpointConsensusEntry {
+int32_t tEncodeStreamTaskRunReq(SEncoder* pEncoder, const SStreamTaskRunReq* pReq);
-  SRestoreCheckpointInfo req;
+int32_t tDecodeStreamTaskRunReq(SDecoder* pDecoder, SStreamTaskRunReq* pReq);
-  int64_t                ts;
-} SCheckpointConsensusEntry;
 
 #ifdef __cplusplus
 }

include/common/tcommon.h

@@ -251,6 +251,7 @@ typedef struct SQueryTableDataCond {
   int32_t      type;       // data block load type:
   bool         skipRollup;
   STimeWindow  twindows;
+  STimeWindow  extTwindows[2];
   int64_t      startVersion;
   int64_t      endVersion;
   bool         notLoadData;  // response the actual data, not only the rows in the attribute of info.row of ssdatablock

include/common/tdataformat.h

@@ -57,9 +57,9 @@ const static uint8_t BIT2_MAP[4] = {0b11111100, 0b11110011, 0b11001111, 0b001111
 #define ONE               ((uint8_t)1)
 #define THREE             ((uint8_t)3)
 #define DIV_8(i)          ((i) >> 3)
-#define MOD_8(i)          ((i)&7)
+#define MOD_8(i)          ((i) & 7)
 #define DIV_4(i)          ((i) >> 2)
-#define MOD_4(i)          ((i)&3)
+#define MOD_4(i)          ((i) & 3)
 #define MOD_4_TIME_2(i)   (MOD_4(i) << 1)
 #define BIT1_SIZE(n)      (DIV_8((n)-1) + 1)
 #define BIT2_SIZE(n)      (DIV_4((n)-1) + 1)
@@ -173,6 +173,8 @@ typedef struct {
 } SColDataCompressInfo;
 
 typedef void *(*xMallocFn)(void *, int32_t);
+typedef int32_t (*checkWKBGeometryFn)(const unsigned char *geoWKB, size_t nGeom);
+typedef int32_t (*initGeosFn)();
 
 void    tColDataDestroy(void *ph);
 void    tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t cflag);
@@ -191,7 +193,8 @@ int32_t tColDataCompress(SColData *colData, SColDataCompressInfo *info, SBuffer
 int32_t tColDataDecompress(void *input, SColDataCompressInfo *info, SColData *colData, SBuffer *assist);
 
 // for stmt bind
-int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen);
+int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen, initGeosFn igeos,
+                               checkWKBGeometryFn cgeos);
 int32_t tColDataSortMerge(SArray **arr);
 
 // for raw block
@@ -378,7 +381,8 @@ int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted,
                           SArray *rowArray);
 
 // stmt2 binding
-int32_t tColDataAddValueByBind2(SColData *pColData, TAOS_STMT2_BIND *pBind, int32_t buffMaxLen);
+int32_t tColDataAddValueByBind2(SColData *pColData, TAOS_STMT2_BIND *pBind, int32_t buffMaxLen, initGeosFn igeos,
+                                checkWKBGeometryFn cgeos);
 
 typedef struct {
   int32_t          columnId;

include/common/tglobal.h

@@ -67,6 +67,7 @@ extern int64_t tsTickPerHour[3];
 extern int32_t tsCountAlwaysReturnValue;
 extern float   tsSelectivityRatio;
 extern int32_t tsTagFilterResCacheSize;
+extern int32_t tsBypassFlag;
 
 // queue & threads
 extern int32_t tsNumOfRpcThreads;

include/common/tmsg.h

@@ -3814,7 +3814,14 @@ typedef struct {
   SMsgHead head;
   int64_t  streamId;
   int32_t  taskId;
-} SVPauseStreamTaskReq, SVResetStreamTaskReq;
+} SVPauseStreamTaskReq;
+
+typedef struct {
+  SMsgHead head;
+  int64_t  streamId;
+  int32_t  taskId;
+  int64_t  chkptId;
+} SVResetStreamTaskReq;
 
 typedef struct {
   char   name[TSDB_STREAM_FNAME_LEN];

include/libs/catalog/catalog.h

@@ -102,6 +102,7 @@ typedef struct SCatalogReq {
   bool    svrVerRequired;
   bool    forceUpdate;
   bool    cloned;
+  bool    forceFetchViewMeta;
 } SCatalogReq;
 
 typedef struct SMetaRes {

include/libs/geometry/geosWrapper.h

@@ -35,6 +35,7 @@ int32_t doGeomFromText(const char *inputWKT, unsigned char **outputGeom, size_t
 
 int32_t initCtxAsText();
 int32_t doAsText(const unsigned char *inputGeom, size_t size, char **outputWKT);
+int32_t checkWKB(const unsigned char *wkb, size_t size);
 
 int32_t initCtxRelationFunc();
 int32_t doIntersects(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2,
@@ -47,11 +48,12 @@ int32_t doCovers(const GEOSGeometry *geom1, const GEOSPreparedGeometry *prepared
                  bool swapped, char *res);
 int32_t doContains(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2,
                    bool swapped, char *res);
-int32_t doContainsProperly(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2,
+int32_t doContainsProperly(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1,
-                           bool swapped, char *res);
+                           const GEOSGeometry *geom2, bool swapped, char *res);
 
-int32_t readGeometry(const unsigned char *input, GEOSGeometry **outputGeom, const GEOSPreparedGeometry **outputPreparedGeom);
+int32_t readGeometry(const unsigned char *input, GEOSGeometry **outputGeom,
-void destroyGeometry(GEOSGeometry **geom, const GEOSPreparedGeometry **preparedGeom);
+                     const GEOSPreparedGeometry **outputPreparedGeom);
+void    destroyGeometry(GEOSGeometry **geom, const GEOSPreparedGeometry **preparedGeom);
 
 #ifdef __cplusplus
 }

include/libs/nodes/nodes.h

@@ -174,6 +174,7 @@ char*   nodesGetNameFromColumnNode(SNode* pNode);
 int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots);
 void    nodesSortList(SNodeList** pList, int32_t (*)(SNode* pNode1, SNode* pNode2));
 void    destroyFuncParam(void* pFuncStruct);
+int32_t nodesListDeduplicate(SNodeList** pList);
 
 #ifdef __cplusplus
 }

include/libs/stream/tstream.h

@@ -319,11 +319,6 @@ typedef struct SSTaskBasicInfo {
   SInterval interval;
 } SSTaskBasicInfo;
 
-typedef struct SStreamRetrieveReq SStreamRetrieveReq;
-typedef struct SStreamDispatchReq SStreamDispatchReq;
-typedef struct STokenBucket       STokenBucket;
-typedef struct SMetaHbInfo        SMetaHbInfo;
-
 typedef struct SDispatchMsgInfo {
   SStreamDispatchReq* pData;  // current dispatch data
 
@@ -626,11 +621,11 @@ typedef struct STaskStatusEntry {
   STaskCkptInfo checkpointInfo;
 } STaskStatusEntry;
 
-typedef struct SNodeUpdateInfo {
+//typedef struct SNodeUpdateInfo {
-  int32_t nodeId;
+//  int32_t nodeId;
-  SEpSet  prevEp;
+//  SEpSet  prevEp;
-  SEpSet  newEp;
+//  SEpSet  newEp;
-} SNodeUpdateInfo;
+//} SNodeUpdateInfo;
 
 typedef struct SStreamTaskState {
   ETaskStatus state;
@@ -643,6 +638,11 @@ typedef struct SCheckpointConsensusInfo {
   int64_t streamId;
 } SCheckpointConsensusInfo;
 
+typedef struct SCheckpointConsensusEntry {
+  SRestoreCheckpointInfo req;
+  int64_t                ts;
+} SCheckpointConsensusEntry;
+
 void    streamSetupScheduleTrigger(SStreamTask* pTask);
 
 // dispatch related
@@ -718,6 +718,7 @@ int32_t streamTaskInitTriggerDispatchInfo(SStreamTask* pTask);
 void    streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId);
 int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId, int32_t downstreamNodeId,
                                            SRpcHandleInfo* pInfo, int32_t code);
+void    streamTaskSetFailedCheckpointId(SStreamTask* pTask, int64_t failedId);
 
 int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue);
 int32_t streamQueueGetNumOfUnAccessedItems(const SStreamQueue* pQueue);

include/util/taoserror.h

@@ -1011,6 +1011,7 @@ int32_t  taosGetErrSize();
 #define TSDB_CODE_STREAM_CONFLICT_EVENT          TAOS_DEF_ERROR_CODE(0, 0x4106)
 #define TSDB_CODE_STREAM_INTERNAL_ERROR          TAOS_DEF_ERROR_CODE(0, 0x4107)
 #define TSDB_CODE_STREAM_INPUTQ_FULL             TAOS_DEF_ERROR_CODE(0, 0x4108)
+#define TSDB_CODE_STREAM_INVLD_CHKPT             TAOS_DEF_ERROR_CODE(0, 0x4109)
 
 // TDLite
 #define TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS         TAOS_DEF_ERROR_CODE(0, 0x5100)

include/util/tbuffer.inc

@@ -186,11 +186,25 @@ static int32_t tBufferGetI16(SBufferReader *reader, int16_t *value) {
 }
 
 static int32_t tBufferGetI32(SBufferReader *reader, int32_t *value) {
-  return tBufferGet(reader, sizeof(*value), value);
+  if (reader->offset + sizeof(int32_t) > reader->buffer->size) {
+    return TSDB_CODE_OUT_OF_RANGE;
+  }
+  if (value) {
+    *value = *(int32_t*)BR_PTR(reader);
+  }
+  reader->offset += sizeof(int32_t);
+  return 0;
 }
 
 static int32_t tBufferGetI64(SBufferReader *reader, int64_t *value) {
-  return tBufferGet(reader, sizeof(*value), value);
+  if (reader->offset + sizeof(int64_t) > reader->buffer->size) {
+    return TSDB_CODE_OUT_OF_RANGE;
+  }
+  if (value) {
+    *value = *(int64_t*)BR_PTR(reader);
+  }
+  reader->offset += sizeof(int64_t);
+  return 0;
 }
 
 static int32_t tBufferGetU8(SBufferReader *reader, uint8_t *value) { return tBufferGet(reader, sizeof(*value), value); }

include/util/tdef.h

@@ -620,6 +620,16 @@ enum {
 
 enum { RAND_ERR_MEMORY = 1, RAND_ERR_FILE = 2, RAND_ERR_NETWORK = 4 };
 
+/**
+ * RB: return before
+ * RA: return after
+ * NR: not return, skip and go on following steps
+ */
+#define TSDB_BYPASS_RB_RPC_SEND_SUBMIT 0x01u
+#define TSDB_BYPASS_RA_RPC_RECV_SUBMIT 0x02u
+#define TSDB_BYPASS_RB_TSDB_WRITE_MEM  0x04u
+#define TSDB_BYPASS_RB_TSDB_COMMIT     0x08u
+
 #define DEFAULT_HANDLE 0
 #define MNODE_HANDLE   1
 #define QNODE_HANDLE   -1

source/client/src/clientImpl.c

@@ -3032,13 +3032,13 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
   if (code != TSDB_CODE_SUCCESS) {
     pRequest->code = code;
     taosMemoryFreeClear(pResultInfo->pData);
-    pRequest->body.fetchFp(((SSyncQueryParam*)pRequest->body.interParam)->userParam, pRequest, 0);
+    pRequest->body.fetchFp(((SSyncQueryParam*)pRequest->body.interParam)->userParam, pRequest, code);
     return;
   }
 
   if (pRequest->code != TSDB_CODE_SUCCESS) {
     taosMemoryFreeClear(pResultInfo->pData);
-    pRequest->body.fetchFp(((SSyncQueryParam*)pRequest->body.interParam)->userParam, pRequest, 0);
+    pRequest->body.fetchFp(((SSyncQueryParam*)pRequest->body.interParam)->userParam, pRequest, pRequest->code);
     return;
   }
 

source/common/CMakeLists.txt

@@ -1,4 +1,7 @@
 aux_source_directory(src COMMON_SRC)
+aux_source_directory(src/msg COMMON_MSG_SRC)
+
+LIST(APPEND COMMON_SRC ${COMMON_MSG_SRC})
 
 if(TD_ENTERPRISE)
   LIST(APPEND COMMON_SRC ${TD_ENTERPRISE_DIR}/src/plugins/common/src/tglobal.c)

source/common/src/msg/streamMsg.c

@@ -15,8 +15,48 @@
 
 #include "streamMsg.h"
 #include "os.h"
-#include "tstream.h"
+#include "tcommon.h"
-#include "streamInt.h"
+
+typedef struct STaskId {
+  int64_t streamId;
+  int64_t taskId;
+} STaskId;
+
+typedef struct STaskCkptInfo {
+  int64_t latestId;          // saved checkpoint id
+  int64_t latestVer;         // saved checkpoint ver
+  int64_t latestTime;        // latest checkpoint time
+  int64_t latestSize;        // latest checkpoint size
+  int8_t  remoteBackup;      // latest checkpoint backup done
+  int64_t activeId;          // current active checkpoint id
+  int32_t activeTransId;     // checkpoint trans id
+  int8_t  failed;            // denote if the checkpoint is failed or not
+  int8_t  consensusChkptId;  // required the consensus-checkpointId
+  int64_t consensusTs;       //
+} STaskCkptInfo;
+
+typedef struct STaskStatusEntry {
+  STaskId       id;
+  int32_t       status;
+  int32_t       statusLastDuration;  // to record the last duration of current status
+  int64_t       stage;
+  int32_t       nodeId;
+  SVersionRange verRange;      // start/end version in WAL, only valid for source task
+  int64_t       processedVer;  // only valid for source task
+  double        inputQUsed;    // in MiB
+  double        inputRate;
+  double        procsThroughput;   // duration between one element put into input queue and being processed.
+  double        procsTotal;        // duration between one element put into input queue and being processed.
+  double        outputThroughput;  // the size of dispatched result blocks in bytes
+  double        outputTotal;       // the size of dispatched result blocks in bytes
+  double        sinkQuota;         // existed quota size for sink task
+  double        sinkDataSize;      // sink to dst data size
+  int64_t       startTime;
+  int64_t       startCheckpointId;
+  int64_t       startCheckpointVer;
+  int64_t       hTaskId;
+  STaskCkptInfo checkpointInfo;
+} STaskStatusEntry;
 
 int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamUpstreamEpInfo* pInfo) {
   TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pInfo->taskId));
@@ -289,7 +329,7 @@ int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* p
   TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->totalLen));
 
   if (taosArrayGetSize(pReq->data) != pReq->blockNum || taosArrayGetSize(pReq->dataLen) != pReq->blockNum) {
-    stError("invalid dispatch req msg");
+    uError("invalid dispatch req msg");
     TAOS_CHECK_EXIT(TSDB_CODE_INVALID_MSG);
   }
 
@@ -605,173 +645,92 @@ void tCleanupStreamHbMsg(SStreamHbMsg* pMsg) {
   pMsg->numOfTasks = -1;
 }
 
-int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
+int32_t tEncodeStreamHbRsp(SEncoder* pEncoder, const SMStreamHbRspMsg* pRsp) {
   int32_t code = 0;
   int32_t lino;
 
   TAOS_CHECK_EXIT(tStartEncode(pEncoder));
-  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->ver));
+  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->msgId));
-  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->id.streamId));
-  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->id.taskId));
-  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->info.trigger));
-  TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->info.taskLevel));
-  TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->outputInfo.type));
-  TAOS_CHECK_EXIT(tEncodeI16(pEncoder, pTask->msgInfo.msgType));
-
-  TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->status.taskStatus));
-  TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->status.schedStatus));
-
-  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->info.selfChildId));
-  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->info.nodeId));
-  TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pTask->info.epSet));
-  TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pTask->info.mnodeEpset));
-
-  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->chkInfo.checkpointId));
-  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->chkInfo.checkpointVer));
-  TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->info.fillHistory));
-
-  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->hTaskInfo.id.streamId));
-  int32_t taskId = pTask->hTaskInfo.id.taskId;
-  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, taskId));
-
-  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->streamTaskId.streamId));
-  taskId = pTask->streamTaskId.taskId;
-  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, taskId));
-
-  TAOS_CHECK_EXIT(tEncodeU64(pEncoder, pTask->dataRange.range.minVer));
-  TAOS_CHECK_EXIT(tEncodeU64(pEncoder, pTask->dataRange.range.maxVer));
-  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->dataRange.window.skey));
-  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->dataRange.window.ekey));
-
-  int32_t epSz = taosArrayGetSize(pTask->upstreamInfo.pList);
-  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, epSz));
-  for (int32_t i = 0; i < epSz; i++) {
-    SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i);
-    TAOS_CHECK_EXIT(tEncodeStreamEpInfo(pEncoder, pInfo));
-  }
-
-  if (pTask->info.taskLevel != TASK_LEVEL__SINK) {
-    TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, pTask->exec.qmsg));
-  }
-
-  if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) {
-    TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->outputInfo.tbSink.stbUid));
-    TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, pTask->outputInfo.tbSink.stbFullName));
-    TAOS_CHECK_EXIT(tEncodeSSchemaWrapper(pEncoder, pTask->outputInfo.tbSink.pSchemaWrapper));
-  } else if (pTask->outputInfo.type == TASK_OUTPUT__SMA) {
-    TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->outputInfo.smaSink.smaId));
-  } else if (pTask->outputInfo.type == TASK_OUTPUT__FETCH) {
-    TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->outputInfo.fetchSink.reserved));
-  } else if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) {
-    TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->outputInfo.fixedDispatcher.taskId));
-    TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->outputInfo.fixedDispatcher.nodeId));
-    TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pTask->outputInfo.fixedDispatcher.epSet));
-  } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
-    TAOS_CHECK_EXIT(tSerializeSUseDbRspImp(pEncoder, &pTask->outputInfo.shuffleDispatcher.dbInfo));
-    TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, pTask->outputInfo.shuffleDispatcher.stbFullName));
-  }
-  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->info.delaySchedParam));
-  TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->subtableWithoutMd5));
-  TAOS_CHECK_EXIT(tEncodeCStrWithLen(pEncoder, pTask->reserve, sizeof(pTask->reserve) - 1));
-
   tEndEncode(pEncoder);
+
 _exit:
   return code;
 }
 
-int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
+int32_t tDecodeStreamHbRsp(SDecoder* pDecoder, SMStreamHbRspMsg* pRsp) {
-  int32_t taskId = 0;
   int32_t code = 0;
   int32_t lino;
 
   TAOS_CHECK_EXIT(tStartDecode(pDecoder));
-  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->ver));
+  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->msgId));
-  if (pTask->ver <= SSTREAM_TASK_INCOMPATIBLE_VER || pTask->ver > SSTREAM_TASK_VER) {
+  tEndDecode(pDecoder);
-    TAOS_CHECK_EXIT(TSDB_CODE_INVALID_MSG);
-  }
 
-  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->id.streamId));
+_exit:
-  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->id.taskId));
+  return code;
-  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->info.trigger));
+}
-  TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->info.taskLevel));
-  TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->outputInfo.type));
-  TAOS_CHECK_EXIT(tDecodeI16(pDecoder, &pTask->msgInfo.msgType));
 
-  TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->status.taskStatus));
+int32_t tEncodeRetrieveChkptTriggerReq(SEncoder* pEncoder, const SRetrieveChkptTriggerReq* pReq) {
-  TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->status.schedStatus));
+  int32_t code = 0;
+  int32_t lino;
 
-  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->info.selfChildId));
+  TAOS_CHECK_EXIT(tStartEncode(pEncoder));
-  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->info.nodeId));
+  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->streamId));
-  TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &pTask->info.epSet));
+  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->checkpointId));
-  TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &pTask->info.mnodeEpset));
+  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->upstreamNodeId));
+  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->upstreamTaskId));
+  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->downstreamNodeId));
+  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->downstreamTaskId));
+  tEndEncode(pEncoder);
 
-  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->chkInfo.checkpointId));
+_exit:
-  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->chkInfo.checkpointVer));
+  return code;
-  TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->info.fillHistory));
+}
 
-  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->hTaskInfo.id.streamId));
+int32_t tDecodeRetrieveChkptTriggerReq(SDecoder* pDecoder, SRetrieveChkptTriggerReq* pReq) {
-  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &taskId));
+  int32_t code = 0;
-  pTask->hTaskInfo.id.taskId = taskId;
+  int32_t lino;
 
-  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->streamTaskId.streamId));
+  TAOS_CHECK_EXIT(tStartDecode(pDecoder));
-  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &taskId));
+  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->streamId));
-  pTask->streamTaskId.taskId = taskId;
+  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->checkpointId));
+  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->upstreamNodeId));
+  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->upstreamTaskId));
+  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->downstreamNodeId));
+  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->downstreamTaskId));
+  tEndDecode(pDecoder);
 
-  TAOS_CHECK_EXIT(tDecodeU64(pDecoder, (uint64_t*)&pTask->dataRange.range.minVer));
+_exit:
-  TAOS_CHECK_EXIT(tDecodeU64(pDecoder, (uint64_t*)&pTask->dataRange.range.maxVer));
+  return code;
-  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->dataRange.window.skey));
+}
-  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->dataRange.window.ekey));
 
-  int32_t epSz = -1;
+int32_t tEncodeCheckpointTriggerRsp(SEncoder* pEncoder, const SCheckpointTriggerRsp* pRsp) {
-  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &epSz) < 0);
+  int32_t code = 0;
+  int32_t lino;
 
-  if ((pTask->upstreamInfo.pList = taosArrayInit(epSz, POINTER_BYTES)) == NULL) {
+  TAOS_CHECK_EXIT(tStartEncode(pEncoder));
-    TAOS_CHECK_EXIT(terrno);
+  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pRsp->streamId));
-  }
+  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pRsp->checkpointId));
-  for (int32_t i = 0; i < epSz; i++) {
+  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->upstreamTaskId));
-    SStreamUpstreamEpInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamUpstreamEpInfo));
+  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->taskId));
-    if (pInfo == NULL) {
+  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->transId));
-      TAOS_CHECK_EXIT(terrno);
+  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->rspCode));
-    }
+  tEndEncode(pEncoder);
-    if ((code = tDecodeStreamEpInfo(pDecoder, pInfo)) < 0) {
-      taosMemoryFreeClear(pInfo);
-      goto _exit;
-    }
-    if (taosArrayPush(pTask->upstreamInfo.pList, &pInfo) == NULL) {
-      TAOS_CHECK_EXIT(terrno);
-    }
-  }
 
-  if (pTask->info.taskLevel != TASK_LEVEL__SINK) {
+_exit:
-    TAOS_CHECK_EXIT(tDecodeCStrAlloc(pDecoder, &pTask->exec.qmsg));
+  return code;
-  }
+}
 
-  if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) {
+int32_t tDecodeCheckpointTriggerRsp(SDecoder* pDecoder, SCheckpointTriggerRsp* pRsp) {
-    TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->outputInfo.tbSink.stbUid));
+  int32_t code = 0;
-    TAOS_CHECK_EXIT(tDecodeCStrTo(pDecoder, pTask->outputInfo.tbSink.stbFullName));
+  int32_t lino;
-    pTask->outputInfo.tbSink.pSchemaWrapper = taosMemoryCalloc(1, sizeof(SSchemaWrapper));
-    if (pTask->outputInfo.tbSink.pSchemaWrapper == NULL) {
-      TAOS_CHECK_EXIT(terrno);
-    }
-    TAOS_CHECK_EXIT(tDecodeSSchemaWrapper(pDecoder, pTask->outputInfo.tbSink.pSchemaWrapper));
-  } else if (pTask->outputInfo.type == TASK_OUTPUT__SMA) {
-    TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->outputInfo.smaSink.smaId));
-  } else if (pTask->outputInfo.type == TASK_OUTPUT__FETCH) {
-    TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->outputInfo.fetchSink.reserved));
-  } else if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) {
-    TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->outputInfo.fixedDispatcher.taskId));
-    TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->outputInfo.fixedDispatcher.nodeId));
-    TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &pTask->outputInfo.fixedDispatcher.epSet));
-  } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
-    TAOS_CHECK_EXIT(tDeserializeSUseDbRspImp(pDecoder, &pTask->outputInfo.shuffleDispatcher.dbInfo));
-    TAOS_CHECK_EXIT(tDecodeCStrTo(pDecoder, pTask->outputInfo.shuffleDispatcher.stbFullName));
-  }
-  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->info.delaySchedParam));
-  if (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER) {
-    TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->subtableWithoutMd5));
-  }
-  TAOS_CHECK_EXIT(tDecodeCStrTo(pDecoder, pTask->reserve));
 
+  TAOS_CHECK_EXIT(tStartDecode(pDecoder));
+  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pRsp->streamId));
+  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pRsp->checkpointId));
+  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->upstreamTaskId));
+  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->taskId));
+  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->transId));
+  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->rspCode));
   tEndDecode(pDecoder);
 
 _exit:
@@ -830,11 +789,7 @@ int32_t tEncodeRestoreCheckpointInfo(SEncoder* pEncoder, const SRestoreCheckpoin
   tEndEncode(pEncoder);
 
 _exit:
-  if (code) {
+  return code;
-    return code;
-  } else {
-    return pEncoder->pos;
-  }
 }
 
 int32_t tDecodeRestoreCheckpointInfo(SDecoder* pDecoder, SRestoreCheckpointInfo* pReq) {
@@ -853,3 +808,31 @@ int32_t tDecodeRestoreCheckpointInfo(SDecoder* pDecoder, SRestoreCheckpointInfo*
 _exit:
   return code;
 }
+
+int32_t tEncodeStreamTaskRunReq (SEncoder* pEncoder, const SStreamTaskRunReq* pReq) {
+  int32_t code = 0;
+  int32_t lino;
+
+  TAOS_CHECK_EXIT(tStartEncode(pEncoder));
+  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->streamId));
+  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->taskId));
+  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->reqType));
+  tEndEncode(pEncoder);
+
+_exit:
+  return code;
+}
+
+int32_t tDecodeStreamTaskRunReq(SDecoder* pDecoder, SStreamTaskRunReq* pReq) {
+  int32_t code = 0;
+  int32_t lino;
+
+  TAOS_CHECK_EXIT(tStartDecode(pDecoder));
+  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->streamId));
+  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->taskId));
+  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->reqType));
+  tEndDecode(pDecoder);
+
+_exit:
+  return code;
+}

source/common/src/tcol.c

@@ -81,26 +81,42 @@ const char* getDefaultEncodeStr(uint8_t type) { return columnEncodeStr(getDefaul
 uint16_t getDefaultCompress(uint8_t type) {
   switch (type) {
     case TSDB_DATA_TYPE_NULL:
+      return TSDB_COLVAL_COMPRESS_LZ4;
     case TSDB_DATA_TYPE_BOOL:
+      return TSDB_COLVAL_COMPRESS_ZSTD;
     case TSDB_DATA_TYPE_TINYINT:
     case TSDB_DATA_TYPE_SMALLINT:
+      return TSDB_COLVAL_COMPRESS_ZLIB;
     case TSDB_DATA_TYPE_INT:
     case TSDB_DATA_TYPE_BIGINT:
+      return TSDB_COLVAL_COMPRESS_LZ4;
     case TSDB_DATA_TYPE_FLOAT:
     case TSDB_DATA_TYPE_DOUBLE:
+      return TSDB_COLVAL_COMPRESS_LZ4;
     case TSDB_DATA_TYPE_VARCHAR:  // TSDB_DATA_TYPE_BINARY
+      return TSDB_COLVAL_COMPRESS_ZSTD;
     case TSDB_DATA_TYPE_TIMESTAMP:
+      return TSDB_COLVAL_COMPRESS_LZ4;
     case TSDB_DATA_TYPE_NCHAR:
+      return TSDB_COLVAL_COMPRESS_ZSTD;
     case TSDB_DATA_TYPE_UTINYINT:
     case TSDB_DATA_TYPE_USMALLINT:
+      return TSDB_COLVAL_COMPRESS_ZLIB;
     case TSDB_DATA_TYPE_UINT:
     case TSDB_DATA_TYPE_UBIGINT:
+      return TSDB_COLVAL_COMPRESS_LZ4;
     case TSDB_DATA_TYPE_JSON:
+      return TSDB_COLVAL_COMPRESS_LZ4;
     case TSDB_DATA_TYPE_VARBINARY:
+      return TSDB_COLVAL_COMPRESS_ZSTD;
     case TSDB_DATA_TYPE_DECIMAL:
+      return TSDB_COLVAL_COMPRESS_LZ4;
     case TSDB_DATA_TYPE_BLOB:
+      return TSDB_COLVAL_COMPRESS_LZ4;
     case TSDB_DATA_TYPE_MEDIUMBLOB:
+      return TSDB_COLVAL_COMPRESS_LZ4;
     case TSDB_DATA_TYPE_GEOMETRY:
+      return TSDB_COLVAL_COMPRESS_LZ4;
     case TSDB_DATA_TYPE_MAX:
       return TSDB_COLVAL_COMPRESS_LZ4;
     default:

source/common/src/tdataformat.c

@@ -3036,7 +3036,8 @@ _exit:
   return code;
 }
 
-int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen) {
+int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen, initGeosFn igeos,
+                               checkWKBGeometryFn cgeos) {
   int32_t code = 0;
 
   if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) {
@@ -3046,6 +3047,12 @@ int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32
   }
 
   if (IS_VAR_DATA_TYPE(pColData->type)) {  // var-length data type
+    if (pColData->type == TSDB_DATA_TYPE_GEOMETRY) {
+      code = igeos();
+      if (code) {
+        return code;
+      }
+    }
     for (int32_t i = 0; i < pBind->num; ++i) {
       if (pBind->is_null && pBind->is_null[i]) {
         if (pColData->cflag & COL_IS_KEY) {
@@ -3055,9 +3062,12 @@ int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32
         code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0);
         if (code) goto _exit;
       } else if (pBind->length[i] > buffMaxLen) {
-        uError("var data length too big, len:%d, max:%d", pBind->length[i], buffMaxLen);
+        return TSDB_CODE_PAR_VALUE_TOO_LONG;
-        return TSDB_CODE_INVALID_PARA;
       } else {
+        if (pColData->type == TSDB_DATA_TYPE_GEOMETRY) {
+          code = cgeos((char *)pBind->buffer + pBind->buffer_length * i, (size_t)pBind->length[i]);
+          if (code) goto _exit;
+        }
         code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](
             pColData, (uint8_t *)pBind->buffer + pBind->buffer_length * i, pBind->length[i]);
       }
@@ -3108,7 +3118,8 @@ _exit:
   return code;
 }
 
-int32_t tColDataAddValueByBind2(SColData *pColData, TAOS_STMT2_BIND *pBind, int32_t buffMaxLen) {
+int32_t tColDataAddValueByBind2(SColData *pColData, TAOS_STMT2_BIND *pBind, int32_t buffMaxLen, initGeosFn igeos,
+                                checkWKBGeometryFn cgeos) {
   int32_t code = 0;
 
   if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) {
@@ -3118,6 +3129,13 @@ int32_t tColDataAddValueByBind2(SColData *pColData, TAOS_STMT2_BIND *pBind, int3
   }
 
   if (IS_VAR_DATA_TYPE(pColData->type)) {  // var-length data type
+    if (pColData->type == TSDB_DATA_TYPE_GEOMETRY) {
+      code = igeos();
+      if (code) {
+        return code;
+      }
+    }
+
     uint8_t *buf = pBind->buffer;
     for (int32_t i = 0; i < pBind->num; ++i) {
       if (pBind->is_null && pBind->is_null[i]) {
@@ -3133,9 +3151,12 @@ int32_t tColDataAddValueByBind2(SColData *pColData, TAOS_STMT2_BIND *pBind, int3
           if (code) goto _exit;
         }
       } else if (pBind->length[i] > buffMaxLen) {
-        uError("var data length too big, len:%d, max:%d", pBind->length[i], buffMaxLen);
+        return TSDB_CODE_PAR_VALUE_TOO_LONG;
-        return TSDB_CODE_INVALID_PARA;
       } else {
+        if (pColData->type == TSDB_DATA_TYPE_GEOMETRY) {
+          code = cgeos(buf, pBind->length[i]);
+          if (code) goto _exit;
+        }
         code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, buf, pBind->length[i]);
         buf += pBind->length[i];
       }

source/common/src/tglobal.c

@@ -217,6 +217,8 @@ float   tsSelectivityRatio = 1.0;
 int32_t tsTagFilterResCacheSize = 1024 * 10;
 char    tsTagFilterCache = 0;
 
+int32_t tsBypassFlag = 0;
+
 // the maximum allowed query buffer size during query processing for each data node.
 // -1 no limit (default)
 // 0  no query allowed, queries are disabled
@@ -612,6 +614,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
       cfgAddInt64(pCfg, "randErrorDivisor", tsRandErrDivisor, 1, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
   TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorScope", tsRandErrScope, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "safetyCheckLevel", tsSafetyCheckLevel, 0, 5, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
+  TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "bypassFlag", tsBypassFlag, 0, INT32_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
 
   tsNumOfRpcThreads = tsNumOfCores / 2;
   tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 1, TSDB_MAX_RPC_THREADS);
@@ -1303,6 +1306,10 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
 
   TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "safetyCheckLevel");
   tsSafetyCheckLevel = pItem->i32;
+
+  TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "bypassFlag");
+  tsBypassFlag = pItem->i32;
+
   TAOS_RETURN(TSDB_CODE_SUCCESS);
 }
 
@@ -2046,7 +2053,8 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) {
                                          {"supportVnodes", &tsNumOfSupportVnodes},
                                          {"experimental", &tsExperimental},
                                          {"maxTsmaNum", &tsMaxTsmaNum},
-                                         {"safetyCheckLevel", &tsSafetyCheckLevel}};
+                                         {"safetyCheckLevel", &tsSafetyCheckLevel},
+                                         {"bypassFlag", &tsBypassFlag}};
 
     if ((code = taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true)) != TSDB_CODE_SUCCESS) {
       code = taosCfgSetOption(options, tListLen(options), pItem, false);
@@ -2302,7 +2310,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) {
                                          {"multiResultFunctionStarReturnTags", &tsMultiResultFunctionStarReturnTags},
                                          {"maxTsmaCalcDelay", &tsMaxTsmaCalcDelay},
                                          {"tsmaDataDeleteMark", &tsmaDataDeleteMark},
-                                         {"safetyCheckLevel", &tsSafetyCheckLevel}};
+                                         {"safetyCheckLevel", &tsSafetyCheckLevel},
+                                         {"bypassFlag", &tsBypassFlag}};
 
     if ((code = taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true)) != TSDB_CODE_SUCCESS) {
       code = taosCfgSetOption(options, tListLen(options), pItem, false);

source/common/test/CMakeLists.txt

@@ -46,7 +46,7 @@ if (${TD_LINUX})
     target_sources(tmsgTest 
         PRIVATE
         "tmsgTest.cpp"
-        "../src/tmsg.c"
+        "../src/msg/tmsg.c"
     )
     target_include_directories(tmsgTest PUBLIC "${TD_SOURCE_DIR}/include/common/")
     target_link_libraries(tmsgTest PUBLIC os util gtest gtest_main)

source/dnode/mgmt/mgmt_snode/src/smWorker.c

@@ -36,14 +36,15 @@ static void smProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
 
     dTrace("msg:%p, get from snode-write queue", pMsg);
     int32_t code = sndProcessWriteMsg(pMgmt->pSnode, pMsg, NULL);
-    if (code < 0) {
+    // if (code < 0) {
-      dGError("snd, msg:%p failed to process write since %s", pMsg, tstrerror(code));
+    //   dGError("snd, msg:%p failed to process write since %s", pMsg, tstrerror(code));
-      if (pMsg->info.handle != NULL) {
+    //   if (pMsg->info.handle != NULL) {
-        tmsgSendRsp(pMsg);
+    //     tmsgSendRsp(pMsg);
-      }
+    //   }
-    } else {
+    // } else {
-      smSendRsp(pMsg, 0);
+    //   smSendRsp(pMsg, 0);
-    }
+    // }
+    smSendRsp(pMsg, code);
 
     dTrace("msg:%p, is freed", pMsg);
     rpcFreeCont(pMsg->pCont);

source/dnode/mgmt/mgmt_vnode/inc/vmInt.h

@@ -37,7 +37,9 @@ typedef struct SVnodeMgmt {
   SSingleWorker         mgmtMultiWorker;
   SHashObj             *hash;
   SHashObj             *closedHash;
+  SHashObj             *creatingHash;
   TdThreadRwlock        lock;
+  TdThreadMutex         mutex;
   SVnodesStat           state;
   STfs                 *pTfs;
   TdThread              thread;
@@ -96,6 +98,7 @@ SVnodeObj *vmAcquireVnodeImpl(SVnodeMgmt *pMgmt, int32_t vgId, bool strict);
 void       vmReleaseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode);
 int32_t    vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl);
 void       vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal, bool keepClosed);
+void vmRemoveFromCreatingHash(SVnodeMgmt *pMgmt, int32_t vgId);
 
 // vmHandle.c
 SArray *vmGetMsgHandles();
@@ -113,6 +116,7 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t
 int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt);
 int32_t vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes);
 int32_t vmGetAllVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes);
+int32_t vmGetAllVnodeListFromHashWithCreating(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes);
 
 // vmWorker.c
 int32_t vmStartWorker(SVnodeMgmt *pMgmt);

source/dnode/mgmt/mgmt_vnode/src/vmFile.c

@@ -67,6 +67,54 @@ int32_t vmGetAllVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnod
   return 0;
 }
 
+int32_t vmGetAllVnodeListFromHashWithCreating(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes) {
+  (void)taosThreadRwlockRdlock(&pMgmt->lock);
+
+  int32_t num = 0;
+  int32_t size = taosHashGetSize(pMgmt->hash);
+  int32_t creatingSize = taosHashGetSize(pMgmt->creatingHash);
+  size += creatingSize;
+  SVnodeObj **pVnodes = taosMemoryCalloc(size, sizeof(SVnodeObj *));
+  if (pVnodes == NULL) {
+    (void)taosThreadRwlockUnlock(&pMgmt->lock);
+    return terrno;
+  }
+
+  void *pIter = taosHashIterate(pMgmt->hash, NULL);
+  while (pIter) {
+    SVnodeObj **ppVnode = pIter;
+    SVnodeObj  *pVnode = *ppVnode;
+    if (pVnode && num < size) {
+      int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
+      dTrace("vgId:%d,acquire vnode, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount);
+      pVnodes[num++] = (*ppVnode);
+      pIter = taosHashIterate(pMgmt->hash, pIter);
+    } else {
+      taosHashCancelIterate(pMgmt->hash, pIter);
+    }
+  }
+
+  pIter = taosHashIterate(pMgmt->creatingHash, NULL);
+  while (pIter) {
+    SVnodeObj **ppVnode = pIter;
+    SVnodeObj  *pVnode = *ppVnode;
+    if (pVnode && num < size) {
+      int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
+      dTrace("vgId:%d, acquire vnode, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount);
+      pVnodes[num++] = (*ppVnode);
+      pIter = taosHashIterate(pMgmt->creatingHash, pIter);
+    } else {
+      taosHashCancelIterate(pMgmt->creatingHash, pIter);
+    }
+  }
+  (void)taosThreadRwlockUnlock(&pMgmt->lock);
+
+  *numOfVnodes = num;
+  *ppVnodes = pVnodes;
+
+  return 0;
+}
+
 int32_t vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes) {
   (void)taosThreadRwlockRdlock(&pMgmt->lock);
 

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -381,6 +381,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   if (vnodeCreate(path, &vnodeCfg, diskPrimary, pMgmt->pTfs) < 0) {
     dError("vgId:%d, failed to create vnode since %s", req.vgId, terrstr());
     vmReleaseVnode(pMgmt, pVnode);
+    vmRemoveFromCreatingHash(pMgmt, req.vgId);
     (void)tFreeSCreateVnodeReq(&req);
     code = terrno != 0 ? terrno : -1;
     return code;
@@ -422,6 +423,8 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   }
 
 _OVER:
+  vmRemoveFromCreatingHash(pMgmt, req.vgId);
+
   if (code != 0) {
     int32_t r = 0;
     r = taosThreadRwlockWrlock(&pMgmt->lock);

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -16,6 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "vmInt.h"
 #include "libs/function/tudf.h"
+#include "osMemory.h"
 #include "tfs.h"
 #include "vnd.h"
 
@@ -62,10 +63,20 @@ int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) {
   int32_t     numOfVnodes = 0;
   SVnodeObj **ppVnodes = NULL;
 
-  code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
+  code = taosThreadMutexLock(&pMgmt->mutex);
   if (code != 0) {
     return code;
   }
+
+  code = vmGetAllVnodeListFromHashWithCreating(pMgmt, &numOfVnodes, &ppVnodes);
+  if (code != 0) {
+    int32_t r = taosThreadMutexUnlock(&pMgmt->mutex);
+    if (r != 0) {
+      dError("vgId:%d, failed to unlock mutex since %s", vgId, tstrerror(r));
+    }
+    return code;
+  }
+
   for (int32_t v = 0; v < numOfVnodes; v++) {
     SVnodeObj *pVnode = ppVnodes[v];
     disks[pVnode->diskPrimary] += 1;
@@ -81,6 +92,51 @@ int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) {
     }
   }
 
+  SVnodeObj *pCreatingVnode = taosMemoryCalloc(1, sizeof(SVnodeObj));
+  if (pCreatingVnode == NULL) {
+    code = -1;
+    if (terrno != 0) code = terrno;
+    dError("failed to alloc vnode since %s", tstrerror(code));
+    int32_t r = taosThreadMutexUnlock(&pMgmt->mutex);
+    if (r != 0) {
+      dError("vgId:%d, failed to unlock mutex since %s", vgId, tstrerror(r));
+    }
+    goto _OVER;
+  }
+  (void)memset(pCreatingVnode, 0, sizeof(SVnodeObj));
+
+  pCreatingVnode->vgId = vgId;
+  pCreatingVnode->diskPrimary = diskId;
+
+  code = taosThreadRwlockWrlock(&pMgmt->lock);
+  if (code != 0) {
+    int32_t r = taosThreadMutexUnlock(&pMgmt->mutex);
+    if (r != 0) {
+      dError("vgId:%d, failed to unlock mutex since %s", vgId, tstrerror(r));
+    }
+    taosMemoryFree(pCreatingVnode);
+    goto _OVER;
+  }
+
+  dTrace("vgId:%d, put vnode into creating hash, pCreatingVnode:%p", vgId, pCreatingVnode);
+  code = taosHashPut(pMgmt->creatingHash, &vgId, sizeof(int32_t), &pCreatingVnode, sizeof(SVnodeObj *));
+  if (code != 0) {
+    dError("vgId:%d, failed to put vnode to creatingHash", vgId);
+    taosMemoryFree(pCreatingVnode);
+  }
+
+  int32_t r = taosThreadRwlockUnlock(&pMgmt->lock);
+  if (r != 0) {
+    dError("vgId:%d, failed to unlock since %s", vgId, tstrerror(r));
+  }
+
+  code = taosThreadMutexUnlock(&pMgmt->mutex);
+  if (code != 0) {
+    goto _OVER;
+  }
+
+_OVER:
+
   for (int32_t i = 0; i < numOfVnodes; ++i) {
     if (ppVnodes == NULL || ppVnodes[i] == NULL) continue;
     vmReleaseVnode(pMgmt, ppVnodes[i]);
@@ -89,8 +145,13 @@ int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) {
     taosMemoryFree(ppVnodes);
   }
 
-  dInfo("vgId:%d, alloc disk:%d of level 0. ndisk:%d, vnodes: %d", vgId, diskId, ndisk, numOfVnodes);
+  if (code != 0) {
-  return diskId;
+    dError("vgId:%d, failed to alloc disk since %s", vgId, tstrerror(code));
+    return code;
+  } else {
+    dInfo("vgId:%d, alloc disk:%d of level 0. ndisk:%d, vnodes: %d", vgId, diskId, ndisk, numOfVnodes);
+    return diskId;
+  }
 }
 
 SVnodeObj *vmAcquireVnodeImpl(SVnodeMgmt *pMgmt, int32_t vgId, bool strict) {
@@ -216,12 +277,12 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal,
   }
   if (keepClosed) {
     SVnodeObj *pClosedVnode = taosMemoryCalloc(1, sizeof(SVnodeObj));
-    (void)memset(pClosedVnode, 0, sizeof(SVnodeObj));
+    if (pClosedVnode == NULL) {
-    if (pVnode == NULL) {
+      dError("failed to alloc vnode since %s", terrstr());
-      dError("vgId:%d, failed to alloc vnode since %s", pVnode->vgId, terrstr());
       (void)taosThreadRwlockUnlock(&pMgmt->lock);
       return;
     }
+    (void)memset(pClosedVnode, 0, sizeof(SVnodeObj));
 
     pClosedVnode->vgId = pVnode->vgId;
     pClosedVnode->dropped = pVnode->dropped;
@@ -427,11 +488,18 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) {
 
   pMgmt->closedHash =
       taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
-  if (pMgmt->hash == NULL) {
+  if (pMgmt->closedHash == NULL) {
     dError("failed to init vnode closed hash since %s", terrstr());
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
+  pMgmt->creatingHash =
+      taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
+  if (pMgmt->creatingHash == NULL) {
+    dError("failed to init vnode creatingHash hash since %s", terrstr());
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
   SWrapperCfg *pCfgs = NULL;
   int32_t      numOfVnodes = 0;
   if (vmGetVnodeListFromFile(pMgmt, &pCfgs, &numOfVnodes) != 0) {
@@ -509,6 +577,32 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) {
   return 0;
 }
 
+void vmRemoveFromCreatingHash(SVnodeMgmt *pMgmt, int32_t vgId) {
+  SVnodeObj *pOld = NULL;
+
+  (void)taosThreadRwlockWrlock(&pMgmt->lock);
+  int32_t    r = taosHashGetDup(pMgmt->creatingHash, &vgId, sizeof(int32_t), (void *)&pOld);
+  if (r != 0) {
+    dError("vgId:%d, failed to get vnode from creating Hash", vgId);
+  }
+  dTrace("vgId:%d, remove from creating Hash", vgId);
+  r = taosHashRemove(pMgmt->creatingHash, &vgId, sizeof(int32_t));
+  if (r != 0) {
+    dError("vgId:%d, failed to remove vnode from hash", vgId);
+  }
+  (void)taosThreadRwlockUnlock(&pMgmt->lock);
+
+  if (pOld) {
+    dTrace("vgId:%d, free vnode pOld:%p", vgId, &pOld);
+    vmFreeVnodeObj(&pOld);
+  }
+
+_OVER:
+  if (r != 0) {
+    dError("vgId:%d, failed to remove vnode from creatingHash since %s", vgId, tstrerror(r));
+  }
+}
+
 static void *vmCloseVnodeInThread(void *param) {
   SVnodeThread *pThread = param;
   SVnodeMgmt   *pMgmt = pThread->pMgmt;
@@ -614,6 +708,18 @@ static void vmCloseVnodes(SVnodeMgmt *pMgmt) {
     pMgmt->closedHash = NULL;
   }
 
+  pIter = taosHashIterate(pMgmt->creatingHash, NULL);
+  while (pIter) {
+    SVnodeObj **ppVnode = pIter;
+    vmFreeVnodeObj(ppVnode);
+    pIter = taosHashIterate(pMgmt->creatingHash, pIter);
+  }
+
+  if (pMgmt->creatingHash != NULL) {
+    taosHashCleanup(pMgmt->creatingHash);
+    pMgmt->creatingHash = NULL;
+  }
+
   dInfo("total vnodes:%d are all closed", numOfVnodes);
 }
 
@@ -622,6 +728,7 @@ static void vmCleanup(SVnodeMgmt *pMgmt) {
   vmStopWorker(pMgmt);
   vnodeCleanup();
   (void)taosThreadRwlockDestroy(&pMgmt->lock);
+  (void)taosThreadMutexDestroy(&pMgmt->mutex);
   (void)taosThreadMutexDestroy(&pMgmt->fileLock);
   taosMemoryFree(pMgmt);
 }
@@ -714,6 +821,12 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
     goto _OVER;
   }
 
+  code = taosThreadMutexInit(&pMgmt->mutex, NULL);
+  if (code != 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto _OVER;
+  }
+
   code = taosThreadMutexInit(&pMgmt->fileLock, NULL);
   if (code != 0) {
     code = TAOS_SYSTEM_ERROR(errno);

source/dnode/mgmt/node_mgmt/src/dmTransport.c

@@ -214,8 +214,6 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
   } else if ((pRpc->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pRpc->code == TSDB_CODE_RPC_BROKEN_LINK) &&
              (!IsReq(pRpc)) && (pRpc->pCont == NULL)) {
     dGError("msg:%p, type:%s pCont is NULL, err: %s", pRpc, TMSG_INFO(pRpc->msgType), tstrerror(pRpc->code));
-    code = pRpc->code;
-    goto _OVER;
   }
 
   if (pHandle->defaultNtype == NODE_END) {

source/dnode/mnode/impl/inc/mndStream.h

@@ -56,6 +56,7 @@ typedef struct SStreamTransMgmt {
 typedef struct SStreamTaskResetMsg {
   int64_t streamId;
   int32_t transId;
+  int64_t checkpointId;
 } SStreamTaskResetMsg;
 
 typedef struct SChkptReportInfo {
@@ -142,9 +143,9 @@ int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pSt
 int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
 int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
 int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList);
-int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
+int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream, int64_t chkptId);
 int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
-int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream);
+int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream, int64_t chkptId);
 int32_t mndStreamSetChkptIdAction(SMnode *pMnode, STrans *pTrans, SStreamTask* pTask, int64_t checkpointId, int64_t ts);
 int32_t mndStreamSetRestartAction(SMnode* pMnode, STrans *pTrans, SStreamObj* pStream);
 int32_t mndStreamSetCheckpointAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, int64_t checkpointId,

source/dnode/mnode/impl/src/mndMain.c

@@ -53,7 +53,7 @@ static inline int32_t mndAcquireRpc(SMnode *pMnode) {
   if (pMnode->stopped) {
     code = TSDB_CODE_APP_IS_STOPPING;
   } else if (!mndIsLeader(pMnode)) {
-    code = -1;
+    code = 1;
   } else {
 #if 1
     (void)atomic_add_fetch_32(&pMnode->rpcRef, 1);
@@ -1002,8 +1002,12 @@ int64_t mndGenerateUid(const char *name, int32_t len) {
 
 int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
                           SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) {
-  int32_t code = 0;
+  int32_t code = mndAcquireRpc(pMnode);
-  TAOS_CHECK_RETURN(mndAcquireRpc(pMnode));
+  if (code < 0) {
+    TAOS_RETURN(code);
+  } else if (code == 1) {
+    TAOS_RETURN(TSDB_CODE_SUCCESS);
+  }
 
   SSdb   *pSdb = pMnode->pSdb;
   int64_t ms = taosGetTimestampMs();

source/dnode/mnode/impl/src/mndStream.c

@@ -2434,7 +2434,12 @@ static void doAddReportStreamTask(SArray *pList, int64_t reportChkptId, const SC
         mDebug("s-task:0x%x expired checkpoint-report msg in checkpoint-report list update from %" PRId64 "->%" PRId64,
                pReport->taskId, p->checkpointId, pReport->checkpointId);
 
-        memcpy(p, pReport, sizeof(STaskChkptInfo));
+        // update the checkpoint report info
+        p->checkpointId = pReport->checkpointId;
+        p->ts = pReport->checkpointTs;
+        p->version = pReport->checkpointVer;
+        p->transId = pReport->transId;
+        p->dropHTask = pReport->dropHTask;
       } else {
         mWarn("taskId:0x%x already in checkpoint-report list", pReport->taskId);
       }

source/dnode/mnode/impl/src/mndStreamErrorInjection.c

@@ -0,0 +1,72 @@
+#include "mndTrans.h"
+
+uint32_t seed = 0;
+
+static SRpcMsg createRpcMsg(STransAction* pAction, int64_t traceId, int64_t signature) {
+  SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen, .info.ahandle = (void *)signature};
+  rpcMsg.pCont = rpcMallocCont(pAction->contLen);
+  if (rpcMsg.pCont == NULL) {
+    return rpcMsg;
+  }
+
+  rpcMsg.info.traceId.rootId = traceId;
+  rpcMsg.info.notFreeAhandle = 1;
+
+  memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen);
+  return rpcMsg;
+}
+
+void streamTransRandomErrorGen(STransAction *pAction, STrans *pTrans, int64_t signature) {
+  if ((pAction->msgType == TDMT_STREAM_TASK_UPDATE_CHKPT && pAction->id > 2) ||
+      (pAction->msgType == TDMT_STREAM_CONSEN_CHKPT) ||
+      (pAction->msgType == TDMT_VND_STREAM_CHECK_POINT_SOURCE && pAction->id > 2)) {
+    if (seed == 0) {
+      seed = taosGetTimestampSec();
+    }
+
+    uint32_t v = taosRandR(&seed);
+    int32_t  choseItem = v % 5;
+
+    if (choseItem == 0) {
+      // 1. one of update-checkpoint not send, restart and send it again
+      taosMsleep(5000);
+      if (pAction->msgType == TDMT_STREAM_TASK_UPDATE_CHKPT) {
+        mError(
+            "***sleep 5s and core dump, following tasks will not recv update-checkpoint info, so the checkpoint will "
+            "rollback***");
+        exit(-1);
+      } else if (pAction->msgType == TDMT_STREAM_CONSEN_CHKPT) {  // pAction->msgType == TDMT_STREAM_CONSEN_CHKPT
+        mError(
+            "***sleep 5s and core dump, following tasks will not recv consen-checkpoint info, so the tasks will "
+            "not started***");
+      } else {  // pAction->msgType == TDMT_VND_STREAM_CHECK_POINT_SOURCE
+        mError(
+            "***sleep 5s and core dump, following tasks will not recv checkpoint-source info, so the tasks will "
+            "started after restart***");
+        exit(-1);
+      }
+    } else if (choseItem == 1) {
+      // 2. repeat send update chkpt msg
+      mError("***repeat send update-checkpoint/consensus/checkpoint trans msg 3times to vnode***");
+
+      mError("***repeat 1***");
+      SRpcMsg rpcMsg1 = createRpcMsg(pAction, pTrans->mTraceId, signature);
+      int32_t code = tmsgSendReq(&pAction->epSet, &rpcMsg1);
+
+      mError("***repeat 2***");
+      SRpcMsg rpcMsg2 = createRpcMsg(pAction, pTrans->mTraceId, signature);
+      code = tmsgSendReq(&pAction->epSet, &rpcMsg2);
+
+      mError("***repeat 3***");
+      SRpcMsg rpcMsg3 = createRpcMsg(pAction, pTrans->mTraceId, signature);
+      code = tmsgSendReq(&pAction->epSet, &rpcMsg3);
+    } else if (choseItem == 2) {
+      // 3. sleep 40s and then send msg
+      mError("***idle for 30s, and then send msg***");
+      taosMsleep(30000);
+    } else {
+      // do nothing
+      //      mInfo("no error triggered");
+    }
+  }
+}

source/dnode/mnode/impl/src/mndStreamHb.c

@@ -24,7 +24,7 @@ typedef struct SFailedCheckpointInfo {
 
 static int32_t mndStreamSendUpdateChkptInfoMsg(SMnode *pMnode);
 static int32_t mndSendDropOrphanTasksMsg(SMnode *pMnode, SArray *pList);
-static int32_t mndSendResetFromCheckpointMsg(SMnode *pMnode, int64_t streamId, int32_t transId);
+static int32_t mndSendResetFromCheckpointMsg(SMnode *pMnode, int64_t streamId, int32_t transId, int64_t checkpointId);
 static void    updateStageInfo(STaskStatusEntry *pTaskEntry, int64_t stage);
 static void    addIntoFailedChkptList(SArray *pList, const SFailedCheckpointInfo *pInfo);
 static int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList);
@@ -68,7 +68,7 @@ void addIntoFailedChkptList(SArray *pList, const SFailedCheckpointInfo *pInfo) {
   }
 }
 
-int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) {
+int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream, int64_t chkptId) {
   STrans *pTrans = NULL;
   int32_t code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_RESET_NAME,
                                " reset from failed checkpoint", &pTrans);
@@ -84,7 +84,7 @@ int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) {
     return code;
   }
 
-  code = mndStreamSetResetTaskAction(pMnode, pTrans, pStream);
+  code = mndStreamSetResetTaskAction(pMnode, pTrans, pStream, chkptId);
   if (code) {
     sdbRelease(pMnode->pSdb, pStream);
     mndTransDrop(pTrans);
@@ -115,7 +115,7 @@ int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) {
   return code;
 }
 
-int32_t mndSendResetFromCheckpointMsg(SMnode *pMnode, int64_t streamId, int32_t transId) {
+int32_t mndSendResetFromCheckpointMsg(SMnode *pMnode, int64_t streamId, int32_t transId, int64_t checkpointId) {
   int32_t size = sizeof(SStreamTaskResetMsg);
 
   int32_t num = taosArrayGetSize(execInfo.pKilledChkptTrans);
@@ -135,8 +135,9 @@ int32_t mndSendResetFromCheckpointMsg(SMnode *pMnode, int64_t streamId, int32_t
     taosArrayRemove(execInfo.pKilledChkptTrans, 0);  // remove this first, append new reset trans in the tail
   }
 
-  SStreamTaskResetMsg p = {.streamId = streamId, .transId = transId};
+  SStreamTaskResetMsg p = {.streamId = streamId, .transId = transId, .checkpointId = checkpointId};
 
+  // let's remember that this trans had been killed already
   void *px = taosArrayPush(execInfo.pKilledChkptTrans, &p);
   if (px == NULL) {
     mError("failed to push reset-msg trans:%d into the killed chkpt trans list, size:%d", transId, num - 1);
@@ -150,6 +151,7 @@ int32_t mndSendResetFromCheckpointMsg(SMnode *pMnode, int64_t streamId, int32_t
 
   pReq->streamId = streamId;
   pReq->transId = transId;
+  pReq->checkpointId = checkpointId;
 
   SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_TASK_RESET, .pCont = pReq, .contLen = size};
   int32_t code = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
@@ -234,7 +236,7 @@ int32_t mndProcessResetStatusReq(SRpcMsg *pReq) {
     } else {
       mDebug("stream:%s (0x%" PRIx64 ") reset checkpoint procedure, transId:%d, create reset trans", pStream->name,
              pStream->uid, pMsg->transId);
-      code = mndCreateStreamResetStatusTrans(pMnode, pStream);
+      code = mndCreateStreamResetStatusTrans(pMnode, pStream, pMsg->checkpointId);
     }
   }
 
@@ -379,9 +381,10 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
     }
 
     if ((pEntry->lastHbMsgId == req.msgId) && (pEntry->lastHbMsgTs == req.ts)) {
-      mError("vgId:%d HbMsgId:%d already handled, bh msg discard", pEntry->nodeId, req.msgId);
+      mError("vgId:%d HbMsgId:%d already handled, bh msg discard, and send HbRsp", pEntry->nodeId, req.msgId);
 
-      terrno = TSDB_CODE_INVALID_MSG;
+      // return directly and after the vnode to continue to send the next HbMsg.
+      terrno = TSDB_CODE_SUCCESS;
       doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId);
 
       streamMutexUnlock(&execInfo.lock);
@@ -495,10 +498,11 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
           continue;
         }
 
-        mInfo("checkpointId:%" PRId64 " transId:%d failed, issue task-reset trans to reset all tasks status",
+        mInfo("stream:0x%" PRIx64 " checkpointId:%" PRId64
-              pInfo->checkpointId, pInfo->transId);
+              " transId:%d failed issue task-reset trans to reset all tasks status",
+              pInfo->streamUid, pInfo->checkpointId, pInfo->transId);
 
-        code = mndSendResetFromCheckpointMsg(pMnode, pInfo->streamUid, pInfo->transId);
+        code = mndSendResetFromCheckpointMsg(pMnode, pInfo->streamUid, pInfo->transId, pInfo->checkpointId);
         if (code) {
           mError("failed to create reset task trans, code:%s", tstrerror(code));
         }
@@ -549,12 +553,37 @@ void cleanupAfterProcessHbMsg(SStreamHbMsg *pReq, SArray *pFailedChkptList, SArr
 }
 
 void doSendHbMsgRsp(int32_t code, SRpcHandleInfo *pRpcInfo, int32_t vgId, int32_t msgId) {
-  SRpcMsg rsp = {.code = code, .info = *pRpcInfo, .contLen = sizeof(SMStreamHbRspMsg)};
+  int32_t ret = 0;
-  rsp.pCont = rpcMallocCont(rsp.contLen);
+  int32_t tlen = 0;
+  void   *buf = NULL;
 
-  SMStreamHbRspMsg *pMsg = rsp.pCont;
+  const SMStreamHbRspMsg msg = {.msgId = msgId};
-  pMsg->head.vgId = htonl(vgId);
+
-  pMsg->msgId = msgId;
+  tEncodeSize(tEncodeStreamHbRsp, &msg, tlen, ret);
+  if (ret < 0) {
+    mError("encode stream hb msg rsp failed, code:%s", tstrerror(code));
+  }
+
+  buf = rpcMallocCont(tlen + sizeof(SMsgHead));
+  if (buf == NULL) {
+    mError("encode stream hb msg rsp failed, code:%s", tstrerror(terrno));
+    return;
+  }
+
+  ((SMStreamHbRspMsg*)buf)->head.vgId = htonl(vgId);
+  void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
+
+  SEncoder encoder;
+  tEncoderInit(&encoder, abuf, tlen);
+  if ((code = tEncodeStreamHbRsp(&encoder, &msg)) < 0) {
+    rpcFreeCont(buf);
+    tEncoderClear(&encoder);
+    mError("encode stream hb msg rsp failed, code:%s", tstrerror(code));
+    return;
+  }
+  tEncoderClear(&encoder);
+
+  SRpcMsg rsp = {.code = code, .info = *pRpcInfo, .contLen = tlen + sizeof(SMsgHead), .pCont = buf};
 
   tmsgSendRsp(&rsp);
   pRpcInfo->handle = NULL;  // disable auto rsp

source/dnode/mnode/impl/src/mndStreamTransAct.c

@@ -295,7 +295,7 @@ static int32_t doSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamTas
   return code;
 }
 
-static int32_t doSetResetAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) {
+static int32_t doSetResetAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, int64_t chkptId) {
   SVResetStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResetStreamTaskReq));
   if (pReq == NULL) {
     mError("failed to malloc in reset stream, size:%" PRIzu ", code:%s", sizeof(SVResetStreamTaskReq),
@@ -306,6 +306,7 @@ static int32_t doSetResetAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
   pReq->head.vgId = htonl(pTask->info.nodeId);
   pReq->taskId = pTask->id.taskId;
   pReq->streamId = pTask->id.streamId;
+  pReq->chkptId = chkptId;
 
   SEpSet  epset = {0};
   bool    hasEpset = false;
@@ -544,7 +545,7 @@ int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray* p
   return 0;
 }
 
-int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
+int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream, int64_t chkptId) {
   SStreamTaskIter *pIter = NULL;
 
   taosWLockLatch(&pStream->lock);
@@ -564,7 +565,7 @@ int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *
       return code;
     }
 
-    code = doSetResetAction(pMnode, pTrans, pTask);
+    code = doSetResetAction(pMnode, pTrans, pTask, chkptId);
     if (code != TSDB_CODE_SUCCESS) {
       destroyStreamTaskIter(pIter);
       taosWUnLockLatch(&pStream->lock);
@@ -606,7 +607,7 @@ int32_t mndStreamSetChkptIdAction(SMnode *pMnode, STrans *pTrans, SStreamTask* p
   tEncoderInit(&encoder, abuf, tlen);
   code = tEncodeRestoreCheckpointInfo(&encoder, &req);
   tEncoderClear(&encoder);
-  if (code == -1) {
+  if (code < 0) {
     taosMemoryFree(pBuf);
     return code;
   }

source/dnode/mnode/impl/src/mndStreamUtil.c

@@ -1521,74 +1521,4 @@ int32_t mndCheckForSnode(SMnode *pMnode, SDbObj *pSrcDb) {
     mError("snode not existed when trying to create stream in db with multiple replica");
     return TSDB_CODE_SNODE_NOT_DEPLOYED;
   }
-}
-
-uint32_t seed = 0;
-static SRpcMsg createRpcMsg(STransAction* pAction, int64_t traceId, int64_t signature) {
-  SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen, .info.ahandle = (void *)signature};
-  rpcMsg.pCont = rpcMallocCont(pAction->contLen);
-  if (rpcMsg.pCont == NULL) {
-    return rpcMsg;
-  }
-
-  rpcMsg.info.traceId.rootId = traceId;
-  rpcMsg.info.notFreeAhandle = 1;
-
-  memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen);
-  return rpcMsg;
-}
-
-void streamTransRandomErrorGen(STransAction *pAction, STrans *pTrans, int64_t signature) {
-  if ((pAction->msgType == TDMT_STREAM_TASK_UPDATE_CHKPT && pAction->id > 2) ||
-      (pAction->msgType == TDMT_STREAM_CONSEN_CHKPT) ||
-      (pAction->msgType == TDMT_VND_STREAM_CHECK_POINT_SOURCE && pAction->id > 2)) {
-    if (seed == 0) {
-      seed = taosGetTimestampSec();
-    }
-
-    uint32_t v = taosRandR(&seed);
-    int32_t  choseItem = v % 5;
-
-    if (choseItem == 0) {
-      // 1. one of update-checkpoint not send, restart and send it again
-      taosMsleep(5000);
-      if (pAction->msgType == TDMT_STREAM_TASK_UPDATE_CHKPT) {
-        mError(
-            "***sleep 5s and core dump, following tasks will not recv update-checkpoint info, so the checkpoint will "
-            "rollback***");
-        exit(-1);
-      } else if (pAction->msgType == TDMT_STREAM_CONSEN_CHKPT) {  // pAction->msgType == TDMT_STREAM_CONSEN_CHKPT
-        mError(
-            "***sleep 5s and core dump, following tasks will not recv consen-checkpoint info, so the tasks will "
-            "not started***");
-      } else {  // pAction->msgType == TDMT_VND_STREAM_CHECK_POINT_SOURCE
-        mError(
-            "***sleep 5s and core dump, following tasks will not recv checkpoint-source info, so the tasks will "
-            "started after restart***");
-        exit(-1);
-      }
-    } else if (choseItem == 1) {
-      // 2. repeat send update chkpt msg
-      mError("***repeat send update-checkpoint/consensus/checkpoint trans msg 3times to vnode***");
-
-      mError("***repeat 1***");
-      SRpcMsg rpcMsg1 = createRpcMsg(pAction, pTrans->mTraceId, signature);
-      int32_t code = tmsgSendReq(&pAction->epSet, &rpcMsg1);
-
-      mError("***repeat 2***");
-      SRpcMsg rpcMsg2 = createRpcMsg(pAction, pTrans->mTraceId, signature);
-      code = tmsgSendReq(&pAction->epSet, &rpcMsg2);
-
-      mError("***repeat 3***");
-      SRpcMsg rpcMsg3 = createRpcMsg(pAction, pTrans->mTraceId, signature);
-      code = tmsgSendReq(&pAction->epSet, &rpcMsg3);
-    } else if (choseItem == 2) {
-      // 3. sleep 40s and then send msg
-      mError("***idle for 30s, and then send msg***");
-      taosMsleep(30000);
-    } else {
-      // do nothing
-      //      mInfo("no error triggered");
-    }
-  }
 }

source/dnode/mnode/impl/test/stream/stream.cpp

@@ -246,7 +246,7 @@ TEST_F(StreamTest, kill_checkpoint_trans) {
   px = taosArrayPush(pStream->tasks, &pLevel);
   ASSERT(px != NULL);
 
-  code = mndCreateStreamResetStatusTrans(pMnode, pStream);
+  code = mndCreateStreamResetStatusTrans(pMnode, pStream, 1);
   ASSERT(code != 0);
 
   tFreeStreamObj(pStream);

source/dnode/vnode/src/meta/metaOpen.c

@@ -324,7 +324,11 @@ static int32_t metaGenerateNewMeta(SMeta **ppMeta) {
           SMetaEntry me = {0};
           tDecoderInit(&dc, value, valueSize);
           if (metaDecodeEntry(&dc, &me) == 0) {
-            if (metaHandleEntry(pNewMeta, &me) != 0) {
+            if (me.type == TSDB_CHILD_TABLE &&
+                tdbTbGet(pMeta->pUidIdx, &me.ctbEntry.suid, sizeof(me.ctbEntry.suid), NULL, NULL) != 0) {
+              metaError("vgId:%d failed to get super table uid:%" PRId64 " for child table uid:%" PRId64,
+                        TD_VID(pVnode), me.ctbEntry.suid, uid);
+            } else if (metaHandleEntry(pNewMeta, &me) != 0) {
               metaError("vgId:%d failed to handle entry, uid:%" PRId64, TD_VID(pVnode), uid);
             }
           }

source/dnode/vnode/src/tq/tq.c

@@ -1009,21 +1009,34 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
 }
 
 int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
-  SStreamTaskRunReq* pReq = pMsg->pCont;
+  int32_t  code = 0;
+  char*    msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
+  int32_t  len = pMsg->contLen - sizeof(SMsgHead);
+  SDecoder decoder;
+
+  SStreamTaskRunReq req = {0};
+  tDecoderInit(&decoder, (uint8_t*)msg, len);
+  if ((code = tDecodeStreamTaskRunReq(&decoder, &req)) < 0) {
+    tqError("vgId:%d failed to decode task run req, code:%s", pTq->pStreamMeta->vgId, tstrerror(code));
+    tDecoderClear(&decoder);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  tDecoderClear(&decoder);
 
   // extracted submit data from wal files for all tasks
-  if (pReq->reqType == STREAM_EXEC_T_EXTRACT_WAL_DATA) {
+  if (req.reqType == STREAM_EXEC_T_EXTRACT_WAL_DATA) {
     return tqScanWal(pTq);
   }
 
-  int32_t code = tqStreamTaskProcessRunReq(pTq->pStreamMeta, pMsg, vnodeIsRoleLeader(pTq->pVnode));
+  code = tqStreamTaskProcessRunReq(pTq->pStreamMeta, pMsg, vnodeIsRoleLeader(pTq->pVnode));
   if (code) {
     tqError("vgId:%d failed to create task run req, code:%s", TD_VID(pTq->pVnode), tstrerror(code));
     return code;
   }
 
   // let's continue scan data in the wal files
-  if (pReq->reqType >= 0 || pReq->reqType == STREAM_EXEC_T_RESUME_TASK) {
+  if (req.reqType >= 0 || req.reqType == STREAM_EXEC_T_RESUME_TASK) {
     code = tqScanWalAsync(pTq, false);  // it's ok to failed
     if (code) {
       tqError("vgId:%d failed to start scan wal file, code:%s", pTq->pStreamMeta->vgId, tstrerror(code));
@@ -1297,7 +1310,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
 int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg) {
   int32_t vgId = TD_VID(pTq->pVnode);
 
-  SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*)pMsg->pCont;
+  SStreamCheckpointReadyMsg* pReq = (SStreamCheckpointReadyMsg*)pMsg->pCont;
   if (!vnodeIsRoleLeader(pTq->pVnode)) {
     tqError("vgId:%d not leader, ignore the retrieve checkpoint-trigger msg from 0x%x", vgId,
             (int32_t)pReq->downstreamTaskId);
@@ -1318,10 +1331,23 @@ int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) {
 int32_t tqProcessTaskRetrieveTriggerReq(STQ* pTq, SRpcMsg* pMsg) {
   int32_t vgId = TD_VID(pTq->pVnode);
 
-  SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*)pMsg->pCont;
   if (!vnodeIsRoleLeader(pTq->pVnode)) {
-    tqError("vgId:%d not leader, ignore the retrieve checkpoint-trigger msg from 0x%x", vgId,
+    SRetrieveChkptTriggerReq req = {0};
-            (int32_t)pReq->downstreamTaskId);
+
+    char*    msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
+    int32_t  len = pMsg->contLen - sizeof(SMsgHead);
+    SDecoder decoder = {0};
+
+    tDecoderInit(&decoder, (uint8_t*)msg, len);
+    if (tDecodeRetrieveChkptTriggerReq(&decoder, &req) < 0) {
+      tDecoderClear(&decoder);
+      tqError("vgId:%d invalid retrieve checkpoint-trigger req received", vgId);
+      return TSDB_CODE_INVALID_MSG;
+    }
+    tDecoderClear(&decoder);
+
+    tqError("vgId:%d not leader, ignore the retrieve checkpoint-trigger msg from s-task:0x%" PRId64, vgId,
+            req.downstreamTaskId);
     return TSDB_CODE_STREAM_NOT_LEADER;
   }
 

source/dnode/vnode/src/tqCommon/tqCommon.c

@@ -828,14 +828,25 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
 }
 
 int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader) {
-  SStreamTaskRunReq* pReq = pMsg->pCont;
+  int32_t  code = 0;
+  int32_t  vgId = pMeta->vgId;
+  char*    msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
+  int32_t  len = pMsg->contLen - sizeof(SMsgHead);
+  SDecoder decoder;
 
-  int32_t type = pReq->reqType;
+  SStreamTaskRunReq req = {0};
-  int32_t vgId = pMeta->vgId;
+  tDecoderInit(&decoder, (uint8_t*)msg, len);
-  int32_t code = 0;
+  if ((code = tDecodeStreamTaskRunReq(&decoder, &req)) < 0) {
+    tqError("vgId:%d failed to decode task run req, code:%s", pMeta->vgId, tstrerror(code));
+    tDecoderClear(&decoder);
+    return TSDB_CODE_SUCCESS;
+  }
 
+  tDecoderClear(&decoder);
+
+  int32_t type = req.reqType;
   if (type == STREAM_EXEC_T_START_ONE_TASK) {
-    code = streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId);
+    code = streamMetaStartOneTask(pMeta, req.streamId, req.taskId);
     return 0;
   } else if (type == STREAM_EXEC_T_START_ALL_TASKS) {
     code = streamMetaStartAllTasks(pMeta);
@@ -847,11 +858,11 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
     code = streamMetaStopAllTasks(pMeta);
     return 0;
   } else if (type == STREAM_EXEC_T_ADD_FAILED_TASK) {
-    code = streamMetaAddFailedTask(pMeta, pReq->streamId, pReq->taskId);
+    code = streamMetaAddFailedTask(pMeta, req.streamId, req.taskId);
     return code;
   } else if (type == STREAM_EXEC_T_RESUME_TASK) {  // task resume to run after idle for a while
     SStreamTask* pTask = NULL;
-    code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
+    code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
 
     if (pTask != NULL && (code == 0)) {
       char* pStatus = NULL;
@@ -873,7 +884,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
   }
 
   SStreamTask* pTask = NULL;
-  code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
+  code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
   if ((pTask != NULL) && (code == 0)) {  // even in halt status, the data in inputQ must be processed
     char* p = NULL;
     if (streamTaskReadyToRun(pTask, &p)) {
@@ -890,7 +901,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
     return 0;
   } else {  // NOTE: pTask->status.schedStatus is not updated since it is not be handled by the run exec.
     // todo add one function to handle this
-    tqError("vgId:%d failed to found s-task, taskId:0x%x may have been dropped", vgId, pReq->taskId);
+    tqError("vgId:%d failed to found s-task, taskId:0x%x may have been dropped", vgId, req.taskId);
     return code;
   }
 }
@@ -939,7 +950,7 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) {
 }
 
 int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
-  SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg;
+  SVResetStreamTaskReq* pReq = (SVResetStreamTaskReq*)pMsg;
 
   SStreamTask* pTask = NULL;
   int32_t      code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
@@ -954,17 +965,13 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
   streamMutexLock(&pTask->lock);
   streamTaskClearCheckInfo(pTask, true);
 
+  streamTaskSetFailedCheckpointId(pTask, pReq->chkptId);
+
   // clear flag set during do checkpoint, and open inputQ for all upstream tasks
   SStreamTaskState pState = streamTaskGetStatus(pTask);
   if (pState.state == TASK_STATUS__CK) {
-    int32_t tranId = 0;
-    int64_t activeChkId = 0;
-    streamTaskGetActiveCheckpointInfo(pTask, &tranId, &activeChkId);
-
-    tqDebug("s-task:%s reset task status from checkpoint, current checkpointingId:%" PRId64 ", transId:%d",
-            pTask->id.idStr, activeChkId, tranId);
-
     streamTaskSetStatusReady(pTask);
+    tqDebug("s-task:%s reset checkpoint status to ready", pTask->id.idStr);
   } else if (pState.state == TASK_STATUS__UNINIT) {
     //    tqDebug("s-task:%s start task by checking downstream tasks", pTask->id.idStr);
     //    tqStreamTaskRestoreCheckpoint(pMeta, pTask->id.streamId, pTask->id.taskId);
@@ -980,25 +987,36 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
 }
 
 int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
-  SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*)pMsg->pCont;
+  SRetrieveChkptTriggerReq req = {0};
+  SStreamTask*             pTask = NULL;
+  char*                    msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
+  int32_t                  len = pMsg->contLen - sizeof(SMsgHead);
+  SDecoder                 decoder = {0};
 
-  SStreamTask* pTask = NULL;
+  tDecoderInit(&decoder, (uint8_t*)msg, len);
-  int32_t      code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId, &pTask);
+  if (tDecodeRetrieveChkptTriggerReq(&decoder, &req) < 0) {
+    tDecoderClear(&decoder);
+    tqError("vgId:%d invalid retrieve checkpoint-trigger req received", pMeta->vgId);
+    return TSDB_CODE_INVALID_MSG;
+  }
+  tDecoderClear(&decoder);
+
+  int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId, &pTask);
   if (pTask == NULL || (code != 0)) {
-    tqError("vgId:%d process retrieve checkpoint trigger, checkpointId:%" PRId64
+    tqError("vgId:%d process retrieve checkpoint-trigger, checkpointId:%" PRId64
             " from s-task:0x%x, failed to acquire task:0x%x, it may have been dropped already",
-            pMeta->vgId, pReq->checkpointId, (int32_t)pReq->downstreamTaskId, pReq->upstreamTaskId);
+            pMeta->vgId, req.checkpointId, (int32_t)req.downstreamTaskId, req.upstreamTaskId);
     return TSDB_CODE_STREAM_TASK_NOT_EXIST;
   }
 
   tqDebug("s-task:0x%x recv retrieve checkpoint-trigger msg from downstream s-task:0x%x, checkpointId:%" PRId64,
-          pReq->upstreamTaskId, (int32_t)pReq->downstreamTaskId, pReq->checkpointId);
+          req.upstreamTaskId, (int32_t)req.downstreamTaskId, req.checkpointId);
 
   if (pTask->status.downstreamReady != 1) {
     tqError("s-task:%s not ready for checkpoint-trigger retrieve from 0x%x, since downstream not ready",
-            pTask->id.idStr, (int32_t)pReq->downstreamTaskId);
+            pTask->id.idStr, (int32_t)req.downstreamTaskId);
 
-    code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
+    code = streamTaskSendCheckpointTriggerMsg(pTask, req.downstreamTaskId, req.downstreamNodeId, &pMsg->info,
                                               TSDB_CODE_STREAM_TASK_IVLD_STATUS);
     streamMetaReleaseTask(pMeta, pTask);
     return code;
@@ -1010,19 +1028,19 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
     int64_t checkpointId = 0;
 
     streamTaskGetActiveCheckpointInfo(pTask, &transId, &checkpointId);
-    if (checkpointId != pReq->checkpointId) {
+    if (checkpointId != req.checkpointId) {
       tqError("s-task:%s invalid checkpoint-trigger retrieve msg from 0x%" PRIx64 ", current checkpointId:%" PRId64
               " req:%" PRId64,
-              pTask->id.idStr, pReq->downstreamTaskId, checkpointId, pReq->checkpointId);
+              pTask->id.idStr, req.downstreamTaskId, checkpointId, req.checkpointId);
       streamMetaReleaseTask(pMeta, pTask);
       return TSDB_CODE_INVALID_MSG;
     }
 
-    if (streamTaskAlreadySendTrigger(pTask, pReq->downstreamNodeId)) {
+    if (streamTaskAlreadySendTrigger(pTask, req.downstreamNodeId)) {
       // re-send the lost checkpoint-trigger msg to downstream task
       tqDebug("s-task:%s re-send checkpoint-trigger to:0x%x, checkpointId:%" PRId64 ", transId:%d", pTask->id.idStr,
-              (int32_t)pReq->downstreamTaskId, checkpointId, transId);
+              (int32_t)req.downstreamTaskId, checkpointId, transId);
-      code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
+      code = streamTaskSendCheckpointTriggerMsg(pTask, req.downstreamTaskId, req.downstreamNodeId, &pMsg->info,
                                                 TSDB_CODE_SUCCESS);
     } else {  // not send checkpoint-trigger yet, wait
       int32_t recv = 0, total = 0;
@@ -1036,7 +1054,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
             "sending checkpoint-source/trigger",
             pTask->id.idStr, recv, total);
       }
-      code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
+      code = streamTaskSendCheckpointTriggerMsg(pTask, req.downstreamTaskId, req.downstreamNodeId, &pMsg->info,
                                                 TSDB_CODE_ACTION_IN_PROGRESS);
     }
   } else {  // upstream not recv the checkpoint-source/trigger till now
@@ -1048,7 +1066,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
         "s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all "
         "upstream sending checkpoint-source/trigger",
         pTask->id.idStr);
-    code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
+    code = streamTaskSendCheckpointTriggerMsg(pTask, req.downstreamTaskId, req.downstreamNodeId, &pMsg->info,
                                               TSDB_CODE_ACTION_IN_PROGRESS);
   }
 
@@ -1057,23 +1075,34 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
 }
 
 int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
-  SCheckpointTriggerRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
+  SCheckpointTriggerRsp rsp = {0};
+  SStreamTask*          pTask = NULL;
+  char*                 msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
+  int32_t               len = pMsg->contLen - sizeof(SMsgHead);
+  SDecoder              decoder = {0};
 
-  SStreamTask* pTask = NULL;
+  tDecoderInit(&decoder, (uint8_t*)msg, len);
-  int32_t      code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId, &pTask);
+  if (tDecodeCheckpointTriggerRsp(&decoder, &rsp) < 0) {
+    tDecoderClear(&decoder);
+    tqError("vgId:%d invalid retrieve checkpoint-trigger rsp received", pMeta->vgId);
+    return TSDB_CODE_INVALID_MSG;
+  }
+  tDecoderClear(&decoder);
+
+  int32_t code = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.taskId, &pTask);
   if (pTask == NULL || (code != 0)) {
     tqError(
         "vgId:%d process retrieve checkpoint-trigger, failed to acquire task:0x%x, it may have been dropped already",
-        pMeta->vgId, pRsp->taskId);
+        pMeta->vgId, rsp.taskId);
     return code;
   }
 
   tqDebug(
-      "s-task:%s recv re-send checkpoint-trigger msg from through retrieve/rsp channel, upstream:0x%x, "
+      "s-task:%s recv re-send checkpoint-trigger msg through retrieve/rsp channel, upstream:0x%x, checkpointId:%" PRId64
-      "checkpointId:%" PRId64 ", transId:%d",
+      ", transId:%d",
-      pTask->id.idStr, pRsp->upstreamTaskId, pRsp->checkpointId, pRsp->transId);
+      pTask->id.idStr, rsp.upstreamTaskId, rsp.checkpointId, rsp.transId);
 
-  code = streamTaskProcessCheckpointTriggerRsp(pTask, pRsp);
+  code = streamTaskProcessCheckpointTriggerRsp(pTask, &rsp);
   streamMetaReleaseTask(pMeta, pTask);
   return code;
 }
@@ -1203,7 +1232,23 @@ int32_t doProcessDummyRspMsg(SStreamMeta* UNUSED_PARAM(pMeta), SRpcMsg* pMsg) {
 }
 
 int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
-  return streamProcessHeartbeatRsp(pMeta, pMsg->pCont);
+  SMStreamHbRspMsg rsp = {0};
+  int32_t          code = 0;
+  SDecoder         decoder;
+  char*            msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
+  int32_t          len = pMsg->contLen - sizeof(SMsgHead);
+
+  tDecoderInit(&decoder, (uint8_t*)msg, len);
+  code = tDecodeStreamHbRsp(&decoder, &rsp);
+  if (code < 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    tDecoderClear(&decoder);
+    tqError("vgId:%d failed to parse hb rsp msg, code:%s", pMeta->vgId, tstrerror(terrno));
+    return terrno;
+  }
+
+  tDecoderClear(&decoder);
+  return streamProcessHeartbeatRsp(pMeta, &rsp);
 }
 
 int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); }
@@ -1237,7 +1282,7 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
   SRestoreCheckpointInfo req = {0};
 
   tDecoderInit(&decoder, (uint8_t*)msg, len);
-  if (tDecodeRestoreCheckpointInfo(&decoder, &req) < 0) {
+  if ((code = tDecodeRestoreCheckpointInfo(&decoder, &req)) < 0) {
     tqError("vgId:%d failed to decode set consensus checkpointId req, code:%s", vgId, tstrerror(code));
     tDecoderClear(&decoder);
     return TSDB_CODE_SUCCESS;

source/dnode/vnode/src/tsdb/tsdbCommit2.c

@@ -667,7 +667,7 @@ int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info) {
   int64_t    nRow = imem->nRow;
   int64_t    nDel = imem->nDel;
 
-  if (nRow == 0 && nDel == 0) {
+  if ((nRow == 0 && nDel == 0) || (tsBypassFlag & TSDB_BYPASS_RB_TSDB_COMMIT)) {
     (void)taosThreadMutexLock(&tsdb->mutex);
     tsdb->imem = NULL;
     (void)taosThreadMutexUnlock(&tsdb->mutex);

source/dnode/vnode/src/tsdb/tsdbDataFileRW.c

@@ -972,7 +972,7 @@ static int32_t tsdbDataFileWriteBrinRecord(SDataFileWriter *writer, const SBrinR
     break;
   }
 
-  if ((writer->brinBlock->numOfRecords) >= writer->config->maxRow) {
+  if ((writer->brinBlock->numOfRecords) >= 256) {
     TAOS_CHECK_GOTO(tsdbDataFileWriteBrinBlock(writer), &lino, _exit);
   }
 

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -122,6 +122,10 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitTbData *pSubmi
   tb_uid_t   suid = pSubmitTbData->suid;
   tb_uid_t   uid = pSubmitTbData->uid;
 
+  if (tsBypassFlag & TSDB_BYPASS_RB_TSDB_WRITE_MEM) {
+    goto _err;
+  }
+
   // create/get STbData to op
   code = tsdbGetOrCreateTbData(pMemTable, suid, uid, &pTbData);
   if (code) {

source/dnode/vnode/src/tsdb/tsdbRead2.c

@@ -836,6 +836,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileRead
   pList = &pReader->status.uidList;
 
   int32_t i = 0;
+  int32_t j = 0;
   while (i < TARRAY2_SIZE(pBlkArray)) {
     pBrinBlk = &pBlkArray->data[i];
     if (pBrinBlk->maxTbid.suid < pReader->info.suid) {
@@ -851,7 +852,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileRead
         (pBrinBlk->minTbid.suid <= pReader->info.suid) && (pBrinBlk->maxTbid.suid >= pReader->info.suid), code, lino,
         _end, TSDB_CODE_INTERNAL_ERROR);
 
-    if (pBrinBlk->maxTbid.suid == pReader->info.suid && pBrinBlk->maxTbid.uid < pList->tableUidList[0]) {
+    if (pBrinBlk->maxTbid.suid == pReader->info.suid && pBrinBlk->maxTbid.uid < pList->tableUidList[j]) {
       i += 1;
       continue;
     }
@@ -864,6 +865,14 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileRead
     TSDB_CHECK_NULL(p1, code, lino, _end, terrno);
 
     i += 1;
+    if (pBrinBlk->maxTbid.suid == pReader->info.suid) {
+      while (j < numOfTables && pList->tableUidList[j] < pBrinBlk->maxTbid.uid) {
+        j++;
+      }
+      if (j >= numOfTables) {
+        break;
+      }
+    }
   }
 
   et2 = taosGetTimestampUs();
@@ -1134,7 +1143,12 @@ static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInf
   *pInfo = NULL;
 
   size_t num = TARRAY_SIZE(pBlockIter->blockList);
-  TSDB_CHECK_CONDITION(num != 0, code, lino, _end, TSDB_CODE_INVALID_PARA);
+  if (num == 0) {
+    // Some callers would attempt to call this function. Filter out certain normal cases and return directly to avoid
+    // generating excessive unnecessary error logs.
+    TSDB_CHECK_CONDITION(num == pBlockIter->numOfBlocks, code, lino, _end, TSDB_CODE_INVALID_PARA);
+    return TSDB_CODE_INVALID_PARA;
+  }
 
   *pInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index);
   TSDB_CHECK_NULL(*pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
@@ -4807,7 +4821,7 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn
   pBlockData = &pReader->status.fileBlockData;
   asc = ASCENDING_TRAVERSE(pReader->info.order);
   pVerRange = &pReader->info.verRange;
-  ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1;
+  step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1;
 
   *state = CHECK_FILEBLOCK_QUIT;
   code = loadNeighborIfOverlap(pFBlock, pScanInfo, pReader, &loadNeighbor);
@@ -5530,12 +5544,10 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi
     // update the SQueryTableDataCond to create inner reader
     int32_t order = pCond->order;
     if (order == TSDB_ORDER_ASC) {
-      pCond->twindows.ekey = window.skey - 1;
+      pCond->twindows = pCond->extTwindows[0];
-      pCond->twindows.skey = INT64_MIN;
       pCond->order = TSDB_ORDER_DESC;
     } else {
-      pCond->twindows.skey = window.ekey + 1;
+      pCond->twindows = pCond->extTwindows[1];
-      pCond->twindows.ekey = INT64_MAX;
       pCond->order = TSDB_ORDER_ASC;
     }
 
@@ -5544,11 +5556,9 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi
     TSDB_CHECK_CODE(code, lino, _end);
 
     if (order == TSDB_ORDER_ASC) {
-      pCond->twindows.skey = window.ekey + 1;
+      pCond->twindows = pCond->extTwindows[1];
-      pCond->twindows.ekey = INT64_MAX;
     } else {
-      pCond->twindows.skey = INT64_MIN;
+      pCond->twindows = pCond->extTwindows[0];
-      pCond->twindows.ekey = window.ekey - 1;
     }
     pCond->order = order;
 
@@ -6115,7 +6125,7 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) {
         TSDB_CHECK_CODE(code, lino, _end);
       }
 
-      goto _end;
+      return code;
     }
   }
 
@@ -6142,7 +6152,7 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) {
       acquired = false;
       TSDB_CHECK_CODE(code, lino, _end);
     }
-    goto _end;
+    return code;
   }
 
   if (pReader->step == EXTERNAL_ROWS_MAIN && pReader->innerReader[1] != NULL) {
@@ -6168,7 +6178,7 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) {
         TSDB_CHECK_CODE(code, lino, _end);
       }
 
-      goto _end;
+      return code;
     }
   }
 

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -362,6 +362,10 @@ static int32_t vnodePreProcessSubmitMsg(SVnode *pVnode, SRpcMsg *pMsg) {
   int32_t code = 0;
   int32_t lino = 0;
 
+  if (tsBypassFlag & TSDB_BYPASS_RA_RPC_RECV_SUBMIT) {
+    return TSDB_CODE_MSG_PREPROCESSED;
+  }
+
   SDecoder *pCoder = &(SDecoder){0};
 
   if (taosHton64(((SSubmitReq2Msg *)pMsg->pCont)->version) != 1) {

source/libs/catalog/inc/catalogInt.h

@@ -271,6 +271,7 @@ typedef struct SCtgViewsCtx {
   SArray* pNames;
   SArray* pResList;
   SArray* pFetchs;
+  bool    forceFetch;
 } SCtgViewsCtx;
 
 typedef enum {

source/libs/catalog/src/ctgAsync.c

@@ -20,6 +20,11 @@
 #include "tref.h"
 #include "trpc.h"
 
+typedef struct SCtgViewTaskParam {
+  bool    forceFetch;
+  SArray* pTableReqs;
+} SCtgViewTaskParam;
+
 void ctgIsTaskDone(SCtgJob* pJob, CTG_TASK_TYPE type, bool* done) {
   SCtgTask* pTask = NULL;
 
@@ -500,7 +505,7 @@ int32_t ctgInitGetTbTagTask(SCtgJob* pJob, int32_t taskIdx, void* param) {
 
 int32_t ctgInitGetViewsTask(SCtgJob* pJob, int32_t taskIdx, void* param) {
   SCtgTask task = {0};
-
+  SCtgViewTaskParam* p = param;
   task.type = CTG_TASK_GET_VIEW;
   task.taskId = taskIdx;
   task.pJob = pJob;
@@ -511,7 +516,8 @@ int32_t ctgInitGetViewsTask(SCtgJob* pJob, int32_t taskIdx, void* param) {
   }
 
   SCtgViewsCtx* ctx = task.taskCtx;
-  ctx->pNames = param;
+  ctx->pNames = p->pTableReqs;
+  ctx->forceFetch = p->forceFetch;
   ctx->pResList = taosArrayInit(pJob->viewNum, sizeof(SMetaRes));
   if (NULL == ctx->pResList) {
     qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->viewNum,
@@ -849,13 +855,12 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const
   int32_t tbCfgNum = (int32_t)taosArrayGetSize(pReq->pTableCfg);
   int32_t tbTagNum = (int32_t)taosArrayGetSize(pReq->pTableTag);
   int32_t viewNum = (int32_t)ctgGetTablesReqNum(pReq->pView);
-  int32_t tbTsmaNum = (int32_t)taosArrayGetSize(pReq->pTableTSMAs);
+  int32_t tbTsmaNum = tsQuerySmaOptimize ? (int32_t)taosArrayGetSize(pReq->pTableTSMAs) : 0;
   int32_t tsmaNum = (int32_t)taosArrayGetSize(pReq->pTSMAs);
   int32_t tbNameNum = (int32_t)ctgGetTablesReqNum(pReq->pTableName);
 
   int32_t taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dnodeNum + svrVerNum + dbCfgNum + indexNum +
                     userNum + dbInfoNum + tbIndexNum + tbCfgNum + tbTagNum + viewNum + tbTsmaNum + tbNameNum;
-
   *job = taosMemoryCalloc(1, sizeof(SCtgJob));
   if (NULL == *job) {
     ctgError("failed to calloc, size:%d,QID:0x%" PRIx64, (int32_t)sizeof(SCtgJob), pConn->requestId);
@@ -1014,7 +1019,8 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const
   }
 
   if (viewNum > 0) {
-    CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_VIEW, pReq->pView, NULL));
+    SCtgViewTaskParam param = {.forceFetch = pReq->forceFetchViewMeta, .pTableReqs = pReq->pView};
+    CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_VIEW, &param, NULL));
   }
   if (tbTsmaNum > 0) {
     CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_TSMA, pReq->pTableTSMAs, NULL));
@@ -3712,16 +3718,14 @@ int32_t ctgLaunchGetViewsTask(SCtgTask* pTask) {
   bool              tbMetaDone = false;
   SName*            pName = NULL;
 
-  /*
+  ctgIsTaskDone(pJob, CTG_TASK_GET_TB_META_BATCH, &tbMetaDone);
-    ctgIsTaskDone(pJob, CTG_TASK_GET_TB_META_BATCH, &tbMetaDone);
+  if (tbMetaDone && !pCtx->forceFetch) {
-    if (tbMetaDone) {
+    CTG_ERR_RET(ctgBuildViewNullRes(pTask, pCtx));
-      CTG_ERR_RET(ctgBuildViewNullRes(pTask, pCtx));
+    TSWAP(pTask->res, pCtx->pResList);
-      TSWAP(pTask->res, pCtx->pResList);
 
-      CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0));
+    CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0));
-      return TSDB_CODE_SUCCESS;
+    return TSDB_CODE_SUCCESS;
-    }
+  }
-  */
 
   int32_t dbNum = taosArrayGetSize(pCtx->pNames);
   int32_t fetchIdx = 0;

source/libs/executor/src/scanoperator.c

@@ -3040,7 +3040,6 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
                                   pBlockInfo->rows, pTaskInfo, &pTableScanInfo->base.metaCache);
     // ignore the table not exists error, since this table may have been dropped during the scan procedure.
     if (code) {
-      blockDataFreeRes((SSDataBlock*)pBlock);
       QUERY_CHECK_CODE(code, lino, _end);
     }
 
@@ -3465,11 +3464,6 @@ void streamScanOperatorDecode(void* pBuff, int32_t len, SStreamScanInfo* pInfo)
     goto _end;
   }
 
-  void* pUpInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo));
-  if (!pUpInfo) {
-    lino = __LINE__;
-    goto _end;
-  }
   SDecoder decoder = {0};
   pDeCoder = &decoder;
   tDecoderInit(pDeCoder, buf, tlen);
@@ -3478,14 +3472,21 @@ void streamScanOperatorDecode(void* pBuff, int32_t len, SStreamScanInfo* pInfo)
     goto _end;
   }
 
-  code = pInfo->stateStore.updateInfoDeserialize(pDeCoder, pUpInfo);
+  if (pInfo->pUpdateInfo != NULL) {
-  if (code == TSDB_CODE_SUCCESS) {
+    void* pUpInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo));
-    pInfo->stateStore.updateInfoDestroy(pInfo->pUpdateInfo);
+    if (!pUpInfo) {
-    pInfo->pUpdateInfo = pUpInfo;
+      lino = __LINE__;
-  } else {
+      goto _end;
-    taosMemoryFree(pUpInfo);
+    }
-    lino = __LINE__;
+    code = pInfo->stateStore.updateInfoDeserialize(pDeCoder, pUpInfo);
-    goto _end;
+    if (code == TSDB_CODE_SUCCESS) {
+      pInfo->stateStore.updateInfoDestroy(pInfo->pUpdateInfo);
+      pInfo->pUpdateInfo = pUpInfo;
+    } else {
+      taosMemoryFree(pUpInfo);
+      lino = __LINE__;
+      goto _end;
+    }
   }
 
   if (tDecodeIsEnd(pDeCoder)) {

source/libs/executor/src/timesliceoperator.c

@@ -1131,6 +1131,47 @@ static int32_t extractPkColumnFromFuncs(SNodeList* pFuncs, bool* pHasPk, SColumn
   return TSDB_CODE_SUCCESS;
 }
 
+/**
+ * @brief Determine the actual time range for reading data based on the RANGE clause and the WHERE conditions.
+ * @param[in] cond The range specified by WHERE condition.
+ * @param[in] range The range specified by RANGE clause.
+ * @param[out] twindow The range to be read in DESC order, and only one record is needed.
+ * @param[out] extTwindow The external range to read for only one record, which is used for FILL clause.
+ * @note `cond` and `twindow` may be the same address.
+ */
+static int32_t getQueryExtWindow(const STimeWindow* cond, const STimeWindow* range, STimeWindow* twindow,
+                                 STimeWindow* extTwindows) {
+  int32_t code = TSDB_CODE_SUCCESS;
+  int32_t lino = 0;
+  STimeWindow tempWindow;
+
+  if (cond->skey > cond->ekey || range->skey > range->ekey) {
+    *twindow = extTwindows[0] = extTwindows[1] = TSWINDOW_DESC_INITIALIZER;
+    return code;
+  }
+
+  if (range->ekey < cond->skey) {
+    extTwindows[1] = *cond;
+    *twindow = extTwindows[0] = TSWINDOW_DESC_INITIALIZER;
+    return code;
+  }
+
+  if (cond->ekey < range->skey) {
+    extTwindows[0] = *cond;
+    *twindow = extTwindows[1] = TSWINDOW_DESC_INITIALIZER;
+    return code;
+  }
+
+  // Only scan data in the time range intersecion.
+  extTwindows[0] = extTwindows[1] = *cond;
+  twindow->skey = TMAX(cond->skey, range->skey);
+  twindow->ekey = TMIN(cond->ekey, range->ekey);
+  extTwindows[0].ekey = twindow->skey - 1;
+  extTwindows[1].skey = twindow->ekey + 1;
+
+  return code;
+}
+
 int32_t createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) {
   QRY_PARAM_CHECK(pOptrInfo);
 
@@ -1206,8 +1247,10 @@ int32_t createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyN
 
   if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
     STableScanInfo* pScanInfo = (STableScanInfo*)downstream->info;
-    pScanInfo->base.cond.twindows = pInfo->win;
+    SQueryTableDataCond *cond = &pScanInfo->base.cond;
-    pScanInfo->base.cond.type = TIMEWINDOW_RANGE_EXTERNAL;
+    cond->type = TIMEWINDOW_RANGE_EXTERNAL;
+    code = getQueryExtWindow(&cond->twindows, &pInfo->win, &cond->twindows, cond->extTwindows);
+    QUERY_CHECK_CODE(code, lino, _error);
   }
 
   setOperatorInfo(pOperator, "TimeSliceOperator", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, false, OP_NOT_OPENED, pInfo,

source/libs/function/src/builtinsimpl.c

@@ -3036,61 +3036,60 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) {
   TSKEY startKey = getRowPTs(pInput->pPTS, 0);
   TSKEY endKey = getRowPTs(pInput->pPTS, pInput->totalRows - 1);
 
-#if 0
+  if (pCtx->order == TSDB_ORDER_ASC && !pCtx->hasPrimaryKey) {
-  int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
-
-  // the optimized version only valid if all tuples in one block are monotonious increasing or descreasing.
-  // this assumption is NOT always works if project operator exists in downstream.
-  if (blockDataOrder == TSDB_ORDER_ASC) {
     for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) {
-      char* data = colDataGetData(pInputCol, i);
+      bool  isNull = colDataIsNull(pInputCol, pInput->numOfRows, i, NULL);
+      char* data = isNull ? NULL : colDataGetData(pInputCol, i);
       TSKEY cts = getRowPTs(pInput->pPTS, i);
       numOfElems++;
 
       if (pResInfo->numOfRes == 0 || pInfo->ts < cts) {
-        doSaveLastrow(pCtx, data, i, cts, pInfo);
+        int32_t code = doSaveLastrow(pCtx, data, i, cts, pInfo);
+        if (code != TSDB_CODE_SUCCESS) return code;
       }
 
       break;
     }
-  } else {  // descending order
+  } else if (!pCtx->hasPrimaryKey && pCtx->order == TSDB_ORDER_DESC) {
+    // the optimized version only valid if all tuples in one block are monotonious increasing or descreasing.
+    // this assumption is NOT always works if project operator exists in downstream.
     for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
-      char* data = colDataGetData(pInputCol, i);
+      bool  isNull = colDataIsNull(pInputCol, pInput->numOfRows, i, NULL);
+      char* data = isNull ? NULL : colDataGetData(pInputCol, i);
       TSKEY cts = getRowPTs(pInput->pPTS, i);
       numOfElems++;
 
       if (pResInfo->numOfRes == 0 || pInfo->ts < cts) {
-        doSaveLastrow(pCtx, data, i, cts, pInfo);
+        int32_t code = doSaveLastrow(pCtx, data, i, cts, pInfo);
+        if (code != TSDB_CODE_SUCCESS) return code;
       }
       break;
     }
-  }
+  } else {
-#else
+    int64_t* pts = (int64_t*)pInput->pPTS->pData;
+    int from = -1;
+    int32_t i = -1;
+    while (funcInputGetNextRowIndex(pInput, from, false, &i, &from)) {
+      bool  isNull = colDataIsNull(pInputCol, pInput->numOfRows, i, NULL);
+      char* data = isNull ? NULL : colDataGetData(pInputCol, i);
+      TSKEY cts = pts[i];
 
-  int64_t* pts = (int64_t*)pInput->pPTS->pData;
+      numOfElems++;
-  int      from = -1;
+      char* pkData = NULL;
-  int32_t  i = -1;
+      if (pCtx->hasPrimaryKey) {
-  while (funcInputGetNextRowIndex(pInput, from, false, &i, &from)) {
+        pkData = colDataGetData(pkCol, i);
-    bool  isNull = colDataIsNull(pInputCol, pInput->numOfRows, i, NULL);
+      }
-    char* data = isNull ? NULL : colDataGetData(pInputCol, i);
+      if (pResInfo->numOfRes == 0 || pInfo->ts < cts ||
-    TSKEY cts = pts[i];
+          (pInfo->ts == pts[i] && pkCompareFn && pkCompareFn(pkData, pInfo->pkData) < 0)) {
-
+        int32_t code = doSaveLastrow(pCtx, data, i, cts, pInfo);
-    numOfElems++;
+        if (code != TSDB_CODE_SUCCESS) {
-    char* pkData = NULL;
+          return code;
-    if (pCtx->hasPrimaryKey) {
+        }
-      pkData = colDataGetData(pkCol, i);
+        pResInfo->numOfRes = 1;
-    }
-    if (pResInfo->numOfRes == 0 || pInfo->ts < cts ||
-        (pInfo->ts == pts[i] && pkCompareFn && pkCompareFn(pkData, pInfo->pkData) < 0)) {
-      int32_t code = doSaveLastrow(pCtx, data, i, cts, pInfo);
-      if (code != TSDB_CODE_SUCCESS) {
-        return code;
       }
-      pResInfo->numOfRes = 1;
     }
-  }
 
-#endif
+  }
 
   SET_VAL(pResInfo, numOfElems, 1);
   return TSDB_CODE_SUCCESS;

source/libs/geometry/src/geosWrapper.c

@@ -63,7 +63,7 @@ int32_t initCtxMakePoint() {
 int32_t doMakePoint(double x, double y, unsigned char **outputGeom, size_t *size) {
   int32_t       code = TSDB_CODE_FAILED;
   SGeosContext *geosCtx = NULL;
-  
+
   TAOS_CHECK_RETURN(getThreadLocalGeosCtx(&geosCtx));
 
   GEOSGeometry  *geom = NULL;
@@ -170,7 +170,7 @@ static int32_t initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData
 int32_t initCtxGeomFromText() {
   int32_t       code = TSDB_CODE_FAILED;
   SGeosContext *geosCtx = NULL;
-  
+
   TAOS_CHECK_RETURN(getThreadLocalGeosCtx(&geosCtx));
 
   if (geosCtx->handle == NULL) {
@@ -208,7 +208,7 @@ int32_t initCtxGeomFromText() {
 int32_t doGeomFromText(const char *inputWKT, unsigned char **outputGeom, size_t *size) {
   int32_t       code = TSDB_CODE_FAILED;
   SGeosContext *geosCtx = NULL;
-  
+
   TAOS_CHECK_RETURN(getThreadLocalGeosCtx(&geosCtx));
 
   GEOSGeometry  *geom = NULL;
@@ -245,7 +245,7 @@ _exit:
 int32_t initCtxAsText() {
   int32_t       code = TSDB_CODE_FAILED;
   SGeosContext *geosCtx = NULL;
-  
+
   TAOS_CHECK_RETURN(getThreadLocalGeosCtx(&geosCtx));
 
   if (geosCtx->handle == NULL) {
@@ -283,11 +283,11 @@ int32_t initCtxAsText() {
 int32_t doAsText(const unsigned char *inputGeom, size_t size, char **outputWKT) {
   int32_t       code = TSDB_CODE_FAILED;
   SGeosContext *geosCtx = NULL;
-  
+
   TAOS_CHECK_RETURN(getThreadLocalGeosCtx(&geosCtx));
 
-  GEOSGeometry  *geom = NULL;
+  GEOSGeometry *geom = NULL;
-  char *wkt = NULL;
+  char         *wkt = NULL;
 
   geom = GEOSWKBReader_read_r(geosCtx->handle, geosCtx->WKBReader, inputGeom, size);
   if (geom == NULL) {
@@ -313,10 +313,35 @@ _exit:
   return code;
 }
 
+int32_t checkWKB(const unsigned char *wkb, size_t size) {
+  int32_t       code = TSDB_CODE_SUCCESS;
+  GEOSGeometry *geom = NULL;
+  SGeosContext *geosCtx = NULL;
+
+  TAOS_CHECK_RETURN(getThreadLocalGeosCtx(&geosCtx));
+
+  geom = GEOSWKBReader_read_r(geosCtx->handle, geosCtx->WKBReader, wkb, size);
+  if (geom == NULL) {
+    return TSDB_CODE_FUNC_FUNTION_PARA_VALUE;
+  }
+
+  if (!GEOSisValid_r(geosCtx->handle, geom)) {
+    code = TSDB_CODE_FUNC_FUNTION_PARA_VALUE;
+    goto _exit;
+  }
+
+_exit:
+  if (geom) {
+    GEOSGeom_destroy_r(geosCtx->handle, geom);
+    geom = NULL;
+  }
+  return code;
+}
+
 int32_t initCtxRelationFunc() {
   int32_t       code = TSDB_CODE_FAILED;
   SGeosContext *geosCtx = NULL;
-  
+
   TAOS_CHECK_RETURN(getThreadLocalGeosCtx(&geosCtx));
 
   if (geosCtx->handle == NULL) {
@@ -343,7 +368,7 @@ int32_t doGeosRelation(const GEOSGeometry *geom1, const GEOSPreparedGeometry *pr
                        _geosPreparedRelationFunc_t preparedRelationFn,
                        _geosPreparedRelationFunc_t swappedPreparedRelationFn) {
   SGeosContext *geosCtx = NULL;
-  
+
   TAOS_CHECK_RETURN(getThreadLocalGeosCtx(&geosCtx));
 
   if (!preparedGeom1) {

source/libs/nodes/src/nodesEqualFuncs.c

@@ -153,6 +153,12 @@ static bool caseWhenNodeEqual(const SCaseWhenNode* a, const SCaseWhenNode* b) {
   return true;
 }
 
+static bool groupingSetNodeEqual(const SGroupingSetNode* a, const SGroupingSetNode* b) {
+  COMPARE_SCALAR_FIELD(groupingSetType);
+  COMPARE_NODE_LIST_FIELD(pParameterList);
+  return true;
+}
+
 bool nodesEqualNode(const SNode* a, const SNode* b) {
   if (a == b) {
     return true;
@@ -181,10 +187,11 @@ bool nodesEqualNode(const SNode* a, const SNode* b) {
       return whenThenNodeEqual((const SWhenThenNode*)a, (const SWhenThenNode*)b);
     case QUERY_NODE_CASE_WHEN:
       return caseWhenNodeEqual((const SCaseWhenNode*)a, (const SCaseWhenNode*)b);
+    case QUERY_NODE_GROUPING_SET:
+      return groupingSetNodeEqual((const SGroupingSetNode*)a, (const SGroupingSetNode*)b);
     case QUERY_NODE_REAL_TABLE:
     case QUERY_NODE_TEMP_TABLE:
     case QUERY_NODE_JOIN_TABLE:
-    case QUERY_NODE_GROUPING_SET:
     case QUERY_NODE_ORDER_BY_EXPR:
     case QUERY_NODE_LIMIT:
       return false;

source/libs/nodes/src/nodesUtilFuncs.c

@@ -3117,3 +3117,46 @@ void nodesSortList(SNodeList** pList, int32_t (*comp)(SNode* pNode1, SNode* pNod
     inSize *= 2;
   }
 }
+
+static SNode* nodesListFindNode(SNodeList* pList, SNode* pNode) {
+  SNode* pFound = NULL;
+  FOREACH(pFound, pList) {
+    if (nodesEqualNode(pFound, pNode)) {
+      break;
+    }
+  }
+  return pFound;
+}
+
+int32_t nodesListDeduplicate(SNodeList** ppList) {
+  if (!ppList || LIST_LENGTH(*ppList) <= 1) return TSDB_CODE_SUCCESS;
+  if (LIST_LENGTH(*ppList) == 2) {
+    SNode* pNode1 = nodesListGetNode(*ppList, 0);
+    SNode* pNode2 = nodesListGetNode(*ppList, 1);
+    if (nodesEqualNode(pNode1, pNode2)) {
+      SListCell* pCell = nodesListGetCell(*ppList, 1);
+      (void)nodesListErase(*ppList, pCell);
+    }
+    return TSDB_CODE_SUCCESS;
+  }
+  SNodeList* pTmp = NULL;
+  int32_t code = nodesMakeList(&pTmp);
+  if (TSDB_CODE_SUCCESS == code) {
+    SNode* pNode = NULL;
+    FOREACH(pNode, *ppList) {
+      SNode* pFound = nodesListFindNode(pTmp, pNode);
+      if (NULL == pFound) {
+        code = nodesCloneNode(pNode, &pFound);
+        if (TSDB_CODE_SUCCESS == code) code = nodesListStrictAppend(pTmp, pFound);
+        if (TSDB_CODE_SUCCESS != code) break;
+      }
+    }
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    nodesDestroyList(*ppList);
+    *ppList = pTmp;
+  } else {
+    nodesDestroyList(pTmp);
+  }
+  return code;
+}

source/libs/parser/inc/parUtil.h

@@ -115,6 +115,7 @@ typedef struct SParseMetaCache {
   SHashObj* pTableName;    // key is tbFUid, elements is STableMeta*(append with tbName)
   SArray*   pDnodes;       // element is SEpSet
   bool      dnodeRequired;
+  bool      forceFetchViewMeta;
 } SParseMetaCache;
 
 int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...);

source/libs/parser/src/parAstParser.c

@@ -811,7 +811,7 @@ static int32_t collectMetaKeyFromShowCreateView(SCollectMetaKeyCxt* pCxt, SShowC
   if (TSDB_CODE_SUCCESS == code) {
     code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, pCxt->pMetaCache);
   }
-
+  pCxt->pMetaCache->forceFetchViewMeta = true;
   return code;
 }
 
@@ -889,6 +889,7 @@ static int32_t collectMetaKeyFromCreateViewStmt(SCollectMetaKeyCxt* pCxt, SCreat
 static int32_t collectMetaKeyFromDropViewStmt(SCollectMetaKeyCxt* pCxt, SDropViewStmt* pStmt) {
   int32_t code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName,
                                             pStmt->viewName, AUTH_TYPE_ALTER, pCxt->pMetaCache);
+  pCxt->pMetaCache->forceFetchViewMeta = true;
   return code;
 }
 

source/libs/parser/src/parCalcConst.c

@@ -336,7 +336,10 @@ static int32_t calcConstGroupBy(SCalcConstContext* pCxt, SSelectStmt* pSelect) {
         }
       }
     }
-    NODES_DESTORY_LIST(pSelect->pGroupByList);
+    FOREACH(pNode, pSelect->pGroupByList) {
+      if (!cell->pPrev) continue;
+      ERASE_NODE(pSelect->pGroupByList);
+    }
   }
   return code;
 }

source/libs/parser/src/parInsertStmt.c

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "geosWrapper.h"
 #include "os.h"
 #include "parInsertUtil.h"
 #include "parInt.h"
@@ -192,6 +193,12 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch
       //      strcpy(val.colName, pTagSchema->name);
       if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || pTagSchema->type == TSDB_DATA_TYPE_VARBINARY ||
           pTagSchema->type == TSDB_DATA_TYPE_GEOMETRY) {
+        if (pTagSchema->type == TSDB_DATA_TYPE_GEOMETRY) {
+          if (initCtxAsText() || checkWKB(bind[c].buffer, colLen)) {
+            code = buildSyntaxErrMsg(&pBuf, "invalid geometry tag", bind[c].buffer);
+            goto end;
+          }
+        }
         val.pData = (uint8_t*)bind[c].buffer;
         val.nData = colLen;
       } else if (pTagSchema->type == TSDB_DATA_TYPE_NCHAR) {
@@ -409,7 +416,8 @@ int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, c
     }
 
     code = tColDataAddValueByBind(pCol, pBind,
-                                  IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE : -1);
+                                  IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE : -1,
+                                  initCtxAsText, checkWKB);
     if (code) {
       goto _return;
     }
@@ -461,7 +469,8 @@ int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bi
   }
 
   code = tColDataAddValueByBind(pCol, pBind,
-                                IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE : -1);
+                                IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE : -1,
+                                initCtxAsText, checkWKB);
 
   qDebug("stmt col %d bind %d rows data", colIdx, rowNum);
 
@@ -544,6 +553,12 @@ int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const c
       //      strcpy(val.colName, pTagSchema->name);
       if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || pTagSchema->type == TSDB_DATA_TYPE_VARBINARY ||
           pTagSchema->type == TSDB_DATA_TYPE_GEOMETRY) {
+        if (pTagSchema->type == TSDB_DATA_TYPE_GEOMETRY) {
+          if (initCtxAsText() || checkWKB(bind[c].buffer, colLen)) {
+            code = buildSyntaxErrMsg(&pBuf, "invalid geometry tag", bind[c].buffer);
+            goto end;
+          }
+        }
         val.pData = (uint8_t*)bind[c].buffer;
         val.nData = colLen;
       } else if (pTagSchema->type == TSDB_DATA_TYPE_NCHAR) {
@@ -824,7 +839,8 @@ int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind,
     }
 
     code = tColDataAddValueByBind2(pCol, pBind,
-                                   IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE : -1);
+                                   IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE : -1,
+                                   initCtxAsText, checkWKB);
     if (code) {
       goto _return;
     }
@@ -876,7 +892,8 @@ int32_t qBindStmtSingleColValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* b
   }
 
   code = tColDataAddValueByBind2(pCol, pBind,
-                                 IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE : -1);
+                                 IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE : -1,
+                                 initCtxAsText, checkWKB);
 
   qDebug("stmt col %d bind %d rows data", colIdx, rowNum);
 

source/libs/parser/src/parUtil.c

@@ -817,6 +817,7 @@ int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalog
   }
 #endif
   pCatalogReq->dNodeRequired = pMetaCache->dnodeRequired;
+  pCatalogReq->forceFetchViewMeta = pMetaCache->forceFetchViewMeta;
   return code;
 }
 

source/libs/parser/src/parser.c

@@ -154,6 +154,9 @@ static int32_t parseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, SParseMetaCa
 }
 
 static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) {
+  if (!pParam || IS_NULL_TYPE(pParam->buffer_type)) {
+    return TSDB_CODE_APP_ERROR;
+  }
   if (IS_VAR_DATA_TYPE(pVal->node.resType.type)) {
     taosMemoryFreeClear(pVal->datum.p);
   }
@@ -441,6 +444,9 @@ int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx
 }
 
 static int32_t setValueByBindParam2(SValueNode* pVal, TAOS_STMT2_BIND* pParam) {
+  if (!pParam || IS_NULL_TYPE(pParam->buffer_type)) {
+    return TSDB_CODE_APP_ERROR;
+  }
   if (IS_VAR_DATA_TYPE(pVal->node.resType.type)) {
     taosMemoryFreeClear(pVal->datum.p);
   }

source/libs/planner/src/planLogicCreater.c

@@ -847,8 +847,11 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect,
   }
 
   if (NULL != pSelect->pGroupByList) {
-    pAgg->pGroupKeys = NULL;
+    code = nodesListDeduplicate(&pSelect->pGroupByList);
-    code = nodesCloneList(pSelect->pGroupByList, &pAgg->pGroupKeys);
+    if (TSDB_CODE_SUCCESS == code) {
+      pAgg->pGroupKeys = NULL;
+      code = nodesCloneList(pSelect->pGroupByList, &pAgg->pGroupKeys);
+    }
   }
 
   // rewrite the expression in subsequent clauses

source/libs/planner/src/planOptimizer.c

@@ -208,6 +208,7 @@ static void optSetParentOrder(SLogicNode* pNode, EOrder order, SLogicNode* pNode
     // case QUERY_NODE_LOGIC_PLAN_WINDOW:
     case QUERY_NODE_LOGIC_PLAN_AGG:
     case QUERY_NODE_LOGIC_PLAN_SORT:
+    case QUERY_NODE_LOGIC_PLAN_FILL:
       if (pNode == pNodeForcePropagate) {
         pNode->outputTsOrder = order;
         break;

source/libs/scheduler/src/schRemote.c

@@ -1345,30 +1345,19 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
       SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
   }
 
-#if 1
+  if ((tsBypassFlag & TSDB_BYPASS_RB_RPC_SEND_SUBMIT) && (TDMT_VND_SUBMIT == msgType)) {
-  SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)};
+    taosMemoryFree(msg);
-  code = schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, (uint32_t)msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL));
+    SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask));
-  msg = NULL;
+  } else {
-  SCH_ERR_JRET(code);
-
-  if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY) {
-    SCH_ERR_RET(schAppendTaskExecNode(pJob, pTask, addr, pTask->execId));
-  }
-#else
-  if (TDMT_VND_SUBMIT != msgType) {
     SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)};
-    code = schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL));
+    code = schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, (uint32_t)msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL));
     msg = NULL;
     SCH_ERR_JRET(code);
 
     if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY) {
       SCH_ERR_RET(schAppendTaskExecNode(pJob, pTask, addr, pTask->execId));
     }
-  } else {
-    taosMemoryFree(msg);
-    SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask));
   }
-#endif
 
   return TSDB_CODE_SUCCESS;
 

source/libs/stream/inc/streamInt.h

@@ -192,7 +192,6 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask);
 int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask);
 int32_t streamTaskSendCheckpointReq(SStreamTask* pTask);
 
-void    streamTaskSetFailedCheckpointId(SStreamTask* pTask);
 int32_t streamTaskGetNumOfDownstream(const SStreamTask* pTask);
 int32_t streamTaskGetNumOfUpstream(const SStreamTask* pTask);
 int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate, const char*);
@@ -245,6 +244,9 @@ int32_t streamCreateSinkResTrigger(SStreamTrigger** pTrigger);
 int32_t streamCreateForcewindowTrigger(SStreamTrigger** pTrigger, int32_t trigger, SInterval* pInterval,
                                        STimeWindow* pLatestWindow, const char* id);
 
+// inject stream errors
+void chkptFailedByRetrieveReqToSource(SStreamTask* pTask, int64_t checkpointId);
+
 #ifdef __cplusplus
 }
 #endif

source/libs/stream/src/streamCheckpoint.c

@@ -161,33 +161,52 @@ int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTri
 
 int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId, int32_t downstreamNodeId,
                                            SRpcHandleInfo* pRpcInfo, int32_t code) {
-  int32_t size = sizeof(SMsgHead) + sizeof(SCheckpointTriggerRsp);
+  int32_t  ret = 0;
-  void*   pBuf = rpcMallocCont(size);
+  int32_t  tlen = 0;
-  if (pBuf == NULL) {
+  void*    buf = NULL;
+  SEncoder encoder;
+
+  SCheckpointTriggerRsp req = {.streamId = pTask->id.streamId,
+                               .upstreamTaskId = pTask->id.taskId,
+                               .taskId = dstTaskId,
+                               .rspCode = code};
+
+  if (code == TSDB_CODE_SUCCESS) {
+    req.checkpointId = pTask->chkInfo.pActiveInfo->activeId;
+    req.transId = pTask->chkInfo.pActiveInfo->transId;
+  } else {
+    req.checkpointId = -1;
+    req.transId = -1;
+  }
+
+  tEncodeSize(tEncodeCheckpointTriggerRsp, &req, tlen, ret);
+  if (ret < 0) {
+    stError("s-task:%s encode checkpoint-trigger rsp msg failed, code:%s", pTask->id.idStr, tstrerror(code));
+    return ret;
+  }
+
+  buf = rpcMallocCont(tlen + sizeof(SMsgHead));
+  if (buf == NULL) {
+    stError("s-task:%s malloc chkpt-trigger rsp failed for task:0x%x, since out of memory", pTask->id.idStr, dstTaskId);
     return terrno;
   }
 
-  SCheckpointTriggerRsp* pRsp = POINTER_SHIFT(pBuf, sizeof(SMsgHead));
+  ((SMsgHead*)buf)->vgId = htonl(downstreamNodeId);
+  void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
 
-  ((SMsgHead*)pBuf)->vgId = htonl(downstreamNodeId);
+  tEncoderInit(&encoder, abuf, tlen);
-
+  if ((ret = tEncodeCheckpointTriggerRsp(&encoder, &req)) < 0) {
-  pRsp->streamId = pTask->id.streamId;
+    rpcFreeCont(buf);
-  pRsp->upstreamTaskId = pTask->id.taskId;
+    tEncoderClear(&encoder);
-  pRsp->taskId = dstTaskId;
+    stError("encode checkpoint-trigger rsp failed, code:%s", tstrerror(code));
-  pRsp->rspCode = code;
+    return ret;
-
-  if (code == TSDB_CODE_SUCCESS) {
-    pRsp->checkpointId = pTask->chkInfo.pActiveInfo->activeId;
-    pRsp->transId = pTask->chkInfo.pActiveInfo->transId;
-  } else {
-    pRsp->checkpointId = -1;
-    pRsp->transId = -1;
   }
+  tEncoderClear(&encoder);
 
-  SRpcMsg rspMsg = {.code = 0, .pCont = pBuf, .contLen = size, .info = *pRpcInfo};
+  SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = tlen + sizeof(SMsgHead), .info = *pRpcInfo};
   tmsgSendRsp(&rspMsg);
 
-  return 0;
+  return ret;
 }
 
 int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStreamTask* pTask) {
@@ -222,14 +241,14 @@ static int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t check
     stError("s-task:%s vgId:%d current checkpointId:%" PRId64
             " recv expired checkpoint-trigger block, checkpointId:%" PRId64 " transId:%d, discard",
             id, vgId, pTask->chkInfo.checkpointId, checkpointId, transId);
-    return code;
+    return TSDB_CODE_STREAM_INVLD_CHKPT;
   }
 
   if (pActiveInfo->failedId >= checkpointId) {
     stError("s-task:%s vgId:%d checkpointId:%" PRId64 " transId:%d, has been marked failed, failedId:%" PRId64
             " discard the checkpoint-trigger block",
             id, vgId, checkpointId, transId, pActiveInfo->failedId);
-    return code;
+    return TSDB_CODE_STREAM_INVLD_CHKPT;
   }
 
   if (pTask->chkInfo.checkpointId == checkpointId) {
@@ -255,8 +274,7 @@ static int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t check
         "the interrupted checkpoint",
         id, vgId, pBlock->srcTaskId);
 
-    streamTaskOpenUpstreamInput(pTask, pBlock->srcTaskId);
+    return TSDB_CODE_STREAM_INVLD_CHKPT;
-    return code;
   }
 
   if (streamTaskGetStatus(pTask).state == TASK_STATUS__CK) {
@@ -264,14 +282,14 @@ static int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t check
       stError("s-task:%s vgId:%d active checkpointId:%" PRId64 ", recv invalid checkpoint-trigger checkpointId:%" PRId64
               " discard",
               id, vgId, pActiveInfo->activeId, checkpointId);
-      return code;
+      return TSDB_CODE_STREAM_INVLD_CHKPT;
     } else {  // checkpointId == pActiveInfo->activeId
       if (pActiveInfo->allUpstreamTriggerRecv == 1) {
         stDebug(
             "s-task:%s vgId:%d all upstream checkpoint-trigger recv, discard this checkpoint-trigger, "
             "checkpointId:%" PRId64 " transId:%d",
             id, vgId, checkpointId, transId);
-        return code;
+        return TSDB_CODE_STREAM_INVLD_CHKPT;
       }
 
       if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) {
@@ -283,17 +301,17 @@ static int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t check
           }
 
           if (p->upstreamTaskId == pBlock->srcTaskId) {
-            stWarn("s-task:%s repeatly recv checkpoint-source msg from task:0x%x vgId:%d, checkpointId:%" PRId64
+            stWarn("s-task:%s repeatly recv checkpoint-trigger msg from task:0x%x vgId:%d, checkpointId:%" PRId64
                    ", prev recvTs:%" PRId64 " discard",
                    pTask->id.idStr, p->upstreamTaskId, p->upstreamNodeId, p->checkpointId, p->recvTs);
-            return code;
+            return TSDB_CODE_STREAM_INVLD_CHKPT;
           }
         }
       }
     }
   }
 
-  return 0;
+  return TSDB_CODE_SUCCESS;
 }
 
 int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) {
@@ -317,6 +335,9 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
   code = doCheckBeforeHandleChkptTrigger(pTask, checkpointId, pBlock, transId);
   streamMutexUnlock(&pTask->lock);
   if (code) {
+    if (taskLevel != TASK_LEVEL__SOURCE) { // the checkpoint-trigger is discard, open the inputQ for upstream tasks
+      streamTaskOpenUpstreamInput(pTask, pBlock->srcTaskId);
+    }
     streamFreeQitem((SStreamQueueItem*)pBlock);
     return code;
   }
@@ -330,6 +351,11 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
     pActiveInfo->activeId = checkpointId;
     pActiveInfo->transId = transId;
 
+    if (pTask->chkInfo.startTs == 0) {
+      pTask->chkInfo.startTs = taosGetTimestampMs();
+      pTask->execInfo.checkpoint += 1;
+    }
+
     code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT);
     if (code != TSDB_CODE_SUCCESS) {
       stError("s-task:%s handle checkpoint-trigger block failed, code:%s", id, tstrerror(code));
@@ -359,6 +385,10 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
     }
   }
 
+#if 0
+  taosMsleep(20*1000);
+#endif
+
   if (taskLevel == TASK_LEVEL__SOURCE) {
     int8_t type = pTask->outputInfo.type;
     pActiveInfo->allUpstreamTriggerRecv = 1;
@@ -373,6 +403,10 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
       return code;
     }
 
+#if 0
+    chkptFailedByRetrieveReqToSource(pTask, checkpointId);
+#endif
+
     if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
       stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId);
       code = continueDispatchCheckpointTriggerBlock(pBlock, pTask);  // todo handle this failure
@@ -382,11 +416,6 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
       streamFreeQitem((SStreamQueueItem*)pBlock);
     }
   } else if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) {
-    if (pTask->chkInfo.startTs == 0) {
-      pTask->chkInfo.startTs = taosGetTimestampMs();
-      pTask->execInfo.checkpoint += 1;
-    }
-
     // todo: handle this
     // update the child Id for downstream tasks
     code = streamAddCheckpointReadyMsg(pTask, pBlock->srcTaskId, pTask->info.selfChildId, checkpointId);
@@ -562,7 +591,7 @@ void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) {
   }
   streamMutexUnlock(&pInfo->lock);
 
-  stDebug("s-task:%s clear active checkpointInfo, failed checkpointId:%" PRId64 ", current checkpointId:%" PRId64,
+  stDebug("s-task:%s clear active checkpointInfo, failed checkpointId:%" PRId64 ", latest checkpointId:%" PRId64,
           pTask->id.idStr, pInfo->failedId, pTask->chkInfo.checkpointId);
 }
 
@@ -682,15 +711,22 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
   return TSDB_CODE_SUCCESS;
 }
 
-void streamTaskSetFailedCheckpointId(SStreamTask* pTask) {
+void streamTaskSetFailedCheckpointId(SStreamTask* pTask, int64_t failedId) {
   struct SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
 
-  if (pInfo->activeId <= 0) {
+  if (failedId <= 0) {
-    stWarn("s-task:%s checkpoint-info is cleared now, not set the failed checkpoint info", pTask->id.idStr);
+    stWarn("s-task:%s failedId is 0, not update the failed checkpoint info, current failedId:%" PRId64
+           " activeId:%" PRId64,
+           pTask->id.idStr, pInfo->failedId, pInfo->activeId);
   } else {
-    pInfo->failedId = pInfo->activeId;
+    if (failedId <= pInfo->failedId) {
-    stDebug("s-task:%s mark and set the failed checkpointId:%" PRId64 " (transId:%d)", pTask->id.idStr, pInfo->activeId,
+      stDebug("s-task:%s failedId:%" PRId64 " not update to:%" PRId64, pTask->id.idStr, pInfo->failedId, failedId);
-            pInfo->transId);
+    } else {
+      stDebug("s-task:%s mark and set the failed checkpointId:%" PRId64 " (transId:%d) activeId:%" PRId64
+              " prev failedId:%" PRId64,
+              pTask->id.idStr, failedId, pInfo->transId, pInfo->activeId, pInfo->failedId);
+      pInfo->failedId = failedId;
+    }
   }
 }
 
@@ -698,7 +734,7 @@ void streamTaskSetCheckpointFailed(SStreamTask* pTask) {
   streamMutexLock(&pTask->lock);
   ETaskStatus status = streamTaskGetStatus(pTask).state;
   if (status == TASK_STATUS__CK) {
-    streamTaskSetFailedCheckpointId(pTask);
+    streamTaskSetFailedCheckpointId(pTask, pTask->chkInfo.pActiveInfo->activeId);
   }
   streamMutexUnlock(&pTask->lock);
 }
@@ -876,8 +912,9 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) {
       code = streamSendChkptReportMsg(pTask, &pTask->chkInfo, dropRelHTask);
     }
   } else {  // clear the checkpoint info if failed
+    // set failed checkpoint id before clear the checkpoint info
     streamMutexLock(&pTask->lock);
-    streamTaskSetFailedCheckpointId(pTask);  // set failed checkpoint id before clear the checkpoint info
+    streamTaskSetFailedCheckpointId(pTask, ckId);
     streamMutexUnlock(&pTask->lock);
 
     code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_CHECKPOINT_DONE);
@@ -1101,23 +1138,43 @@ int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList) {
       return TSDB_CODE_INVALID_PARA;
     }
 
-    SRetrieveChkptTriggerReq* pReq = rpcMallocCont(sizeof(SRetrieveChkptTriggerReq));
+    int32_t  ret = 0;
-    if (pReq == NULL) {
+    int32_t  tlen = 0;
-      code = terrno;
+    void*    buf = NULL;
-      stError("vgId:%d failed to create msg to retrieve trigger msg for task:%s exec, code:out of memory", vgId, pId);
+    SRpcMsg  rpcMsg = {0};
+    SEncoder encoder;
+
+    SRetrieveChkptTriggerReq req = {.streamId = pTask->id.streamId,
+                                    .downstreamTaskId = pTask->id.taskId,
+                                    .downstreamNodeId = vgId,
+                                    .upstreamTaskId = pUpstreamTask->taskId,
+                                    .upstreamNodeId = pUpstreamTask->nodeId,
+                                    .checkpointId = checkpointId};
+
+    tEncodeSize(tEncodeRetrieveChkptTriggerReq, &req, tlen, ret);
+    if (ret < 0) {
+      stError("encode retrieve checkpoint-trigger msg failed, code:%s", tstrerror(code));
+    }
+
+    buf = rpcMallocCont(tlen + sizeof(SMsgHead));
+    if (buf == NULL) {
+      stError("vgId:%d failed to create retrieve checkpoint-trigger msg for task:%s exec, code:out of memory", vgId, pId);
       continue;
     }
 
-    pReq->head.vgId = htonl(pUpstreamTask->nodeId);
+    ((SRetrieveChkptTriggerReq*)buf)->head.vgId = htonl(pUpstreamTask->nodeId);
-    pReq->streamId = pTask->id.streamId;
+    void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
-    pReq->downstreamTaskId = pTask->id.taskId;
-    pReq->downstreamNodeId = vgId;
-    pReq->upstreamTaskId = pUpstreamTask->taskId;
-    pReq->upstreamNodeId = pUpstreamTask->nodeId;
-    pReq->checkpointId = checkpointId;
 
-    SRpcMsg rpcMsg = {0};
+    tEncoderInit(&encoder, abuf, tlen);
-    initRpcMsg(&rpcMsg, TDMT_STREAM_RETRIEVE_TRIGGER, pReq, sizeof(SRetrieveChkptTriggerReq));
+    if ((code = tEncodeRetrieveChkptTriggerReq(&encoder, &req)) < 0) {
+      rpcFreeCont(buf);
+      tEncoderClear(&encoder);
+      stError("encode retrieve checkpoint-trigger req failed, code:%s", tstrerror(code));
+      continue;
+    }
+    tEncoderClear(&encoder);
+
+    initRpcMsg(&rpcMsg, TDMT_STREAM_RETRIEVE_TRIGGER, buf, tlen + sizeof(SMsgHead));
 
     code = tmsgSendReq(&pUpstreamTask->epSet, &rpcMsg);
     if (code == TSDB_CODE_SUCCESS) {

source/libs/stream/src/streamDispatch.c

@@ -1170,6 +1170,7 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) {
   if (taosArrayGetSize(pTask->upstreamInfo.pList) != num) {
     stError("s-task:%s invalid number of sent readyMsg:%d to upstream:%d", id, num,
             (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList));
+    streamMutexUnlock(&pActiveInfo->lock);
     return TSDB_CODE_STREAM_INTERNAL_ERROR;
   }
 
@@ -1412,6 +1413,7 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa
   if (size > 0) {
     STaskCheckpointReadyInfo* pReady = taosArrayGet(pActiveInfo->pReadyMsgList, 0);
     if (pReady == NULL) {
+      streamMutexUnlock(&pActiveInfo->lock);
       return terrno;
     }
 

source/libs/stream/src/streamErrorInjection.c

@@ -0,0 +1,17 @@
+#include "streamInt.h"
+
+/**
+ *  pre-request: checkpoint interval should be 60s
+ * @param pTask
+ * @param checkpointId
+ */
+void chkptFailedByRetrieveReqToSource(SStreamTask* pTask, int64_t checkpointId) {
+  streamMutexLock(&pTask->lock);
+
+  // set current checkpoint failed immediately, set failed checkpoint id before clear the checkpoint info
+  streamTaskSetFailedCheckpointId(pTask, checkpointId);
+  streamMutexUnlock(&pTask->lock);
+
+  // the checkpoint interval should be 60s, and the next checkpoint req should be issued by mnode
+  taosMsleep(65*1000);
+}

source/libs/stream/src/streamSched.c

@@ -83,13 +83,37 @@ int32_t streamTrySchedExec(SStreamTask* pTask) {
 }
 
 int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId, int32_t execType) {
-  SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq));
+  int32_t code = 0;
-  if (pRunReq == NULL) {
+  int32_t tlen = 0;
+
+  SStreamTaskRunReq req = {.streamId = streamId, .taskId = taskId, .reqType = execType};
+
+  tEncodeSize(tEncodeStreamTaskRunReq, &req, tlen, code);
+  if (code < 0) {
+    stError("s-task:0x%" PRIx64 " vgId:%d encode stream task run req failed, code:%s", streamId, vgId, tstrerror(code));
+    return code;
+  }
+
+  void* buf = rpcMallocCont(tlen + sizeof(SMsgHead));
+  if (buf == NULL) {
     stError("vgId:%d failed to create msg to start stream task:0x%x exec, type:%d, code:%s", vgId, taskId, execType,
             tstrerror(terrno));
     return terrno;
   }
 
+  ((SMsgHead*)buf)->vgId = vgId;
+  char* bufx = POINTER_SHIFT(buf, sizeof(SMsgHead));
+
+  SEncoder encoder;
+  tEncoderInit(&encoder, (uint8_t*)bufx, tlen);
+  if ((code = tEncodeStreamTaskRunReq(&encoder, &req)) < 0) {
+    rpcFreeCont(buf);
+    tEncoderClear(&encoder);
+    stError("s-task:0x%x vgId:%d encode run task msg failed, code:%s", taskId, vgId, tstrerror(code));
+    return code;
+  }
+  tEncoderClear(&encoder);
+
   if (streamId != 0) {
     stDebug("vgId:%d create msg to for task:0x%x, exec type:%d, %s", vgId, taskId, execType,
             streamTaskGetExecType(execType));
@@ -97,13 +121,8 @@ int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int3
     stDebug("vgId:%d create msg to exec, type:%d, %s", vgId, execType, streamTaskGetExecType(execType));
   }
 
-  pRunReq->head.vgId = vgId;
+  SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = buf, .contLen = tlen + sizeof(SMsgHead)};
-  pRunReq->streamId = streamId;
+  code = tmsgPutToQueue(pMsgCb, STREAM_QUEUE, &msg);
-  pRunReq->taskId = taskId;
-  pRunReq->reqType = execType;
-
-  SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)};
-  int32_t code = tmsgPutToQueue(pMsgCb, STREAM_QUEUE, &msg);
   if (code) {
     stError("vgId:%d failed to put msg into stream queue, code:%s, %x", vgId, tstrerror(code), taskId);
   }

source/libs/stream/src/streamStartTask.c

@@ -433,6 +433,7 @@ int32_t streamMetaStopAllTasks(SStreamMeta* pMeta) {
   // send hb msg to mnode before closing all tasks.
   int32_t code = streamMetaSendMsgBeforeCloseTasks(pMeta, &pTaskList);
   if (code != TSDB_CODE_SUCCESS) {
+    streamMetaRUnLock(pMeta);
     return code;
   }
 

source/libs/stream/src/streamTask.c

@@ -22,6 +22,7 @@
 #include "tstream.h"
 #include "ttimer.h"
 #include "wal.h"
+#include "streamMsg.h"
 
 static void streamTaskDestroyUpstreamInfo(SUpstreamInfo* pUpstreamInfo);
 static int32_t streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool* pUpdated);
@@ -1246,13 +1247,13 @@ void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo) {
   taosMemoryFree(pInfo);
 }
 
-//NOTE: clear the checkpoint id, and keep the failed id
+// NOTE: clear the checkpoint id, and keep the failed id
+// failedId for a task will increase as the checkpoint I.D. increases.
 void streamTaskClearActiveInfo(SActiveCheckpointInfo* pInfo) {
   pInfo->activeId = 0;
   pInfo->transId = 0;
   pInfo->allUpstreamTriggerRecv = 0;
   pInfo->dispatchTrigger = false;
-//  pInfo->failedId = 0;
 
   taosArrayClear(pInfo->pDispatchTriggerList);
   taosArrayClear(pInfo->pCheckpointReadyRecvList);
@@ -1303,4 +1304,178 @@ void streamTaskFreeRefId(int64_t* pRefId) {
   }
 
   metaRefMgtRemove(pRefId);
+}
+
+
+int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
+  int32_t code = 0;
+  int32_t lino;
+
+  TAOS_CHECK_EXIT(tStartEncode(pEncoder));
+  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->ver));
+  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->id.streamId));
+  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->id.taskId));
+  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->info.trigger));
+  TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->info.taskLevel));
+  TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->outputInfo.type));
+  TAOS_CHECK_EXIT(tEncodeI16(pEncoder, pTask->msgInfo.msgType));
+
+  TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->status.taskStatus));
+  TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->status.schedStatus));
+
+  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->info.selfChildId));
+  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->info.nodeId));
+  TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pTask->info.epSet));
+  TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pTask->info.mnodeEpset));
+
+  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->chkInfo.checkpointId));
+  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->chkInfo.checkpointVer));
+  TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->info.fillHistory));
+
+  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->hTaskInfo.id.streamId));
+  int32_t taskId = pTask->hTaskInfo.id.taskId;
+  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, taskId));
+
+  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->streamTaskId.streamId));
+  taskId = pTask->streamTaskId.taskId;
+  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, taskId));
+
+  TAOS_CHECK_EXIT(tEncodeU64(pEncoder, pTask->dataRange.range.minVer));
+  TAOS_CHECK_EXIT(tEncodeU64(pEncoder, pTask->dataRange.range.maxVer));
+  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->dataRange.window.skey));
+  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->dataRange.window.ekey));
+
+  int32_t epSz = taosArrayGetSize(pTask->upstreamInfo.pList);
+  TAOS_CHECK_EXIT(tEncodeI32(pEncoder, epSz));
+  for (int32_t i = 0; i < epSz; i++) {
+    SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i);
+    TAOS_CHECK_EXIT(tEncodeStreamEpInfo(pEncoder, pInfo));
+  }
+
+  if (pTask->info.taskLevel != TASK_LEVEL__SINK) {
+    TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, pTask->exec.qmsg));
+  }
+
+  if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) {
+    TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->outputInfo.tbSink.stbUid));
+    TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, pTask->outputInfo.tbSink.stbFullName));
+    TAOS_CHECK_EXIT(tEncodeSSchemaWrapper(pEncoder, pTask->outputInfo.tbSink.pSchemaWrapper));
+  } else if (pTask->outputInfo.type == TASK_OUTPUT__SMA) {
+    TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->outputInfo.smaSink.smaId));
+  } else if (pTask->outputInfo.type == TASK_OUTPUT__FETCH) {
+    TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->outputInfo.fetchSink.reserved));
+  } else if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) {
+    TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->outputInfo.fixedDispatcher.taskId));
+    TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->outputInfo.fixedDispatcher.nodeId));
+    TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pTask->outputInfo.fixedDispatcher.epSet));
+  } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
+    TAOS_CHECK_EXIT(tSerializeSUseDbRspImp(pEncoder, &pTask->outputInfo.shuffleDispatcher.dbInfo));
+    TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, pTask->outputInfo.shuffleDispatcher.stbFullName));
+  }
+  TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->info.delaySchedParam));
+  TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->subtableWithoutMd5));
+  TAOS_CHECK_EXIT(tEncodeCStrWithLen(pEncoder, pTask->reserve, sizeof(pTask->reserve) - 1));
+
+  tEndEncode(pEncoder);
+_exit:
+  return code;
+}
+
+int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
+  int32_t taskId = 0;
+  int32_t code = 0;
+  int32_t lino;
+
+  TAOS_CHECK_EXIT(tStartDecode(pDecoder));
+  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->ver));
+  if (pTask->ver <= SSTREAM_TASK_INCOMPATIBLE_VER || pTask->ver > SSTREAM_TASK_VER) {
+    TAOS_CHECK_EXIT(TSDB_CODE_INVALID_MSG);
+  }
+
+  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->id.streamId));
+  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->id.taskId));
+  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->info.trigger));
+  TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->info.taskLevel));
+  TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->outputInfo.type));
+  TAOS_CHECK_EXIT(tDecodeI16(pDecoder, &pTask->msgInfo.msgType));
+
+  TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->status.taskStatus));
+  TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->status.schedStatus));
+
+  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->info.selfChildId));
+  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->info.nodeId));
+  TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &pTask->info.epSet));
+  TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &pTask->info.mnodeEpset));
+
+  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->chkInfo.checkpointId));
+  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->chkInfo.checkpointVer));
+  TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->info.fillHistory));
+
+  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->hTaskInfo.id.streamId));
+  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &taskId));
+  pTask->hTaskInfo.id.taskId = taskId;
+
+  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->streamTaskId.streamId));
+  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &taskId));
+  pTask->streamTaskId.taskId = taskId;
+
+  TAOS_CHECK_EXIT(tDecodeU64(pDecoder, (uint64_t*)&pTask->dataRange.range.minVer));
+  TAOS_CHECK_EXIT(tDecodeU64(pDecoder, (uint64_t*)&pTask->dataRange.range.maxVer));
+  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->dataRange.window.skey));
+  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->dataRange.window.ekey));
+
+  int32_t epSz = -1;
+  TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &epSz) < 0);
+
+  if ((pTask->upstreamInfo.pList = taosArrayInit(epSz, POINTER_BYTES)) == NULL) {
+    TAOS_CHECK_EXIT(terrno);
+  }
+  for (int32_t i = 0; i < epSz; i++) {
+    SStreamUpstreamEpInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamUpstreamEpInfo));
+    if (pInfo == NULL) {
+      TAOS_CHECK_EXIT(terrno);
+    }
+    if ((code = tDecodeStreamEpInfo(pDecoder, pInfo)) < 0) {
+      taosMemoryFreeClear(pInfo);
+      goto _exit;
+    }
+    if (taosArrayPush(pTask->upstreamInfo.pList, &pInfo) == NULL) {
+      TAOS_CHECK_EXIT(terrno);
+    }
+  }
+
+  if (pTask->info.taskLevel != TASK_LEVEL__SINK) {
+    TAOS_CHECK_EXIT(tDecodeCStrAlloc(pDecoder, &pTask->exec.qmsg));
+  }
+
+  if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) {
+    TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->outputInfo.tbSink.stbUid));
+    TAOS_CHECK_EXIT(tDecodeCStrTo(pDecoder, pTask->outputInfo.tbSink.stbFullName));
+    pTask->outputInfo.tbSink.pSchemaWrapper = taosMemoryCalloc(1, sizeof(SSchemaWrapper));
+    if (pTask->outputInfo.tbSink.pSchemaWrapper == NULL) {
+      TAOS_CHECK_EXIT(terrno);
+    }
+    TAOS_CHECK_EXIT(tDecodeSSchemaWrapper(pDecoder, pTask->outputInfo.tbSink.pSchemaWrapper));
+  } else if (pTask->outputInfo.type == TASK_OUTPUT__SMA) {
+    TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->outputInfo.smaSink.smaId));
+  } else if (pTask->outputInfo.type == TASK_OUTPUT__FETCH) {
+    TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->outputInfo.fetchSink.reserved));
+  } else if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) {
+    TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->outputInfo.fixedDispatcher.taskId));
+    TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->outputInfo.fixedDispatcher.nodeId));
+    TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &pTask->outputInfo.fixedDispatcher.epSet));
+  } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
+    TAOS_CHECK_EXIT(tDeserializeSUseDbRspImp(pDecoder, &pTask->outputInfo.shuffleDispatcher.dbInfo));
+    TAOS_CHECK_EXIT(tDecodeCStrTo(pDecoder, pTask->outputInfo.shuffleDispatcher.stbFullName));
+  }
+  TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->info.delaySchedParam));
+  if (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER) {
+    TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->subtableWithoutMd5));
+  }
+  TAOS_CHECK_EXIT(tDecodeCStrTo(pDecoder, pTask->reserve));
+
+  tEndDecode(pDecoder);
+
+_exit:
+  return code;
 }

source/libs/sync/inc/syncIndexMgr.h

@@ -29,6 +29,7 @@ typedef struct SSyncIndexMgr {
   SyncTerm   privateTerm[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA];  // for advanced function
   int64_t    startTimeArr[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA];
   int64_t    recvTimeArr[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA];
+  int64_t    sentTimeArr[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA];
   int32_t    replicaNum;
   int32_t    totalReplicaNum;
   SSyncNode *pNode;
@@ -45,7 +46,9 @@ void           syncIndexMgrCopyIfExist(SSyncIndexMgr * pNewIndex, SSyncIndexMgr
 void     syncIndexMgrSetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, int64_t startTime);
 int64_t  syncIndexMgrGetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId);
 void     syncIndexMgrSetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, int64_t recvTime);
+void     syncIndexMgrSetSentTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, int64_t sentTime);
 int64_t  syncIndexMgrGetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId);
+int64_t  syncIndexMgrGetSentTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId);
 void     syncIndexMgrSetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, SyncTerm term);
 SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId);