diff --git a/.github/workflows/taosd-ci-build.yml b/.github/workflows/taosd-ci-build.yml
index 372008b585..4972aebebe 100644
--- a/.github/workflows/taosd-ci-build.yml
+++ b/.github/workflows/taosd-ci-build.yml
@@ -11,7 +11,7 @@ on:
- 'packaging/**'
- 'tests/**'
- '*.md'
-
+
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
@@ -43,15 +43,33 @@ jobs:
if: runner.os == 'Linux'
run: |
sudo apt update -y
- sudo apt install -y build-essential cmake \
- libgeos-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev \
- zlib1g-dev pkg-config libssl-dev gawk
+ sudo apt install -y \
+ build-essential \
+ cmake \
+ gawk \
+ libgeos-dev \
+ libjansson-dev \
+ liblzma-dev \
+ libsnappy-dev \
+ libssl-dev \
+ libz-dev \
+ pkg-config \
+ zlib1g
- name: Install dependencies on macOS
if: runner.os == 'macOS'
run: |
brew update
- brew install argp-standalone gflags pkg-config snappy zlib geos jansson gawk openssl
+ brew install \
+ argp-standalone \
+ gawk \
+ gflags \
+ geos \
+ jansson \
+ openssl \
+ pkg-config \
+ snappy \
+ zlib
- name: Build and install TDengine
run: |
@@ -80,7 +98,7 @@ jobs:
run: |
taosBenchmark -t 10 -n 10 -y
taos -s "select count(*) from test.meters"
-
+
- name: Clean up
if: always()
run: |
diff --git a/Jenkinsfile2 b/Jenkinsfile2
index 395beb72db..3135b7b683 100644
--- a/Jenkinsfile2
+++ b/Jenkinsfile2
@@ -70,7 +70,7 @@ def check_docs(){
returnStdout: true
)
- def file_no_doc_changed = sh (
+ file_no_doc_changed = sh (
script: '''
cd ${WKC}
git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" || :
diff --git a/README-CN.md b/README-CN.md
index ca046dbe13..40e97de2ba 100644
--- a/README-CN.md
+++ b/README-CN.md
@@ -69,6 +69,8 @@ TDengine 是一款开源、高性能、云原生的时序数据库 (Time-Series
TDengine 目前可以在 Linux、 Windows、macOS 等平台上安装和运行。任何 OS 的应用也可以选择 taosAdapter 的 RESTful 接口连接服务端 taosd。CPU 支持 X64/ARM64，后续会支持 MIPS64、Alpha64、ARM32、RISC-V 等 CPU 架构。目前不支持使用交叉编译器构建。
+如果你想要编译 taosAdapter 或者 taosKeeper，需要安装 Go 1.18 及以上版本。
+
## 3.1 Linux系统
@@ -153,6 +155,10 @@ cmake .. -DBUILD_TOOLS=true -DBUILD_CONTRIB=true
make
```
+如果你想要编译 taosAdapter，需要添加 `-DBUILD_HTTP=false` 选项。
+
+如果你想要编译 taosKeeper，需要添加 `--DBUILD_KEEPER=true` 选项。
+
可以使用Jemalloc作为内存分配器，而不是使用glibc:
```bash
@@ -180,6 +186,10 @@ mkdir debug && cd debug
cmake .. && cmake --build .
```
+如果你想要编译 taosAdapter，需要添加 `-DBUILD_HTTP=false` 选项。
+
+如果你想要编译 taosKeeper，需要添加 `--DBUILD_KEEPER=true` 选项。
+
## 4.3 Windows系统上构建
diff --git a/README.md b/README.md
index 6cfd1980b1..f7db2a7ea2 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@
[](https://www.linkedin.com/company/tdengine)
[](https://stackoverflow.com/questions/tagged/tdengine)
-English | [简体中文](README-CN.md) | [TDengine Cloud](https://cloud.tdengine.com) | [Learn more about TSDB](https://tdengine.com/tsdb/)
+English | [简体中文](README-CN.md) | [TDengine Cloud](https://cloud.tdengine.com) | [Learn more about TSDB](https://tdengine.com/time-series-database/)
# Table of Contents
@@ -82,6 +82,8 @@ For contributing/building/testing TDengine Connectors, please check the followin
At the moment, TDengine server supports running on Linux/Windows/MacOS systems. Any application can also choose the RESTful interface provided by taosAdapter to connect the taosd service. TDengine supports X64/ARM64 CPU, and it will support MIPS64, Alpha64, ARM32, RISC-V and other CPU architectures in the future. Right now we don't support build with cross-compiling environment.
+If you want to compile taosAdapter or taosKeeper, you need to install Go 1.18 or above.
+
## 3.1 On Linux
@@ -168,6 +170,10 @@ cmake .. -DBUILD_TOOLS=true -DBUILD_CONTRIB=true
make
```
+If you want to compile taosAdapter, you need to add the `-DBUILD_HTTP=false` option.
+
+If you want to compile taosKeeper, you need to add the `--DBUILD_KEEPER=true` option.
+
You can use Jemalloc as memory allocator instead of glibc:
```bash
@@ -196,6 +202,10 @@ mkdir debug && cd debug
cmake .. && cmake --build .
```
+If you want to compile taosAdapter, you need to add the `-DBUILD_HTTP=false` option.
+
+If you want to compile taosKeeper, you need to add the `--DBUILD_KEEPER=true` option.
+
## 4.3 Build on Windows
diff --git a/cmake/cmake.define b/cmake/cmake.define
index a794d927ad..dae8020dcc 100644
--- a/cmake/cmake.define
+++ b/cmake/cmake.define
@@ -117,7 +117,7 @@ ELSE()
ENDIF()
# force set all platform to JEMALLOC_ENABLED = false
-SET(JEMALLOC_ENABLED OFF)
+# SET(JEMALLOC_ENABLED OFF)
IF(TD_WINDOWS)
MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}")
@@ -258,3 +258,14 @@ ELSE()
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reserved-user-defined-literal -g3 -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k")
ENDIF()
ENDIF()
+
+
+IF(TD_LINUX)
+ IF(${JEMALLOC_ENABLED})
+ MESSAGE(STATUS "JEMALLOC_ENABLED Enabled")
+ SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-error=attributes")
+ SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=attributes")
+ ELSE()
+ MESSAGE(STATUS "JEMALLOC_ENABLED Disabled")
+ ENDIF()
+ENDIF()
\ No newline at end of file
diff --git a/docs/en/07-develop/04-schemaless.md b/docs/en/07-develop/04-schemaless.md
index ba20d63f1c..eb1e4f25eb 100644
--- a/docs/en/07-develop/04-schemaless.md
+++ b/docs/en/07-develop/04-schemaless.md
@@ -304,7 +304,7 @@ Not supported
## Querying the Written Data
-By running the example code from the previous section, tables will be automatically created in the power database. We can query the data using taos shell or an application. Below is an example of querying the data from the supertable and meters table using taos shell.
+By running the example code from the previous section, tables will be automatically created in the power database. We can query the data using TDengine CLI or an application. Below is an example of querying the data from the supertable and meters table using TDengine CLI.
```shell
taos> show power.stables;
diff --git a/docs/en/07-develop/07-tmq.md b/docs/en/07-develop/07-tmq.md
index 6b92ace6a2..cf370039c3 100644
--- a/docs/en/07-develop/07-tmq.md
+++ b/docs/en/07-develop/07-tmq.md
@@ -10,7 +10,7 @@ TDengine provides data subscription and consumption interfaces similar to those
## Creating Topics
-Please use taos shell or refer to the [Execute SQL](../running-sql-statements/) section to execute the SQL for creating topics: `CREATE TOPIC IF NOT EXISTS topic_meters AS SELECT ts, current, voltage, phase, groupid, location FROM meters`
+Please use TDengine CLI or refer to the [Execute SQL](../running-sql-statements/) section to execute the SQL for creating topics: `CREATE TOPIC IF NOT EXISTS topic_meters AS SELECT ts, current, voltage, phase, groupid, location FROM meters`
The above SQL will create a subscription named topic_meters. Each record in the messages obtained using this subscription is composed of the columns selected by this query statement `SELECT ts, current, voltage, phase, groupid, location FROM meters`.
@@ -44,6 +44,9 @@ There are many parameters for creating consumers, which flexibly support various
| `enable.replay` | boolean | Whether to enable data replay function | Default is off |
| `session.timeout.ms` | integer | Timeout after consumer heartbeat is lost, after which rebalance logic is triggered, and upon success, that consumer will be removed (supported from version 3.3.3.0) | Default is 12000, range [6000, 1800000] |
| `max.poll.interval.ms` | integer | The longest time interval for consumer poll data fetching, exceeding this time will be considered as the consumer being offline, triggering rebalance logic, and upon success, that consumer will be removed (supported from version 3.3.3.0) | Default is 300000, range [1000, INT32_MAX] |
+| `fetch.max.wait.ms` | integer | The maximum time it takes for the server to return data once (supported from version 3.3.6.0) | Default is 1000, range [1, INT32_MAX] |
+| `min.poll.rows` | integer | The minimum number of data returned by the server once (supported from version 3.3.6.0) | Default is 4096, range [1, INT32_MAX]
+| `msg.consume.rawdata` | integer | When consuming data, the data type pulled is binary and cannot be parsed. It is an internal parameter and is only used for taosx data migration（supported from version 3.3.6.0） | The default value of 0 indicates that it is not effective, and non-zero indicates that it is effective |
Below are the connection parameters for connectors in various languages:
diff --git a/docs/en/08-operation/02-planning.md b/docs/en/08-operation/02-planning.md
index a5715c1f34..d6589b3c5d 100644
--- a/docs/en/08-operation/02-planning.md
+++ b/docs/en/08-operation/02-planning.md
@@ -87,7 +87,7 @@ Additionally, to accelerate the data processing process, TDengine specifically s
```text
Users can use the performance testing tool taosBenchmark to assess the data compression effect of TDengine. By using the -f option to specify the write configuration file, taosBenchmark can write a specified number of CSV sample data into the specified database parameters and table structure.
-After completing the data writing, users can execute the flush database command in the taos shell to force all data to be written to the disk. Then, use the du command of the Linux operating system to get the size of the data folder of the specified vnode. Finally, divide the original data size by the actual storage data size to calculate the real compression ratio.
+After completing the data writing, users can execute the flush database command in the TDengine CLI to force all data to be written to the disk. Then, use the du command of the Linux operating system to get the size of the data folder of the specified vnode. Finally, divide the original data size by the actual storage data size to calculate the real compression ratio.
```
The following command can be used to obtain the storage space occupied by TDengine.
diff --git a/docs/en/08-operation/03-deployment/01-manual.md b/docs/en/08-operation/03-deployment/01-manual.md
index 3b07364f1d..4e4fbeceaf 100644
--- a/docs/en/08-operation/03-deployment/01-manual.md
+++ b/docs/en/08-operation/03-deployment/01-manual.md
@@ -55,7 +55,7 @@ For dnodes wishing to join the cluster, it is essential to ensure that the param
### 5. Start
-Start the first dnode, such as `h1.tdengine.com`, following the steps mentioned above. Then execute taos in the terminal to start TDengine's CLI program taos, and execute the `show dnodes` command within it to view all dnode information in the current cluster.
+Start the first dnode, such as `h1.tdengine.com`, following the steps mentioned above. Then execute taos in the terminal to start TDengine CLI program taos, and execute the `show dnodes` command within it to view all dnode information in the current cluster.
```shell
taos> show dnodes;
@@ -68,7 +68,7 @@ You can see that the endpoint of the dnode node that has just started is `h1.tde
### 6. Adding dnode
-Follow the steps mentioned earlier, start taosd on each physical node. Each dnode needs to configure the firstEp parameter in the taos.cfg file to the endpoint of the first node of the new cluster, which in this case is `h1.tdengine.com:6030`. On the machine where the first dnode is located, run taos in the terminal, open TDengine's CLI program taos, then log into the TDengine cluster, and execute the following SQL.
+Follow the steps mentioned earlier, start taosd on each physical node. Each dnode needs to configure the firstEp parameter in the taos.cfg file to the endpoint of the first node of the new cluster, which in this case is `h1.tdengine.com:6030`. On the machine where the first dnode is located, run taos in the terminal, open TDengine CLI program taos, then log into the TDengine cluster, and execute the following SQL.
```shell
create dnode "h2.tdengine.com:6030"
@@ -84,13 +84,13 @@ In the logs, please confirm that the fqdn and port of the output dnode are consi
**Tips**
-- Any dnode that has joined the cluster can serve as the firstEp for subsequent nodes to be added. The firstEp parameter only functions when that dnode first joins the cluster. After joining, the dnode will save the latest mnode's endpoint list, and subsequently, it no longer depends on this parameter. The firstEp parameter in the configuration file is mainly used for client connections, and if no parameters are set for TDengine's CLI, it will default to connecting to the node specified by firstEp.
+- Any dnode that has joined the cluster can serve as the firstEp for subsequent nodes to be added. The firstEp parameter only functions when that dnode first joins the cluster. After joining, the dnode will save the latest mnode's endpoint list, and subsequently, it no longer depends on this parameter. The firstEp parameter in the configuration file is mainly used for client connections, and if no parameters are set for TDengine CLI, it will default to connecting to the node specified by firstEp.
- Two dnodes that have not configured the firstEp parameter will run independently after starting. At this time, it is not possible to join one dnode to another to form a cluster.
- TDengine does not allow merging two independent clusters into a new cluster.
### 7. Adding mnode
-When creating a TDengine cluster, the first dnode automatically becomes the mnode of the cluster, responsible for managing and coordinating the cluster. To achieve high availability of mnode, subsequent dnodes need to manually create mnode. Please note that a cluster can create up to 3 mnodes, and only one mnode can be created on each dnode. When the number of dnodes in the cluster reaches or exceeds 3, you can create mnode for the existing cluster. In the first dnode, first log into TDengine through the CLI program taos, then execute the following SQL.
+When creating a TDengine cluster, the first dnode automatically becomes the mnode of the cluster, responsible for managing and coordinating the cluster. To achieve high availability of mnode, subsequent dnodes need to manually create mnode. Please note that a cluster can create up to 3 mnodes, and only one mnode can be created on each dnode. When the number of dnodes in the cluster reaches or exceeds 3, you can create mnode for the existing cluster. In the first dnode, first log into TDengine through TDengine CLI program taos, then execute the following SQL.
```shell
create mnode on dnode
diff --git a/docs/en/08-operation/03-deployment/03-kubernetes.md b/docs/en/08-operation/03-deployment/03-kubernetes.md
index 8a6edcead3..dbb6022ce6 100644
--- a/docs/en/08-operation/03-deployment/03-kubernetes.md
+++ b/docs/en/08-operation/03-deployment/03-kubernetes.md
@@ -456,7 +456,7 @@ export POD_NAME=$(kubectl get pods --namespace default \
kubectl --namespace default exec $POD_NAME -- taos -s "show dnodes; show mnodes"
-3. Run into taos shell:
+3. Run into TDengine CLI:
kubectl --namespace default exec -it $POD_NAME -- taos
```
diff --git a/docs/en/10-third-party/01-collection/flink.md b/docs/en/10-third-party/01-collection/flink.md
index b225a2d610..698f5644f8 100644
--- a/docs/en/10-third-party/01-collection/flink.md
+++ b/docs/en/10-third-party/01-collection/flink.md
@@ -26,6 +26,7 @@ Flink Connector supports all platforms that can run Flink 1.19 and above version
| Flink Connector Version | Major Changes | TDengine Version|
|-------------------------| ------------------------------------ | ---------------- |
+| 2.1.0 | Fix the issue of writing varchar types from different data sources.| - |
| 2.0.2 | The Table Sink supports types such as RowKind.UPDATE_BEFORE, RowKind.UPDATE_AFTER, and RowKind.DELETE.| - |
| 2.0.1 | Sink supports writing types from Rowdata implementations.| - |
| 2.0.0 | 1.Support SQL queries on data in TDengine database.
2. Support CDC subscription to data in TDengine database.
3. Supports reading and writing to TDengine database using Table SQL. | 3.3.5.1 and higher|
@@ -86,7 +87,8 @@ TDengine currently supports timestamp, number, character, and boolean types, and
| SMALLINT | Short |
| TINYINT | Byte |
| BOOL | Boolean |
-| BINARY | byte[] |
+| VARCHAR | StringData |
+| BINARY | StringData |
| NCHAR | StringData |
| JSON | StringData |
| VARBINARY | byte[] |
@@ -116,7 +118,7 @@ If using Maven to manage a project, simply add the following dependencies in pom
com.taosdata.flink
flink-connector-tdengine
- 2.0.2
+ 2.1.0
```
diff --git a/docs/en/10-third-party/05-bi/12-tableau.md b/docs/en/10-third-party/05-bi/12-tableau.md
new file mode 100644
index 0000000000..31292644b2
--- /dev/null
+++ b/docs/en/10-third-party/05-bi/12-tableau.md
@@ -0,0 +1,41 @@
+---
+sidebar_label: Tableau
+title: Integration With Tableau
+toc_max_heading_level: 4
+---
+
+Tableau is a well-known business intelligence tool that supports multiple data sources, making it easy to connect, import, and integrate data. And through an intuitive user interface, users can create rich and diverse visual charts, with powerful analysis and filtering functions, providing strong support for data decision-making. Users can import tag data, raw time-series data, or time-series data aggregated over time from TDengine into Tableau via the TDengine ODBC Connector to create reports or dashboards, and no code writing is required throughout the entire process.
+
+## Prerequisites
+
+Prepare the following environment:
+
+- TDengine 3.3.5.4 and above version is installed and running normally (both Enterprise and Community versions are available)
+- taosAdapter is running normally, refer to [taosAdapter Reference](../../../tdengine-reference/components/taosadapter/)
+- Install and run Tableau Desktop (if not installed, please download and install Windows operating system 64-bit [Download Tableau Desktop](https://www.tableau.com/products/desktop/download)). Install Tableau please refer to [Tableau Desktop](https://www.tableau.com).
+- Download the latest Windows operating system X64 client driver from the TDengine official website and install it, refer to [Install ODBC Driver](../../../tdengine-reference/client-libraries/odbc/#Installation).
+
+## Configure Data Source
+
+**Step 1**, Search and open the "ODBC Data Source (64 bit)" management tool in the Start menu of the Windows operating system and configure it, refer to [Install ODBC Driver](../../../tdengine-reference/client-libraries/odbc/#Installation).
+
+**Step 2**, Start Tableau in the Windows system environment, then search for "ODBC" on its connection page and select "Other Databases (ODBC)".
+
+**Step 3**, Click the `DSN` radio button, then select the configured data source (MyTDengine), and click the `Connect` button. After the connection is successful, delete the content of the string attachment, and finally click the `Sign In` button.
+
+
+
+## Data Analysis
+
+**Step 1**, In the workbook page, the connected data sources will be displayed. Clicking on the dropdown list of databases will display the databases that require data analysis. On this basis, click the search button in the table options to display all tables in the database. Then, drag the table to be analyzed to the right area to display the table structure.
+
+
+
+**Step 2**, Click the `Update Now` button below to display the data in the table.
+
+
+
+**Step 3**, Click on the "Worksheet" at the bottom of the window to pop up the data analysis window, which displays all the fields of the analysis table. Drag the fields to the rows and columns to display the chart.
+
+
+
diff --git a/docs/en/10-third-party/05-bi/13-excel.md b/docs/en/10-third-party/05-bi/13-excel.md
new file mode 100644
index 0000000000..aff4b7adf6
--- /dev/null
+++ b/docs/en/10-third-party/05-bi/13-excel.md
@@ -0,0 +1,42 @@
+---
+sidebar_label: Excel
+title: Integration With Excel
+toc_max_heading_level: 4
+---
+
+ By configuring the use of the ODBC connector, Excel can quickly access data from TDengine. Users can import tag data, raw time-series data, or time-aggregated time series data from TDengine into Excel to create reports or dashboards, all without the need for any coding.
+
+## Prerequisites
+
+Prepare the following environment:
+
+- TDengine 3.3.5.7 and above version is installed and running normally (both Enterprise and Community versions are available).
+- taosAdapter is running normally, refer to [taosAdapter Reference](../../../tdengine-reference/components/taosadapter/).
+- Install and run Excel. If not installed, please download and install it. For specific instructions, please refer to Microsoft's official documentation.
+- Download the latest Windows operating system X64 client driver from the TDengine official website and install it, refer to [Install ODBC Driver](../../../tdengine-reference/client-libraries/odbc/#Installation).
+
+## Configure Data Source
+
+**Step 1**, Search and open the [ODBC Data Source (64 bit)] management tool in the Start menu of the Windows operating system and configure it, refer to [Install ODBC Driver](../../../tdengine-reference/client-libraries/odbc/#Installation).
+
+**Step 2**, Start Excel in the Windows system environment, then select [Data] -> [Get Data] -> [From Other Sources] -> [From ODBC].
+
+
+
+**Step 3**, In the pop-up window, select the data source you need to connect to from the drop-down list of [Data source name (DSN)], and then click the [OK] button.
+
+
+
+**Step 4**, Enter the username and password for TDengine.
+
+
+
+**Step 5**, In the pop-up [Navigator] dialog box, select the database tables you want to load, and then click [Load] to complete the data loading.
+
+
+
+## Data Analysis
+
+Select the imported data. On the [Insert] tab, choose the column chart, and then configure the data fields in the [PivotChart Fields] pane on the right.
+
+
diff --git a/docs/en/10-third-party/05-bi/excel/odbc-config.jpg b/docs/en/10-third-party/05-bi/excel/odbc-config.jpg
new file mode 100644
index 0000000000..a62c9eb18f
Binary files /dev/null and b/docs/en/10-third-party/05-bi/excel/odbc-config.jpg differ
diff --git a/docs/en/10-third-party/05-bi/excel/odbc-data.jpg b/docs/en/10-third-party/05-bi/excel/odbc-data.jpg
new file mode 100644
index 0000000000..9a16033e9b
Binary files /dev/null and b/docs/en/10-third-party/05-bi/excel/odbc-data.jpg differ
diff --git a/docs/en/10-third-party/05-bi/excel/odbc-load.jpg b/docs/en/10-third-party/05-bi/excel/odbc-load.jpg
new file mode 100644
index 0000000000..7a6f829ff9
Binary files /dev/null and b/docs/en/10-third-party/05-bi/excel/odbc-load.jpg differ
diff --git a/docs/en/10-third-party/05-bi/excel/odbc-menu.jpg b/docs/en/10-third-party/05-bi/excel/odbc-menu.jpg
new file mode 100644
index 0000000000..f7dedc9cbb
Binary files /dev/null and b/docs/en/10-third-party/05-bi/excel/odbc-menu.jpg differ
diff --git a/docs/en/10-third-party/05-bi/excel/odbc-select.jpg b/docs/en/10-third-party/05-bi/excel/odbc-select.jpg
new file mode 100644
index 0000000000..fe72c975d5
Binary files /dev/null and b/docs/en/10-third-party/05-bi/excel/odbc-select.jpg differ
diff --git a/docs/en/10-third-party/05-bi/tableau/tableau-analysis.jpg b/docs/en/10-third-party/05-bi/tableau/tableau-analysis.jpg
new file mode 100644
index 0000000000..600bd286ac
Binary files /dev/null and b/docs/en/10-third-party/05-bi/tableau/tableau-analysis.jpg differ
diff --git a/docs/en/10-third-party/05-bi/tableau/tableau-data.jpg b/docs/en/10-third-party/05-bi/tableau/tableau-data.jpg
new file mode 100644
index 0000000000..9b8c7ba546
Binary files /dev/null and b/docs/en/10-third-party/05-bi/tableau/tableau-data.jpg differ
diff --git a/docs/en/10-third-party/05-bi/tableau/tableau-odbc.jpg b/docs/en/10-third-party/05-bi/tableau/tableau-odbc.jpg
new file mode 100644
index 0000000000..fe7c03f963
Binary files /dev/null and b/docs/en/10-third-party/05-bi/tableau/tableau-odbc.jpg differ
diff --git a/docs/en/10-third-party/05-bi/tableau/tableau-table.jpg b/docs/en/10-third-party/05-bi/tableau/tableau-table.jpg
new file mode 100644
index 0000000000..cb77a54197
Binary files /dev/null and b/docs/en/10-third-party/05-bi/tableau/tableau-table.jpg differ
diff --git a/docs/en/14-reference/02-tools/08-taos-cli.md b/docs/en/14-reference/02-tools/08-taos-cli.md
index 9309da831d..b173de495a 100644
--- a/docs/en/14-reference/02-tools/08-taos-cli.md
+++ b/docs/en/14-reference/02-tools/08-taos-cli.md
@@ -6,6 +6,9 @@ slug: /tdengine-reference/tools/tdengine-cli
The TDengine command line program (hereinafter referred to as TDengine CLI) is the simplest and most commonly used tool for users to operate and interact with TDengine instances. It requires the installation of either the TDengine Server package or the TDengine Client package before use.
+## Get
+TDengine CLI is the default installation component in the TDengine server and client installation package. It can be used after installation, refer to [TDengine Installation](../../../get-started/)
+
## Startup
To enter the TDengine CLI, simply execute `taos` in the terminal.
@@ -29,15 +32,83 @@ To exit the TDengine CLI, execute `q`, `quit`, or `exit` and press enter.
taos> quit
```
+## Command Line Parameters
+
+### Basic Parameters
+
+You can change the behavior of the TDengine CLI by configuring command line parameters. Below are some commonly used command line parameters:
+
+- -h HOST: The FQDN of the server where the TDengine service is located, default is to connect to the local service.
+- -P PORT: Specifies the port number used by the server.
+- -u USER: Username to use when connecting.
+- -p PASSWORD: Password to use when connecting to the server.
+- -?, --help: Prints out all command line parameters.
+- -s COMMAND: SQL command executed in non-interactive mode.
+ Use the `-s` parameter to execute SQL non interactively, and exit after execution. This mode is suitable for use in automated scripts.
+ For example, connect to the server h1.taos.com with the following command, and execute the SQL specified by `-s`:
+ ```bash
+ taos -h my-server -s "use db; show tables;"
+ ```
+
+- -c CONFIGDIR: Specify the configuration file directory.
+ In Linux, the default is `/etc/tao`. The default name of the configuration file in this directory is `taos.cfg`.
+ Use the `-c` parameter to change the location where the `taosc` client loads the configuration file. For client configuration parameters, refer to [Client Configuration](../../components/taosc).
+ The following command specifies the `taos.cfg` configuration file under `/root/cfg/` loaded by the `taosc` client.
+
+ ```bash
+ taos -c /root/cfg/
+ ```
+
+### Advanced Parameters
+
+- -a AUTHSTR: Authorization information for connecting to the server.
+- -A: Calculate authorization information using username and password.
+- -B: Set BI tool display mode, after setting, all outputs follow the format of BI tools.
+- -C: Print the configuration parameters of `taos.cfg` in the directory specified by -c.
+- -d DATABASE: Specifies the database to use when connecting to the server.
+- -E dsn: Connect to cloud services or servers providing WebSocket connections using WebSocket DSN.
+- -f FILE: Execute SQL script file in non-interactive mode. Each SQL statement in the file must occupy one line.
+- -k: Test the running status of the server, 0: unavailable, 1: network ok, 2: service ok, 3: service degraded, 4: exiting.
+- -l PKTLEN: Test packet size used during network testing.
+- -n NETROLE: Test range during network connection testing, default is `client`, options are `client`, `server`.
+- -N PKTNUM: Number of test packets used during network testing.
+- -r: Convert time columns to unsigned 64-bit integer type output (i.e., uint64_t in C language).
+- -R: Connect to the server using RESTful mode.
+- -t: Test the startup status of the server, status same as -k.
+- -w DISPLAYWIDTH: Client column display width.
+- -z TIMEZONE: Specifies the timezone, default is the local timezone.
+- -V: Print the current version number.
+
+## Data Export/Import
+
+### Data Export To a File
+
+- You can use the symbol “>>” to export query results to a file, the syntax is: sql query statement >> 'output file name'; If no path is written for the output file, it will be output to the current directory. For example, `select * from d0 >> '/root/d0.csv';` will output the query results to /root/d0.csv.
+
+### Data Import From a File
+
+- You can use insert into table_name file 'input file name', to import the data file exported in the previous step back into the specified table. For example, `insert into d0 file '/root/d0.csv';` means to import all the data exported above back into the d0 table.
+
+
## Execute SQL Scripts
-In the TDengine CLI, you can run multiple SQL commands from a script file using the `source` command.
+In the TDengine CLI, you can run multiple SQL commands from a script file using the `source` command, multiple SQL statements in the script file can be written in line.
```sql
taos> source ;
```
-## Online Modification of Display Character Width
+## TDengine CLI Tips
+
+### TAB Key Completion
+
+- Pressing the TAB key when no command is present will list all commands supported by TDengine CLI.
+- Pressing the TAB key when preceded by a space will display the first of all possible command words at this position, pressing TAB again will switch to the next one.
+- If a string precedes the TAB key, it will search for all command words that match the prefix of this string and display the first one, pressing TAB again will switch to the next one.
+- Entering a backslash `\` + TAB key, will automatically complete to the column display mode command word `\G;`.
+
+
+### Modification of Display Character Width
You can adjust the display character width in the TDengine CLI using the following command:
@@ -47,71 +118,16 @@ taos> SET MAX_BINARY_DISPLAY_WIDTH ;
If the displayed content ends with ..., it indicates that the content has been truncated. You can modify the display character width with this command to display the full content.
-## Command Line Parameters
+### Other
-You can change the behavior of the TDengine CLI by configuring command line parameters. Below are some commonly used command line parameters:
+- You can use the up and down arrow keys to view previously entered commands.
+- In TDengine CLI, use the `alter user` command to change user passwords, the default password is `taosdata`.
+- Ctrl+C to stop a query that is in progress.
+- Execute `RESET QUERY CACHE` to clear the cache of the local table Schema.
+- Batch execute SQL statements.
+ You can store a series of TDengine CLI commands (ending with a semicolon `;`, each SQL statement on a new line) in a file, and execute the command `source ` in TDengine CLI to automatically execute all SQL statements in that file.
-- -h HOST: The FQDN of the server where the TDengine service is located, default is to connect to the local service
-- -P PORT: Specifies the port number used by the server
-- -u USER: Username to use when connecting
-- -p PASSWORD: Password to use when connecting to the server
-- -?, --help: Prints out all command line parameters
-
-There are many other parameters:
-
-- -a AUTHSTR: Authorization information for connecting to the server
-- -A: Calculate authorization information using username and password
-- -B: Set BI tool display mode, after setting, all outputs follow the format of BI tools
-- -c CONFIGDIR: Specify the configuration file directory, default in Linux environment is `/etc/taos`, the default name of the configuration file in this directory is `taos.cfg`
-- -C: Print the configuration parameters of `taos.cfg` in the directory specified by -c
-- -d DATABASE: Specifies the database to use when connecting to the server
-- -E dsn: Connect to cloud services or servers providing WebSocket connections using WebSocket DSN
-- -f FILE: Execute SQL script file in non-interactive mode. Each SQL statement in the file must occupy one line
-- -k: Test the running status of the server, 0: unavailable, 1: network ok, 2: service ok, 3: service degraded, 4: exiting
-- -l PKTLEN: Test packet size used during network testing
-- -n NETROLE: Test range during network connection testing, default is `client`, options are `client`, `server`
-- -N PKTNUM: Number of test packets used during network testing
-- -r: Convert time columns to unsigned 64-bit integer type output (i.e., uint64_t in C language)
-- -R: Connect to the server using RESTful mode
-- -s COMMAND: SQL command executed in non-interactive mode
-- -t: Test the startup status of the server, status same as -k
-- -w DISPLAYWIDTH: Client column display width
-- -z TIMEZONE: Specifies the timezone, default is the local timezone
-- -V: Print the current version number
-
-Example:
-
-```shell
-taos -h h1.taos.com -s "use db; show tables;"
-```
-
-## Configuration File
-
-You can also control the behavior of the TDengine CLI through parameters set in the configuration file. For available configuration parameters, please refer to [Client Configuration](../../components/taosc)
## Error Code Table
Starting from TDengine version 3.3.4.8, TDengine CLI returns specific error codes in error messages. Users can visit the TDengine official website's error code page to find specific reasons and solutions, see: [Error Code Reference](../../error-codes/)
-
-## TDengine CLI TAB Key Completion
-
-- Pressing the TAB key when no command is present will list all commands supported by TDengine CLI
-- Pressing the TAB key when preceded by a space will display the first of all possible command words at this position, pressing TAB again will switch to the next one
-- If a string precedes the TAB key, it will search for all command words that match the prefix of this string and display the first one, pressing TAB again will switch to the next one
-- Entering a backslash `\` + TAB key, will automatically complete to the column display mode command word `\G;`
-
-## TDengine CLI Tips
-
-- You can use the up and down arrow keys to view previously entered commands
-- In TDengine CLI, use the `alter user` command to change user passwords, the default password is `taosdata`
-- Ctrl+C to stop a query that is in progress
-- Execute `RESET QUERY CACHE` to clear the cache of the local table Schema
-- Batch execute SQL statements. You can store a series of TDengine CLI commands (ending with a semicolon `;`, each SQL statement on a new line) in a file, and execute the command `source ` in TDengine CLI to automatically execute all SQL statements in that file
-
-## TDengine CLI Export Query Results to a File
-
-- You can use the symbol “>>” to export query results to a file, the syntax is: sql query statement >> 'output file name'; If no path is written for the output file, it will be output to the current directory. For example, select * from d0 >> '/root/d0.csv'; will output the query results to /root/d0.csv.
-
-## TDengine CLI Import Data from a File into a Table
-
-- You can use insert into table_name file 'input file name', to import the data file exported in the previous step back into the specified table. For example, insert into d0 file '/root/d0.csv'; means to import all the data exported above back into the d0 table.
diff --git a/docs/en/14-reference/02-tools/09-taosdump.md b/docs/en/14-reference/02-tools/09-taosdump.md
index 1ce08ff825..ec9d98fe6a 100644
--- a/docs/en/14-reference/02-tools/09-taosdump.md
+++ b/docs/en/14-reference/02-tools/09-taosdump.md
@@ -6,45 +6,26 @@ slug: /tdengine-reference/tools/taosdump
`taosdump` is a TDengine data backup/recovery tool provided for open source users, and the backed up data files adopt the standard [Apache AVRO](https://avro.apache.org/)
Format, convenient for exchanging data with the external ecosystem.
- Taosdump provides multiple data backup and recovery options to meet different data needs, and all supported options can be viewed through --help.
+ taosdump provides multiple data backup and recovery options to meet different data needs, and all supported options can be viewed through --help.
-## Installation
+## Get
-taosdump is the default installation component in the TDengine installation package, which can be used after installing TDengine. For how to install TDengine, please refer to [TDengine Installation](../../../get-started/)
+taosdump is the default installation component in the TDengine server and client installation package. It can be used after installation, refer to [TDengine Installation](../../../get-started/)
-## Common Use Cases
+## Startup
-### taosdump Backup Data
+taosdump needs to be run in the command line terminal. It must be run with parameters to indicate backup or restore operations, such as:
+``` bash
+taosdump -h my-server -D test -o /root/test/
+```
+The above command means to backup the `test` database on the `my server` machine to the `/root/test/` directory.
-1. Backup all databases: specify the `-A` or `--all-databases` parameter;
-2. Backup multiple specified databases: use the `-D db1,db2,...` parameter;
-3. Backup certain supertables or basic tables in a specified database: use the `dbname stbname1 stbname2 tbname1 tbname2 ...` parameter, note that this input sequence starts with the database name, supports only one database, and the second and subsequent parameters are the names of the supertables or basic tables in that database, separated by spaces;
-4. Backup the system log database: TDengine clusters usually include a system database named `log`, which contains data for TDengine's own operation, taosdump does not back up the log database by default. If there is a specific need to back up the log database, you can use the `-a` or `--allow-sys` command line parameter.
-5. "Tolerant" mode backup: Versions after taosdump 1.4.1 provide the `-n` and `-L` parameters, used for backing up data without using escape characters and in "tolerant" mode, which can reduce backup data time and space occupied when table names, column names, and label names do not use escape characters. If unsure whether to use `-n` and `-L`, use the default parameters for "strict" mode backup. For an explanation of escape characters, please refer to the [official documentation](../../sql-manual/escape-characters/).
-6. If a backup file already exists in the directory specified by the `-o` parameter, to prevent data from being overwritten, taosdump will report an error and exit. Please replace it with another empty directory or clear the original data before backing up.
-7. Currently, taosdump does not support data breakpoint backup function. Once the data backup is interrupted, it needs to be started from scratch.
- If the backup takes a long time, it is recommended to use the (-S -E options) method to specify the start/end time for segmented backup.
+``` bash
+taosdump -h my-server -i /root/test/
+```
+The above command means to restore the previously backed up data files in the `/root/test/` directory to the host named `my server`.
-:::tip
-
-- Versions after taosdump 1.4.1 provide the `-I` parameter, used for parsing avro file schema and data, specifying the `-s` parameter will only parse the schema.
-- Backups after taosdump 1.4.2 use the `-B` parameter to specify the number of batches, the default value is 16384. If "Error actual dump .. batch .." occurs due to insufficient network speed or disk performance in some environments, you can try adjusting the `-B` parameter to a smaller value.
-- taosdump's export does not support interruption recovery, so the correct way to handle an unexpected termination of the process is to delete all related files that have been exported or generated.
-- taosdump's import supports interruption recovery, but when the process restarts, you may receive some "table already exists" prompts, which can be ignored.
-
-:::
-
-### taosdump Restore Data
-
-- Restore data files from a specified path: use the `-i` parameter along with the data file path. As mentioned earlier, the same directory should not be used to back up different data sets, nor should the same path be used to back up the same data set multiple times, otherwise, the backup data will cause overwriting or multiple backups.
-- taosdump supports data recovery to a new database name with the parameter `-W`, please refer to the command line parameter description for details.
-
-:::tip
-taosdump internally uses the TDengine stmt binding API to write restored data, currently using 16384 as a batch for writing. If there are many columns in the backup data, it may cause a "WAL size exceeds limit" error, in which case you can try adjusting the `-B` parameter to a smaller value.
-
-:::
-
-## Detailed Command Line Parameters List
+## Command Line Parameters
Below is the detailed command line parameters list for taosdump:
@@ -65,8 +46,8 @@ Usage: taosdump [OPTION...] dbname [tbname ...]
-c, --config-dir=CONFIG_DIR Configure directory. Default is /etc/taos
-i, --inpath=INPATH Input file path.
-o, --outpath=OUTPATH Output file path.
- -r, --resultFile=RESULTFILE DumpOut/In Result file path and name.
- -a, --allow-sys Allow to dump system database
+ -r, --resultFile=RESULTFILE DumpOut/In Result file path and name.
+ -a, --allow-sys Allow to dump system database.
-A, --all-databases Dump all databases.
-D, --databases=DATABASES Dump inputted databases. Use comma to separate
databases' name.
@@ -98,23 +79,54 @@ Usage: taosdump [OPTION...] dbname [tbname ...]
-n, --no-escape No escape char '`'. Default is using it.
-Q, --dot-replace Replace dot character with underline character in
the table name.(Version 2.5.3)
- -T, --thread-num=THREAD_NUM Number of thread for dump in file. Default is
- 8.
- -W, --rename=RENAME-LIST Rename database name with new name during
+ -T, --thread-num=THREAD_NUM Number of thread for dump in file. Default is 8.
+ -W, --rename=RENAME-LIST Rename database name with new name during.
importing data. RENAME-LIST:
"db1=newDB1|db2=newDB2" means rename db1 to newDB1
- and rename db2 to newDB2 (Version 2.5.4)
+ and rename db2 to newDB2 (Version 2.5.4).
-k, --retry-count=VALUE Set the number of retry attempts for connection or
- query failures
- -z, --retry-sleep-ms=VALUE retry interval sleep time, unit ms
- -C, --cloud=CLOUD_DSN specify a DSN to access TDengine cloud service
- -R, --restful Use RESTful interface to connect TDengine
+ query failures.
+ -z, --retry-sleep-ms=VALUE retry interval sleep time, unit ms.
+ -C, --cloud=CLOUD_DSN specify a DSN to access TDengine cloud service.
+ -R, --restful Use RESTful interface to connect TDengine.
-t, --timeout=SECONDS The timeout seconds for websocket to interact.
-g, --debug Print debug info.
- -?, --help Give this help list
- --usage Give a short usage message
- -V, --version Print program version
+ -?, --help Give this help list.
+ --usage Give a short usage message.
+ -V, --version Print program version.
Mandatory or optional arguments to long options are also mandatory or optional
for any corresponding short options.
```
+
+## Common Use Cases
+
+### Backup Data
+
+1. Backup all databases: specify the `-A` or `--all-databases` parameter.
+2. Backup multiple specified databases: use the `-D db1,db2,...` parameter.
+3. Backup certain supertables or basic tables in a specified database: use the `dbname stbname1 stbname2 tbname1 tbname2 ...` parameter, note that this input sequence starts with the database name, supports only one database, and the second and subsequent parameters are the names of the supertables or basic tables in that database, separated by spaces.
+4. Backup the system log database: TDengine clusters usually include a system database named `log`, which contains data for TDengine's own operation, taosdump does not back up the log database by default. If there is a specific need to back up the log database, you can use the `-a` or `--allow-sys` command line parameter.
+5. "Tolerant" mode backup: Versions after taosdump 1.4.1 provide the `-n` and `-L` parameters, used for backing up data without using escape characters and in "tolerant" mode, which can reduce backup data time and space occupied when table names, column names, and label names do not use escape characters. If unsure whether to use `-n` and `-L`, use the default parameters for "strict" mode backup. For an explanation of escape characters, please refer to the [official documentation](../../sql-manual/escape-characters/)
+6. If a backup file already exists in the directory specified by the `-o` parameter, to prevent data from being overwritten, taosdump will report an error and exit. Please replace it with another empty directory or clear the original data before backing up.
+7. Currently, taosdump does not support data breakpoint backup function. Once the data backup is interrupted, it needs to be started from scratch.
+ If the backup takes a long time, it is recommended to use the (-S -E options) method to specify the start/end time for segmented backup.
+
+:::tip
+
+- Versions after taosdump 1.4.1 provide the `-I` parameter, used for parsing avro file schema and data, specifying the `-s` parameter will only parse the schema.
+- Backups after taosdump 1.4.2 use the `-B` parameter to specify the number of batches, the default value is 16384. If "Error actual dump .. batch .." occurs due to insufficient network speed or disk performance in some environments, you can try adjusting the `-B` parameter to a smaller value.
+- taosdump's export does not support interruption recovery, so the correct way to handle an unexpected termination of the process is to delete all related files that have been exported or generated.
+- taosdump's import supports interruption recovery, but when the process restarts, you may receive some "table already exists" prompts, which can be ignored.
+
+:::
+
+### Restore Data
+
+- Restore data files from a specified path: use the `-i` parameter along with the data file path. As mentioned earlier, the same directory should not be used to back up different data sets, nor should the same path be used to back up the same data set multiple times, otherwise, the backup data will cause overwriting or multiple backups.
+- taosdump supports data recovery to a new database name with the parameter `-W`, please refer to the command line parameter description for details.
+
+:::tip
+taosdump internally uses the TDengine stmt binding API to write restored data, currently using 16384 as a batch for writing. If there are many columns in the backup data, it may cause a "WAL size exceeds limit" error, in which case you can try adjusting the `-B` parameter to a smaller value.
+
+:::
diff --git a/docs/en/14-reference/02-tools/10-taosbenchmark.md b/docs/en/14-reference/02-tools/10-taosbenchmark.md
index f146d7134c..cfc92b4e0b 100644
--- a/docs/en/14-reference/02-tools/10-taosbenchmark.md
+++ b/docs/en/14-reference/02-tools/10-taosbenchmark.md
@@ -2,30 +2,33 @@
title: taosBenchmark Reference
sidebar_label: taosBenchmark
slug: /tdengine-reference/tools/taosbenchmark
+toc_max_heading_level: 4
---
TaosBenchmark is a performance benchmarking tool for TDengine products, providing insertion, query, and subscription performance testing for TDengine products, and outputting performance indicators.
-## Installation
+## Get
-taosBenchmark is the default installation component in the TDengine installation package, which can be used after installing TDengine. For how to install TDengine, please refer to [TDengine Installation](../../../get started/)
+taosBenchmark is the default installation component in the TDengine server and client installation package. It can be used after installation, refer to [TDengine Installation](../../../get-started/)
-## Operation
+## Startup
-### Configuration and Operation Methods
+taosbenchmark supports three operating modes:
+- No Parameter Mode
+- Command Line Mode
+- JSON Configuration File Mode
-taosBbenchmark supports three operating modes:
-- No parameter mode
-- Command line mode
-- JSON configuration file mode
-The command-line approach is a subset of the functionality of JSON configuration files, which immediately uses the command line and then the configuration file, with the parameters specified by the command line taking precedence.
+The `Command Line Mode` is a subset of the `JSON Configuration File Mode` function. When both are used at the same time, the `Command Line Mode` takes precedence.
-**Ensure that the TDengine cluster is running correctly before running taosBenchmark.**
+:::tip
+Ensure that the TDengine cluster is running correctly before running taosBenchmark.
+:::
-### Running Without Command Line Arguments
+### No Parameter Mode
Execute the following command to quickly experience taosBenchmark performing a write performance test on TDengine based on the default configuration.
+
```shell
taosBenchmark
```
@@ -33,17 +36,18 @@ taosBenchmark
When running without parameters, taosBenchmark defaults to connecting to the TDengine cluster specified in `/etc/taos/taos.cfg `.
After successful connection, a smart meter example database test, super meters, and 10000 sub meters will be created, with 10000 records per sub meter. If the test database already exists, it will be deleted before creating a new one.
-### Running Using Command Line Configuration Parameters
+### Command Line Mode
-When running taosBenchmark using command line parameters and controlling its behavior, the `-f ` parameter cannot be used. All configuration parameters must be specified through the command line. Below is an example of using command line mode to test the write performance of taosBenchmark.
+The parameters supported by the command line are those frequently used in the write function. The query and subscription function does not support the command line mode.
+For example:
-```shell
-taosBenchmark -I stmt -n 200 -t 100
+```bash
+taosBenchmark -d db -t 100 -n 1000 -T 4 -I stmt -y
```
-The above command `taosBenchmark` will create a database named `test`, establish a supertable `meters` within it, create 100 subtables in the supertable, and insert 200 records into each subtable using parameter binding.
+This command means that using `taosBenchmark` will create a database named `db`, create the default super table `meters`, sub table 100, and use parameter binding (stmt) to write 1000 records for each sub table.
-### Running Using a Configuration File
+### JSON Configuration File Mode
Running in configuration file mode provides all functions, so parameters can be configured to run in the configuration file.
@@ -51,42 +55,8 @@ Running in configuration file mode provides all functions, so parameters can be
taosBenchmark -f
```
-**Below are a few examples of configuration files:**
-#### JSON Configuration File Example for Insertion Scenario
-
-
-insert.json
-
-```json
-{{#include /TDengine/tools/taos-tools/example/insert.json}}
-```
-
-
-
-#### Example JSON Configuration File for Query Scenario
-
-
-query.json
-
-```json
-{{#include /TDengine/tools/taos-tools/example/query.json}}
-```
-
-
-
-#### Example JSON Configuration File for Subscription Scenario
-
-
-tmq.json
-
-```json
-{{#include /TDengine/tools/taos-tools/example/tmq.json}}
-```
-
-
-
-## Detailed Explanation of Command Line Parameters
+## Command Line Parameters
- **-f/--file \** :
The JSON configuration file to use, specifying all parameters. This parameter cannot be used simultaneously with other command line parameters. There is no default value.
@@ -212,62 +182,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\)
Displays help information and exits. Cannot be used with other parameters.
-## Output performance indicators
-
-#### Write indicators
-
-After writing is completed, a summary performance metric will be output in the last two lines in the following format:
-``` bash
-SUCC: Spent 8.527298 (real 8.117379) seconds to insert rows: 10000000 with 8 thread(s) into test 1172704.41 (real 1231924.74) records/second
-SUCC: insert delay, min: 19.6780ms, avg: 64.9390ms, p90: 94.6900ms, p95: 105.1870ms, p99: 130.6660ms, max: 157.0830ms
-```
-First line write speed statistics:
-- Spent: Total write time, in seconds, counting from the start of writing the first data to the end of the last data. This indicates that a total of 8.527298 seconds were spent
-- Real: Total write time (calling the engine), excluding the time spent preparing data for the testing framework. Purely counting the time spent on engine calls, The time spent is 8.117379 seconds. If 8.527298-8.117379=0.409919 seconds, it is the time spent preparing data for the testing framework
-- Rows: Write the total number of rows, which is 10 million pieces of data
-- Threads: The number of threads being written, which is 8 threads writing simultaneously
-- Records/second write speed = `total write time` / `total number of rows written`, real in parentheses is the same as before, indicating pure engine write speed
-
-Second line single write delay statistics:
-- min: Write minimum delay
-- avg: Write normal delay
-- p90: Write delay p90 percentile delay number
-- p95: Write delay p95 percentile delay number
-- p99: Write delay p99 percentile delay number
-- max: maximum write delay
-Through this series of indicators, the distribution of write request latency can be observed
-
-#### Query indicators
-The query performance test mainly outputs the QPS indicator of query request speed, and the output format is as follows:
-
-``` bash
-complete query with 3 threads and 10000 query delay avg: 0.002686s min: 0.001182s max: 0.012189s p90: 0.002977s p95: 0.003493s p99: 0.004645s SQL command: select ...
-INFO: Spend 26.9530 second completed total queries: 30000, the QPS of all threads: 1113.049
-```
-
-- The first line represents the percentile distribution of query execution and query request delay for each of the three threads executing 10000 queries. The SQL command is the test query statement
-- The second line indicates that the total query time is 26.9653 seconds, and the query rate per second (QPS) is 1113.049 times/second
-- If the `continue_if_fail` option is set to `yes` in the query, the last line will output the number of failed requests and error rate, the format like "error + number of failed requests (error rate)"
-- QPS = number of successful requests / time spent (in seconds)
-- Error rate = number of failed requests / (number of successful requests + number of failed requests)
-
-#### Subscription metrics
-
-The subscription performance test mainly outputs consumer consumption speed indicators, with the following output format:
-``` bash
-INFO: consumer id 0 has poll total msgs: 376, period rate: 37.592 msgs/s, total rows: 3760000, period rate: 375924.815 rows/s
-INFO: consumer id 1 has poll total msgs: 362, period rate: 36.131 msgs/s, total rows: 3620000, period rate: 361313.504 rows/s
-INFO: consumer id 2 has poll total msgs: 364, period rate: 36.378 msgs/s, total rows: 3640000, period rate: 363781.731 rows/s
-INFO: consumerId: 0, consume msgs: 1000, consume rows: 10000000
-INFO: consumerId: 1, consume msgs: 1000, consume rows: 10000000
-INFO: consumerId: 2, consume msgs: 1000, consume rows: 10000000
-INFO: Consumed total msgs: 3000, total rows: 30000000
-```
-- Lines 1 to 3 real-time output of the current consumption speed of each consumer, msgs/s represents the number of consumption messages, each message contains multiple rows of data, and rows/s represents the consumption speed calculated by rows
-- Lines 4 to 6 show the overall statistics of each consumer after the test is completed, including the total number of messages consumed and the total number of lines
-- The overall statistics of all consumers in line 7, `msgs` represents how many messages were consumed in total, `rows` represents how many rows of data were consumed in total
-
-## Configuration File Parameters Detailed Explanation
+## Configuration File Parameters
### General Configuration Parameters
@@ -284,7 +199,7 @@ The parameters listed in this section apply to all functional modes.
- **password** : Password for connecting to the TDengine server, default value is taosdata.
-### Configuration Parameters for Insertion Scenarios
+### Insertion Configuration Parameters
In insertion scenarios, `filetype` must be set to `insert`. For this parameter and other common parameters, see Common Configuration Parameters.
@@ -299,7 +214,7 @@ In insertion scenarios, `filetype` must be set to `insert`. For this parameter a
“continue_if_fail”: “yes”, taosBenchmark warns the user and continues writing
“continue_if_fail”: “smart”, if the child table does not exist upon failure, taosBenchmark will create the child table and continue writing
-#### Database Related Configuration Parameters
+#### Database Parameters
Parameters related to database creation are configured in the `dbinfo` section of the json configuration file, specific parameters are as follows. Other parameters correspond to those specified in TDengine's `create database`, see [../../taos-sql/database]
@@ -307,23 +222,7 @@ Parameters related to database creation are configured in the `dbinfo` section o
- **drop**: Whether to delete the database before insertion, options are "yes" or "no", "no" means do not create. Default is to delete.
-#### Stream Computing Related Configuration Parameters
-
-Parameters related to stream computing are configured in the `stream` section of the json configuration file, specific parameters are as follows.
-
-- **stream_name**: Name of the stream computing, mandatory.
-
-- **stream_stb**: Name of the supertable corresponding to the stream computing, mandatory.
-
-- **stream_sql**: SQL statement for the stream computing, mandatory.
-
-- **trigger_mode**: Trigger mode for the stream computing, optional.
-
-- **watermark**: Watermark for the stream computing, optional.
-
-- **drop**: Whether to create stream computing, options are "yes" or "no", "no" means do not create.
-
-#### Supertable Related Configuration Parameters
+#### Supertable Parameters
Parameters related to supertable creation are configured in the `super_tables` section of the json configuration file, specific parameters are as follows.
@@ -357,7 +256,7 @@ Parameters related to supertable creation are configured in the `super_tables` s
- **childtable_limit** : Effective only when child_table_exists is yes, specifies the limit when getting the subtable list from the supertable.
-- **interlace_rows** : Enables interlaced insertion mode and specifies the number of rows to insert into each subtable at a time. Interlaced insertion mode means inserting the number of rows specified by this parameter into each subtable in turn and repeating this process until all subtable data is inserted. The default value is 0, i.e., data is inserted into one subtable before moving to the next subtable.
+- **interlace_rows** : Enables interlaced insertion mode and specifies the number of rows to insert into each subtable at a time. Interlaced insertion mode means inserting the number of rows specified by this parameter into each subtable in turn and repeating this process until all subtable data is inserted. The default is 0, i.e., data is inserted into one subtable before moving to the next subtable.
- **insert_interval** : Specifies the insertion interval for interlaced insertion mode, in ms, default value is 0. Only effective when `-B/--interlace-rows` is greater than 0. It means that the data insertion thread will wait for the time interval specified by this value after inserting interlaced records for each subtable before proceeding to the next round of writing.
@@ -385,7 +284,7 @@ Parameters related to supertable creation are configured in the `super_tables` s
- **sqls** : Array of strings type, specifies the array of sql to be executed after the supertable is successfully created, the table name specified in sql must be prefixed with the database name, otherwise an unspecified database error will occur
-#### Tag and Data Column Configuration Parameters
+#### Tag and Data Columns
Specify the configuration parameters for tag and data columns in `super_tables` under `columns` and `tag`.
@@ -420,7 +319,7 @@ Specify the configuration parameters for tag and data columns in `super_tables`
- **fillNull**: String type, specifies whether this column randomly inserts NULL values, can be specified as "true" or "false", only effective when generate_row_rule is 2.
-#### Insertion Behavior Configuration Parameters
+#### Insertion Behavior Parameters
- **thread_count**: The number of threads for inserting data, default is 8.
@@ -432,7 +331,7 @@ Specify the configuration parameters for tag and data columns in `super_tables`
- **confirm_parameter_prompt** : A toggle parameter that requires user confirmation after a prompt to continue. The value can be "yes" or "no", by default "no".
-- **interlace_rows** : Enables interleaved insertion mode and specifies the number of rows to insert into each subtable at a time. Interleaved insertion mode refers to inserting the specified number of rows into each subtable in sequence and repeating this process until all subtable data has been inserted. The default value is 0, meaning data is inserted into one subtable completely before moving to the next.
+- **interlace_rows** : Enables interleaved insertion mode and specifies the number of rows to insert into each subtable at a time. Interleaved insertion mode refers to inserting the specified number of rows into each subtable in sequence and repeating this process until all subtable data has been inserted. The default is 0, meaning data is inserted into one subtable completely before moving to the next.
This parameter can also be configured in `super_tables`; if configured, the settings in `super_tables` take higher priority and override the global settings.
- **insert_interval** :
@@ -442,68 +341,84 @@ Specify the configuration parameters for tag and data columns in `super_tables`
- **num_of_records_per_req** :
The number of data rows requested per write to TDengine, default is 30000. If set too high, the TDengine client driver will return corresponding error messages, and this parameter needs to be reduced to meet the writing requirements.
-- **prepare_rand** : The number of unique values in the generated random data. If it is 1, it means all data are the same. The default value is 10000.
+- **prepare_rand** : The number of unique values in the generated random data. If it is 1, it means all data are the same. The default is 10000.
- **pre_load_tb_meta** : Whether to pre-load the meta data of subtables, values are “yes” or "no". When there are a large number of subtables, turning on this option can improve the writing speed.
-### Configuration Parameters for Query Scenarios
+### Query Parameters
In query scenarios, `filetype` must be set to `query`.
`query_times` specifies the number of times to run the query, numeric type.
Query scenarios can control the execution of slow query statements by setting `kill_slow_query_threshold` and `kill_slow_query_interval` parameters, where threshold controls that queries exceeding the specified exec_usec time will be killed by taosBenchmark, in seconds; interval controls the sleep time to avoid continuous slow query CPU consumption, in seconds.
-For other common parameters, see Common Configuration Parameters.
+For other common parameters, see [General Configuration Parameters](#general-configuration-parameters)
-#### Configuration Parameters for Executing Specified Query Statements
-
-Configuration parameters for querying specified tables (can specify supertables, subtables, or regular tables) are set in `specified_table_query`.
-
-`General Query`: Each SQL in `sqls` starts `threads` threads to query this SQL, Each thread exits after executing the `query_times` queries, and only after all threads executing this SQL have completed can the next SQL be executed.
-The total number of queries(`General Query`) = the number of `sqls` * `query_times` * `threads`
-- `Mixed Query` : All SQL statements in `sqls` are divided into `threads` groups, with each thread executing one group. Each SQL statement needs to execute `query_times` queries.
-The total number of queries(`Mixed Query`) = the number of `sqls` * `query_times`
+#### Specified Query
+Configuration parameters for querying specified tables (can specify supertables, subtables, or regular tables) are set in `specified_table_query`.
+- **mixed_query** : Query Mode . "yes" is `Mixed Query`, "no" is `General Query`, default is "no".
+ `General Query`:
+ Each SQL in `sqls` starts `threads` threads to query this SQL, Each thread exits after executing the `query_times` queries, and only after all threads executing this SQL have completed can the next SQL be executed.
+ The total number of queries(`General Query`) = the number of `sqls` * `query_times` * `threads`
+ `Mixed Query`:
+ All SQL statements in `sqls` are divided into `threads` groups, with each thread executing one group. Each SQL statement needs to execute `query_times` queries.
+ The total number of queries(`Mixed Query`) = the number of `sqls` * `query_times`
- **query_interval** : Query interval, in millisecond, default is 0.
-
- **threads** : Number of threads executing the SQL query, default is 1.
-
- **sqls**:
- **sql**: The SQL command to execute, required.
- **result**: File to save the query results, if not specified, results are not saved.
-#### Configuration Parameters for Querying Supertables
+#### Supertables
Configuration parameters for querying supertables are set in `super_table_query`.
The thread mode of the super table query is the same as the `Normal Query` mode of the specified query statement described above, except that `sqls` is filled all sub tables.
- **stblname** : The name of the supertable to query, required.
-
- **query_interval** : Query interval, in seconds, default is 0.
-
- **threads** : Number of threads executing the SQL query, default is 1.
-
- **sqls** :
- **sql** : The SQL command to execute, required; for supertable queries, keep "xxxx" in the SQL command, the program will automatically replace it with all subtable names of the supertable.
- **result** : File to save the query results, if not specified, results are not saved.
- **Note**: The maximum number of SQL arrays configured under SQL is 100.
-### Configuration Parameters for Subscription Scenarios
+### Subscription Parameters
-In subscription scenarios, `filetype` must be set to `subscribe`, this parameter and other common parameters see Common Configuration Parameters.
+In the subscription scenario, `filetype` must be set to `subscribe`. For details of this parameter and other general parameters, see [General Configuration Parameters](#general-configuration-parameters)
+The subscription configuration parameters are set under `tmq_info`. The parameters are as follows:
-#### Configuration Parameters for Executing Specified Subscription Statements
+- **concurrent**: the number of consumers who consume subscriptions, or the number of concurrent consumers. The default value is 1.
+- **create_mode**: create a consumer mode.
+ Which can be sequential: create in sequence. parallel: It is created at the same time. It is required and has no default value.
+- **group_mode**: generate the consumer groupId mode.
+ Which can take the value share: all consumers can only generate one groupId independent: Each consumer generates an independent groupId. If `group.id` is not set, this item is mandatory and has no default value.
+-**poll_delay**: The polling timeout time passed in by calling tmq_consumer_poll.
+ The unit is milliseconds. A negative number means the default timeout is 1 second.
+-**enable.manual.commit**: whether manual submission is allowed.
+ The value can be true: manual submission is allowed, after consuming messages, manually call tmq_commit_sync to complete the submission. false：Do not submit, default value: false.
+-**rows_file**: a file that stores consumption data.
+ It can be a full path or a relative path with a file name.The actual saved file will be followed by the consumer serial number. For example, rows_file is result, and the actual file name is result_1 (consumer 1) result_2 (consumer 2).
+-**expect_rows**: the number of rows and data types expected to be consumed by each consumer.
+ When the consumption reaches this number, the consumption will exit, and the consumption will continue without setting.
+-**topic_list**: specifies the topic list and array type of consumption.
+ Example of topic list format: `{"name": "topic1", "sql": "select * from test.meters;"}`.
+ name: Specify the topic name.
+ sql: Specify the sql statement for creating topic, Ensure that the sql is correct, and the framework will automatically create topic.
-Configuration parameters for subscribing to specified tables (can specify supertables, subtables, or regular tables) are set in `specified_table_query`.
+For the following parameters, see the description of [Subscription](../../../advanced-features/data-subscription/):
+- **client.id**
+- **auto.offset.reset**
+- **enable.auto.commit**
+- **enable.auto.commit**
+- **msg.with.table.name**
+- **auto.commit.interval.ms**
+- **group.id**: If this value is not specified, the groupId will be generated by the rule specified by `group_mode`. If this value is specified, the `group_mode` parameter is ignore.
-- **threads/concurrent** : Number of threads executing the SQL, default is 1.
-- **sqls** :
- - **sql** : The SQL command to execute, required.
+### Data Type Comparison Table
-### Data Type Writing Comparison Table in Configuration File
-
-| # | **Engine** | **taosBenchmark**
+| # | **TDengine** | **taosBenchmark**
| --- | :----------------: | :---------------:
| 1 | TIMESTAMP | timestamp
| 2 | INT | int
@@ -525,3 +440,97 @@ Configuration parameters for subscribing to specified tables (can specify supert
| 18 | JSON | json
Note: Data types in the taosBenchmark configuration file must be in lowercase to be recognized.
+
+## Example Of Configuration Files
+
+**Below are a few examples of configuration files:**
+
+#### Insertion Example
+
+
+insert.json
+
+```json
+{{#include /TDengine/tools/taos-tools/example/insert.json}}
+```
+
+
+
+#### Query Example
+
+
+query.json
+
+```json
+{{#include /TDengine/tools/taos-tools/example/query.json}}
+```
+
+
+
+#### Subscription Example
+
+
+tmq.json
+
+```json
+{{#include /TDengine/tools/taos-tools/example/tmq.json}}
+```
+
+
+
+Other json examples see [here](https://github.com/taosdata/TDengine/tree/main/tools/taos-tools/example)
+
+## Output Performance Indicators
+
+#### Write Indicators
+
+After writing is completed, a summary performance metric will be output in the last two lines in the following format:
+``` bash
+SUCC: Spent 8.527298 (real 8.117379) seconds to insert rows: 10000000 with 8 thread(s) into test 1172704.41 (real 1231924.74) records/second
+SUCC: insert delay, min: 19.6780ms, avg: 64.9390ms, p90: 94.6900ms, p95: 105.1870ms, p99: 130.6660ms, max: 157.0830ms
+```
+First line write speed statistics:
+- Spent: Total write time, in seconds, counting from the start of writing the first data to the end of the last data. This indicates that a total of 8.527298 seconds were spent.
+- Real: Total write time (calling the engine), excluding the time spent preparing data for the testing framework. Purely counting the time spent on engine calls, The time spent is 8.117379 seconds. If 8.527298-8.117379=0.409919 seconds, it is the time spent preparing data for the testing framework.
+- Rows: Write the total number of rows, which is 10 million pieces of data.
+- Threads: The number of threads being written, which is 8 threads writing simultaneously.
+- Records/second write speed = `total write time` / `total number of rows written`, real in parentheses is the same as before, indicating pure engine write speed.
+
+Second line single write delay statistics:
+- min: Write minimum delay.
+- avg: Write normal delay.
+- p90: Write delay p90 percentile delay number.
+- p95: Write delay p95 percentile delay number.
+- p99: Write delay p99 percentile delay number.
+- max: maximum write delay.
+Through this series of indicators, the distribution of write request latency can be observed.
+
+#### Query indicators
+The query performance test mainly outputs the QPS indicator of query request speed, and the output format is as follows:
+
+``` bash
+complete query with 3 threads and 10000 query delay avg: 0.002686s min: 0.001182s max: 0.012189s p90: 0.002977s p95: 0.003493s p99: 0.004645s SQL command: select ...
+INFO: Spend 26.9530 second completed total queries: 30000, the QPS of all threads: 1113.049
+```
+
+- The first line represents the percentile distribution of query execution and query request delay for each of the three threads executing 10000 queries. The SQL command is the test query statement.
+- The second line indicates that the total query time is 26.9653 seconds, and the query rate per second (QPS) is 1113.049 times/second.
+- If the `continue_if_fail` option is set to `yes` in the query, the last line will output the number of failed requests and error rate, the format like "error + number of failed requests (error rate)".
+- QPS = number of successful requests / time spent (in seconds)
+- Error rate = number of failed requests / (number of successful requests + number of failed requests)
+
+#### Subscription indicators
+
+The subscription performance test mainly outputs consumer consumption speed indicators, with the following output format:
+``` bash
+INFO: consumer id 0 has poll total msgs: 376, period rate: 37.592 msgs/s, total rows: 3760000, period rate: 375924.815 rows/s
+INFO: consumer id 1 has poll total msgs: 362, period rate: 36.131 msgs/s, total rows: 3620000, period rate: 361313.504 rows/s
+INFO: consumer id 2 has poll total msgs: 364, period rate: 36.378 msgs/s, total rows: 3640000, period rate: 363781.731 rows/s
+INFO: consumerId: 0, consume msgs: 1000, consume rows: 10000000
+INFO: consumerId: 1, consume msgs: 1000, consume rows: 10000000
+INFO: consumerId: 2, consume msgs: 1000, consume rows: 10000000
+INFO: Consumed total msgs: 3000, total rows: 30000000
+```
+- Lines 1 to 3 real-time output of the current consumption speed of each consumer, msgs/s represents the number of consumption messages, each message contains multiple rows of data, and rows/s represents the consumption speed calculated by rows.
+- Lines 4 to 6 show the overall statistics of each consumer after the test is completed, including the total number of messages consumed and the total number of lines.
+- The overall statistics of all consumers in line 7, `msgs` represents how many messages were consumed in total, `rows` represents how many rows of data were consumed in total.
\ No newline at end of file
diff --git a/docs/en/14-reference/03-taos-sql/14-stream.md b/docs/en/14-reference/03-taos-sql/14-stream.md
index 704a7ebcbc..24d9bae468 100644
--- a/docs/en/14-reference/03-taos-sql/14-stream.md
+++ b/docs/en/14-reference/03-taos-sql/14-stream.md
@@ -9,7 +9,7 @@ import imgStream from './assets/stream-processing-01.png';
## Creating Stream Computing
```sql
-CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name[(field1_name, field2_name [PRIMARY KEY], ...)] [TAGS (create_definition [, create_definition] ...)] SUBTABLE(expression) AS subquery
+CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name[(field1_name, field2_name [PRIMARY KEY], ...)] [TAGS (create_definition [, create_definition] ...)] SUBTABLE(expression) AS subquery [notification_definition]
stream_options: {
TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time | FORCE_WINDOW_CLOSE]
WATERMARK time
@@ -85,6 +85,8 @@ CREATE STREAM streams1 IGNORE EXPIRED 1 WATERMARK 100s INTO streamt1 AS
SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WINDOW(10);
```
+notification_definition clause specifies the addresses to which notifications should be sent when designated events occur during window computations, such as window opening or closing. For more details, see [Stream Computing Event Notifications](#stream-computing-event-notifications).
+
## Stream Computation Partitioning
You can use `PARTITION BY TBNAME`, tags, regular columns, or expressions to partition a stream for multi-partition computation. Each partition's timeline and window are independent, aggregating separately, and writing into different subtables of the target table.
@@ -305,3 +307,223 @@ CREATE SNODE ON DNODE [id]
The id is the serial number of the dnode in the cluster. Please be mindful of the selected dnode, as the intermediate state of stream computing will automatically be backed up on it.
Starting from version 3.3.4.0, in a multi-replica environment, creating a stream will perform an **existence check** of snode, requiring the snode to be created first. If the snode does not exist, the stream cannot be created.
+
+## Stream Computing Event Notifications
+
+### User Guide
+
+Stream computing supports sending event notifications to external systems when windows open or close. Users can specify the events to be notified and the target addresses for receiving notification messages using the notification_definition clause.
+
+```sql
+notification_definition:
+ NOTIFY (url [, url] ...) ON (event_type [, event_type] ...) [notification_options]
+
+event_type:
+ 'WINDOW_OPEN'
+ | 'WINDOW_CLOSE'
+
+notification_options: {
+ NOTIFY_HISTORY [0|1]
+ ON_FAILURE [DROP|PAUSE]
+}
+```
+
+The rules for the syntax above are as follows:
+1. `url`: Specifies the target address for the notification. It must include the protocol, IP or domain name, port, and may include a path and parameters. Currently, only the websocket protocol is supported. For example: 'ws://localhost:8080', 'ws://localhost:8080/notify', 'wss://localhost:8080/notify?key=foo'.
+2. `event_type`: Defines the events that trigger notifications. Supported event types include:
+ 1. 'WINDOW_OPEN': Window open event; triggered when any type of window opens.
+ 2. 'WINDOW_CLOSE': Window close event; triggered when any type of window closes.
+3. `NOTIFY_HISTORY`: Controls whether to trigger notifications during the computation of historical data. The default value is 0, which means no notifications are sent.
+4. `ON_FAILURE`: Determines whether to allow dropping some events if sending notifications fails (e.g., in poor network conditions). The default value is `PAUSE`:
+ 1. PAUSE means that the stream computing task is paused if sending a notification fails. taosd will retry until the notification is successfully delivered and the task resumes.
+ 2. DROP means that if sending a notification fails, the event information is discarded, and the stream computing task continues running unaffected.
+
+For example, the following creates a stream that computes the per-minute average current from electric meters and sends notifications to two target addresses when the window opens and closes. It does not send notifications for historical data and does not allow dropping notifications on failure:
+
+```sql
+CREATE STREAM avg_current_stream FILL_HISTORY 1
+ AS SELECT _wstart, _wend, AVG(current) FROM meters
+ INTERVAL (1m)
+ NOTIFY ('ws://localhost:8080/notify', 'wss://192.168.1.1:8080/notify?key=foo')
+ ON ('WINDOW_OPEN', 'WINDOW_CLOSE');
+ NOTIFY_HISTORY 0
+ ON_FAILURE PAUSE;
+```
+
+When the specified events are triggered, taosd will send a POST request to the given URL(s) with a JSON message body. A single request may contain events from several streams, and the event types may differ.
+
+The details of the event information depend on the type of window:
+
+1. Time Window: At the opening, the start time is sent; at the closing, the start time, end time, and computation result are sent.
+2. State Window: At the opening, the start time, previous window's state, and current window's state are sent; at closing, the start time, end time, computation result, current window state, and next window state are sent.
+3. Session Window: At the opening, the start time is sent; at the closing, the start time, end time, and computation result are sent.
+4. Event Window: At the opening, the start time along with the data values and corresponding condition index that triggered the window opening are sent; at the closing, the start time, end time, computation result, and the triggering data value and condition index for window closure are sent.
+5. Count Window: At the opening, the start time is sent; at the closing, the start time, end time, and computation result are sent.
+
+An example structure for the notification message is shown below:
+
+```json
+{
+ "messageId": "unique-message-id-12345",
+ "timestamp": 1733284887203,
+ "streams": [
+ {
+ "streamName": "avg_current_stream",
+ "events": [
+ {
+ "tableName": "t_a667a16127d3b5a18988e32f3e76cd30",
+ "eventType": "WINDOW_OPEN",
+ "eventTime": 1733284887097,
+ "windowId": "window-id-67890",
+ "windowType": "Time",
+ "windowStart": 1733284800000
+ },
+ {
+ "tableName": "t_a667a16127d3b5a18988e32f3e76cd30",
+ "eventType": "WINDOW_CLOSE",
+ "eventTime": 1733284887197,
+ "windowId": "window-id-67890",
+ "windowType": "Time",
+ "windowStart": 1733284800000,
+ "windowEnd": 1733284860000,
+ "result": {
+ "_wstart": 1733284800000,
+ "avg(current)": 1.3
+ }
+ }
+ ]
+ },
+ {
+ "streamName": "max_voltage_stream",
+ "events": [
+ {
+ "tableName": "t_96f62b752f36e9b16dc969fe45363748",
+ "eventType": "WINDOW_OPEN",
+ "eventTime": 1733284887231,
+ "windowId": "window-id-13579",
+ "windowType": "Event",
+ "windowStart": 1733284800000,
+ "triggerCondition": {
+ "conditionIndex": 0,
+ "fieldValue": {
+ "c1": 10,
+ "c2": 15
+ }
+ },
+ },
+ {
+ "tableName": "t_96f62b752f36e9b16dc969fe45363748",
+ "eventType": "WINDOW_CLOSE",
+ "eventTime": 1733284887231,
+ "windowId": "window-id-13579",
+ "windowType": "Event",
+ "windowStart": 1733284800000,
+ "windowEnd": 1733284810000,
+ "triggerCondition": {
+ "conditionIndex": 1,
+ "fieldValue": {
+ "c1": 20
+ "c2": 3
+ }
+ },
+ "result": {
+ "_wstart": 1733284800000,
+ "max(voltage)": 220
+ }
+ }
+ ]
+ }
+ ]
+}
+```
+
+The following sections explain the fields in the notification message.
+
+### Root-Level Field Descriptions
+
+1. "messageId": A string that uniquely identifies the notification message. It ensures that the entire message can be tracked and de-duplicated.
+2. "timestamp": A long integer timestamp representing the time when the notification message was generated, accurate to the millisecond (i.e., the number of milliseconds since '00:00, Jan 1 1970 UTC').
+3. "streams": An array containing the event information for multiple stream tasks. (See the following sections for details.)
+
+### "stream" Object Field Descriptions
+
+1. "streamName": A string representing the name of the stream task, used to identify which stream the events belong to.
+2. "events": An array containing the list of event objects for the stream task. Each event object includes detailed information. (See the next sections for details.)
+
+### "event" Object Field Descriptions
+
+#### Common Fields
+
+These fields are common to all event objects.
+1. "tableName": A string indicating the name of the target subtable.
+2. "eventType": A string representing the event type ("WINDOW_OPEN", "WINDOW_CLOSE", or "WINDOW_INVALIDATION").
+3. "eventTime": A long integer timestamp that indicates when the event was generated, accurate to the millisecond (i.e., the number of milliseconds since '00:00, Jan 1 1970 UTC').
+4. "windowId": A string representing the unique identifier for the window. This ID ensures that the open and close events for the same window can be correlated. In the case that taosd restarts due to a fault, some events may be sent repeatedly, but the windowId remains constant for the same window.
+5. "windowType": A string that indicates the window type ("Time", "State", "Session", "Event", or "Count").
+
+#### Fields for Time Windows
+
+These fields are present only when "windowType" is "Time".
+1. When "eventType" is "WINDOW_OPEN", the following field is included:
+ 1. "windowStart": A long integer timestamp representing the start time of the window, matching the time precision of the result table.
+2. When "eventType" is "WINDOW_CLOSE", the following fields are included:
+ 1. "windowStart": A long integer timestamp representing the start time of the window.
+ 1. "windowEnd": A long integer timestamp representing the end time of the window.
+ 1. "result": An object containing key-value pairs of the computed result columns and their corresponding values.
+
+#### Fields for State Windows
+
+These fields are present only when "windowType" is "State".
+1. When "eventType" is "WINDOW_OPEN", the following fields are included:
+ 1. "windowStart": A long integer timestamp representing the start time of the window.
+ 1. "prevState": A value of the same type as the state column, representing the state of the previous window. If there is no previous window (i.e., this is the first window), it will be NULL.
+ 1. "curState": A value of the same type as the state column, representing the current window's state.
+2. When "eventType" is "WINDOW_CLOSE", the following fields are included:
+ 1. "windowStart": A long integer timestamp representing the start time of the window.
+ 1. "windowEnd": A long integer timestamp representing the end time of the window.
+ 1. "curState": The current window's state.
+ 1. "nextState": The state for the next window.
+ 1. "result": An object containing key-value pairs of the computed result columns and their corresponding values.
+
+#### Fields for Session Windows
+
+These fields are present only when "windowType" is "Session".
+1. When "eventType" is "WINDOW_OPEN", the following field is included:
+ 1. "windowStart": A long integer timestamp representing the start time of the window.
+2. When "eventType" is "WINDOW_CLOSE", the following fields are included:
+ 1. "windowStart": A long integer timestamp representing the start time of the window.
+ 1. "windowEnd": A long integer timestamp representing the end time of the window.
+ 1. "result": An object containing key-value pairs of the computed result columns and their corresponding values.
+
+#### Fields for Event Windows
+
+These fields are present only when "windowType" is "Event".
+1. When "eventType" is "WINDOW_OPEN", the following fields are included:
+ 1. "windowStart": A long integer timestamp representing the start time of the window.
+ 1. "triggerCondition": An object that provides information about the condition that triggered the window to open. It includes:
+ 1. "conditionIndex": An integer representing the index of the condition that triggered the window, starting from 0.
+ 1. "fieldValue": An object containing key-value pairs of the column names related to the condition and their respective values.
+2. When "eventType" is "WINDOW_CLOSE", the following fields are included:
+ 1. "windowStart": A long integer timestamp representing the start time of the window.
+ 1. "windowEnd": A long integer timestamp representing the end time of the window.
+ 1. "triggerCondition": An object that provides information about the condition that triggered the window to close. It includes:
+ 1. "conditionIndex": An integer representing the index of the condition that triggered the closure, starting from 0.
+ 1. "fieldValue": An object containing key-value pairs of the related column names and their respective values.
+ 1. "result": An object containing key-value pairs of the computed result columns and their corresponding values.
+
+#### Fields for Count Windows
+
+These fields are present only when "windowType" is "Count".
+1. When "eventType" is "WINDOW_OPEN", the following field is included:
+ 1. "windowStart": A long integer timestamp representing the start time of the window.
+2. When "eventType" is "WINDOW_CLOSE", the following fields are included:
+ 1. "windowStart": A long integer timestamp representing the start time of the window.
+ 1. "windowEnd": A long integer timestamp representing the end time of the window.
+ 1. "result": An object containing key-value pairs of the computed result columns and their corresponding values.
+
+#### Fields for Window Invalidation
+
+Due to scenarios such as data disorder, updates, or deletions during stream computing, windows that have already been generated might be removed or their results need to be recalculated. In such cases, a notification with the eventType "WINDOW_INVALIDATION" is sent to inform which windows have been invalidated.
+For events with "eventType" as "WINDOW_INVALIDATION", the following fields are included:
+1. "windowStart": A long integer timestamp representing the start time of the window.
+1. "windowEnd": A long integer timestamp representing the end time of the window.
diff --git a/docs/en/14-reference/05-connector/40-csharp.md b/docs/en/14-reference/05-connector/40-csharp.md
index 01f4f0e81d..e3db8bc351 100644
--- a/docs/en/14-reference/05-connector/40-csharp.md
+++ b/docs/en/14-reference/05-connector/40-csharp.md
@@ -25,6 +25,7 @@ import RequestId from "../../assets/resources/_request_id.mdx";
| Connector Version | Major Changes | TDengine Version |
|-------------------|------------------------------------------------------------|--------------------|
+| 3.1.6 | Optimize WebSocket connection message handling. | - |
| 3.1.5 | Fix WebSocket encoding error for Chinese character length. | - |
| 3.1.4 | Improved WebSocket query and insert performance. | 3.3.2.0 and higher |
| 3.1.3 | Supported WebSocket auto-reconnect. | - |
@@ -39,25 +40,25 @@ For error reporting in other TDengine modules, please refer to [Error Codes](../
## Data Type Mapping
-| TDengine DataType | C# Type |
-|-------------------|------------------|
-| TIMESTAMP | DateTime |
-| TINYINT | sbyte |
-| SMALLINT | short |
-| INT | int |
-| BIGINT | long |
-| TINYINT UNSIGNED | byte |
-| SMALLINT UNSIGNED | ushort |
-| INT UNSIGNED | uint |
-| BIGINT UNSIGNED | ulong |
-| FLOAT | float |
-| DOUBLE | double |
-| BOOL | bool |
-| BINARY | byte[] |
-| NCHAR | string (utf-8 encoded) |
-| JSON | byte[] |
-| VARBINARY | byte[] |
-| GEOMETRY | byte[] |
+| TDengine DataType | C# Type |
+|-------------------|----------|
+| TIMESTAMP | DateTime |
+| TINYINT | sbyte |
+| SMALLINT | short |
+| INT | int |
+| BIGINT | long |
+| TINYINT UNSIGNED | byte |
+| SMALLINT UNSIGNED | ushort |
+| INT UNSIGNED | uint |
+| BIGINT UNSIGNED | ulong |
+| FLOAT | float |
+| DOUBLE | double |
+| BOOL | bool |
+| BINARY | byte[] |
+| NCHAR | string |
+| JSON | byte[] |
+| VARBINARY | byte[] |
+| GEOMETRY | byte[] |
**Note**: JSON type is only supported in tags.
The GEOMETRY type is binary data in little endian byte order, conforming to the WKB standard. For more details, please refer to [Data Types](../../sql-manual/data-types/)
diff --git a/docs/en/14-reference/05-connector/50-odbc.md b/docs/en/14-reference/05-connector/50-odbc.md
index b4c3e23e13..2dcab047a7 100644
--- a/docs/en/14-reference/05-connector/50-odbc.md
+++ b/docs/en/14-reference/05-connector/50-odbc.md
@@ -644,6 +644,3 @@ This section summarizes the ODBC API by functionality. For a complete ODBC API r
- **Standard**: ODBC
- **Function**: Closes the cursor associated with the current statement handle and releases all resources used by the cursor
-## Integration with Third Parties
-
-As an example of using the TDengine ODBC driver, you can use Power BI to analyze time-series data with TDengine. For more details, please refer to [Power BI](../../../third-party-tools/analytics/power-bi/)
diff --git a/docs/en/27-train-faq/index.md b/docs/en/27-train-faq/index.md
index aa85729102..4e89ede4f6 100644
--- a/docs/en/27-train-faq/index.md
+++ b/docs/en/27-train-faq/index.md
@@ -73,7 +73,7 @@ If the client encounters a connection failure, please follow the steps below to
### 5. What to do if you encounter the error "Unable to resolve FQDN"?
-This error occurs because the client or data node cannot resolve the FQDN (Fully Qualified Domain Name). For the TAOS Shell or client applications, please check the following:
+This error occurs because the client or data node cannot resolve the FQDN (Fully Qualified Domain Name). For the TDengine CLI or client applications, please check the following:
1. Check if the FQDN of the server you are connecting to is correct.
2. If there is a DNS server in the network configuration, check if it is working properly
@@ -244,15 +244,15 @@ launchctl limit maxfiles
This prompt indicates that the number of vnodes required for creating the db is not enough, exceeding the upper limit of vnodes in the dnode. By default, a dnode contains twice the number of CPU cores worth of vnodes, which can also be controlled by the supportVnodes parameter in the configuration file.
Normally, increase the supportVnodes parameter in taos.cfg.
-### 21 Why can data from a specified time period be queried using taos-CLI on the server, but not on the client machine?
+### 21 Why can data from a specified time period be queried using TDengine CLI on the server, but not on the client machine?
This issue is due to the client and server having different time zone settings. Adjusting the client's time zone to match the server's will resolve the issue.
### 22 The table name is confirmed to exist, but returns "table name does not exist" when writing or querying, why?
-In TDengine, all names, including database names and table names, are case-sensitive. If these names are not enclosed in backticks (\`) in the program or taos-CLI, even if you input them in uppercase, the engine will convert them to lowercase for use. If the names are enclosed in backticks, the engine will not convert them to lowercase and will use them as is.
+In TDengine, all names, including database names and table names, are case-sensitive. If these names are not enclosed in backticks (\`) in the program or TDengine CLI, even if you input them in uppercase, the engine will convert them to lowercase for use. If the names are enclosed in backticks, the engine will not convert them to lowercase and will use them as is.
-### 23 How to fully display field content in taos-CLI queries?
+### 23 How to fully display field content in TDengine CLI queries?
You can use the \G parameter for vertical display, such as `show databases\G\;` (for ease of input, press TAB after "\" to automatically complete the content).
@@ -315,4 +315,4 @@ Problem solving: You should configure the automatic mount of the dataDir directo
Directly querying from child table is fast. The query from super table with TAG filter is designed to meet the convenience of querying. It can filter data from multiple child tables at the same time. If the goal is to pursue performance and the child table has been clearly queried, directly querying from the sub table can achieve higher performance
### 35 How to view data compression ratio indicators?
-Currently, TDengine only provides compression ratios based on tables, not databases or the entire system. To view the compression ratios, execute the `SHOW TABLE DISTRIBUTED table_name;` command in the client taos-CLI. The table_name can be a super table, regular table, or subtable. For details [Click Here](https://docs.tdengine.com/tdengine-reference/sql-manual/show-commands/#show-table-distributed)
\ No newline at end of file
+Currently, TDengine only provides compression ratios based on tables, not databases or the entire system. To view the compression ratios, execute the `SHOW TABLE DISTRIBUTED table_name;` command in the client TDengine CLI. The table_name can be a super table, regular table, or subtable. For details [Click Here](https://docs.tdengine.com/tdengine-reference/sql-manual/show-commands/#show-table-distributed)
\ No newline at end of file
diff --git a/docs/en/28-releases/03-notes/3.3.2.0.md b/docs/en/28-releases/03-notes/3.3.2.0.md
index f6f5f39589..b539467bb1 100644
--- a/docs/en/28-releases/03-notes/3.3.2.0.md
+++ b/docs/en/28-releases/03-notes/3.3.2.0.md
@@ -56,4 +56,4 @@ slug: /release-history/release-notes/3-3-2-0
13. Upgrading to 3.3.0.0 and enabling `cachemodel` causes incorrect row count returns for `last + group by`
14. `taos-explorer` navigation bar does not display all supertable names (Enterprise Edition only)
15. Querying causes `taosd` to crash when compound primary key VARCHAR length exceeds 125
-16. High CPU usage by `taos CLI` and `taosAdapter`
+16. High CPU usage by `TDengine CLI` and `taosAdapter`
diff --git a/docs/en/28-releases/03-notes/3.3.3.0.md b/docs/en/28-releases/03-notes/3.3.3.0.md
index 15dd21ff9f..9fa267d7dc 100644
--- a/docs/en/28-releases/03-notes/3.3.3.0.md
+++ b/docs/en/28-releases/03-notes/3.3.3.0.md
@@ -53,7 +53,7 @@ slug: /release-history/release-notes/3-3-3-0
22. Cursor error in data filling during cache update causes taosd to exit abnormally
23. Random incorrect results of STDDEV function calculation
24. Unable to add offline nodes in multi-tier storage and encryption scenarios
-25. taos CLI cannot input passwords longer than 20 bytes
+25. TDengine CLI cannot input passwords longer than 20 bytes
26. SQL write error: int data overflow
27. Metadata consistency in scenarios of high query concurrency
28. Attempt to solve the issue where manually clicking the stop button does not stop the task
@@ -90,4 +90,4 @@ slug: /release-history/release-notes/3-3-3-0
59. Client memory leak
60. Open-source users unable to modify other database options after upgrading stt_trigger value
61. Incorrect results for NOT IN (NULL) queries
-62. taos shell and taosBenchmark unable to successfully connect to cloud service instances
+62. TDengine CLI and taosBenchmark unable to successfully connect to cloud service instances
diff --git a/docs/en/28-releases/03-notes/3.3.4.3.md b/docs/en/28-releases/03-notes/3.3.4.3.md
index 4b5e817a7e..a5a4be19e8 100644
--- a/docs/en/28-releases/03-notes/3.3.4.3.md
+++ b/docs/en/28-releases/03-notes/3.3.4.3.md
@@ -52,13 +52,13 @@ slug: /release-history/release-notes/3-3-4-3
1. fix: memory leak caused by repeated addition and deletion of tables on the Windows platform.
1. fix(stream): check the right return code for concurrent checkpoint trans.
1. fix: the "too many session" problem while perform large concurrent queries.
-1. fix: the problem of taos shell crashing in slow query scenarios on the Windows platform.
-1. fix: the encrypted database cannot be recovered when opening the dnode log.
-1. fix: the problem that taosd cannot be started due to mnode synchronization timeout.
-1. fix: the slow sorting of file group data during snapshot synchronization leads to the inability of Vnode to recover.
-1. fix: when writing data with escape characters to a varchar field throug line protocol, taosd will crash.
-1. fix: metadata file damage caused by incorrect logic processing of error code
-1. fix: when a query statement contains multiple nested "not" conditional statements, not setting the scalar mode will lead to query errors.
-1. fix: the problem of dnode going offline due to timeout of vnode stat report.
-1. fix: taosd failed to start on servers that not support AVX instructions.
-1. fix(taosX): handle 0x09xx error codes in migration
+2. fix: the problem of TDengine CLI crashing in slow query scenarios on the Windows platform.
+3. fix: the encrypted database cannot be recovered when opening the dnode log.
+4. fix: the problem that taosd cannot be started due to mnode synchronization timeout.
+5. fix: the slow sorting of file group data during snapshot synchronization leads to the inability of Vnode to recover.
+6. fix: when writing data with escape characters to a varchar field throug line protocol, taosd will crash.
+7. fix: metadata file damage caused by incorrect logic processing of error code
+8. fix: when a query statement contains multiple nested "not" conditional statements, not setting the scalar mode will lead to query errors.
+9. fix: the problem of dnode going offline due to timeout of vnode stat report.
+10. fix: taosd failed to start on servers that not support AVX instructions.
+11. fix(taosX): handle 0x09xx error codes in migration
diff --git a/docs/en/28-releases/03-notes/3.3.5.0.md b/docs/en/28-releases/03-notes/3.3.5.0.md
index 6f23205770..c840c70d0a 100755
--- a/docs/en/28-releases/03-notes/3.3.5.0.md
+++ b/docs/en/28-releases/03-notes/3.3.5.0.md
@@ -10,7 +10,7 @@ slug: /release-history/release-notes/3-3-5-0
2. feat: refactor taosX incremental backup-restore
3. feat: add stmt2 apis in JDBC via websocket connection
4. feat: add stmt2 api in Rust connector
- 5. feat: add error codes in error prompts in taos-CLI
+ 5. feat: add error codes in error prompts in TDengine CLI
6. feat: superSet can connect TDengine with python connector
7. feat: configurable grafana dashboards in explorer management
8. feat: add taosX-agent in-memory cache queu capacity option
diff --git a/docs/en/28-releases/03-notes/3.3.5.2.md b/docs/en/28-releases/03-notes/3.3.5.2.md
index ce0141cccf..a70376bc69 100755
--- a/docs/en/28-releases/03-notes/3.3.5.2.md
+++ b/docs/en/28-releases/03-notes/3.3.5.2.md
@@ -38,6 +38,6 @@ slug: /release-history/release-notes/3.3.5.2
20. fix: column names were not correctly copied when using SELECT * FROM subqueries
21. fix: when performing max/min function on string type data, the results are inaccurate and taosd will crash
22. fix: stream computing does not support the use of the HAVING clause, but no error is reported during creation
- 23. fix: the version information displayed by taos shell for the server is inaccurate, such as being unable to correctly distinguish between the community edition and the enterprise edition
+ 23. fix: the version information displayed by TDengine CLI for the server is inaccurate, such as being unable to correctly distinguish between the community edition and the enterprise edition
24. fix: in certain specific query scenarios, when JOIN and CAST are used together, taosd may crash
diff --git a/docs/examples/JDBC/taosdemo/pom.xml b/docs/examples/JDBC/taosdemo/pom.xml
index 0435a8736b..0f8955d958 100644
--- a/docs/examples/JDBC/taosdemo/pom.xml
+++ b/docs/examples/JDBC/taosdemo/pom.xml
@@ -77,13 +77,6 @@
fastjson
1.2.83
-
-
- mysql
- mysql-connector-java
- 8.0.28
- test
-
org.apache.logging.log4j
diff --git a/docs/examples/flink/Main.java b/docs/examples/flink/Main.java
index 50a507d1de..333b780056 100644
--- a/docs/examples/flink/Main.java
+++ b/docs/examples/flink/Main.java
@@ -198,7 +198,7 @@ splitSql.setSelect("ts, current, voltage, phase, groupid, location")
", current: " + rowData.getFloat(1) +
", voltage: " + rowData.getInt(2) +
", phase: " + rowData.getFloat(3) +
- ", location: " + new String(rowData.getBinary(4)));
+ ", location: " + rowData.getString(4).toString());
sb.append("\n");
return sb.toString();
});
@@ -273,7 +273,7 @@ splitSql.setSelect("ts, current, voltage, phase, groupid, location")
", current: " + row.getFloat(1) +
", voltage: " + row.getInt(2) +
", phase: " + row.getFloat(3) +
- ", location: " + new String(row.getBinary(4)));
+ ", location: " + rowData.getString(4).toString());
sb.append("\n");
totalVoltage.addAndGet(row.getInt(2));
}
@@ -311,7 +311,7 @@ splitSql.setSelect("ts, current, voltage, phase, groupid, location")
", current: " + rowData.getFloat(1) +
", voltage: " + rowData.getInt(2) +
", phase: " + rowData.getFloat(3) +
- ", location: " + new String(rowData.getBinary(4)));
+ ", location: " + rowData.getString(4).toString());
sb.append("\n");
totalVoltage.addAndGet(rowData.getInt(2));
return sb.toString();
@@ -353,7 +353,7 @@ splitSql.setSelect("ts, current, voltage, phase, groupid, location")
", current: " + row.getFloat(1) +
", voltage: " + row.getInt(2) +
", phase: " + row.getFloat(3) +
- ", location: " + new String(row.getBinary(4)));
+ ", location: " + rowData.getString(4).toString());
sb.append("\n");
totalVoltage.addAndGet(row.getInt(2));
}
@@ -489,9 +489,9 @@ splitSql.setSelect("ts, current, voltage, phase, groupid, location")
" `current` FLOAT," +
" voltage INT," +
" phase FLOAT," +
- " location VARBINARY," +
+ " location VARCHAR(255)," +
" groupid INT," +
- " tbname VARBINARY" +
+ " tbname VARCHAR(255)" +
") WITH (" +
" 'connector' = 'tdengine-connector'," +
" 'td.jdbc.url' = 'jdbc:TAOS-WS://localhost:6041/power?user=root&password=taosdata'," +
@@ -506,9 +506,9 @@ splitSql.setSelect("ts, current, voltage, phase, groupid, location")
" `current` FLOAT," +
" voltage INT," +
" phase FLOAT," +
- " location VARBINARY," +
+ " location VARCHAR(255)," +
" groupid INT," +
- " tbname VARBINARY" +
+ " tbname VARCHAR(255)" +
") WITH (" +
" 'connector' = 'tdengine-connector'," +
" 'td.jdbc.mode' = 'sink'," +
@@ -535,9 +535,9 @@ splitSql.setSelect("ts, current, voltage, phase, groupid, location")
" `current` FLOAT," +
" voltage INT," +
" phase FLOAT," +
- " location VARBINARY," +
+ " location VARCHAR(255)," +
" groupid INT," +
- " tbname VARBINARY" +
+ " tbname VARCHAR(255)" +
") WITH (" +
" 'connector' = 'tdengine-connector'," +
" 'bootstrap.servers' = 'localhost:6041'," +
@@ -554,12 +554,12 @@ splitSql.setSelect("ts, current, voltage, phase, groupid, location")
" `current` FLOAT," +
" voltage INT," +
" phase FLOAT," +
- " location VARBINARY," +
+ " location VARCHAR(255)," +
" groupid INT," +
- " tbname VARBINARY" +
+ " tbname VARCHAR(255)" +
") WITH (" +
" 'connector' = 'tdengine-connector'," +
- " 'td.jdbc.mode' = 'cdc'," +
+ " 'td.jdbc.mode' = 'sink'," +
" 'td.jdbc.url' = 'jdbc:TAOS-WS://localhost:6041/power_sink?user=root&password=taosdata'," +
" 'sink.db.name' = 'power_sink'," +
" 'sink.supertable.name' = 'sink_meters'" +
diff --git a/docs/zh/01-index.md b/docs/zh/01-index.md
index 02c6105d43..9014a24598 100644
--- a/docs/zh/01-index.md
+++ b/docs/zh/01-index.md
@@ -8,7 +8,7 @@ TDengine 是一款[开源](https://www.taosdata.com/tdengine/open_source_time-se
TDengine 充分利用了时序数据的特点，提出了“一个数据采集点一张表”与“超级表”的概念，设计了创新的存储引擎，让数据的写入、查询和存储效率都得到极大的提升。为正确理解并使用 TDengine，无论你在工作中是什么角色，请您仔细阅读[数据模型](./basic/model)一章。
-如果你是开发工程师，请一定仔细阅读[开发指南](./develop)一章，该部分对数据库连接、建模、插入数据、查询、流式计算、缓存、数据订阅、用户自定义函数等功能都做了详细介绍，并配有各种编程语言的示例代码。大部分情况下，你只要复制粘贴示例代码，针对自己的应用稍作改动，就能跑起来。对 REST API、各种编程语言的连接器（Connector）想做更多详细了解的话，请看[连接器](./reference/connector)一章。
+如果你是开发工程师，请一定仔细阅读[开发指南](./develop)一章，该部分对数据库连接、建模、写入、查询、流式计算、缓存、数据订阅、用户自定义函数等功能都做了详细介绍，并配有各种编程语言的示例代码。大部分情况下，只要复制粘贴示例代码，针对自己的应用稍作改动，就能跑起来。对 REST API、各种编程语言的连接器（Connector）想做更多详细了解，请看[连接器](./reference/connector)一章。
我们已经生活在大数据时代，纵向扩展已经无法满足日益增长的业务需求，任何系统都必须具有水平扩展的能力，集群成为大数据以及 Database 系统的不可缺失功能。TDengine 团队不仅实现了集群功能，而且将这一重要核心功能开源。怎么部署、管理和维护 TDengine 集群，请仔细参考[运维管理](./operation)一章。
@@ -16,7 +16,7 @@ TDengine 采用 SQL 作为查询语言，大大降低学习成本、降低迁移
如果你是系统管理员，关心安装、升级、容错灾备、关心数据导入、导出、配置参数，如何监测 TDengine 是否健康运行，如何提升系统运行的性能，请仔细参考[运维指南](./operation)一章。
-如果你对数据库内核设计感兴趣，或是开源爱好者，建议仔细阅读[技术内幕](./tdinternal)一章。该章从分布式架构到存储引擎、查询引擎、数据订阅，再到流计算引擎都做了详细阐述。建议对照文档，查看TDengine在GitHub的源代码，对TDengine的设计和编码做深入了解，更欢迎加入开源社区，贡献代码。
+如果你对数据库内核设计感兴趣，或是开源爱好者，建议仔细阅读[技术内幕](./tdinternal)一章。该章从分布式架构到存储引擎、查询引擎、数据订阅，再到流计算引擎都做了详细阐述。建议对照文档，查看 TDengine 在 GitHub 的源代码，对 TDengine 的设计和编码做深入了解，更欢迎加入开源社区，贡献代码。
最后，作为一个开源软件，欢迎大家的参与。如果发现文档有任何错误、描述不清晰的地方，请在每个页面的最下方，点击“编辑本文档”直接进行修改。
diff --git a/docs/zh/02-concept.md b/docs/zh/02-concept.md
index 17acc50892..96e5a306cf 100644
--- a/docs/zh/02-concept.md
+++ b/docs/zh/02-concept.md
@@ -9,7 +9,7 @@ toc_max_heading_level: 4
时序数据，即时间序列数据（Time-Series Data），它们是一组按照时间发生先后顺序进行排列的序列数据。日常生活中，设备、传感器采集的数据就是时序数据，证券交易的记录也是时序数据。因此时序数据的处理并不陌生，特别在是工业自动化以及证券金融行业，专业的时序数据处理软件早已存在，比如工业领域的 PI System 以及金融行业的 KDB。
-这些时序数据是周期、准周期产生的，或事件触发产生的，有的采集频率高，有的采集频率低。一般被发送至服务器中进行汇总并进行实时分析和处理，对系统的运行做出实时监测或预警，对股市行情进行预测。这些数据也可以被长期保存下来，用以进行离线数据分析。比如统计时间区间内设备的运行节奏与产出，分析如何进一步优化配置来提升生产效率；统计一段时间内生产过程中的成本分布，分析如何降低生产成本；统计一段时间内的设备异常值，结合业务分析潜在的安全隐患，以降低故障时长等等。
+这些时序数据是周期、准周期产生的，或事件触发产生的，有的采集频率高，有的采集频率低。一般被发送至服务器进行汇总并进行实时分析和处理，对系统的运行做出实时监测或预警，对股市行情进行预测。这些数据也可以被长期保存下来，用以进行离线数据分析。比如统计时间区间内设备的运行节奏与产出，分析如何进一步优化配置来提升生产效率；统计一段时间内生产过程中的成本分布，分析如何降低生产成本；统计一段时间内的设备异常值，结合业务分析潜在的安全隐患，以降低故障时长等等。
过去的二十年，随着数据通讯成本的急剧下降，以及各种传感技术和智能设备的出现，特别是物联网与工业 4.0 的推动，工业、物联网企业为了监测设备、环境、生产线及整个系统的运行状态，在各个关键点都配有传感器，采集各种数据。从手环、共享出行、智能电表、环境监测设备到电梯、数控机床、挖掘机、工业生产线等都在源源不断的产生海量的实时数据，时序数据的体量正指数级的增长。以智能电表为例，智能电表每隔 15 分钟采集一次数据，每天会自动生成 96 条记录。现在全中国已经有超过 10 亿台智能电表，一天就产生 960 亿条时序数据。一台联网的汽车往往每隔 10 到 15 秒采集一次数据发到云端，那么一天下来就很容易产生 1000 条记录。假设中国有 2 亿车辆联网，它们每天将产生总计 2000 亿条甚至更多的时序数据。
@@ -33,7 +33,7 @@ toc_max_heading_level: 4
7. 用户关注的是一段时间的趋势：对于一条银行交易记录，或者一条微博、微信，对于它的用户而言，每一条都很重要。但对于物联网、工业时序数据，每个数据点与数据点的变化并不大，大家关心的更多是一段时间，比如过去五分钟、一小时数据变化的趋势，不会只针对一个时间点进行。
-8. 数据是有保留期限的：采集的数据一般都有基于时长的保留策略，比如仅仅保留一天、一周、一个月、一年甚至更长时间，该类数据的价值往往是由时间段决定的，因此对于不在重要时间段内的数据，都是可以被视为过期数据整块删除的。
+8. 数据是有保留期限的：采集的数据一般都有基于时长的保留策略，比如仅仅保留一天、一周、一个月、一年甚至更长时间，该类数据的价值往往是由时间段决定的，因此对于不在重要时间段内的数据，都是可以被视为过期数据整块删除的。
9. 需要实时分析计算操作：对于大部分互联网大数据应用，更多的是离线分析，即使有实时分析，但要求并不高。比如用户画像、可以积累一定的用户行为数据后进行，早一天晚一天画不会特别影响结果。但是对于工业、物联网的平台应用以及交易系统，对数据的实时计算要求就往往很高，因为需要根据计算结果进行实时报警、监控，从而避免事故的发生、决策时机的错过。
@@ -47,7 +47,7 @@ toc_max_heading_level: 4
1. 电力能源领域：电力能源领域范围较大，不论是在发电、输电、配电、用电还是其他环节中，各种电力设备都会产生大量时序数据，以风力发电为例，风电机作为大型设备，拥有可能高达数百的数据采集点，因此每日所产生的时序数据量极其之大，对这些数据的监控分析是确保发电环节准确无误的必要工作。在用电环节，对智能电表实时采集回来的电流、电压等数据进行快速计算，实时了解最新的用电总量、尖、峰、平、谷用电量，判断设备是否正常工作。有些时候，电力系统可能需要拉取历史上某一年的全量数据，通过机器学习等技术分析用户的用电习惯、进行负荷预测、节能方案设计、帮助电力公司合理规划电力的供应。或者拉取上个月的尖峰平谷用电量，根据不同价位进行周期性的电费结算，以上都是时序数据在电力能源领域的典型应用。
-2. 车联网/轨道交通领域：车辆的 GPS 、速度、油耗、故障信息等，都是典型的时序数据，通过对它们科学合理地数据分析，可以为车辆管理和优化提供强有力的支持。但是，不同车型采集的点位信息从数百点到数千点之间不一而同，随着联网的交通设备数量越来越多，这些海量的时序数据如何安全上传、数据存储、查询和分析，成为了一个亟待解决的行业问题。对于交通工具的本身，科学合理地处理时序数据可以实现车辆轨迹追踪、无人驾驶、故障预警等功能。对于交通工具的整体配套服务，也可以提供良好的支持。比如，在新一代的智能地铁管理系统中，通过地铁站中各种传感器的时序数据采集分析，可以在站中实时展示各个车厢的拥挤度、温度、舒适度等数据，让用户可以自行选择体验度最佳的出行方案，对于地铁运营商，也可以更好地实现乘客流量的调度管理。
+2. 车联网/轨道交通领域：车辆的 GPS 、速度、油耗、故障信息等，都是典型的时序数据，通过科学合理地数据分析，可以为车辆管理和优化提供强有力的支持。但是，不同车型采集的点位信息从数百点到数千点之间不一而同，随着联网的交通设备数量越来越多，这些海量的时序数据如何安全上传、数据存储、查询和分析，成为了一个亟待解决的行业问题。对于交通工具的本身，科学合理地处理时序数据可以实现车辆轨迹追踪、无人驾驶、故障预警等功能。对于交通工具的整体配套服务，也可以提供良好的支持。比如，在新一代的智能地铁管理系统中，通过地铁站中各种传感器的时序数据采集分析，可以在站中实时展示各个车厢的拥挤度、温度、舒适度等数据，让用户可以自行选择体验度最佳的出行方案，对于地铁运营商，也可以更好地实现乘客流量的调度管理。
3. 智能制造领域：过去的十几年间，许多传统工业企业的数字化得到了长足的发展，单个工厂从传统的几千个数据采集点，到如今数十万点、上百万点，部分远程运维场景面临上万设备、千万点的数据采集存储的需求，这些数据都属于典型的时序数据。就整个工业大数据系统而言，时序数据的处理是相当复杂的。以烟草行业的数据采集为例，设备的工业数据协议各式各样、数据采集单位随着设备类型的不同而不同。数据的实时处理能力随着数据采集点的持续增加而难以跟上，与此同时还要兼顾数据的高性能、高可用、可拓展性等等诸多特性。但从另一个角度来看，如果大数据平台能够解决以上困难，满足企业对于时序数据存储分析的需求，就可以帮助企业实现更加智能化、自动化的生产模式，从而完成质的飞升。
@@ -55,7 +55,7 @@ toc_max_heading_level: 4
5. IT 运维领域：IT 领域中，基础设施（如服务器、网络设备、存储设备）、应用程序运行的过程中会产生大量的时序数据。通过对这些时序数据的监控，可以很快地发现基础设施/应用的运行状态和服务可用性，包括系统是否在线、服务是否正常响应等；也能看到具体到某一个具体的点位的性能指标：如 CPU 利用率、内存利用率、磁盘空间利用率、网络带宽利用率等； 还可以监控系统产生的错误日志和异常事件，包括入侵检测、安全事件日志、权限控制等，最终通过设置报警规则，及时通知管理员或运维人员具体的情况，从而及时发现问题、预防故障，并优化系统性能，确保系统稳定可靠地运行。
-6. 金融领域：金融领域目前正经历着数据管理的一场革命，它们的行情数据属于典型的时序数据，由于保留行情数据的储存期限往往需长达 5 至 10 年，甚至超过 30 年，而且可能全世界各个国家/地区的主流金融市场的交易数据都需要全量保存，因此行情数据的总量数据体量庞大，会轻松达到 TB 级别，造成存储、查询等等各方面的瓶颈。在金融领域中，量化交易平台是最能凸显时序数据处理重要性的革命性应用之一：通过对大量时序行情数据的读取分析来及时响应市场变化，帮助交易者把握投资机会，同时规避不必要的风险，实现资产的稳健增长。可以实现包括但不限于：资产管理、情绪监控、股票回测、交易信号模拟、报表自动生成等等诸多功能。
+6. 金融领域：金融领域目前正经历着数据管理的一场革命，行情数据属于典型的时序数据，由于保留行情数据的储存期限往往需长达 5 至 10 年，甚至超过 30 年，而且可能全世界各个国家/地区的主流金融市场的交易数据都需要全量保存，因此行情数据的总量数据体量庞大，会轻松达到 TB 级别，造成存储、查询等等各方面的瓶颈。在金融领域中，量化交易平台是最能凸显时序数据处理重要性的革命性应用之一：通过对大量时序行情数据的读取分析来及时响应市场变化，帮助交易者把握投资机会，同时规避不必要的风险，实现资产的稳健增长。可以实现包括但不限于：资产管理、情绪监控、股票回测、交易信号模拟、报表自动生成等等诸多功能。
## 处理时序数据所需要的工具
@@ -71,11 +71,11 @@ toc_max_heading_level: 4
5. 缓存（Cache）：物联网、工业、金融应用需要实时展示一些设备或股票的最新状态，因此平台需要缓存技术提供快速的数据访问。原因是：由于时序数据体量极大，如果不使用缓存技术，而是进行常规的读取、筛选，那么对于监控设备最新状态之类的计算是十分困难的，将会导致很大的延迟，从而失去“实时”的意义。因此，缓存技术是时序数据处理平台不可缺少的一环， Redis 就是这样一种常用的缓存工具。
-处理时序数据需要一系列模块的协同作业，从数据采集到存储、计算、分析与可视化，再到专用的时序数据算法库，每个环节都有相应的工具支持。这些工具的选择取决于具体的业务需求和数据特点，合理地选用和搭配它们才能做到高效地处理各种类型的时序数据，挖掘数据背后的价值。
+处理时序数据需要一系列模块的协同作业，从数据采集到存储、计算、分析与可视化，再到专用的时序数据算法库，每个环节都有相应的工具支持。这些工具的选择取决于具体的业务需求和数据特点，合理地选用和搭配才能做到高效地处理各种类型的时序数据，挖掘数据背后的价值。
## 专用时序数据处理工具的必要性
-在时序数据的十大特征一节中提到，对于一个优秀的时序大数据处理平台来说，它必然需要具备处理时序数据十大特征的能力。在处理时序数据所需要的工具一节中介绍了时序大数据平台处理时序数据所需要的主要模块/组件。 结合这两节的内容与实际情况，可以发现：处理海量时序数据，其实是一个很庞大复杂的系统。
+在时序数据的十大特征一节中提到，对于一个优秀的时序大数据处理平台来说，必然需要具备处理时序数据十大特征的能力。在处理时序数据所需要的工具一节中介绍了时序大数据平台处理时序数据所需要的主要模块/组件。结合这两节的内容与实际情况，可以发现：处理海量时序数据，其实是一个很庞大复杂的系统。
早些年，为处理日益增长的互联网数据，众多的工具开始出现，最流行的便是 Hadoop 体系。除使用大家所熟悉的 Hadoop 组件如 HDFS、MapReduce、HBase 和 Hive 外，通用的大数据处理平台往往还使用 Kafka 或其他消息队列工具，Redis 或其他缓存软件，Flink 或其他实时流式数据处理软件。存储上也有人选用 MongoDB、Cassandra 或其他 NoSQL 数据库。这样一个典型的大数据处理平台基本上能很好的处理互联网行业的引用，比如典型的用户画像、舆情分析等。
diff --git a/docs/zh/03-intro.md b/docs/zh/03-intro.md
index 4207ab4eb6..d146fd62b6 100644
--- a/docs/zh/03-intro.md
+++ b/docs/zh/03-intro.md
@@ -14,7 +14,7 @@ TDengine 是一个高性能、分布式的时序数据库。通过集成的缓
TDengine OSS 是一个开源的高性能时序数据库，与其他时序数据库相比，它的核心优势在于其集群开源、高性能和云原生架构。而且除了基础的写入、查询和存储功能外，TDengine OSS 还集成了缓存、流式计算和数据订阅等高级功能，这些功能显著简化了系统设计，降低了企业的研发和运营成本。
-在 TDengine OSS 的基础上，企业版 TDengine Enterprise 提供了增强的辅助功能，包括数据的备份恢复、异地容灾、多级存储、视图、权限控制、安全加密、IP 白名单、支持 MQTT、OPC-UA、OPC-DA、PI、Wonderware、Kafka 等各种数据源。这些功能为企业提供了更为全面、安全、可靠和高效的时序数据管理解决方案。更多的细节请看 [TDengine Enterprise](https://www.taosdata.com/tdengine-pro)
+在 TDengine OSS 的基础上，TDengine Enterprise 提供了增强的辅助功能，包括数据的备份恢复、异地容灾、多级存储、视图、权限控制、安全加密、IP 白名单、支持 MQTT、OPC-UA、OPC-DA、PI、Wonderware、Kafka 等各种数据源。这些功能为企业提供了更为全面、安全、可靠和高效的时序数据管理解决方案。更多的细节请看 [TDengine Enterprise](https://www.taosdata.com/tdengine-pro)。
此外，TDengine Cloud 作为一种全托管的云服务，存储与计算分离，分开计费，为企业提供了企业级的工具和服务，彻底解决了运维难题，尤其适合中小规模的用户使用。更多的细节请看[TDengine 云服务](https://cloud.taosdata.com/?utm_source=menu&utm_medium=webcn)
@@ -30,19 +30,19 @@ TDengine 经过特别优化，以适应时间序列数据的独特需求，引
4. 流式计算：TDengine 流式计算引擎提供了实时处理写入的数据流的能力，不仅支持连续查询，还支持事件驱动的流式计算。它提供了替代复杂流处理系统的轻量级解决方案，并能够在高吞吐的数据写入的情况下，提供毫秒级的计算结果延迟。
-5. 数据订阅：TDengine 提供了类似 Kafka 的数据订阅功能。但用户可以通过 SQL 来灵活控制订阅的数据内容，并使用 Kafka 相同的 API 来订阅一张表、一组表、全部列或部分列、甚至整个数据库的数据。TDengine 可以替代需要集成消息队列产品的场景， 从而简化系统设计的复杂度，降低运营维护成本。
+5. 数据订阅：TDengine 提供了类似 Kafka 的数据订阅功能。但用户可以通过 SQL 来灵活控制订阅的数据内容，并使用和 Kafka 相同的 API 来订阅一张表、一组表、全部列或部分列、甚至整个数据库的数据。TDengine 可以替代需要集成消息队列产品的场景， 从而简化系统设计的复杂度，降低运营维护成本。
-6. 可视化/BI：TDengine 本身不提供可视化或 BI 的功能。但通过其 RESTful API， 标准的 JDBC、ODBC 接口，TDengine 能够 Grafana、Google Data Studio、Power BI、Tableau 以及国产 BI 工具无缝集成。
+6. 可视化/BI：TDengine 本身不提供可视化或 BI 的功能。但通过其 RESTful API， 标准的 JDBC、ODBC 接口，TDengine 能够和 Grafana、Google Data Studio、Power BI、Tableau 以及国产 BI 工具无缝集成。
-7. 集群功能：TDengine 支持集群部署，能够随着业务数据量的增长，通过增加节点线性提升系统处理能力，实现水平扩展。同时，通过多副本技术提供高可用性，并支持 Kubernetes 部署。同时还提供了多种运维工具，方便系统管理员更好地管理和维护集群的健壮运行。
+7. 集群功能：TDengine 支持集群部署，能够随着业务数据量的增长，通过增加节点线性提升系统处理能力，实现水平扩展。同时，通过多副本技术提供高可用性，支持 Kubernetes 部署，提供了多种运维工具，方便系统管理员更好地管理和维护集群的健壮运行。
8. 数据迁移：TDengine 提供了多种便捷的数据导入导出功能，包括脚本文件导入导出、数据文件导入导出、taosdump 工具导入导出等。
-9. 编程连接器：TDengine 提供不同语言的连接器，包括 C/C++、Java、Go、Node.js、Rust、Python、C#、R、PHP 等。这些连接器大多都支持原生连接和 WebSocket 两种连接方式。TDengine 也提供 RESTful 接口，任何语言的应用程序可以直接通过 HTTP 请求访问数据库。
+9. 编程连接器：TDengine 提供多种语言的连接器，包括 C/C++、Java、Go、Node.js、Rust、Python、C#、R、PHP 等。这些连接器大多都支持原生连接和 WebSocket 两种连接方式。TDengine 也提供 RESTful 接口，任何语言的应用程序可以直接通过 HTTP 请求访问数据库。
10. 数据安全：TDengine 提供了丰富的用户管理和权限管理功能以控制不同用户对数据库和表的访问权限，提供了 IP 白名单功能以控制不同帐号只能从特定的服务器接入集群。TDengine 支持系统管理员对不同数据库按需加密，数据加密后对读写完全透明且对性能的影响很小。还提供了审计日志功能以记录系统中的敏感操作。
-11. 常用工具：TDengine 还提供了交互式命令行程序（CLI），便于管理集群、检查系统状态、做即时查询。压力测试工具 taosBenchmark，用于测试 TDengine 的性能。TDengine 还提供了图形化管理界面，简化了操作和管理过程。
+11. 常用工具：TDengine 提供了交互式命令行程序（CLI），便于管理集群、检查系统状态、做即时查询。压力测试工具 taosBenchmark，用于测试 TDengine 的性能。TDengine 还提供了图形化管理界面，简化了操作和管理过程。
12. 零代码数据接入：TDengine 企业版提供了丰富的数据接入功能，依托强大的数据接入平台，无需一行代码，只需要做简单的配置即可实现多种数据源的数据接入，目前已经支持的数据源包括：OPC-UA、OPC-DA、PI、MQTT、Kafka、InfluxDB、OpenTSDB、MySQL、SQL Server、Oracle、Wonderware Historian、MongoDB。
@@ -63,8 +63,11 @@ TDengine 经过特别优化，以适应时间序列数据的独特需求，引
6. 核心开源：TDengine 的核心代码，包括集群功能，均在开源协议下公开发布。它在 GitHub 网站全球趋势排行榜上多次位居榜首，显示出其受欢迎程度。同时，TDengine 拥有一个活跃的开发者社区，为技术的持续发展和创新提供了有力支持。
采用 TDengine，企业可以在物联网、车联网、工业互联网等典型场景中显著降低大数据平台的总拥有成本，主要体现在以下几个方面：
+
1. 高性能带来的成本节约：TDengine 卓越的写入、查询和存储性能意味着系统所需的计算资源和存储资源可以大幅度减少。这不仅降低了硬件成本，还减少了能源消耗和维护费用。
+
2. 标准化与兼容性带来的成本效益：由于 TDengine 支持标准 SQL，并与众多第三方软件实现了无缝集成，用户可以轻松地将现有系统迁移到 TDengine 上，无须重写大量代码。这种标准化和兼容性大大降低了学习和迁移成本，缩短了项目周期。
+
3. 简化系统架构带来的成本降低：作为一个极简的时序数据平台，TDengine 集成了消息队列、缓存、流计算等必要功能，避免了额外集成众多其他组件的需要。这种简化的系统架构显著降低了系统的复杂度，从而减少了研发和运营成本，提高了整体运营效率。
## 技术生态
@@ -78,7 +81,7 @@ TDengine 经过特别优化，以适应时间序列数据的独特需求，引
图 1. TDengine 技术生态图
-上图中，左侧是各种数据采集或消息队列，包括 OPC-UA、MQTT、Telegraf、也包括 Kafka，他们的数据将被源源不断的写入到 TDengine。右侧则是可视化、BI 工具、组态软件、应用程序。下侧则是 TDengine 自身提供的命令行程序（CLI）以及可视化管理工具。
+上图中，左侧是各种数据采集或消息队列，包括 OPC-UA、MQTT、Telegraf、也包括 Kafka，它们的数据将被源源不断的写入到 TDengine。右侧是可视化、BI 工具、组态软件、应用程序。下侧是 TDengine 自身提供的命令行程序（CLI）以及可视化管理工具。
## 典型适用场景
diff --git a/docs/zh/04-get-started/01-docker.md b/docs/zh/04-get-started/01-docker.md
index c75c8bafd0..a148c57172 100644
--- a/docs/zh/04-get-started/01-docker.md
+++ b/docs/zh/04-get-started/01-docker.md
@@ -1,5 +1,5 @@
---
-sidebar_label: 用Docker快速体验
+sidebar_label: 用 Docker 快速体验
title: 用 Docker 快速体验 TDengine
description: 使用 Docker 快速体验 TDengine 的高效写入和查询
---
@@ -91,7 +91,7 @@ taosBenchmark 提供了丰富的选项，允许用户自定义测试参数，如
taosBenchmark --help
```
-有关taosBenchmark 的详细使用方法，请参考[taosBenchmark 参考手册](../../reference/tools/taosbenchmark)
+有关 taosBenchmark 的详细使用方法，请参考 [taosBenchmark 参考手册](../../reference/tools/taosbenchmark)
### 体验查询
diff --git a/docs/zh/04-get-started/03-package.md b/docs/zh/04-get-started/03-package.md
index 344b9412df..6d68c2eade 100644
--- a/docs/zh/04-get-started/03-package.md
+++ b/docs/zh/04-get-started/03-package.md
@@ -8,7 +8,7 @@ import Tabs from "@theme/Tabs";
import TabItem from "@theme/TabItem";
import PkgListV3 from "/components/PkgListV3";
-TDengine 完整的软件包包括服务端（taosd）、应用驱动（taosc）、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、命令行程序（CLI，taos）和一些工具软件。目前 TDinsight 仅在 Linux 系统上安装和运行，后续将支持 Windows、macOS 等系统。TDengine 除了提供多种语言的连接器之外，还通过 [taosAdapter](../../reference/components/taosadapter/) 提供 [RESTful 接口](../../reference/connector/rest-api/)。
+TDengine 完整的软件包包括服务端（taosd）、应用驱动（taosc）、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、命令行程序（TDengine CLI）和一些工具软件。目前 TDinsight 仅在 Linux 系统上安装和运行，后续将支持 Windows、macOS 等系统。TDengine 除了提供多种语言的连接器之外，还通过 [taosAdapter](../../reference/components/taosadapter/) 提供 [RESTful 接口](../../reference/connector/rest-api/)。
为方便使用，标准的服务端安装包包含了 taosd、taosAdapter、taosc、taos、taosdump、taosBenchmark、TDinsight 安装脚本和示例代码；如果您只需要用到服务端程序和客户端连接的 C/C++ 语言支持，也可以仅下载 Lite 版本的安装包。
@@ -17,30 +17,27 @@ TDengine 完整的软件包包括服务端（taosd）、应用驱动（taosc）
此外，TDengine 也提供 macOS x64/m1 平台的 pkg 安装包。
## 运行环境要求
+
在linux系统中，运行环境最低要求如下:
+1. linux 内核版本：3.10.0-1160.83.1.el7.x86_64 或以上
+2. glibc 版本：2.17 或以上
-linux 内核版本 - 3.10.0-1160.83.1.el7.x86_64;
-
-glibc 版本 - 2.17;
-
-如果通过clone源码进行编译安装，还需要满足:
-
-cmake版本 - 3.26.4或以上;
-
-gcc 版本 - 9.3.1或以上;
-
+如果通过 Clone 源码进行编译安装，还需要满足:
+1. cmake 版本：3.26.4 或以上
+2. gcc 版本：9.3.1 或以上
## 安装
**注意**
-从TDengine 3.0.6.0 开始，不再提供单独的 taosTools 安装包，原 taosTools 安装包中包含的工具都在 TDengine-server 安装包中，如果需要请直接下载 TDengine -server 安装包。
+从 TDengine 3.0.6.0 开始，不再提供单独的 taosTools 安装包，原 taosTools 安装包中包含的工具都在 TDengine-server 安装包中，如果需要请直接下载 TDengine-server 安装包。
-1. 从列表中下载获得 Deb 安装包；
+1. 从列表中下载获得 Deb 安装包：
+
2. 进入到安装包所在目录，执行如下的安装命令：
> 请将 `` 替换为下载的安装包版本
@@ -53,8 +50,9 @@ sudo dpkg -i TDengine-server--Linux-x64.deb
-1. 从列表中下载获得 RPM 安装包；
+1. 从列表中下载获得 RPM 安装包：
+
2. 进入到安装包所在目录，执行如下的安装命令：
> 请将 `` 替换为下载的安装包版本
@@ -67,7 +65,7 @@ sudo rpm -ivh TDengine-server--Linux-x64.rpm
-1. 从列表中下载获得 tar.gz 安装包；
+1. 从列表中下载获得 tar.gz 安装包：
2. 进入到安装包所在目录，使用 `tar` 解压安装包；
3. 进入到安装包所在目录，先解压文件后，进入子目录，执行其中的 install.sh 安装脚本。
@@ -126,14 +124,14 @@ apt-get 方式只适用于 Debian 或 Ubuntu 系统。
**注意**
- 目前 TDengine 在 Windows 平台上只支持 Windows Server 2016/2019 和 Windows 10/11。
- 从 TDengine 3.1.0.0 开始，只提供 Windows 客户端安装包。如果需要 Windows 服务端安装包，请联系 TDengine 销售团队升级为企业版。
-- Windows 上需要安装 VC 运行时库，可在此下载安装 [VC运行时库](https://learn.microsoft.com/zh-cn/cpp/windows/latest-supported-vc-redist?view=msvc-170), 如果已经安装此运行库可忽略。
+- Windows 上需要安装 VC 运行时库，可在此下载安装 [VC 运行时库](https://learn.microsoft.com/zh-cn/cpp/windows/latest-supported-vc-redist?view=msvc-170)，如果已经安装此运行库可忽略。
按照以下步骤安装：
-1. 从列表中下载获得 exe 安装程序；
+1. 从列表中下载获得 exe 安装程序：
2. 运行可执行程序来安装 TDengine。
-Note: 从 3.0.1.7 开始，只提供 TDengine 客户端的 Windows 客户端的下载。想要使用TDengine 服务端的 Windows 版本，请联系销售升级为企业版本。
+Note: 从 3.0.1.7 版本开始，只提供 TDengine 客户端的 Windows 客户端的下载。想要使用 TDengine 服务端的 Windows 版本，请联系 TDengine 销售团队升级为企业版。
@@ -210,12 +208,12 @@ sudo launchctl start com.tdengine.taoskeeper
sudo launchctl start com.tdengine.taos-explorer
```
-你也可以直接运行 start-all.sh 脚本来启动上面的所有服务
+你也可以直接运行 `start-all.sh` 脚本来启动上面的所有服务
```bash
start-all.sh
```
-可以使用 `launchctl` 命令管理上面提到的每个 TDengine 服务，以下示例使用 `taosd` ：
+可以使用 `launchctl` 命令管理上面提到的每个 TDengine 服务，以下示例使用 `taosd` ：
```bash
sudo launchctl start com.tdengine.taosd
diff --git a/docs/zh/04-get-started/05-cloud.md b/docs/zh/04-get-started/05-cloud.md
index 1bca09ee91..929ec5ad88 100644
--- a/docs/zh/04-get-started/05-cloud.md
+++ b/docs/zh/04-get-started/05-cloud.md
@@ -4,7 +4,7 @@ title: 通过云服务 快速体验 TDengine
toc_max_heading_level: 4
---
-TDengine Cloud 作为一个全托管的时序大数据云服务平台，致力于让用户迅速领略TDengine 的强大功能。该平台不仅继承了 TDengine Enterprise 的核心功能特性，还充分发挥了 TDengine 的云原生优势。TDengine Cloud 以其极致的资源弹性伸缩、高可用性、容器化部署以及按需付费等特点，灵活满足各类用户需求，为用户打造高效、可靠且经济的时序大数据处理解决方案。
+TDengine Cloud 作为一个全托管的时序大数据云服务平台，致力于让用户迅速领略 TDengine 的强大功能。该平台不仅继承了 TDengine Enterprise 的核心功能特性，还充分发挥了 TDengine 的云原生优势。TDengine Cloud 以其极致的资源弹性伸缩、高可用性、容器化部署以及按需付费等特点，灵活满足各类用户需求，为用户打造高效、可靠且经济的时序大数据处理解决方案。
TDengine Cloud 大幅减轻了用户在部署、运维等方面的人力负担，同时提供了全方位的企业级服务。这些服务涵盖多角色、多层次的用户管理、数据共享功能，以适应各种异构网络环境。此外，TDengine Cloud 还提供私有链接服务和极简的数据备份与恢复功能，确保数据安全无忧。
@@ -25,11 +25,10 @@ TDengine Cloud 大幅减轻了用户在部署、运维等方面的人力负担
要在 TDengine Cloud 中创建 TDengine 实例，只须遵循以下 3 个简单步骤。
-1. 第 1 步，选择公共数据库。在此步骤中，TDengine Cloud 提供了可供公共访问的智能电表等数据库。通过浏览和查询这些数据库，你可以立即体验 TDengine 的各种功能和高性能。你可以根据需求在此步骤启动数据库访问，或在后续使用过程中再进行启动。若不需要此步骤，可直接点击“下一步”按钮跳过。
+1. 选择公共数据库。在此步骤中，TDengine Cloud 提供了可供公共访问的智能电表等数据库。通过浏览和查询这些数据库，你可以立即体验 TDengine 的各种功能和高性能。你可以根据需求在此步骤启动数据库访问，或在后续使用过程中再进行启动。若不需要此步骤，可直接点击“下一步”按钮跳过。
-2. 第 2 步，创建组织。在此步骤中，请输入一个具有意义的名称，代表你的公司或组织，这将有助于你和平台更好地管理云上资源。
-
-3. 第 3 步，创建实例。在此步骤中，你需要填写实例的区域、名称、是否选择高可用选项以及计费方案等必填信息。确认无误后，点击“创建”按钮。大约等待 1min，新的TDengine 实例便会创建完成。随后，你可以在控制台中对该实例进行各种操作，如查询数据、创建订阅、创建流等。
+2. 创建组织。在此步骤中，请输入一个具有意义的名称，代表你的公司或组织，这将有助于你和平台更好地管理云上资源。
+3. 创建实例。在此步骤中，你需要填写实例的区域、名称、是否选择高可用选项以及计费方案等必填信息。确认无误后，点击“创建”按钮。大约等待 1min，新的 TDengine 实例便会创建完成。随后，你可以在控制台中对该实例进行各种操作，如查询数据、创建订阅、创建流等。
TDengine Cloud 提供多种级别的计费方案，包括入门版、基础版、标准版、专业版和旗舰版，以满足不同客户的需求。如果你觉得现有计费方案无法满足自己的特定需求，请联系 TDengine Cloud 的客户支持团队，他们将为你量身定制计费方案。注册后，你将获得一定的免费额度，以便体验服务
diff --git a/docs/zh/04-get-started/index.md b/docs/zh/04-get-started/index.md
index 5a7192f2c6..2dea09a4d8 100644
--- a/docs/zh/04-get-started/index.md
+++ b/docs/zh/04-get-started/index.md
@@ -8,7 +8,7 @@ import xiaot_new from './xiaot-20231007.png'
import channel from './channel.webp'
import official_account from './official-account.webp'
-TDengine 完整的软件包包括服务端（taosd）、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、应用驱动（taosc）、命令行程序 (CLI，taos) 和一些工具软件。TDengine 除了提供多种语言的连接器之外，还通过 [taosAdapter](../reference/components/taosadapter) 提供 [RESTful 接口](../reference/connector/rest-api)。
+TDengine 完整的软件包包括服务端（taosd）、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、应用驱动（taosc）、命令行程序 (TDengine CLI) 和一些工具软件。TDengine 除了提供多种语言的连接器之外，还通过 [taosAdapter](../reference/components/taosadapter) 提供 [RESTful 接口](../reference/connector/rest-api)。
本章主要介绍如何快速设置 TDengine 环境并体验其高效写入和查询。
diff --git a/docs/zh/05-basic/01-model.md b/docs/zh/05-basic/01-model.md
index f49db17892..fc5c3a0a2e 100644
--- a/docs/zh/05-basic/01-model.md
+++ b/docs/zh/05-basic/01-model.md
@@ -25,11 +25,11 @@ toc_max_heading_level: 4
### 采集量
采集量是指通过各种传感器、设备或其他类型的采集点所获取的物理量，如电流、电压、温度、压力、GPS 等。由于这些物理量随时间不断变化，因此采集的数据类型多
-样，包括整型、浮点型、布尔型以及字符串等。随着时间的积累，存储的数据将持续增长。以智能电表为例，其中的 current（电流）、voltage（电压）和 phase（相位）便是典型的采集量。
+样，包括整型、浮点型、布尔型以及字符串等。随着时间的积累，存储的数据将持续增长。以智能电表为例，其中的 current、voltage 和 phase 便是典型的采集量。
### 标签
-标签是指附着在传感器、设备或其他类型采集点上的静态属性，这些属性不会随时间发生变化，例如设备型号、颜色、设备所在地等。标签的数据类型可以是任意类型。尽管标签本身是静态的，但在实际应用中，用户可能需要对标签进行修改、删除或添加。与采集量不同，随着时间的推移，存储的标签数据量保持相对稳定，不会呈现明显的增长趋势。在智能电表的示例中，location（位置）和 Group ID（分组 ID）就是典型的标签。
+标签是指附着在传感器、设备或其他类型采集点上的静态属性，这些属性不会随时间发生变化，例如设备型号、颜色、设备所在地等。标签的数据类型可以是任意类型。尽管标签本身是静态的，但在实际应用中，用户可能需要对标签进行修改、删除或添加。与采集量不同，随着时间的推移，存储的标签数据量保持相对稳定，不会呈现明显的增长趋势。在智能电表的示例中，location 和 Group ID 就是典型的标签。
### 数据采集点
@@ -49,9 +49,9 @@ toc_max_heading_level: 4
4. 一个数据块内部，采用列式存储，对于不同的数据类型，可以采用不同压缩算法来提高压缩率。并且，由于采集量的变化通常是缓慢的，压缩率会更高。
-如果采用传统的方式，将多个数据采集点的数据写入一张表，由于网络延时不可控，不同数据采集点的数据到达服务器的时序是无法保证的，写入操作是要有锁保护的，而且一个数据采集点的数据是难以保证连续存储在一起的。采用一个数据采集点一张表的方式，能最大程度的保证单个数据采集点的插入和查询的性能是最优的，，而且数据压缩率最高。
+如果采用传统的方式，将多个数据采集点的数据写入一张表，由于网络延时不可控，不同数据采集点的数据到达服务器的时序是无法保证的，写入操作是要有锁保护的，而且一个数据采集点的数据是难以保证连续存储在一起的。采用一个数据采集点一张表的方式，能最大程度的保证单个数据采集点的插入和查询的性能是最优的，而且数据压缩率最高。
-在 TDengine 中，通常使用数据采集点的名称（如：d1001）来做表名，每个数据采集点可以有多个采集量（如：current、voltage、phase 等），每个采集量对应一张表的一列。采集量的数据类型可以是整型、浮点型、字符串等。
+在 TDengine 中，通常使用数据采集点的名称（如 d1001）来做表名，每个数据采集点可以有多个采集量（如 current、voltage、phase 等），每个采集量对应一张表的一列。采集量的数据类型可以是整型、浮点型、字符串等。
此外，表的第一列必须是时间戳，即数据类型为 Timestamp。对于每个采集量，TDengine 将使用第一列时间戳建立索引，采用列式存储。对于复杂的设备，比如汽车，它有多个数据采集点，则需要为一辆汽车建立多张表。
@@ -86,12 +86,12 @@ toc_max_heading_level: 4
### 时间戳
时间戳在时序数据处理中扮演着至关重要的角色，特别是在应用程序需要从多个不同时区访问数据库时，这一问题变得更加复杂。在深入了解 TDengine 如何处理时间戳与时区之前，我们先介绍以下几个基本概念。
-- 本地日期时间：指特定地区的当地时间，通常表示为 yyyy-MM-dd hh:mm:ss.SSS 格 式 的 字 符 串。 这 种 时 间 表 示 不 包 含 任 何 时 区 信 息， 如“2021-07-21 12:00:00.000”。
-- 时区：地球上不同地理位置的标准时间。协调世界时（Universal Time Coordinated，UTC）或格林尼治时间是国际时间标准，其他时区通常表示为相对于 UTC 的偏移量，如“UTC+8”代表东八区时间。 UTC 时间戳：表示自 UNIX 纪 元（即 UTC 时 间 1970 年 1 月 1 日 0 点） 起 经 过的毫秒数。例如，“1700000000000”对应的日期时间是“2023-11-14 22:13:20（UTC+0）”。 在 TDengine 中保存时序数据时，实际上保存的是 UTC 时间戳。TDengine 在写入数据时，时间戳的处理分为如下两种情况。
-- RFC-3339 格式：当使用这种格式时，TDengine 能够正确解析带有时区信息的时间字符串为 UTC 时间戳。例如，“2018-10-03T14:38:05.000+08:00”会被转换为UTC 时间戳。
+- 本地日期时间：指特定地区的当地时间，通常表示为 yyyy-MM-dd hh:mm:ss.SSS 格式的字符串。这种时间表示不包含任何时区信息，如 “2021-07-21 12:00:00.000”。
+- 时区：地球上不同地理位置的标准时间。协调世界时（Universal Time Coordinated，UTC）或格林尼治时间是国际时间标准，其他时区通常表示为相对于 UTC 的偏移量，如 “UTC+8” 代表东八区时间。 UTC 时间戳：表示自 UNIX 纪元（即 UTC 时间 1970 年 1 月 1 日 0 点）起经过的毫秒数。例如，“1700000000000” 对应的日期时间是 “2023-11-14 22:13:20（UTC+0）”。 在 TDengine 中保存时序数据时，实际上保存的是 UTC 时间戳。TDengine 在写入数据时，时间戳的处理分为如下两种情况。
+- RFC-3339 格式：当使用这种格式时，TDengine 能够正确解析带有时区信息的时间字符串为 UTC 时间戳。例如，“2018-10-03T14:38:05.000+08:00” 会被转换为 UTC 时间戳。
- 非 RFC-3339 格式：如果时间字符串不包含时区信息，TDengine 将使用应用程序所在的时区设置自动将时间转换为 UTC 时间戳。
-在查询数据时，TDengine 客户端会根据应用程序当前的时区设置，自动将保存的UTC 时间戳转换成本地时间进行显示，确保用户在不同时区下都能看到正确的时间信息。
+在查询数据时，TDengine 客户端会根据应用程序当前的时区设置，自动将保存的 UTC 时间戳转换成本地时间进行显示，确保用户在不同时区下都能看到正确的时间信息。
## 数据建模
@@ -110,7 +110,7 @@ CREATE DATABASE power PRECISION 'ms' KEEP 3650 DURATION 10 BUFFER 16;
- `DURATION 10` ：每 10 天的数据放在一个数据文件中
- `BUFFER 16` ：写入使用大小为 16MB 的内存池。
-在创建power数据库后，可以执行 USE 语句来使用切换数据库。
+在创建 power 数据库后，可以执行 USE 语句来使用切换数据库。
```sql
use power;
@@ -134,10 +134,10 @@ CREATE STABLE meters (
在 TDengine 中，创建超级表的 SQL 语句与关系型数据库类似。例如，上面的 SQL 中，`CREATE STABLE` 为关键字，表示创建超级表；接着，`meters` 是超级表的名称；在表名后面的括号中，定义超级表的列（列名、数据类型等），规则如下：
-1. 第 1 列必须为时间戳列。例如：`ts timestamp` 表示，时间戳列名是 `t`s，数据类型为 `timestamp`；
-2. 从第 2 列开始是采集量列。采集量的数据类型可以为整型、浮点型、字符串等。例如：`current float` 表示，采集量电流 `current`，数据类型为 `float`；
+1. 第 1 列必须为时间戳列。例如：`ts timestamp` 表示，时间戳列名是 `ts`，数据类型为 `timestamp`；
+2. 第 2 列开始是采集量列。采集量的数据类型可以为整型、浮点型、字符串等。例如：`current float` 表示，采集量电流 `current`，数据类型为 `float`。
-最后，TAGS是关键字，表示标签，在 TAGS 后面的括号中，定义超级表的标签（标签名、数据类型等）。
+最后，TAGS 是关键字，表示标签，在 TAGS 后面的括号中，定义超级表的标签（标签名、数据类型等）。
1. 标签的数据类型可以为整型、浮点型、字符串等。例如：`location varchar(64)` 表示，标签地区 `location`，数据类型为 `varchar(64)`；
2. 标签的名称不能与采集量列的名称相同。
@@ -155,7 +155,7 @@ USING meters (
);
```
-上面的 SQL 中，`CREATE TABLE` 为关键字，表示创建表；`d1001` 是子表的名称；`USING` 是关键字，表示要使用超级表作为模版；`meters` 是超级表的名称；在超级表名后的括号中，`location`, `group_id` 表示，是超级表的标签列名列表；`TAGS` 是关键字，在后面的括号中指定子表的标签列的值。`"California.SanFrancisco"` 和 `2` 表示子表 `d1001` 的位置为 `California.SanFrancisco`，分组 ID 为 `2` 。
+上面的 SQL 中，`CREATE TABLE` 为关键字，表示创建表；`d1001` 是子表的名称；`USING` 是关键字，表示要使用超级表作为模版；`meters` 是超级表的名称；在超级表名后的括号中，`location`、`group_id` 表示，是超级表的标签列名列表；`TAGS` 是关键字，在后面的括号中指定子表的标签列的值。`"California.SanFrancisco"` 和 `2` 表示子表 `d1001` 的位置为 `California.SanFrancisco`，分组 ID 为 `2`。
当对超级表进行写入或查询操作时，用户可以使用伪列 tbname 来指定或输出对应操作的子表名。
@@ -178,7 +178,7 @@ TAGS (
);
```
-上面的 SQL 中，`INSERT INTO d1002` 表示，向子表 `d1002` 中写入数据；`USING meters` 表示，使用超级表 `meters` 作为模版；`TAGS ("California.SanFrancisco", 2)` 表示，子表 `d1002` 的标签值分别为 `California.SanFrancisco` 和 `2`；`VALUES (NOW, 10.2, 219, 0.32)` 表示，向子表 `d1002` 插入一行记录，值分别为NOW（当前时间戳）、10.2（电流）、219（电压）、0.32（相位）。在 TDengine 执行这条 SQL 时，如果子表 `d1002` 已经存在，则直接写入数据；当子表 `d1002` 不存在，会先自动创建子表，再写入数据。
+上面的 SQL 中，`INSERT INTO d1002` 表示，向子表 `d1002` 中写入数据；`USING meters` 表示，使用超级表 `meters` 作为模版；`TAGS ("California.SanFrancisco", 2)` 表示，子表 `d1002` 的标签值分别为 `California.SanFrancisco` 和 `2`；`VALUES (NOW, 10.2, 219, 0.32)` 表示，向子表 `d1002` 插入一行记录，值分别为 NOW（当前时间戳）、10.2（电流）、219（电压）、0.32（相位）。在 TDengine 执行这条 SQL 时，如果子表 `d1002` 已经存在，则直接写入数据；当子表 `d1002` 不存在，会先自动创建子表，再写入数据。
### 创建普通表
@@ -204,7 +204,7 @@ CREATE TABLE d1003(
);
```
-上面的 SQL 表示，创建普通表 `d1003` ，表结构包括 `ts`、`current`、`voltage`、`phase`、`location`、`group_id`，共 6 个列。这样的数据模型，与关系型数据库完全一致。
+上面的 SQL 表示，创建普通表 `d1003`，表结构包括 `ts`、`current`、`voltage`、`phase`、`location`、`group_id`，共 6 个列。这样的数据模型，与关系型数据库完全一致。
采用普通表作为数据模型意味着静态标签数据（如 location 和 group_id）会重复存储在表的每一行中。这种做法不仅增加了存储空间的消耗，而且在进行查询时，由于无法直接利用标签数据进行过滤，查询性能会显著低于使用超级表的数据模型。
diff --git a/docs/zh/05-basic/02-insert.md b/docs/zh/05-basic/02-insert.md
index 88d131e832..b129fdbff1 100644
--- a/docs/zh/05-basic/02-insert.md
+++ b/docs/zh/05-basic/02-insert.md
@@ -12,9 +12,9 @@ toc_max_heading_level: 4
### 一次写入一条
-假设设备 ID 为 d1001 的智能电表在 2018 年 10 月 3 日 14:38:05 采集到数据：电流10.3A，电压 219V，相位 0.31。在第 3 章中，我们已经在 TDengine 的 power 数据库中创建了属于超级表 meters 的子表 d1001。接下来可以通过下面的 insert 语句在子表 d1001 中写入时序数据。
+假设设备 ID 为 d1001 的智能电表在 2018 年 10 月 3 日 14:38:05 采集到数据：电流 10.3A，电压 219V，相位 0.31。在第 3 章中，我们已经在 TDengine 的 power 数据库中创建了属于超级表 meters 的子表 d1001。接下来可以通过下面的 insert 语句在子表 d1001 中写入时序数据。
-1. 可以通过下面的 INSERT 语句向子表d1001中写入时序数据。
+1. 可以通过下面的 INSERT 语句向子表 d1001 中写入时序数据。
```sql
insert into d1001 (ts, current, voltage, phase) values ( "2018-10-03 14:38:05", 10.3, 219, 0.31)
@@ -120,7 +120,7 @@ values( "d1001, "2018-10-03 14:38:05", 10.2, 220, 0.23, "California.SanFrancisco
## 更新
-可以通过写入重复时间戳的一条数据来更新时序数据，新写入的数据会替换旧值。 下面的 SQL，通过指定列的方式，向子表 `d1001` 中写入 1 行数据；当子表 `d1001` 中已经存在日期时间为 `2018-10-03 14:38:05` 的数据时，`current`（电流）的新值22，会替换旧值。
+可以通过写入重复时间戳的一条数据来更新时序数据，新写入的数据会替换旧值。下面的 SQL，通过指定列的方式，向子表 `d1001` 中写入 1 行数据；当子表 `d1001` 中已经存在日期时间为 `2018-10-03 14:38:05` 的数据时，`current`（电流）的新值 22，会替换旧值。
```sql
INSERT INTO d1001 (ts, current) VALUES ("2018-10-03 14:38:05", 22);
@@ -128,7 +128,7 @@ INSERT INTO d1001 (ts, current) VALUES ("2018-10-03 14:38:05", 22);
## 删除
-为方便用户清理由于设备故障等原因产生的异常数据，TDengine 支持根据时间戳删除时序数据。 下面的 SQL，将超级表 `meters` 中所有时间戳早于 `2021-10-01 10:40:00.100` 的数据删除。数据删除后不可恢复，请慎重使用。为了确保删除的数据确实是自己要删除的，建议可以先使用 select 语句加 where 后的删除条件查看要删除的数据内容，确认无误后再执行 delete 。
+为方便用户清理由于设备故障等原因产生的异常数据，TDengine 支持根据时间戳删除时序数据。下面的 SQL，将超级表 `meters` 中所有时间戳早于 `2021-10-01 10:40:00.100` 的数据删除。数据删除后不可恢复，请慎重使用。为了确保删除的数据确实是自己要删除的，建议可以先使用 select 语句加 where 后的删除条件查看要删除的数据内容，确认无误后再执行 delete 。
```sql
delete from meters where ts < '2021-10-01 10:40:00.100' ;
diff --git a/docs/zh/05-basic/03-query.md b/docs/zh/05-basic/03-query.md
index f3c9eb099e..0b2f290667 100644
--- a/docs/zh/05-basic/03-query.md
+++ b/docs/zh/05-basic/03-query.md
@@ -14,7 +14,7 @@ toc_max_heading_level: 4
taosBenchmark --start-timestamp=1600000000000 --tables=100 --records=10000000 --time-step=10000
```
-上面的命令，taosBenchmark 工具在 TDengine 中生成了一个用于测试的数据库，产生共 10 亿条时序数据。时序数据的时间戳从 `1600000000000`（2020-09-13T20:26:40+08:00）开始，包含 `100` 个设备（子表），每个设备有 `10000000` 条数据，时序数据的采集频率是 10 秒/ 条。
+上面的命令，taosBenchmark 工具在 TDengine 中生成了一个用于测试的数据库，产生共 10 亿条时序数据。时序数据的时间戳从 `1600000000000`（2020-09-13T20:26:40+08:00）开始，包含 `100` 个设备（子表），每个设备有 `10000000` 条数据，时序数据的采集频率是 10 秒/条。
在 TDengine 中，用户可以通过 WHERE 语句指定条件，查询时序数据。以智能电表的数据为例
@@ -74,22 +74,22 @@ GROUP BY groupid;
Query OK, 10 row(s) in set (0.042446s)
```
-**注意**: group by 子句在聚合数据时，并不保证结果集按照特定顺序排列。为了获得有序的结果集，可以使用 order by 子句对结果进行排序。这样，可以根据需要调整输出结果的顺序，以满足特定的业务需求或报告要求。
+**注意**：group by 子句在聚合数据时，并不保证结果集按照特定顺序排列。为了获得有序的结果集，可以使用 order by 子句对结果进行排序。这样，可以根据需要调整输出结果的顺序，以满足特定的业务需求或报告要求。
TDengine 提供了多种内置的聚合函数。如下表所示：
| 聚合函数 | 功能说明 |
|:----------------------:|:--------------------------------------------------------------:|
-|APERCENTILE | 统计表/超级表中指定列的值的近似百分比分位数，与 PERCENTILE 函数相似，但是返回近似结果。 |
-|AVG | 统计指定字段的平均值 |
-|COUNT | 统计指定字段的记录行数 |
+|APERCENTILE | 统计表/超级表中指定列的值的近似百分比分位数，与 PERCENTILE 函数相似，但是返回近似结果。|
+|AVG | 统计指定字段的平均值。|
+|COUNT | 统计指定字段的记录行数。|
|ELAPSED|elapsed 函数表达了统计周期内连续的时间长度，和 twa 函数配合使用可以计算统计曲线下的面积。在通过 INTERVAL 子句指定窗口的情况下，统计在给定时间范围内的每个窗口内有数据覆盖的时间范围；如果没有 INTERVAL 子句，则返回整个给定时间范围内的有数据覆盖的时间范围。注意，ELAPSED 返回的并不是时间范围的绝对值，而是绝对值除以 time_unit 所得到的单位个数。|
-|LEASTSQUARES | 统计表中某列的值的拟合直线方程。start_val 是自变量初始值，step_val 是自变量的步长值。 |
+|LEASTSQUARES | 统计表中某列的值的拟合直线方程。start_val 是自变量初始值，step_val 是自变量的步长值。|
|SPREAD | 统计表中某列的最大值和最小值之差。|
-|STDDEV | 统计表中某列的均方差。 |
-|SUM | 统计表/超级表中某列的和。 |
-|HYPERLOGLOG | 采用 hyperloglog 算法，返回某列的基数。该算法在数据量很大的情况下，可以明显降低内存的占用，求出来的基数是个估算值，标准误差（标准误差是多次实验，每次的平均数的标准差，不是与真实结果的误差）为 0.81%。在数据量较少的时候该算法不是很准确，可以使用 select count（data） from （select unique（col） as data from table） 的方法。 |
-|HISTOGRAM | 统计数据按照用户指定区间的分布。 |
+|STDDEV | 统计表中某列的均方差。|
+|SUM | 统计表/超级表中某列的和。|
+|HYPERLOGLOG | 采用 hyperloglog 算法，返回某列的基数。该算法在数据量很大的情况下，可以明显降低内存的占用，求出来的基数是个估算值，标准误差（标准误差是多次实验，每次的平均数的标准差，不是与真实结果的误差）为 0.81%。在数据量较少的时候该算法不是很准确，可以使用 select count(data) from (select unique(col) as data from table) 的方法。|
+|HISTOGRAM | 统计数据按照用户指定区间的分布。|
|PERCENTILE | 统计表中某列的值百分比分位数。|
## 数据切分查询
@@ -101,12 +101,12 @@ PARTITION BY part_list
`part_list` 可以是任意的标量表达式，包括列、常量、标量函数和它们的组合。
-TDengine 按如下方式处理数据切分子句。
+TDengine 按如下方式处理数据切分子句：
1. 数据切分子句位于 WHERE 子句之后；
2. 数据切分子句将表数据按指定的维度进行切分，每个切分的分片进行指定的计算。计算由之后的子句定义（窗口子句、GROUP BY 子句或 SELECT 子句）；
3. 数据切分子句可以和窗口切分子句（或 GROUP BY 子句）一起使用，此时后面的子句作用在每个切分的分片上。
-数据切分的 SQL 如下：s
+数据切分的 SQL 如下：
```sql
SELECT location, avg(voltage)
@@ -141,6 +141,7 @@ Query OK, 10 row(s) in set (2.415961s)
- 状态窗口（status window）
- 会话窗口（session window）
- 事件窗口（event window）
+- 计数窗口（count window）
窗口划分逻辑如下图所示：
@@ -152,14 +153,15 @@ Query OK, 10 row(s) in set (2.415961s)
window_clause: {
SESSION(ts_col, tol_val)
| STATE_WINDOW(col)
- | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [FILL(fill_mod_and_val)]
+ | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [WATERMARK(watermark_val)] [FILL(fill_mod_and_val)]
| EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition
+ | COUNT_WINDOW(count_val[, sliding_val])
}
```
**注意** 在使用窗口子句时应注意以下规则：
1. 窗口子句位于数据切分子句之后，不可以和 GROUP BY 子句一起使用。
-2. 窗口子句将数据按窗口进行切分，对每个窗口进行 SELECT 列表中的表达式的计算，SELECT 列表中的表达式只能包含：常量；伪列：_wstart 伪列、_wend 伪列和 _wduration 伪列；聚合函数（包括选择函数和可以由参数确定输出行数的时序特有函数）
+2. 窗口子句将数据按窗口进行切分，对每个窗口进行 SELECT 列表中的表达式的计算，SELECT 列表中的表达式只能包含：常量；伪列：_wstart、_wend 和 _wduration；聚合函数：包括选择函数和可以由参数确定输出行数的时序特有函数。
3. WHERE 语句可以指定查询的起止时间和其他过滤条件。
### 时间戳伪列
@@ -177,16 +179,15 @@ INTERVAL(interval_val [, interval_offset])
```
时间窗口子句包括 3 个子句：
-- INTERVAL 子句：用于产生相等时间周期的窗口，interval_val 指定每个时间窗口的大小，interval_offset 指定；
+- INTERVAL 子句：用于产生相等时间周期的窗口，interval_val 指定每个时间窗口的大小，interval_offset 指定窗口偏移量；
- SLIDING 子句：用于指定窗口向前滑动的时间；
- FILL：用于指定窗口区间数据缺失的情况下，数据的填充模式。
-对于时间窗口，interval_val 和 sliding_val 都表示时间段， 语法上支持三种方式。例如：
-1. INTERVAL(1s, 500a) SLIDING(1s)，带时间单位的形式，其中的时间单位是单字符表示， 分别为： a （毫秒）， b （纳秒）， d （天）， h （小时）， m （分钟）， n （月）， s （秒）， u （微秒）， w （周）， y （年）；
+对于时间窗口，interval_val 和 sliding_val 都表示时间段，语法上支持三种方式。例如：
+1. INTERVAL(1s, 500a) SLIDING(1s)，带时间单位的形式，其中的时间单位是单字符表示，分别为：a（毫秒）、b（纳秒），d（天）、h（小时）、m（分钟）、n（月）、s（秒）、u（微秒）、w（周）、y（年）；
2. INTERVAL(1000, 500) SLIDING(1000)，不带时间单位的形式，将使用查询库的时间精度作为默认时间单位，当存在多个库时默认采用精度更高的库；
3. INTERVAL('1s', '500a') SLIDING('1s')，带时间单位的字符串形式，字符串内部不能有任何空格等其它字符。
-
示例 SQL 如下：
```sql
SELECT tbname, _wstart, _wend, avg(voltage)
@@ -220,7 +221,7 @@ Query OK, 12 row(s) in set (0.021265s)
#### 滑动窗口
-每次执行的查询是一个时间窗口，时间窗口随着时间流动向前滑动。在定义连续查询的时候需要指定时间窗口（time window ）大小和每次前向增量时间（forward sliding times）。如下图，[t0s， t0e] ，[t1s ， t1e]， [t2s， t2e] 是分别是执行三次连续查询的时间窗口范围，窗口的前向滑动的时间范围 sliding time 标识 。查询过滤、聚合等操作按照每个时间窗口为独立的单位执行。
+每次执行的查询是一个时间窗口，时间窗口随着时间流动向前滑动。在定义连续查询的时候需要指定时间窗口（time window ）大小和每次前向增量时间（forward sliding times）。如下图，[t0s, t0e]、[t1s, t1e]、[t2s, t2e] 是分别是执行三次连续查询的时间窗口范围，窗口的前向滑动的时间范围 sliding time 标识。查询过滤、聚合等操作按照每个时间窗口为独立的单位执行。
@@ -238,7 +239,7 @@ SELECT COUNT(*) FROM temp_tb_1 INTERVAL(1m) SLIDING(2m);
**使用时间窗口需要注意**
1. 聚合时间段的窗口宽度由关键词 INTERVAL 指定，最短时间间隔 10 毫秒（10a）；并且支持偏移 offset（偏移必须小于间隔），也即时间窗口划分与“UTC 时刻 0”相比的偏移量。SLIDING 语句用于指定聚合时间段的前向增量，也即每次窗口向前滑动的时长。
-2. 使用 INTERVAL 语句时，除非极特殊的情况，都要求把客户端和服务端的 taos.cfg 配置文件中的 timezone 参数配置为相同的取值，以避免时间处理函数频繁进行跨时区转换而导致的严重性能影响。
+2. 使用 INTERVAL 语句时，除非极特殊的情况，都要求把客户端和服务端的 timezone 参数配置为相同的取值，以避免时间处理函数频繁进行跨时区转换而导致的严重性能影响。
3. 返回的结果中时间序列严格单调递增。
示例：
@@ -274,7 +275,7 @@ Query OK, 11 row(s) in set (0.013153s)
#### 翻转窗口
-当 SLIDING 与 INTERVAL 相等的时候，滑动窗口即为翻转窗口。翻转窗口和滑动窗口的区别在于，滑动窗口因为 interval_val 和 sliding_val 不同，不同时间窗口之间，会存在数据重叠，翻转窗口则没有数据重叠。本质上，翻转窗口就是按照 interval_val 进行了时间窗口划分，INTERVAL(1m)和INTERVAL(1m) SLIDING(1m)是等效的。
+当 SLIDING 与 INTERVAL 相等的时候，滑动窗口即为翻转窗口。翻转窗口和滑动窗口的区别在于，滑动窗口因为 interval_val 和 sliding_val 不同，不同时间窗口之间，会存在数据重叠，翻转窗口则没有数据重叠。本质上，翻转窗口就是按照 interval_val 进行了时间窗口划分，INTERVAL(1m) 和 INTERVAL(1m) SLIDING(1m) 是等效的。
示例：
@@ -304,7 +305,7 @@ Query OK, 5 row(s) in set (0.016812s)
#### FILL 子句
1. 不进行填充：NONE（默认填充模式）。
-2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。这里需要注意，最终填充的值受由相应列的类型决定，如 FILL(VALUE, 1.23)，相应列为 INT 类型，则填充值为 1, 若查询列表中有多列需要 FILL, 则需要给每一个 FILL 列指定 VALUE, 如 `SELECT _wstart, min(c1), max(c1) FROM ... FILL(VALUE, 0, 0)`, 注意, SELECT 表达式中只有包含普通列时才需要指定 FILL VALUE, 如 `_wstart`, `_wstart+1a`, `now`, `1+1` 以及使用 partition by 时的 partition key (如 tbname)都不需要指定 VALUE, 如 `timediff(last(ts), _wstart)` 则需要指定VALUE。
+2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。这里需要注意，最终填充的值受由相应列的类型决定，如 FILL(VALUE, 1.23)，相应列为 INT 类型，则填充值为 1，若查询列表中有多列需要 FILL，则需要给每一个 FILL 列指定 VALUE，如 `SELECT _wstart, min(c1), max(c1) FROM ... FILL(VALUE, 0, 0)`。注意，SELECT 表达式中只有包含普通列时才需要指定 FILL VALUE，如 `_wstart`、`_wstart+1a`、`now`、`1+1` 以及使用 partition by 时的 partition key (如 tbname)都不需要指定 VALUE,，如 `timediff(last(ts), _wstart)` 则需要指定VALUE。
3. PREV 填充：使用前一个非 NULL 值填充数据。例如：FILL(PREV)。
4. NULL 填充：使用 NULL 填充数据。例如：FILL(NULL)。
5. LINEAR 填充：根据前后距离最近的非 NULL 值做线性插值填充。例如：FILL(LINEAR)。
@@ -313,11 +314,11 @@ Query OK, 5 row(s) in set (0.016812s)
以上填充模式中，除了 NONE 模式默认不填充值之外，其他模式在查询的整个时间范围内如果没有数据 FILL 子句将被忽略，即不产生填充数据，查询结果为空。这种行为在部分模式（PREV、NEXT、LINEAR）下具有合理性，因为在这些模式下没有数据意味着无法产生填充数值。
对另外一些模式（NULL、VALUE）来说，理论上是可以产生填充数值的，至于需不需要输出填充数值，取决于应用的需求。所以为了满足这类需要强制填充数据或 NULL 的应用的需求，同时不破坏现有填充模式的行为兼容性，TDengine 还支持两种新的填充模式：
-1. NULL_F: 强制填充 NULL 值
-2. VALUE_F: 强制填充 VALUE 值
+1. NULL_F：强制填充 NULL 值
+2. VALUE_F：强制填充 VALUE 值
-NULL、 NULL_F、 VALUE、 VALUE_F 这几种填充模式针对不同场景区别如下：
-1. INTERVAL 子句： NULL_F， VALUE_F 为强制填充模式；NULL， VALUE 为非强制模式。在这种模式下下各自的语义与名称相符
+NULL、NULL_F、VALUE、VALUE_F 这几种填充模式针对不同场景区别如下：
+1. INTERVAL 子句：NULL_F、VALUE_F 为强制填充模式；NULL、VALUE 为非强制模式。在这种模式下下各自的语义与名称相符
2. 流计算中的 INTERVAL 子句：NULL_F 与 NULL 行为相同，均为非强制模式；VALUE_F 与 VALUE 行为相同，均为非强制模式。即流计算中的 INTERVAL 没有强制模式
3. INTERP 子句：NULL 与 NULL_F 行为相同，均为强制模式；VALUE 与 VALUE_F 行为相同，均为强制模式。即 INTERP 中没有非强制模式。
@@ -405,7 +406,7 @@ Query OK, 22 row(s) in set (0.153403s)
### 会话窗口
-会话窗口根据记录的时间戳主键的值来确定是否属于同一个会话。如下图所示，如果设置时间戳的连续的间隔小于等于 12 秒，则以下 6 条记录构成 2 个会话窗口，分别是：[2019-04-28 14:22:10，2019-04-28 14:22:30]和[2019-04-28 14:23:10，2019-04-28 14:23:30]。因为 2019-04-28 14:22:30 与 2019-04-28 14:23:10 之间的时间间隔是 40 秒，超过了连续时间间隔（12 秒）。
+会话窗口根据记录的时间戳主键的值来确定是否属于同一个会话。如下图所示，如果设置时间戳的连续的间隔小于等于 12 秒，则以下 6 条记录构成 2 个会话窗口，分别是：[2019-04-28 14:22:10，2019-04-28 14:22:30] 和 [2019-04-28 14:23:10，2019-04-28 14:23:30]。因为 2019-04-28 14:22:30 与 2019-04-28 14:23:10 之间的时间间隔是 40 秒，超过了连续时间间隔（12 秒）。
@@ -452,7 +453,7 @@ Query OK, 10 row(s) in set (0.043489s)
事件窗口无法关闭时，不构成一个窗口，不会被输出。即有数据满足 start_trigger_condition，此时窗口打开，但后续数据都不能满足 end_trigger_condition，这个窗口无法被关闭，这部分数据不够成一个窗口，不会被输出。
-如果直接在超级表上进行事件窗口查询，TDengine 会将超级表的数据汇总成一条时间线，然后进行事件窗口的计算。 如果需要对子查询的结果集进行事件窗口查询，那么子查询的结果集需要满足按时间线输出的要求，且可以输出有效的时间戳列。
+如果直接在超级表上进行事件窗口查询，TDengine 会将超级表的数据汇总成一条时间线，然后进行事件窗口的计算。如果需要对子查询的结果集进行事件窗口查询，那么子查询的结果集需要满足按时间线输出的要求，且可以输出有效的时间戳列。
以下面的 SQL 语句为例，事件窗口切分如下图所示。
@@ -474,7 +475,7 @@ EVENT_WINDOW START WITH voltage >= 225 END WITH voltage < 235
LIMIT 5;
```
-上面的 SQL，查询超级表meters中，时间戳大于等于2022-01-01T00:00:00+08:00，且时间戳小于2022-01-01T00:10:00+08:00的数据；数据先按照子表名tbname进行数据切分，再根据事件窗口条件：电压大于等于 225V，且小于 235V 进行切分；最后，取每个分片的前 5 行的数据作为结果，返回子表名、窗口开始时间、窗口结束时间、窗口宽度、窗口内数据条数。查询结果如下：
+上面的 SQL，查询超级表 meters 中，时间戳大于等于 2022-01-01T00:00:00+08:00，且时间戳小于 2022-01-01T00:10:00+08:00 的数据；数据先按照子表名 tbname 进行数据切分，再根据事件窗口条件：电压大于等于 225V，且小于 235V 进行切分；最后，取每个分片的前 5 行的数据作为结果，返回子表名、窗口开始时间、窗口结束时间、窗口宽度、窗口内数据条数。查询结果如下：
```text
tbname | _wstart | _wend | _wduration | count(*) |
@@ -529,25 +530,25 @@ Query OK, 10 row(s) in set (0.062794s)
时序数据特有函数是 TDengine 针对时序数据查询场景专门设计的一组函数。在通用数据库中，要实现类似的功能通常需要编写复杂的查询语句，而且效率较低。为了降低用户的使用成本和简化查询过程，TDengine 将这些功能以内置函数的形式提供，从而实现了高效且易于使用的时序数据处理能力。时序数据特有函数如下表所示。
-| 函数 | 功能说明 |
-|:---------------:|:--------------------------------------------------------------------:|
-|CSUM | 累加和（Cumulative sum），忽略 NULL 值。 |
-|DERIVATIVE | 统计表中某列数值的单位变化率。其中单位时间区间的长度可以通过 time_interval 参数指定，最小可以是 1 秒（1s）；ignore_negative 参数的值可以是 0 或 1，为 1 时表示忽略负值。 |
-|DIFF | 统计表中某列的值与前一行对应值的差。 ignore_negative 取值为 0|1 ， 可以不填，默认值为 0。 不忽略负值。ignore_negative 为 1 时表示忽略负数。|
-|IRATE | 计算瞬时增长率。使用时间区间中最后两个样本数据来计算瞬时增长速率；如果这两个值呈递减关系，那么只取最后一个数用于计算，而不是使用二者差值。 |
-|MAVG | 计算连续 k 个值的移动平均数（moving average）。如果输入行数小于 k，则无结果输出。参数 k 的合法输入范围是 1≤ k ≤ 1000。|
-|STATECOUNT | 返回满足某个条件的连续记录的个数，结果作为新的一列追加在每行后面。条件根据参数计算，如果条件为 true 则加 1，条件为 false 则重置为 -1，如果数据为 NULL，跳过该条数据。 |
+| 函数 | 功能说明 |
+|:------------:|:--------------------------------------------------------------------:|
+|CSUM | 累加和（Cumulative sum），忽略 NULL 值。|
+|DERIVATIVE | 统计表中某列数值的单位变化率。其中单位时间区间的长度可以通过 time_interval 参数指定，最小可以是 1 秒（1s）；ignore_negative 参数的值可以是 0 或 1，为 1 时表示忽略负值。|
+|DIFF | 统计表中某列的值与前一行对应值的差。ignore_negative 取值为 0|1 ，可以不填，默认值为 0。不忽略负值。ignore_negative 为 1 时表示忽略负数。|
+|IRATE | 计算瞬时增长率。使用时间区间中最后两个样本数据来计算瞬时增长速率；如果这两个值呈递减关系，那么只取最后一个数用于计算，而不是使用二者差值。|
+|MAVG | 计算连续 k 个值的移动平均数（moving average）。如果输入行数小于 k，则无结果输出。参数 k 的合法输入范围是 1≤ k ≤ 1000。|
+|STATECOUNT | 返回满足某个条件的连续记录的个数，结果作为新的一列追加在每行后面。条件根据参数计算，如果条件为 true 则加 1，条件为 false 则重置为 -1，如果数据为 NULL，跳过该条数据。|
|STATEDURATION | 返回满足某个条件的连续记录的时间长度，结果作为新的一列追加在每行后面。条件根据参数计算，如果条件为 true 则加上两个记录之间的时间长度（第一个满足条件的记录时间长度记为 0），条件为 false 则重置为 -1，如果数据为 NULL，跳过该条数据|
-|TWA | 时间加权平均函数。统计表中某列在一段时间内的时间加权平均。 |
+|TWA | 时间加权平均函数。统计表中某列在一段时间内的时间加权平均。|
## 嵌套查询
嵌套查询，也称为 subquery（子查询），是指在一个 SQL 中，内层查询的计算结果可以作为外层查询的计算对象来使用。TDengine 支持在 from 子句中使用非关联 subquery。非关联是指 subquery 不会用到父查询中的参数。在 select 查询的 from 子句之后，可以接一个独立的 select 语句，这个 select 语句被包含在英文圆括号内。通过使用嵌套查询，你可以在一个查询中引用另一个查询的结果，从而实现更复杂的数据处理和分析。以智能电表为例进行说明，SQL 如下
```sql
-SELECT max(voltage),*
+SELECT max(voltage), *
FROM (
- SELECT tbname,last_row(ts),voltage,current,phase,groupid,location
+ SELECT tbname, last_row(ts), voltage, current, phase, groupid, location
FROM meters
PARTITION BY tbname
)
@@ -559,12 +560,12 @@ GROUP BY groupid;
TDengine 的嵌套查询遵循以下规则：
1. 内层查询的返回结果将作为“虚拟表”供外层查询使用，此虚拟表建议起别名，以便于外层查询中方便引用。
2. 外层查询支持直接通过列名或列名的形式引用内层查询的列或伪列。
-3. 在内层和外层查询中，都支持普通的表间/超级表间 JOIN。内层查询的计算结果也可以再参与数据子表的 JOIN 操作。
+3. 在内层和外层查询中，都支持普通表间/超级表间 JOIN。内层查询的计算结果也可以再参与数据子表的 JOIN 操作。
4. 内层查询支持的功能特性与非嵌套的查询语句能力是一致的。内层查询的 ORDER BY 子句一般没有意义，建议避免这样的写法以免无谓的资源消耗。
5. 与非嵌套的查询语句相比，外层查询所能支持的功能特性存在如下限制：
-6. 如果内层查询的结果数据未提供时间戳，那么计算过程隐式依赖时间戳的函数在外层会无法正常工作。例如：INTERP， DERIVATIVE， IRATE， LAST_ROW， FIRST， LAST， TWA， STATEDURATION， TAIL， UNIQUE。
-7. 如果内层查询的结果数据不是按时间戳有序，那么计算过程依赖数据按时间有序的函数在外层会无法正常工作。例如：LEASTSQUARES， ELAPSED， INTERP， DERIVATIVE， IRATE， TWA， DIFF， STATECOUNT， STATEDURATION， CSUM， MAVG， TAIL， UNIQUE。
-8. 计算过程需要两遍扫描的函数，在外层查询中无法正常工作。例如：此类函数包括：PERCENTILE。
+6. 如果内层查询的结果数据未提供时间戳，那么计算过程隐式依赖时间戳的函数在外层会无法正常工作。例如：INTERP、DERIVATIVE、IRATE、LAST_ROW、FIRST、LAST、TWA、STATEDURATION、TAIL、UNIQUE。
+7. 如果内层查询的结果数据不是按时间戳有序，那么计算过程依赖数据按时间有序的函数在外层会无法正常工作。例如：LEASTSQUARES、ELAPSED、INTERP、DERIVATIVE、IRATE、TWA、DIFF、STATECOUNT、STATEDURATION、CSUM、MAVG、TAIL、UNIQUE。
+8. 计算过程需要两遍扫描的函数，在外层查询中无法正常工作。例如：PERCENTILE。
## UNION 子句
@@ -573,11 +574,11 @@ TDengine 支持 UNION 操作符。也就是说，如果多个 SELECT 子句返
示例：
```sql
-(SELECT tbname,* FROM d1 limit 1)
+(SELECT tbname, * FROM d1 limit 1)
UNION ALL
-(SELECT tbname,* FROM d11 limit 2)
+(SELECT tbname, * FROM d11 limit 2)
UNION ALL
-(SELECT tbname,* FROM d21 limit 3);
+(SELECT tbname, * FROM d21 limit 3);
```
上面的 SQL，分别查询：子表 d1 的 1 条数据，子表 d11 的 2 条数据，子表 d21 的 3 条数据，并将结果合并。返回的结果如下：
@@ -594,7 +595,7 @@ UNION ALL
Query OK, 6 row(s) in set (0.006438s)
```
-在同一个 sql 语句中，最多支持 100 个 UNION 子句。
+在同一个 SQL 语句中，最多支持 100 个 UNION 子句。
## 关联查询
@@ -640,9 +641,9 @@ select a.* from meters a left asof join meters b on timetruncate(a.ts, 1s) < tim
### 语法说明
-在接下来的内容中，我们将通过统一的方式并行介绍 Left Join 和 Right Join 系列。因此，在后续关于 Outer、Semi、Anti-Semi、ASOF、Window 等系列内容的介绍中，我们采用了“ Left/Right”这种表述方式来同时涵盖 Left Join 和 Right Join 的相关知识。这里的“ /”符号前的描述专指应用于 Left Join，而“ /”符号后的描述则专指应用于 Right Join。通过这种表述方式，我们可以更加清晰地展示这两种 Join 操作的特点和用法。
+在接下来的内容中，我们将通过统一的方式并行介绍 Left Join 和 Right Join 系列。因此，在后续关于 Outer、Semi、Anti-Semi、ASOF、Window 等系列内容的介绍中，我们采用了“Left/Right”这种表述方式来同时涵盖 Left Join 和 Right Join 的相关知识。这里的“/”符号前的描述专指应用于 Left Join，而“/”符号后的描述则专指应用于 Right Join。通过这种表述方式，我们可以更加清晰地展示这两种 Join 操作的特点和用法。
-例如，当我们提及“左 / 右表”时，对于 Left Join，它特指左表，而对于 Right Join，它则特指右表。同理，当我们提及“右 / 左表”时，对于 Left Join，它特指右表，而对于 Right Join，它则特指左表。
+例如，当我们提及“左/右表”时，对于 Left Join，它特指左表，而对于 Right Join，它则特指右表。同理，当我们提及“右/左表”时，对于 Left Join，它特指右表，而对于 Right Join，它则特指左表。
### Join 功能
@@ -650,13 +651,13 @@ select a.* from meters a left asof join meters b on timetruncate(a.ts, 1s) < tim
| Join 类型 | 定义 |
|:------------------------:|:--------------------------------------------------------:|
-|Inner Join | 内连接，只有左右表中同时符合连接条件的数据才会被返回，可以视为两张表符合连接条件的数据的交集 |
-|Left/Right Outer Join | 左 / 右（外）连接，既包含左右表中同时符合连接条件的数据集合，也包括左 / 右表中不符合连接条件的数据集合 |
-|Left/Right Semi Join | 左 / 右半连接，通常表达的是 in、exists 的含义，即对左 / 右表任意一条数据来说，只有当右 / 左表中存在任一符合连接条件的数据时才返回左 / 右表行数据 |
+|Inner Join | 内连接，只有左右表中同时符合连接条件的数据才会被返回，可以视为两张表符合连接条件的数据的交集 |
+|Left/Right Outer Join | 左 / 右（外）连接，既包含左右表中同时符合连接条件的数据集合，也包括左 / 右表中不符合连接条件的数据集合 |
+|Left/Right Semi Join | 左 / 右半连接，通常表达的是 in、exists 的含义，即对左 / 右表任意一条数据来说，只有当右 / 左表中存在任一符合连接条件的数据时才返回左 / 右表行数据 |
|Left/Right Anti-Semi Join | 左 / 右反连接，同左 / 右半连接的逻辑正好相反，通常表达的是 not in、not exists 的含义，即对左 / 右表任意一条数据来说，只有当右 / 左表中不存在任何符合连接条件的数据时才返回左 / 右表行数据 |
-|left/Right ASOF Join | 左 / 右不完全匹配连接，不同于其他传统 Join 操作的完全匹配模式，ASOF Join 允许以指定的匹配模式进行不完全匹配，即按照主键时间戳最接近的方式进行匹配 |
-|Left/Right Window Join | 左 / 右窗口连接，根据左 / 右表中每一行的主键时间戳和窗口边界构造窗口并据此进行窗口连接，支持在窗口内进行投影、标量和聚合操作 |
-|Full Outer Join | 全（外）连接，既包含左右表中同时符合连接条件的数据集合，也包括左右表中不符合连接条件的数据集合 |
+|left/Right ASOF Join | 左 / 右不完全匹配连接，不同于其他传统 Join 操作的完全匹配模式，ASOF Join 允许以指定的匹配模式进行不完全匹配，即按照主键时间戳最接近的方式进行匹配 |
+|Left/Right Window Join | 左 / 右窗口连接，根据左 / 右表中每一行的主键时间戳和窗口边界构造窗口并据此进行窗口连接，支持在窗口内进行投影、标量和聚合操作 |
+|Full Outer Join | 全（外）连接，既包含左右表中同时符合连接条件的数据集合，也包括左右表中不符合连接条件的数据集合 |
### 约束和限制
diff --git a/docs/zh/06-advanced/01-subscription.md b/docs/zh/06-advanced/01-subscription.md
index 5d458b186c..0760f44446 100644
--- a/docs/zh/06-advanced/01-subscription.md
+++ b/docs/zh/06-advanced/01-subscription.md
@@ -12,7 +12,7 @@ toc_max_heading_level: 4
为实现上述功能，TDengine 会为预写数据日志（Write-Ahead Logging，WAL）文件自动创建索引，以支持快速随机访问，并提供了灵活可配置的文件切换与保留机制。用户可以根据需求指定 WAL 文件的保留时间和大小。通过这些方法，WAL 被改造成一个保留事件到达顺序的、可持久化的存储引擎。对于以主题形式创建的查询，TDengine 将从 WAL 读取数据。在消费过程中，TDengine 根据当前消费进度从 WAL 直接读取数据，并使用统一的查询引擎实现过滤、变换等操作，然后将数据推送给消费者。
-从 3.2.0.0 版本开始，数据订阅支持 vnode 迁移和分裂。 由于数据订阅依赖 wal文件，而在 vnode 迁移和分裂的过程中，wal 并不会同步过去，所以迁移或分裂后，之前没消费完的 wal数据后消费不到。所以请保证之前把数据全部消费完后，再进行 vnode 迁移或分裂，否则，消费会丢失数据。
+从 3.2.0.0 版本开始，数据订阅支持 vnode 迁移和分裂。由于数据订阅依赖 wal 文件，而在 vnode 迁移和分裂的过程中，wal 文件并不会进行同步。因此，在迁移或分裂操作完成后，您将无法继续消费之前尚未消费完 wal 数据。请务必在执行 vnode 迁移或分裂之前，将所有 wal 数据消费完毕。
## 主题类型
@@ -31,7 +31,7 @@ CREATE TOPIC [IF NOT EXISTS] topic_name as subquery
3. 若发生表结构变更，新增的列不出现在结果中。
4. 对于 select *，则订阅展开为创建时所有的列（子表、普通表为数据列，超级表为数据列加标签列）
-假设需要订阅所有智能电表中电压值大于 200 的数据，而且仅仅返回时间戳、电流、电压 3 个采集量（不返回相位），那么可以通过下面的 SQL 创建 power_topic 这个主题。
+假设需要订阅所有智能电表中电压值大于 200 的数据，且仅仅返回时间戳、电流、电压 3 个采集量（不返回相位），那么可以通过下面的 SQL 创建 power_topic 这个主题。
```sql
CREATE TOPIC power_topic AS SELECT ts, current, voltage FROM power.meters WHERE voltage > 200;
```
@@ -45,21 +45,21 @@ CREATE TOPIC [IF NOT EXISTS] topic_name [with meta] AS STABLE stb_name [where_co
与使用 `SELECT * from stbName` 订阅的区别是：
-1. 不会限制用户的表结构变更，即表结构变更以及变更后的新数据都能够订阅到
+1. 不会限制用户的表结构变更，即表结构变更以及变更后的新数据都能够订阅到。
2. 返回的是非结构化的数据，返回数据的结构会随着超级表的表结构变化而变化。
-3. with meta 参数可选，选择时将返回创建超级表，子表等语句，主要用于 taosx 做超级表迁移。
-4. where_condition 参数可选，选择时将用来过滤符合条件的子表，订阅这些子表。where 条件里不能有普通列，只能是 tag 或 tbname，where 条件里可以用函数，用来过滤 tag，但是不能是聚合函数，因为子表 tag 值无法做聚合。也可以是常量表达式，比如 2 > 1（订阅全部子表），或者 false（订阅 0 个子表）。
+3. with meta 参数可选，选择时将返回创建超级表，子表等语句，主要用于 taosX 做超级表迁移。
+4. where_condition 参数可选，选择时将用来过滤符合条件的子表，订阅这些子表。where 条件里不能有普通列，只能是 tag 或 tbname，where 条件里可以用函数，用来过滤 tag，但是不能是聚合函数，因为子表 tag 值无法做聚合。可以是常量表达式，比如 2 > 1（订阅全部子表），或者 false（订阅 0 个子表）。
5. 返回数据不包含标签。
### 数据库主题
-订阅一个数据库里所有数据，其语法如下
+订阅一个数据库里所有数据，其语法如下：
```sql
CREATE TOPIC [IF NOT EXISTS] topic_name [with meta] AS DATABASE db_name;
```
-通过该语句可创建一个包含数据库所有表数据的订阅
-1. with meta 参数可选，选择时将返回数据库里所有超级表，子表、普通表的元数据创建、删除、修改语句，主要用于 taosx 做数据库迁移。
+通过该语句可创建一个包含数据库所有表数据的订阅：
+1. with meta 参数可选，选择时将返回数据库里所有超级表，子表、普通表的元数据创建、删除、修改语句，主要用于 taosX 做数据库迁移。
2. 超级表订阅和库订阅属于高级订阅模式，容易出错，如确实要使用，请咨询技术支持人员。
## 删除主题
@@ -128,7 +128,7 @@ TDengine 的数据订阅功能支持回放（replay）功能，允许用户按
2023/09/22 00:00:08.000
```
-使用数据订阅的回放功能时需要注意如下几项。
+使用数据订阅的回放功能时需要注意如下几项：
- 数据订阅的回放功能仅查询订阅支持数据回放，超级表和库订阅不支持回放。
- 回放不支持进度保存。
- 因为数据回放本身需要处理时间，所以回放的精度存在几十毫秒的误差。
diff --git a/docs/zh/06-advanced/02-cache.md b/docs/zh/06-advanced/02-cache.md
index 875452205b..5054bd6acd 100644
--- a/docs/zh/06-advanced/02-cache.md
+++ b/docs/zh/06-advanced/02-cache.md
@@ -29,14 +29,14 @@ TDengine 采用时间驱动的缓存管理策略，将最新数据优先存储
## TDengine 的读缓存配置
-在创建数据库时，用户可以选择是否启用缓存机制以存储该数据库中每张子表的最新数据。这一缓存机制由数据库创建参数 cachemodel 进行控制。参数 cachemodel 具有如 下 4 种情况：
+在创建数据库时，用户可以选择是否启用缓存机制以存储该数据库中每张子表的最新数据。这一缓存机制由数据库创建参数 cachemodel 进行控制。参数 cachemodel 具有如下 4 种情况：
- none：不缓存
- last_row：缓存子表最近一行数据，这将显著改善 last_row 函数的性能
-- last_value：缓存子表每一列最近的非 NULL 值，这将显著改善无特殊影响（比如 WHERE,ORDER BY,GROUP BY, INTERVAL）时的 last 函数的性能
+- last_value：缓存子表每一列最近的非 NULL 值，这将显著改善无特殊影响（比如 WHERE、ORDER BY、GROUP BY、INTERVAL）时的 last 函数的性能
- both：同时缓存最近的行和列，即等同于上述 cachemodel 值为 last_row 和 last_value 的行为同时生效
当使用数据库读缓存时，可以使用参数 cachesize 来配置每个 vnode 的内存大小。
-- cachesize：表示每个 vnode 中用于缓存子表最近数据的内存大小。默认为 1 ，范围是[1，65536]，单位是 MB。需要根据机器内存合理配置。
+- cachesize：表示每个 vnode 中用于缓存子表最近数据的内存大小。默认为 1，范围是 [1，65536]，单位是 MB。需要根据机器内存合理配置。
关于数据库的具体创建，相关参数和操作说明请参考[创建数据库](../../reference/taos-sql/database/)
@@ -48,25 +48,25 @@ TDengine 采用时间驱动的缓存管理策略，将最新数据优先存储
# taosBenchmark -d power -Q --start-timestamp=1600000000000 --tables=10000 --records=10000 --time-step=10000 -y
```
-上面的命令，taosBenchmark 工具在 TDengine 中生成了一个用于测试的 电表数据库 power，产生共 10 亿条时序数据。时序数据的时间戳从 `1600000000000（2020-09-13T20:26:40+08:00）`开始，超级表为 `meter`s，包含 10000 个设备（子表），每个设备有 10000 条数据，时序数据的采集频率是 10 秒/ 条。
+上面的命令，taosBenchmark 工具在 TDengine 中生成了一个用于测试的 电表数据库 power，产生共 10 亿条时序数据。时序数据的时间戳从 `1600000000000（2020-09-13T20:26:40+08:00）` 开始，超级表为 `meters`，包含 10000 个设备（子表），每个设备有 10000 条数据，时序数据的采集频率是 10 秒/条。
查询任意一个电表的最新的电流和时间戳数据，执行如下 SQL
```sql
-taos> select last(ts,current) from meters;
+taos> select last(ts, current) from meters;
last(ts) | last(current) |
=================================================
2020-09-15 00:13:10.000 | 1.1294620 |
Query OK, 1 row(s) in set (0.353815s)
-taos> select last_row(ts,current) from meters;
+taos> select last_row(ts, current) from meters;
last_row(ts) | last_row(current) |
=================================================
2020-09-15 00:13:10.000 | 1.1294620 |
Query OK, 1 row(s) in set (0.344070s)
```
-希望使用缓存来查询任意一个电表的最新时间戳数据，执行如下 SQL ，并检查数据库的缓存生效。
+希望使用缓存来查询任意一个电表的最新时间戳数据，执行如下 SQL，并检查数据库的缓存生效。
```sql
taos> alter database power cachemodel 'both' ;
@@ -82,13 +82,13 @@ Query OK, 1 row(s) in set (0.000282s)
再次查询电表的最新的实时数据，第一次查询会做缓存计算，后续的查询时延就大大缩减。
```sql
-taos> select last(ts,current) from meters;
+taos> select last(ts, current) from meters;
last(ts) | last(current) |
=================================================
2020-09-15 00:13:10.000 | 1.1294620 |
Query OK, 1 row(s) in set (0.044021s)
-taos> select last_row(ts,current) from meters;
+taos> select last_row(ts, current) from meters;
last_row(ts) | last_row(current) |
=================================================
2020-09-15 00:13:10.000 | 1.1294620 |
diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md
index 8692afad45..74e7e81675 100644
--- a/docs/zh/06-advanced/03-stream.md
+++ b/docs/zh/06-advanced/03-stream.md
@@ -60,20 +60,20 @@ subquery 支持会话窗口、状态窗口、时间窗口、事件窗口与计
3. INTERVAL 是时间窗口，又可分为滑动时间窗口和翻转时间窗口。INTERVAL 子句用于指定窗口相等时间周期，SLIDING 字句用于指定窗口向前滑动的时间。当 interval_val 与 sliding_val 相等的时候，时间窗口即为翻转时间窗口，否则为滑动时间窗口，注意：sliding_val 必须小于等于 interval_val。
-4. EVENT_WINDOW 是事件窗口，根据开始条件和结束条件来划定窗口。当 start_trigger_condition 满足时则窗口开始，直到 end_trigger_condition 满足时窗口关闭。 start_trigger_condition 和 end_trigger_condition 可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。
+4. EVENT_WINDOW 是事件窗口，根据开始条件和结束条件来划定窗口。当 start_trigger_condition 满足时则窗口开始，直到 end_trigger_condition 满足时窗口关闭。start_trigger_condition 和 end_trigger_condition 可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。
-5. COUNT_WINDOW 是计数窗口，按固定的数据行数来划分窗口。 count_val 是常量，是正整数，必须大于等于 2，小于 2147483648。 count_val 表示每个 COUNT_WINDOW 包含的最大数据行数，总数据行数不能整除 count_val 时，最后一个窗口的行数会小于 count_val 。 sliding_val 是常量，表示窗口滑动的数量，类似于 INTERVAL 的 SLIDING 。
+5. COUNT_WINDOW 是计数窗口，按固定的数据行数来划分窗口。count_val 是常量，是正整数，必须大于等于 2，小于 2147483648。count_val 表示每个 COUNT_WINDOW 包含的最大数据行数，总数据行数不能整除 count_val 时，最后一个窗口的行数会小于 count_val。sliding_val 是常量，表示窗口滑动的数量，类似于 INTERVAL 的 SLIDING 。
窗口的定义与时序数据窗口查询中的定义完全相同，具体可参考 TDengine 窗口函数部分。
-如下 SQL 将创建一个流计算，执行后 TDengine 会自动创建名为avg_vol 的超级表，此流计算以 1min 为时间窗口、30s 为前向增量统计这些智能电表的平均电压，并将来自 meters 的数据的计算结果写入 avg_vol，不同分区的数据会分别创建子表并写入不同子表。
+如下 SQL 将创建一个流计算，执行后 TDengine 会自动创建名为 avg_vol 的超级表，此流计算以 1min 为时间窗口、30s 为前向增量统计这些智能电表的平均电压，并将来自 meters 的数据的计算结果写入 avg_vol，不同分区的数据会分别创建子表并写入不同子表。
```sql
CREATE STREAM avg_vol_s INTO avg_vol AS
SELECT _wstart, count(*), avg(voltage) FROM power.meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s);
```
本节涉及的相关参数的说明如下。
-- stb_name 是保存计算结果的超级表的表名，如果该超级表不存在，则会自动创建；如果已存在，则检查列的 schema 信息。详见 6.3.8 节。
+- stb_name 是保存计算结果的超级表的表名，如果该超级表不存在，则会自动创建；如果已存在，则检查列的 schema 信息。
- tags 子句定义了流计算中创建标签的规则。通过 tags 字段可以为每个分区对应的子表生成自定义的标签值。
## 流式计算的规则和策略
diff --git a/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md b/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md
index 087d6540e6..560e6cdcc0 100644
--- a/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md
+++ b/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md
@@ -3,6 +3,9 @@ title: 预测算法
description: 预测算法
---
+import fc_result from '../pic/fc.png';
+import fc_result_figure from '../pic/fc-result.png';
+
时序数据预测处理以持续一个时间段的时序数据作为输入，预测接下来一个连续时间区间内时间序列数据趋势。用户可以指定输出的（预测）时间序列数据点的数量，因此其输出的结果行数不确定。为此，TDengine 使用新 SQL 函数 `FORECAST` 提供时序数据预测服务。基础数据（用于预测的历史时间序列数据）是该函数的输入，预测结果是该函数的输出。用户可以通过 `FORECAST` 函数调用 Anode 提供的预测算法提供的服务。
在后续章节中，使用时序数据表`foo`作为示例，介绍预测和异常检测算法的使用方式，`foo` 表的模式如下：
@@ -106,3 +109,37 @@ taos> select _flow, _fhigh, _frowts, forecast(i32) from foo;
- PatchTST (Patch Time Series Transformer)
- Temporal Fusion Transformer
- TimesNet
+
+## 算法有效性评估工具
+TDgpt 提供预测分析算法有效性评估工具 `analytics_compare`，调用该工具并设置合适的参数，能够使用 TDengine 中的数据作为回测依据，评估不同预测算法或相同的预测算法在不同的参数或训练模型的下的预测有效性。预测有效性的评估使用 `MSE` 和 `MAE` 指标作为依据，后续还将增加 `MAPE`指标。
+
+```ini
+[forecast]
+# 训练数据的周期，每个周期包含多少个输入点
+period = 10
+
+# 使用范围内最后 10 条记录作为预测结果
+rows = 10
+
+# 训练数据开始时间
+start_time = 1949-01-01T00:00:00
+
+# 训练数据结束时间
+end_time = 1960-12-01T00:00:00
+
+# 输出结果的起始时间
+res_start_time = 1730000000000
+
+# 是否绘制预测结果图
+gen_figure = true
+```
+
+算法对比分析运行完成以后，生成 fc-results.xlsx 文件，其中包含了调用算法的预测分析误差、执行时间、调用参数等信息。如下图所示：
+
+
+
+
+如果设置了 `gen_figure` 为 true，分析结果中还会有绘制的分析预测结果图（如下图所示）。
+
+
+
diff --git a/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/index.md b/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/index.md
index 632492ce72..4802fa2e4f 100644
--- a/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/index.md
+++ b/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/index.md
@@ -4,6 +4,8 @@ description: 异常检测算法
---
import ad from '../pic/anomaly-detection.png';
+import ad_result from '../pic/ad-result.png';
+import ad_result_figure from '../pic/ad-result-figure.png';
TDengine 中定义了异常（状态）窗口来提供异常检测服务。异常窗口可以视为一种特殊的**事件窗口（Event Window）**，即异常检测算法确定的连续异常时间序列数据所在的时间窗口。与普通事件窗口区别在于——时间窗口的起始时间和结束时间均是分析算法识别确定，不是用户给定的表达式进行判定。因此，在 `WHERE` 子句中使用 `ANOMALY_WINDOW` 关键词即可调用时序数据异常检测服务，同时窗口伪列（`_WSTART`, `_WEND`, `_WDURATION`）也能够像其他时间窗口一样用于描述异常窗口的起始时间(`_WSTART`)、结束时间(`_WEND`)、持续时间(`_WDURATION`)。例如：
@@ -67,3 +69,38 @@ Query OK, 1 row(s) in set (0.028946s)
### 内置异常检测算法
分析平台内置了6个异常检查模型，分为3个类别，分别是[基于统计学的算法](./02-statistics-approach.md)、[基于数据密度的算法](./03-data-density.md)、以及[基于机器学习的算法](./04-machine-learning.md)。在不指定异常检测使用的方法的情况下，默认调用 IQR 进行异常检测。
+### 异常检测算法有效性比较工具
+TDgpt 提供自动化的工具对比不同数据集的不同算法监测有效性，针对异常检测算法提供查全率（recall）和查准率（precision）两个指标衡量不同算法的有效性。
+通过在配置文件中(analysis.ini)设置以下的选项可以调用需要使用的异常检测算法，异常检测算法测试用数据的时间范围、是否生成标注结果的图片、调用的异常检测算法以及相应的参数。
+调用异常检测算法比较之前，需要人工手动标注异常监测数据集的结果，即设置[anno_res]选项的数值，第几个数值是异常点，需要标注在数组中，如下测试集中，第 9 个点是异常点，我们就标注异常结果为 [9].
+
+```bash
+[ad]
+# training data start time
+start_time = 2021-01-01T01:01:01
+
+# training data end time
+end_time = 2021-01-01T01:01:11
+
+# draw the results or not
+gen_figure = true
+
+# annotate the anomaly_detection result
+anno_res = [9]
+
+# algorithms list that is involved in the comparion
+[ad.algos]
+ksigma={"k": 2}
+iqr={}
+grubbs={}
+lof={"algo":"auto", "n_neighbor": 3}
+```
+
+对比程序执行完成以后，会自动生成名称为`ad_result.xlsx` 的文件，第一个卡片是算法运行结果（如下图所示），分别包含了算法名称、执行调用参数、查全率、查准率、执行时间 5 个指标。
+
+
+
+如果设置了 `gen_figure` 为 `true`，比较程序会自动将每个参与比较的算法分析结果采用图片方式呈现出来（如下图所示为 ksigma 的异常检测结果标注）。
+
+
+
diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md b/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md
index 7c85d41c50..9af66201ae 100644
--- a/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md
+++ b/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md
@@ -47,7 +47,7 @@ class _MyAnomalyDetectionService(AbstractAnomalyDetectionService):
return super().set_params(params)
```
-将该文件保存在 `./lib/taosanalytics/algo/ad/` 目录下，然后重启 taosanode 服务。在 TDengine 命令行接口 taos 中执行 `SHOW ANODES FULL` 就能够看到新加入的算法，然后就可以通过 SQL 语句调用该算法。
+将该文件保存在 `./lib/taosanalytics/algo/ad/` 目录下，然后重启 taosanode 服务。在 TDengine CLI 中执行 `SHOW ANODES FULL` 就能够看到新加入的算法，然后就可以通过 SQL 语句调用该算法。
```SQL
--- 对 col 列进行异常检测，通过指定 algo 参数为 myad 来调用新添加的异常检测类
diff --git a/docs/zh/06-advanced/06-TDgpt/pic/ad-result-figure.png b/docs/zh/06-advanced/06-TDgpt/pic/ad-result-figure.png
new file mode 100644
index 0000000000..b1bf231786
Binary files /dev/null and b/docs/zh/06-advanced/06-TDgpt/pic/ad-result-figure.png differ
diff --git a/docs/zh/06-advanced/06-TDgpt/pic/ad-result.png b/docs/zh/06-advanced/06-TDgpt/pic/ad-result.png
new file mode 100644
index 0000000000..80667c5733
Binary files /dev/null and b/docs/zh/06-advanced/06-TDgpt/pic/ad-result.png differ
diff --git a/docs/zh/06-advanced/06-TDgpt/pic/fc-result.png b/docs/zh/06-advanced/06-TDgpt/pic/fc-result.png
new file mode 100644
index 0000000000..6bcca3934c
Binary files /dev/null and b/docs/zh/06-advanced/06-TDgpt/pic/fc-result.png differ
diff --git a/docs/zh/06-advanced/06-TDgpt/pic/fc.png b/docs/zh/06-advanced/06-TDgpt/pic/fc.png
new file mode 100644
index 0000000000..ff11a5dc48
Binary files /dev/null and b/docs/zh/06-advanced/06-TDgpt/pic/fc.png differ
diff --git a/docs/zh/07-develop/01-connect/index.md b/docs/zh/07-develop/01-connect/index.md
index 494e93f6ef..234a6b9792 100644
--- a/docs/zh/07-develop/01-connect/index.md
+++ b/docs/zh/07-develop/01-connect/index.md
@@ -20,7 +20,7 @@ import VerifyLinux from "../../14-reference/05-connector/_verify_linux.mdx";
import VerifyMacOS from "../../14-reference/05-connector/_verify_macos.mdx";
import VerifyWindows from "../../14-reference/05-connector/_verify_windows.mdx";
-TDengine 提供了丰富的应用程序开发接口，为了便于用户快速开发自己的应用，TDengine 支持了多种编程语言的连接器，其中官方连接器包括支持 C/C++、Java、Python、Go、Node.js、C#、Rust、Lua（社区贡献）和 PHP （社区贡献）的连接器。这些连接器支持使用原生接口（taosc）和 REST 接口（部分语言暂不支持）连接 TDengine 集群。社区开发者也贡献了多个非官方连接器，例如 ADO.NET 连接器、Lua 连接器和 PHP 连接器。另外 TDengine 还可以直接调用 taosadapter 提供的 REST API 接口，进行数据写入和查询操作。
+TDengine 提供了丰富的应用程序开发接口，为了便于用户快速开发自己的应用，TDengine 支持了多种编程语言的连接器，其中官方连接器包括支持 C/C++、Java、Python、Go、Node.js、C#、Rust、Lua（社区贡献）和 PHP （社区贡献）的连接器。这些连接器支持使用原生接口（taosc）和 REST 接口（部分语言暂不支持）连接 TDengine 集群。社区开发者也贡献了多个非官方连接器，例如 ADO.NET 连接器、Lua 连接器和 PHP 连接器。另外 TDengine 还可以直接调用 taosAdapter 提供的 REST API 接口，进行数据写入和查询操作。
## 连接方式
@@ -33,7 +33,7 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速
无论使用何种方式建立连接，连接器都提供了相同或相似的 API 操作数据库，都可以执行 SQL 语句，只是初始化连接的方式稍有不同，用户在使用上不会感到什么差别。
-各种连接方式和各语言连接器支持情况请参考：[连接器功能特性](../../reference/connector/#功能特性)
+各种连接方式和各语言连接器支持情况请参考 [连接器功能特性](../../reference/connector/#功能特性)
关键不同点在于：
diff --git a/docs/zh/07-develop/04-schemaless.md b/docs/zh/07-develop/04-schemaless.md
index bf10b41736..53a034dbdc 100644
--- a/docs/zh/07-develop/04-schemaless.md
+++ b/docs/zh/07-develop/04-schemaless.md
@@ -299,7 +299,7 @@ writer.write(lineDemo, SchemalessProtocolType.LINE, SchemalessTimestampType.NANO
## 查询写入的数据
-运行上节的样例代码，会在 power 数据库中自动建表，我们可以通过 taos shell 或者应用程序来查询数据。下面给出用 taos shell 查询超级表和 meters 表数据的样例。
+运行上节的样例代码，会在 power 数据库l中自动建表，我们可以通过 TDengine CLI 或者应用程序来查询数据。下面给出用 TDengine CLI 查询超级表和 meters 表数据的样例。
```shell
taos> show power.stables;
diff --git a/docs/zh/07-develop/07-tmq.md b/docs/zh/07-develop/07-tmq.md
index c38a43f3fb..7ae09aaeef 100644
--- a/docs/zh/07-develop/07-tmq.md
+++ b/docs/zh/07-develop/07-tmq.md
@@ -10,7 +10,7 @@ import TabItem from "@theme/TabItem";
TDengine 提供了类似于消息队列产品的数据订阅和消费接口。在许多场景中，采用 TDengine 的时序大数据平台，无须再集成消息队列产品，从而简化应用程序设计并降低运维成本。本章介绍各语言连接器数据订阅的相关 API 以及使用方法。 数据订阅的基础知识请参考 [数据订阅](../../advanced/subscription/)
## 创建主题
-请用 taos shell 或者 参考 [执行 SQL](../sql/) 章节用程序执行创建主题的 SQL：`CREATE TOPIC IF NOT EXISTS topic_meters AS SELECT ts, current, voltage, phase, groupid, location FROM meters`
+请用 TDengine CLI 或者 参考 [执行 SQL](../sql/) 章节用程序执行创建主题的 SQL：`CREATE TOPIC IF NOT EXISTS topic_meters AS SELECT ts, current, voltage, phase, groupid, location FROM meters`
上述 SQL 将创建一个名为 topic_meters 的订阅。使用该订阅所获取的消息中的每条记录都由此查询语句 `SELECT ts, current, voltage, phase, groupid, location FROM meters` 所选择的列组成。
@@ -43,6 +43,9 @@ TDengine 消费者的概念跟 Kafka 类似，消费者通过订阅主题来接
| `enable.replay` | boolean | 是否开启数据回放功能 | 默认关闭 |
| `session.timeout.ms` | integer | consumer 心跳丢失后超时时间，超时后会触发 rebalance 逻辑，成功后该 consumer 会被删除（从3.3.3.0版本开始支持） | 默认值为 12000，取值范围 [6000， 1800000] |
| `max.poll.interval.ms` | integer | consumer poll 拉取数据间隔的最长时间，超过该时间，会认为该 consumer 离线，触发rebalance 逻辑，成功后该 consumer 会被删除（从3.3.3.0版本开始支持） | 默认值为 300000，[1000，INT32_MAX] |
+| `fetch.max.wait.ms` | integer | 服务端单次返回数据的最大耗时（从3.3.6.0版本开始支持） | 默认值为 1000，[1，INT32_MAX] |
+| `min.poll.rows` | integer | 服务端单次返回数据的最小条数（从3.3.6.0版本开始支持） | 默认值为 4096，[1，INT32_MAX] |
+| `msg.consume.rawdata` | integer | 消费数据时拉取数据类型为二进制类型，不可做解析操作，内部参数，只用于 taosx 数据迁移（从3.3.6.0版本开始支持） | 默认值为 0 表示不起效， 非 0 为 起效 |
下面是各语言连接器创建参数：
diff --git a/docs/zh/07-develop/index.md b/docs/zh/07-develop/index.md
index 4759027344..c870c09f42 100644
--- a/docs/zh/07-develop/index.md
+++ b/docs/zh/07-develop/index.md
@@ -5,19 +5,19 @@ description: 让开发者能够快速上手的指南
开发一个应用，如果你准备采用 TDengine 作为时序数据处理的工具，那么有如下几个事情要做：
-1. 确定应用到 TDengine 的连接方式。无论你使用何种编程语言，你总是可以使用 REST 接口, 但也可以使用每种编程语言独有的连接器进行方便的连接。
+1. 确定应用到 TDengine 的连接方式。无论你使用何种编程语言，你总是可以使用 REST 接口，但也可以使用每种编程语言独有的连接器进行方便的连接。
2. 根据自己的应用场景，确定数据模型。根据数据特征，决定建立一个还是多个库；分清静态标签、采集量，建立正确的超级表，建立子表。
3. 决定插入数据的方式。TDengine 支持使用标准的 SQL 写入，但同时也支持 Schemaless 模式写入，这样不用手工建表，可以将数据直接写入。
4. 根据业务要求，看需要撰写哪些 SQL 查询语句。
-5. 如果你要基于时序数据做轻量级的实时统计分析，包括各种监测看板，那么建议你采用 TDengine 3.0 的流式计算功能，而不用额外部署 Spark, Flink 等复杂的流式计算系统。
+5. 如果你要基于时序数据做轻量级的实时统计分析，包括各种监测看板，那么建议你采用 TDengine 3.0 的流式计算功能，而不用额外部署 Spark、Flink 等复杂的流式计算系统。
6. 如果你的应用有模块需要消费插入的数据，希望有新的数据插入时，就能获取通知，那么建议你采用 TDengine 提供的数据订阅功能，而无需专门部署 Kafka 或其他消息队列软件。
7. 在很多场景下（如车辆管理），应用需要获取每个数据采集点的最新状态，那么建议你采用 TDengine 的 Cache 功能，而不用单独部署 Redis 等缓存软件。
8. 如果你发现 TDengine 的函数无法满足你的要求，那么你可以使用用户自定义函数（UDF）来解决问题。
本部分内容就是按照上述顺序组织的。为便于理解，TDengine 为每个功能和每个支持的编程语言都提供了示例代码，位于 [示例代码](https://github.com/taosdata/TDengine/tree/main/docs/examples)。所有示例代码都会有 CI 保证正确性，脚本位于 [示例代码 CI](https://github.com/taosdata/TDengine/tree/main/tests/docs-examples-test)。
-如果你希望深入了解 SQL 的使用，需要查看[SQL 手册](../reference/taos-sql/)。如果想更深入地了解各连接器的使用，请阅读[连接器参考指南](../reference/connector/)。如果还希望想将 TDengine 与第三方系统集成起来，比如 Grafana, 请参考[第三方工具](../third-party/)。
+如果你希望深入了解 SQL 的使用，需要查看 [SQL 手册](../reference/taos-sql/)。如果想更深入地了解各连接器的使用，请阅读 [连接器参考指南](../reference/connector/)。如果还希望想将 TDengine 与第三方系统集成起来，比如 Grafana，请参考 [第三方工具](../third-party/)。
-如果在开发过程中遇到任何问题，请点击每个页面下方的["反馈问题"](https://github.com/taosdata/TDengine/issues/new/choose), 在 GitHub 上直接递交 Issue。
+如果在开发过程中遇到任何问题，请点击每个页面下方的 [反馈问题](https://github.com/taosdata/TDengine/issues/new/choose)，在 GitHub 上直接递交 Issue。
```mdx-code-block
import DocCardList from '@theme/DocCardList';
diff --git a/docs/zh/08-operation/02-planning.md b/docs/zh/08-operation/02-planning.md
index 89f4c42fe8..e07d45fb68 100644
--- a/docs/zh/08-operation/02-planning.md
+++ b/docs/zh/08-operation/02-planning.md
@@ -87,7 +87,7 @@ RawDataSize = numOfTables × rowSizePerTable × rowsPerTable
**技巧** 如何估算 TDengine 压缩率
```text
用户可以利用性能测试工具 taosBenchmark 来评估 TDengine 的数据压缩效果。通过使用 -f 选项指定写入配置文件，taosBenchmark 可以将指定数量的 CSV 样例数据写入指定的库参数和表结构中。
-在完成数据写入后，用户可以在 taos shell 中执行 flush database 命令，将所有数据强制写入硬盘。接着，通过 Linux 操作系统的 du 命令获取指定 vnode 的数据文件夹大小。最后，将原始数据大小除以实际存储的数据大小，即可计算出真实的压缩率。
+在完成数据写入后，用户可以在 TDengine CLI 中执行 flush database 命令，将所有数据强制写入硬盘。接着，通过 Linux 操作系统的 du 命令获取指定 vnode 的数据文件夹大小。最后，将原始数据大小除以实际存储的数据大小，即可计算出真实的压缩率。
```
通过如下命令可以获得 TDengine 占用的存储空间。
diff --git a/docs/zh/08-operation/03-deployment.md b/docs/zh/08-operation/03-deployment.md
index e549e8613d..0e59be5f55 100644
--- a/docs/zh/08-operation/03-deployment.md
+++ b/docs/zh/08-operation/03-deployment.md
@@ -58,7 +58,7 @@ serverPort 6030
#### 5. 启动
-按照前述步骤启动第 1 个 dnode，例如 h1.taosdata.com。接着在终端中执行 taos，启动 TDengine 的 CLI 程序 taos，并在其中执行 show dnodes 命令，以查看当前集群中的所有 dnode 信息。
+按照前述步骤启动第 1 个 dnode，例如 h1.taosdata.com。接着在终端中执行 taos，启动 TDengine CLI 程序 taos，并在其中执行 show dnodes 命令，以查看当前集群中的所有 dnode 信息。
```shell
taos> show dnodes;
@@ -71,7 +71,7 @@ taos> show dnodes;
#### 6. 添加 dnode
-按照前述步骤，在每个物理节点启动 taosd。每个 dnode 都需要在 taos.cfg 文件中将 firstEp 参数配置为新建集群首个节点的 endpoint，在本例中是 h1.taosdata.com:6030。在第 1 个 dnode 所在机器，在终端中运行 taos，打开 TDengine 的 CLI 程序 taos，然后登录TDengine 集群，执行如下 SQL。
+按照前述步骤，在每个物理节点启动 taosd。每个 dnode 都需要在 taos.cfg 文件中将 firstEp 参数配置为新建集群首个节点的 endpoint，在本例中是 h1.taosdata.com:6030。在第 1 个 dnode 所在机器，在终端中运行 taos，打开 TDengine CLI 程序 taos，然后登录TDengine 集群，执行如下 SQL。
```shell
create dnode "h2.taosdata.com:6030"
@@ -86,13 +86,13 @@ show dnodes;
在日志中，请确认输出的 dnode 的 fqdn 和端口是否与你刚刚尝试添加的 endpoint 一致。如果不一致，请修正为正确的 endpoint。遵循上述步骤，你可以持续地将新的 dnode 逐个加入集群，从而扩展集群规模并提高整体性能。确保在添加新节点时遵循正确的流程，这有助于维持集群的稳定性和可靠性。
**Tips**
-- 任何已经加入集群的 dnode 都可以作为后续待加入节点的 firstEp。firstEp 参数仅仅在该 dnode 首次加入集群时起作用，加入集群后，该 dnode 会保存最新的 mnode 的 endpoint 列表，后续不再依赖这个参数。之后配置文件中的 firstEp 参数主要用于客户端连接，如果没有为 TDengine 的 CLI 设置参数，则默认连接由 firstEp 指定的节点。
+- 任何已经加入集群的 dnode 都可以作为后续待加入节点的 firstEp。firstEp 参数仅仅在该 dnode 首次加入集群时起作用，加入集群后，该 dnode 会保存最新的 mnode 的 endpoint 列表，后续不再依赖这个参数。之后配置文件中的 firstEp 参数主要用于客户端连接，如果没有为 TDengine CLI 设置参数，则默认连接由 firstEp 指定的节点。
- 两个没有配置 firstEp 参数的 dnode 在启动后会独立运行。这时无法将其中一个dnode 加入另外一个 dnode，形成集群。
- TDengine 不允许将两个独立的集群合并成新的集群。
#### 7. 添加 mnode
-在创建 TDengine 集群时，首个 dnode 将自动成为集群的 mnode，负责集群的管理和协调工作。为了实现 mnode 的高可用性，后续添加的 dnode 需要手动创建 mnode。请注意，一个集群最多允许创建 3 个 mnode，且每个 dnode 上只能创建一个 mnode。当集群中的 dnode 数量达到或超过 3 个时，你可以为现有集群创建 mnode。在第 1个 dnode 中，首先通过 TDengine 的 CLI 程序 taos 登录 TDengine，然后执行如下 SQL。
+在创建 TDengine 集群时，首个 dnode 将自动成为集群的 mnode，负责集群的管理和协调工作。为了实现 mnode 的高可用性，后续添加的 dnode 需要手动创建 mnode。请注意，一个集群最多允许创建 3 个 mnode，且每个 dnode 上只能创建一个 mnode。当集群中的 dnode 数量达到或超过 3 个时，你可以为现有集群创建 mnode。在第 1个 dnode 中，首先通过 TDengine CLI 程序 taos 登录 TDengine，然后执行如下 SQL。
```shell
create mnode on dnode
diff --git a/docs/zh/08-operation/06-gui.md b/docs/zh/08-operation/06-gui.md
index cdf19b3ac1..8f60c5758d 100644
--- a/docs/zh/08-operation/06-gui.md
+++ b/docs/zh/08-operation/06-gui.md
@@ -203,7 +203,7 @@ toc_max_heading_level: 4
通过 “工具” 页面，用户可以了解如下 TDengine 周边工具的使用方法。
- TDengine CLI。
- taosBenchmark。
-- taosDump。
+- taosdump。
- TDengine 与 BI 工具的集成，例如 Google Data Studio、Power BI、永洪 BI 等。
- TDengine 与 Grafana、Seeq 的集成。
diff --git a/docs/zh/08-operation/09-backup.md b/docs/zh/08-operation/09-backup.md
index 5b02b4fa55..fbc2e612e1 100644
--- a/docs/zh/08-operation/09-backup.md
+++ b/docs/zh/08-operation/09-backup.md
@@ -61,7 +61,7 @@ TDengine Enterprise 的备份和恢复功能包括以下几个概念：
## 2.2. 数据备份
-通过浏览器访问 taosExplorer 服务，访问地址通常为 TDengine 集群所在 IP 地址的端口 6060，如 http://localhost:6060。 在
+通过浏览器访问 taosExplorer 服务，访问地址通常为 TDengine 集群所在 IP 地址的端口 6060，如 `http://localhost:6060`。 在
taosExplorer 服务页面中，进入“系统管理 - 备份”页面，在“备份计划”标签页下，点击“创建备份计划”，填写备份计划的相关信息。
需要填写的信息包括：
diff --git a/docs/zh/08-operation/10-disaster.md b/docs/zh/08-operation/10-disaster.md
index b274e1373b..96f9822725 100644
--- a/docs/zh/08-operation/10-disaster.md
+++ b/docs/zh/08-operation/10-disaster.md
@@ -23,7 +23,7 @@ TDengine 支持 WAL 机制，实现数据的容错能力，保证数据的高可
- 第 1 步，在集群 A 中创建一个数据库 db1，并向该数据库持续写入数据。
-- 第 2 步， 通过 Web 浏览器访问集群 A 的 taosExplorer 服务， 访问地址通常 为TDengine 集群所在 IP 地址的端口 6060，如 http://localhost:6060。
+- 第 2 步， 通过 Web 浏览器访问集群 A 的 taosExplorer 服务， 访问地址通常 为TDengine 集群所在 IP 地址的端口 6060，如 `http://localhost:6060`。
- 第 3 步，访问 TDengine 集群 B，创建一个与集群 A 中数据库 db1 参数配置相同的数据库 db2。
diff --git a/docs/zh/10-third-party/01-collection/12-flink.md b/docs/zh/10-third-party/01-collection/12-flink.md
index d74f6cad6d..054d7288a5 100644
--- a/docs/zh/10-third-party/01-collection/12-flink.md
+++ b/docs/zh/10-third-party/01-collection/12-flink.md
@@ -24,6 +24,7 @@ Flink Connector 支持所有能运行 Flink 1.19 及以上版本的平台。
## 版本历史
| Flink Connector 版本 | 主要变化 | TDengine 版本 |
| ------------------| ------------------------------------ | ---------------- |
+| 2.1.0 | 修复不同数据源varchar类型写入问题| - |
| 2.0.2 | Table Sink 支持 RowKind.UPDATE_BEFORE、RowKind.UPDATE_AFTER 和 RowKind.DELETE 类型| - |
| 2.0.1 | Sink 支持对所有继承自 RowData 并已实现的类型进行数据写入| - |
| 2.0.0 | 1. 支持 SQL 查询 TDengine 数据库中的数据
2. 支持 CDC 订阅 TDengine 数据库中的数据
3. 支持 Table SQL 方式读取和写入 TDengine 数据库| 3.3.5.1 及以上版本 |
@@ -84,7 +85,8 @@ TDengine 目前支持时间戳、数字、字符、布尔类型，与 Flink RowD
| SMALLINT | Short |
| TINYINT | Byte |
| BOOL | Boolean |
-| BINARY | byte[] |
+| VARCHAR | StringData |
+| BINARY | StringData |
| NCHAR | StringData |
| JSON | StringData |
| VARBINARY | byte[] |
@@ -113,7 +115,7 @@ env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.AT_LEAST_ONCE);
com.taosdata.flink
flink-connector-tdengine
- 2.0.2
+ 2.1.0
```
diff --git a/docs/zh/10-third-party/05-bi/03-powerbi.md b/docs/zh/10-third-party/05-bi/03-powerbi.md
index bd51b5591d..b864fa2835 100644
--- a/docs/zh/10-third-party/05-bi/03-powerbi.md
+++ b/docs/zh/10-third-party/05-bi/03-powerbi.md
@@ -23,12 +23,12 @@ Power BI是由Microsoft提供的一种商业分析工具。通过配置使用ODB
第3步，在“选择您想为其安装数据源的驱动程序”列表中选择“TDengine”，点击“完成”按钮，进入TDengine ODBC数据源配置页面。填写如下必要信息。
- DSN：数据源名称，必填，比如“MyTDengine”。
- 连接类型：勾选“WebSocket”复选框。
- - URL：ODBC 数据源 URL，必填，比如“http://127.0.0.1:6041”。
+ - URL：ODBC 数据源 URL，必填，比如`http://127.0.0.1:6041`。
- 数据库：表示需要连接的数据库，可选，比如“test”。
- 用户名：输入用户名，如果不填，默认为“root”。
- 密码：输入用户密码，如果不填，默认为“taosdata”。
-第4步，点击“测试连接”按钮，测试连接情况，如果成功连接，则会提示“成功连接到http://127.0.0.1:6041”。
+第4步，点击“测试连接”按钮，测试连接情况，如果成功连接，则会提示“成功连接到`http://127.0.0.1:6041`”。
第5步，点击“确定”按钮，即可保存配置并退出。
## 导入TDengine数据到Power BI
diff --git a/docs/zh/10-third-party/05-bi/12-tableau.md b/docs/zh/10-third-party/05-bi/12-tableau.md
index f14e8c1594..0478b3f7d5 100644
--- a/docs/zh/10-third-party/05-bi/12-tableau.md
+++ b/docs/zh/10-third-party/05-bi/12-tableau.md
@@ -3,33 +3,37 @@ sidebar_label: Tableau
title: 与 Tableau 集成
---
-Tableau 是一款知名的商业智能工具，它支持多种数据源，可方便地连接、导入和整合数据。并且可以通过直观的操作界面，让用户创建丰富多样的可视化图表，并具备强大的分析和筛选功能，为数据决策提供有力支持。
+Tableau 是一款知名的商业智能工具，它支持多种数据源，可方便地连接、导入和整合数据。并且可以通过直观的操作界面，让用户创建丰富多样的可视化图表，并具备强大的分析和筛选功能，为数据决策提供有力支持。用户可通过 TDengine ODBC Connector 将标签数据、原始时序数据或者经时间聚合后的时序数据从 TDengine 导入到 Tableau，用以制作报表或仪表盘，且整个过程无需编写任何代码。
## 前置条件
准备以下环境：
-- TDengine 3.3.5.4以上版本集群已部署并正常运行（企业及社区版均可）
-- taosAdapter 能够正常运行。详细参考 [taosAdapter 使用手册](../../../reference/components/taosadapter)
-- Tableau 桌面版安装并运行（如未安装，请下载并安装 Windows 操作系统 32/64 位 [Tableau 桌面版](https://www.tableau.com/products/desktop/download) ）。安装 Tableau 桌面版请参考 [官方文档](https://www.tableau.com)。
-- ODBC 驱动安装成功。详细参考[安装 ODBC 驱动](../../../reference/connector/odbc/#安装)
-- ODBC 数据源配置成功。详细参考[配置ODBC数据源](../../../reference/connector/odbc/#配置数据源)
+- TDengine 3.3.5.4 以上版本集群已部署并正常运行（企业及社区版均可）
+- taosAdapter 能够正常运行。详细参考 [taosAdapter 参考手册](../../../reference/components/taosadapter)
+- Tableau 桌面版安装并运行（如未安装，请下载并安装 Windows 操作系统 64 位 [Tableau 桌面版](https://www.tableau.com/products/desktop/download) ）。安装 Tableau 桌面版请参考 [官方文档](https://www.tableau.com)。
+- 从TDengine官网下载最新的Windows操作系统X64客户端驱动程序，并进行安装。详细参考 [安装 ODBC 驱动](../../../reference/connector/odbc/#安装)。
-## 加载和分析 TDengine 数据
-**第 1 步**，在 Windows 系统环境下启动 Tableau，之后在其连接页面中搜索 “ODBC”，并选择 “其他数据库 (ODBC)”。
+## 配置数据源
-**第 2 步**，点击 DNS 单选框，接着选择已配置好的数据源(MyTDengine)，然后点击连接按钮。待连接成功后，删除字符串附加部分的内容，最后点击登录按钮即可。
+**第 1 步**，在Windows操作系统的开始菜单中搜索并打开“ODBC数据源（64位）”管理工具并进行配置。详细参考[配置ODBC数据源](../../../reference/connector/odbc/#配置数据源)。
+
+**第 2 步**，在 Windows 系统环境下启动 Tableau，之后在其连接页面中搜索 “ODBC”，并选择 “其他数据库 (ODBC)”。
+
+**第 3 步**，点击 `DSN` 单选框，接着选择已配置好的数据源(MyTDengine)，然后点击`连接`按钮。待连接成功后，删除字符串附加部分的内容，最后点击`登录`按钮即可。
-**第 3 步**，在弹出的工作簿页面中，会显示已连接的数据源。点击数据库的下拉列表，会显示需要进行数据分析的数据库。在此基础上，点击表选项中的查找按钮，即可将该数据库下的所有表显示出来。然后，拖动需要分析的表到右侧区域，即可显示出表结构。
+## 数据分析
+
+**第 1 步**，在工作簿页面中，选择已连接的数据源。点击数据库的下拉列表，会显示需要进行数据分析的数据库。在此基础上，点击表选项中的查找按钮，即可将该数据库下的所有表显示出来。然后，拖动需要分析的表到右侧区域，即可显示出表结构。
-**第 4 步**，点击下方的"立即更新"按钮，即可将表中的数据展示出来。
+**第 2 步**，点击下方的"立即更新"按钮，即可将表中的数据展示出来。
-**第 5 步**，点击窗口下方的"工作表"，弹出数据分析窗口， 并展示分析表的所有字段,将字段拖动到行列即可展示出图表。
+**第 3 步**，点击窗口下方的"工作表"，弹出数据分析窗口， 并展示分析表的所有字段,将字段拖动到行列即可展示出图表。
\ No newline at end of file
diff --git a/docs/zh/10-third-party/05-bi/13-excel.md b/docs/zh/10-third-party/05-bi/13-excel.md
new file mode 100644
index 0000000000..442d0175b4
--- /dev/null
+++ b/docs/zh/10-third-party/05-bi/13-excel.md
@@ -0,0 +1,40 @@
+---
+sidebar_label: Excel
+title: 与 Excel 集成
+---
+
+通过配置使用 ODBC 连接器，Excel 可以快速访问 TDengine 的数据。用户可以将标签数据、原始时序数据或按时间聚合后的时序数据从 TDengine 导入到 Excel，用以制作报表整个过程不需要任何代码编写过程。
+
+## 前置条件
+
+准备以下环境：
+- TDengine 3.3.5.7 以上版本集群已部署并正常运行（企业及社区版均可）。
+- taosAdapter 能够正常运行，详细参考 [taosAdapter 参考手册](../../../reference/components/taosadapter)。
+- Excel 安装并运行, 如未安装，请下载并安装, 具体操作请参考 Microsoft 官方文档。
+- 从 TDengine 官网下载最新的 Windows 操作系统 X64 客户端驱动程序并进行安装，详细参考 [安装 ODBC 驱动](../../../reference/connector/odbc/#安装)。
+
+## 配置数据源
+
+**第 1 步**，在 Windows 操作系统的开始菜单中搜索并打开【ODBC数据源（64位）】管理工具并进行配置。详细参考 [配置ODBC数据源](../../../reference/connector/odbc/#配置数据源)。
+
+**第 2 步**，在 Windows 系统环境下启动 Excel，之后选择【数据】->【获取数据】->【自其他源】->【从ODBC】。
+
+
+
+**第 3 步**，在弹出窗口的【数据源名称(DSN)】下拉列表中选择需要连接的数据源后，点击【确定】按钮。
+
+
+
+**第 4 步**，输入 TDengine 的用户名密码。
+
+
+
+**第 5 步**，在弹出的【导航器】对话框中，选择要加载的库表, 并点击【加载】完成数据加载。
+
+
+
+## 数据分析
+
+选中导入的数据，在【插入】选项卡中选择柱状图，并且在右侧的【数据透视图】中配置数据字段。
+
+
diff --git a/docs/zh/10-third-party/05-bi/excel/odbc-config.jpg b/docs/zh/10-third-party/05-bi/excel/odbc-config.jpg
new file mode 100644
index 0000000000..58d3395e1b
Binary files /dev/null and b/docs/zh/10-third-party/05-bi/excel/odbc-config.jpg differ
diff --git a/docs/zh/10-third-party/05-bi/excel/odbc-data.jpg b/docs/zh/10-third-party/05-bi/excel/odbc-data.jpg
new file mode 100644
index 0000000000..abe54bba2f
Binary files /dev/null and b/docs/zh/10-third-party/05-bi/excel/odbc-data.jpg differ
diff --git a/docs/zh/10-third-party/05-bi/excel/odbc-load.jpg b/docs/zh/10-third-party/05-bi/excel/odbc-load.jpg
new file mode 100644
index 0000000000..87d5958d35
Binary files /dev/null and b/docs/zh/10-third-party/05-bi/excel/odbc-load.jpg differ
diff --git a/docs/zh/10-third-party/05-bi/excel/odbc-menu.jpg b/docs/zh/10-third-party/05-bi/excel/odbc-menu.jpg
new file mode 100644
index 0000000000..eed00a8a75
Binary files /dev/null and b/docs/zh/10-third-party/05-bi/excel/odbc-menu.jpg differ
diff --git a/docs/zh/10-third-party/05-bi/excel/odbc-select.jpg b/docs/zh/10-third-party/05-bi/excel/odbc-select.jpg
new file mode 100644
index 0000000000..a6b66f4ea8
Binary files /dev/null and b/docs/zh/10-third-party/05-bi/excel/odbc-select.jpg differ
diff --git a/docs/zh/14-reference/01-components/02-taosc.md b/docs/zh/14-reference/01-components/02-taosc.md
index 09235557c8..262027bff6 100755
--- a/docs/zh/14-reference/01-components/02-taosc.md
+++ b/docs/zh/14-reference/01-components/02-taosc.md
@@ -490,7 +490,7 @@ TDengine 客户端驱动提供了应用编程所需要的全部 API，并且在
- 支持版本：从 v3.3.4.3 版本开始引入
#### bypassFlag
-- 说明：配置文件所在目录
+- 说明：用于短路测试 `内部参数`
- 类型：整数；
- 取值范围：0：正常写入，1：写入消息在 taos 客户端发送 RPC 消息前返回，2：写入消息在 taosd 服务端收到 RPC 消息后返回，4：写入消息在 taosd 服务端写入内存缓存前返回，8：写入消息在 taosd 服务端数据落盘前返回
- 默认值：0
diff --git a/docs/zh/14-reference/02-tools/08-taos-cli.md b/docs/zh/14-reference/02-tools/08-taos-cli.md
index 2964b7c184..ed1eca15a6 100644
--- a/docs/zh/14-reference/02-tools/08-taos-cli.md
+++ b/docs/zh/14-reference/02-tools/08-taos-cli.md
@@ -1,6 +1,6 @@
---
title: TDengine CLI 参考手册
-sidebar_label: taos
+sidebar_label: TDengine CLI
toc_max_heading_level: 4
---
@@ -27,102 +27,103 @@ taos>
```
进入 TDengine CLI 后，可执行各种 SQL 语句，包括插入、查询以及各种管理命令。
-退出 TDengine CLI， 执行 `q` 或 `quit` 或 `exit` 回车即可
+退出 TDengine CLI， 执行 `q` 或 `quit` 或 `exit` 回车即可。
```shell
taos> quit
```
## 命令行参数
-### 常用参数
+### 基础参数
可通过配置命令行参数来改变 TDengine CLI 的行为。以下为常用的几个命令行参数：
-- -h HOST: 要连接的 TDengine 服务端所在服务器的 FQDN, 默认值： 127.0.0.1
-- -P PORT: 指定服务端所用端口号，默认值：6030
-- -u USER: 连接时使用的用户名，默认值：root
-- -p PASSWORD: 连接服务端时使用的密码，特殊字符如 `! & ( ) < > ; |` 需使用字符 `\` 进行转义处理, 默认值：taosdata
-- -?, --help: 打印出所有命令行参数
+- -h HOST: 要连接的 TDengine 服务端所在服务器的 FQDN, 默认值： 127.0.0.1 。
+- -P PORT: 指定服务端所用端口号，默认值：6030 。
+- -u USER: 连接时使用的用户名，默认值：root 。
+- -p PASSWORD: 连接服务端时使用的密码，特殊字符如 `! & ( ) < > ; |` 需使用字符 `\` 进行转义处理, 默认值：taosdata 。
+- -?, --help: 打印出所有命令行参数。
+- -s COMMAND: 以非交互模式执行的 SQL 命令。
-### 更多参数
+ 使用 `-s` 参数可进行非交互式执行 SQL，执行完成后退出，此模式适合在自动化脚本中使用。
+ 如以下命令连接到服务器 h1.taos.com, 执行 -s 指定的 SQL:
+ ```bash
+ taos -h my-server -s "use db; show tables;"
+ ```
-- -a AUTHSTR: 连接服务端的授权信息
-- -A: 通过用户名和密码计算授权信息
-- -B: 设置 BI 工具显示模式，设置后所有输出都遵循 BI 工具的格式进行输出
-- -c CONFIGDIR: 指定配置文件目录，Linux 环境下默认为 `/etc/taos`，该目录下的配置文件默认名称为 `taos.cfg`
-- -C: 打印 -c 指定的目录中 `taos.cfg` 的配置参数
-- -d DATABASE: 指定连接到服务端时使用的数据库
-- -E dsn: 使用 WebSocket DSN 连接云服务或者提供 WebSocket 连接的服务端
-- -f FILE: 以非交互模式执行 SQL 脚本文件。文件中一个 SQL 语句只能占一行
-- -k: 测试服务端运行状态，0: unavailable，1: network ok，2: service ok，3: service degraded，4: exiting
-- -l PKTLEN: 网络测试时使用的测试包大小
-- -n NETROLE: 网络连接测试时的测试范围，默认为 `client`, 可选值为 `client`、`server`
-- -N PKTNUM: 网络测试时使用的测试包数量
-- -r: 将时间列转化为无符号 64 位整数类型输出(即 C 语言中 uint64_t)
-- -R: 使用 RESTful 模式连接服务端
-- -s COMMAND: 以非交互模式执行的 SQL 命令
-- -t: 测试服务端启动状态，状态同-k
-- -w DISPLAYWIDTH: 客户端列显示宽度
-- -z TIMEZONE: 指定时区，默认为本地时区
-- -V: 打印出当前版本号
+- -c CONFIGDIR: 指定配置文件目录。
+
+ Linux 环境下默认为 `/etc/taos`，该目录下的配置文件默认名称为 `taos.cfg` 。
+ 使用 `-c` 参数改变 `taosc` 客户端加载配置文件的位置，客户端配置参数参考 [客户端配置](../../components/taosc) 。
+ 以下命令指定了 `taosc` 客户端加载 `/root/cfg/` 下的 `taos.cfg` 配置文件。
+ ```bash
+ taos -c /root/cfg/
+ ```
-### 非交互式执行
+### 高级参数
-使用 `-s` 参数可进行非交互式执行 SQL，执行完成后退出，此模式适合在自动化脚本中使用。
-如以下命令连接到服务器 h1.taos.com, 执行 -s 指定的 SQL:
-```bash
-taos -h h1.taos.com -s "use db; show tables;"
-```
+- -a AUTHSTR: 连接服务端的授权信息。
+- -A: 通过用户名和密码计算授权信息。
+- -B: 设置 BI 工具显示模式，设置后所有输出都遵循 BI 工具的格式进行输出。
+- -C: 打印 -c 指定的目录中 `taos.cfg` 的配置参数。
+- -d DATABASE: 指定连接到服务端时使用的数据库。
+- -E dsn: 使用 WebSocket DSN 连接云服务或者提供 WebSocket 连接的服务端。
+- -f FILE: 以非交互模式执行 SQL 脚本文件。文件中一个 SQL 语句只能占一行。
+- -k: 测试服务端运行状态，0: unavailable，1: network ok，2: service ok，3: service degraded，4: exiting 。
+- -l PKTLEN: 网络测试时使用的测试包大小。
+- -n NETROLE: 网络连接测试时的测试范围，默认为 `client`, 可选值为 `client`、`server` 。
+- -N PKTNUM: 网络测试时使用的测试包数量。
+- -r: 将时间列转化为无符号 64 位整数类型输出(即 C 语言中 uint64_t) 。
+- -R: 使用 RESTful 模式连接服务端。
+- -t: 测试服务端启动状态，状态同 -k 。
+- -w DISPLAYWIDTH: 客户端列显示宽度。
+- -z TIMEZONE: 指定时区，默认为本地时区。
+- -V: 打印出当前版本号。
-### taosc 配置文件
-使用 `-c` 参数改变 `taosc` 客户端加载配置文件的位置，客户端配置参数参考 [客户端配置](../../components/taosc)
-以下命令指定了 `taosc` 客户端加载 `/root/cfg/` 下的 `taos.cfg` 配置文件
-```bash
-taos -c /root/cfg/
-```
+## 数据导出/导入
+
+### 数据导出
+
+- 可以使用符号 “>>” 导出查询结果到某个文件中，语法为： sql 查询语句 >> ‘输出文件名’; 输出文件如果不写路径的话，将输出至当前目录下。如 `select * from d0 >> ‘/root/d0.csv’;` 将把查询结果输出到 /root/d0.csv 中。
+
+### 数据导入
+
+- 可以使用 insert into table_name file '输入文件名'，把上一步中导出的数据文件再导入到指定表中。如 `insert into d0 file '/root/d0.csv';` 表示把上面导出的数据全部再导致至 d0 表中。
## 执行 SQL 脚本
-在 TDengine CLI 里可以通过 `source` 命令来运行脚本文件中的多条 SQL 命令。
-
+在 TDengine CLI 里可以通过 `source` 命令来运行脚本文件中的多条 SQL 命令，脚本文件中多条 SQL 按行书写即可
```sql
taos> source ;
```
-## 数据导入/导出
-### 导出查询结果
+## 使用小技巧
-- 可以使用符号 “>>” 导出查询结果到某个文件中，语法为： sql 查询语句 >> ‘输出文件名’; 输出文件如果不写路径的话，将输出至当前目录下。如 select * from d0 >> ‘/root/d0.csv’; 将把查询结果输出到 /root/d0.csv 中。
+### TAB 键自动补全
-### 数据从文件导入
+- TAB 键前为空命令状态下按 TAB 键，会列出 TDengine CLI 支持的所有命令。
+- TAB 键前为空格状态下按 TAB 键，会显示此位置可以出现的所有命令词的第一个，再次按 TAB 键切为下一个。
+- TAB 键前为字符串，会搜索与此字符串前缀匹配的所有可出现命令词，并显示第一个，再次按 TAB 键切为下一个。
+- 输入反斜杠 `\` + TAB 键, 会自动补全为列显示模式命令词 `\G;` 。
-- 可以使用 insert into table_name file '输入文件名'，把上一步中导出的数据文件再导入到指定表中。如 insert into d0 file '/root/d0.csv'; 表示把上面导出的数据全部再导致至 d0 表中。
-
-## 设置字符类型显示宽度
-
-可以在 TDengine CLI 里使用如下命令调整字符显示宽度
+### 设置字符列显示宽度
+可以在 TDengine CLI 里使用如下命令调整字符串类型字段列显示宽度，默认显示宽度为 30 个字符。
+以下命令设置了显示宽度为 120 个字符：
```sql
-taos> SET MAX_BINARY_DISPLAY_WIDTH ;
+taos> SET MAX_BINARY_DISPLAY_WIDTH 120;
```
如显示的内容后面以 ... 结尾时，表示该内容已被截断，可通过本命令修改显示字符宽度以显示完整的内容。
-## TAB 键自动补全
+### 其它
-- TAB 键前为空命令状态下按 TAB 键，会列出 TDengine CLI 支持的所有命令
-- TAB 键前为空格状态下按 TAB 键，会显示此位置可以出现的所有命令词的第一个，再次按 TAB 键切为下一个
-- TAB 键前为字符串，会搜索与此字符串前缀匹配的所有可出现命令词，并显示第一个，再次按 TAB 键切为下一个
-- 输入反斜杠 `\` + TAB 键, 会自动补全为列显示模式命令词 `\G;`
-
-## 使用小技巧
-
-- 可以使用上下光标键查看历史输入的指令
-- 在 TDengine CLI 中使用 `alter user` 命令可以修改用户密码，缺省密码为 `taosdata`
-- Ctrl+C 中止正在进行中的查询
-- 执行 `RESET QUERY CACHE` 可清除本地表 Schema 的缓存
-- 批量执行 SQL 语句。可以将一系列的 TDengine CLI 命令（以英文 ; 结尾，每个 SQL 语句为一行）按行存放在文件里，在 TDengine CLI 里执行命令 `source ` 自动执行该文件里所有的 SQL 语句
+- 可以使用上下光标键查看历史输入的指令。
+- 在 TDengine CLI 中使用 `alter user` 命令可以修改用户密码，缺省密码为 `taosdata` 。
+- Ctrl+C 中止正在进行中的查询。
+- 执行 `RESET QUERY CACHE` 可清除本地表 Schema 的缓存。
+- 批量执行 SQL 语句。可以将一系列的 TDengine CLI 命令（以英文 ; 结尾，每个 SQL 语句为一行）按行存放在文件里，在 TDengine CLI 里执行命令 `source ` 自动执行该文件里所有的 SQL 语句。
## 错误代码表
在 TDengine 3.3.4.8 版本后 TDengine CLI 在返回错误信息中返回了具体错误码，用户可到 TDengine 官网错误码页面查找具体原因及解决措施，见：[错误码参考表](https://docs.taosdata.com/reference/error-code/)
diff --git a/docs/zh/14-reference/02-tools/09-taosdump.md b/docs/zh/14-reference/02-tools/09-taosdump.md
index 56f8ecc704..5435935467 100644
--- a/docs/zh/14-reference/02-tools/09-taosdump.md
+++ b/docs/zh/14-reference/02-tools/09-taosdump.md
@@ -8,19 +8,19 @@ taosdump 是为开源用户提供的 TDengine 数据备份/恢复工具，备份
## 工具获取
-taosdump 是 TDengine 服务器及客户端安装包中默认安装组件，安装后即可使用，参考 [TDengine 安装](../../../get-started/)
+taosdump 是 TDengine 服务器及客户端安装包中默认安装组件，安装后即可使用，参考 [TDengine 安装](../../../get-started/)
## 运行
taosdump 需在命令行终端中运行，运行时必须带参数，指明是备份操作或还原操作，如：
``` bash
-taosdump -h dev126 -D test -o /root/test/
+taosdump -h my-server -D test -o /root/test/
```
-以上命令表示备份主机名为 `dev126` 机器上的 `test` 数据库到 `/root/test/` 目录下
+以上命令表示备份主机名为 `my-server` 机器上的 `test` 数据库到 `/root/test/` 目录下。
``` bash
-taosdump -h dev126 -i /root/test/
+taosdump -h my-server -i /root/test/
```
-以上命令表示把 `/root/test/` 目录下之前备份的数据文件恢复到主机名为 `dev126` 的主机上
+以上命令表示把 `/root/test/` 目录下之前备份的数据文件恢复到主机名为 `my-server` 的主机上。
## 命令行参数
@@ -71,28 +71,27 @@ Usage: taosdump [OPTION...] dbname [tbname ...]
restore, please adjust the value to a smaller one
and try. The workable value is related to the
length of the row and type of table schema.
- -I, --inspect inspect avro file content and print on screen
+ -I, --inspect inspect avro file content and print on screen.
-L, --loose-mode Using loose mode if the table name and column name
use letter and number only. Default is NOT.
-n, --no-escape No escape char '`'. Default is using it.
-Q, --dot-replace Repalce dot character with underline character in
- the table name.(Version 2.5.3)
- -T, --thread-num=THREAD_NUM Number of thread for dump in file. Default is
- 8.
+ the table name.(Version 2.5.3).
+ -T, --thread-num=THREAD_NUM Number of thread for dump in file. Default is 8
-W, --rename=RENAME-LIST Rename database name with new name during
importing data. RENAME-LIST:
"db1=newDB1|db2=newDB2" means rename db1 to newDB1
- and rename db2 to newDB2 (Version 2.5.4)
+ and rename db2 to newDB2 (Version 2.5.4).
-k, --retry-count=VALUE Set the number of retry attempts for connection or
- query failures
- -z, --retry-sleep-ms=VALUE retry interval sleep time, unit ms
- -C, --cloud=CLOUD_DSN specify a DSN to access TDengine cloud service
- -R, --restful Use RESTful interface to connect TDengine
+ query failures.
+ -z, --retry-sleep-ms=VALUE retry interval sleep time, unit ms.
+ -C, --cloud=CLOUD_DSN specify a DSN to access TDengine cloud service.
+ -R, --restful Use RESTful interface to connect TDengine.
-t, --timeout=SECONDS The timeout seconds for websocket to interact.
-g, --debug Print debug info.
- -?, --help Give this help list
- --usage Give a short usage message
- -V, --version Print program version
+ -?, --help Give this help list.
+ --usage Give a short usage message.
+ -V, --version Print program version.
Mandatory or optional arguments to long options are also mandatory or optional
for any corresponding short options.
@@ -102,15 +101,15 @@ Report bugs to .
## 常用使用场景
-### taosdump 备份数据
+### 备份数据
-1. 备份所有数据库：指定 `-A` 或 `--all-databases` 参数；
-2. 备份多个指定数据库：使用 `-D db1,db2,...` 参数；
-3. 备份指定数据库中某些超级表或普通表：使用 `dbname stbname1 stbname2 tbname1 tbname2 ...` 参数，注意这种输入序列第一个参数为数据库名称，且只支持一个数据库，第二个和之后的参数为该数据库中的超级表或普通表名称，中间以空格分隔；
+1. 备份所有数据库：指定 `-A` 或 `--all-databases` 参数。
+2. 备份多个指定数据库：使用 `-D db1,db2,...` 参数。
+3. 备份指定数据库中某些超级表或普通表：使用 `dbname stbname1 stbname2 tbname1 tbname2 ...` 参数，注意这种输入序列第一个参数为数据库名称，且只支持一个数据库，第二个和之后的参数为该数据库中的超级表或普通表名称，中间以空格分隔。
4. 备份系统 log 库：TDengine 集群通常会包含一个系统数据库，名为 `log`，这个数据库内的数据为 TDengine 自我运行的数据，taosdump 默认不会对 log 库进行备份。如果有特定需求对 log 库进行备份，可以使用 `-a` 或 `--allow-sys` 命令行参数。
-5. “宽容”模式备份：taosdump 1.4.1 之后的版本提供 `-n` 参数和 `-L` 参数，用于备份数据时不使用转义字符和“宽容”模式，可以在表名、列名、标签名没使用转义字符的情况下减少备份数据时间和备份数据占用空间。如果不确定符合使用 `-n` 和 `-L` 条件时请使用默认参数进行“严格”模式进行备份。转义字符的说明请参考[官方文档](../../taos-sql/escape)。
+5. “宽容”模式备份：taosdump 1.4.1 之后的版本提供 `-n` 参数和 `-L` 参数，用于备份数据时不使用转义字符和“宽容”模式，可以在表名、列名、标签名没使用转义字符的情况下减少备份数据时间和备份数据占用空间。如果不确定符合使用 `-n` 和 `-L` 条件时请使用默认参数进行“严格”模式进行备份。转义字符的说明请参考 [官方文档](../../taos-sql/escape)
6. `-o` 参数指定的目录下如果已存在备份文件，为防止数据被覆盖，taosdump 会报错并退出，请更换其它空目录或清空原来数据后再备份。
-7. 目前 taosdump 不支持数据断点继备功能，一旦数据备份中断，需要从头开始。如果备份需要很长时间，建议使用（-S -E 选项）指定开始/结束时间进行分段备份的方法，
+7. 目前 taosdump 不支持数据断点继备功能，一旦数据备份中断，需要从头开始。如果备份需要很长时间，建议使用（-S -E 选项）指定开始/结束时间进行分段备份的方法。
:::tip
- taosdump 1.4.1 之后的版本提供 `-I` 参数，用于解析 avro 文件 schema 和数据，如果指定 `-s` 参数将只解析 schema。
@@ -120,13 +119,13 @@ Report bugs to .
:::
-### taosdump 恢复数据
+### 恢复数据
- 恢复指定路径下的数据文件：使用 `-i` 参数加上数据文件所在路径。如前面提及，不应该使用同一个目录备份不同数据集合，也不应该在同一路径多次备份同一数据集，否则备份数据会造成覆盖或多次备份。
- taosdump 支持数据恢复至新数据库名下，参数是 -W, 详细见命令行参数说明。
:::tip
-taosdump 内部使用 TDengine stmt binding API 进行恢复数据的写入，为提高数据恢复性能，目前使用 16384 为一次写入批次。如果备份数据中有比较多列数据，可能会导致产生 "WAL size exceeds limit" 错误，此时可以通过使用 `-B` 参数调整为一个更小的值进行尝试。
+taosdump 内部使用 TDengine stmt binding API 进行恢复数据的写入，为提高数据恢复性能，目前使用 16384 为一次写入批次。如果备份数据中有较多列数据，可能会导致产生 "WAL size exceeds limit" 错误，此时可以通过使用 `-B` 参数调整为一个更小的值进行尝试。
:::
\ No newline at end of file
diff --git a/docs/zh/14-reference/02-tools/10-taosbenchmark.md b/docs/zh/14-reference/02-tools/10-taosbenchmark.md
index c95c76a837..67a823ad5e 100644
--- a/docs/zh/14-reference/02-tools/10-taosbenchmark.md
+++ b/docs/zh/14-reference/02-tools/10-taosbenchmark.md
@@ -12,18 +12,13 @@ taosBenchmark 是 TDengine 服务器及客户端安装包中默认安装组件
## 运行
-### 运行方式
+taosBenchmark 支持无参数、命令行、配置文件三种运行模式，`命令行` 为 `配置文件` 功能子集，两者同时使用时，以命令行方式优先。
-taosBenchmark 支持三种运行模式:
-- 无参数模式
-- 命令行模式
-- JSON 配置文件模式
-`命令行方式` 为 `JSON 配置文件方式` 功能子集，两者都使用时，命令行方式优先。
+:::tip
+在运行 taosBenchmark 之前要确保 TDengine 集群已经在正确运行。
+:::
-
-**在运行 taosBenchmark 之前要确保 TDengine 集群已经在正确运行。**
-
-### 无命令行参数运行
+### 无参数模式
```bash
taosBenchmark
@@ -32,9 +27,9 @@ taosBenchmark
在无参数运行时，taosBenchmark 默认连接 `/etc/taos/taos.cfg` 中指定的 TDengine 集群。
连接成功后，会默认创建智能电表示例数据库 test，创建超级表 meters, 创建子表 1 万，每子写入数据 1 万条，若 test 库已存在，默认会先删再建。
-### 使用命令行参数运行
+### 命令行模式
-命令行支持的参数为写入功能中使用较为频繁的参数，查询与订阅功能不支持命令行方式
+命令行支持的参数为写入功能中使用较为频繁的参数，查询与订阅功能不支持命令行方式。
示例：
```bash
taosBenchmark -d db -t 100 -n 1000 -T 4 -I stmt -y
@@ -42,51 +37,14 @@ taosBenchmark -d db -t 100 -n 1000 -T 4 -I stmt -y
此命令表示使用 `taosBenchmark` 将创建一个名为 `db` 的数据库，并建立默认超级表 `meters`，子表 100 ，使用参数绑定(stmt)方式为每张子表写入 1000 条记录。
-### 使用配置文件运行
+### 配置文件模式
-配置文件方式运行提供了全部功能，所有命令行参数都可以在配置文件中配置运行
+以 JSON 配置文件方式运行提供了全部功能，所有命令行参数都可以在配置文件中配置运行。
```bash
taosBenchmark -f
```
-**下面为支持的写入、查询、订阅三大功能的配置文件示例：**
-
-#### 写入场景 JSON 配置文件示例
-
-
-insert.json
-
-```json
-{{#include /TDengine/tools/taos-tools/example/insert.json}}
-```
-
-
-
-#### 查询场景 JSON 配置文件示例
-
-
-query.json
-
-```json
-{{#include /TDengine/tools/taos-tools/example/query.json}}
-```
-
-
-
-#### 订阅场景 JSON 配置文件示例
-
-
-tmq.json
-
-```json
-{{#include /TDengine/tools/taos-tools/example/tmq.json}}
-```
-
-
-
-查看更多 json 配置文件示例可 [点击这里](https://github.com/taosdata/TDengine/tree/main/tools/taos-tools/example)
-
## 命令行参数
| 命令行参数 | 功能说明 |
| ---------------------------- | ----------------------------------------------- |
@@ -129,60 +87,6 @@ taosBenchmark -f
| -V/--version | 显示版本信息并退出。不能与其它参数混用|
| -?/--help | 显示帮助信息并退出。不能与其它参数混用|
-
-## 输出性能指标
-
-#### 写入指标
-
-写入结束后会在最后两行输出总体性能指标，格式如下：
-``` bash
-SUCC: Spent 8.527298 (real 8.117379) seconds to insert rows: 10000000 with 8 thread(s) into test 1172704.41 (real 1231924.74) records/second
-SUCC: insert delay, min: 19.6780ms, avg: 64.9390ms, p90: 94.6900ms, p95: 105.1870ms, p99: 130.6660ms, max: 157.0830ms
-```
-第一行写入速度统计：
- - Spent: 写入总耗时，单位秒，从开始写入第一个数据开始计时到最后一条数据结束，这里表示共花了 8.527298 秒
- - real : 写入总耗时（调用引擎），此耗时已抛去测试框架准备数据时间，纯统计在引擎调用上花费的时间，示例为 8.117379 秒，8.527298 - 8.117379 = 0.409919 秒则为测试框架准备数据消耗时间
- - rows : 写入总行数，为 1000 万条数据
- - threads: 写入线程数，这里是 8 个线程同时写入
- - records/second 写入速度 = `写入总耗时`/ `写入总行数` ， 括号中 `real` 同前，表示纯引擎写入速度
-第二行单个写入延时统计：
- - min : 写入最小延时
- - avg : 写入平时延时
- - p90 : 写入延时 p90 百分位上的延时数
- - p95 : 写入延时 p95 百分位上的延时数
- - p99 : 写入延时 p99 百分位上的延时数
- - max : 写入最大延时
-通过此系列指标，可观察到写入请求延时分布情况
-
-#### 查询指标
-
-查询性能测试主要输出查询请求速度 QPS 指标, 输出格式如下：
-``` bash
-complete query with 3 threads and 10000 query delay avg: 0.002686s min: 0.001182s max: 0.012189s p90: 0.002977s p95: 0.003493s p99: 0.004645s SQL command: select ...
-INFO: Spend 26.9530 second completed total queries: 30000, the QPS of all threads: 1113.049
-```
-- 第一行表示 3 个线程每个线程执行 10000 次查询及查询请求延时百分位分布情况，`SQL command` 为测试的查询语句
-- 第二行表示查询总耗时为 26.9653 秒，每秒查询率(QPS)为：1113.049 次/秒
-- 如果在查询中设置了 `continue_if_fail` 选项为 `yes`，在最后一行中会输出失败请求个数及错误率，格式 error + 失败请求个数 (错误率)
-- QPS = 成功请求数量 / 花费时间(单位秒)
-- 错误率 = 失败请求数量 /（成功请求数量 + 失败请求数量）
-
-#### 订阅指标
-
-订阅性能测试主要输出消费者消费速度指标，输出格式如下：
-``` bash
-INFO: consumer id 0 has poll total msgs: 376, period rate: 37.592 msgs/s, total rows: 3760000, period rate: 375924.815 rows/s
-INFO: consumer id 1 has poll total msgs: 362, period rate: 36.131 msgs/s, total rows: 3620000, period rate: 361313.504 rows/s
-INFO: consumer id 2 has poll total msgs: 364, period rate: 36.378 msgs/s, total rows: 3640000, period rate: 363781.731 rows/s
-INFO: consumerId: 0, consume msgs: 1000, consume rows: 10000000
-INFO: consumerId: 1, consume msgs: 1000, consume rows: 10000000
-INFO: consumerId: 2, consume msgs: 1000, consume rows: 10000000
-INFO: Consumed total msgs: 3000, total rows: 30000000
-```
-- 1 ~ 3 行实时输出每个消费者当前的消费速度，`msgs/s` 表示消费消息个数，每个消息中包含多行数据，`rows/s` 表示按行数统计的消费速度
-- 4 ~ 6 行是测试完成后每个消费者总体统计，统计共消费了多少条消息，共计多少行
-- 第 7 行所有消费者总体统计，`msgs` 表示共消费了多少条消息， `rows` 表示共消费了多少行数据
-
## 配置文件参数
### 通用配置参数
@@ -192,28 +96,28 @@ INFO: Consumed total msgs: 3000, total rows: 30000000
- **filetype** : 功能分类，可选值为 `insert`, `query` 和 `subscribe`。分别对应插入、查询和订阅功能。每个配置文件中只能指定其中之一。
- **cfgdir** : TDengine 客户端配置文件所在的目录，默认路径是 /etc/taos 。
-- **host** : 指定要连接的 TDengine 服务端的 FQDN，默认值为 localhost。
+- **host** : 指定要连接的 TDengine 服务端的 FQDN，默认值为 localhost 。
-- **port** : 要连接的 TDengine 服务器的端口号，默认值为 6030。
+- **port** : 要连接的 TDengine 服务器的端口号，默认值为 6030 。
-- **user** : 用于连接 TDengine 服务端的用户名，默认为 root。
+- **user** : 用于连接 TDengine 服务端的用户名，默认为 root 。
- **password** : 用于连接 TDengine 服务端的密码，默认值为 taosdata。
-### 插入场景配置参数
+### 写入配置参数
-插入场景下 `filetype` 必须设置为 `insert`，该参数及其它通用参数详见[通用配置参数](#通用配置参数)
+写入场景下 `filetype` 必须设置为 `insert`，该参数及其它通用参数详见 [通用配置参数](#通用配置参数)
- **keep_trying** : 失败后进行重试的次数，默认不重试。需使用 v3.0.9 以上版本。
- **trying_interval** : 失败重试间隔时间，单位为毫秒，仅在 keep_trying 指定重试后有效。需使用 v3.0.9 以上版本。
-- **childtable_from 和 childtable_to** : 指定写入子表范围，开闭区间为 [childtable_from, childtable_to).
+- **childtable_from 和 childtable_to** : 指定写入子表范围，开闭区间为 [childtable_from, childtable_to) 。
-- **continue_if_fail** : 允许用户定义失败后行为
+- **continue_if_fail** : 允许用户定义失败后行为。
- “continue_if_fail”: “no”, 失败 taosBenchmark 自动退出，默认行为
- “continue_if_fail”: “yes”, 失败 taosBenchmark 警告用户，并继续写入
- “continue_if_fail”: “smart”, 如果子表不存在失败，taosBenchmark 会建立子表并继续写入
+ “continue_if_fail”: “no”, 失败 taosBenchmark 自动退出，默认行为。
+ “continue_if_fail”: “yes”, 失败 taosBenchmark 警告用户，并继续写入。
+ “continue_if_fail”: “smart”, 如果子表不存在失败，taosBenchmark 会建立子表并继续写入。
#### 数据库相关
@@ -221,7 +125,7 @@ INFO: Consumed total msgs: 3000, total rows: 30000000
- **name** : 数据库名。
-- **drop** : 数据库已存在时是否删除，可选项为 "yes" 或 "no", 默认为 “yes”
+- **drop** : 数据库已存在时是否删除，可选项为 "yes" 或 "no", 默认为 “yes” 。
#### 超级表相关
@@ -229,13 +133,13 @@ INFO: Consumed total msgs: 3000, total rows: 30000000
- **name**: 超级表名，必须配置，没有默认值。
-- **child_table_exists** : 子表是否已经存在，默认值为 "no"，可选值为 "yes" 或 "no"。
+- **child_table_exists** : 子表是否已经存在，默认值为 "no"，可选值为 "yes" 或 "no" 。
- **childtable_count** : 子表的数量，默认值为 10。
- **childtable_prefix** : 子表名称的前缀，必选配置项，没有默认值。
-- **escape_character** : 超级表和子表名称中是否包含转义字符，默认值为 "no"，可选值为 "yes" 或 "no"。
+- **escape_character** : 超级表和子表名称中是否包含转义字符，默认值为 "no"，可选值为 "yes" 或 "no" 。
- **auto_create_table** : 仅当 insert_mode 为 taosc, rest, stmt 并且 child_table_exists 为 "no" 时生效，该参数为 "yes" 表示 taosBenchmark 在插入数据时会自动创建不存在的表；为 "no" 则表示先提前建好所有表再进行插入。
@@ -247,7 +151,7 @@ INFO: Consumed total msgs: 3000, total rows: 30000000
- **non_stop_mode** : 指定是否持续写入，若为 "yes" 则 insert_rows 失效，直到 Ctrl + C 停止程序，写入才会停止。默认值为 "no"，即写入指定数量的记录后停止。注：即使在持续写入模式下 insert_rows 失效，但其也必须被配置为一个非零正整数。
-- **line_protocol** : 使用行协议插入数据，仅当 insert_mode 为 sml 或 sml-rest 时生效，可选项为 line, telnet, json。
+- **line_protocol** : 使用行协议插入数据，仅当 insert_mode 为 sml 或 sml-rest 时生效，可选项为 line, telnet, json 。
- **tcp_transfer** : telnet 模式下的通信协议，仅当 insert_mode 为 sml-rest 并且 line_protocol 为 telnet 时生效。如果不配置，则默认为 http 协议。
@@ -267,9 +171,9 @@ INFO: Consumed total msgs: 3000, total rows: 30000000
- **disorder_range** : 指定乱序数据的时间戳回退范围。所生成的乱序时间戳为非乱序情况下应该使用的时间戳减去这个范围内的一个随机值。仅在 `-O/--disorder` 指定的乱序数据百分比大于 0 时有效。
-- **timestamp_step** : 每个子表中插入数据的时间戳步长，单位与数据库的 `precision` 一致，默认值是 1。
+- **timestamp_step** : 每个子表中插入数据的时间戳步长，单位与数据库的 `precision` 一致，默认值是 1 。
-- **start_timestamp** : 每个子表的时间戳起始值，默认值是 now。
+- **start_timestamp** : 每个子表的时间戳起始值，默认值是 now 。
- **sample_format** : 样本数据文件的类型，现在只支持 "csv" 。
@@ -279,10 +183,10 @@ INFO: Consumed total msgs: 3000, total rows: 30000000
- **tags_file** : 仅当 insert_mode 为 taosc, rest 的模式下生效。 最终的 tag 的数值与 childtable_count 有关，如果 csv 文件内的 tag 数据行小于给定的子表数量，那么会循环读取 csv 文件数据直到生成 childtable_count 指定的子表数量；否则则只会读取 childtable_count 行 tag 数据。也即最终生成的子表数量为二者取小。
-- **primary_key** : 指定超级表是否有复合主键，取值 1 和 0， 复合主键列只能是超级表的第二列，指定生成复合主键后要确保第二列符合复合主键的数据类型，否则会报错
-- **repeat_ts_min** : 数值类型，复合主键开启情况下指定生成相同时间戳记录的最小个数，生成相同时间戳记录的个数是在范围[repeat_ts_min, repeat_ts_max] 内的随机值, 最小值等于最大值时为固定个数
-- **repeat_ts_max** : 数值类型，复合主键开启情况下指定生成相同时间戳记录的最大个数
-- **sqls** : 字符串数组类型，指定超级表创建成功后要执行的 sql 数组，sql 中指定表名前面要带数据库名，否则会报未指定数据库错误
+- **primary_key** : 指定超级表是否有复合主键，取值 1 和 0， 复合主键列只能是超级表的第二列，指定生成复合主键后要确保第二列符合复合主键的数据类型，否则会报错。
+- **repeat_ts_min** : 数值类型，复合主键开启情况下指定生成相同时间戳记录的最小个数，生成相同时间戳记录的个数是在范围[repeat_ts_min, repeat_ts_max] 内的随机值, 最小值等于最大值时为固定个数。
+- **repeat_ts_max** : 数值类型，复合主键开启情况下指定生成相同时间戳记录的最大个数。
+- **sqls** : 字符串数组类型，指定超级表创建成功后要执行的 sql 数组，sql 中指定表名前面要带数据库名，否则会报未指定数据库错误。
#### 标签列与数据列
@@ -308,27 +212,27 @@ INFO: Consumed total msgs: 3000, total rows: 30000000
- **values** : nchar/binary 列/标签的值域，将从值中随机选择。
-- **sma**: 将该列加入 SMA 中，值为 "yes" 或者 "no"，默认为 "no"。
+- **sma**: 将该列加入 SMA 中，值为 "yes" 或者 "no"，默认为 "no" 。
-- **encode**: 字符串类型，指定此列两级压缩中的第一级编码算法，详细参见创建超级表
+- **encode**: 字符串类型，指定此列两级压缩中的第一级编码算法，详细参见创建超级表。
-- **compress**: 字符串类型，指定此列两级压缩中的第二级加密算法，详细参见创建超级表
+- **compress**: 字符串类型，指定此列两级压缩中的第二级加密算法，详细参见创建超级表。
-- **level**: 字符串类型，指定此列两级压缩中的第二级加密算法的压缩率高低，详细参见创建超级表
+- **level**: 字符串类型，指定此列两级压缩中的第二级加密算法的压缩率高低，详细参见创建超级表。
-- **gen**: 字符串类型，指定此列生成数据的方式，不指定为随机，若指定为 “order”, 会按自然数顺序增长
+- **gen**: 字符串类型，指定此列生成数据的方式，不指定为随机，若指定为 “order”, 会按自然数顺序增长。
-- **fillNull**: 字符串类型，指定此列是否随机插入 NULL 值，可指定为 “true” 或 "false", 只有当 generate_row_rule 为 2 时有效
+- **fillNull**: 字符串类型，指定此列是否随机插入 NULL 值，可指定为 “true” 或 "false", 只有当 generate_row_rule 为 2 时有效。
-#### 插入行为相关
+#### 写入行为相关
- **thread_count** : 插入数据的线程数量，默认为 8。
-**thread_bind_vgroup** : 写入时 vgroup 是否和写入线程绑定，绑定后可提升写入速度, 取值为 "yes" 或 "no"，默认值为 “no”, 设置为 “no” 后与原来行为一致。 当设为 “yes” 时，如果 thread_count 大于写入数据库 vgroups 数量， thread_count 自动调整为 vgroups 数量；如果 thread_count 小于 vgroups 数量，写入线程数量不做调整，一个线程写完一个 vgroup 数据后再写下一个，同时保持一个 vgroup 同时只能由一个线程写入的规则。
+- **thread_bind_vgroup** : 写入时 vgroup 是否和写入线程绑定，绑定后可提升写入速度, 取值为 "yes" 或 "no"，默认值为 “no”, 设置为 “no” 后与原来行为一致。 当设为 “yes” 时，如果 thread_count 大于写入数据库 vgroups 数量， thread_count 自动调整为 vgroups 数量；如果 thread_count 小于 vgroups 数量，写入线程数量不做调整，一个线程写完一个 vgroup 数据后再写下一个，同时保持一个 vgroup 同时只能由一个线程写入的规则。
- **create_table_thread_count** : 建表的线程数量，默认为 8。
-- **result_file** : 结果输出文件的路径，默认值为 ./output.txt。
+- **result_file** : 结果输出文件的路径，默认值为 ./output.txt 。
- **confirm_parameter_prompt** : 开关参数，要求用户在提示后确认才能继续， 可取值 "yes" or "no"。默认值为 "no" 。
@@ -346,15 +250,15 @@ INFO: Consumed total msgs: 3000, total rows: 30000000
- **pre_load_tb_meta** ：是否提前加载子表的 meta 数据，取值为 “yes” or "no"。当子表数量非常多时，打开此选项可提高写入速度。
-### 查询场景配置参数
+### 查询配置参数
查询场景下 `filetype` 必须设置为 `query`。
-`query_times` 指定运行查询的次数，数值类型
+`query_times` 指定运行查询的次数，数值类型。
-查询场景可以通过设置 `kill_slow_query_threshold` 和 `kill_slow_query_interval` 参数来控制杀掉慢查询语句的执行，threshold 控制如果 exec_usec 超过指定时间的查询将被 taosBenchmark 杀掉，单位为秒；
+查询场景可以通过设置 `kill_slow_query_threshold` 和 `kill_slow_query_interval` 参数来控制杀掉慢查询语句的执行，threshold 控制如果 exec_usec 超过指定时间的查询将被 taosBenchmark 杀掉，单位为秒。
interval 控制休眠时间，避免持续查询慢查询消耗 CPU ，单位为秒。
-其它通用参数详见[通用配置参数](#通用配置参数)。
+其它通用参数详见 [通用配置参数](#通用配置参数)
#### 执行指定查询语句
@@ -391,23 +295,35 @@ interval 控制休眠时间，避免持续查询慢查询消耗 CPU ，单位为
- **sqls** ：
- **sql** : 执行的 SQL 命令，必填；对于超级表的查询 SQL，在 SQL 命令中必须保留 "xxxx"，会替换为超级下所有子表名后再执行。
- **result** : 保存查询结果的文件，未指定则不保存。
- - **限制项** : sqls 下配置 sql 数组最大为 100 个
+ - **限制项** : sqls 下配置 sql 数组最大为 100 个。
-### 订阅场景配置参数
+### 订阅配置参数
-订阅场景下 `filetype` 必须设置为 `subscribe`，该参数及其它通用参数详见[通用配置参数](#通用配置参数)
+订阅场景下 `filetype` 必须设置为 `subscribe`，该参数及其它通用参数详见 [通用配置参数](#通用配置参数)
-订阅指定表（可以指定超级表、子表或者普通表）的配置参数在 `specified_table_query` 中设置。
+订阅配置参数在 `tmq_info` 项下设置，参数如下：
-- **threads/concurrent** : 执行 SQL 的线程数，默认为 1。
+- **concurrent** : 消费订阅的消费者数量，或称并发消费数量，默认值：1。
+- **create_mode** : 创建消费者模式，可取值 sequential：顺序创建， parallel：并发同时创建，必填项，无默认值。
+- **group_mode** : 生成消费者 groupId 模式，可取值 share：所有消费者只生成一个 groupId， independent：每个消费者生成一个独立的 groupId，如果 `group.id` 未设置，此项为必填项，无默认值。
+- **poll_delay** : 调用 tmq_consumer_poll 传入的轮询超时时间，单位为毫秒，负数表示默认超时 1 秒。
+- **enable.manual.commit** : 是否允许手动提交，可取值 true：允许手动提交，每次消费完消息后手动调用 tmq_commit_sync 完成提交， false：不进行提交，默认值: false。
+- **rows_file** : 存储消费数据的文件，可以为全路径或相对路径，带文件名。实际保存的文件会在后面加上消费者序号，如 rows_file 为 result, 实际文件名为 result_1（消费者 1） result_2（消费者 2） ...
+- **expect_rows** : 期望每个消费者消费的行数，数据类型，当消费达到这个数，消费会退出，不设置会一直消费。
+- **topic_list** : 指定消费的 topic 列表，数组类型。topic 列表格式示例： `{"name": "topic1", "sql": "select * from test.meters;"}` ，name：指定 topic 名，sql：指定创建 topic 的 sql 语句，需保证 sql 正确，框架会自动创建出 topic。
-- **sqls** ：
- - **sql** : 执行的 SQL 命令，必填。
-
+以下参数透传订阅属性，参见 [订阅创建参数](../../../develop/tmq/#创建参数) 说明：
+- **client.id**
+- **auto.offset.reset**
+- **enable.auto.commit**
+- **enable.auto.commit**
+- **msg.with.table.name**
+- **auto.commit.interval.ms**
+- **group.id** : 若此值不指定，将由 `group_mode` 指定规则生成 groupId，若指定此值，`group_mode` 参数不再有效。
-### 配置文件中数据类型书写对照表
+### 数据类型对照表
-| # | **引擎** | **taosBenchmark**
+| # | **TDengine** | **taosBenchmark**
| --- | :----------------: | :---------------:
| 1 | TIMESTAMP | timestamp
| 2 | INT | int
@@ -428,7 +344,96 @@ interval 控制休眠时间，避免持续查询慢查询消耗 CPU ，单位为
| 17 | GEOMETRY | geometry
| 18 | JSON | json
-注意：taosBenchmark 配置文件中数据类型必须小写方可识别
+注意：taosBenchmark 配置文件中数据类型必须小写方可识别。
+## 配置文件示例
+**下面为支持的写入、查询、订阅三大功能的配置文件示例：**
+### 写入 JSON 示例
+
+
+insert.json
+
+```json
+{{#include /TDengine/tools/taos-tools/example/insert.json}}
+```
+
+
+
+### 查询 JSON 示例
+
+
+query.json
+
+```json
+{{#include /TDengine/tools/taos-tools/example/query.json}}
+```
+
+
+
+### 订阅 JSON 示例
+
+
+tmq.json
+
+```json
+{{#include /TDengine/tools/taos-tools/example/tmq.json}}
+```
+
+
+
+查看更多 json 配置文件示例可 [点击这里](https://github.com/taosdata/TDengine/tree/main/tools/taos-tools/example)
+
+## 输出性能指标
+
+#### 写入指标
+
+写入结束后会在最后两行输出总体性能指标，格式如下：
+``` bash
+SUCC: Spent 8.527298 (real 8.117379) seconds to insert rows: 10000000 with 8 thread(s) into test 1172704.41 (real 1231924.74) records/second
+SUCC: insert delay, min: 19.6780ms, avg: 64.9390ms, p90: 94.6900ms, p95: 105.1870ms, p99: 130.6660ms, max: 157.0830ms
+```
+第一行写入速度统计：
+ - Spent: 写入总耗时，单位秒，从开始写入第一个数据开始计时到最后一条数据结束，这里表示共花了 8.527298 秒。
+ - real : 写入总耗时（调用引擎），此耗时已抛去测试框架准备数据时间，纯统计在引擎调用上花费的时间，示例为 8.117379 秒，8.527298 - 8.117379 = 0.409919 秒则为测试框架准备数据消耗时间
+ - rows : 写入总行数，为 1000 万条数据。
+ - threads: 写入线程数，这里是 8 个线程同时写入。
+ - records/second 写入速度 = `写入总耗时`/ `写入总行数` ， 括号中 `real` 同前，表示纯引擎写入速度。
+第二行单个写入延时统计：
+ - min : 写入最小延时。
+ - avg : 写入平时延时。
+ - p90 : 写入延时 p90 百分位上的延时数。
+ - p95 : 写入延时 p95 百分位上的延时数。
+ - p99 : 写入延时 p99 百分位上的延时数。
+ - max : 写入最大延时。
+通过此系列指标，可观察到写入请求延时分布情况。
+
+#### 查询指标
+
+查询性能测试主要输出查询请求速度 QPS 指标, 输出格式如下：
+``` bash
+complete query with 3 threads and 10000 query delay avg: 0.002686s min: 0.001182s max: 0.012189s p90: 0.002977s p95: 0.003493s p99: 0.004645s SQL command: select ...
+INFO: Spend 26.9530 second completed total queries: 30000, the QPS of all threads: 1113.049
+```
+- 第一行表示 3 个线程每个线程执行 10000 次查询及查询请求延时百分位分布情况，`SQL command` 为测试的查询语句。
+- 第二行表示查询总耗时为 26.9653 秒，每秒查询率(QPS)为：1113.049 次/秒。
+- 如果在查询中设置了 `continue_if_fail` 选项为 `yes`，在最后一行中会输出失败请求个数及错误率，格式 error + 失败请求个数 (错误率)。
+- QPS = 成功请求数量 / 花费时间(单位秒)
+- 错误率 = 失败请求数量 /（成功请求数量 + 失败请求数量）
+
+#### 订阅指标
+
+订阅性能测试主要输出消费者消费速度指标，输出格式如下：
+``` bash
+INFO: consumer id 0 has poll total msgs: 376, period rate: 37.592 msgs/s, total rows: 3760000, period rate: 375924.815 rows/s
+INFO: consumer id 1 has poll total msgs: 362, period rate: 36.131 msgs/s, total rows: 3620000, period rate: 361313.504 rows/s
+INFO: consumer id 2 has poll total msgs: 364, period rate: 36.378 msgs/s, total rows: 3640000, period rate: 363781.731 rows/s
+INFO: consumerId: 0, consume msgs: 1000, consume rows: 10000000
+INFO: consumerId: 1, consume msgs: 1000, consume rows: 10000000
+INFO: consumerId: 2, consume msgs: 1000, consume rows: 10000000
+INFO: Consumed total msgs: 3000, total rows: 30000000
+```
+- 1 ~ 3 行实时输出每个消费者当前的消费速度，`msgs/s` 表示消费消息个数，每个消息中包含多行数据，`rows/s` 表示按行数统计的消费速度。
+- 4 ~ 6 行是测试完成后每个消费者总体统计，统计共消费了多少条消息，共计多少行。
+- 第 7 行所有消费者总体统计，`msgs` 表示共消费了多少条消息， `rows` 表示共消费了多少行数据。
diff --git a/docs/zh/14-reference/03-taos-sql/02-database.md b/docs/zh/14-reference/03-taos-sql/02-database.md
index 9e42f240d0..35cb99a3dd 100644
--- a/docs/zh/14-reference/03-taos-sql/02-database.md
+++ b/docs/zh/14-reference/03-taos-sql/02-database.md
@@ -37,6 +37,9 @@ database_option: {
| WAL_FSYNC_PERIOD value
| WAL_RETENTION_PERIOD value
| WAL_RETENTION_SIZE value
+ | COMPACT_INTERVAL value
+ | COMPACT_TIME_RANGE value
+ | COMPACT_TIME_OFFSET value
}
```
@@ -81,6 +84,10 @@ database_option: {
- WAL_FSYNC_PERIOD：当 WAL_LEVEL 参数设置为 2 时，用于设置落盘的周期。默认为 3000，单位毫秒。最小为 0，表示每次写入立即落盘；最大为 180000，即三分钟。
- WAL_RETENTION_PERIOD: 为了数据订阅消费，需要 WAL 日志文件额外保留的最大时长策略。WAL 日志清理，不受订阅客户端消费状态影响。单位为 s。默认为 3600，表示在 WAL 保留最近 3600 秒的数据，请根据数据订阅的需要修改这个参数为适当值。
- WAL_RETENTION_SIZE：为了数据订阅消费，需要 WAL 日志文件额外保留的最大累计大小策略。单位为 KB。默认为 0，表示累计大小无上限。
+- COMPACT_INTERVAL：自动 compact 触发周期（从 1970-01-01T00:00:00Z 开始切分的时间周期)。取值范围：0 或 [10m, keep2]，单位：m（分钟），h（小时），d（天）。不加时间单位默认单位为天，默认值为 0，即不触发自动 compact 功能。如果 db 中有未完成的 compact 任务，不重复下发 compact 任务。仅企业版 3.3.5.0 版本开始支持。
+- COMPACT_TIME_RANGE：自动 compact 任务触发的 compact 时间范围，取值范围：[-keep2, -duration]，单位：m（分钟），h（小时），d（天）。不加时间单位时默认单位为天，默认值为 [0, 0]。取默认值 [0, 0] 时，如果 COMPACT_INTERVAL 大于 0，会按照 [-keep2, -duration] 下发自动 compact。因此，要关闭自动 compact 功能，需要将 COMPACT_INTERVAL 设置为 0。仅企业版 3.3.5.0 版本开始支持。
+- COMPACT_TIME_OFFSET：自动 compact 任务触发的 compact 时间相对本地时间的偏移量。取值范围：[0,23]，单位: h（小时），默认值为 0。以 UTC 0 时区为例，如果 COMPACT_INTERVAL 为 1d，当 COMPACT_TIME_OFFSET 为 0 时，在每天 0 点下发自动 compact，如果 COMPACT_TIME_OFFSET 为 2，在每天 2 点下发自动 compact。仅企业版 3.3.5.0 版本开始支持。
+-
### 创建数据库示例
@@ -127,6 +134,9 @@ alter_database_option: {
| WAL_RETENTION_PERIOD value
| WAL_RETENTION_SIZE value
| MINROWS value
+ | COMPACT_INTERVAL value
+ | COMPACT_TIME_RANGE value
+ | COMPACT_TIME_OFFSET value
}
```
diff --git a/docs/zh/14-reference/03-taos-sql/14-stream.md b/docs/zh/14-reference/03-taos-sql/14-stream.md
index 5e5a82b95c..666cf97ea7 100644
--- a/docs/zh/14-reference/03-taos-sql/14-stream.md
+++ b/docs/zh/14-reference/03-taos-sql/14-stream.md
@@ -8,7 +8,7 @@ description: 流式计算的相关 SQL 的详细语法
## 创建流式计算
```sql
-CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name[(field1_name, field2_name [PRIMARY KEY], ...)] [TAGS (create_definition [, create_definition] ...)] SUBTABLE(expression) AS subquery
+CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name[(field1_name, field2_name [PRIMARY KEY], ...)] [TAGS (create_definition [, create_definition] ...)] SUBTABLE(expression) AS subquery [notification_definition]
stream_options: {
TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time | FORCE_WINDOW_CLOSE]
WATERMARK time
@@ -30,11 +30,11 @@ subquery: SELECT select_list
window_clause
```
-支持会话窗口、状态窗口、滑动窗口、事件窗口和计数窗口，其中，状态窗口、事件窗口和计数窗口搭配超级表时必须与partition by tbname一起使用。对于数据源表是复合主键的流，不支持状态窗口、事件窗口、计数窗口的计算。
+支持会话窗口、状态窗口、滑动窗口、事件窗口和计数窗口。其中，状态窗口、事件窗口和计数窗口搭配超级表时必须与 partition by tbname 一起使用。对于数据源表是复合主键的流，不支持状态窗口、事件窗口、计数窗口的计算。
-stb_name 是保存计算结果的超级表的表名，如果该超级表不存在，会自动创建；如果已存在，则检查列的schema信息。详见 写入已存在的超级表。
+stb_name 是保存计算结果的超级表的表名，如果该超级表不存在，会自动创建；如果已存在，则检查列的schema信息。详见 [写入已存在的超级表](#写入已存在的超级表) 。
-TAGS 子句定义了流计算中创建TAG的规则，可以为每个partition对应的子表生成自定义的TAG值，详见 自定义TAG
+TAGS 子句定义了流计算中创建TAG的规则，可以为每个 partition 对应的子表生成自定义的TAG值，详见 [自定义 TAG](#自定义 TAG)
```sql
create_definition:
col_name column_definition
@@ -42,7 +42,7 @@ column_definition:
type_name [COMMENT 'string_value']
```
-subtable 子句定义了流式计算中创建的子表的命名规则，详见 流式计算的 partition 部分。
+subtable 子句定义了流式计算中创建的子表的命名规则，详见 [流式计算的 partition](#流式计算的 partition)。
```sql
window_clause: {
@@ -54,23 +54,25 @@ window_clause: {
}
```
-其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。该窗口的 _wend 等于最后一条数据的时间加上 tol_val。
+其中：
-STATE_WINDOW 是状态窗口，col 用来标识状态量，相同的状态量数值则归属于同一个状态窗口，col 数值改变后则当前窗口结束，自动开启下一个窗口。
+- SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。该窗口的 _wend 等于最后一条数据的时间加上 tol_val。
-INTERVAL 是时间窗口，又可分为滑动时间窗口和翻转时间窗口。INTERVAL 子句用于指定窗口相等时间周期，SLIDING 字句用于指定窗口向前滑动的时间。当 interval_val 与 sliding_val 相等的时候，时间窗口即为翻转时间窗口，否则为滑动时间窗口，注意：sliding_val 必须小于等于 interval_val。
+- STATE_WINDOW 是状态窗口，col 用来标识状态量，相同的状态量数值则归属于同一个状态窗口，col 数值改变后则当前窗口结束，自动开启下一个窗口。
-EVENT_WINDOW 是事件窗口，根据开始条件和结束条件来划定窗口。当 start_trigger_condition 满足时则窗口开始，直到 end_trigger_condition 满足时窗口关闭。 start_trigger_condition 和 end_trigger_condition 可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。
+- INTERVAL 是时间窗口，又可分为滑动时间窗口和翻转时间窗口。INTERVAL 子句用于指定窗口相等时间周期，SLIDING 字句用于指定窗口向前滑动的时间。当 interval_val 与 sliding_val 相等的时候，时间窗口即为翻转时间窗口，否则为滑动时间窗口，注意：sliding_val 必须小于等于 interval_val。
-COUNT_WINDOW 是计数窗口,按固定的数据行数来划分窗口。 count_val 是常量，是正整数，必须大于等于2，小于2147483648。 count_val 表示每个 COUNT_WINDOW 包含的最大数据行数，总数据行数不能整除 count_val 时，最后一个窗口的行数会小于 count_val 。 sliding_val 是常量，表示窗口滑动的数量，类似于 INTERVAL 的 SLIDING 。
+- EVENT_WINDOW 是事件窗口，根据开始条件和结束条件来划定窗口。当 start_trigger_condition 满足时则窗口开始，直到 end_trigger_condition 满足时窗口关闭。 start_trigger_condition 和 end_trigger_condition 可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。
+
+- COUNT_WINDOW 是计数窗口，按固定的数据行数来划分窗口。 count_val 是常量，是正整数，必须大于等于2，小于2147483648。 count_val 表示每个 COUNT_WINDOW 包含的最大数据行数，总数据行数不能整除 count_val 时，最后一个窗口的行数会小于 count_val 。 sliding_val 是常量，表示窗口滑动的数量，类似于 INTERVAL 的 SLIDING 。
窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished)
例如，如下语句创建流式计算。第一个流计算，自动创建名为 avg_vol 的超级表，以一分钟为时间窗口、30 秒为前向增量统计这些电表的平均电压，并将来自 meters 表的数据的计算结果写入 avg_vol 表，不同 partition 的数据会分别创建子表并写入不同子表。
-第二个流计算，自动创建名为 streamt0 的超级表，将数据按时间戳的顺序，以 voltage < 0 作为窗口的开始条件，voltage > 9作为窗口的结束条件，划分窗口做聚合运算，并将来自 meters 表的数据的计算结果写入 streamt0 表，不同 partition 的数据会分别创建子表并写入不同子表。
+第二个流计算，自动创建名为 streamt0 的超级表，将数据按时间戳的顺序，以 voltage < 0 作为窗口的开始条件，voltage > 9 作为窗口的结束条件，划分窗口做聚合运算，并将来自 meters 表的数据的计算结果写入 streamt0 表，不同 partition 的数据会分别创建子表并写入不同子表。
-第三个流计算，自动创建名为 streamt1 的超级表，将数据按时间戳的顺序，以10条数据为一组，划分窗口做聚合运算，并将来自 meters 表的数据的计算结果写入 streamt1 表，不同 partition 的数据会分别创建子表并写入不同子表。
+第三个流计算，自动创建名为 streamt1 的超级表，将数据按时间戳的顺序，以 10 条数据为一组，划分窗口做聚合运算，并将来自 meters 表的数据的计算结果写入 streamt1 表，不同 partition 的数据会分别创建子表并写入不同子表。
```sql
CREATE STREAM avg_vol_s INTO avg_vol AS
@@ -83,6 +85,8 @@ CREATE STREAM streams1 IGNORE EXPIRED 1 WATERMARK 100s INTO streamt1 AS
SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WINDOW(10);
```
+notification_definition 子句定义了窗口计算过程中，在窗口打开/关闭等指定事件发生时，需要向哪些地址发送通知。详见 [流式计算的事件通知](#流式计算的事件通知)
+
## 流式计算的 partition
可以使用 PARTITION BY TBNAME，tag，普通列或者表达式，对一个流进行多分区的计算，每个分区的时间线与时间窗口是独立的，会各自聚合，并写入到目的表中的不同子表。
@@ -97,19 +101,19 @@ SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WIN
CREATE STREAM avg_vol_s INTO avg_vol SUBTABLE(CONCAT('new-', tname)) AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname tname INTERVAL(1m);
```
-PARTITION 子句中，为 tbname 定义了一个别名 tname, 在PARTITION 子句中的别名可以用于 SUBTABLE 子句中的表达式计算，在上述示例中，流新创建的子表将以前缀 'new-' 连接原表名作为表名(从3.2.3.0开始，为了避免 SUBTABLE 中的表达式无法区分各个子表，即误将多个相同时间线写入一个子表，在指定的子表名后面加上 _stableName_groupId)。
+PARTITION 子句中，为 tbname 定义了一个别名 tname, 在 PARTITION 子句中的别名可以用于 SUBTABLE 子句中的表达式计算，在上述示例中，流新创建的子表将以前缀 'new-' 连接原表名作为表名（从 3.2.3.0 版本开始，为了避免 SUBTABLE 中的表达式无法区分各个子表，即误将多个相同时间线写入一个子表，在指定的子表名后面加上 _stableName_groupId）。
注意，子表名的长度若超过 TDengine 的限制，将被截断。若要生成的子表名已经存在于另一超级表，由于 TDengine 的子表名是唯一的，因此对应新子表的创建以及数据的写入将会失败。
## 流式计算读取历史数据
-正常情况下，流式计算不会处理创建前已经写入源表中的数据，若要处理已经写入的数据，可以在创建流时设置 fill_history 1 选项，这样创建的流式计算会自动处理创建前、创建中、创建后写入的数据。流计算处理历史数据的最大窗口数是2000万，超过限制会报错。例如：
+正常情况下，流式计算不会处理创建前已经写入源表中的数据，若要处理已经写入的数据，可以在创建流时设置 fill_history 1 选项，这样创建的流式计算会自动处理创建前、创建中、创建后写入的数据。流计算处理历史数据的最大窗口数是 2000万，超过限制会报错。例如：
```sql
create stream if not exists s1 fill_history 1 into st1 as select count(*) from t1 interval(10s)
```
-结合 fill_history 1 选项，可以实现只处理特定历史时间范围的数据，例如：只处理某历史时刻（2020年1月30日）之后的数据
+结合 fill_history 1 选项，可以实现只处理特定历史时间范围的数据，例如：只处理某历史时刻（2020 年 1 月 30 日）之后的数据
```sql
create stream if not exists s1 fill_history 1 into st1 as select count(*) from t1 where ts > '2020-01-30' interval(10s)
@@ -154,7 +158,7 @@ SELECT * from information_schema.`ins_streams`;
2. WINDOW_CLOSE：窗口关闭时触发（窗口关闭由事件时间决定，可配合 watermark 使用）
3. MAX_DELAY time：若窗口关闭，则触发计算。若窗口未关闭，且未关闭时长超过 max delay 指定的时间，则触发计算。
-4. FORCE_WINDOW_CLOSE：以操作系统当前时间为准，只计算当前关闭窗口的结果，并推送出去。窗口只会在被关闭的时刻计算一次，后续不会再重复计算。该模式当前只支持 INTERVAL 窗口（不支持滑动）；FILL_HISTORY 必须为 0，IGNORE EXPIRED 必须为 1，IGNORE UPDATE 必须为 1；FILL 只支持 PREV 、NULL、NONE、VALUE。
+4. FORCE_WINDOW_CLOSE：以操作系统当前时间为准，只计算当前关闭窗口的结果，并推送出去。窗口只会在被关闭的时刻计算一次，后续不会再重复计算。该模式当前只支持 INTERVAL 窗口（不支持滑动）；FILL_HISTORY 必须为 0，IGNORE EXPIRED 必须为 1，IGNORE UPDATE 必须为 1；FILL 只支持 PREV、NULL、NONE、VALUE。
由于窗口关闭是由事件时间决定的，如事件流中断、或持续延迟，则事件时间无法更新，可能导致无法得到最新的计算结果。
@@ -215,33 +219,33 @@ TDengine 对于修改数据提供两种处理方式，由 IGNORE UPDATE 选项
## 写入已存在的超级表
```sql
-[field1_name,...]
+[field1_name, ...]
```
-在本页文档顶部的 [field1_name,...] 是用来指定 stb_name 的列与 subquery 输出结果的对应关系的。如果 stb_name 的列与 subquery 输出结果的位置、数量全部匹配，则不需要显示指定对应关系。如果 stb_name 的列与 subquery 输出结果的数据类型不匹配，会把 subquery 输出结果的类型转换成对应的 stb_name 的列的类型。创建流计算时不能指定 stb_name 的列和 TAG 的数据类型，否则会报错。
+在本页文档顶部的 [field1_name, ...] 是用来指定 stb_name 的列与 subquery 输出结果的对应关系的。如果 stb_name 的列与 subquery 输出结果的位置、数量全部匹配，则不需要显示指定对应关系。如果 stb_name 的列与 subquery 输出结果的数据类型不匹配，会把 subquery 输出结果的类型转换成对应的 stb_name 的列的类型。创建流计算时不能指定 stb_name 的列和 TAG 的数据类型，否则会报错。
对于已经存在的超级表，检查列的schema信息
1. 检查列的 schema 信息是否匹配，对于不匹配的，则自动进行类型转换，当前只有数据长度大于 4096byte 时才报错，其余场景都能进行类型转换。
2. 检查列的个数是否相同，如果不同，需要显示的指定超级表与 subquery 的列的对应关系，否则报错；如果相同，可以指定对应关系，也可以不指定，不指定则按位置顺序对应。
-## 自定义TAG
+## 自定义 TAG
-用户可以为每个 partition 对应的子表生成自定义的TAG值。
+用户可以为每个 partition 对应的子表生成自定义的 TAG 值。
```sql
-CREATE STREAM streams2 trigger at_once INTO st1 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as cc interval(10s));
+CREATE STREAM streams2 trigger at_once INTO st1 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as cc interval(10s));
```
-PARTITION 子句中，为 concat("tag-", tbname)定义了一个别名cc, 对应超级表st1的自定义TAG的名字。在上述示例中，流新创建的子表的TAG将以前缀 'new-' 连接原表名作为TAG的值。
+PARTITION 子句中，为 concat("tag-", tbname) 定义了一个别名 cc，对应超级表 st1 的自定义 TAG 的名字。在上述示例中，流新创建的子表的 TAG 将以前缀 'new-' 连接原表名作为 TAG 的值。
会对TAG信息进行如下检查
-1. 检查tag的schema信息是否匹配，对于不匹配的，则自动进行数据类型转换，当前只有数据长度大于4096byte时才报错，其余场景都能进行类型转换。
-2. 检查tag的个数是否相同，如果不同，需要显示的指定超级表与subquery的tag的对应关系，否则报错；如果相同，可以指定对应关系，也可以不指定，不指定则按位置顺序对应。
+1. 检查 tag 的 schema 信息是否匹配，对于不匹配的，则自动进行数据类型转换，当前只有数据长度大于 4096byte 时才报错，其余场景都能进行类型转换。
+2. 检查 tag 的个数是否相同，如果不同，需要显示的指定超级表与 subquery 的 tag 的对应关系，否则报错；如果相同，可以指定对应关系，也可以不指定，不指定则按位置顺序对应。
## 清理中间状态
```
-DELETE_MARK time
+DELETE_MARK time
```
-DELETE_MARK用于删除缓存的窗口状态，也就是删除流计算的中间结果。如果不设置，默认值是10年
+DELETE_MARK 用于删除缓存的窗口状态，也就是删除流计算的中间结果。如果不设置，默认值是 10 年
T = 最新事件时间 - DELETE_MARK
## 流式计算支持的函数
@@ -272,15 +276,15 @@ T = 最新事件时间 - DELETE_MARK
## 暂停、恢复流计算
1.流计算暂停计算任务
PAUSE STREAM [IF EXISTS] stream_name;
-没有指定IF EXISTS，如果该stream不存在，则报错；如果存在，则暂停流计算。指定了IF EXISTS，如果该stream不存在，则返回成功；如果存在，则暂停流计算
+没有指定 IF EXISTS，如果该 stream 不存在，则报错；如果存在，则暂停流计算。指定了 IF EXISTS，如果该 stream 不存在，则返回成功；如果存在，则暂停流计算。
2.流计算恢复计算任务
RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name;
-没有指定IF EXISTS，如果该stream不存在，则报错，如果存在，则恢复流计算；指定了IF EXISTS，如果stream不存在，则返回成功；如果存在，则恢复流计算。如果指定IGNORE UNTREATED，则恢复流计算时，忽略流计算暂停期间写入的数据。
+没有指定 IF EXISTS，如果该 stream 不存在，则报错，如果存在，则恢复流计算；指定了 IF EXISTS，如果 stream 不存在，则返回成功；如果存在，则恢复流计算。如果指定 IGNORE UNTREATED，则恢复流计算时，忽略流计算暂停期间写入的数据。
## 状态数据备份与同步
-流计算的中间结果成为计算的状态数据，需要在流计算整个生命周期中进行持久化保存。为了确保流计算中间状态能够在集群环境下在不同的节点间可靠地同步和迁移，至3.3.2.1 版本开始，需要在运行环境中部署 rsync 软件，还需要增加以下的步骤：
-1. 在配置文件中配置 snode 的地址（IP+端口）和状态数据备份目录（该目录系 snode 所在的物理节点的目录）。
+流计算的中间结果成为计算的状态数据，需要在流计算整个生命周期中进行持久化保存。为了确保流计算中间状态能够在集群环境下在不同的节点间可靠地同步和迁移，从 3.3.2.1 版本开始，需要在运行环境中部署 rsync 软件，还需要增加以下的步骤：
+1. 在配置文件中配置 snode 的地址（IP + 端口）和状态数据备份目录（该目录系 snode 所在的物理节点的目录）。
2. 然后创建 snode。
完成上述两个步骤以后才能创建流。
如果没有创建 snode 并正确配置 snode 的地址，流计算过程中将无法生成检查点（checkpoint），并可能导致后续的计算结果产生错误。
@@ -291,9 +295,228 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name;
## 创建 snode 的方式
-使用以下命令创建 snode（stream node）， snode 是流计算中有状态的计算节点，可用于部署聚合任务，同时负责备份不同的流计算任务生成的检查点数据。
+使用以下命令创建 snode（stream node），snode 是流计算中有状态的计算节点，可用于部署聚合任务，同时负责备份不同的流计算任务生成的检查点数据。
```sql
CREATE SNODE ON DNODE [id]
```
其中的 id 是集群中的 dnode 的序号。请注意选择的dnode，流计算的中间状态将自动在其上进行备份。
从 3.3.4.0 版本开始，在多副本环境中创建流会进行 snode 的**存在性检查**，要求首先创建 snode。如果 snode 不存在，无法创建流。
+
+## 流式计算的事件通知
+
+### 使用说明
+
+流式计算支持在窗口打开/关闭时，向外部系统发送相关的事件通知。用户通过 `notification_definition` 来指定需要通知的事件，以及用于接收通知消息的目标地址。
+
+```sql
+notification_definition:
+ NOTIFY (url [, url] ...) ON (event_type [, event_type] ...) [notification_options]
+
+event_type:
+ 'WINDOW_OPEN'
+ | 'WINDOW_CLOSE'
+
+notification_options: {
+ NOTIFY_HISTORY [0|1]
+ ON_FAILURE [DROP|PAUSE]
+}
+```
+
+上述语法中的相关规则含义如下：
+1. `url`：指定通知的目标地址，必须包括协议、IP 或域名、端口号，并允许包含路径、参数。目前仅支持 websocket 协议。例如：`ws://localhost:8080`、`ws://localhost:8080/notify`、`wss://localhost:8080/notify?key=foo`。
+1. `event_type`：定义需要通知的事件，支持的事件类型有：
+ 1. WINDOW_OPEN：窗口打开事件，所有类型的窗口打开时都会触发。
+ 1. WINDOW_CLOSE：窗口关闭事件，所有类型的窗口关闭时都会触发。
+1. `NOTIFY_HISTORY`：控制是否在计算历史数据时触发通知，默认值为 0，即不触发。
+1. `ON_FAILURE`：向通知地址发送通知失败时(比如网络不佳场景)是否允许丢弃部分事件，默认值为 `PAUSE`。
+ 1. PAUSE 表示发送通知失败时暂停流计算任务。taosd 会重试发送通知，直到发送成功后，任务自动恢复运行。
+ 1. DROP 表示发送通知失败时直接丢弃事件信息，流计算任务继续运行，不受影响。
+
+比如，以下示例创建一个流，计算电表电流的每分钟平均值，并在窗口打开、关闭时向两个通知地址发送通知，计算历史数据时不发送通知，不允许在通知发送失败时丢弃通知：
+
+```sql
+CREATE STREAM avg_current_stream FILL_HISTORY 1
+ AS SELECT _wstart, _wend, AVG(current) FROM meters
+ INTERVAL (1m)
+ NOTIFY ('ws://localhost:8080/notify', 'wss://192.168.1.1:8080/notify?key=foo')
+ ON ('WINDOW_OPEN', 'WINDOW_CLOSE');
+ NOTIFY_HISTORY 0
+ ON_FAILURE PAUSE;
+```
+
+当触发指定的事件时，taosd 会向指定的 URL 发送 POST 请求，消息体为 JSON 格式。一个请求可能包含若干个流的若干个事件，且事件类型不一定相同。
+事件信息视窗口类型而定：
+
+1. 时间窗口：开始时发送起始时间；结束时发送起始时间、结束时间、计算结果。
+1. 状态窗口：开始时发送起始时间、前一个窗口的状态值、当前窗口的状态值；结束时发送起始时间、结束时间、计算结果、当前窗口的状态值、下一个窗口的状态值。
+1. 会话窗口：开始时发送起始时间；结束时发送起始时间、结束时间、计算结果。
+1. 事件窗口：开始时发送起始时间，触发窗口打开的数据值和对应条件编号；结束时发送起始时间、结束时间、计算结果、触发窗口关闭的数据值和对应条件编号。
+1. 计数窗口：开始时发送起始时间；结束时发送起始时间、结束时间、计算结果。
+
+通知消息的结构示例如下：
+
+```json
+{
+ "messageId": "unique-message-id-12345",
+ "timestamp": 1733284887203,
+ "streams": [
+ {
+ "streamName": "avg_current_stream",
+ "events": [
+ {
+ "tableName": "t_a667a16127d3b5a18988e32f3e76cd30",
+ "eventType": "WINDOW_OPEN",
+ "eventTime": 1733284887097,
+ "windowId": "window-id-67890",
+ "windowType": "Time",
+ "windowStart": 1733284800000
+ },
+ {
+ "tableName": "t_a667a16127d3b5a18988e32f3e76cd30",
+ "eventType": "WINDOW_CLOSE",
+ "eventTime": 1733284887197,
+ "windowId": "window-id-67890",
+ "windowType": "Time",
+ "windowStart": 1733284800000,
+ "windowEnd": 1733284860000,
+ "result": {
+ "_wstart": 1733284800000,
+ "avg(current)": 1.3
+ }
+ }
+ ]
+ },
+ {
+ "streamName": "max_voltage_stream",
+ "events": [
+ {
+ "tableName": "t_96f62b752f36e9b16dc969fe45363748",
+ "eventType": "WINDOW_OPEN",
+ "eventTime": 1733284887231,
+ "windowId": "window-id-13579",
+ "windowType": "Event",
+ "windowStart": 1733284800000,
+ "triggerCondition": {
+ "conditionIndex": 0,
+ "fieldValue": {
+ "c1": 10,
+ "c2": 15
+ }
+ },
+ },
+ {
+ "tableName": "t_96f62b752f36e9b16dc969fe45363748",
+ "eventType": "WINDOW_CLOSE",
+ "eventTime": 1733284887231,
+ "windowId": "window-id-13579",
+ "windowType": "Event",
+ "windowStart": 1733284800000,
+ "windowEnd": 1733284810000,
+ "triggerCondition": {
+ "conditionIndex": 1,
+ "fieldValue": {
+ "c1": 20
+ "c2": 3
+ }
+ },
+ "result": {
+ "_wstart": 1733284800000,
+ "max(voltage)": 220
+ }
+ }
+ ]
+ }
+ ]
+}
+```
+
+后续小节是通知消息中各个字段的说明。
+
+### 根级字段说明
+
+1. messageId：字符串类型，是通知消息的唯一标识符，确保整条消息可以被追踪和去重。
+1. timestamp：长整型时间戳，表示通知消息生成的时间，精确到毫秒，即: '00:00, Jan 1 1970 UTC' 以来的毫秒数。
+1. streams：对象数组，包含多个流任务的事件信息。(详细信息见下节)
+
+### stream 对象的字段说明
+
+1. streamName：字符串类型，流任务的名称，用于标识事件所属的流。
+1. events：对象数组，该流任务下的事件列表，包含一个或多个事件对象。(详细信息见下节)
+
+### event 对象的字段说明
+
+#### 通用字段
+
+这部分是所有 event 对象所共有的字段。
+1. tableName：字符串类型，是对应目标子表的表名。
+1. eventType：字符串类型，表示事件类型，支持 WINDOW_OPEN、WINDOW_CLOSE、WINDOW_INVALIDATION 三种类型。
+1. eventTime：长整型时间戳，表示事件生成时间，精确到毫秒，即：'00:00, Jan 1 1970 UTC' 以来的毫秒数。
+1. windowId：字符串类型，窗口的唯一标识符，确保打开和关闭事件的 ID 一致，便于外部系统将两者关联。如果 taosd 发生故障重启，部分事件可能会重复发送，会保证同一窗口的 windowId 保持不变。
+1. windowType：字符串类型，表示窗口类型，支持 Time、State、Session、Event、Count 五种类型。
+
+#### 时间窗口相关字段
+
+这部分是 windowType 为 Time 时 event 对象才有的字段。
+1. 如果 eventType 为 WINDOW_OPEN，则包含如下字段：
+ 1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。
+1. 如果 eventType 为 WINDOW_CLOSE，则包含如下字段：
+ 1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。
+ 1. windowEnd：长整型时间戳，表示窗口的结束时间，精度与结果表的时间精度一致。
+ 1. result：计算结果，为键值对形式，包含窗口计算的结果列列名及其对应的值。
+
+#### 状态窗口相关字段
+
+这部分是 windowType 为 State 时 event 对象才有的字段。
+1. 如果 eventType 为 WINDOW_OPEN，则包含如下字段：
+ 1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。
+ 1. prevState：与状态列的类型相同，表示上一个窗口的状态值。如果没有上一个窗口(即：现在是第一个窗口)，则为 NULL。
+ 1. curState：与状态列的类型相同，表示当前窗口的状态值。
+1. 如果 eventType 为 WINDOW_CLOSE，则包含如下字段：
+ 1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。
+ 1. windowEnd：长整型时间戳，表示窗口的结束时间，精度与结果表的时间精度一致。
+ 1. curState：与状态列的类型相同，表示当前窗口的状态值。
+ 1. nextState：与状态列的类型相同，表示下一个窗口的状态值。
+ 1. result：计算结果，为键值对形式，包含窗口计算的结果列列名及其对应的值。
+
+#### 会话窗口相关字段
+
+这部分是 windowType 为 Session 时 event 对象才有的字段。
+1. 如果 eventType 为 WINDOW_OPEN，则包含如下字段：
+ 1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。
+1. 如果 eventType 为 WINDOW_CLOSE，则包含如下字段：
+ 1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。
+ 1. windowEnd：长整型时间戳，表示窗口的结束时间，精度与结果表的时间精度一致。
+ 1. result：计算结果，为键值对形式，包含窗口计算的结果列列名及其对应的值。
+
+#### 事件窗口相关字段
+
+这部分是 windowType 为 Event 时 event 对象才有的字段。
+1. 如果 eventType 为 WINDOW_OPEN，则包含如下字段：
+ 1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。
+ 1. triggerCondition：触发窗口开始的条件信息，包括以下字段：
+ 1. conditionIndex：整型，表示满足的触发窗口开始的条件的索引，从0开始编号。
+ 1. fieldValue：键值对形式，包含条件列列名及其对应的值。
+1. 如果 eventType 为 WINDOW_CLOSE，则包含如下字段：
+ 1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。
+ 1. windowEnd：长整型时间戳，表示窗口的结束时间，精度与结果表的时间精度一致。
+ 1. triggerCondition：触发窗口关闭的条件信息，包括以下字段：
+ 1. conditionIndex：整型，表示满足的触发窗口关闭的条件的索引，从0开始编号。
+ 1. fieldValue：键值对形式，包含条件列列名及其对应的值。
+ 1. result：计算结果，为键值对形式，包含窗口计算的结果列列名及其对应的值。
+
+#### 计数窗口相关字段
+
+这部分是 windowType 为 Count 时 event 对象才有的字段。
+1. 如果 eventType 为 WINDOW_OPEN，则包含如下字段：
+ 1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。
+1. 如果 eventType 为 WINDOW_CLOSE，则包含如下字段：
+ 1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。
+ 1. windowEnd：长整型时间戳，表示窗口的结束时间，精度与结果表的时间精度一致。
+ 1. result：计算结果，为键值对形式，包含窗口计算的结果列列名及其对应的值。
+
+#### 窗口失效相关字段
+
+因为流计算过程中会遇到数据乱序、更新、删除等情况，可能造成已生成的窗口被删除，或者结果需要重新计算。此时会向通知地址发送一条 WINDOW_INVALIDATION 的通知，说明哪些窗口已经被删除。
+这部分是 eventType 为 WINDOW_INVALIDATION 时，event 对象才有的字段。
+1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。
+1. windowEnd: 长整型时间戳，表示窗口的结束时间，精度与结果表的时间精度一致。
diff --git a/docs/zh/14-reference/03-taos-sql/20-keywords.md b/docs/zh/14-reference/03-taos-sql/20-keywords.md
index 0a1bdc4ce8..9c023e0f92 100644
--- a/docs/zh/14-reference/03-taos-sql/20-keywords.md
+++ b/docs/zh/14-reference/03-taos-sql/20-keywords.md
@@ -37,6 +37,7 @@ description: TDengine 保留关键字的详细列表
| ASOF | |
| AT_ONCE | |
| ATTACH | |
+| AUTO | 3.3.5.0 及后续版本 |
### B
|关键字|说明|
@@ -82,6 +83,9 @@ description: TDengine 保留关键字的详细列表
| COMP | |
| COMPACT | |
| COMPACTS | |
+| COMPACT_INTERVAL | 3.3.5.0 及后续版本 |
+| COMPACT_TIME_OFFSET | 3.3.5.0 及后续版本 |
+| COMPACT_TIME_RANGE | 3.3.5.0 及后续版本 |
| CONCAT | |
| CONFLICT | |
| CONNECTION | |
@@ -111,6 +115,7 @@ description: TDengine 保留关键字的详细列表
| DESC | |
| DESCRIBE | |
| DETACH | |
+| DISK_INFO | 3.3.5.0 及后续版本 |
| DISTINCT | |
| DISTRIBUTED | |
| DIVIDE | |
@@ -201,6 +206,7 @@ description: TDengine 保留关键字的详细列表
| INTO | |
| IPTOKEN | |
| IROWTS | |
+| IROWTS_ORIGIN | 3.3.5.0 及后续版本 |
| IS | |
| IS_IMPORT | |
| ISFILLED | |
@@ -269,6 +275,8 @@ description: TDengine 保留关键字的详细列表
| NONE | |
| NORMAL | |
| NOT | |
+| NOTIFY | 3.3.6.0 及后续版本 |
+| NOTIFY_HISTORY | 3.3.6.0 及后续版本 |
| NOTNULL | |
| NOW | |
| NULL | |
@@ -282,6 +290,7 @@ description: TDengine 保留关键字的详细列表
| OFFSET | |
| ON | |
| ONLY | |
+| ON_FAILURE | 3.3.6.0 及后续版本 |
| OR | |
| ORDER | |
| OUTER | |
@@ -329,6 +338,7 @@ description: TDengine 保留关键字的详细列表
| RATIO | |
| READ | |
| RECURSIVE | |
+| REGEXP | 3.3.6.0 及后续版本 |
| REDISTRIBUTE | |
| REM | |
| REPLACE | |
@@ -475,7 +485,7 @@ description: TDengine 保留关键字的详细列表
| WINDOW_OFFSET | |
| WITH | |
| WRITE | |
-| WSTART | |
+| WSTART | |
### \_
diff --git a/docs/zh/14-reference/05-connector/40-csharp.mdx b/docs/zh/14-reference/05-connector/40-csharp.mdx
index f58b243689..89bba2e443 100644
--- a/docs/zh/14-reference/05-connector/40-csharp.mdx
+++ b/docs/zh/14-reference/05-connector/40-csharp.mdx
@@ -22,14 +22,15 @@ import RequestId from "./_request_id.mdx";
## 版本历史
-| Connector 版本 | 主要变化 | TDengine 版本 |
-|:-------------|:---------------------------|:--------------|
-| 3.1.5 | 修复 websocket 协议编码中文时长度错误 | - |
-| 3.1.4 | 提升 websocket 查询和写入性能 | 3.3.2.0 及更高版本 |
-| 3.1.3 | 支持 WebSocket 自动重连 | - |
-| 3.1.2 | 修复 schemaless 资源释放 | - |
-| 3.1.1 | 支持 varbinary 和 geometry 类型 | - |
-| 3.1.0 | WebSocket 使用原生实现 | 3.2.1.0 及更高版本 |
+| Connector 版本 | 主要变化 | TDengine 版本 |
+|:-------------|:----------------------------|:--------------|
+| 3.1.6 | 优化 WebSocket 连接接收消息处理。 | - |
+| 3.1.5 | 修复 WebSocket 协议编码中文时长度错误。 | - |
+| 3.1.4 | 提升 WebSocket 查询和写入性能。 | 3.3.2.0 及更高版本 |
+| 3.1.3 | 支持 WebSocket 自动重连。 | - |
+| 3.1.2 | 修复 schemaless 资源释放。 | - |
+| 3.1.1 | 支持 varbinary 和 geometry 类型。 | - |
+| 3.1.0 | WebSocket 使用原生实现。 | 3.2.1.0 及更高版本 |
## 处理异常
@@ -53,14 +54,14 @@ TDengine 其他功能模块的报错，请参考 [错误码](../../../reference/
| DOUBLE | double |
| BOOL | bool |
| BINARY | byte[] |
-| NCHAR | string (utf-8编码) |
+| NCHAR | string |
| JSON | byte[] |
| VARBINARY | byte[] |
| GEOMETRY | byte[] |
**注意**：JSON 类型仅在 tag 中支持。
-GEOMETRY类型是 little endian 字节序的二进制数据，符合 WKB 规范。详细信息请参考 [数据类型](../../taos-sql/data-type/#数据类型)
-WKB规范请参考[Well-Known Binary (WKB)](https://libgeos.org/specifications/wkb/)
+GEOMETRY类型是 little endian 字节序的二进制数据，符合 WKB 规范。详细信息请参考 [数据类型](../../taos-sql/data-type/#数据类型)。
+WKB规范请参考[Well-Known Binary (WKB)](https://libgeos.org/specifications/wkb/)。
## 示例程序汇总
diff --git a/docs/zh/14-reference/05-connector/50-odbc.mdx b/docs/zh/14-reference/05-connector/50-odbc.mdx
index 7d71b847c6..579386dd5a 100644
--- a/docs/zh/14-reference/05-connector/50-odbc.mdx
+++ b/docs/zh/14-reference/05-connector/50-odbc.mdx
@@ -652,7 +652,3 @@ TDengine ODBC 支持两种连接 TDengine 数据库方式：WebSocket 连接与
- **标准**: ODBC
- **作用**: 关闭与当前语句句柄关联的游标，并释放游标所使用的所有资源
-
-## 与第三方集成
-
-作为使用 TDengine ODBC driver 的一个示例，你可以使用 Power BI 与 TDengine 分析时序数据。更多细节请参考 [Power BI](../../../third-party/bi/powerbi)
diff --git a/docs/zh/27-train-faq/01-faq.md b/docs/zh/27-train-faq/01-faq.md
index 12a21df10b..d1120954da 100644
--- a/docs/zh/27-train-faq/01-faq.md
+++ b/docs/zh/27-train-faq/01-faq.md
@@ -81,7 +81,7 @@ description: 一些常见问题的解决方法汇总
### 5. 遇到错误 Unable to resolve FQDN” 怎么办？
-产生这个错误，是由于客户端或数据节点无法解析 FQDN(Fully Qualified Domain Name)导致。对于 TAOS Shell 或客户端应用，请做如下检查：
+产生这个错误，是由于客户端或数据节点无法解析 FQDN(Fully Qualified Domain Name)导致。对于 TDengine CLI 或客户端应用，请做如下检查：
1. 请检查连接的服务器的 FQDN 是否正确，FQDN 配置参考：[一篇文章说清楚 TDengine 的 FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)
2. 如果网络配置有 DNS server，请检查是否正常工作
@@ -247,13 +247,13 @@ launchctl limit maxfiles
该提示是创建 db 的 vnode 数量不够了，需要的 vnode 不能超过了 dnode 中 vnode 的上限。因为系统默认是一个 dnode 中有 CPU 核数两倍的 vnode，也可以通过配置文件中的参数 supportVnodes 控制。
正常调大 taos.cfg 中 supportVnodes 参数即可。
-### 21 在服务器上的使用 taos-CLI 能查到指定时间段的数据，但在客户端机器上查不到？
+### 21 在服务器上的使用 TDengine CLI能查到指定时间段的数据，但在客户端机器上查不到？
这种情况是因为客户端与服务器上设置的时区不一致导致的，调整客户端与服务器的时区一致即可解决。
### 22 表名确认是存在的，但在写入或查询时返回表名不存在，什么原因？
-TDengine 中的所有名称，包括数据库名、表名等都是区分大小写的，如果这些名称在程序或 taos-CLI 中没有使用反引号（`）括起来使用，即使你输入的是大写的，引擎也会转化成小写来使用，如果名称前后加上了反引号，引擎就不会再转化成小写，会保持原样来使用。
+TDengine 中的所有名称，包括数据库名、表名等都是区分大小写的，如果这些名称在程序或 TDengine CLI中没有使用反引号（`）括起来使用，即使你输入的是大写的，引擎也会转化成小写来使用，如果名称前后加上了反引号，引擎就不会再转化成小写，会保持原样来使用。
-### 23 在 taos-CLI 中查询，字段内容不能完全显示出来怎么办？
+### 23 在 TDengine CLI中查询，字段内容不能完全显示出来怎么办？
可以使用 \G 参数来竖式显示，如 show databases\G; （为了输入方便，在"\"后加 TAB 键，会自动补全后面的内容）
### 24 使用 taosBenchmark 测试工具写入数据查询很快，为什么我写入的数据查询非常慢？
@@ -311,7 +311,7 @@ TDinsight插件中展示的数据是通过taosKeeper和taosAdapter服务收集
直接查子表更快。超级表带 TAG 过滤查询子查数据是为满足查询方便性，同时可对多个子表中数据进行过滤，如果目的是追求性能并已明确查询子表，直接从子表查性能更高
### 35 如何查看数据库的数据压缩率和磁盘占用指标？
-TDengine 3.3.5.0 之前的版本，只提供以表为统计单位的压缩率，数据库及整体还未提供，查看命令是在客户端 taos-CLI 中执行 `SHOW TABLE DISTRIBUTED table_name;` 命令，table_name 为要查看压缩率的表，可以为超级表、普通表及子表，详细可 [查看此处](https://docs.taosdata.com/reference/taos-sql/show/#show-table-distributed)
+TDengine 3.3.5.0 之前的版本，只提供以表为统计单位的压缩率，数据库及整体还未提供，查看命令是在客户端 TDengine CLI 中执行 `SHOW TABLE DISTRIBUTED table_name;` 命令，table_name 为要查看压缩率的表，可以为超级表、普通表及子表，详细可 [查看此处](https://docs.taosdata.com/reference/taos-sql/show/#show-table-distributed)
TDengine 3.3.5.0 及以上的版本，还提供了数据库整体压缩率和磁盘空间占用统计。查看数据库整体的数据压缩率和磁盘空间占用的命令为 `SHOW db_name.disk_info;`，查看数据库各个模块的磁盘空间占用的命令为 `SELECT * FROM INFORMATION_SCHEMA.INS_DISK_USAGE WHERE db_name='db_name';`，db_name 为要查看的数据库名称。详细可 [查看此处](https://docs.taosdata.com/reference/taos-sql/database/#%E6%9F%A5%E7%9C%8B-db-%E7%9A%84%E7%A3%81%E7%9B%98%E7%A9%BA%E9%97%B4%E5%8D%A0%E7%94%A8)
@@ -324,3 +324,4 @@ TDengine 3.3.5.1 之前的版本，StartLimitInterval 为 60 秒。若在 60 秒
问题解决：
1）通过 systemd 重启 taosd 服务：推荐方法是先执行命令 `systemctl reset-failed taosd.service` 重置失败计数器，然后再通过 `systemctl restart taosd.service` 重启；若需长期调整，可手动修改 /etc/systemd/system/taosd.service 文件，将 StartLimitInterval 调小或将 StartLimitBurst 调大(注：重新安装 taosd 会重置该参数，需要重新修改)，执行 `systemctl daemon-reload` 重新加载配置，然后再重启。2）也可以不通过 systemd 而是通过 taosd 命令直接重启 taosd 服务，此时不受 StartLimitInterval 和 StartLimitBurst 参数限制。
+
diff --git a/docs/zh/28-releases/03-notes/3.3.2.0.md b/docs/zh/28-releases/03-notes/3.3.2.0.md
index 8a6b20a5fe..57e76cb4e1 100644
--- a/docs/zh/28-releases/03-notes/3.3.2.0.md
+++ b/docs/zh/28-releases/03-notes/3.3.2.0.md
@@ -50,4 +50,4 @@ description: 3.3.2.0 版本说明
13. 升级至3.3.0.0开启cachemodel后查询last+group by返回行数不正确
14. taos-explorer导航栏未显示所有超级表名（仅企业版）
15. 复合主键VARCHAR长度超125遇查询导致taosd异常退出
-16. taos CLI和taosAdapter占用CPU过高
+16. TDengine CLI 和taosAdapter占用CPU过高
diff --git a/docs/zh/28-releases/03-notes/3.3.3.0.md b/docs/zh/28-releases/03-notes/3.3.3.0.md
index bb7bc0f831..dffe1960ad 100644
--- a/docs/zh/28-releases/03-notes/3.3.3.0.md
+++ b/docs/zh/28-releases/03-notes/3.3.3.0.md
@@ -49,7 +49,7 @@ description: 3.3.3.0 版本说明
22. 缓存更新时数据填充的游标错误导致的 taosd 异常退出
23. STDDEV 函数计算结果随机不正确的问题
24. 多级存储以及加密场景下无法添加离线节点
-25. taos CLI 无法输入大于20字节长度的密码
+25. TDengine CLI 无法输入大于20字节长度的密码
26. 拼接 sql 写入报错: int data overflow
27. 大量查询并发场景下元数据的一致性
28. 尝试解决手动点击停止按钮，任务无法停止的问题
@@ -86,4 +86,4 @@ description: 3.3.3.0 版本说明
59. 客户端内存泄漏
60. 开源用户修改stt_trigger值升级后无法修改其他数据库选项
61. NOT IN (NULL) 查询结果不正确
-62. taos shell和taosBenchmark不能成功连接云服务实例
+62. TDengine CLI 和 taosBenchmark 不能成功连接云服务实例
diff --git a/docs/zh/28-releases/03-notes/3.3.4.3.md b/docs/zh/28-releases/03-notes/3.3.4.3.md
index 8ffd5802ed..7a2a1c7c79 100644
--- a/docs/zh/28-releases/03-notes/3.3.4.3.md
+++ b/docs/zh/28-releases/03-notes/3.3.4.3.md
@@ -57,7 +57,7 @@ description: 3.3.4.3 版本说明
1. 修复：Windows 平台重复增删表的内存泄漏
1. 修复：无法限制并发拉起 checkpoint 数量导致流计算消耗资源过多
1. 修复：并发查询时的 too many session 问题
-1. 修复：Windows 平台下 taos shell 在慢查询场景中崩溃的问题
+1. 修复：Windows 平台下 TDengine CLI 在慢查询场景中崩溃的问题
1. 修复：当打开 dnode日志时，加密数据库无法恢复的问题
1. 修复：由于 mnode 同步超时，进而导致 taosd 无法启动的问题
1. 修复：由于在快照同步过程中整理文件组数据的速度过慢，从而导致 Vnode（虚拟节点）无法恢复的问题
diff --git a/docs/zh/28-releases/03-notes/3.3.5.0.md b/docs/zh/28-releases/03-notes/3.3.5.0.md
index 7e16516344..278692b98a 100755
--- a/docs/zh/28-releases/03-notes/3.3.5.0.md
+++ b/docs/zh/28-releases/03-notes/3.3.5.0.md
@@ -9,7 +9,7 @@ description: 3.3.5.0 版本说明
2. 特性：taosX 增量备份与恢复
3. 特性：JDBC WebSocket 连接支持 STMT2 接口
4. 特性：Rust 连接器支持 STMT2 接口
- 5. 特性：taos-CLI 中在错误提示中增加错误码
+ 5. 特性：TDengine CLI 中在错误提示中增加错误码
6. 特性：Python 连接器对接 SuperSet
7. 特性：Explorer 可配置 Grafana Dashboard
8. 特性：taosX-agent 支持配置内存缓存队列长度
diff --git a/docs/zh/28-releases/03-notes/3.3.5.2.md b/docs/zh/28-releases/03-notes/3.3.5.2.md
index dc2734c50b..33bfedc96e 100755
--- a/docs/zh/28-releases/03-notes/3.3.5.2.md
+++ b/docs/zh/28-releases/03-notes/3.3.5.2.md
@@ -37,6 +37,6 @@ description: 3.3.5.2 版本说明
20. 修复：在使用 SELECT * FROM 子查询时，列名未能正确复制到外层查询
21. 修复：对字符串类型数据执行 max/min 函数时，结果不准确且 taosd 可能会崩溃
22. 修复：流式计算不支持使用 HAVING 语句，但在创建时未报告错误
- 23. 修复：taos shell 显示的服务端版本信息不准确，例如无法正确区分社区版和企业版
+ 23. 修复：TDengine CLI 显示的服务端版本信息不准确，例如无法正确区分社区版和企业版
24. 修复：在某些特定的查询场景下，当 JOIN 和 CAST 联合使用时，taosd 可能会崩溃
diff --git a/include/client/taos.h b/include/client/taos.h
index 17f97d3d3d..433779f811 100644
--- a/include/client/taos.h
+++ b/include/client/taos.h
@@ -247,11 +247,13 @@ typedef struct TAOS_STMT2_BINDV {
DLL_EXPORT TAOS_STMT2 *taos_stmt2_init(TAOS *taos, TAOS_STMT2_OPTION *option);
DLL_EXPORT int taos_stmt2_prepare(TAOS_STMT2 *stmt, const char *sql, unsigned long length);
DLL_EXPORT int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col_idx);
-DLL_EXPORT int taos_stmt2_exec(TAOS_STMT2 *stmt, int *affected_rows);
-DLL_EXPORT int taos_stmt2_close(TAOS_STMT2 *stmt);
-DLL_EXPORT int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert);
-DLL_EXPORT int taos_stmt2_get_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_ALL **fields);
-DLL_EXPORT void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_ALL *fields);
+DLL_EXPORT int taos_stmt2_bind_param_a(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col_idx, __taos_async_fn_t fp,
+ void *param);
+DLL_EXPORT int taos_stmt2_exec(TAOS_STMT2 *stmt, int *affected_rows);
+DLL_EXPORT int taos_stmt2_close(TAOS_STMT2 *stmt);
+DLL_EXPORT int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert);
+DLL_EXPORT int taos_stmt2_get_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_ALL **fields);
+DLL_EXPORT void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_ALL *fields);
DLL_EXPORT TAOS_RES *taos_stmt2_result(TAOS_STMT2 *stmt);
DLL_EXPORT char *taos_stmt2_error(TAOS_STMT2 *stmt);
@@ -360,7 +362,8 @@ typedef enum tmq_res_t {
TMQ_RES_INVALID = -1,
TMQ_RES_DATA = 1,
TMQ_RES_TABLE_META = 2,
- TMQ_RES_METADATA = 3
+ TMQ_RES_METADATA = 3,
+ TMQ_RES_RAWDATA = 4
} tmq_res_t;
typedef struct tmq_topic_assignment {
diff --git a/include/common/tcommon.h b/include/common/tcommon.h
index c30f2ab4ec..bd5bdb927d 100644
--- a/include/common/tcommon.h
+++ b/include/common/tcommon.h
@@ -121,10 +121,11 @@ enum {
TMQ_MSG_TYPE__POLL_DATA_META_RSP,
TMQ_MSG_TYPE__WALINFO_RSP,
TMQ_MSG_TYPE__POLL_BATCH_META_RSP,
+ TMQ_MSG_TYPE__POLL_RAW_DATA_RSP,
};
static char* tmqMsgTypeStr[] = {
- "data", "meta", "ask ep", "meta data", "wal info", "batch meta"
+ "data", "meta", "ask ep", "meta data", "wal info", "batch meta", "raw data"
};
enum {
diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h
index 96478047ca..aa07183592 100644
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@@ -195,6 +195,8 @@ int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const
// For the VAR_DATA_TYPE type, if a row already has data before inserting it (judged by offset != -1),
// it will be inserted at the original position and the old data will be overwritten.
int32_t colDataSetValOrCover(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull);
+int32_t varColSetVarData(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pVarData, int32_t varDataLen,
+ bool isNull);
int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx, const char* pData);
int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows,
bool trimValue);
diff --git a/include/common/tglobal.h b/include/common/tglobal.h
index bb3f0964de..1b59f1de8a 100644
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@@ -296,6 +296,8 @@ extern int32_t tsMaxStreamBackendCache;
extern int32_t tsPQSortMemThreshold;
extern bool tsStreamCoverage;
extern int8_t tsS3EpNum;
+extern int32_t tsStreamNotifyMessageSize;
+extern int32_t tsStreamNotifyFrameSize;
extern bool tsExperimental;
// #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index c347c3f38a..6d58748a3b 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -1355,10 +1355,10 @@ typedef struct {
int8_t encryptAlgorithm;
char dnodeListStr[TSDB_DNODE_LIST_LEN];
// 1. add auto-compact parameters
- int32_t compactInterval; // minutes
- int32_t compactStartTime; // minutes
- int32_t compactEndTime; // minutes
- int8_t compactTimeOffset; // hour
+ int32_t compactInterval; // minutes
+ int32_t compactStartTime; // minutes
+ int32_t compactEndTime; // minutes
+ int8_t compactTimeOffset; // hour
} SCreateDbReq;
int32_t tSerializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq);
@@ -1777,6 +1777,8 @@ typedef struct {
int64_t numOfBatchInsertSuccessReqs;
int32_t numOfCachedTables;
int32_t learnerProgress; // use one reservered
+ int64_t syncAppliedIndex;
+ int64_t syncCommitIndex;
} SVnodeLoad;
typedef struct {
@@ -2312,6 +2314,10 @@ typedef struct SSysTableSchema {
int32_t tSerializeSRetrieveTableReq(void* buf, int32_t bufLen, SRetrieveTableReq* pReq);
int32_t tDeserializeSRetrieveTableReq(void* buf, int32_t bufLen, SRetrieveTableReq* pReq);
+#define RETRIEVE_TABLE_RSP_VERSION 0
+#define RETRIEVE_TABLE_RSP_TMQ_VERSION 1
+#define RETRIEVE_TABLE_RSP_TMQ_RAW_VERSION 2
+
typedef struct {
int64_t useconds;
int8_t completed; // all results are returned to client
@@ -2966,6 +2972,22 @@ typedef struct {
int8_t notifyHistory;
} SCMCreateStreamReq;
+typedef struct STaskNotifyEventStat {
+ int64_t notifyEventAddTimes; // call times of add function
+ int64_t notifyEventAddElems; // elements added by add function
+ double notifyEventAddCostSec; // time cost of add function
+ int64_t notifyEventPushTimes; // call times of push function
+ int64_t notifyEventPushElems; // elements pushed by push function
+ double notifyEventPushCostSec; // time cost of push function
+ int64_t notifyEventPackTimes; // call times of pack function
+ int64_t notifyEventPackElems; // elements packed by pack function
+ double notifyEventPackCostSec; // time cost of pack function
+ int64_t notifyEventSendTimes; // call times of send function
+ int64_t notifyEventSendElems; // elements sent by send function
+ double notifyEventSendCostSec; // time cost of send function
+ int64_t notifyEventHoldElems; // elements hold due to watermark
+} STaskNotifyEventStat;
+
typedef struct {
int64_t streamId;
} SCMCreateStreamRsp;
@@ -2980,8 +3002,10 @@ enum {
TOPIC_SUB_TYPE__COLUMN,
};
-#define DEFAULT_MAX_POLL_INTERVAL 300000
-#define DEFAULT_SESSION_TIMEOUT 12000
+#define DEFAULT_MAX_POLL_INTERVAL 300000
+#define DEFAULT_SESSION_TIMEOUT 12000
+#define DEFAULT_MAX_POLL_WAIT_TIME 1000
+#define DEFAULT_MIN_POLL_ROWS 4096
typedef struct {
char name[TSDB_TOPIC_FNAME_LEN]; // accout.topic
@@ -3939,8 +3963,8 @@ typedef struct {
int8_t igExists;
int8_t intervalUnit;
int8_t slidingUnit;
- int8_t timezone; // int8_t is not enough, timezone is unit of second
- int32_t dstVgId; // for stream
+ int8_t timezone; // int8_t is not enough, timezone is unit of second
+ int32_t dstVgId; // for stream
int64_t interval;
int64_t offset;
int64_t sliding;
@@ -4184,7 +4208,10 @@ typedef struct {
STqOffsetVal reqOffset;
int8_t enableReplay;
int8_t sourceExcluded;
+ int8_t rawData;
+ int32_t minPollRows;
int8_t enableBatchMeta;
+ SHashObj *uidHash; // to find if uid is duplicated
} SMqPollReq;
int32_t tSerializeSMqPollReq(void* buf, int32_t bufLen, SMqPollReq* pReq);
@@ -4258,13 +4285,21 @@ typedef struct {
SArray* createTableLen;
SArray* createTableReq;
};
+ struct{
+ int32_t len;
+ void* rawData;
+ };
};
+ void* data; //for free in client, only effected if type is data or metadata. raw data not effected
+ bool blockDataElementFree; // if true, free blockDataElement in blockData,(true in server, false in client)
} SMqDataRsp;
int32_t tEncodeMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pObj);
int32_t tDecodeMqDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp);
+int32_t tDecodeMqRawDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp);
void tDeleteMqDataRsp(SMqDataRsp* pRsp);
+void tDeleteMqRawDataRsp(SMqDataRsp* pRsp);
int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp);
int32_t tDecodeSTaosxRsp(SDecoder* pDecoder, SMqDataRsp* pRsp);
@@ -4514,6 +4549,7 @@ typedef struct {
typedef struct {
SArray* aSubmitTbData; // SArray
+ bool raw;
} SSubmitReq2;
typedef struct {
@@ -4522,8 +4558,9 @@ typedef struct {
char data[]; // SSubmitReq2
} SSubmitReq2Msg;
+int32_t transformRawSSubmitTbData(void* data, int64_t suid, int64_t uid, int32_t sver);
int32_t tEncodeSubmitReq(SEncoder* pCoder, const SSubmitReq2* pReq);
-int32_t tDecodeSubmitReq(SDecoder* pCoder, SSubmitReq2* pReq);
+int32_t tDecodeSubmitReq(SDecoder* pCoder, SSubmitReq2* pReq, SArray* rawList);
void tDestroySubmitTbData(SSubmitTbData* pTbData, int32_t flag);
void tDestroySubmitReq(SSubmitReq2* pReq, int32_t flag);
diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h
index 9a7c3912b0..f05234b82f 100644
--- a/include/libs/executor/executor.h
+++ b/include/libs/executor/executor.h
@@ -99,7 +99,7 @@ int32_t qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId);
int32_t qSetStreamOpOpen(qTaskInfo_t tinfo);
int32_t qSetStreamNotifyInfo(qTaskInfo_t tinfo, int32_t eventTypes, const SSchemaWrapper* pSchemaWrapper,
- const char* stbFullName, bool newSubTableRule);
+ const char* stbFullName, bool newSubTableRule, STaskNotifyEventStat* pNotifyEventStat);
/**
* Set multiple input data blocks for the stream scan.
diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h
index 3cc2acf30f..75252e8d9f 100644
--- a/include/libs/executor/storageapi.h
+++ b/include/libs/executor/storageapi.h
@@ -431,6 +431,7 @@ typedef struct SStateStore {
int32_t (*updateInfoSerialize)(SEncoder* pEncoder, const SUpdateInfo* pInfo);
int32_t (*updateInfoDeserialize)(SDecoder* pDeCoder, SUpdateInfo* pInfo);
+ SStreamStateCur* (*streamStateSessionSeekKeyPrev)(SStreamState* pState, const SSessionKey* key);
SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key);
SStreamStateCur* (*streamStateCountSeekKeyPrev)(SStreamState* pState, const SSessionKey* pKey, COUNT_TYPE count);
SStreamStateCur* (*streamStateSessionSeekKeyCurrentPrev)(SStreamState* pState, const SSessionKey* key);
diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h
index 76db5e29a4..2cd743b37e 100644
--- a/include/libs/nodes/cmdnodes.h
+++ b/include/libs/nodes/cmdnodes.h
@@ -573,6 +573,7 @@ typedef enum EStreamNotifyOptionSetFlag {
} EStreamNotifyOptionSetFlag;
typedef enum EStreamNotifyEventType {
+ SNOTIFY_EVENT_WINDOW_INVALIDATION = 0,
SNOTIFY_EVENT_WINDOW_OPEN = BIT_FLAG_MASK(0),
SNOTIFY_EVENT_WINDOW_CLOSE = BIT_FLAG_MASK(1),
} EStreamNotifyEventType;
diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h
index 3e95f1e286..1afec35c3c 100644
--- a/include/libs/nodes/plannodes.h
+++ b/include/libs/nodes/plannodes.h
@@ -128,6 +128,7 @@ typedef struct SScanLogicNode {
bool paraTablesSort; // for table merge scan
bool smallDataTsSort; // disable row id sort for table merge scan
bool needSplit;
+ bool noPseudoRefAfterGrp; // no pseudo columns referenced ater group/partition clause
} SScanLogicNode;
typedef struct SJoinLogicNode {
diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h
index 95f522f504..f4bf5fafd4 100644
--- a/include/libs/parser/parser.h
+++ b/include/libs/parser/parser.h
@@ -180,8 +180,11 @@ int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsS
STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl,
char* msgBuf, int32_t msgBufLen, void* charsetCxt);
int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash);
+int32_t smlBuildOutputRaw(SQuery* handle, SHashObj* pVgHash);
+int rawBlockBindRawData(SHashObj* pVgroupHash, SArray* pVgroupList, STableMeta* pTableMeta, void* data);
int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, void* fields,
int numFields, bool needChangeLength, char* errstr, int32_t errstrLen, bool raw);
+int32_t checkSchema(SSchema* pColSchema, int8_t* fields, char* errstr, int32_t errstrLen);
int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray);
int32_t serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap, SArray** pOut);
diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h
index 5b28eadc4f..a17e67279c 100644
--- a/include/libs/qcom/query.h
+++ b/include/libs/qcom/query.h
@@ -300,6 +300,7 @@ int32_t cleanupTaskQueue();
int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code);
int32_t taosAsyncWait();
int32_t taosAsyncRecover();
+int32_t taosStmt2AsyncBind(__async_exec_fn_t execFn, void* execParam);
void destroySendMsgInfo(SMsgSendInfo* pMsgBody);
diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h
index b4e0087b1a..52a61e9452 100644
--- a/include/libs/stream/streamState.h
+++ b/include/libs/stream/streamState.h
@@ -65,6 +65,7 @@ int32_t streamStateCountGetKeyByRange(SStreamState* pState, const SSessionKey* r
int32_t streamStateSessionAllocWinBuffByNextPosition(SStreamState* pState, SStreamStateCur* pCur,
const SSessionKey* pKey, void** pVal, int32_t* pVLen);
+SStreamStateCur *streamStateSessionSeekKeyPrev(SStreamState *pState, const SSessionKey *key);
SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key);
SStreamStateCur* streamStateCountSeekKeyPrev(SStreamState* pState, const SSessionKey* pKey, COUNT_TYPE count);
SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key);
@@ -162,4 +163,4 @@ int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2);
}
#endif
-#endif /* ifndef _STREAM_STATE_H_ */
\ No newline at end of file
+#endif /* ifndef _STREAM_STATE_H_ */
diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 9cd6dd13ca..041d888d33 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -463,6 +463,7 @@ struct SStreamTask {
SUpstreamInfo upstreamInfo;
STaskCheckInfo taskCheckInfo;
SNotifyInfo notifyInfo;
+ STaskNotifyEventStat notifyEventStat;
// the followings attributes don't be serialized
SScanhistorySchedInfo schedHistoryInfo;
@@ -632,6 +633,7 @@ typedef struct STaskStatusEntry {
int64_t startCheckpointVer;
int64_t hTaskId;
STaskCkptInfo checkpointInfo;
+ STaskNotifyEventStat notifyEventStat;
} STaskStatusEntry;
//typedef struct SNodeUpdateInfo {
diff --git a/include/libs/stream/tstreamFileState.h b/include/libs/stream/tstreamFileState.h
index f47c308e18..f07034adda 100644
--- a/include/libs/stream/tstreamFileState.h
+++ b/include/libs/stream/tstreamFileState.h
@@ -100,10 +100,12 @@ void sessionWinStateCleanup(void* pBuff);
SStreamStateCur* createStateCursor(SStreamFileState* pFileState);
SStreamStateCur* sessionWinStateSeekKeyCurrentPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey);
SStreamStateCur* sessionWinStateSeekKeyCurrentNext(SStreamFileState* pFileState, const SSessionKey* pWinKey);
+SStreamStateCur* sessionWinStateSeekKeyPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey);
SStreamStateCur* sessionWinStateSeekKeyNext(SStreamFileState* pFileState, const SSessionKey* pWinKey);
SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey, COUNT_TYPE count);
int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
void sessionWinStateMoveToNext(SStreamStateCur* pCur);
+void sessionWinStateMoveToPrev(SStreamStateCur* pCur);
int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey,
range_cmpr_fn cmpFn);
diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h
index c191e96d0e..643d459791 100644
--- a/include/libs/sync/sync.h
+++ b/include/libs/sync/sync.h
@@ -293,6 +293,7 @@ int32_t syncBecomeAssignedLeader(SSyncNode* ths, SRpcMsg* pRpcMsg);
int32_t syncUpdateArbTerm(int64_t rid, SyncTerm arbTerm);
SSyncState syncGetState(int64_t rid);
+void syncGetCommitIndex(int64_t rid, int64_t* syncCommitIndex);
int32_t syncGetArbToken(int64_t rid, char* outToken);
int32_t syncCheckSynced(int64_t rid);
void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet);
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 464dffa937..397118411c 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -1016,6 +1016,8 @@ int32_t taosGetErrSize();
#define TSDB_CODE_TMQ_NO_TABLE_QUALIFIED TAOS_DEF_ERROR_CODE(0, 0x4015)
#define TSDB_CODE_TMQ_NO_NEED_REBALANCE TAOS_DEF_ERROR_CODE(0, 0x4016)
#define TSDB_CODE_TMQ_INVALID_STATUS TAOS_DEF_ERROR_CODE(0, 0x4017)
+#define TSDB_CODE_TMQ_INVALID_DATA TAOS_DEF_ERROR_CODE(0, 0x4018)
+#define TSDB_CODE_TMQ_RAW_DATA_SPLIT TAOS_DEF_ERROR_CODE(0, 0x4019)
// stream
#define TSDB_CODE_STREAM_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x4100)
diff --git a/include/util/tdef.h b/include/util/tdef.h
index 6d346290bc..41cb147622 100644
--- a/include/util/tdef.h
+++ b/include/util/tdef.h
@@ -239,6 +239,7 @@ typedef enum ELogicConditionType {
#define TSDB_USER_CGROUP_LEN (TSDB_USER_LEN + TSDB_CGROUP_LEN) // it is a null-terminated string
#define TSDB_STREAM_NAME_LEN 193 // it is a null-terminated string
#define TSDB_STREAM_NOTIFY_URL_LEN 128 // it includes the terminating '\0'
+#define TSDB_STREAM_NOTIFY_STAT_LEN 350 // it includes the terminating '\0'
#define TSDB_DB_NAME_LEN 65
#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_PRIVILEDGE_CONDITION_LEN 48 * 1024
@@ -321,8 +322,8 @@ typedef enum ELogicConditionType {
#define TSDB_ARB_GROUP_MEMBER_NUM 2
#define TSDB_ARB_TOKEN_SIZE 32
-#define TSDB_TRANS_STAGE_LEN 12
-#define TSDB_TRANS_TYPE_LEN 16
+#define TSDB_TRANS_STAGE_LEN 12
+#define TSDB_TRANS_TYPE_LEN 16
#define TSDB_TRANS_ERROR_LEN 512
#define TSDB_TRANS_OBJTYPE_LEN 40
#define TSDB_TRANS_RESULT_LEN 100
@@ -356,6 +357,8 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_REPLICA 5
#define TSDB_MAX_LEARNER_REPLICA 10
+#define TSDB_SYNC_RESTORE_lEN 20
+#define TSDB_SYNC_APPLY_COMMIT_LEN 41
#define TSDB_SYNC_LOG_BUFFER_SIZE 4096
#define TSDB_SYNC_LOG_BUFFER_RETENTION 256
#define TSDB_SYNC_LOG_BUFFER_THRESHOLD (1024 * 1024 * 5)
@@ -671,7 +674,7 @@ typedef enum {
TSDB_VERSION_END,
} EVersionType;
-#define MIN_RESERVE_MEM_SIZE 1024 // MB
+#define MIN_RESERVE_MEM_SIZE 1024 // MB
#ifdef __cplusplus
}
diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h
index 2543a1f3ec..c9e0410341 100644
--- a/source/client/inc/clientInt.h
+++ b/source/client/inc/clientInt.h
@@ -45,6 +45,7 @@ enum {
RES_TYPE__TMQ_META,
RES_TYPE__TMQ_METADATA,
RES_TYPE__TMQ_BATCH_META,
+ RES_TYPE__TMQ_RAWDATA,
};
#define SHOW_VARIABLES_RESULT_COLS 5
@@ -55,6 +56,7 @@ enum {
#define SHOW_VARIABLES_RESULT_FIELD5_LEN (TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE)
#define TD_RES_QUERY(res) (*(int8_t*)(res) == RES_TYPE__QUERY)
+#define TD_RES_TMQ_RAW(res) (*(int8_t*)(res) == RES_TYPE__TMQ_RAWDATA)
#define TD_RES_TMQ(res) (*(int8_t*)(res) == RES_TYPE__TMQ)
#define TD_RES_TMQ_META(res) (*(int8_t*)(res) == RES_TYPE__TMQ_META)
#define TD_RES_TMQ_METADATA(res) (*(int8_t*)(res) == RES_TYPE__TMQ_METADATA)
diff --git a/source/client/inc/clientStmt2.h b/source/client/inc/clientStmt2.h
index 05a4c849f8..283573803e 100644
--- a/source/client/inc/clientStmt2.h
+++ b/source/client/inc/clientStmt2.h
@@ -133,6 +133,19 @@ SStmtQNode* tail;
uint64_t qRemainNum;
} SStmtQueue;
*/
+typedef struct {
+ TAOS_STMT2 *stmt;
+ TAOS_STMT2_BINDV *bindv;
+ int32_t col_idx;
+ __taos_async_fn_t fp;
+ void *param;
+} ThreadArgs;
+
+typedef struct AsyncBindParam {
+ TdThreadMutex mutex;
+ TdThreadCond waitCond;
+ uint8_t asyncBindNum;
+} AsyncBindParam;
typedef struct {
STscObj *taos;
@@ -150,12 +163,13 @@ typedef struct {
SStmtExecInfo exec;
SStmtBindInfo bInfo;
- char *db;
- int64_t reqid;
- int32_t errCode;
- tsem_t asyncQuerySem;
- bool semWaited;
- SStmtStatInfo stat;
+ char *db;
+ int64_t reqid;
+ int32_t errCode;
+ tsem_t asyncExecSem;
+ bool execSemWaited;
+ AsyncBindParam asyncBindParam;
+ SStmtStatInfo stat;
} STscStmt2;
/*
extern char *gStmtStatusStr[];
@@ -226,6 +240,8 @@ int stmtGetParamNum2(TAOS_STMT2 *stmt, int *nums);
int stmtIsInsert2(TAOS_STMT2 *stmt, int *insert);
TAOS_RES *stmtUseResult2(TAOS_STMT2 *stmt);
const char *stmtErrstr2(TAOS_STMT2 *stmt);
+int stmt2AsyncBind(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col_idx, __taos_async_fn_t fp, void *param);
+int stmtAsyncBindThreadFunc(void *args);
#ifdef __cplusplus
}
diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c
index 0d4af75d04..11bdb16eca 100644
--- a/source/client/src/clientEnv.c
+++ b/source/client/src/clientEnv.c
@@ -38,7 +38,6 @@
#include "tversion.h"
#include "tconv.h"
-
#include "cus_name.h"
#define TSC_VAR_NOT_RELEASE 1
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index 27e21d9a58..184f73c664 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -502,7 +502,7 @@ void taos_close(TAOS *taos) {
}
int taos_errno(TAOS_RES *res) {
- if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
+ if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return terrno;
}
@@ -514,7 +514,7 @@ int taos_errno(TAOS_RES *res) {
}
const char *taos_errstr(TAOS_RES *res) {
- if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
+ if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return (const char *)tstrerror(terrno);
}
@@ -554,6 +554,8 @@ void taos_free_result(TAOS_RES *res) {
tDeleteMqMetaRsp(&pRsp->metaRsp);
} else if (TD_RES_TMQ_BATCH_META(res)) {
tDeleteMqBatchMetaRsp(&pRsp->batchMetaRsp);
+ } else if (TD_RES_TMQ_RAW(res)) {
+ tDeleteMqRawDataRsp(&pRsp->dataRsp);
}
taosMemoryFree(pRsp);
}
@@ -572,7 +574,7 @@ void taos_kill_query(TAOS *taos) {
}
int taos_field_count(TAOS_RES *res) {
- if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
+ if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return 0;
}
@@ -583,7 +585,7 @@ int taos_field_count(TAOS_RES *res) {
int taos_num_fields(TAOS_RES *res) { return taos_field_count(res); }
TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
- if (taos_num_fields(res) == 0 || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
+ if (taos_num_fields(res) == 0 || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return NULL;
}
@@ -643,7 +645,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
return NULL;
} else {
tscError("invalid result passed to taos_fetch_row");
- terrno = TSDB_CODE_TSC_INTERNAL_ERROR;
+ terrno = TSDB_CODE_TMQ_INVALID_DATA;
return NULL;
}
}
@@ -764,7 +766,7 @@ int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD
}
int *taos_fetch_lengths(TAOS_RES *res) {
- if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
+ if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return NULL;
}
@@ -773,7 +775,7 @@ int *taos_fetch_lengths(TAOS_RES *res) {
}
TAOS_ROW *taos_result_block(TAOS_RES *res) {
- if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
+ if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
@@ -841,7 +843,7 @@ const char *taos_get_client_info() { return td_version; }
// return int32_t
int taos_affected_rows(TAOS_RES *res) {
- if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) ||
+ if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) ||
TD_RES_TMQ_BATCH_META(res)) {
return 0;
}
@@ -853,7 +855,7 @@ int taos_affected_rows(TAOS_RES *res) {
// return int64_t
int64_t taos_affected_rows64(TAOS_RES *res) {
- if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) ||
+ if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) ||
TD_RES_TMQ_BATCH_META(res)) {
return 0;
}
@@ -864,7 +866,7 @@ int64_t taos_affected_rows64(TAOS_RES *res) {
}
int taos_result_precision(TAOS_RES *res) {
- if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
+ if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return TSDB_TIME_PRECISION_MILLI;
}
@@ -904,7 +906,7 @@ int taos_select_db(TAOS *taos, const char *db) {
}
void taos_stop_query(TAOS_RES *res) {
- if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) ||
+ if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) ||
TD_RES_TMQ_BATCH_META(res)) {
return;
}
@@ -913,7 +915,7 @@ void taos_stop_query(TAOS_RES *res) {
}
bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) {
- if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
+ if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return true;
}
SReqResultInfo *pResultInfo = tscGetCurResInfo(res);
@@ -938,7 +940,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
}
int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
- if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
+ if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return 0;
}
@@ -973,7 +975,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
return 0;
} else {
tscError("taos_fetch_block_s invalid res type");
- return -1;
+ return TSDB_CODE_TMQ_INVALID_DATA;
}
}
@@ -981,7 +983,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
*numOfRows = 0;
*pData = NULL;
- if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
+ if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return 0;
}
@@ -1018,7 +1020,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
}
int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) {
- if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
+ if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return 0;
}
@@ -1038,7 +1040,7 @@ int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) {
int taos_is_null_by_column(TAOS_RES *res, int columnIndex, bool result[], int *rows) {
if (res == NULL || result == NULL || rows == NULL || *rows <= 0 || columnIndex < 0 || TD_RES_TMQ_META(res) ||
- TD_RES_TMQ_BATCH_META(res)) {
+ TD_RES_TMQ_RAW(res) || TD_RES_TMQ_BATCH_META(res)) {
return TSDB_CODE_INVALID_PARA;
}
@@ -2166,11 +2168,16 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col
}
STscStmt2 *pStmt = (STscStmt2 *)stmt;
- if (pStmt->options.asyncExecFn && !pStmt->semWaited) {
- if (tsem_wait(&pStmt->asyncQuerySem) != 0) {
- tscError("wait async query sem failed");
+ if( atomic_load_8((int8_t*)&pStmt->asyncBindParam.asyncBindNum)>1) {
+ tscError("async bind param is still working, please try again later");
+ return TSDB_CODE_TSC_STMT_API_ERROR;
+ }
+
+ if (pStmt->options.asyncExecFn && !pStmt->execSemWaited) {
+ if (tsem_wait(&pStmt->asyncExecSem) != 0) {
+ tscError("wait asyncExecSem failed");
}
- pStmt->semWaited = true;
+ pStmt->execSemWaited = true;
}
SSHashObj *hashTbnames = tSimpleHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR));
@@ -2243,6 +2250,35 @@ out:
return code;
}
+int taos_stmt2_bind_param_a(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col_idx, __taos_async_fn_t fp,
+ void *param) {
+ if (stmt == NULL || bindv == NULL || fp == NULL) {
+ terrno = TSDB_CODE_INVALID_PARA;
+ return terrno;
+ }
+
+ STscStmt2 *pStmt = (STscStmt2 *)stmt;
+ if (atomic_load_8((int8_t *)&pStmt->asyncBindParam.asyncBindNum) > 0) {
+ tscError("async bind param is still working, please try again later");
+ return TSDB_CODE_TSC_STMT_API_ERROR;
+ }
+
+ ThreadArgs *args = (ThreadArgs *)taosMemoryMalloc(sizeof(ThreadArgs));
+ args->stmt = stmt;
+ args->bindv = bindv;
+ args->col_idx = col_idx;
+ args->fp = fp;
+ args->param = param;
+ (void)atomic_add_fetch_8(&pStmt->asyncBindParam.asyncBindNum, 1);
+ int code_s = taosStmt2AsyncBind(stmtAsyncBindThreadFunc, (void *)args);
+ if (code_s != TSDB_CODE_SUCCESS) {
+ (void)atomic_sub_fetch_8(&pStmt->asyncBindParam.asyncBindNum, 1);
+ // terrno = TAOS_SYSTEM_ERROR(errno);
+ }
+
+ return code_s;
+}
+
int taos_stmt2_exec(TAOS_STMT2 *stmt, int *affected_rows) {
if (stmt == NULL) {
tscError("NULL parameter for %s", __FUNCTION__);
diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 4ff50692b7..cf40e109e7 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -799,7 +799,7 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) {
if (vAlterTbReq.tagType == TSDB_DATA_TYPE_VARBINARY) {
bufSize = vAlterTbReq.nTagVal * 2 + 2 + 3;
} else {
- bufSize = vAlterTbReq.nTagVal + 3;
+ bufSize = vAlterTbReq.nTagVal + 32;
}
buf = taosMemoryCalloc(bufSize, 1);
RAW_NULL_CHECK(buf);
@@ -1844,9 +1844,10 @@ static void* getRawDataFromRes(void* pRetrieve) {
}
void* rawData = NULL;
// deal with compatibility
- if (*(int64_t*)pRetrieve == 0) {
+ if (*(int64_t*)pRetrieve == RETRIEVE_TABLE_RSP_VERSION) {
rawData = ((SRetrieveTableRsp*)pRetrieve)->data;
- } else if (*(int64_t*)pRetrieve == 1) {
+ } else if (*(int64_t*)pRetrieve == RETRIEVE_TABLE_RSP_TMQ_VERSION ||
+ *(int64_t*)pRetrieve == RETRIEVE_TABLE_RSP_TMQ_RAW_VERSION) {
rawData = ((SRetrieveTableRspForTmq*)pRetrieve)->data;
}
return rawData;
@@ -1947,7 +1948,10 @@ static int32_t initRawCacheHash() {
}
static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrapper* pSW) {
- if (rawData == NULL || pTableMeta == NULL || pSW == NULL) {
+ if (rawData == NULL || pSW == NULL){
+ return false;
+ }
+ if (pTableMeta == NULL) {
uError("invalid parameter in %s", __func__);
return false;
}
@@ -1965,6 +1969,7 @@ static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrappe
if (pSW->nCols != pTableMeta->tableInfo.numOfColumns) {
return true;
}
+
for (int i = 0; i < pSW->nCols; i++) {
int j = 0;
for (; j < pTableMeta->tableInfo.numOfColumns; j++) {
@@ -1972,7 +1977,7 @@ static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrappe
char* fieldName = pSW->pSchema[i].name;
if (strcmp(pColSchema->name, fieldName) == 0) {
- if (*fields != pColSchema->type || *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
+ if (checkSchema(pColSchema, fields, NULL, 0) != 0){
return true;
}
break;
@@ -2069,7 +2074,7 @@ static int32_t processCacheMeta(SHashObj* pVgHash, SHashObj* pNameHash, SHashObj
SVCreateTbReq* pCreateReqDst, SCatalog* pCatalog, SRequestConnInfo* conn, SName* pName,
STableMeta** pMeta, SSchemaWrapper* pSW, void* rawData, int32_t retry) {
if (pVgHash == NULL || pNameHash == NULL || pMetaHash == NULL || pCatalog == NULL || conn == NULL || pName == NULL ||
- pMeta == NULL || pSW == NULL || rawData == NULL) {
+ pMeta == NULL) {
uError("invalid parameter in %s", __func__);
return TSDB_CODE_INVALID_PARA;
}
@@ -2168,7 +2173,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, uint32_t dataLen) {
void* rawData = getRawDataFromRes(pRetrieve);
RAW_NULL_CHECK(rawData);
- uDebug(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName);
+ uTrace(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName);
SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN);
tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN);
@@ -2251,7 +2256,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, uint32_t dataLen)
void* rawData = getRawDataFromRes(pRetrieve);
RAW_NULL_CHECK(rawData);
- uDebug(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName);
+ uTrace(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName);
SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN);
tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN);
@@ -2298,6 +2303,90 @@ end:
return code;
}
+static int32_t tmqWriteRawRawDataImpl(TAOS* taos, void* data, uint32_t dataLen) {
+ if (taos == NULL || data == NULL) {
+ uError("invalid parameter in %s", __func__);
+ return TSDB_CODE_INVALID_PARA;
+ }
+ int32_t code = TSDB_CODE_SUCCESS;
+ SQuery* pQuery = NULL;
+ SHashObj* pVgroupHash = NULL;
+ SMqRspObj rspObj = {0};
+ SDecoder decoder = {0};
+
+ SRequestObj* pRequest = NULL;
+ SCatalog* pCatalog = NULL;
+ SRequestConnInfo conn = {0};
+
+ RAW_RETURN_CHECK(buildRawRequest(taos, &pRequest, &pCatalog, &conn));
+ uDebug(LOG_ID_TAG " write raw rawdata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
+ RAW_RETURN_CHECK(decodeRawData(&decoder, data, dataLen, tDecodeMqDataRsp, &rspObj));
+
+ SHashObj* pVgHash = NULL;
+ SHashObj* pNameHash = NULL;
+ SHashObj* pMetaHash = NULL;
+ RAW_RETURN_CHECK(getRawCache(&pVgHash, &pNameHash, &pMetaHash, taos));
+ int retry = 0;
+ while (1) {
+ RAW_RETURN_CHECK(smlInitHandle(&pQuery));
+ uDebug(LOG_ID_TAG " write raw rawdata block num:%d", LOG_ID_VALUE, rspObj.dataRsp.blockNum);
+ SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)(pQuery)->pRoot;
+ pVgroupHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
+ RAW_NULL_CHECK(pVgroupHash);
+ pStmt->pVgDataBlocks = taosArrayInit(8, POINTER_BYTES);
+ RAW_NULL_CHECK(pStmt->pVgDataBlocks);
+
+ while (++rspObj.resIter < rspObj.dataRsp.blockNum) {
+ const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter);
+ RAW_NULL_CHECK(tbName);
+ void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter);
+ RAW_NULL_CHECK(pRetrieve);
+ void* rawData = getRawDataFromRes(pRetrieve);
+ RAW_NULL_CHECK(rawData);
+
+ uTrace(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName);
+ SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
+ tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN);
+ tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN);
+
+ // find schema data info
+ STableMeta* pTableMeta = NULL;
+ RAW_RETURN_CHECK(processCacheMeta(pVgHash, pNameHash, pMetaHash, NULL, pCatalog, &conn, &pName,
+ &pTableMeta, NULL, NULL, retry));
+ char err[ERR_MSG_LEN] = {0};
+ code = rawBlockBindRawData(pVgroupHash, pStmt->pVgDataBlocks, pTableMeta, rawData);
+ if (code != TSDB_CODE_SUCCESS) {
+ SET_ERROR_MSG("table:%s, err:%s", pName.tname, err);
+ goto end;
+ }
+ }
+ taosHashCleanup(pVgroupHash);
+ pVgroupHash = NULL;
+
+ RAW_RETURN_CHECK(smlBuildOutputRaw(pQuery, pVgHash));
+ launchQueryImpl(pRequest, pQuery, true, NULL);
+ code = pRequest->code;
+
+ if (NEED_CLIENT_HANDLE_ERROR(code) && retry++ < 3) {
+ uInfo("write raw retry:%d/3 end code:%d, msg:%s", retry, code, tstrerror(code));
+ qDestroyQuery(pQuery);
+ pQuery = NULL;
+ rspObj.resIter = -1;
+ continue;
+ }
+ break;
+ }
+
+ end:
+ uDebug(LOG_ID_TAG " write raw rawdata return, msg:%s", LOG_ID_VALUE, tstrerror(code));
+ tDeleteMqDataRsp(&rspObj.dataRsp);
+ tDecoderClear(&decoder);
+ qDestroyQuery(pQuery);
+ taosHashCleanup(pVgroupHash);
+ destroyRequest(pRequest);
+ return code;
+}
+
static void processSimpleMeta(SMqMetaRsp* pMetaRsp, cJSON** meta) {
if (pMetaRsp == NULL || meta == NULL) {
uError("invalid parameter in %s", __func__);
@@ -2315,8 +2404,6 @@ static void processSimpleMeta(SMqMetaRsp* pMetaRsp, cJSON** meta) {
processAlterTable(pMetaRsp, meta);
} else if (pMetaRsp->resMsgType == TDMT_VND_DROP_TABLE) {
processDropTable(pMetaRsp, meta);
- } else if (pMetaRsp->resMsgType == TDMT_VND_DROP_TABLE) {
- processDropTable(pMetaRsp, meta);
} else if (pMetaRsp->resMsgType == TDMT_VND_DELETE) {
processDeleteTable(pMetaRsp, meta);
}
@@ -2472,6 +2559,7 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) {
uError("invalid parameter in %s", __func__);
return TSDB_CODE_INVALID_PARA;
}
+ *raw = (tmq_raw_data){0};
SMqRspObj* rspObj = ((SMqRspObj*)res);
if (TD_RES_TMQ_META(res)) {
raw->raw = rspObj->metaRsp.metaRsp;
@@ -2499,6 +2587,12 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) {
raw->raw_len = rspObj->batchMetaRsp.metaBuffLen;
raw->raw_type = rspObj->resType;
uDebug("tmq get raw batch meta:%p", raw);
+ } else if (TD_RES_TMQ_RAW(res)) {
+ raw->raw = rspObj->dataRsp.rawData;
+ rspObj->dataRsp.rawData = NULL;
+ raw->raw_len = rspObj->dataRsp.len;
+ raw->raw_type = rspObj->resType;
+ uDebug("tmq get raw raw:%p", raw);
} else {
uError("tmq get raw error type:%d", *(int8_t*)res);
return TSDB_CODE_TMQ_INVALID_MSG;
@@ -2508,8 +2602,11 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) {
void tmq_free_raw(tmq_raw_data raw) {
uDebug("tmq free raw data type:%d", raw.raw_type);
- if (raw.raw_type == RES_TYPE__TMQ || raw.raw_type == RES_TYPE__TMQ_METADATA) {
+ if (raw.raw_type == RES_TYPE__TMQ ||
+ raw.raw_type == RES_TYPE__TMQ_METADATA) {
taosMemoryFree(raw.raw);
+ } else if(raw.raw_type == RES_TYPE__TMQ_RAWDATA && raw.raw != NULL){
+ taosMemoryFree(POINTER_SHIFT(raw.raw, - sizeof(SMqRspHead)));
}
(void)memset(terrMsg, 0, ERR_MSG_LEN);
}
@@ -2559,6 +2656,8 @@ static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type)
return taosDeleteData(taos, buf, len);
} else if (type == RES_TYPE__TMQ_METADATA) {
return tmqWriteRawMetaDataImpl(taos, buf, len);
+ } else if (type == RES_TYPE__TMQ_RAWDATA) {
+ return tmqWriteRawRawDataImpl(taos, buf, len);
} else if (type == RES_TYPE__TMQ) {
return tmqWriteRawDataImpl(taos, buf, len);
} else if (type == RES_TYPE__TMQ_BATCH_META) {
diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c
index fe303bc314..67066e1fdd 100644
--- a/source/client/src/clientStmt2.c
+++ b/source/client/src/clientStmt2.c
@@ -752,6 +752,14 @@ static int32_t stmtInitQueue(STscStmt2* pStmt) {
return TSDB_CODE_SUCCESS;
}
+static int32_t stmtIniAsyncBind(STscStmt2* pStmt) {
+ (void)taosThreadCondInit(&pStmt->asyncBindParam.waitCond, NULL);
+ (void)taosThreadMutexInit(&pStmt->asyncBindParam.mutex, NULL);
+ pStmt->asyncBindParam.asyncBindNum = 0;
+
+ return TSDB_CODE_SUCCESS;
+}
+
static int32_t stmtInitTableBuf(STableBufInfo* pTblBuf) {
pTblBuf->buffUnit = sizeof(SStmtQNode);
pTblBuf->buffSize = pTblBuf->buffUnit * 1000;
@@ -812,13 +820,13 @@ TAOS_STMT2* stmtInit2(STscObj* taos, TAOS_STMT2_OPTION* pOptions) {
pStmt->sql.siInfo.mgmtEpSet = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp);
pStmt->sql.siInfo.pTableHash = tSimpleHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY));
if (NULL == pStmt->sql.siInfo.pTableHash) {
- (void)stmtClose(pStmt);
+ (void)stmtClose2(pStmt);
return NULL;
}
pStmt->sql.siInfo.pTableCols = taosArrayInit(STMT_TABLE_COLS_NUM, POINTER_BYTES);
if (NULL == pStmt->sql.siInfo.pTableCols) {
terrno = terrno;
- (void)stmtClose(pStmt);
+ (void)stmtClose2(pStmt);
return NULL;
}
@@ -831,20 +839,27 @@ TAOS_STMT2* stmtInit2(STscObj* taos, TAOS_STMT2_OPTION* pOptions) {
}
if (TSDB_CODE_SUCCESS != code) {
terrno = code;
- (void)stmtClose(pStmt);
+ (void)stmtClose2(pStmt);
return NULL;
}
}
pStmt->sql.siInfo.tableColsReady = true;
if (pStmt->options.asyncExecFn) {
- if (tsem_init(&pStmt->asyncQuerySem, 0, 1) != 0) {
+ if (tsem_init(&pStmt->asyncExecSem, 0, 1) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
- (void)stmtClose(pStmt);
+ (void)stmtClose2(pStmt);
return NULL;
}
}
- pStmt->semWaited = false;
+ code = stmtIniAsyncBind(pStmt);
+ if (TSDB_CODE_SUCCESS != code) {
+ terrno = code;
+ (void)stmtClose2(pStmt);
+ return NULL;
+ }
+
+ pStmt->execSemWaited = false;
STMT_LOG_SEQ(STMT_INIT);
@@ -1659,8 +1674,8 @@ static void asyncQueryCb(void* userdata, TAOS_RES* res, int code) {
(void)stmtCleanExecInfo(pStmt, (code ? false : true), false);
++pStmt->sql.runTimes;
- if (tsem_post(&pStmt->asyncQuerySem) != 0) {
- tscError("failed to post asyncQuerySem");
+ if (tsem_post(&pStmt->asyncExecSem) != 0) {
+ tscError("failed to post asyncExecSem");
}
}
@@ -1675,6 +1690,12 @@ int stmtExec2(TAOS_STMT2* stmt, int* affected_rows) {
return pStmt->errCode;
}
+ taosThreadMutexLock(&pStmt->asyncBindParam.mutex);
+ if (atomic_load_8((int8_t*)&pStmt->asyncBindParam.asyncBindNum) > 0) {
+ (void)taosThreadCondWait(&pStmt->asyncBindParam.waitCond, &pStmt->asyncBindParam.mutex);
+ }
+ taosThreadMutexUnlock(&pStmt->asyncBindParam.mutex);
+
if (pStmt->sql.stbInterlaceMode) {
STMT_ERR_RET(stmtAddBatch2(pStmt));
}
@@ -1749,7 +1770,7 @@ int stmtExec2(TAOS_STMT2* stmt, int* affected_rows) {
pRequest->body.queryFp = asyncQueryCb;
((SSyncQueryParam*)(pRequest)->body.interParam)->userParam = pStmt;
- pStmt->semWaited = false;
+ pStmt->execSemWaited = false;
launchAsyncQuery(pRequest, pStmt->sql.pQuery, NULL, pWrapper);
}
@@ -1775,12 +1796,20 @@ int stmtClose2(TAOS_STMT2* stmt) {
pStmt->bindThreadInUse = false;
}
+ taosThreadMutexLock(&pStmt->asyncBindParam.mutex);
+ if (atomic_load_8((int8_t*)&pStmt->asyncBindParam.asyncBindNum) > 0) {
+ (void)taosThreadCondWait(&pStmt->asyncBindParam.waitCond, &pStmt->asyncBindParam.mutex);
+ }
+ taosThreadMutexUnlock(&pStmt->asyncBindParam.mutex);
(void)taosThreadCondDestroy(&pStmt->queue.waitCond);
(void)taosThreadMutexDestroy(&pStmt->queue.mutex);
- if (pStmt->options.asyncExecFn && !pStmt->semWaited) {
- if (tsem_wait(&pStmt->asyncQuerySem) != 0) {
- tscError("failed to wait asyncQuerySem");
+ (void)taosThreadCondDestroy(&pStmt->asyncBindParam.waitCond);
+ (void)taosThreadMutexDestroy(&pStmt->asyncBindParam.mutex);
+
+ if (pStmt->options.asyncExecFn && !pStmt->execSemWaited) {
+ if (tsem_wait(&pStmt->asyncExecSem) != 0) {
+ tscError("failed to wait asyncExecSem");
}
}
@@ -1798,8 +1827,8 @@ int stmtClose2(TAOS_STMT2* stmt) {
STMT_ERR_RET(stmtCleanSQLInfo(pStmt));
if (pStmt->options.asyncExecFn) {
- if (tsem_destroy(&pStmt->asyncQuerySem) != 0) {
- tscError("failed to destroy asyncQuerySem");
+ if (tsem_destroy(&pStmt->asyncExecSem) != 0) {
+ tscError("failed to destroy asyncExecSem");
}
}
taosMemoryFree(stmt);
@@ -1928,3 +1957,22 @@ TAOS_RES* stmtUseResult2(TAOS_STMT2* stmt) {
return pStmt->exec.pRequest;
}
+
+int32_t stmtAsyncBindThreadFunc(void* args) {
+ qInfo("async stmt bind thread started");
+
+ ThreadArgs* targs = (ThreadArgs*)args;
+ STscStmt2* pStmt = (STscStmt2*)targs->stmt;
+
+ int code = taos_stmt2_bind_param(targs->stmt, targs->bindv, targs->col_idx);
+ targs->fp(targs->param, NULL, code);
+ (void)taosThreadMutexLock(&(pStmt->asyncBindParam.mutex));
+ (void)atomic_sub_fetch_8(&pStmt->asyncBindParam.asyncBindNum, 1);
+ (void)taosThreadCondSignal(&(pStmt->asyncBindParam.waitCond));
+ (void)taosThreadMutexUnlock(&(pStmt->asyncBindParam.mutex));
+ taosMemoryFree(args);
+
+ qInfo("async stmt bind thread stopped");
+
+ return code;
+}
\ No newline at end of file
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 1d461d5a49..4adc738d35 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -42,18 +42,19 @@
#define PROCESS_POLL_RSP(FUNC,DATA) \
SDecoder decoder = {0}; \
- tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); \
+ tDecoderInit(&decoder, POINTER_SHIFT(pRspWrapper->pollRsp.data, sizeof(SMqRspHead)), pRspWrapper->pollRsp.len - sizeof(SMqRspHead)); \
if (FUNC(&decoder, DATA) < 0) { \
tDecoderClear(&decoder); \
code = terrno; \
goto END;\
}\
tDecoderClear(&decoder);\
- (void)memcpy(DATA, pMsg->pData, sizeof(SMqRspHead));
+ (void)memcpy(DATA, pRspWrapper->pollRsp.data, sizeof(SMqRspHead));
#define DELETE_POLL_RSP(FUNC,DATA) \
SMqPollRspWrapper* pRsp = &rspWrapper->pollRsp;\
- taosMemoryFreeClear(pRsp->pEpset);\
+ taosMemoryFreeClear(pRsp->pEpset); \
+ taosMemoryFreeClear(pRsp->data); \
FUNC(DATA);
enum {
@@ -92,6 +93,9 @@ struct tmq_conf_t {
int8_t snapEnable;
int8_t replayEnable;
int8_t sourceExcluded; // do not consume, bit
+ int8_t rawData; // fetch raw data
+ int32_t maxPollWaitTime;
+ int32_t minPollRows;
uint16_t port;
int32_t autoCommitInterval;
int32_t sessionTimeoutMs;
@@ -121,6 +125,9 @@ struct tmq_t {
int8_t resetOffsetCfg;
int8_t replayEnable;
int8_t sourceExcluded; // do not consume, bit
+ int8_t rawData; // fetch raw data
+ int32_t maxPollWaitTime;
+ int32_t minPollRows;
int64_t consumerId;
tmq_commit_cb* commitCb;
void* commitCbUserParam;
@@ -188,6 +195,8 @@ typedef struct {
SMqClientTopic* topicHandle;
uint64_t reqId;
SEpSet* pEpset;
+ void* data;
+ uint32_t len;
union {
struct{
SMqRspHead head;
@@ -302,6 +311,8 @@ tmq_conf_t* tmq_conf_new() {
conf->heartBeatIntervalMs = DEFAULT_HEARTBEAT_INTERVAL;
conf->maxPollIntervalMs = DEFAULT_MAX_POLL_INTERVAL;
conf->sessionTimeoutMs = DEFAULT_SESSION_TIMEOUT;
+ conf->maxPollWaitTime = DEFAULT_MAX_POLL_WAIT_TIME;
+ conf->minPollRows = DEFAULT_MIN_POLL_ROWS;
return conf;
}
@@ -491,11 +502,39 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
}
}
if (strcasecmp(key, "msg.consume.excluded") == 0) {
- int64_t tmp;
+ int64_t tmp = 0;
code = taosStr2int64(value, &tmp);
conf->sourceExcluded = (0 == code && tmp != 0) ? TD_REQ_FROM_TAOX : 0;
return TMQ_CONF_OK;
}
+ if (strcasecmp(key, "msg.consume.rawdata") == 0) {
+ int64_t tmp = 0;
+ code = taosStr2int64(value, &tmp);
+ conf->rawData = (0 == code && tmp != 0) ? 1 : 0;
+ return TMQ_CONF_OK;
+ }
+
+ if (strcasecmp(key, "fetch.max.wait.ms") == 0) {
+ int64_t tmp = 0;
+ code = taosStr2int64(value, &tmp);
+ if (tmp <= 0 || tmp > INT32_MAX || code != 0) {
+ tqErrorC("invalid value for fetch.max.wait.ms: %s", value);
+ return TMQ_CONF_INVALID;
+ }
+ conf->maxPollWaitTime = tmp;
+ return TMQ_CONF_OK;
+ }
+
+ if (strcasecmp(key, "min.poll.rows") == 0) {
+ int64_t tmp = 0;
+ code = taosStr2int64(value, &tmp);
+ if (tmp <= 0 || tmp > INT32_MAX || code != 0) {
+ tqErrorC("invalid value for min.poll.rows: %s", value);
+ return TMQ_CONF_INVALID;
+ }
+ conf->minPollRows = tmp;
+ return TMQ_CONF_OK;
+ }
if (strcasecmp(key, "td.connect.db") == 0) {
return TMQ_CONF_OK;
@@ -812,7 +851,7 @@ static void asyncCommitFromResult(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_c
goto end;
}
- if (TD_RES_TMQ(pRes) || TD_RES_TMQ_META(pRes) ||
+ if (TD_RES_TMQ(pRes) || TD_RES_TMQ_RAW(pRes) || TD_RES_TMQ_META(pRes) ||
TD_RES_TMQ_METADATA(pRes) || TD_RES_TMQ_BATCH_META(pRes)) {
SMqRspObj* pRspObj = (SMqRspObj*)pRes;
pTopicName = pRspObj->topic;
@@ -1082,6 +1121,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
sendInfo->fp = tmqHbCb;
sendInfo->msgType = TDMT_MND_TMQ_HB;
+
SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
int32_t code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo);
@@ -1122,6 +1162,8 @@ static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) {
DELETE_POLL_RSP(tDeleteMqMetaRsp,&pRsp->metaRsp)
} else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) {
DELETE_POLL_RSP(tDeleteMqBatchMetaRsp,&pRsp->batchMetaRsp)
+ } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RAW_DATA_RSP) {
+ DELETE_POLL_RSP(tDeleteMqRawDataRsp, &pRsp->dataRsp)
}
}
@@ -1733,6 +1775,9 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
pTmq->resetOffsetCfg = conf->resetOffset;
pTmq->replayEnable = conf->replayEnable;
pTmq->sourceExcluded = conf->sourceExcluded;
+ pTmq->rawData = conf->rawData;
+ pTmq->maxPollWaitTime = conf->maxPollWaitTime;
+ pTmq->minPollRows = conf->minPollRows;
pTmq->enableBatchMeta = conf->enableBatchMeta;
tstrncpy(pTmq->user, user, TSDB_USER_LEN);
if (taosGetFqdn(pTmq->fqdn) != 0) {
@@ -2057,23 +2102,14 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
goto END;
}
rspType = ((SMqRspHead*)pMsg->pData)->mqMsgType;
- tqDebugC("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, type %d,QID:0x%" PRIx64, tmq->consumerId, vgId, rspType, requestId);
- if (rspType == TMQ_MSG_TYPE__POLL_DATA_RSP) {
- PROCESS_POLL_RSP(tDecodeMqDataRsp, &pRspWrapper->pollRsp.dataRsp)
- } else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) {
- PROCESS_POLL_RSP(tDecodeMqMetaRsp, &pRspWrapper->pollRsp.metaRsp)
- } else if (rspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) {
- PROCESS_POLL_RSP(tDecodeSTaosxRsp, &pRspWrapper->pollRsp.dataRsp)
- } else if (rspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) {
- PROCESS_POLL_RSP(tSemiDecodeMqBatchMetaRsp, &pRspWrapper->pollRsp.batchMetaRsp)
- } else { // invalid rspType
- tqErrorC("consumer:0x%" PRIx64 " invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType);
- code = TSDB_CODE_TSC_INTERNAL_ERROR;
- goto END;
- }
+ tqDebugC("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, type %d(%s),QID:0x%" PRIx64, tmq->consumerId, vgId, rspType, tmqMsgTypeStr[rspType], requestId);
+
pRspWrapper->tmqRspType = rspType;
pRspWrapper->pollRsp.reqId = requestId;
pRspWrapper->pollRsp.pEpset = pMsg->pEpSet;
+ pRspWrapper->pollRsp.data = pMsg->pData;
+ pRspWrapper->pollRsp.len = pMsg->len;
+ pMsg->pData = NULL;
pMsg->pEpSet = NULL;
END:
@@ -2087,8 +2123,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
taosFreeQitem(pRspWrapper);
tqErrorC("consumer:0x%" PRIx64 " put poll res into mqueue failed, code:%d", tmq->consumerId, code);
} else {
- tqDebugC("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d,QID:0x%" PRIx64,
- tmq ? tmq->consumerId : 0, rspType, vgId, taosQueueItemSize(tmq->mqueue), requestId);
+ tqDebugC("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d(%s), vgId:%d, total in queue:%d,QID:0x%" PRIx64,
+ tmq ? tmq->consumerId : 0, rspType, tmqMsgTypeStr[rspType], vgId, taosQueueItemSize(tmq->mqueue), requestId);
}
}
@@ -2107,14 +2143,15 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
return code;
}
-void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) {
+void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, SMqClientTopic* pTopic, SMqClientVg* pVg) {
if (pReq == NULL || tmq == NULL || pTopic == NULL || pVg == NULL) {
return;
}
(void)snprintf(pReq->subKey, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", tmq->groupId, TMQ_SEPARATOR, pTopic->topicName);
pReq->withTbName = tmq->withTbName;
pReq->consumerId = tmq->consumerId;
- pReq->timeout = timeout < 0 ? INT32_MAX : timeout;
+ pReq->timeout = tmq->maxPollWaitTime;
+ pReq->minPollRows = tmq->minPollRows;
pReq->epoch = tmq->epoch;
pReq->reqOffset = pVg->offsetInfo.endOffset;
pReq->head.vgId = pVg->vgId;
@@ -2122,6 +2159,7 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl
pReq->reqId = generateRequestId();
pReq->enableReplay = tmq->replayEnable;
pReq->sourceExcluded = tmq->sourceExcluded;
+ pReq->rawData = tmq->rawData;
pReq->enableBatchMeta = tmq->enableBatchMeta;
}
@@ -2217,14 +2255,14 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg
}
}
-static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* pVg, int64_t timeout) {
+static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* pVg) {
SMqPollReq req = {0};
char* msg = NULL;
SMqPollCbParam* pParam = NULL;
SMsgSendInfo* sendInfo = NULL;
int code = 0;
int lino = 0;
- tmqBuildConsumeReqImpl(&req, pTmq, timeout, pTopic, pVg);
+ tmqBuildConsumeReqImpl(&req, pTmq, pTopic, pVg);
int32_t msgSize = tSerializeSMqPollReq(NULL, 0, &req);
TSDB_CHECK_CONDITION(msgSize >= 0, code, lino, END, TSDB_CODE_INVALID_MSG);
@@ -2276,7 +2314,7 @@ END:
return code;
}
-static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
+static int32_t tmqPollImpl(tmq_t* tmq) {
if (tmq == NULL) {
return TSDB_CODE_INVALID_MSG;
}
@@ -2329,7 +2367,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
}
atomic_store_32(&pVg->vgSkipCnt, 0);
- code = doTmqPollImpl(tmq, pTopic, pVg, timeout);
+ code = doTmqPollImpl(tmq, pTopic, pVg);
if (code != TSDB_CODE_SUCCESS) {
goto end;
}
@@ -2415,6 +2453,35 @@ static int32_t processMqRspError(tmq_t* tmq, SMqRspWrapper* pRspWrapper){
return code;
}
+
+static int32_t processWrapperData(SMqRspWrapper* pRspWrapper){
+ int32_t code = 0;
+ if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) {
+ PROCESS_POLL_RSP(tDecodeMqDataRsp, &pRspWrapper->pollRsp.dataRsp)
+ pRspWrapper->pollRsp.dataRsp.data = pRspWrapper->pollRsp.data;
+ pRspWrapper->pollRsp.data = NULL;
+ } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) {
+ PROCESS_POLL_RSP(tDecodeMqMetaRsp, &pRspWrapper->pollRsp.metaRsp)
+ } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) {
+ PROCESS_POLL_RSP(tDecodeSTaosxRsp, &pRspWrapper->pollRsp.dataRsp)
+ pRspWrapper->pollRsp.dataRsp.data = pRspWrapper->pollRsp.data;
+ pRspWrapper->pollRsp.data = NULL;
+ } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) {
+ PROCESS_POLL_RSP(tSemiDecodeMqBatchMetaRsp, &pRspWrapper->pollRsp.batchMetaRsp)
+ } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RAW_DATA_RSP) {
+ PROCESS_POLL_RSP(tDecodeMqRawDataRsp, &pRspWrapper->pollRsp.dataRsp)
+ pRspWrapper->pollRsp.dataRsp.len = pRspWrapper->pollRsp.len - sizeof(SMqRspHead);
+ pRspWrapper->pollRsp.dataRsp.rawData = POINTER_SHIFT(pRspWrapper->pollRsp.data, sizeof(SMqRspHead));
+ pRspWrapper->pollRsp.data = NULL;
+ } else {
+ tqErrorC("invalid rsp msg received, type:%d ignored", pRspWrapper->tmqRspType);
+ code = TSDB_CODE_TSC_INTERNAL_ERROR;
+ goto END;
+ }
+ END:
+ return code;
+}
+
static SMqRspObj* processMqRsp(tmq_t* tmq, SMqRspWrapper* pRspWrapper){
int32_t code = 0;
SMqRspObj* pRspObj = NULL;
@@ -2427,6 +2494,10 @@ static SMqRspObj* processMqRsp(tmq_t* tmq, SMqRspWrapper* pRspWrapper){
return pRspObj;
}
+ code = processWrapperData(pRspWrapper);
+ if (code != 0) {
+ goto END;
+ }
SMqPollRspWrapper* pollRspWrapper = &pRspWrapper->pollRsp;
taosWLockLatch(&tmq->lock);
SMqClientVg* pVg = NULL;
@@ -2442,7 +2513,9 @@ static SMqRspObj* processMqRsp(tmq_t* tmq, SMqRspWrapper* pRspWrapper){
pVg->epSet = *pollRspWrapper->pEpset;
}
- if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP || pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) {
+ if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP ||
+ pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP ||
+ pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RAW_DATA_RSP) {
updateVgInfo(pVg, &pollRspWrapper->dataRsp.reqOffset, &pollRspWrapper->dataRsp.rspOffset, pollRspWrapper->head.walsver, pollRspWrapper->head.walever,
tmq->consumerId, pollRspWrapper->dataRsp.blockNum != 0);
@@ -2459,12 +2532,15 @@ static SMqRspObj* processMqRsp(tmq_t* tmq, SMqRspWrapper* pRspWrapper){
tqErrorC("consumer:0x%" PRIx64 " failed to allocate memory for meta rsp", tmq->consumerId);
goto END;
}
- pRspObj->resType = pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP ? RES_TYPE__TMQ : RES_TYPE__TMQ_METADATA;
+ pRspObj->resType = pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RAW_DATA_RSP ? RES_TYPE__TMQ_RAWDATA :
+ (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP ? RES_TYPE__TMQ : RES_TYPE__TMQ_METADATA);
int64_t numOfRows = 0;
- tmqBuildRspFromWrapperInner(pollRspWrapper, pVg, &numOfRows, pRspObj);
- tmq->totalRows += numOfRows;
+ if (pRspWrapper->tmqRspType != TMQ_MSG_TYPE__POLL_RAW_DATA_RSP){
+ tmqBuildRspFromWrapperInner(pollRspWrapper, pVg, &numOfRows, pRspObj);
+ tmq->totalRows += numOfRows;
+ }
pVg->emptyBlockReceiveTs = 0;
- if (tmq->replayEnable) {
+ if (tmq->replayEnable && pRspWrapper->tmqRspType != TMQ_MSG_TYPE__POLL_RAW_DATA_RSP) {
pVg->blockReceiveTs = taosGetTimestampMs();
pVg->blockSleepForReplay = pRspObj->dataRsp.sleepTime;
if (pVg->blockSleepForReplay > 0) {
@@ -2523,6 +2599,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq) {
returnVal = processMqRsp(tmq, pRspWrapper);
code = terrno;
}
+
tmqFreeRspWrapper(pRspWrapper);
taosFreeQitem(pRspWrapper);
if(returnVal != NULL || code != 0){
@@ -2552,7 +2629,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
code = tmqHandleAllDelayedTask(tmq);
TSDB_CHECK_CODE(code, lino, END);
- code = tmqPollImpl(tmq, timeout);
+ code = tmqPollImpl(tmq);
TSDB_CHECK_CODE(code, lino, END);
rspObj = tmqHandleAllRsp(tmq);
@@ -2575,7 +2652,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
END:
terrno = code;
- if (tmq != NULL) {
+ if (tmq != NULL && terrno != 0) {
tqErrorC("consumer:0x%" PRIx64 " poll error at line:%d, msg:%s", tmq->consumerId, lino, tstrerror(terrno));
}
return NULL;
@@ -2688,6 +2765,8 @@ tmq_res_t tmq_get_res_type(TAOS_RES* res) {
return TMQ_RES_METADATA;
} else if (TD_RES_TMQ_BATCH_META(res)) {
return TMQ_RES_TABLE_META;
+ } else if (TD_RES_TMQ_RAW(res)) {
+ return TMQ_RES_RAWDATA;
} else {
return TMQ_RES_INVALID;
}
@@ -2697,7 +2776,7 @@ const char* tmq_get_topic_name(TAOS_RES* res) {
if (res == NULL) {
return NULL;
}
- if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) ||
+ if (TD_RES_TMQ_RAW(res) || TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) ||
TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
char* tmp = strchr(((SMqRspObj*)res)->topic, '.');
if (tmp == NULL) {
@@ -2714,7 +2793,7 @@ const char* tmq_get_db_name(TAOS_RES* res) {
return NULL;
}
- if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) ||
+ if (TD_RES_TMQ_RAW(res) || TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) ||
TD_RES_TMQ_BATCH_META(res) || TD_RES_TMQ_META(res)) {
char* tmp = strchr(((SMqRspObj*)res)->db, '.');
if (tmp == NULL) {
@@ -2730,7 +2809,7 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) {
if (res == NULL) {
return TSDB_CODE_INVALID_PARA;
}
- if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) ||
+ if (TD_RES_TMQ_RAW(res) || TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) ||
TD_RES_TMQ_BATCH_META(res) || TD_RES_TMQ_META(res)) {
return ((SMqRspObj*)res)->vgId;
} else {
@@ -2742,7 +2821,7 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) {
if (res == NULL) {
return TSDB_CODE_INVALID_PARA;
}
- if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
+ if (TD_RES_TMQ_RAW(res) || TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
SMqRspObj* pRspObj = (SMqRspObj*)res;
STqOffsetVal* pOffset = &pRspObj->dataRsp.reqOffset;
if (pOffset->type == TMQ_OFFSET__LOG) {
@@ -2767,7 +2846,7 @@ const char* tmq_get_table_name(TAOS_RES* res) {
if (res == NULL) {
return NULL;
}
- if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
+ if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) ) {
SMqRspObj* pRspObj = (SMqRspObj*)res;
SMqDataRsp* data = &pRspObj->dataRsp;
if (!data->withTbName || data->blockTbName == NULL || pRspObj->resIter < 0 ||
@@ -3427,7 +3506,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
pParam->pCommon = pCommon;
SMqPollReq req = {0};
- tmqBuildConsumeReqImpl(&req, tmq, 10, pTopic, pClientVg);
+ tmqBuildConsumeReqImpl(&req, tmq, pTopic, pClientVg);
req.reqOffset = pClientVg->offsetInfo.beginOffset;
int32_t msgSize = tSerializeSMqPollReq(NULL, 0, &req);
@@ -3670,4 +3749,4 @@ TAOS* tmq_get_connect(tmq_t* tmq) {
return (TAOS*)(&(tmq->pTscObj->id));
}
return NULL;
-}
\ No newline at end of file
+}
diff --git a/source/client/test/stmt2Test.cpp b/source/client/test/stmt2Test.cpp
index 3a648042a6..72b2e6929c 100644
--- a/source/client/test/stmt2Test.cpp
+++ b/source/client/test/stmt2Test.cpp
@@ -37,7 +37,7 @@ namespace {
void checkError(TAOS_STMT2* stmt, int code) {
if (code != TSDB_CODE_SUCCESS) {
STscStmt2* pStmt = (STscStmt2*)stmt;
- if (pStmt == nullptr || pStmt->sql.sqlStr == nullptr) {
+ if (pStmt == nullptr || pStmt->sql.sqlStr == nullptr || pStmt->exec.pRequest == nullptr) {
printf("stmt api error\n stats : %d\n errstr : %s\n", pStmt->sql.status, taos_stmt_errstr(stmt));
} else {
printf("stmt api error\n sql : %s\n stats : %d\n errstr : %s\n", pStmt->sql.sqlStr, pStmt->sql.status,
@@ -255,48 +255,41 @@ TEST(stmt2Case, stmt2_test_limit) {
do_query(taos, "create database IF NOT EXISTS stmt2_testdb_7");
do_query(taos, "create stable stmt2_testdb_7.stb (ts timestamp, b binary(10)) tags(t1 int, t2 binary(10))");
do_query(taos,
- "insert into stmt2_testdb_7.tb2 using stmt2_testdb_7.stb tags(2,'xyz') values(1591060628000, "
- "'abc'),(1591060628001,'def'),(1591060628004, 'hij')");
+ "insert into stmt2_testdb_7.tb2 using stmt2_testdb_7.stb tags(2,'xyz') values(1591060628000, "
+ "'abc'),(1591060628001,'def'),(1591060628004, 'hij')");
do_query(taos, "use stmt2_testdb_7");
-
TAOS_STMT2_OPTION option = {0, true, true, NULL, NULL};
-
TAOS_STMT2* stmt = taos_stmt2_init(taos, &option);
ASSERT_NE(stmt, nullptr);
-
const char* sql = "select * from stmt2_testdb_7.tb2 where ts > ? and ts < ? limit ?";
- int code = taos_stmt2_prepare(stmt, sql, 0);
+ int code = taos_stmt2_prepare(stmt, sql, 0);
checkError(stmt, code);
-
- int t64_len[1] = {sizeof(int64_t)};
- int b_len[1] = {3};
- int x = 2;
- int x_len = sizeof(int);
- int64_t ts[2] = {1591060627000, 1591060628005};
- TAOS_STMT2_BIND params[3] = {{TSDB_DATA_TYPE_TIMESTAMP, &ts[0], t64_len, NULL, 1},
- {TSDB_DATA_TYPE_TIMESTAMP, &ts[1], t64_len, NULL, 1},
- {TSDB_DATA_TYPE_INT, &x, &x_len, NULL, 1}};
+ int t64_len[1] = {sizeof(int64_t)};
+ int b_len[1] = {3};
+ int x = 2;
+ int x_len = sizeof(int);
+ int64_t ts[2] = {1591060627000, 1591060628005};
+ TAOS_STMT2_BIND params[3] = {{TSDB_DATA_TYPE_TIMESTAMP, &ts[0], t64_len, NULL, 1},
+ {TSDB_DATA_TYPE_TIMESTAMP, &ts[1], t64_len, NULL, 1},
+ {TSDB_DATA_TYPE_INT, &x, &x_len, NULL, 1}};
TAOS_STMT2_BIND* paramv = ¶ms[0];
TAOS_STMT2_BINDV bindv = {1, NULL, NULL, ¶mv};
code = taos_stmt2_bind_param(stmt, &bindv, -1);
checkError(stmt, code);
-
taos_stmt2_exec(stmt, NULL);
checkError(stmt, code);
-
TAOS_RES* pRes = taos_stmt2_result(stmt);
ASSERT_NE(pRes, nullptr);
-
int getRecordCounts = 0;
while ((taos_fetch_row(pRes))) {
- getRecordCounts++;
+ getRecordCounts++;
}
ASSERT_EQ(getRecordCounts, 2);
taos_stmt2_close(stmt);
@@ -304,7 +297,6 @@ TEST(stmt2Case, stmt2_test_limit) {
taos_close(taos);
}
-
TEST(stmt2Case, insert_stb_get_fields_Test) {
TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(taos, nullptr);
@@ -1051,6 +1043,37 @@ TEST(stmt2Case, stmt2_insert_non_statndard) {
taos_stmt2_close(stmt);
}
+ // pk error
+ {
+ TAOS_STMT2* stmt = taos_stmt2_init(taos, &option);
+ ASSERT_NE(stmt, nullptr);
+ const char* sql =
+ "INSERT INTO stmt2_testdb_6.? using stmt2_testdb_6.stb1 (int_tag)tags(1) (int_col,ts)VALUES (?,?)";
+ int code = taos_stmt2_prepare(stmt, sql, 0);
+ checkError(stmt, code);
+
+ int tag_i = 0;
+ int tag_l = sizeof(int);
+ int tag_bl = 3;
+ int64_t ts[2] = {1591060628000, NULL};
+ int t64_len[2] = {sizeof(int64_t), sizeof(int64_t)};
+ int coli[2] = {1, 2};
+ int ilen[2] = {sizeof(int), sizeof(int)};
+ int total_affect_rows = 0;
+ char is_null[2] = {1, 1};
+
+ TAOS_STMT2_BIND params1[2] = {{TSDB_DATA_TYPE_INT, &coli, &ilen[0], is_null, 2},
+ {TSDB_DATA_TYPE_TIMESTAMP, &ts, &t64_len[0], is_null, 2}};
+
+ TAOS_STMT2_BIND* paramv = ¶ms1[0];
+ char* tbname = "tb3";
+ TAOS_STMT2_BINDV bindv = {1, &tbname, NULL, ¶mv};
+ code = taos_stmt2_bind_param(stmt, &bindv, -1);
+ ASSERT_EQ(code, TSDB_CODE_PAR_PRIMARY_KEY_IS_NULL);
+
+ taos_stmt2_close(stmt);
+ }
+
do_query(taos, "drop database if exists stmt2_testdb_6");
taos_close(taos);
}
@@ -1617,4 +1640,233 @@ TEST(stmt2Case, errcode) {
code = taos_stmt_prepare(stmt, sql, 0);
checkError(stmt, code);
}
+
+void stmtAsyncBindCb(void* param, TAOS_RES* pRes, int code) {
+ bool* finish = (bool*)param;
+ ASSERT_EQ(code, TSDB_CODE_SUCCESS);
+ taosMsleep(500);
+ *finish = true;
+ return;
+}
+
+void stmtAsyncQueryCb2(void* param, TAOS_RES* pRes, int code) {
+ ASSERT_EQ(code, TSDB_CODE_SUCCESS);
+ taosMsleep(500);
+ return;
+}
+
+void stmtAsyncBindCb2(void* param, TAOS_RES* pRes, int code) {
+ bool* finish = (bool*)param;
+ taosMsleep(500);
+ *finish = true;
+ return;
+}
+
+TEST(stmt2Case, async_order) {
+ int CTB_NUMS = 2;
+ int ROW_NUMS = 2;
+ int CYC_NUMS = 2;
+
+ TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
+ TAOS_STMT2_OPTION option = {0, true, true, stmtAsyncQueryCb2, NULL};
+ char* sql = "insert into ? values(?,?)";
+
+ do_query(taos, "drop database if exists stmt2_testdb_15");
+ do_query(taos, "create database IF NOT EXISTS stmt2_testdb_15");
+ do_query(taos, "create stable stmt2_testdb_15.stb (ts timestamp, b binary(10)) tags(t1 int, t2 binary(10))");
+ do_query(taos, "use stmt2_testdb_15");
+
+ TAOS_STMT2* stmt = taos_stmt2_init(taos, &option);
+ ASSERT_NE(stmt, nullptr);
+ int code = taos_stmt2_prepare(stmt, sql, 0);
+ checkError(stmt, code);
+ int total_affected = 0;
+
+ // tbname
+ char** tbs = (char**)taosMemoryMalloc(CTB_NUMS * sizeof(char*));
+ for (int i = 0; i < CTB_NUMS; i++) {
+ tbs[i] = (char*)taosMemoryMalloc(sizeof(char) * 20);
+ sprintf(tbs[i], "ctb_%d", i);
+ char* tmp = (char*)taosMemoryMalloc(sizeof(char) * 100);
+ sprintf(tmp, "create table stmt2_testdb_15.%s using stmt2_testdb_15.stb tags(0, 'after')", tbs[i]);
+ do_query(taos, tmp);
+ }
+ // params
+ TAOS_STMT2_BIND** paramv = (TAOS_STMT2_BIND**)taosMemoryMalloc(CTB_NUMS * sizeof(TAOS_STMT2_BIND*));
+ // col params
+ int64_t** ts = (int64_t**)taosMemoryMalloc(CTB_NUMS * sizeof(int64_t*));
+ char** b = (char**)taosMemoryMalloc(CTB_NUMS * sizeof(char*));
+ int* ts_len = (int*)taosMemoryMalloc(ROW_NUMS * sizeof(int));
+ int* b_len = (int*)taosMemoryMalloc(ROW_NUMS * sizeof(int));
+ for (int i = 0; i < ROW_NUMS; i++) {
+ ts_len[i] = sizeof(int64_t);
+ b_len[i] = 1;
+ }
+ for (int i = 0; i < CTB_NUMS; i++) {
+ ts[i] = (int64_t*)taosMemoryMalloc(ROW_NUMS * sizeof(int64_t));
+ b[i] = (char*)taosMemoryMalloc(ROW_NUMS * sizeof(char));
+ for (int j = 0; j < ROW_NUMS; j++) {
+ ts[i][j] = 1591060628000 + 100000 + j;
+ b[i][j] = 'a' + j;
+ }
+ }
+ // bind params
+ for (int i = 0; i < CTB_NUMS; i++) {
+ // create col params
+ paramv[i] = (TAOS_STMT2_BIND*)taosMemoryMalloc(2 * sizeof(TAOS_STMT2_BIND));
+ paramv[i][0] = {TSDB_DATA_TYPE_TIMESTAMP, &ts[i][0], &ts_len[0], NULL, ROW_NUMS};
+ paramv[i][1] = {TSDB_DATA_TYPE_BINARY, &b[i][0], &b_len[0], NULL, ROW_NUMS};
+ }
+
+ // case 1 : bind_a->exec_a->bind_a->exec_a->...
+ {
+ printf("case 1 : bind_a->exec_a->bind_a->exec_a->...\n");
+ for (int r = 0; r < CYC_NUMS; r++) {
+ // bind
+ TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, NULL, paramv};
+ bool finish = false;
+ code = taos_stmt2_bind_param_a(stmt, &bindv, -1, stmtAsyncBindCb, (void*)&finish);
+
+ checkError(stmt, code);
+
+ // exec
+ code = taos_stmt2_exec(stmt, NULL);
+ checkError(stmt, code);
+ }
+ }
+
+ // case 2 : bind_a->bind_a->bind_a->exec_a->...
+ {
+ printf("case 2 : bind_a->bind_a->bind_a->exec_a->...\n");
+ for (int r = 0; r < CYC_NUMS; r++) {
+ // bind params
+ TAOS_STMT2_BIND** paramv = (TAOS_STMT2_BIND**)taosMemoryMalloc(CTB_NUMS * sizeof(TAOS_STMT2_BIND*));
+ for (int i = 0; i < CTB_NUMS; i++) {
+ // create col params
+ paramv[i] = (TAOS_STMT2_BIND*)taosMemoryMalloc(2 * sizeof(TAOS_STMT2_BIND));
+ paramv[i][0] = {TSDB_DATA_TYPE_TIMESTAMP, &ts[i][0], &ts_len[0], NULL, ROW_NUMS};
+ paramv[i][1] = {TSDB_DATA_TYPE_BINARY, &b[i][0], &b_len[0], NULL, ROW_NUMS};
+ }
+ // bind
+ TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, NULL, paramv};
+ bool finish = false;
+ code = taos_stmt2_bind_param_a(stmt, &bindv, -1, stmtAsyncBindCb, (void*)&finish);
+ while (!finish) {
+ taosMsleep(100);
+ }
+ checkError(stmt, code);
+ }
+ // exec
+ code = taos_stmt2_exec(stmt, NULL);
+ checkError(stmt, code);
+ }
+
+ // case 3 : bind->exec_a->bind->exec_a->...
+ {
+ printf("case 3 : bind->exec_a->bind->exec_a->...\n");
+ for (int r = 0; r < CYC_NUMS; r++) {
+ // bind
+ TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, NULL, paramv};
+ bool finish = false;
+ code = taos_stmt2_bind_param(stmt, &bindv, -1);
+
+ checkError(stmt, code);
+
+ // exec
+ code = taos_stmt2_exec(stmt, NULL);
+ checkError(stmt, code);
+ }
+ }
+
+ // case 4 : bind_a->close
+ {
+ printf("case 4 : bind_a->close\n");
+ // bind
+ TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, NULL, paramv};
+ bool finish = false;
+ code = taos_stmt2_bind_param_a(stmt, &bindv, -1, stmtAsyncBindCb, (void*)&finish);
+ checkError(stmt, code);
+ taos_stmt2_close(stmt);
+ checkError(stmt, code);
+ }
+
+ // case 5 : bind_a->exec_a->close
+ {
+ printf("case 5 : bind_a->exec_a->close\n");
+ // init
+ TAOS_STMT2* stmt = taos_stmt2_init(taos, &option);
+ ASSERT_NE(stmt, nullptr);
+ int code = taos_stmt2_prepare(stmt, sql, 0);
+ checkError(stmt, code);
+ // bind
+ TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, NULL, paramv};
+ bool finish = false;
+ code = taos_stmt2_bind_param_a(stmt, &bindv, -1, stmtAsyncBindCb, (void*)&finish);
+ checkError(stmt, code);
+ // exec
+ code = taos_stmt2_exec(stmt, NULL);
+ checkError(stmt, code);
+ // close
+ taos_stmt2_close(stmt);
+ checkError(stmt, code);
+ }
+
+ option = {0, false, false, NULL, NULL};
+ stmt = taos_stmt2_init(taos, &option);
+ ASSERT_NE(stmt, nullptr);
+ code = taos_stmt2_prepare(stmt, sql, 0);
+ checkError(stmt, code);
+
+ // case 6 : bind_a->exec->bind_a->exec->...
+ {
+ printf("case 6 : bind_a->exec->bind_a->exec->...\n");
+ // init
+
+ checkError(stmt, code);
+ for (int r = 0; r < CYC_NUMS; r++) {
+ // bind
+ TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, NULL, paramv};
+ bool finish = false;
+ code = taos_stmt2_bind_param_a(stmt, &bindv, -1, stmtAsyncBindCb, (void*)&finish);
+ checkError(stmt, code);
+ // exec
+ code = taos_stmt2_exec(stmt, NULL);
+ checkError(stmt, code);
+ }
+ }
+
+ // case 7 (error:no wait error) : bind_a->bind_a
+ {
+ printf("case 7 (error:no wait error) : bind_a->bind_a\n");
+ // bind
+ TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, NULL, paramv};
+ bool finish = false;
+ code = taos_stmt2_bind_param_a(stmt, &bindv, -1, stmtAsyncBindCb2, (void*)&finish);
+ checkError(stmt, code);
+ taosMsleep(200);
+ code = taos_stmt2_bind_param_a(stmt, &bindv, -1, stmtAsyncBindCb2, (void*)&finish);
+ ASSERT_EQ(code, TSDB_CODE_TSC_STMT_API_ERROR);
+ while (!finish) {
+ taosMsleep(100);
+ }
+ }
+ // close
+ taos_stmt2_close(stmt);
+
+ // free memory
+ for (int i = 0; i < CTB_NUMS; i++) {
+ taosMemoryFree(paramv[i]);
+ taosMemoryFree(ts[i]);
+ taosMemoryFree(b[i]);
+ }
+ taosMemoryFree(ts);
+ taosMemoryFree(b);
+ taosMemoryFree(ts_len);
+ taosMemoryFree(b_len);
+ taosMemoryFree(paramv);
+ for (int i = 0; i < CTB_NUMS; i++) {
+ taosMemoryFree(tbs[i]);
+ }
+ taosMemoryFree(tbs);
+}
#pragma GCC diagnostic pop
diff --git a/source/common/src/msg/streamMsg.c b/source/common/src/msg/streamMsg.c
index 54b17b14d1..7e7952eb60 100644
--- a/source/common/src/msg/streamMsg.c
+++ b/source/common/src/msg/streamMsg.c
@@ -56,6 +56,7 @@ typedef struct STaskStatusEntry {
int64_t startCheckpointVer;
int64_t hTaskId;
STaskCkptInfo checkpointInfo;
+ STaskNotifyEventStat notifyEventStat;
} STaskStatusEntry;
int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamUpstreamEpInfo* pInfo) {
@@ -523,6 +524,19 @@ int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pReq) {
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->startCheckpointId));
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->startCheckpointVer));
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->hTaskId));
+ TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventAddTimes));
+ TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventAddElems));
+ TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->notifyEventStat.notifyEventAddCostSec));
+ TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventPushTimes));
+ TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventPushElems));
+ TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->notifyEventStat.notifyEventPushCostSec));
+ TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventPackTimes));
+ TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventPackElems));
+ TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->notifyEventStat.notifyEventPackCostSec));
+ TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventSendTimes));
+ TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventSendElems));
+ TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->notifyEventStat.notifyEventSendCostSec));
+ TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventHoldElems));
}
int32_t numOfVgs = taosArrayGetSize(pReq->pUpdateNodes);
@@ -596,6 +610,20 @@ int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) {
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.startCheckpointVer));
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.hTaskId));
+ TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventAddTimes));
+ TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventAddElems));
+ TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.notifyEventStat.notifyEventAddCostSec));
+ TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventPushTimes));
+ TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventPushElems));
+ TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.notifyEventStat.notifyEventPushCostSec));
+ TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventPackTimes));
+ TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventPackElems));
+ TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.notifyEventStat.notifyEventPackCostSec));
+ TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventSendTimes));
+ TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventSendElems));
+ TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.notifyEventStat.notifyEventSendCostSec));
+ TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventHoldElems));
+
entry.id.taskId = taskId;
if (taosArrayPush(pReq->pTaskStatus, &entry) == NULL) {
TAOS_CHECK_EXIT(terrno);
@@ -837,4 +865,4 @@ int32_t tDecodeStreamTaskRunReq(SDecoder* pDecoder, SStreamTaskRunReq* pReq) {
_exit:
return code;
-}
\ No newline at end of file
+}
diff --git a/source/common/src/msg/tmsg.c b/source/common/src/msg/tmsg.c
index 052fcbe2ba..6a3e1948c8 100644
--- a/source/common/src/msg/tmsg.c
+++ b/source/common/src/msg/tmsg.c
@@ -14,8 +14,8 @@
*/
#define _DEFAULT_SOURCE
-#include "tmsg.h"
#include "tglobal.h"
+#include "tmsg.h"
#undef TD_MSG_NUMBER_
#undef TD_MSG_DICT_
@@ -1471,6 +1471,12 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->analVer));
TAOS_CHECK_EXIT(tSerializeSMonitorParas(&encoder, &pReq->clusterCfg.monitorParas));
+ for (int32_t i = 0; i < vlen; ++i) {
+ SVnodeLoad *pload = taosArrayGet(pReq->pVloads, i);
+ TAOS_CHECK_EXIT(tEncodeI64(&encoder, pload->syncAppliedIndex));
+ TAOS_CHECK_EXIT(tEncodeI64(&encoder, pload->syncCommitIndex));
+ }
+
tEndEncode(&encoder);
_exit:
@@ -1600,6 +1606,14 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
TAOS_CHECK_EXIT(tDeserializeSMonitorParas(&decoder, &pReq->clusterCfg.monitorParas));
}
+ if (!tDecodeIsEnd(&decoder)) {
+ for (int32_t i = 0; i < vlen; ++i) {
+ SVnodeLoad *pLoad = taosArrayGet(pReq->pVloads, i);
+ TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pLoad->syncAppliedIndex));
+ TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pLoad->syncCommitIndex));
+ }
+ }
+
tEndDecode(&decoder);
_exit:
@@ -9219,6 +9233,8 @@ int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) {
TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->enableReplay));
TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->sourceExcluded));
TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->enableBatchMeta));
+ TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->rawData));
+ TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->minPollRows));
tEndEncode(&encoder);
@@ -9272,6 +9288,11 @@ int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) {
pReq->enableBatchMeta = false;
}
+ if (!tDecodeIsEnd(&decoder)) {
+ TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->rawData));
+ TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->minPollRows));
+ }
+
tEndDecode(&decoder);
_exit:
@@ -9279,7 +9300,13 @@ _exit:
return code;
}
-void tDestroySMqPollReq(SMqPollReq *pReq) { tOffsetDestroy(&pReq->reqOffset); }
+void tDestroySMqPollReq(SMqPollReq *pReq) {
+ tOffsetDestroy(&pReq->reqOffset);
+ if (pReq->uidHash != NULL) {
+ taosHashCleanup(pReq->uidHash);
+ pReq->uidHash = NULL;
+ }
+}
int32_t tSerializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq) {
int32_t code = 0;
int32_t lino;
@@ -11448,12 +11475,14 @@ int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRsp *pRsp) {
}
for (int32_t i = 0; i < pRsp->blockNum; i++) {
- void *data;
- uint64_t bLen;
- TAOS_CHECK_EXIT(tDecodeBinaryAlloc(pDecoder, &data, &bLen));
+ void *data = NULL;
+ uint32_t bLen = 0;
+ TAOS_CHECK_EXIT(tDecodeBinary(pDecoder, (uint8_t**)&data, &bLen));
if (taosArrayPush(pRsp->blockData, &data) == NULL) {
TAOS_CHECK_EXIT(terrno);
}
+ pRsp->blockDataElementFree = false;
+
int32_t len = bLen;
if (taosArrayPush(pRsp->blockDataLen, &len) == NULL) {
TAOS_CHECK_EXIT(terrno);
@@ -11499,10 +11528,25 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
return 0;
}
+int32_t tDecodeMqRawDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
+ int32_t code = 0;
+ int32_t lino;
+
+ TAOS_CHECK_EXIT(tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset));
+ TAOS_CHECK_EXIT(tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset));
+ TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->blockNum));
+_exit:
+ return code;
+}
+
static void tDeleteMqDataRspCommon(SMqDataRsp *pRsp) {
taosArrayDestroy(pRsp->blockDataLen);
pRsp->blockDataLen = NULL;
- taosArrayDestroyP(pRsp->blockData, NULL);
+ if (pRsp->blockDataElementFree){
+ taosArrayDestroyP(pRsp->blockData, NULL);
+ } else {
+ taosArrayDestroy(pRsp->blockData);
+ }
pRsp->blockData = NULL;
taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSchemaWrapper);
pRsp->blockSchema = NULL;
@@ -11510,6 +11554,7 @@ static void tDeleteMqDataRspCommon(SMqDataRsp *pRsp) {
pRsp->blockTbName = NULL;
tOffsetDestroy(&pRsp->reqOffset);
tOffsetDestroy(&pRsp->rspOffset);
+ taosMemoryFreeClear(pRsp->data);
}
void tDeleteMqDataRsp(SMqDataRsp *rsp) { tDeleteMqDataRspCommon(rsp); }
@@ -11572,6 +11617,14 @@ void tDeleteSTaosxRsp(SMqDataRsp *pRsp) {
pRsp->createTableReq = NULL;
}
+void tDeleteMqRawDataRsp(SMqDataRsp *pRsp) {
+ tOffsetDestroy(&pRsp->reqOffset);
+ tOffsetDestroy(&pRsp->rspOffset);
+ if (pRsp->rawData != NULL){
+ taosMemoryFree(POINTER_SHIFT(pRsp->rawData, - sizeof(SMqRspHead)));
+ }
+}
+
int32_t tEncodeSSingleDeleteReq(SEncoder *pEncoder, const SSingleDeleteReq *pReq) {
TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pReq->tbname));
TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pReq->startTs));
@@ -11658,6 +11711,26 @@ int32_t tDecodeSBatchDeleteReqSetCtime(SDecoder *pDecoder, SBatchDeleteReq *pReq
_exit:
return code;
}
+int32_t transformRawSSubmitTbData(void* data, int64_t suid, int64_t uid, int32_t sver){
+ int32_t code = 0;
+ int32_t lino = 0;
+ SDecoder decoder = {0};
+ tDecoderInit(&decoder, (uint8_t *)POINTER_SHIFT(data, INT_BYTES), *(uint32_t*)data);
+
+ int32_t flags = 0;
+ TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &flags));
+ flags |= TD_REQ_FROM_TAOX;
+ flags &= ~SUBMIT_REQ_AUTO_CREATE_TABLE;
+
+ SEncoder encoder = {0};
+ tEncoderInit(&encoder, (uint8_t *)POINTER_SHIFT(data, INT_BYTES), *(uint32_t*)data);
+ TAOS_CHECK_EXIT(tEncodeI32v(&encoder, flags));
+ TAOS_CHECK_EXIT(tEncodeI64(&encoder, suid));
+ TAOS_CHECK_EXIT(tEncodeI64(&encoder, uid));
+ TAOS_CHECK_EXIT(tEncodeI32v(&encoder, sver));
+ _exit:
+ return code;
+}
static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubmitTbData) {
int32_t code = 0;
@@ -11705,14 +11778,20 @@ _exit:
return code;
}
-static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbData) {
+static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbData, void* rawData) {
int32_t code = 0;
int32_t lino;
int32_t flags;
uint8_t version;
+ uint8_t* dataAfterCreate = NULL;
+ uint8_t* dataStart = pCoder->data + pCoder->pos;
+ uint32_t posAfterCreate = 0;
+
TAOS_CHECK_EXIT(tStartDecode(pCoder));
+ uint32_t pos = pCoder->pos;
TAOS_CHECK_EXIT(tDecodeI32v(pCoder, &flags));
+ uint32_t flagsLen = pCoder->pos - pos;
pSubmitTbData->flags = flags & 0xff;
version = (flags >> 8) & 0xff;
@@ -11724,6 +11803,8 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa
}
TAOS_CHECK_EXIT(tDecodeSVCreateTbReq(pCoder, pSubmitTbData->pCreateTbReq));
+ dataAfterCreate = pCoder->data + pCoder->pos;
+ posAfterCreate = pCoder->pos;
}
// submit data
@@ -11732,7 +11813,7 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa
TAOS_CHECK_EXIT(tDecodeI32v(pCoder, &pSubmitTbData->sver));
if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {
- uint64_t nColData;
+ uint64_t nColData = 0;
TAOS_CHECK_EXIT(tDecodeU64v(pCoder, &nColData));
@@ -11745,7 +11826,7 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa
TAOS_CHECK_EXIT(tDecodeColData(version, pCoder, taosArrayReserve(pSubmitTbData->aCol, 1)));
}
} else {
- uint64_t nRow;
+ uint64_t nRow = 0;
TAOS_CHECK_EXIT(tDecodeU64v(pCoder, &nRow));
pSubmitTbData->aRowP = taosArrayInit(nRow, sizeof(SRow *));
@@ -11768,6 +11849,15 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa
TAOS_CHECK_EXIT(tDecodeI64(pCoder, &pSubmitTbData->ctimeMs));
}
+ if (rawData != NULL){
+ if (dataAfterCreate != NULL){
+ TAOS_MEMCPY(dataAfterCreate - INT_BYTES - flagsLen, dataStart, INT_BYTES + flagsLen);
+ *(int32_t*)(dataAfterCreate - INT_BYTES - flagsLen) = pCoder->pos - posAfterCreate + flagsLen;
+ *(void**)rawData = dataAfterCreate - INT_BYTES - flagsLen;
+ }else{
+ *(void**)rawData = dataStart;
+ }
+ }
tEndDecode(pCoder);
_exit:
@@ -11780,15 +11870,27 @@ int32_t tEncodeSubmitReq(SEncoder *pCoder, const SSubmitReq2 *pReq) {
TAOS_CHECK_EXIT(tStartEncode(pCoder));
TAOS_CHECK_EXIT(tEncodeU64v(pCoder, taosArrayGetSize(pReq->aSubmitTbData)));
- for (uint64_t i = 0; i < taosArrayGetSize(pReq->aSubmitTbData); i++) {
- TAOS_CHECK_EXIT(tEncodeSSubmitTbData(pCoder, taosArrayGet(pReq->aSubmitTbData, i)));
+ if (pReq->raw){
+ for (uint64_t i = 0; i < taosArrayGetSize(pReq->aSubmitTbData); i++) {
+ void* data = taosArrayGetP(pReq->aSubmitTbData, i);
+ if (pCoder->data != NULL){
+ TAOS_MEMCPY(pCoder->data + pCoder->pos, data, *(uint32_t*)data + INT_BYTES);
+
+ }
+ pCoder->pos += *(uint32_t*)data + INT_BYTES;
+ }
+ } else{
+ for (uint64_t i = 0; i < taosArrayGetSize(pReq->aSubmitTbData); i++) {
+ TAOS_CHECK_EXIT(tEncodeSSubmitTbData(pCoder, taosArrayGet(pReq->aSubmitTbData, i)));
+ }
}
+
tEndEncode(pCoder);
_exit:
return code;
}
-int32_t tDecodeSubmitReq(SDecoder *pCoder, SSubmitReq2 *pReq) {
+int32_t tDecodeSubmitReq(SDecoder *pCoder, SSubmitReq2 *pReq, SArray* rawList) {
int32_t code = 0;
memset(pReq, 0, sizeof(*pReq));
@@ -11812,7 +11914,8 @@ int32_t tDecodeSubmitReq(SDecoder *pCoder, SSubmitReq2 *pReq) {
}
for (uint64_t i = 0; i < nSubmitTbData; i++) {
- if (tDecodeSSubmitTbData(pCoder, taosArrayReserve(pReq->aSubmitTbData, 1)) < 0) {
+ SSubmitTbData* data = taosArrayReserve(pReq->aSubmitTbData, 1);
+ if (tDecodeSSubmitTbData(pCoder, data, rawList != NULL ? taosArrayReserve(rawList, 1) : NULL) < 0) {
code = TSDB_CODE_INVALID_MSG;
goto _exit;
}
@@ -11821,13 +11924,6 @@ int32_t tDecodeSubmitReq(SDecoder *pCoder, SSubmitReq2 *pReq) {
tEndDecode(pCoder);
_exit:
- if (code) {
- if (pReq->aSubmitTbData) {
- // todo
- taosArrayDestroy(pReq->aSubmitTbData);
- pReq->aSubmitTbData = NULL;
- }
- }
return code;
}
@@ -11883,12 +11979,15 @@ void tDestroySubmitTbData(SSubmitTbData *pTbData, int32_t flag) {
void tDestroySubmitReq(SSubmitReq2 *pReq, int32_t flag) {
if (pReq->aSubmitTbData == NULL) return;
- int32_t nSubmitTbData = TARRAY_SIZE(pReq->aSubmitTbData);
- SSubmitTbData *aSubmitTbData = (SSubmitTbData *)TARRAY_DATA(pReq->aSubmitTbData);
+ if (!pReq->raw){
+ int32_t nSubmitTbData = TARRAY_SIZE(pReq->aSubmitTbData);
+ SSubmitTbData *aSubmitTbData = (SSubmitTbData *)TARRAY_DATA(pReq->aSubmitTbData);
- for (int32_t i = 0; i < nSubmitTbData; i++) {
- tDestroySubmitTbData(&aSubmitTbData[i], flag);
+ for (int32_t i = 0; i < nSubmitTbData; i++) {
+ tDestroySubmitTbData(&aSubmitTbData[i], flag);
+ }
}
+
taosArrayDestroy(pReq->aSubmitTbData);
pReq->aSubmitTbData = NULL;
}
diff --git a/source/common/src/systable.c b/source/common/src/systable.c
index 4deb1bba24..10c5eb71c1 100644
--- a/source/common/src/systable.c
+++ b/source/common/src/systable.c
@@ -212,6 +212,7 @@ static const SSysDbTableSchema streamTaskSchema[] = {
{.name = "extra_info", .bytes = 25 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "history_task_id", .bytes = 16 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "history_task_status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+ {.name = "notify_event_stat", .bytes = TSDB_STREAM_NOTIFY_STAT_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
};
static const SSysDbTableSchema userTblsSchema[] = {
@@ -292,12 +293,16 @@ static const SSysDbTableSchema vgroupsSchema[] = {
{.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "v1_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
{.name = "v1_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+ {.name = "v1_applied/committed", .bytes = TSDB_SYNC_APPLY_COMMIT_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "v2_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
{.name = "v2_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+ {.name = "v2_applied/committed", .bytes = TSDB_SYNC_APPLY_COMMIT_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "v3_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
{.name = "v3_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+ {.name = "v3_applied/committed", .bytes = TSDB_SYNC_APPLY_COMMIT_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "v4_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
{.name = "v4_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+ {.name = "v4_applied/committed", .bytes = TSDB_SYNC_APPLY_COMMIT_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "cacheload", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "cacheelements", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "tsma", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
@@ -368,6 +373,8 @@ static const SSysDbTableSchema vnodesSchema[] = {
{.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
{.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
{.name = "restored", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL, .sysInfo = true},
+ {.name = "apply_finish_time", .bytes = TSDB_SYNC_RESTORE_lEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+ {.name = "unapplied", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
};
static const SSysDbTableSchema userUserPrivilegesSchema[] = {
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index c3e0fff578..4a9a5ac274 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -165,6 +165,55 @@ int32_t colDataSetValOrCover(SColumnInfoData* pColumnInfoData, uint32_t rowIndex
return colDataSetValHelp(pColumnInfoData, rowIndex, pData, isNull);
}
+int32_t varColSetVarData(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pVarData, int32_t varDataLen,
+ bool isNull) {
+ if (!IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
+ return TSDB_CODE_INVALID_PARA;
+ }
+
+ if (isNull || pVarData == NULL) {
+ pColumnInfoData->varmeta.offset[rowIndex] = -1; // it is a null value of VAR type.
+ pColumnInfoData->hasNull = true;
+ return TSDB_CODE_SUCCESS;
+ }
+
+ int32_t dataLen = VARSTR_HEADER_SIZE + varDataLen;
+ if (pColumnInfoData->varmeta.offset[rowIndex] > 0) {
+ pColumnInfoData->varmeta.length = pColumnInfoData->varmeta.offset[rowIndex];
+ }
+
+ SVarColAttr* pAttr = &pColumnInfoData->varmeta;
+ if (pAttr->allocLen < pAttr->length + dataLen) {
+ uint32_t newSize = pAttr->allocLen;
+ if (newSize <= 1) {
+ newSize = 8;
+ }
+
+ while (newSize < pAttr->length + dataLen) {
+ newSize = newSize * 1.5;
+ if (newSize > UINT32_MAX) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+ }
+
+ char* buf = taosMemoryRealloc(pColumnInfoData->pData, newSize);
+ if (buf == NULL) {
+ return terrno;
+ }
+
+ pColumnInfoData->pData = buf;
+ pAttr->allocLen = newSize;
+ }
+
+ uint32_t len = pColumnInfoData->varmeta.length;
+ pColumnInfoData->varmeta.offset[rowIndex] = len;
+
+ (void)memmove(varDataVal(pColumnInfoData->pData + len), pVarData, varDataLen);
+ varDataSetLen(pColumnInfoData->pData + len, varDataLen);
+ pColumnInfoData->varmeta.length += dataLen;
+ return TSDB_CODE_SUCCESS;
+}
+
int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx,
const char* pData) {
int32_t type = pColumnInfoData->info.type;
@@ -2682,7 +2731,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDat
terrno = 0;
if (NULL == pReq) {
- if (!(pReq = taosMemoryMalloc(sizeof(SSubmitReq2)))) {
+ if (!(pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2)))) {
code = terrno;
goto _end;
}
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index f8f3c0f770..a02ca85a1f 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -152,10 +152,10 @@ static int32_t tRowBuildScan(SArray *colVals, const STSchema *schema, SRowBuildS
return TSDB_CODE_INVALID_PARA;
}
if (!(colValArray[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID)) {
- return TSDB_CODE_INVALID_PARA;
+ return TSDB_CODE_PAR_INVALID_FIRST_COLUMN;
}
if (!(colValArray[0].value.type == TSDB_DATA_TYPE_TIMESTAMP)) {
- return TSDB_CODE_INVALID_PARA;
+ return TSDB_CODE_PAR_INVALID_FIRST_COLUMN;;
}
*sinfo = (SRowBuildScanInfo){
@@ -3627,13 +3627,13 @@ int32_t tColDataSortMerge(SArray **arr) {
SColData *aColData = (SColData *)TARRAY_DATA(colDataArr);
if (!(aColData[0].type == TSDB_DATA_TYPE_TIMESTAMP)) {
- return TSDB_CODE_INVALID_PARA;
+ return TSDB_CODE_PAR_INVALID_FIRST_COLUMN;
}
if (!(aColData[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID)) {
- return TSDB_CODE_INVALID_PARA;
+ return TSDB_CODE_PAR_INVALID_FIRST_COLUMN;
}
if (!(aColData[0].flag == HAS_VALUE)) {
- return TSDB_CODE_INVALID_PARA;
+ return TSDB_CODE_PAR_PRIMARY_KEY_IS_NULL;
}
if (aColData[0].nVal <= 1) goto _exit;
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 223418feed..2fd16b4f67 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -356,6 +356,9 @@ int32_t tsMaxTsmaCalcDelay = 600;
int64_t tsmaDataDeleteMark = 1000 * 60 * 60 * 24; // in ms, default to 1d
void *pTimezoneNameMap = NULL;
+int32_t tsStreamNotifyMessageSize = 8 * 1024; // KB, default 8MB
+int32_t tsStreamNotifyFrameSize = 256; // KB, default 256KB
+
int32_t taosCheckCfgStrValueLen(const char *name, const char *value, int32_t len);
#define TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, pName) \
@@ -956,7 +959,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey[0], CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint[0], CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_GLOBAL));
- TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_GLOBAL));
+ TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_LOCAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL));
@@ -965,6 +968,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "minDiskFreeSize", tsMinDiskFreeSize, TFS_MIN_DISK_FREE_SIZE, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_LOCAL));
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL));
+ TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "streamNotifyMessageSize", tsStreamNotifyMessageSize, 8, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL));
+ TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "streamNotifyFrameSize", tsStreamNotifyFrameSize, 8, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL));
+
// clang-format on
// GRANT_CFG_ADD;
@@ -1850,6 +1856,12 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "minReservedMemorySize");
tsMinReservedMemorySize = pItem->i32;
+ TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "streamNotifyMessageSize");
+ tsStreamNotifyMessageSize = pItem->i32;
+
+ TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "streamNotifyFrameSize");
+ tsStreamNotifyFrameSize = pItem->i32;
+
// GRANT_CFG_GET;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
@@ -3241,4 +3253,4 @@ int32_t taosCheckCfgStrValueLen(const char *name, const char *value, int32_t len
TAOS_RETURN(TSDB_CODE_INVALID_CFG_VALUE);
}
TAOS_RETURN(TSDB_CODE_SUCCESS);
-}
\ No newline at end of file
+}
diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c
index 51e4d86acb..bd7da3f4d6 100644
--- a/source/dnode/mgmt/exe/dmMain.c
+++ b/source/dnode/mgmt/exe/dmMain.c
@@ -21,12 +21,15 @@
#include "tglobal.h"
#include "version.h"
#include "tconv.h"
-#ifdef TD_JEMALLOC_ENABLED
-#include "jemalloc/jemalloc.h"
-#endif
#include "dmUtil.h"
#include "tcs.h"
#include "qworker.h"
+
+#ifdef TD_JEMALLOC_ENABLED
+#define ALLOW_FORBID_FUNC
+#include "jemalloc/jemalloc.h"
+#endif
+
#include "cus_name.h"
// clang-format off
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
index 3677fc5616..234d4f41e1 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@@ -1016,15 +1016,15 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_CHECK_POINT_SOURCE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
- if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
- if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
- if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
- if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, vmPutMsgToStreamCtrlQueue, 0) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY_RSP, vmPutMsgToStreamCtrlQueue, 0) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER, vmPutMsgToStreamCtrlQueue, 0) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER_RSP, vmPutMsgToStreamCtrlQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, vmPutMsgToStreamCtrlQueue, 0) == NULL) goto _OVER;
- if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
- if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, vmPutMsgToStreamCtrlQueue, 0) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT_RSP, vmPutMsgToStreamCtrlQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
index 00fa2a8c95..d71e0b02c4 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
@@ -396,7 +396,8 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal,
tqNotifyClose(pVnode->pImpl->pTq);
- dInfo("vgId:%d, wait for vnode stream queue:%p is empty", pVnode->vgId, pVnode->pStreamQ);
+ dInfo("vgId:%d, wait for vnode stream queue:%p is empty, %d remains", pVnode->vgId,
+ pVnode->pStreamQ, taosQueueItemSize(pVnode->pStreamQ));
while (!taosQueueEmpty(pVnode->pStreamQ)) taosMsleep(10);
dInfo("vgId:%d, wait for vnode stream ctrl queue:%p is empty", pVnode->vgId, pVnode->pStreamCtrlQ);
diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h
index 71da8beed9..050c722167 100644
--- a/source/dnode/mnode/impl/inc/mndDef.h
+++ b/source/dnode/mnode/impl/inc/mndDef.h
@@ -136,7 +136,7 @@ typedef enum {
typedef enum {
TRN_KILL_MODE_SKIP = 0,
TRN_KILL_MODE_INTERUPT = 1,
- //TRN_KILL_MODE_ROLLBACK = 2,
+ // TRN_KILL_MODE_ROLLBACK = 2,
} ETrnKillMode;
typedef enum {
@@ -478,6 +478,10 @@ typedef struct {
int32_t dnodeId;
ESyncState syncState;
int64_t syncTerm;
+ int64_t syncAppliedIndex;
+ int64_t lastSyncAppliedIndexUpdateTime;
+ double appliedRate;
+ int64_t syncCommitIndex;
bool syncRestore;
bool syncCanRead;
int64_t roleTimeMs;
diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c
index ca119191eb..1a74573490 100644
--- a/source/dnode/mnode/impl/src/mndDnode.c
+++ b/source/dnode/mnode/impl/src/mndDnode.c
@@ -535,11 +535,37 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, SDnodeObj *pDnode, const S
return DND_REASON_ONLINE;
}
+double calcAppliedRate(int64_t currentCount, int64_t lastCount, int64_t currentTimeMs, int64_t lastTimeMs) {
+ if ((currentTimeMs <= lastTimeMs) || (currentCount <= lastCount)) {
+ return 0.0;
+ }
+
+ int64_t deltaCount = currentCount - lastCount;
+ int64_t deltaMs = currentTimeMs - lastTimeMs;
+ double rate = (double)deltaCount / (double)deltaMs;
+ return rate;
+}
+
static bool mndUpdateVnodeState(int32_t vgId, SVnodeGid *pGid, SVnodeLoad *pVload) {
bool stateChanged = false;
bool roleChanged = pGid->syncState != pVload->syncState ||
(pVload->syncTerm != -1 && pGid->syncTerm != pVload->syncTerm) ||
pGid->roleTimeMs != pVload->roleTimeMs;
+
+ if (pVload->syncCommitIndex > pVload->syncAppliedIndex) {
+ if (pGid->lastSyncAppliedIndexUpdateTime == 0) {
+ pGid->lastSyncAppliedIndexUpdateTime = taosGetTimestampMs();
+ } else if (pGid->syncAppliedIndex != pVload->syncAppliedIndex) {
+ int64_t currentTimeMs = taosGetTimestampMs();
+ pGid->appliedRate = calcAppliedRate(pVload->syncAppliedIndex, pGid->syncAppliedIndex, currentTimeMs,
+ pGid->lastSyncAppliedIndexUpdateTime);
+
+ pGid->lastSyncAppliedIndexUpdateTime = currentTimeMs;
+ }
+ }
+
+ pGid->syncAppliedIndex = pVload->syncAppliedIndex;
+ pGid->syncCommitIndex = pVload->syncCommitIndex;
if (roleChanged || pGid->syncRestore != pVload->syncRestore || pGid->syncCanRead != pVload->syncCanRead ||
pGid->startTimeMs != pVload->startTimeMs) {
mInfo(
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index 65de7be704..a0c68f8b00 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -765,7 +765,7 @@ static int32_t addStreamTaskNotifyInfo(const SCMCreateStreamReq *createReq, cons
TSDB_CHECK_NULL(pTask->notifyInfo.pNotifyAddrUrls, code, lino, _end, terrno);
pTask->notifyInfo.notifyEventTypes = createReq->notifyEventTypes;
pTask->notifyInfo.notifyErrorHandle = createReq->notifyErrorHandle;
- pTask->notifyInfo.streamName = taosStrdup(createReq->name);
+ pTask->notifyInfo.streamName = taosStrdup(mndGetDbStr(createReq->name));
TSDB_CHECK_NULL(pTask->notifyInfo.streamName, code, lino, _end, terrno);
pTask->notifyInfo.stbFullName = taosStrdup(createReq->targetStbFullName);
TSDB_CHECK_NULL(pTask->notifyInfo.stbFullName, code, lino, _end, terrno);
@@ -1239,51 +1239,22 @@ int32_t extractStreamNodeList(SMnode *pMnode) {
}
static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
- bool ready = true;
+ int32_t code = 0;
if (mndStreamNodeIsUpdated(pMnode)) {
- TAOS_RETURN(TSDB_CODE_STREAM_TASK_IVLD_STATUS);
+ return TSDB_CODE_STREAM_TASK_IVLD_STATUS;
}
streamMutexLock(&execInfo.lock);
if (taosArrayGetSize(execInfo.pNodeList) == 0) {
mDebug("stream task node change checking done, no vgroups exist, do nothing");
if (taosArrayGetSize(execInfo.pTaskList) != 0) {
- streamMutexUnlock(&execInfo.lock);
mError("stream task node change checking done, no vgroups exist, but task list is not empty");
- return TSDB_CODE_FAILED;
- }
- }
-
- for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
- STaskId *p = taosArrayGet(execInfo.pTaskList, i);
- if (p == NULL) {
- continue;
- }
-
- STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p));
- if (pEntry == NULL) {
- continue;
- }
-
- if (pEntry->status != TASK_STATUS__READY) {
- mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s, checkpoint not issued", pEntry->id.streamId,
- (int32_t)pEntry->id.taskId, pEntry->nodeId, streamTaskGetStatusStr(pEntry->status));
- ready = false;
- break;
- }
-
- if (pEntry->hTaskId != 0) {
- mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s related fill-history task:0x%" PRIx64
- " exists, checkpoint not issued",
- pEntry->id.streamId, (int32_t)pEntry->id.taskId, pEntry->nodeId, streamTaskGetStatusStr(pEntry->status),
- pEntry->hTaskId);
- ready = false;
- break;
+ code = TSDB_CODE_STREAM_TASK_IVLD_STATUS;
}
}
streamMutexUnlock(&execInfo.lock);
- return ready ? 0 : -1;
+ return code;
}
int64_t getStreamTaskLastReadyState(SArray *pTaskList, int64_t streamId) {
@@ -1297,7 +1268,22 @@ int64_t getStreamTaskLastReadyState(SArray *pTaskList, int64_t streamId) {
continue;
}
- if (pEntry->status == TASK_STATUS__READY && ts < pEntry->startTime) {
+ // -1 denote not ready now or never ready till now
+ if (pEntry->hTaskId != 0) {
+ mInfo("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s related fill-history task:0x%" PRIx64
+ " exists, checkpoint not issued",
+ pEntry->id.streamId, (int32_t)pEntry->id.taskId, pEntry->nodeId, streamTaskGetStatusStr(pEntry->status),
+ pEntry->hTaskId);
+ return -1;
+ }
+
+ if (pEntry->status != TASK_STATUS__READY) {
+ mInfo("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s, not ready for checkpoint", pEntry->id.streamId,
+ (int32_t)pEntry->id.taskId, pEntry->nodeId, streamTaskGetStatusStr(pEntry->status));
+ return -1;
+ }
+
+ if (ts < pEntry->startTime) {
ts = pEntry->startTime;
taskId = pEntry->id.taskId;
}
@@ -1330,11 +1316,11 @@ static bool isStreamReadyHelp(int64_t now, SStreamObj* pStream) {
int64_t lastReadyTs = getStreamTaskLastReadyState(execInfo.pTaskList, pStream->uid);
if ((lastReadyTs == -1) || ((lastReadyTs != -1) && ((now - lastReadyTs) < tsStreamCheckpointInterval * 1000))) {
+
if (lastReadyTs != -1) {
- mInfo("not start checkpoint, stream:0x%"PRIx64" last ready ts:%"PRId64" ready duration:%"PRId64" less than threshold",
- pStream->uid, lastReadyTs, now - lastReadyTs);
- } else {
- mInfo("not start checkpoint, stream:0x%"PRIx64" not ready now", pStream->uid);
+ mInfo("not start checkpoint, stream:0x%" PRIx64 " last ready ts:%" PRId64 " ready duration:%" PRId64
+ "ms less than threshold",
+ pStream->uid, lastReadyTs, (now - lastReadyTs));
}
ready = false;
@@ -1355,7 +1341,7 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
int32_t numOfCheckpointTrans = 0;
if ((code = mndCheckTaskAndNodeStatus(pMnode)) != 0) {
- TAOS_RETURN(TSDB_CODE_STREAM_TASK_IVLD_STATUS);
+ return TSDB_CODE_STREAM_TASK_IVLD_STATUS;
}
SArray *pList = taosArrayInit(4, sizeof(SCheckpointInterval));
@@ -1407,7 +1393,7 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
}
int32_t numOfQual = taosArrayGetSize(pList);
- if (numOfCheckpointTrans > tsMaxConcurrentCheckpoint) {
+ if (numOfCheckpointTrans >= tsMaxConcurrentCheckpoint) {
mDebug(
"%d stream(s) checkpoint interval longer than %ds, ongoing checkpoint trans:%d reach maximum allowed:%d, new "
"checkpoint trans are not allowed, wait for 30s",
@@ -2713,20 +2699,51 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId,
}
}
+static int32_t doCleanReqList(SArray* pList, SCheckpointConsensusInfo* pInfo) {
+ int32_t alreadySend = taosArrayGetSize(pList);
+
+ for (int32_t i = 0; i < alreadySend; ++i) {
+ int32_t *taskId = taosArrayGet(pList, i);
+ if (taskId == NULL) {
+ continue;
+ }
+
+ for (int32_t k = 0; k < taosArrayGetSize(pInfo->pTaskList); ++k) {
+ SCheckpointConsensusEntry *pe = taosArrayGet(pInfo->pTaskList, k);
+ if ((pe != NULL) && (pe->req.taskId == *taskId)) {
+ taosArrayRemove(pInfo->pTaskList, k);
+ break;
+ }
+ }
+ }
+
+ return alreadySend;
+}
+
int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) {
SMnode *pMnode = pMsg->info.node;
int64_t now = taosGetTimestampMs();
+ bool allReady = true;
+ SArray *pNodeSnapshot = NULL;
+ int32_t maxAllowedTrans = 50;
+ int32_t numOfTrans = 0;
+ int32_t code = 0;
+ void *pIter = NULL;
+
+ SArray *pList = taosArrayInit(4, sizeof(int32_t));
+ if (pList == NULL) {
+ return terrno;
+ }
+
SArray *pStreamList = taosArrayInit(4, sizeof(int64_t));
if (pStreamList == NULL) {
+ taosArrayDestroy(pList);
return terrno;
}
mDebug("start to process consensus-checkpointId in tmr");
- bool allReady = true;
- SArray *pNodeSnapshot = NULL;
-
- int32_t code = mndTakeVgroupSnapshot(pMnode, &allReady, &pNodeSnapshot);
+ code = mndTakeVgroupSnapshot(pMnode, &allReady, &pNodeSnapshot);
taosArrayDestroy(pNodeSnapshot);
if (code) {
mError("failed to get the vgroup snapshot, ignore it and continue");
@@ -2735,28 +2752,30 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) {
if (!allReady) {
mWarn("not all vnodes are ready, end to process the consensus-checkpointId in tmr process");
taosArrayDestroy(pStreamList);
+ taosArrayDestroy(pList);
return 0;
}
streamMutexLock(&execInfo.lock);
- void *pIter = NULL;
while ((pIter = taosHashIterate(execInfo.pStreamConsensus, pIter)) != NULL) {
SCheckpointConsensusInfo *pInfo = (SCheckpointConsensusInfo *)pIter;
- int64_t streamId = -1;
- int32_t num = taosArrayGetSize(pInfo->pTaskList);
- SArray *pList = taosArrayInit(4, sizeof(int32_t));
- if (pList == NULL) {
- continue;
- }
+ taosArrayClear(pList);
+ int64_t streamId = -1;
+ int32_t num = taosArrayGetSize(pInfo->pTaskList);
SStreamObj *pStream = NULL;
+
code = mndGetStreamObj(pMnode, pInfo->streamId, &pStream);
if (pStream == NULL || code != 0) { // stream has been dropped already
mDebug("stream:0x%" PRIx64 " dropped already, continue", pInfo->streamId);
void *p = taosArrayPush(pStreamList, &pInfo->streamId);
- taosArrayDestroy(pList);
+ if (p == NULL) {
+ mError("failed to record the missing stream id in concensus-stream list, streamId:%" PRId64
+ " code:%s, continue",
+ pInfo->streamId, tstrerror(terrno));
+ }
continue;
}
@@ -2766,7 +2785,9 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) {
continue;
}
- streamId = pe->req.streamId;
+ if (streamId == -1) {
+ streamId = pe->req.streamId;
+ }
int32_t existed = 0;
bool allSame = true;
@@ -2785,8 +2806,12 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) {
taosArrayDestroy(pStreamList);
mError("s-task:0x%x checkpointId:%" PRId64 " is updated to %" PRId64 ", update it", pe->req.taskId,
pe->req.checkpointId, chkId);
+
+ mndReleaseStream(pMnode, pStream);
+ taosHashCancelIterate(execInfo.pStreamConsensus, pIter);
return TSDB_CODE_FAILED;
}
+
code = mndCreateSetConsensusChkptIdTrans(pMnode, pStream, pe->req.taskId, chkId, pe->req.startTs);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("failed to create consensus-checkpoint trans, stream:0x%" PRIx64, pStream->uid);
@@ -2796,7 +2821,6 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) {
if (p == NULL) {
mError("failed to put into task list, taskId:0x%x", pe->req.taskId);
}
- streamId = pe->req.streamId;
} else {
mDebug("s-task:0x%x sendTs:%" PRId64 " wait %.2fs already, wait for next round to check", pe->req.taskId,
pe->req.startTs, (now - pe->ts) / 1000.0);
@@ -2805,38 +2829,27 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) {
mndReleaseStream(pMnode, pStream);
- if (taosArrayGetSize(pList) > 0) {
- for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
- int32_t *taskId = taosArrayGet(pList, i);
- if (taskId == NULL) {
- continue;
- }
-
- for (int32_t k = 0; k < taosArrayGetSize(pInfo->pTaskList); ++k) {
- SCheckpointConsensusEntry *pe = taosArrayGet(pInfo->pTaskList, k);
- if ((pe != NULL) && (pe->req.taskId == *taskId)) {
- taosArrayRemove(pInfo->pTaskList, k);
- break;
- }
- }
- }
- }
-
- taosArrayDestroy(pList);
+ int32_t alreadySend = doCleanReqList(pList, pInfo);
+ // clear request stream item with empty task list
if (taosArrayGetSize(pInfo->pTaskList) == 0) {
mndClearConsensusRspEntry(pInfo);
if (streamId == -1) {
- streamMutexUnlock(&execInfo.lock);
- taosArrayDestroy(pStreamList);
- mError("streamId is -1, streamId:%" PRIx64, pInfo->streamId);
- return TSDB_CODE_FAILED;
+ mError("streamId is -1, streamId:%" PRIx64" in consensus-checkpointId hashMap, cont", pInfo->streamId);
}
+
void *p = taosArrayPush(pStreamList, &streamId);
if (p == NULL) {
- mError("failed to put into stream list, stream:0x%" PRIx64, streamId);
+ mError("failed to put into stream list, stream:0x%" PRIx64 " not remove it in consensus-chkpt list", streamId);
}
}
+
+ numOfTrans += alreadySend;
+ if (numOfTrans > maxAllowedTrans) {
+ mInfo("already send consensus-checkpointId trans:%d, try next time", alreadySend);
+ taosHashCancelIterate(execInfo.pStreamConsensus, pIter);
+ break;
+ }
}
for (int32_t i = 0; i < taosArrayGetSize(pStreamList); ++i) {
@@ -2851,7 +2864,9 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) {
streamMutexUnlock(&execInfo.lock);
taosArrayDestroy(pStreamList);
- mDebug("end to process consensus-checkpointId in tmr");
+ taosArrayDestroy(pList);
+
+ mDebug("end to process consensus-checkpointId in tmr, send consensus-checkpoint trans:%d", numOfTrans);
return code;
}
diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c
index d896434f3b..0caaccc28d 100644
--- a/source/dnode/mnode/impl/src/mndStreamUtil.c
+++ b/source/dnode/mnode/impl/src/mndStreamUtil.c
@@ -814,17 +814,18 @@ int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) {
int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, int32_t taskId, int64_t checkpointId,
int64_t ts) {
- char msg[128] = {0};
+ char msg[128] = {0};
+ STrans *pTrans = NULL;
+ SStreamTask *pTask = NULL;
+
snprintf(msg, tListLen(msg), "set consen-chkpt-id for task:0x%x", taskId);
- STrans *pTrans = NULL;
int32_t code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_CONSEN_NAME, msg, &pTrans);
if (pTrans == NULL || code != 0) {
return terrno;
}
- STaskId id = {.streamId = pStream->uid, .taskId = taskId};
- SStreamTask *pTask = NULL;
+ STaskId id = {.streamId = pStream->uid, .taskId = taskId};
code = mndGetStreamTask(&id, pStream, &pTask);
if (code) {
mError("failed to get task:0x%x in stream:%s, failed to create consensus-checkpointId", taskId, pStream->name);
@@ -1309,8 +1310,8 @@ int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SSDataBlo
TSDB_CHECK_CODE(code, lino, _end);
// input queue
- char vbuf[40] = {0};
- char buf[38] = {0};
+ char vbuf[TSDB_STREAM_NOTIFY_STAT_LEN + 2] = {0};
+ char buf[TSDB_STREAM_NOTIFY_STAT_LEN] = {0};
const char *queueInfoStr = "%4.2f MiB (%6.2f%)";
snprintf(buf, tListLen(buf), queueInfoStr, pe->inputQUsed, pe->inputRate);
STR_TO_VARSTR(vbuf, buf);
@@ -1503,6 +1504,47 @@ int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SSDataBlo
code = colDataSetVal(pColInfo, numOfRows, 0, true);
TSDB_CHECK_CODE(code, lino, _end);
+ // notify_event_stat
+ int32_t offset =0;
+ if (pe->notifyEventStat.notifyEventAddTimes > 0) {
+ offset += tsnprintf(buf + offset, sizeof(buf) - offset, "Add %" PRId64 "x, %" PRId64 " elems in %lfs; ",
+ pe->notifyEventStat.notifyEventAddTimes, pe->notifyEventStat.notifyEventAddElems,
+ pe->notifyEventStat.notifyEventAddCostSec);
+ }
+ if (pe->notifyEventStat.notifyEventPushTimes > 0) {
+ offset += tsnprintf(buf + offset, sizeof(buf) - offset, "Push %" PRId64 "x, %" PRId64 " elems in %lfs; ",
+ pe->notifyEventStat.notifyEventPushTimes, pe->notifyEventStat.notifyEventPushElems,
+ pe->notifyEventStat.notifyEventPushCostSec);
+ }
+ if (pe->notifyEventStat.notifyEventPackTimes > 0) {
+ offset += tsnprintf(buf + offset, sizeof(buf) - offset, "Pack %" PRId64 "x, %" PRId64 " elems in %lfs; ",
+ pe->notifyEventStat.notifyEventPackTimes, pe->notifyEventStat.notifyEventPackElems,
+ pe->notifyEventStat.notifyEventPackCostSec);
+ }
+ if (pe->notifyEventStat.notifyEventSendTimes > 0) {
+ offset += tsnprintf(buf + offset, sizeof(buf) - offset, "Send %" PRId64 "x, %" PRId64 " elems in %lfs; ",
+ pe->notifyEventStat.notifyEventSendTimes, pe->notifyEventStat.notifyEventSendElems,
+ pe->notifyEventStat.notifyEventSendCostSec);
+ }
+ if (pe->notifyEventStat.notifyEventHoldElems > 0) {
+ offset += tsnprintf(buf + offset, sizeof(buf) - offset, "[Hold %" PRId64 " elems] ",
+ pe->notifyEventStat.notifyEventHoldElems);
+ }
+ TSDB_CHECK_CONDITION(offset < sizeof(buf), code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
+ buf[offset] = '\0';
+
+ STR_TO_VARSTR(vbuf, buf);
+
+ pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+ TSDB_CHECK_NULL(pColInfo, code, lino, _end, terrno);
+
+ if (offset == 0) {
+ colDataSetNULL(pColInfo, numOfRows);
+ } else {
+ code = colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false);
+ TSDB_CHECK_CODE(code, lino, _end);
+ }
+
_end:
if (code) {
mError("error happens during build task attr result blocks, lino:%d, code:%s", lino, tstrerror(code));
@@ -1689,4 +1731,4 @@ int32_t mndCheckForSnode(SMnode *pMnode, SDbObj *pSrcDb) {
mError("snode not existed when trying to create stream in db with multiple replica");
return TSDB_CODE_SNODE_NOT_DEPLOYED;
}
-}
\ No newline at end of file
+}
diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c
index e20afb7201..311beb0daa 100644
--- a/source/dnode/mnode/impl/src/mndVgroup.c
+++ b/source/dnode/mnode/impl/src/mndVgroup.c
@@ -244,6 +244,8 @@ static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew) {
pNewGid->syncState = pOldGid->syncState;
pNewGid->syncRestore = pOldGid->syncRestore;
pNewGid->syncCanRead = pOldGid->syncCanRead;
+ pNewGid->syncAppliedIndex = pOldGid->syncAppliedIndex;
+ pNewGid->syncCommitIndex = pOldGid->syncCommitIndex;
}
}
}
@@ -1122,10 +1124,25 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
mError("vgId:%d, failed to set role, since %s", pVgroup->vgId, tstrerror(code));
return code;
}
+
+ char applyStr[TSDB_SYNC_APPLY_COMMIT_LEN + 1] = {0};
+ char buf[TSDB_SYNC_APPLY_COMMIT_LEN + VARSTR_HEADER_SIZE + 1] = {0};
+ snprintf(applyStr, sizeof(applyStr), "%" PRId64 "/%" PRId64, pVgroup->vnodeGid[i].syncAppliedIndex,
+ pVgroup->vnodeGid[i].syncCommitIndex);
+ STR_WITH_MAXSIZE_TO_VARSTR(buf, applyStr, pShow->pMeta->pSchemas[cols].bytes);
+
+ pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+ code = colDataSetVal(pColInfo, numOfRows, (const char *)&buf, false);
+ if (code != 0) {
+ mError("vgId:%d, failed to set role, since %s", pVgroup->vgId, tstrerror(code));
+ return code;
+ }
} else {
colDataSetNULL(pColInfo, numOfRows);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetNULL(pColInfo, numOfRows);
+ pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+ colDataSetNULL(pColInfo, numOfRows);
}
}
@@ -1232,6 +1249,21 @@ int64_t mndGetVnodesMemory(SMnode *pMnode, int32_t dnodeId) {
return vnodeMemory;
}
+void calculateRstoreFinishTime(double rate, int64_t applyCount, char *restoreStr, size_t restoreStrSize) {
+ if (rate == 0) {
+ snprintf(restoreStr, restoreStrSize, "0:0:0");
+ return;
+ }
+
+ int64_t costTime = applyCount / rate;
+ int64_t totalSeconds = costTime / 1000;
+ int64_t hours = totalSeconds / 3600;
+ totalSeconds %= 3600;
+ int64_t minutes = totalSeconds / 60;
+ int64_t seconds = totalSeconds % 60;
+ snprintf(restoreStr, restoreStrSize, "%" PRId64 ":%" PRId64 ":%" PRId64, hours, minutes, seconds);
+}
+
static int32_t mndRetrieveVnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
@@ -1319,6 +1351,26 @@ static int32_t mndRetrieveVnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
return code;
}
+ int64_t unappliedCount = pGid->syncCommitIndex - pGid->syncAppliedIndex;
+ pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+ char restoreStr[20] = {0};
+ if (unappliedCount > 0) {
+ calculateRstoreFinishTime(pGid->appliedRate, unappliedCount, restoreStr, sizeof(restoreStr));
+ }
+ STR_TO_VARSTR(buf, restoreStr);
+ code = colDataSetVal(pColInfo, numOfRows, (const char *)&buf, false);
+ if (code != 0) {
+ mError("vgId:%d, failed to set syncRestore finish time, since %s", pVgroup->vgId, tstrerror(code));
+ return code;
+ }
+
+ pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+ code = colDataSetVal(pColInfo, numOfRows, (const char *)&unappliedCount, false);
+ if (code != 0) {
+ mError("vgId:%d, failed to set syncRestore, since %s", pVgroup->vgId, tstrerror(code));
+ return code;
+ }
+
numOfRows++;
sdbRelease(pSdb, pDnode);
}
@@ -2771,7 +2823,7 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb
pVgroup->vnodeGid[0].dnodeId);
// add second
- if (pNewVgroup->replica == 1){
+ if (pNewVgroup->replica == 1) {
TAOS_CHECK_RETURN(mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray));
}
@@ -2792,8 +2844,8 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb
TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup));
// add third
- if (pNewVgroup->replica == 2){
- TAOS_CHECK_RETURN (mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray));
+ if (pNewVgroup->replica == 2) {
+ TAOS_CHECK_RETURN(mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray));
}
pNewVgroup->vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER;
@@ -2823,7 +2875,7 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb
TAOS_CHECK_RETURN(mndRemoveVnodeFromVgroup(pMnode, pTrans, pNewVgroup, pArray, &del2));
TAOS_CHECK_RETURN(mndAddDropVnodeAction(pMnode, pTrans, pNewDb, pNewVgroup, &del2, true));
TAOS_CHECK_RETURN(
- mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId));
+ mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId));
TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup));
} else if (pNewDb->cfg.replications == 2) {
mInfo("db:%s, vgId:%d, will add 1 vnode, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId,
diff --git a/source/dnode/snode/src/snodeInitApi.c b/source/dnode/snode/src/snodeInitApi.c
index 68dc981338..54ec15a558 100644
--- a/source/dnode/snode/src/snodeInitApi.c
+++ b/source/dnode/snode/src/snodeInitApi.c
@@ -97,6 +97,7 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->updateInfoSerialize = updateInfoSerialize;
pStore->updateInfoDeserialize = updateInfoDeserialize;
+ pStore->streamStateSessionSeekKeyPrev = streamStateSessionSeekKeyPrev;
pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext;
pStore->streamStateCountSeekKeyPrev = streamStateCountSeekKeyPrev;
pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev;
@@ -123,4 +124,4 @@ void initStateStoreAPI(SStateStore* pStore) {
void initFunctionStateStore(SFunctionStateStore* pStore) {
pStore->streamStateFuncPut = streamStateFuncPut;
pStore->streamStateFuncGet = streamStateFuncGet;
-}
\ No newline at end of file
+}
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index 50d75c4838..f0e7af50f3 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -242,11 +242,11 @@ SSDataBlock *tqGetResultBlock(STqReader *pReader);
int64_t tqGetResultBlockTime(STqReader *pReader);
int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, int64_t maxVer, const char *id);
-int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver);
+int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver, SArray* rawList);
+void tqReaderClearSubmitMsg(STqReader *pReader);
bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char *idstr);
-int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet,
- int64_t *createTime);
+int32_t tqRetrieveTaosxBlock(STqReader *pReader, SMqDataRsp* pRsp, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet, SArray* rawList, int8_t fetchMeta);
int32_t tqGetStreamExecInfo(SVnode *pVnode, int64_t streamId, int64_t *pDelay, bool *fhFinished);
// sma
diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h
index e0bf51b333..cd66e82687 100644
--- a/source/dnode/vnode/src/inc/tq.h
+++ b/source/dnode/vnode/src/inc/tq.h
@@ -90,6 +90,7 @@ typedef struct {
// for replay
SSDataBlock* block;
int64_t blockTime;
+ SHashObj* tableCreateTimeHash; // for process create table msg in submit if fetch raw data
} STqHandle;
struct STQ {
@@ -112,13 +113,13 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle);
void tqDestroyTqHandle(void* data);
// tqRead
-int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBatchMetaRsp* pBatchMetaRsp, STqOffsetVal* offset, int64_t timeout);
+int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBatchMetaRsp* pBatchMetaRsp, STqOffsetVal* offset, const SMqPollReq* pRequest);
int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset, const SMqPollReq* pRequest);
int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t reqId);
// tqExec
-int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded);
-int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp,
+int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqDataRsp* pRsp, int32_t* totalRows, const SMqPollReq* pRequest);
+int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, SMqDataRsp* pRsp,
int32_t type, int32_t vgId);
void tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId);
@@ -147,7 +148,7 @@ int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name);
// tq util
int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type, EStreamType blockType);
int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg);
-int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId,
+int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId,
int32_t type, int64_t sver, int64_t ever);
int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset);
void tqUpdateNodeStage(STQ* pTq, bool isLeader);
@@ -182,7 +183,6 @@ int32_t tqSendAllNotifyEvents(const SArray* pBlocks, SStreamTask* pTask, SVnode*
#define TQ_SUBSCRIBE_NAME "subscribe"
#define TQ_OFFSET_NAME "offset-ver0"
-#define TQ_POLL_MAX_TIME 1000
#ifdef __cplusplus
}
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 27ee54ec70..0feb2e8e4f 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -167,7 +167,8 @@ int32_t metaDropMultipleTables(SMeta* pMeta, int64_t version, SArray* tb
int metaTtlFindExpired(SMeta* pMeta, int64_t timePointMs, SArray* tbUids, int32_t ttlDropMaxCount);
int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp);
int metaUpdateChangeTimeWithLock(SMeta* pMeta, tb_uid_t uid, int64_t changeTimeMs);
-SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, int64_t* createTime);
+SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock);
+int64_t metaGetTableCreateTime(SMeta *pMeta, tb_uid_t uid, int lock);
int32_t metaGetTbTSchemaNotNull(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, STSchema** ppTSchema);
int32_t metaGetTbTSchemaMaybeNull(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, STSchema** ppTSchema);
STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock);
diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c
index 93347c810f..9b85ca0b45 100644
--- a/source/dnode/vnode/src/meta/metaCache.c
+++ b/source/dnode/vnode/src/meta/metaCache.c
@@ -402,6 +402,7 @@ int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo) {
if (*ppEntry) { // update
(*ppEntry)->info.ctbNum = pInfo->ctbNum;
+ (*ppEntry)->info.colNum = pInfo->colNum;
} else { // insert
if (pCache->sStbStatsCache.nEntry >= pCache->sStbStatsCache.nBucket) {
TAOS_UNUSED(metaRehashStatsCache(pCache, 1));
diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c
index 4e2c93ec2f..b23059079a 100644
--- a/source/dnode/vnode/src/meta/metaEntry2.c
+++ b/source/dnode/vnode/src/meta/metaEntry2.c
@@ -10,14 +10,16 @@
#include "meta.h"
+extern SDmNotifyHandle dmNotifyHdl;
+
int32_t metaCloneEntry(const SMetaEntry *pEntry, SMetaEntry **ppEntry);
void metaCloneEntryFree(SMetaEntry **ppEntry);
void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey);
int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
-void metaTimeSeriesNotifyCheck(SMeta *pMeta);
int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
+static void metaTimeSeriesNotifyCheck(SMeta *pMeta);
static int32_t metaGetChildUidsOfSuperTable(SMeta *pMeta, tb_uid_t suid, SArray **childList);
static int32_t metaFetchTagIdxKey(SMeta *pMeta, const SMetaEntry *pEntry, const SSchema *pTagColumn,
STagIdxKey **ppTagIdxKey, int32_t *pTagIdxKeySize);
@@ -990,6 +992,20 @@ static int32_t metaTtlIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) {
return code;
}
+static void metaTimeSeriesNotifyCheck(SMeta *pMeta) {
+#if defined(TD_ENTERPRISE)
+ int64_t nTimeSeries = metaGetTimeSeriesNum(pMeta, 0);
+ int64_t deltaTS = nTimeSeries - pMeta->pVnode->config.vndStats.numOfReportedTimeSeries;
+ if (deltaTS > tsTimeSeriesThreshold) {
+ if (0 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 2)) {
+ if (tsem_post(&dmNotifyHdl.sem) != 0) {
+ metaError("vgId:%d, failed to post semaphore, errno:%d", TD_VID(pMeta->pVnode), errno);
+ }
+ }
+ }
+#endif
+}
+
static int32_t (*metaTableOpFn[META_TABLE_MAX][META_TABLE_OP_MAX])(SMeta *pMeta, const SMetaHandleParam *pParam) =
{
[META_ENTRY_TABLE] =
@@ -1139,6 +1155,7 @@ static int32_t metaHandleNormalTableCreate(SMeta *pMeta, const SMetaEntry *pEntr
metaError("vgId:%d, failed to create table:%s since %s", TD_VID(pMeta->pVnode), pEntry->name, tstrerror(rc));
}
}
+ metaTimeSeriesNotifyCheck(pMeta);
} else {
metaErr(TD_VID(pMeta->pVnode), code);
}
@@ -1214,7 +1231,7 @@ static int32_t metaHandleChildTableCreate(SMeta *pMeta, const SMetaEntry *pEntry
if (ret < 0) {
metaErr(TD_VID(pMeta->pVnode), ret);
}
- pMeta->pVnode->config.vndStats.numOfNTimeSeries += (nCols - 1);
+ pMeta->pVnode->config.vndStats.numOfTimeSeries += (nCols > 0 ? nCols - 1 : 0);
}
if (!TSDB_CACHE_NO(pMeta->pVnode->config)) {
@@ -1228,7 +1245,7 @@ static int32_t metaHandleChildTableCreate(SMeta *pMeta, const SMetaEntry *pEntry
} else {
metaErr(TD_VID(pMeta->pVnode), code);
}
-
+ metaTimeSeriesNotifyCheck(pMeta);
metaFetchEntryFree(&pSuperEntry);
return code;
}
@@ -1595,6 +1612,10 @@ static int32_t metaHandleSuperTableUpdateImpl(SMeta *pMeta, SMetaHandleParam *pP
}
}
+ if (TSDB_CODE_SUCCESS == code) {
+ metaUpdateStbStats(pMeta, pEntry->uid, 0, pEntry->stbEntry.schemaRow.nCols - pOldEntry->stbEntry.schemaRow.nCols);
+ }
+
return code;
}
@@ -1673,7 +1694,16 @@ static int32_t metaHandleSuperTableUpdate(SMeta *pMeta, const SMetaEntry *pEntry
tsdbCacheInvalidateSchema(pTsdb, pEntry->uid, -1, pEntry->stbEntry.schemaRow.version);
}
-
+ if (updStat) {
+ int64_t ctbNum = 0;
+ int32_t ret = metaGetStbStats(pMeta->pVnode, pEntry->uid, &ctbNum, NULL);
+ if (ret < 0) {
+ metaError("vgId:%d, failed to get stb stats:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pEntry->name,
+ pEntry->uid, tstrerror(ret));
+ }
+ pMeta->pVnode->config.vndStats.numOfTimeSeries += (ctbNum * deltaCol);
+ if (deltaCol > 0) metaTimeSeriesNotifyCheck(pMeta);
+ }
metaFetchEntryFree(&pOldEntry);
return code;
}
@@ -1772,7 +1802,9 @@ static int32_t metaHandleNormalTableUpdate(SMeta *pMeta, const SMetaEntry *pEntr
#endif
tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, pEntry->uid, pEntry->ntbEntry.schemaRow.version);
}
- metaTimeSeriesNotifyCheck(pMeta);
+ int32_t deltaCol = pEntry->ntbEntry.schemaRow.nCols - pOldEntry->ntbEntry.schemaRow.nCols;
+ pMeta->pVnode->config.vndStats.numOfNTimeSeries += deltaCol;
+ if (deltaCol > 0) metaTimeSeriesNotifyCheck(pMeta);
metaFetchEntryFree(&pOldEntry);
return code;
}
diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c
index b367232e2d..169adf219f 100644
--- a/source/dnode/vnode/src/meta/metaQuery.c
+++ b/source/dnode/vnode/src/meta/metaQuery.c
@@ -378,7 +378,7 @@ int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType) {
return 0;
}
-SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock, int64_t *createTime) {
+SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) {
void *pData = NULL;
int nData = 0;
int64_t version;
@@ -414,9 +414,6 @@ _query:
}
} else if (me.type == TSDB_CHILD_TABLE) {
uid = me.ctbEntry.suid;
- if (createTime != NULL){
- *createTime = me.ctbEntry.btime;
- }
tDecoderClear(&dc);
goto _query;
} else {
@@ -455,6 +452,46 @@ _err:
return NULL;
}
+int64_t metaGetTableCreateTime(SMeta *pMeta, tb_uid_t uid, int lock) {
+ void *pData = NULL;
+ int nData = 0;
+ int64_t version = 0;
+ SDecoder dc = {0};
+ int64_t createTime = INT64_MAX;
+ if (lock) {
+ metaRLock(pMeta);
+ }
+
+ if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pData, &nData) < 0) {
+ goto _exit;
+ }
+
+ version = ((SUidIdxVal *)pData)[0].version;
+
+ if (tdbTbGet(pMeta->pTbDb, &(STbDbKey){.uid = uid, .version = version}, sizeof(STbDbKey), &pData, &nData) != 0) {
+ goto _exit;
+ }
+
+ SMetaEntry me = {0};
+ tDecoderInit(&dc, pData, nData);
+ int32_t code = metaDecodeEntry(&dc, &me);
+ if (code) {
+ tDecoderClear(&dc);
+ goto _exit;
+ }
+ if (me.type == TSDB_CHILD_TABLE) {
+ createTime = me.ctbEntry.btime;
+ }
+ tDecoderClear(&dc);
+
+ _exit:
+ if (lock) {
+ metaULock(pMeta);
+ }
+ tdbFree(pData);
+ return createTime;
+}
+
SMCtbCursor *metaOpenCtbCursor(void *pVnode, tb_uid_t uid, int lock) {
SMeta *pMeta = ((SVnode *)pVnode)->pMeta;
SMCtbCursor *pCtbCur = NULL;
@@ -627,7 +664,7 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) {
STSchema *pTSchema = NULL;
SSchemaWrapper *pSW = NULL;
- pSW = metaGetTableSchema(pMeta, uid, sver, lock, NULL);
+ pSW = metaGetTableSchema(pMeta, uid, sver, lock);
if (!pSW) return NULL;
pTSchema = tBuildTSchema(pSW->pSchema, pSW->nCols, pSW->version);
diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c
index 7374b9ceb5..b227653e5e 100644
--- a/source/dnode/vnode/src/meta/metaSnapshot.c
+++ b/source/dnode/vnode/src/meta/metaSnapshot.c
@@ -548,7 +548,7 @@ int32_t setForSnapShot(SSnapContext* ctx, int64_t uid) {
void taosXSetTablePrimaryKey(SSnapContext* ctx, int64_t uid) {
bool ret = false;
- SSchemaWrapper* schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1, NULL);
+ SSchemaWrapper* schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1);
if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) {
ret = true;
}
diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index 2dbc89f58f..25c98d0e56 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -261,20 +261,6 @@ _exception:
return code;
}
-void metaTimeSeriesNotifyCheck(SMeta *pMeta) {
-#if defined(TD_ENTERPRISE)
- int64_t nTimeSeries = metaGetTimeSeriesNum(pMeta, 0);
- int64_t deltaTS = nTimeSeries - pMeta->pVnode->config.vndStats.numOfReportedTimeSeries;
- if (deltaTS > tsTimeSeriesThreshold) {
- if (0 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 2)) {
- if (tsem_post(&dmNotifyHdl.sem) != 0) {
- metaError("vgId:%d, failed to post semaphore, errno:%d", TD_VID(pMeta->pVnode), errno);
- }
- }
- }
-#endif
-}
-
static int32_t metaDropTables(SMeta *pMeta, SArray *tbUids) {
int32_t code = 0;
if (taosArrayGetSize(tbUids) == 0) return TSDB_CODE_SUCCESS;
diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c
index 6ff4cd6fdc..abab15ff58 100644
--- a/source/dnode/vnode/src/meta/metaTable2.c
+++ b/source/dnode/vnode/src/meta/metaTable2.c
@@ -378,10 +378,6 @@ static int32_t metaCreateChildTable(SMeta *pMeta, int64_t version, SVCreateTbReq
pReq->ctb.suid, version);
}
return code;
-
-#if 0
- metaTimeSeriesNotifyCheck(pMeta);
-#endif
}
// Drop Child Table
@@ -489,9 +485,6 @@ static int32_t metaCreateNormalTable(SMeta *pMeta, int64_t version, SVCreateTbRe
__func__, __FILE__, __LINE__, tstrerror(code), pReq->uid, pReq->name, version);
}
TAOS_RETURN(code);
-#if 0
- metaTimeSeriesNotifyCheck(pMeta);
-#endif
}
// Drop Normal Table
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 1c0f73249c..66e64b91d9 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -57,6 +57,7 @@ void tqDestroyTqHandle(void* data) {
if (pData->pRef) {
walCloseRef(pData->pRef->pWal, pData->pRef->refId);
}
+ taosHashCleanup(pData->tableCreateTimeHash);
}
static bool tqOffsetEqual(const STqOffset* pLeft, const STqOffset* pRight) {
@@ -211,7 +212,7 @@ void tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) {
tDeleteMqDataRsp(&dataRsp);
}
-int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t type,
+int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, SMqDataRsp* pRsp, int32_t type,
int32_t vgId) {
if (pHandle == NULL || pMsg == NULL || pReq == NULL || pRsp == NULL) {
return TSDB_CODE_INVALID_PARA;
@@ -414,7 +415,14 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
terrno = TSDB_CODE_INVALID_MSG;
goto END;
}
-
+ if (req.rawData == 1){
+ req.uidHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
+ if (req.uidHash == NULL) {
+ tqError("tq poll rawData taosHashInit failed");
+ code = terrno;
+ goto END;
+ }
+ }
int64_t consumerId = req.consumerId;
int32_t reqEpoch = req.epoch;
STqOffsetVal reqOffset = req.reqOffset;
diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c
index 580828b089..b84dcc4703 100644
--- a/source/dnode/vnode/src/tq/tqMeta.c
+++ b/source/dnode/vnode/src/tq/tqMeta.c
@@ -343,6 +343,7 @@ static int tqMetaInitHandle(STQ* pTq, STqHandle* handle) {
tqReaderSetTbUidList(handle->execHandle.pTqReader, tbUidList, NULL);
taosArrayDestroy(tbUidList);
}
+ handle->tableCreateTimeHash = (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
END:
return code;
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index 937ca80bcc..d650ae9751 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -283,7 +283,7 @@ void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid) {
return;
}
bool ret = false;
- SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1, NULL);
+ SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1);
if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) {
ret = true;
}
@@ -352,7 +352,7 @@ int32_t tqReaderSeek(STqReader* pReader, int64_t ver, const char* id) {
return TSDB_CODE_INVALID_PARA;
}
if (walReaderSeekVer(pReader->pWalReader, ver) < 0) {
- return -1;
+ return terrno;
}
tqDebug("wal reader seek to ver:%" PRId64 " %s", ver, id);
return 0;
@@ -485,14 +485,14 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) {
void* pBody = POINTER_SHIFT(pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg));
int32_t bodyLen = pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg);
int64_t ver = pWalReader->pHead->head.version;
- if (tqReaderSetSubmitMsg(pReader, pBody, bodyLen, ver) != 0) {
+ if (tqReaderSetSubmitMsg(pReader, pBody, bodyLen, ver, NULL) != 0) {
return false;
}
pReader->nextBlk = 0;
}
}
-int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) {
+int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver, SArray* rawList) {
if (pReader == NULL) {
return TSDB_CODE_INVALID_PARA;
}
@@ -500,11 +500,11 @@ int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, i
pReader->msg.msgLen = msgLen;
pReader->msg.ver = ver;
- tqDebug("tq reader set msg pointer:%p, msg len:%d", msgStr, msgLen);
+ tqTrace("tq reader set msg pointer:%p, msg len:%d", msgStr, msgLen);
SDecoder decoder = {0};
tDecoderInit(&decoder, pReader->msg.msgStr, pReader->msg.msgLen);
- int32_t code = tDecodeSubmitReq(&decoder, &pReader->submit);
+ int32_t code = tDecodeSubmitReq(&decoder, &pReader->submit, rawList);
tDecoderClear(&decoder);
if (code != 0) {
@@ -514,6 +514,13 @@ int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, i
return code;
}
+void tqReaderClearSubmitMsg(STqReader *pReader) {
+ tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE);
+ pReader->nextBlk = 0;
+ pReader->msg.msgStr = NULL;
+}
+
+
SWalReader* tqGetWalReader(STqReader* pReader) {
if (pReader == NULL) {
return NULL;
@@ -551,17 +558,15 @@ bool tqNextBlockImpl(STqReader* pReader, const char* idstr) {
void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t));
TSDB_CHECK_CONDITION(ret == NULL, code, lino, END, true);
- tqDebug("iterator data block in hash continue, progress:%d/%d, total queried tables:%d, uid:%"PRId64, pReader->nextBlk, blockSz, taosHashGetSize(pReader->tbIdHash), uid);
+ tqTrace("iterator data block in hash jump block, progress:%d/%d, uid:%" PRId64 "", pReader->nextBlk, blockSz, uid);
pReader->nextBlk++;
}
- tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE);
- pReader->nextBlk = 0;
- pReader->msg.msgStr = NULL;
- tqDebug("iterator data block end, block progress:%d/%d, uid:%"PRId64, pReader->nextBlk, blockSz, uid);
+ tqReaderClearSubmitMsg(pReader);
+ tqTrace("iterator data block end, total block num:%d, uid:%"PRId64, blockSz, uid);
END:
- tqDebug("%s:%d return:%s, uid:%"PRId64, __FUNCTION__, lino, code?"true":"false", uid);
+ tqTrace("%s:%d return:%s, uid:%"PRId64, __FUNCTION__, lino, code?"true":"false", uid);
return code;
}
@@ -581,17 +586,14 @@ bool tqNextDataBlockFilterOut(STqReader* pReader, SHashObj* filterOutUids) {
uid = pSubmitTbData->uid;
void* ret = taosHashGet(filterOutUids, &pSubmitTbData->uid, sizeof(int64_t));
TSDB_CHECK_NULL(ret, code, lino, END, true);
- tqDebug("iterator data block in hash continue, progress:%d/%d, uid:%" PRId64 "", pReader->nextBlk, blockSz, uid);
+ tqTrace("iterator data block in hash jump block, progress:%d/%d, uid:%" PRId64 "", pReader->nextBlk, blockSz, uid);
pReader->nextBlk++;
}
-
- tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE);
- pReader->nextBlk = 0;
- pReader->msg.msgStr = NULL;
- tqDebug("iterator data block end, block progress:%d/%d, uid:%"PRId64, pReader->nextBlk, blockSz, uid);
+ tqReaderClearSubmitMsg(pReader);
+ tqTrace("iterator data block end, total block num:%d, uid:%"PRId64, blockSz, uid);
END:
- tqDebug("%s:%d return:%s, uid:%"PRId64, __FUNCTION__, lino, code?"true":"false", uid);
+ tqTrace("%s:%d get data:%s, uid:%"PRId64, __FUNCTION__, lino, code?"true":"false", uid);
return code;
}
@@ -740,7 +742,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char*
(pReader->cachedSchemaVer != sversion)) {
tDeleteSchemaWrapper(pReader->pSchemaWrapper);
- pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1, NULL);
+ pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1);
if (pReader->pSchemaWrapper == NULL) {
tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", uid:%" PRId64
"version %d, possibly dropped table",
@@ -934,7 +936,7 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData
TQ_NULL_GO_TO_END(pCol);
int32_t numOfRows = pCol->nVal;
int32_t numOfCols = taosArrayGetSize(pCols);
- tqDebug("vgId:%d, tqProcessColData start, col num: %d, rows:%d", pReader->pWalReader->pWal->cfg.vgId, numOfCols, numOfRows);
+ tqTrace("vgId:%d, tqProcessColData start, col num: %d, rows:%d", pReader->pWalReader->pWal->cfg.vgId, numOfCols, numOfRows);
for (int32_t i = 0; i < numOfRows; i++) {
bool buildNew = false;
@@ -974,7 +976,7 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData
}
SSDataBlock* pLastBlock = taosArrayGetLast(blocks);
pLastBlock->info.rows = curRow - lastRow;
- tqDebug("vgId:%d, tqProcessColData end, col num: %d, rows:%d, block num:%d", pReader->pWalReader->pWal->cfg.vgId, numOfCols, numOfRows, (int)taosArrayGetSize(blocks));
+ tqTrace("vgId:%d, tqProcessColData end, col num: %d, rows:%d, block num:%d", pReader->pWalReader->pWal->cfg.vgId, numOfCols, numOfRows, (int)taosArrayGetSize(blocks));
END:
if (code != TSDB_CODE_SUCCESS) {
tqError("vgId:%d, process col data failed, code:%d", pReader->pWalReader->pWal->cfg.vgId, code);
@@ -997,7 +999,7 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra
int32_t numOfRows = taosArrayGetSize(pRows);
pTSchema = tBuildTSchema(pSchemaWrapper->pSchema, pSchemaWrapper->nCols, pSchemaWrapper->version);
TQ_NULL_GO_TO_END(pTSchema);
- tqDebug("vgId:%d, tqProcessRowData start, rows:%d", pReader->pWalReader->pWal->cfg.vgId, numOfRows);
+ tqTrace("vgId:%d, tqProcessRowData start, rows:%d", pReader->pWalReader->pWal->cfg.vgId, numOfRows);
for (int32_t i = 0; i < numOfRows; i++) {
bool buildNew = false;
@@ -1036,7 +1038,7 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra
SSDataBlock* pLastBlock = taosArrayGetLast(blocks);
pLastBlock->info.rows = curRow - lastRow;
- tqDebug("vgId:%d, tqProcessRowData end, rows:%d, block num:%d", pReader->pWalReader->pWal->cfg.vgId, numOfRows, (int)taosArrayGetSize(blocks));
+ tqTrace("vgId:%d, tqProcessRowData end, rows:%d, block num:%d", pReader->pWalReader->pWal->cfg.vgId, numOfRows, (int)taosArrayGetSize(blocks));
END:
if (code != TSDB_CODE_SUCCESS) {
tqError("vgId:%d, process row data failed, code:%d", pReader->pWalReader->pWal->cfg.vgId, code);
@@ -1046,8 +1048,46 @@ END:
return code;
}
-int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet, int64_t *createTime) {
- tqDebug("tq reader retrieve data block msg pointer:%p, index:%d", pReader->msg.msgStr, pReader->nextBlk);
+static int32_t buildCreateTbInfo(SMqDataRsp* pRsp, SVCreateTbReq* pCreateTbReq){
+ int32_t code = 0;
+ int32_t lino = 0;
+ void* createReq = NULL;
+ TSDB_CHECK_NULL(pRsp, code, lino, END, TSDB_CODE_INVALID_PARA);
+ TSDB_CHECK_NULL(pCreateTbReq, code, lino, END, TSDB_CODE_INVALID_PARA);
+
+ if (pRsp->createTableNum == 0) {
+ pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t));
+ TSDB_CHECK_NULL(pRsp->createTableLen, code, lino, END, terrno);
+ pRsp->createTableReq = taosArrayInit(0, sizeof(void*));
+ TSDB_CHECK_NULL(pRsp->createTableReq, code, lino, END, terrno);
+ }
+
+ uint32_t len = 0;
+ tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len, code);
+ TSDB_CHECK_CODE(code, lino, END);
+ createReq = taosMemoryCalloc(1, len);
+ TSDB_CHECK_NULL(createReq, code, lino, END, terrno);
+
+ SEncoder encoder = {0};
+ tEncoderInit(&encoder, createReq, len);
+ code = tEncodeSVCreateTbReq(&encoder, pCreateTbReq);
+ tEncoderClear(&encoder);
+ TSDB_CHECK_CODE(code, lino, END);
+ TSDB_CHECK_NULL(taosArrayPush(pRsp->createTableLen, &len), code, lino, END, terrno);
+ TSDB_CHECK_NULL(taosArrayPush(pRsp->createTableReq, &createReq), code, lino, END, terrno);
+ pRsp->createTableNum++;
+ tqTrace("build create table info msg success");
+
+ END:
+ if (code != 0){
+ tqError("%s failed at %d, failed to build create table info msg:%s", __FUNCTION__, lino, tstrerror(code));
+ taosMemoryFree(createReq);
+ }
+ return code;
+}
+
+int32_t tqRetrieveTaosxBlock(STqReader* pReader, SMqDataRsp* pRsp, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet, SArray* rawList, int8_t fetchMeta) {
+ tqTrace("tq reader retrieve data block msg pointer:%p, index:%d", pReader->msg.msgStr, pReader->nextBlk);
SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
if (pSubmitTbData == NULL) {
return terrno;
@@ -1063,7 +1103,7 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas
pReader->lastBlkUid = uid;
tDeleteSchemaWrapper(pReader->pSchemaWrapper);
- pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1, createTime);
+ pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1);
if (pReader->pSchemaWrapper == NULL) {
tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table",
pReader->pWalReader->pWal->cfg.vgId, uid, pReader->cachedSchemaVer);
@@ -1071,6 +1111,19 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas
return TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
}
+ if (pSubmitTbData->pCreateTbReq != NULL) {
+ int32_t code = buildCreateTbInfo(pRsp, pSubmitTbData->pCreateTbReq);
+ if (code != 0) {
+ return code;
+ }
+ } else if (rawList != NULL) {
+ if (taosArrayPush(schemas, &pReader->pSchemaWrapper) == NULL){
+ return terrno;
+ }
+ pReader->pSchemaWrapper = NULL;
+ return 0;
+ }
+
if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {
return tqProcessColData(pReader, pSubmitTbData, blocks, schemas);
} else {
diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c
index a65b118aea..70a165906e 100644
--- a/source/dnode/vnode/src/tq/tqScan.c
+++ b/source/dnode/vnode/src/tq/tqScan.c
@@ -14,6 +14,33 @@
*/
#include "tq.h"
+static int32_t tqAddRawDataToRsp(const void* rawData, SMqDataRsp* pRsp, int8_t precision) {
+ int32_t code = TDB_CODE_SUCCESS;
+ int32_t lino = 0;
+ void* buf = NULL;
+
+ int32_t dataStrLen = sizeof(SRetrieveTableRspForTmq) + *(uint32_t *)rawData + INT_BYTES;
+ buf = taosMemoryCalloc(1, dataStrLen);
+ TSDB_CHECK_NULL(buf, code, lino, END, terrno);
+
+ SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)buf;
+ pRetrieve->version = RETRIEVE_TABLE_RSP_TMQ_RAW_VERSION;
+ pRetrieve->precision = precision;
+ pRetrieve->compressed = 0;
+
+ memcpy(pRetrieve->data, rawData, *(uint32_t *)rawData + INT_BYTES);
+ TSDB_CHECK_NULL(taosArrayPush(pRsp->blockDataLen, &dataStrLen), code, lino, END, terrno);
+ TSDB_CHECK_NULL(taosArrayPush(pRsp->blockData, &buf), code, lino, END, terrno);
+ pRsp->blockDataElementFree = true;
+
+ tqTrace("tqAddRawDataToRsp add block data to block array, blockDataLen:%d, blockData:%p", dataStrLen, buf);
+ END:
+ if (code != TSDB_CODE_SUCCESS) {
+ taosMemoryFree(buf);
+ tqError("%s failed at %d, failed to add block data to response:%s", __FUNCTION__, lino, tstrerror(code));
+ }
+ return code;
+}
static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision) {
int32_t code = 0;
@@ -25,7 +52,7 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp,
TSDB_CHECK_NULL(buf, code, lino, END, terrno);
SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)buf;
- pRetrieve->version = 1;
+ pRetrieve->version = RETRIEVE_TABLE_RSP_TMQ_VERSION;
pRetrieve->precision = precision;
pRetrieve->compressed = 0;
pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows);
@@ -36,13 +63,14 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp,
actualLen += sizeof(SRetrieveTableRspForTmq);
TSDB_CHECK_NULL(taosArrayPush(pRsp->blockDataLen, &actualLen), code, lino, END, terrno);
TSDB_CHECK_NULL(taosArrayPush(pRsp->blockData, &buf), code, lino, END, terrno);
+ pRsp->blockDataElementFree = true;
+ tqTrace("tqAddBlockDataToRsp add block data to block array, blockDataLen:%d, blockData:%p", dataStrLen, buf);
- buf = NULL;
END:
- if (code != 0){
+ if (code != TSDB_CODE_SUCCESS){
+ taosMemoryFree(buf);
tqError("%s failed at line %d with msg:%s", __func__, lino, tstrerror(code));
}
- taosMemoryFree(buf);
return code;
}
@@ -66,7 +94,7 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, i
tqError("failed to push tbName to blockTbName:%s, uid:%"PRId64, tbName, uid);
continue;
}
- tqDebug("add tbName to response success tbname:%s, uid:%"PRId64, tbName, uid);
+ tqTrace("add tbName to response success tbname:%s, uid:%"PRId64, tbName, uid);
}
END:
@@ -190,7 +218,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal*
pRsp->blockNum++;
totalRows += pDataBlock->info.rows;
- if (totalRows >= tmqRowSize || (taosGetTimestampMs() - st > TMIN(TQ_POLL_MAX_TIME, pRequest->timeout))) {
+ if (totalRows >= pRequest->minPollRows || (taosGetTimestampMs() - st > pRequest->timeout)) {
break;
}
}
@@ -205,7 +233,7 @@ END:
return code;
}
-int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBatchMetaRsp* pBatchMetaRsp, STqOffsetVal* pOffset, int64_t timeout) {
+int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBatchMetaRsp* pBatchMetaRsp, STqOffsetVal* pOffset, const SMqPollReq* pRequest) {
int32_t code = 0;
int32_t lino = 0;
char* tbName = NULL;
@@ -234,7 +262,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBat
tbName = NULL;
}
if (pRsp->withSchema) {
- SSchemaWrapper* pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task));
+ pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task));
TSDB_CHECK_NULL(pSW, code, lino, END, terrno);
TSDB_CHECK_NULL(taosArrayPush(pRsp->blockSchema, &pSW), code, lino, END, terrno);
pSW = NULL;
@@ -246,7 +274,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBat
pRsp->blockNum++;
rowCnt += pDataBlock->info.rows;
- if (rowCnt <= tmqRowSize && (taosGetTimestampMs() - st <= TMIN(TQ_POLL_MAX_TIME, timeout))) {
+ if (rowCnt <= pRequest->minPollRows && (taosGetTimestampMs() - st <= pRequest->timeout)) {
continue;
}
}
@@ -290,45 +318,8 @@ END:
return code;
}
-static int32_t buildCreateTbInfo(SMqDataRsp* pRsp, SVCreateTbReq* pCreateTbReq){
- int32_t code = 0;
- int32_t lino = 0;
- void* createReq = NULL;
- TSDB_CHECK_NULL(pRsp, code, lino, END, TSDB_CODE_INVALID_PARA);
- TSDB_CHECK_NULL(pCreateTbReq, code, lino, END, TSDB_CODE_INVALID_PARA);
-
- if (pRsp->createTableNum == 0) {
- pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t));
- TSDB_CHECK_NULL(pRsp->createTableLen, code, lino, END, terrno);
- pRsp->createTableReq = taosArrayInit(0, sizeof(void*));
- TSDB_CHECK_NULL(pRsp->createTableReq, code, lino, END, terrno);
- }
-
- uint32_t len = 0;
- tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len, code);
- TSDB_CHECK_CODE(code, lino, END);
- createReq = taosMemoryCalloc(1, len);
- TSDB_CHECK_NULL(createReq, code, lino, END, terrno);
-
- SEncoder encoder = {0};
- tEncoderInit(&encoder, createReq, len);
- code = tEncodeSVCreateTbReq(&encoder, pCreateTbReq);
- tEncoderClear(&encoder);
- TSDB_CHECK_CODE(code, lino, END);
- TSDB_CHECK_NULL(taosArrayPush(pRsp->createTableLen, &len), code, lino, END, terrno);
- TSDB_CHECK_NULL(taosArrayPush(pRsp->createTableReq, &createReq), code, lino, END, terrno);
- pRsp->createTableNum++;
- tqDebug("build create table info msg success");
-
-END:
- if (code != 0){
- tqError("%s failed at %d, failed to build create table info msg:%s", __FUNCTION__, lino, tstrerror(code));
- taosMemoryFree(createReq);
- }
- return code;
-}
-
-static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded){
+static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int32_t* totalRows,
+ const SMqPollReq* pRequest, SArray* rawList){
int32_t code = 0;
int32_t lino = 0;
SArray* pBlocks = NULL;
@@ -342,78 +333,161 @@ static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int
pSchemas = taosArrayInit(0, sizeof(void*));
TSDB_CHECK_NULL(pSchemas, code, lino, END, terrno);
- SSubmitTbData* pSubmitTbDataRet = NULL;
- int64_t createTime = INT64_MAX;
- code = tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas, &pSubmitTbDataRet, &createTime);
+ SSubmitTbData* pSubmitTbData = NULL;
+ code = tqRetrieveTaosxBlock(pReader, pRsp, pBlocks, pSchemas, &pSubmitTbData, rawList, pHandle->fetchMeta);
TSDB_CHECK_CODE(code, lino, END);
- bool tmp = (pSubmitTbDataRet->flags & sourceExcluded) != 0;
+ bool tmp = (pSubmitTbData->flags & pRequest->sourceExcluded) != 0;
TSDB_CHECK_CONDITION(!tmp, code, lino, END, TSDB_CODE_SUCCESS);
+
+
+ int32_t blockNum = taosArrayGetSize(pBlocks) == 0 ? 1 : taosArrayGetSize(pBlocks);
if (pRsp->withTbName) {
int64_t uid = pExec->pTqReader->lastBlkUid;
- code = tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks));
+ code = tqAddTbNameToRsp(pTq, uid, pRsp, blockNum);
TSDB_CHECK_CODE(code, lino, END);
}
- if (pHandle->fetchMeta != WITH_DATA && pSubmitTbDataRet->pCreateTbReq != NULL) {
- if (pSubmitTbDataRet->ctimeMs - createTime <= 1000) { // judge if table is already created to avoid sending crateTbReq
- code = buildCreateTbInfo(pRsp, pSubmitTbDataRet->pCreateTbReq);
- TSDB_CHECK_CODE(code, lino, END);
- }
- }
- tmp = (pHandle->fetchMeta == ONLY_META && pSubmitTbDataRet->pCreateTbReq == NULL);
+
+ tmp = (pHandle->fetchMeta == ONLY_META && pSubmitTbData->pCreateTbReq == NULL);
TSDB_CHECK_CONDITION(!tmp, code, lino, END, TSDB_CODE_SUCCESS);
- for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) {
- SSDataBlock* pBlock = taosArrayGet(pBlocks, i);
- if (pBlock == NULL) {
- continue;
+ for (int32_t i = 0; i < blockNum; i++) {
+ if (taosArrayGetSize(pBlocks) == 0){
+ void* rawData = taosArrayGetP(rawList, pReader->nextBlk - 1);
+ if (rawData == NULL) {
+ continue;
+ }
+ if (tqAddRawDataToRsp(rawData, pRsp, pTq->pVnode->config.tsdbCfg.precision) != 0){
+ tqError("vgId:%d, failed to add block to rsp msg", pTq->pVnode->config.vgId);
+ continue;
+ }
+ *totalRows += *(uint32_t *)rawData + INT_BYTES; // bytes actually
+ } else {
+ SSDataBlock* pBlock = taosArrayGet(pBlocks, i);
+ if (pBlock == NULL) {
+ continue;
+ }
+
+ if (tqAddBlockDataToRsp(pBlock, pRsp, taosArrayGetSize(pBlock->pDataBlock), pTq->pVnode->config.tsdbCfg.precision) != 0){
+ tqError("vgId:%d, failed to add block to rsp msg", pTq->pVnode->config.vgId);
+ continue;
+ }
+ *totalRows += pBlock->info.rows;
}
- if (tqAddBlockDataToRsp(pBlock, pRsp, taosArrayGetSize(pBlock->pDataBlock), pTq->pVnode->config.tsdbCfg.precision) != 0){
- tqError("vgId:%d, failed to add block to rsp msg", pTq->pVnode->config.vgId);
- continue;
- }
- *totalRows += pBlock->info.rows;
- blockDataFreeRes(pBlock);
- SSchemaWrapper* pSW = taosArrayGetP(pSchemas, i);
- if (taosArrayPush(pRsp->blockSchema, &pSW) == NULL){
+
+ void** pSW = taosArrayGet(pSchemas, i);
+ if (taosArrayPush(pRsp->blockSchema, pSW) == NULL){
tqError("vgId:%d, failed to add schema to rsp msg", pTq->pVnode->config.vgId);
continue;
}
+ *pSW = NULL;
pRsp->blockNum++;
}
- tqDebug("vgId:%d, process sub data success, response blocknum:%d, rows:%d", pTq->pVnode->config.vgId, pRsp->blockNum, *totalRows);
+ tqTrace("vgId:%d, process sub data success, response blocknum:%d, rows:%d", pTq->pVnode->config.vgId, pRsp->blockNum, *totalRows);
END:
- if (code != 0){
+ if (code != 0) {
tqError("%s failed at %d, failed to process sub data:%s", __FUNCTION__, lino, tstrerror(code));
- taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes);
- taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper);
- } else {
- taosArrayDestroy(pBlocks);
- taosArrayDestroy(pSchemas);
+ }
+ taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes);
+ taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper);
+}
+
+static void preProcessSubmitMsg(STqHandle* pHandle, const SMqPollReq* pRequest, SArray** rawList){
+ STqExecHandle* pExec = &pHandle->execHandle;
+ STqReader* pReader = pExec->pTqReader;
+ int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData);
+ for (int32_t i = 0; i < blockSz; i++){
+ SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, i);
+ if (pSubmitTbData== NULL){
+ taosArrayDestroy(*rawList);
+ *rawList = NULL;
+ return;
+ }
+
+ int64_t uid = pSubmitTbData->uid;
+ if (taosHashGet(pRequest->uidHash, &uid, LONG_BYTES) != NULL) {
+ tqDebug("poll rawdata split,uid:%" PRId64 " is already exists", uid);
+ terrno = TSDB_CODE_TMQ_RAW_DATA_SPLIT;
+ return;
+ } else {
+ int32_t code = taosHashPut(pRequest->uidHash, &uid, LONG_BYTES, &uid, LONG_BYTES);
+ if (code != 0){
+ tqError("failed to add table uid to hash, code:%d, uid:%"PRId64, code, uid);
+ }
+ }
+
+ if (pSubmitTbData->pCreateTbReq == NULL){
+ continue;
+ }
+
+ int64_t createTime = INT64_MAX;
+ int64_t *cTime = (int64_t*)taosHashGet(pHandle->tableCreateTimeHash, &uid, LONG_BYTES);
+ if (cTime != NULL){
+ createTime = *cTime;
+ } else{
+ createTime = metaGetTableCreateTime(pReader->pVnodeMeta, uid, 1);
+ if (createTime != INT64_MAX){
+ int32_t code = taosHashPut(pHandle->tableCreateTimeHash, &uid, LONG_BYTES, &createTime, LONG_BYTES);
+ if (code != 0){
+ tqError("failed to add table create time to hash,code:%d, uid:%"PRId64, code, uid);
+ }
+ }
+ }
+ if (pHandle->fetchMeta == WITH_DATA || pSubmitTbData->ctimeMs > createTime){
+ tDestroySVCreateTbReq(pSubmitTbData->pCreateTbReq, TSDB_MSG_FLG_DECODE);
+ pSubmitTbData->pCreateTbReq = NULL;
+ } else{
+ taosArrayDestroy(*rawList);
+ *rawList = NULL;
+ }
}
}
-int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded) {
+int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqDataRsp* pRsp, int32_t* totalRows, const SMqPollReq* pRequest) {
int32_t code = 0;
int32_t lino = 0;
- TSDB_CHECK_NULL(pRsp, code, lino, END, TSDB_CODE_INVALID_PARA);
- TSDB_CHECK_NULL(pTq, code, lino, END, TSDB_CODE_INVALID_PARA);
- TSDB_CHECK_NULL(pHandle, code, lino, END, TSDB_CODE_INVALID_PARA);
- TSDB_CHECK_NULL(totalRows, code, lino, END, TSDB_CODE_INVALID_PARA);
STqExecHandle* pExec = &pHandle->execHandle;
STqReader* pReader = pExec->pTqReader;
- code = tqReaderSetSubmitMsg(pReader, submit.msgStr, submit.msgLen, submit.ver);
+ SArray *rawList = NULL;
+ if (pRequest->rawData){
+ rawList = taosArrayInit(0, POINTER_BYTES);
+ TSDB_CHECK_NULL(rawList, code, lino, END, terrno);
+ }
+ code = tqReaderSetSubmitMsg(pReader, submit.msgStr, submit.msgLen, submit.ver, rawList);
TSDB_CHECK_CODE(code, lino, END);
+ if (pRequest->rawData) {
+ preProcessSubmitMsg(pHandle, pRequest, &rawList);
+ }
+ // data could not contains same uid data in rawdata mode
+ if (pRequest->rawData != 0 && terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT){
+ goto END;
+ }
+
+ // this submit data is metadata and previous data is rawdata
+ if (pRequest->rawData != 0 && *totalRows > 0 && pRsp->createTableNum == 0 && rawList == NULL){
+ tqDebug("poll rawdata split,vgId:%d, this wal submit data contains metadata and previous data is data", pTq->pVnode->config.vgId);
+ terrno = TSDB_CODE_TMQ_RAW_DATA_SPLIT;
+ goto END;
+ }
+
+ // this submit data is rawdata and previous data is metadata
+ if (pRequest->rawData != 0 && pRsp->createTableNum > 0 && rawList != NULL){
+ tqDebug("poll rawdata split,vgId:%d, this wal submit data is data and previous data is metadata", pTq->pVnode->config.vgId);
+ terrno = TSDB_CODE_TMQ_RAW_DATA_SPLIT;
+ goto END;
+ }
if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
while (tqNextBlockImpl(pReader, NULL)) {
- tqProcessSubData(pTq, pHandle, pRsp, totalRows, sourceExcluded);
+ tqProcessSubData(pTq, pHandle, pRsp, totalRows, pRequest, rawList);
}
} else if (pExec->subType == TOPIC_SUB_TYPE__DB) {
while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) {
- tqProcessSubData(pTq, pHandle, pRsp, totalRows, sourceExcluded);
+ tqProcessSubData(pTq, pHandle, pRsp, totalRows, pRequest, rawList);
}
}
END:
+ tqReaderClearSubmitMsg(pReader);
+ taosArrayDestroy(rawList);
if (code != 0){
tqError("%s failed at %d, failed to scan log:%s", __FUNCTION__, lino, tstrerror(code));
}
diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c
index 98ea92125c..7e0b118474 100644
--- a/source/dnode/vnode/src/tq/tqSink.c
+++ b/source/dnode/vnode/src/tq/tqSink.c
@@ -420,7 +420,7 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
reqs.nReqs = taosArrayGetSize(reqs.pArray);
code = tqPutReqToQueue(pVnode, &reqs, encodeCreateChildTableForRPC, TDMT_VND_CREATE_TABLE);
if (code != TSDB_CODE_SUCCESS) {
- tqError("s-task:%s failed to send create table msg", id);
+ tqError("s-task:%s failed to send create table msg, code:%s", id, tstrerror(code));
}
_end:
@@ -859,6 +859,8 @@ int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkI
int32_t vgId = TD_VID(pVnode);
int64_t suid = pTask->outputInfo.tbSink.stbUid;
const char* id = pTask->id.idStr;
+ int32_t timeout = 300; // 5min
+ int64_t start = taosGetTimestampSec();
while (pTableSinkInfo->uid == 0) {
if (streamTaskShouldStop(pTask)) {
@@ -866,6 +868,12 @@ int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkI
return TSDB_CODE_STREAM_EXEC_CANCELLED;
}
+ int64_t waitingDuration = taosGetTimestampSec() - start;
+ if (waitingDuration > timeout) {
+ tqError("s-task:%s wait for table-creating:%s more than %dsec, failed", id, dstTableName, timeout);
+ return TSDB_CODE_PAR_TABLE_NOT_EXIST;
+ }
+
// wait for the table to be created
SMetaReader mr = {0};
metaReaderDoInit(&mr, pVnode->pMeta, META_READER_LOCK);
diff --git a/source/dnode/vnode/src/tq/tqStreamNotify.c b/source/dnode/vnode/src/tq/tqStreamNotify.c
index 46ee95d3b9..1c64c404e6 100644
--- a/source/dnode/vnode/src/tq/tqStreamNotify.c
+++ b/source/dnode/vnode/src/tq/tqStreamNotify.c
@@ -20,14 +20,13 @@
#include "curl/curl.h"
#endif
-#define STREAM_EVENT_NOTIFY_RETRY_MS 50 // 50ms
-
+#define STREAM_EVENT_NOTIFY_RETRY_MS 50 // 50 ms
typedef struct SStreamNotifyHandle {
TdThreadMutex mutex;
#ifndef WINDOWS
- CURL* curl;
+ CURL* curl;
#endif
- char* url;
+ char* url;
} SStreamNotifyHandle;
struct SStreamNotifyHandleMap {
@@ -49,6 +48,7 @@ static void stopStreamNotifyConn(SStreamNotifyHandle* pHandle) {
}
// TODO: add wait mechanism for peer connection close response
curl_easy_cleanup(pHandle->curl);
+ pHandle->curl = NULL;
#endif
}
@@ -258,7 +258,8 @@ _end:
}
static int32_t packupStreamNotifyEvent(const char* streamName, const SArray* pBlocks, char** pMsg,
- int32_t* nNotifyEvents) {
+ int32_t* nNotifyEvents, STaskNotifyEventStat* pNotifyEventStat,
+ int32_t* pBlockIdx) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t numOfBlocks = 0;
@@ -268,15 +269,20 @@ static int32_t packupStreamNotifyEvent(const char* streamName, const SArray* pBl
char* msgHeader = NULL;
const char* msgTail = "]}]}";
char* msg = NULL;
+ int64_t startTime = 0;
+ int64_t endTime = 0;
+ int32_t nBlocks = 0;
TSDB_CHECK_NULL(pMsg, code, lino, _end, TSDB_CODE_INVALID_PARA);
+ TSDB_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
*pMsg = NULL;
numOfBlocks = taosArrayGetSize(pBlocks);
*nNotifyEvents = 0;
- for (int32_t i = 0; i < numOfBlocks; ++i) {
+ for (int32_t i = *pBlockIdx; i < numOfBlocks; ++i) {
SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
+ nBlocks++;
if (pDataBlock == NULL || pDataBlock->info.type != STREAM_NOTIFY_EVENT) {
continue;
}
@@ -287,13 +293,19 @@ static int32_t packupStreamNotifyEvent(const char* streamName, const SArray* pBl
msgLen += varDataLen(val) + 1;
}
*nNotifyEvents += pDataBlock->info.rows;
+ if (msgLen >= tsStreamNotifyMessageSize * 1024) {
+ break;
+ }
}
+ *pBlockIdx += nBlocks;
+
if (msgLen == 0) {
// skip since no notification events found
goto _end;
}
+ startTime = taosGetMonoTimestampMs();
code = getStreamNotifyEventHeader(streamName, &msgHeader);
TSDB_CHECK_CODE(code, lino, _end);
msgHeaderLen = strlen(msgHeader);
@@ -306,7 +318,7 @@ static int32_t packupStreamNotifyEvent(const char* streamName, const SArray* pBl
TAOS_STRNCPY(p, msgHeader, msgHeaderLen);
p += msgHeaderLen - msgTailLen;
- for (int32_t i = 0; i < numOfBlocks; ++i) {
+ for (int32_t i = *pBlockIdx - nBlocks; i < *pBlockIdx; ++i) {
SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
if (pDataBlock == NULL || pDataBlock->info.type != STREAM_NOTIFY_EVENT) {
continue;
@@ -328,6 +340,11 @@ static int32_t packupStreamNotifyEvent(const char* streamName, const SArray* pBl
*pMsg = msg;
msg = NULL;
+ endTime = taosGetMonoTimestampMs();
+ pNotifyEventStat->notifyEventPackTimes++;
+ pNotifyEventStat->notifyEventPackElems += *nNotifyEvents;
+ pNotifyEventStat->notifyEventPackCostSec += (endTime - startTime) / 1000.0;
+
_end:
if (code != TSDB_CODE_SUCCESS) {
tqError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
@@ -354,9 +371,21 @@ static int32_t sendSingleStreamNotify(SStreamNotifyHandle* pHandle, char* msg) {
TSDB_CHECK_NULL(pHandle->curl, code, lino, _end, TSDB_CODE_INVALID_PARA);
totalLen = strlen(msg);
- while (sentLen < totalLen) {
- res = curl_ws_send(pHandle->curl, msg + sentLen, totalLen - sentLen, &nbytes, 0, CURLWS_TEXT);
+ if (totalLen > 0) {
+ // send PING frame to check if the connection is still alive
+ res = curl_ws_send(pHandle->curl, "", 0, (size_t*)&sentLen, 0, CURLWS_PING);
TSDB_CHECK_CONDITION(res == CURLE_OK, code, lino, _end, TSDB_CODE_FAILED);
+ }
+ sentLen = 0;
+ while (sentLen < totalLen) {
+ size_t chunkSize = TMIN(totalLen - sentLen, tsStreamNotifyFrameSize * 1024);
+ if (sentLen == 0) {
+ res = curl_ws_send(pHandle->curl, msg, chunkSize, &nbytes, totalLen, CURLWS_TEXT | CURLWS_OFFSET);
+ TSDB_CHECK_CONDITION(res == CURLE_OK, code, lino, _end, TSDB_CODE_FAILED);
+ } else {
+ res = curl_ws_send(pHandle->curl, msg + sentLen, chunkSize, &nbytes, 0, CURLWS_TEXT | CURLWS_OFFSET);
+ TSDB_CHECK_CONDITION(res == CURLE_OK, code, lino, _end, TSDB_CODE_FAILED);
+ }
sentLen += nbytes;
}
@@ -379,6 +408,9 @@ int32_t tqSendAllNotifyEvents(const SArray* pBlocks, SStreamTask* pTask, SVnode*
int32_t nNotifyAddr = 0;
int32_t nNotifyEvents = 0;
SStreamNotifyHandle* pHandle = NULL;
+ int64_t startTime = 0;
+ int64_t endTime = 0;
+ int32_t blockIdx = 0;
TSDB_CHECK_NULL(pTask, code, lino, _end, TSDB_CODE_INVALID_PARA);
TSDB_CHECK_NULL(pVnode, code, lino, _end, TSDB_CODE_INVALID_PARA);
@@ -388,50 +420,61 @@ int32_t tqSendAllNotifyEvents(const SArray* pBlocks, SStreamTask* pTask, SVnode*
goto _end;
}
- code = packupStreamNotifyEvent(pTask->notifyInfo.streamName, pBlocks, &msg, &nNotifyEvents);
- TSDB_CHECK_CODE(code, lino, _end);
- if (msg == NULL) {
- goto _end;
- }
+ while (blockIdx < taosArrayGetSize(pBlocks)) {
+ code = packupStreamNotifyEvent(pTask->notifyInfo.streamName, pBlocks, &msg, &nNotifyEvents, &pTask->notifyEventStat,
+ &blockIdx);
+ TSDB_CHECK_CODE(code, lino, _end);
+ if (msg == NULL) {
+ continue;
+ }
- tqDebug("stream task %s prepare to send %d notify events, total msg length: %" PRIu64, pTask->notifyInfo.streamName,
- nNotifyEvents, (uint64_t)strlen(msg));
+ tqDebug("stream task %s prepare to send %d notify events, total msg length: %" PRIu64, pTask->notifyInfo.streamName,
+ nNotifyEvents, (uint64_t)strlen(msg));
- for (int32_t i = 0; i < nNotifyAddr; ++i) {
- if (streamTaskShouldStop(pTask)) {
- break;
- }
- const char* url = taosArrayGetP(pTask->notifyInfo.pNotifyAddrUrls, i);
- code = acquireStreamNotifyHandle(pVnode->pNotifyHandleMap, url, &pHandle);
- if (code != TSDB_CODE_SUCCESS) {
- tqError("failed to get stream notify handle of %s", url);
- if (pTask->notifyInfo.notifyErrorHandle == SNOTIFY_ERROR_HANDLE_PAUSE) {
- // retry for event message sending in PAUSE error handling mode
- taosMsleep(STREAM_EVENT_NOTIFY_RETRY_MS);
- --i;
- continue;
- } else {
- // simply ignore the failure in DROP error handling mode
- code = TSDB_CODE_SUCCESS;
- continue;
+ startTime = taosGetMonoTimestampMs();
+ for (int32_t i = 0; i < nNotifyAddr; ++i) {
+ if (streamTaskShouldStop(pTask)) {
+ break;
}
- }
- code = sendSingleStreamNotify(pHandle, msg);
- if (code != TSDB_CODE_SUCCESS) {
- tqError("failed to send stream notify handle to %s since %s", url, tstrerror(code));
- if (pTask->notifyInfo.notifyErrorHandle == SNOTIFY_ERROR_HANDLE_PAUSE) {
- // retry for event message sending in PAUSE error handling mode
- taosMsleep(STREAM_EVENT_NOTIFY_RETRY_MS);
- --i;
- } else {
- // simply ignore the failure in DROP error handling mode
- code = TSDB_CODE_SUCCESS;
+ const char* url = taosArrayGetP(pTask->notifyInfo.pNotifyAddrUrls, i);
+ code = acquireStreamNotifyHandle(pVnode->pNotifyHandleMap, url, &pHandle);
+ if (code != TSDB_CODE_SUCCESS) {
+ tqError("failed to get stream notify handle of %s", url);
+ if (pTask->notifyInfo.notifyErrorHandle == SNOTIFY_ERROR_HANDLE_PAUSE) {
+ // retry for event message sending in PAUSE error handling mode
+ taosMsleep(STREAM_EVENT_NOTIFY_RETRY_MS);
+ --i;
+ continue;
+ } else {
+ // simply ignore the failure in DROP error handling mode
+ code = TSDB_CODE_SUCCESS;
+ continue;
+ }
}
- } else {
- tqDebug("stream task %s send %d notify events to %s successfully", pTask->notifyInfo.streamName, nNotifyEvents,
- url);
+ code = sendSingleStreamNotify(pHandle, msg);
+ if (code != TSDB_CODE_SUCCESS) {
+ tqError("failed to send stream notify handle to %s since %s", url, tstrerror(code));
+ if (pTask->notifyInfo.notifyErrorHandle == SNOTIFY_ERROR_HANDLE_PAUSE) {
+ // retry for event message sending in PAUSE error handling mode
+ taosMsleep(STREAM_EVENT_NOTIFY_RETRY_MS);
+ --i;
+ } else {
+ // simply ignore the failure in DROP error handling mode
+ code = TSDB_CODE_SUCCESS;
+ }
+ } else {
+ tqDebug("stream task %s send %d notify events to %s successfully", pTask->notifyInfo.streamName, nNotifyEvents,
+ url);
+ }
+ releaseStreamNotifyHandle(&pHandle);
}
- releaseStreamNotifyHandle(&pHandle);
+
+ endTime = taosGetMonoTimestampMs();
+ pTask->notifyEventStat.notifyEventSendTimes++;
+ pTask->notifyEventStat.notifyEventSendElems += nNotifyEvents;
+ pTask->notifyEventStat.notifyEventSendCostSec += (endTime - startTime) / 1000.0;
+
+ taosMemoryFreeClear(msg);
}
_end:
diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c
index 197a45cdb9..9866528446 100644
--- a/source/dnode/vnode/src/tq/tqUtil.c
+++ b/source/dnode/vnode/src/tq/tqUtil.c
@@ -131,7 +131,7 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
}
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId,
pHandle->subKey, vgId, dataRsp.rspOffset.version);
- code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
+ code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, (pRequest->rawData == 1) ? TMQ_MSG_TYPE__POLL_RAW_DATA_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
tDeleteMqDataRsp(&dataRsp);
*pBlockReturned = true;
@@ -222,6 +222,10 @@ end:
static void tDeleteCommon(void* parm) {}
+#define POLL_RSP_TYPE(pRequest,taosxRsp) \
+taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : \
+(pRequest->rawData == 1 ? TMQ_MSG_TYPE__POLL_RAW_DATA_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP)
+
static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
SRpcMsg* pMsg, STqOffsetVal* offset) {
int32_t vgId = TD_VID(pTq->pVnode);
@@ -231,7 +235,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
TQ_ERR_GO_TO_END(tqInitTaosxRsp(&taosxRsp, *offset));
if (offset->type != TMQ_OFFSET__LOG) {
- TQ_ERR_GO_TO_END(tqScanTaosx(pTq, pHandle, &taosxRsp, &btMetaRsp, offset, pRequest->timeout));
+ TQ_ERR_GO_TO_END(tqScanTaosx(pTq, pHandle, &taosxRsp, &btMetaRsp, offset, pRequest));
if (taosArrayGetSize(btMetaRsp.batchMetaReq) > 0) {
code = tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId);
@@ -274,7 +278,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
}
tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp,
- taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
+ POLL_RSP_TYPE(pRequest, taosxRsp), vgId);
goto END;
}
@@ -287,7 +291,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
if (totalRows > 0) {
tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp,
- taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
+ POLL_RSP_TYPE(pRequest, taosxRsp), vgId);
goto END;
}
@@ -349,7 +353,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
TQ_NULL_GO_TO_END (taosArrayPush(btMetaRsp.batchMetaReq, &tBuf));
TQ_NULL_GO_TO_END (taosArrayPush(btMetaRsp.batchMetaLen, &tLen));
totalMetaRows++;
- if ((taosArrayGetSize(btMetaRsp.batchMetaReq) >= tmqRowSize) || (taosGetTimestampMs() - st > TMIN(TQ_POLL_MAX_TIME, pRequest->timeout))) {
+ if ((taosArrayGetSize(btMetaRsp.batchMetaReq) >= tmqRowSize) || (taosGetTimestampMs() - st > pRequest->timeout)) {
tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer);
code = tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId);
goto END;
@@ -370,12 +374,18 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
.ver = pHead->version,
};
- TQ_ERR_GO_TO_END(tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows, pRequest->sourceExcluded));
+ TQ_ERR_GO_TO_END(tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows, pRequest));
- if (totalRows >= tmqRowSize || (taosGetTimestampMs() - st > TMIN(TQ_POLL_MAX_TIME, pRequest->timeout))) {
- tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer + 1);
+ if ((pRequest->rawData == 0 && totalRows >= pRequest->minPollRows) ||
+ (taosGetTimestampMs() - st > pRequest->timeout) ||
+ (pRequest->rawData != 0 && (taosArrayGetSize(taosxRsp.blockData) > pRequest->minPollRows ||
+ terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT))) {
+// tqDebug("start to send rsp, block num:%d, totalRows:%d, createTableNum:%d, terrno:%d",
+// (int)taosArrayGetSize(taosxRsp.blockData), totalRows, taosxRsp.createTableNum, terrno);
+ tqOffsetResetToLog(&taosxRsp.rspOffset, terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT ? fetchVer : fetchVer + 1);
code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp,
- taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
+ POLL_RSP_TYPE(pRequest, taosxRsp), vgId);
+ if (terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT){terrno = 0;}
goto END;
} else {
fetchVer++;
@@ -526,7 +536,7 @@ int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPoll
return 0;
}
-int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId,
+int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId,
int32_t type, int64_t sver, int64_t ever) {
if (pRpcHandleInfo == NULL || pRsp == NULL) {
return TSDB_CODE_TMQ_INVALID_MSG;
@@ -534,7 +544,12 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp*
int32_t len = 0;
int32_t code = 0;
- if (type == TMQ_MSG_TYPE__POLL_DATA_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) {
+ if (type == TMQ_MSG_TYPE__POLL_RAW_DATA_RSP){
+ pRsp->withSchema = 0;
+ }
+ if (type == TMQ_MSG_TYPE__POLL_DATA_RSP ||
+ type == TMQ_MSG_TYPE__WALINFO_RSP ||
+ type == TMQ_MSG_TYPE__POLL_RAW_DATA_RSP) {
tEncodeSize(tEncodeMqDataRsp, pRsp, len, code);
} else if (type == TMQ_MSG_TYPE__POLL_DATA_META_RSP) {
tEncodeSize(tEncodeSTaosxRsp, pRsp, len, code);
@@ -558,7 +573,9 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp*
SEncoder encoder = {0};
tEncoderInit(&encoder, abuf, len);
- if (type == TMQ_MSG_TYPE__POLL_DATA_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) {
+ if (type == TMQ_MSG_TYPE__POLL_DATA_RSP ||
+ type == TMQ_MSG_TYPE__WALINFO_RSP ||
+ type == TMQ_MSG_TYPE__POLL_RAW_DATA_RSP) {
code = tEncodeMqDataRsp(&encoder, pRsp);
} else if (type == TMQ_MSG_TYPE__POLL_DATA_META_RSP) {
code = tEncodeSTaosxRsp(&encoder, pRsp);
diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c
index e9eda68cb3..0c4a3932b7 100644
--- a/source/dnode/vnode/src/tqCommon/tqCommon.c
+++ b/source/dnode/vnode/src/tqCommon/tqCommon.c
@@ -87,9 +87,9 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) {
return code;
}
- code =
- qSetStreamNotifyInfo(pTask->exec.pExecutor, pTask->notifyInfo.notifyEventTypes,
- pTask->notifyInfo.pSchemaWrapper, pTask->notifyInfo.stbFullName, IS_NEW_SUBTB_RULE(pTask));
+ code = qSetStreamNotifyInfo(pTask->exec.pExecutor, pTask->notifyInfo.notifyEventTypes,
+ pTask->notifyInfo.pSchemaWrapper, pTask->notifyInfo.stbFullName,
+ IS_NEW_SUBTB_RULE(pTask), &pTask->notifyEventStat);
if (code) {
tqError("s-task:%s failed to set stream notify info, code:%s", pTask->id.idStr, tstrerror(code));
return code;
diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c
index b8682028cf..b29d9add1b 100644
--- a/source/dnode/vnode/src/vnd/vnodeInitApi.c
+++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c
@@ -223,6 +223,7 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->updateInfoSerialize = updateInfoSerialize;
pStore->updateInfoDeserialize = updateInfoDeserialize;
+ pStore->streamStateSessionSeekKeyPrev = streamStateSessionSeekKeyPrev;
pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext;
pStore->streamStateCountSeekKeyPrev = streamStateCountSeekKeyPrev;
pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev;
diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c
index c7b1a816cd..d765861381 100644
--- a/source/dnode/vnode/src/vnd/vnodeQuery.c
+++ b/source/dnode/vnode/src/vnd/vnodeQuery.c
@@ -530,6 +530,8 @@ _exit:
int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
SSyncState state = syncGetState(pVnode->sync);
+ pLoad->syncAppliedIndex = pVnode->state.applied;
+ syncGetCommitIndex(pVnode->sync, &pLoad->syncCommitIndex);
pLoad->vgId = TD_VID(pVnode);
pLoad->syncState = state.state;
@@ -739,7 +741,7 @@ int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) {
}
int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) {
- SSchemaWrapper *pSW = metaGetTableSchema(pVnode->pMeta, suid, -1, 0, NULL);
+ SSchemaWrapper *pSW = metaGetTableSchema(pVnode->pMeta, suid, -1, 0);
if (pSW) {
*num = pSW->nCols;
tDeleteSchemaWrapper(pSW);
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 476b1388d0..abaa61744d 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -314,25 +314,25 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int
uint64_t nColData;
if (tDecodeU64v(pCoder, &nColData) < 0) {
code = TSDB_CODE_INVALID_MSG;
- goto _exit;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
SColData colData = {0};
code = tDecodeColData(version, pCoder, &colData);
if (code) {
code = TSDB_CODE_INVALID_MSG;
- goto _exit;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
if (colData.flag != HAS_VALUE) {
code = TSDB_CODE_INVALID_MSG;
- goto _exit;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
for (int32_t iRow = 0; iRow < colData.nVal; iRow++) {
if (((TSKEY *)colData.pData)[iRow] < minKey || ((TSKEY *)colData.pData)[iRow] > maxKey) {
code = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE;
- goto _exit;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
}
@@ -340,14 +340,14 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int
code = tDecodeColData(version, pCoder, &colData);
if (code) {
code = TSDB_CODE_INVALID_MSG;
- goto _exit;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
}
} else {
uint64_t nRow;
if (tDecodeU64v(pCoder, &nRow) < 0) {
code = TSDB_CODE_INVALID_MSG;
- goto _exit;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
for (int32_t iRow = 0; iRow < nRow; ++iRow) {
@@ -356,7 +356,7 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int
if (pRow->ts < minKey || pRow->ts > maxKey) {
code = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE;
- goto _exit;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
}
}
@@ -369,6 +369,9 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int
tEndDecode(pCoder);
_exit:
+ if (code) {
+ vError("vgId:%d, %s:%d failed to vnodePreProcessSubmitTbData submit request since %s", TD_VID(pVnode), __func__, lino, tstrerror(code));
+ }
return code;
}
static int32_t vnodePreProcessSubmitMsg(SVnode *pVnode, SRpcMsg *pMsg) {
@@ -947,20 +950,8 @@ int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo)
return tqProcessTaskRetrieveRsp(pVnode->pTq, pMsg);
case TDMT_VND_STREAM_SCAN_HISTORY:
return tqProcessTaskScanHistory(pVnode->pTq, pMsg);
- case TDMT_STREAM_TASK_CHECKPOINT_READY:
- return tqProcessTaskCheckpointReadyMsg(pVnode->pTq, pMsg);
- case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP:
- return tqProcessTaskCheckpointReadyRsp(pVnode->pTq, pMsg);
- case TDMT_STREAM_RETRIEVE_TRIGGER:
- return tqProcessTaskRetrieveTriggerReq(pVnode->pTq, pMsg);
- case TDMT_STREAM_RETRIEVE_TRIGGER_RSP:
- return tqProcessTaskRetrieveTriggerRsp(pVnode->pTq, pMsg);
- case TDMT_MND_STREAM_REQ_CHKPT_RSP:
- return tqProcessStreamReqCheckpointRsp(pVnode->pTq, pMsg);
case TDMT_VND_GET_STREAM_PROGRESS:
return tqStreamProgressRetrieveReq(pVnode->pTq, pMsg);
- case TDMT_MND_STREAM_CHKPT_REPORT_RSP:
- return tqProcessTaskChkptReportRsp(pVnode->pTq, pMsg);
default:
vError("unknown msg type:%d in stream queue", pMsg->msgType);
return TSDB_CODE_APP_ERROR;
@@ -987,6 +978,18 @@ int32_t vnodeProcessStreamCtrlMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pIn
return tqProcessTaskCheckReq(pVnode->pTq, pMsg);
case TDMT_VND_STREAM_TASK_CHECK_RSP:
return tqProcessTaskCheckRsp(pVnode->pTq, pMsg);
+ case TDMT_STREAM_TASK_CHECKPOINT_READY:
+ return tqProcessTaskCheckpointReadyMsg(pVnode->pTq, pMsg);
+ case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP:
+ return tqProcessTaskCheckpointReadyRsp(pVnode->pTq, pMsg);
+ case TDMT_STREAM_RETRIEVE_TRIGGER:
+ return tqProcessTaskRetrieveTriggerReq(pVnode->pTq, pMsg);
+ case TDMT_STREAM_RETRIEVE_TRIGGER_RSP:
+ return tqProcessTaskRetrieveTriggerRsp(pVnode->pTq, pMsg);
+ case TDMT_MND_STREAM_REQ_CHKPT_RSP:
+ return tqProcessStreamReqCheckpointRsp(pVnode->pTq, pMsg);
+ case TDMT_MND_STREAM_CHKPT_REPORT_RSP:
+ return tqProcessTaskChkptReportRsp(pVnode->pTq, pMsg);
default:
vError("unknown msg type:%d in stream ctrl queue", pMsg->msgType);
return TSDB_CODE_APP_ERROR;
@@ -1889,7 +1892,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in
len -= sizeof(SSubmitReq2Msg);
SDecoder dc = {0};
tDecoderInit(&dc, pReq, len);
- if (tDecodeSubmitReq(&dc, pSubmitReq) < 0) {
+ if (tDecodeSubmitReq(&dc, pSubmitReq, NULL) < 0) {
code = TSDB_CODE_INVALID_MSG;
goto _exit;
}
diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h
index 84eba69acb..e7bc1f67e1 100644
--- a/source/libs/executor/inc/executorInt.h
+++ b/source/libs/executor/inc/executorInt.h
@@ -450,16 +450,17 @@ typedef struct STimeWindowAggSupp {
} STimeWindowAggSupp;
typedef struct SStreamNotifyEventSupp {
- SArray* pWindowEvents; // Array of SStreamNotifyEvent, storing window events and trigger values.
- SHashObj* pTableNameHashMap; // Hash map from groupid to the dest child table name.
- SHashObj* pResultHashMap; // Hash map from groupid+skey to the window agg result.
- SSDataBlock* pEventBlock; // The datablock contains all window events and results.
+ SHashObj* pWindowEventHashMap; // Hash map from gorupid+skey+eventType to the list node of window event.
+ SHashObj* pTableNameHashMap; // Hash map from groupid to the dest child table name.
+ SSDataBlock* pEventBlock; // The datablock contains all window events and results.
+ SArray* pSessionKeys;
+ const char* windowType;
} SStreamNotifyEventSupp;
typedef struct SSteamOpBasicInfo {
int32_t primaryPkIndex;
bool updateOperatorInfo;
- SStreamNotifyEventSupp windowEventSup;
+ SStreamNotifyEventSupp notifyEventSup;
} SSteamOpBasicInfo;
typedef struct SStreamFillSupporter {
@@ -1053,7 +1054,7 @@ int32_t saveDeleteRes(SSHashObj* pStDelete, SSessionKey key);
void removeSessionResult(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey);
void doBuildDeleteDataBlock(struct SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite);
void doBuildSessionResult(struct SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo,
- SSDataBlock* pBlock);
+ SSDataBlock* pBlock, SArray* pSessionKeys);
int32_t getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SResultWindowInfo* pWinInfo);
void getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin,
SResultWindowInfo* pNextWin);
@@ -1090,7 +1091,7 @@ void freeResetOperatorParams(struct SOperatorInfo* pOperator, SOperatorParamT
int32_t getNextBlockFromDownstreamImpl(struct SOperatorInfo* pOperator, int32_t idx, bool clearParam,
SSDataBlock** pResBlock);
void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, EStreamType mode, SSessionKey* pDelRange);
-void doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey);
+void doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey);
int32_t saveDeleteInfo(SArray* pWins, SSessionKey key);
void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SArray* pWins);
diff --git a/source/libs/executor/inc/querytask.h b/source/libs/executor/inc/querytask.h
index 86ee6f4124..7e621e3df5 100644
--- a/source/libs/executor/inc/querytask.h
+++ b/source/libs/executor/inc/querytask.h
@@ -18,9 +18,9 @@
#ifdef __cplusplus
extern "C" {
-#endif
-
-#include "executorInt.h"
+ #endif
+
+ #include "executorInt.h"
#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str)
@@ -59,22 +59,23 @@ typedef struct STaskStopInfo {
} STaskStopInfo;
typedef struct {
- STqOffsetVal currentOffset; // for tmq
- SMqBatchMetaRsp btMetaRsp; // for tmq fetching meta
- int8_t sourceExcluded;
- int64_t snapshotVer;
- SSchemaWrapper* schema;
- char tbName[TSDB_TABLE_NAME_LEN]; // this is the current scan table: todo refactor
- int8_t recoverStep;
- int8_t recoverScanFinished;
- SQueryTableDataCond tableCond;
- SVersionRange fillHistoryVer;
- STimeWindow fillHistoryWindow;
- SStreamState* pState;
- int32_t eventTypes; // event types to notify
- SSchemaWrapper* notifyResultSchema; // agg result to notify
- char* stbFullName; // used to generate dest child table name
- bool newSubTableRule; // used to generate dest child table name
+ STqOffsetVal currentOffset; // for tmq
+ SMqBatchMetaRsp btMetaRsp; // for tmq fetching meta
+ int8_t sourceExcluded;
+ int64_t snapshotVer;
+ SSchemaWrapper* schema;
+ char tbName[TSDB_TABLE_NAME_LEN]; // this is the current scan table: todo refactor
+ int8_t recoverStep;
+ int8_t recoverScanFinished;
+ SQueryTableDataCond tableCond;
+ SVersionRange fillHistoryVer;
+ STimeWindow fillHistoryWindow;
+ SStreamState* pState;
+ int32_t eventTypes; // event types to notify
+ SSchemaWrapper* notifyResultSchema; // agg result to notify
+ char* stbFullName; // used to generate dest child table name
+ bool newSubTableRule; // used to generate dest child table name
+ STaskNotifyEventStat* pNotifyEventStat; // used to store notify event statistics
} SStreamTaskInfo;
struct SExecTaskInfo {
diff --git a/source/libs/executor/inc/streamexecutorInt.h b/source/libs/executor/inc/streamexecutorInt.h
index 7b3c828351..3195b2b67d 100644
--- a/source/libs/executor/inc/streamexecutorInt.h
+++ b/source/libs/executor/inc/streamexecutorInt.h
@@ -30,20 +30,31 @@ extern "C" {
#define FILL_POS_MID 2
#define FILL_POS_END 3
-#define HAS_NON_ROW_DATA(pRowData) (pRowData->key == INT64_MIN)
-#define HAS_ROW_DATA(pRowData) (pRowData && pRowData->key != INT64_MIN)
+#define HAS_NON_ROW_DATA(pRowData) (pRowData->key == INT64_MIN)
+#define HAS_ROW_DATA(pRowData) (pRowData && pRowData->key != INT64_MIN)
-#define IS_INVALID_WIN_KEY(ts) ((ts) == INT64_MIN)
-#define IS_VALID_WIN_KEY(ts) ((ts) != INT64_MIN)
-#define SET_WIN_KEY_INVALID(ts) ((ts) = INT64_MIN)
+#define IS_INVALID_WIN_KEY(ts) ((ts) == INT64_MIN)
+#define IS_VALID_WIN_KEY(ts) ((ts) != INT64_MIN)
+#define SET_WIN_KEY_INVALID(ts) ((ts) = INT64_MIN)
#define IS_NORMAL_INTERVAL_OP(op) \
((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || \
(op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL)
+#define IS_NORMAL_SESSION_OP(op) \
+ ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || \
+ (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
+
+#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE)
+
+#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT)
+
+#define IS_NORMAL_COUNT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT)
+
#define IS_CONTINUE_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_INTERVAL)
-#define IS_FILL_CONST_VALUE(type) ((type == TSDB_FILL_NULL || type == TSDB_FILL_NULL_F || type == TSDB_FILL_SET_VALUE || type == TSDB_FILL_SET_VALUE_F))
+#define IS_FILL_CONST_VALUE(type) \
+ ((type == TSDB_FILL_NULL || type == TSDB_FILL_NULL_F || type == TSDB_FILL_SET_VALUE || type == TSDB_FILL_SET_VALUE_F))
typedef struct SSliceRowData {
TSKEY key;
@@ -57,11 +68,13 @@ typedef struct SSlicePoint {
SRowBuffPos* pResPos;
} SSlicePoint;
-void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type);
-bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo);
-void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo);
-int32_t initStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo);
-void destroyStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo);
+void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type);
+bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo);
+void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo);
+int32_t initStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo, const struct SOperatorInfo* pOperator);
+void destroyStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo);
+int32_t encodeStreamBasicInfo(void** buf, const SSteamOpBasicInfo* pBasicInfo);
+int32_t decodeStreamBasicInfo(void** buf, SSteamOpBasicInfo* pBasicInfo);
int64_t getDeleteMarkFromOption(SStreamNodeOption* pOption);
void removeDeleteResults(SSHashObj* pUpdatedMap, SArray* pDelWins);
@@ -98,7 +111,7 @@ SResultCellData* getSliceResultCell(SResultCellData* pRowVal, int32_t index, int
int32_t getDownstreamRes(struct SOperatorInfo* downstream, SSDataBlock** ppRes, SColumnInfo** ppPkCol);
void destroyFlusedppPos(void* ppRes);
void doBuildStreamIntervalResult(struct SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,
- SGroupResInfo* pGroupResInfo);
+ SGroupResInfo* pGroupResInfo, SArray* pSessionKeys);
void transBlockToSliceResultRow(const SSDataBlock* pBlock, int32_t rowId, TSKEY ts, SSliceRowData* pRowVal,
int32_t rowSize, void* pPkData, SColumnInfoData* pPkCol, int32_t* pCellOffsetInfo);
int32_t getQualifiedRowNumDesc(SExprSupp* pExprSup, SSDataBlock* pBlock, TSKEY* tsCols, int32_t rowId, bool ignoreNull);
@@ -112,10 +125,25 @@ TSKEY compareTs(void* pKey);
int32_t addEventAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
const SSDataBlock* pInputBlock, const SNodeList* pCondCols, int32_t ri,
- SStreamNotifyEventSupp* sup);
-int32_t addAggResultNotifyEvent(const SSDataBlock* pResultBlock, const SSchemaWrapper* pSchemaWrapper,
- SStreamNotifyEventSupp* sup);
-int32_t buildNotifyEventBlock(const SExecTaskInfo* pTaskInfo, SStreamNotifyEventSupp* sup);
+ SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat);
+int32_t addStateAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
+ const SStateKeys* pCurState, const SStateKeys* pAnotherState, bool onlyUpdate,
+ SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat);
+int32_t addIntervalAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
+ SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat);
+int32_t addSessionAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
+ SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat);
+int32_t addCountAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
+ SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat);
+int32_t addAggResultNotifyEvent(const SSDataBlock* pResultBlock, const SArray* pSessionKeys,
+ const SSchemaWrapper* pSchemaWrapper, SStreamNotifyEventSupp* sup,
+ STaskNotifyEventStat* pNotifyEventStat);
+int32_t addAggDeleteNotifyEvent(const SSDataBlock* pDeleteBlock, SStreamNotifyEventSupp* sup,
+ STaskNotifyEventStat* pNotifyEventStat);
+int32_t buildNotifyEventBlock(const SExecTaskInfo* pTaskInfo, SStreamNotifyEventSupp* sup,
+ STaskNotifyEventStat* pNotifyEventStat);
+int32_t removeOutdatedNotifyEvents(STimeWindowAggSupp* pTwSup, SStreamNotifyEventSupp* sup,
+ STaskNotifyEventStat* pNotifyEventStat);
#ifdef __cplusplus
}
diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c
index 649a7a4524..7349feb70f 100644
--- a/source/libs/executor/src/cachescanoperator.c
+++ b/source/libs/executor/src/cachescanoperator.c
@@ -398,16 +398,16 @@ static int32_t doScanCacheNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
// check for tag values
if (pInfo->pRes->info.rows > 0) {
- if (pInfo->pseudoExprSup.numOfExprs > 0) {
- SExprSupp* pSup = &pInfo->pseudoExprSup;
+ SExprSupp* pSup = &pInfo->pseudoExprSup;
- STableKeyInfo* pKeyInfo = &((STableKeyInfo*)pList)[0];
- pInfo->pRes->info.id.groupId = pKeyInfo->groupId;
+ STableKeyInfo* pKeyInfo = &((STableKeyInfo*)pList)[0];
+ pInfo->pRes->info.id.groupId = pKeyInfo->groupId;
- if (taosArrayGetSize(pInfo->pUidList) > 0) {
- void* pUid = taosArrayGet(pInfo->pUidList, 0);
- QUERY_CHECK_NULL(pUid, code, lino, _end, terrno);
- pInfo->pRes->info.id.uid = *(tb_uid_t*)pUid;
+ if (taosArrayGetSize(pInfo->pUidList) > 0) {
+ void* pUid = taosArrayGet(pInfo->pUidList, 0);
+ QUERY_CHECK_NULL(pUid, code, lino, _end, terrno);
+ pInfo->pRes->info.id.uid = *(tb_uid_t*)pUid;
+ if (pInfo->pseudoExprSup.numOfExprs > 0) {
code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes,
pInfo->pRes->info.rows, pTaskInfo, NULL);
QUERY_CHECK_CODE(code, lino, _end);
diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c
index ec041cba70..3bd6f4e64a 100644
--- a/source/libs/executor/src/dataInserter.c
+++ b/source/libs/executor/src/dataInserter.c
@@ -187,7 +187,7 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
terrno = TSDB_CODE_SUCCESS;
if (NULL == pReq) {
- if (!(pReq = taosMemoryMalloc(sizeof(SSubmitReq2)))) {
+ if (!(pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2)))) {
goto _end;
}
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 39bef9c95f..2a6b77c53f 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -251,7 +251,7 @@ int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) {
}
int32_t qSetStreamNotifyInfo(qTaskInfo_t tinfo, int32_t eventTypes, const SSchemaWrapper* pSchemaWrapper,
- const char* stbFullName, bool newSubTableRule) {
+ const char* stbFullName, bool newSubTableRule, STaskNotifyEventStat* pNotifyEventStat) {
int32_t code = TSDB_CODE_SUCCESS;
SStreamTaskInfo *pStreamInfo = NULL;
@@ -267,6 +267,7 @@ int32_t qSetStreamNotifyInfo(qTaskInfo_t tinfo, int32_t eventTypes, const SSchem
}
pStreamInfo->stbFullName = taosStrdup(stbFullName);
pStreamInfo->newSubTableRule = newSubTableRule;
+ pStreamInfo->pNotifyEventStat = pNotifyEventStat;
_end:
return code;
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 1060cbcffe..4f647a2e9c 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3953,7 +3953,7 @@ FETCH_NEXT_BLOCK:
QUERY_CHECK_NULL(pSubmit, code, lino, _end, terrno);
qDebug("set %d/%d as the input submit block, %s", current + 1, totalBlocks, id);
- if (pAPI->tqReaderFn.tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) <
+ if (pAPI->tqReaderFn.tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver, NULL) <
0) {
qError("submit msg messed up when initializing stream submit block %p, current %d/%d, %s", pSubmit, current,
totalBlocks, id);
diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c
index b8c3ec90f9..c33abb3d89 100644
--- a/source/libs/executor/src/streamcountwindowoperator.c
+++ b/source/libs/executor/src/streamcountwindowoperator.c
@@ -25,7 +25,6 @@
#include "tlog.h"
#include "ttime.h"
-#define IS_NORMAL_COUNT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT)
#define STREAM_COUNT_OP_STATE_NAME "StreamCountHistoryState"
#define STREAM_COUNT_OP_CHECKPOINT_NAME "StreamCountOperator_Checkpoint"
@@ -56,6 +55,8 @@ void destroyStreamCountAggOperatorInfo(void* param) {
&pInfo->groupResInfo);
pInfo->pOperator = NULL;
}
+
+ destroyStreamBasicInfo(&pInfo->basic);
destroyStreamAggSupporter(&pInfo->streamAggSup);
cleanupExprSupp(&pInfo->scalarSupp);
clearGroupResInfo(&pInfo->groupResInfo);
@@ -79,10 +80,9 @@ void destroyStreamCountAggOperatorInfo(void* param) {
bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { return pAggSup->windowCount != pAggSup->windowSliding; }
int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, SCountWindowInfo* pCurWin,
- SBuffInfo* pBuffInfo) {
+ SBuffInfo* pBuffInfo, int32_t* pWinCode) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
- int32_t winCode = TSDB_CODE_SUCCESS;
int32_t size = pAggSup->resultRowSize;
pCurWin->winInfo.sessionWin.groupId = groupId;
pCurWin->winInfo.sessionWin.win.skey = ts;
@@ -90,19 +90,21 @@ int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group
if (isSlidingCountWindow(pAggSup)) {
if (pBuffInfo->winBuffOp == CREATE_NEW_WINDOW) {
- code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount,
- (void**)&pCurWin->winInfo.pStatePos, &size);
+ code =
+ pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
+ pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size);
QUERY_CHECK_CODE(code, lino, _end);
- winCode = TSDB_CODE_FAILED;
+ *pWinCode = TSDB_CODE_FAILED;
} else if (pBuffInfo->winBuffOp == MOVE_NEXT_WINDOW) {
QUERY_CHECK_NULL(pBuffInfo->pCur, code, lino, _end, terrno);
pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pBuffInfo->pCur);
- winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
- (void**)&pCurWin->winInfo.pStatePos, &size);
- if (winCode == TSDB_CODE_FAILED) {
- code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount,
- (void**)&pCurWin->winInfo.pStatePos, &size);
+ *pWinCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
+ (void**)&pCurWin->winInfo.pStatePos, &size);
+ if (*pWinCode == TSDB_CODE_FAILED) {
+ code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
+ pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos,
+ &size);
QUERY_CHECK_CODE(code, lino, _end);
} else {
reuseOutputBuf(pAggSup->pState, pCurWin->winInfo.pStatePos, &pAggSup->stateStore);
@@ -110,11 +112,12 @@ int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group
} else {
pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin,
pAggSup->windowCount);
- winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
- (void**)&pCurWin->winInfo.pStatePos, &size);
- if (winCode == TSDB_CODE_FAILED) {
- code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount,
- (void**)&pCurWin->winInfo.pStatePos, &size);
+ *pWinCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
+ (void**)&pCurWin->winInfo.pStatePos, &size);
+ if (*pWinCode == TSDB_CODE_FAILED) {
+ code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
+ pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos,
+ &size);
QUERY_CHECK_CODE(code, lino, _end);
} else {
reuseOutputBuf(pAggSup->pState, pCurWin->winInfo.pStatePos, &pAggSup->stateStore);
@@ -126,11 +129,11 @@ int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group
} else {
code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin,
pAggSup->windowCount,
- (void**)&pCurWin->winInfo.pStatePos, &size, &winCode);
+ (void**)&pCurWin->winInfo.pStatePos, &size, pWinCode);
QUERY_CHECK_CODE(code, lino, _end);
}
- if (winCode == TSDB_CODE_SUCCESS) {
+ if (*pWinCode == TSDB_CODE_SUCCESS) {
pCurWin->winInfo.isOutput = true;
}
pCurWin->pWindowCount =
@@ -297,10 +300,18 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
continue;
}
SCountWindowInfo curWin = {0};
+ int32_t winCode = TSDB_CODE_SUCCESS;
buffInfo.rebuildWindow = false;
- code = setCountOutputBuf(pAggSup, startTsCols[i], groupId, &curWin, &buffInfo);
+ code = setCountOutputBuf(pAggSup, startTsCols[i], groupId, &curWin, &buffInfo, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
+ if (winCode != TSDB_CODE_SUCCESS &&
+ BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {
+ code = addCountAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &curWin.winInfo.sessionWin, &pInfo->basic.notifyEventSup,
+ pTaskInfo->streamInfo.pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
+
if (!inCountSlidingWindow(pAggSup, &curWin.winInfo.sessionWin.win, &pSDataBlock->info)) {
buffInfo.winBuffOp = MOVE_NEXT_WINDOW;
continue;
@@ -375,23 +386,54 @@ _end:
static int32_t buildCountResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
int32_t code = TSDB_CODE_SUCCESS;
+ int32_t lino = 0;
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SOptrBasicInfo* pBInfo = &pInfo->binfo;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
+ STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
+ bool addNotifyEvent = false;
+ addNotifyEvent = BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
if (pInfo->pDelRes->info.rows > 0) {
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+ if (addNotifyEvent) {
+ code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
(*ppRes) = pInfo->pDelRes;
return code;
}
- doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes);
+ doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes,
+ addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
if (pBInfo->pRes->info.rows > 0) {
printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+ if (addNotifyEvent) {
+ code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,
+ pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
(*ppRes) = pBInfo->pRes;
return code;
}
+
+ code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
+ printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+ (*ppRes) = pNotifySup->pEventBlock;
+ return code;
+ }
+
+ code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+
+_end:
+ if (code != TSDB_CODE_SUCCESS) {
+ qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
+ }
(*ppRes) = NULL;
return code;
}
@@ -423,7 +465,10 @@ int32_t doStreamCountEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper
// 3.dataVersion
tlen += taosEncodeFixedI32(buf, pInfo->dataVersion);
- // 4.checksum
+ // 4.basicInfo
+ tlen += encodeStreamBasicInfo(buf, &pInfo->basic);
+
+ // 5.checksum
if (isParent) {
if (buf) {
uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));
@@ -441,12 +486,13 @@ int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
int32_t lino = 0;
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ void* pDataEnd = POINTER_SHIFT(buf, len);
if (!pInfo) {
code = TSDB_CODE_FAILED;
QUERY_CHECK_CODE(code, lino, _end);
}
- // 4.checksum
+ // 5.checksum
if (isParent) {
int32_t dataLen = len - sizeof(uint32_t);
void* pCksum = POINTER_SHIFT(buf, dataLen);
@@ -454,6 +500,7 @@ int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
code = TSDB_CODE_FAILED;
QUERY_CHECK_CODE(code, lino, _end);
}
+ pDataEnd = pCksum;
}
// 1.streamAggSup.pResultRows
@@ -462,9 +509,10 @@ int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
for (int32_t i = 0; i < mapSize; i++) {
SSessionKey key = {0};
SCountWindowInfo curWin = {0};
+ int32_t winCode = TSDB_CODE_SUCCESS;
buf = decodeSSessionKey(buf, &key);
SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL};
- code = setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo);
+ code = setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize);
@@ -479,6 +527,12 @@ int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
// 3.dataVersion
buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);
+ // 4.basicInfo
+ if (buf < pDataEnd) {
+ code = decodeStreamBasicInfo(&buf, &pInfo->basic);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
+
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
@@ -851,7 +905,7 @@ int32_t createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);
pInfo->binfo.pRes = pResBlock;
- SExprInfo* pExprInfo = NULL;
+ SExprInfo* pExprInfo = NULL;
code = createExprInfo(pCountNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);
QUERY_CHECK_CODE(code, lino, _error);
@@ -925,6 +979,9 @@ int32_t createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamCountReleaseState, streamCountReloadState);
+ code = initStreamBasicInfo(&pInfo->basic, pOperator);
+ QUERY_CHECK_CODE(code, lino, _error);
+
if (downstream) {
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c
index 5f4d6b30fa..ab2aa600bb 100644
--- a/source/libs/executor/src/streameventwindowoperator.c
+++ b/source/libs/executor/src/streameventwindowoperator.c
@@ -30,7 +30,6 @@
#include "tlog.h"
#include "ttime.h"
-#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT)
#define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState"
#define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint"
@@ -135,7 +134,8 @@ void reuseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) {
}
int32_t setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupId, bool* pStart, bool* pEnd,
- int32_t index, int32_t rows, SEventWindowInfo* pCurWin, SSessionKey* pNextWinKey, int32_t* pWinCode) {
+ int32_t index, int32_t rows, SEventWindowInfo* pCurWin, SSessionKey* pNextWinKey,
+ int32_t* pWinCode) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t winCode = TSDB_CODE_SUCCESS;
@@ -179,7 +179,7 @@ int32_t setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t gro
SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId};
code = pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len);
QUERY_CHECK_CODE(code, lino, _error);
- (*pWinCode) = TSDB_CODE_FAILED;
+ (*pWinCode) = TSDB_CODE_FAILED;
setEventWindowInfo(pAggSup, &winKey, pVal, pCurWin);
pCurWin->pWinFlag->startFlag = start;
@@ -335,6 +335,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SColumnInfoData* pColStart = NULL;
SColumnInfoData* pColEnd = NULL;
+ SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
+ STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);
pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;
@@ -395,17 +397,19 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
&nextWinKey, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
- if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN) &&
- *(bool*)colDataGetNumData(pColStart, i) && winCode != TSDB_CODE_SUCCESS) {
+ if (winCode != TSDB_CODE_SUCCESS &&
+ BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN) &&
+ *(bool*)colDataGetNumData(pColStart, i)) {
code = addEventAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &curWin.winInfo.sessionWin, pSDataBlock,
- pInfo->pStartCondCols, i, &pInfo->basic.windowEventSup);
+ pInfo->pStartCondCols, i, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo);
bool rebuild = false;
- code = updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData,
- rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild, &winRows);
+ code = updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData,
+ (bool*)pColEnd->pData, rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild,
+ &winRows);
QUERY_CHECK_CODE(code, lino, _end);
if (rebuild) {
@@ -471,7 +475,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE)) {
code = addEventAggNotifyEvent(SNOTIFY_EVENT_WINDOW_CLOSE, &curWin.winInfo.sessionWin, pSDataBlock,
- pInfo->pEndCondCols, i + winRows - 1, &pInfo->basic.windowEventSup);
+ pInfo->pEndCondCols, i + winRows - 1, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
}
@@ -513,7 +517,10 @@ int32_t doStreamEventEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper
// 3.dataVersion
tlen += taosEncodeFixedI32(buf, pInfo->dataVersion);
- // 4.checksum
+ // 4.basicInfo
+ tlen += encodeStreamBasicInfo(buf, &pInfo->basic);
+
+ // 5.checksum
if (buf) {
uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));
tlen += taosEncodeFixedU32(buf, cksum);
@@ -529,13 +536,14 @@ int32_t doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
int32_t lino = 0;
SStreamEventAggOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ void* pDataEnd = POINTER_SHIFT(buf, len);
if (!pInfo) {
code = TSDB_CODE_FAILED;
QUERY_CHECK_CODE(code, lino, _end);
}
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
- // 4.checksum
+ // 5.checksum
int32_t dataLen = len - sizeof(uint32_t);
void* pCksum = POINTER_SHIFT(buf, dataLen);
if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {
@@ -543,6 +551,7 @@ int32_t doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
code = TSDB_CODE_FAILED;
QUERY_CHECK_CODE(code, lino, _end);
}
+ pDataEnd = pCksum;
// 1.streamAggSup.pResultRows
int32_t mapSize = 0;
@@ -567,6 +576,12 @@ int32_t doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
// 3.dataVersion
buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);
+ // 4.basicInfo
+ if (buf < pDataEnd) {
+ code = decodeStreamBasicInfo(&buf, &pInfo->basic);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
+
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
@@ -598,33 +613,45 @@ static int32_t buildEventResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
SStreamEventAggOperatorInfo* pInfo = pOperator->info;
SOptrBasicInfo* pBInfo = &pInfo->binfo;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
-
+ SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
+ STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
+ bool addNotifyEvent = false;
+ addNotifyEvent = BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
if (pInfo->pDelRes->info.rows > 0) {
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+ if (addNotifyEvent) {
+ code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
(*ppRes) = pInfo->pDelRes;
return code;
}
- doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes);
+ doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes,
+ addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
if (pBInfo->pRes->info.rows > 0) {
printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
- if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE)) {
- code = addAggResultNotifyEvent(pBInfo->pRes, pTaskInfo->streamInfo.notifyResultSchema, &pInfo->basic.windowEventSup);
+ if (addNotifyEvent) {
+ code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,
+ pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
(*ppRes) = pBInfo->pRes;
return code;
}
- code = buildNotifyEventBlock(pTaskInfo, &pInfo->basic.windowEventSup);
+ code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
- if (pInfo->basic.windowEventSup.pEventBlock->info.rows > 0) {
- printDataBlock(pInfo->basic.windowEventSup.pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
- (*ppRes) = pInfo->basic.windowEventSup.pEventBlock;
+ if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
+ printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+ (*ppRes) = pNotifySup->pEventBlock;
return code;
}
+ code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+
_end:
(*ppRes) = NULL;
if (code != TSDB_CODE_SUCCESS) {
@@ -800,7 +827,7 @@ void streamEventReleaseState(SOperatorInfo* pOperator) {
int32_t resSize = winSize + sizeof(TSKEY);
char* pBuff = taosMemoryCalloc(1, resSize);
if (!pBuff) {
- return ;
+ return;
}
memcpy(pBuff, pInfo->historyWins->pData, winSize);
memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY));
@@ -954,9 +981,9 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);
pInfo->binfo.pRes = pResBlock;
- SExprSupp* pExpSup = &pOperator->exprSupp;
- int32_t numOfCols = 0;
- SExprInfo* pExprInfo = NULL;
+ SExprSupp* pExpSup = &pOperator->exprSupp;
+ int32_t numOfCols = 0;
+ SExprInfo* pExprInfo = NULL;
code = createExprInfo(pEventNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);
QUERY_CHECK_CODE(code, lino, _error);
@@ -1006,7 +1033,6 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
pInfo->pPkDeleted = tSimpleHashInit(64, hashFn);
QUERY_CHECK_NULL(pInfo->pPkDeleted, code, lino, _error, terrno);
pInfo->destHasPrimaryKey = pEventNode->window.destHasPrimaryKey;
- initStreamBasicInfo(&pInfo->basic);
pInfo->pOperator = pOperator;
setOperatorInfo(pOperator, "StreamEventAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, true, OP_NOT_OPENED,
@@ -1026,6 +1052,10 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAggNext, NULL, destroyStreamEventOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState);
+
+ code = initStreamBasicInfo(&pInfo->basic, pOperator);
+ QUERY_CHECK_CODE(code, lino, _error);
+
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
QUERY_CHECK_CODE(code, lino, _error);
diff --git a/source/libs/executor/src/streamexecutorInt.c b/source/libs/executor/src/streamexecutorInt.c
index 9cafdfff0c..635de21b6e 100644
--- a/source/libs/executor/src/streamexecutorInt.c
+++ b/source/libs/executor/src/streamexecutorInt.c
@@ -16,53 +16,58 @@
#include "streamexecutorInt.h"
#include "executorInt.h"
+#include "operator.h"
#include "tdatablock.h"
#define NOTIFY_EVENT_NAME_CACHE_LIMIT_MB 16
typedef struct SStreamNotifyEvent {
uint64_t gid;
- TSKEY skey;
- char* content;
- bool isEnd;
+ int64_t eventType;
+ STimeWindow win;
+ cJSON* pJson;
} SStreamNotifyEvent;
+#define NOTIFY_EVENT_KEY_SIZE \
+ ((sizeof(((struct SStreamNotifyEvent*)0)->gid) + sizeof(((struct SStreamNotifyEvent*)0)->eventType)) + \
+ sizeof(((struct SStreamNotifyEvent*)0)->win.skey))
+
void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type) {
if (type != STREAM_GET_ALL && type != STREAM_CHECKPOINT) {
pBasicInfo->updateOperatorInfo = true;
}
}
-bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo) {
- return pBasicInfo->updateOperatorInfo;
-}
+bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo) { return pBasicInfo->updateOperatorInfo; }
-void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo) {
- pBasicInfo->updateOperatorInfo = false;
-}
+void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo) { pBasicInfo->updateOperatorInfo = false; }
static void destroyStreamWindowEvent(void* ptr) {
- SStreamNotifyEvent* pEvent = ptr;
- if (pEvent == NULL || pEvent->content == NULL) return;
- cJSON_free(pEvent->content);
+ SStreamNotifyEvent* pEvent = (SStreamNotifyEvent*)ptr;
+ if (pEvent) {
+ if (pEvent->pJson) {
+ cJSON_Delete(pEvent->pJson);
+ }
+ *pEvent = (SStreamNotifyEvent){0};
+ }
}
static void destroyStreamNotifyEventSupp(SStreamNotifyEventSupp* sup) {
if (sup == NULL) return;
- taosArrayDestroyEx(sup->pWindowEvents, destroyStreamWindowEvent);
+ taosHashCleanup(sup->pWindowEventHashMap);
taosHashCleanup(sup->pTableNameHashMap);
- taosHashCleanup(sup->pResultHashMap);
blockDataDestroy(sup->pEventBlock);
+ taosArrayDestroy(sup->pSessionKeys);
*sup = (SStreamNotifyEventSupp){0};
}
-static int32_t initStreamNotifyEventSupp(SStreamNotifyEventSupp *sup) {
- int32_t code = TSDB_CODE_SUCCESS;
- int32_t lino = 0;
- SSDataBlock* pBlock = NULL;
+static int32_t initStreamNotifyEventSupp(SStreamNotifyEventSupp* sup, const char* windowType, int32_t resCapacity) {
+ int32_t code = TSDB_CODE_SUCCESS;
+ int32_t lino = 0;
+ SSDataBlock* pBlock = NULL;
SColumnInfoData infoData = {0};
- if (sup == NULL) {
+ if (sup == NULL || sup->pWindowEventHashMap != NULL) {
goto _end;
}
@@ -77,15 +82,18 @@ static int32_t initStreamNotifyEventSupp(SStreamNotifyEventSupp *sup) {
code = blockDataAppendColInfo(pBlock, &infoData);
QUERY_CHECK_CODE(code, lino, _end);
- sup->pWindowEvents = taosArrayInit(0, sizeof(SStreamNotifyEvent));
- QUERY_CHECK_NULL(sup->pWindowEvents, code, lino, _end, terrno);
+ sup->pWindowEventHashMap = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+ QUERY_CHECK_NULL(sup->pWindowEventHashMap, code, lino, _end, terrno);
+ taosHashSetFreeFp(sup->pWindowEventHashMap, destroyStreamWindowEvent);
sup->pTableNameHashMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
QUERY_CHECK_NULL(sup->pTableNameHashMap, code, lino, _end, terrno);
- sup->pResultHashMap = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
- QUERY_CHECK_NULL(sup->pResultHashMap, code, lino, _end, terrno);
- taosHashSetFreeFp(sup->pResultHashMap, destroyStreamWindowEvent);
sup->pEventBlock = pBlock;
pBlock = NULL;
+ code = blockDataEnsureCapacity(sup->pEventBlock, resCapacity);
+ QUERY_CHECK_CODE(code, lino, _end);
+ sup->windowType = windowType;
+ sup->pSessionKeys = taosArrayInit(resCapacity, sizeof(SSessionKey));
+ QUERY_CHECK_NULL(sup->pSessionKeys, code, lino, _end, terrno);
_end:
if (code != TSDB_CODE_SUCCESS) {
@@ -100,17 +108,99 @@ _end:
return code;
}
-int32_t initStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo) {
+int32_t initStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo, const struct SOperatorInfo* pOperator) {
pBasicInfo->primaryPkIndex = -1;
pBasicInfo->updateOperatorInfo = false;
- return initStreamNotifyEventSupp(&pBasicInfo->windowEventSup);
+ const char* windowType = NULL;
+ if (IS_NORMAL_INTERVAL_OP(pOperator)) {
+ windowType = "Time";
+ } else if (IS_NORMAL_SESSION_OP(pOperator)) {
+ windowType = "Session";
+ } else if (IS_NORMAL_STATE_OP(pOperator)) {
+ windowType = "State";
+ } else if (IS_NORMAL_EVENT_OP(pOperator)) {
+ windowType = "Event";
+ } else if (IS_NORMAL_COUNT_OP(pOperator)) {
+ windowType = "Count";
+ } else {
+ return TSDB_CODE_SUCCESS;
+ }
+ return initStreamNotifyEventSupp(&pBasicInfo->notifyEventSup, windowType, pOperator->resultInfo.capacity);
}
void destroyStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo) {
- destroyStreamNotifyEventSupp(&pBasicInfo->windowEventSup);
+ destroyStreamNotifyEventSupp(&pBasicInfo->notifyEventSup);
}
-static void streamNotifyGetEventWindowId(const SSessionKey* pSessionKey, char *buf) {
+static int32_t encodeStreamNotifyEventSupp(void** buf, const SStreamNotifyEventSupp* sup) {
+ int32_t tlen = 0;
+ void* pIter = NULL;
+ char* str = NULL;
+
+ if (sup == NULL) {
+ return tlen;
+ }
+
+ tlen += taosEncodeFixedI32(buf, taosHashGetSize(sup->pWindowEventHashMap));
+ pIter = taosHashIterate(sup->pWindowEventHashMap, NULL);
+ while (pIter) {
+ const SStreamNotifyEvent* pEvent = (const SStreamNotifyEvent*)pIter;
+ str = cJSON_PrintUnformatted(pEvent->pJson);
+
+ tlen += taosEncodeFixedU64(buf, pEvent->gid);
+ tlen += taosEncodeFixedI64(buf, pEvent->eventType);
+ tlen += taosEncodeFixedI64(buf, pEvent->win.skey);
+ tlen += taosEncodeFixedI64(buf, pEvent->win.ekey);
+ tlen += taosEncodeString(buf, str);
+ cJSON_free(str);
+ pIter = taosHashIterate(sup->pWindowEventHashMap, pIter);
+ }
+ return tlen;
+}
+
+static int32_t decodeStreamNotifyEventSupp(void** buf, SStreamNotifyEventSupp* sup) {
+ int32_t code = TSDB_CODE_SUCCESS;
+ int32_t lino = 0;
+ void* p = *buf;
+ int32_t size = 0;
+ uint64_t len = 0;
+ SStreamNotifyEvent item = {0};
+
+ p = taosDecodeFixedI32(p, &size);
+ for (int32_t i = 0; i < size; i++) {
+ p = taosDecodeFixedU64(p, &item.gid);
+ p = taosDecodeFixedI64(p, &item.eventType);
+ p = taosDecodeFixedI64(p, &item.win.skey);
+ p = taosDecodeFixedI64(p, &item.win.ekey);
+ p = taosDecodeVariantU64(p, &len);
+ item.pJson = cJSON_Parse(p);
+ if (item.pJson == NULL) {
+ qWarn("failed to parse the json content since %s", cJSON_GetErrorPtr());
+ }
+ QUERY_CHECK_NULL(item.pJson, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
+ p = POINTER_SHIFT(p, len);
+ code = taosHashPut(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE, &item, sizeof(SStreamNotifyEvent));
+ QUERY_CHECK_CODE(code, lino, _end);
+ item.pJson = NULL;
+ }
+ *buf = p;
+_end:
+ if (code != TSDB_CODE_SUCCESS) {
+ qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
+ }
+ destroyStreamWindowEvent(&item);
+ return code;
+}
+
+int32_t encodeStreamBasicInfo(void** buf, const SSteamOpBasicInfo* pBasicInfo) {
+ return encodeStreamNotifyEventSupp(buf, &pBasicInfo->notifyEventSup);
+}
+
+int32_t decodeStreamBasicInfo(void** buf, SSteamOpBasicInfo* pBasicInfo) {
+ return decodeStreamNotifyEventSupp(buf, &pBasicInfo->notifyEventSup);
+}
+
+static void streamNotifyGetEventWindowId(const SSessionKey* pSessionKey, char* buf) {
uint64_t hash = 0;
uint64_t ar[2];
@@ -123,60 +213,60 @@ static void streamNotifyGetEventWindowId(const SSessionKey* pSessionKey, char *b
#define JSON_CHECK_ADD_ITEM(obj, str, item) \
QUERY_CHECK_CONDITION(cJSON_AddItemToObjectCS(obj, str, item), code, lino, _end, TSDB_CODE_OUT_OF_MEMORY)
-static int32_t jsonAddColumnField(const char* colName, const SColumnInfoData* pColData, int32_t ri, cJSON* obj) {
+static int32_t jsonAddColumnField(const char* colName, int16_t type, bool isNull, const char* pData, cJSON* obj) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
char* temp = NULL;
QUERY_CHECK_NULL(colName, code, lino, _end, TSDB_CODE_INVALID_PARA);
- QUERY_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA);
+ QUERY_CHECK_CONDITION(isNull || (pData != NULL), code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(obj, code, lino, _end, TSDB_CODE_INVALID_PARA);
- if (colDataIsNull_s(pColData, ri)) {
+ if (isNull) {
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNull());
goto _end;
}
- switch (pColData->info.type) {
+ switch (type) {
case TSDB_DATA_TYPE_BOOL: {
- bool val = *(bool*)colDataGetNumData(pColData, ri);
+ bool val = *(const bool*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateBool(val));
break;
}
case TSDB_DATA_TYPE_TINYINT: {
- int8_t val = *(int8_t*)colDataGetNumData(pColData, ri);
+ int8_t val = *(const int8_t*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
break;
}
case TSDB_DATA_TYPE_SMALLINT: {
- int16_t val = *(int16_t*)colDataGetNumData(pColData, ri);
+ int16_t val = *(const int16_t*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
break;
}
case TSDB_DATA_TYPE_INT: {
- int32_t val = *(int32_t*)colDataGetNumData(pColData, ri);
+ int32_t val = *(const int32_t*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
break;
}
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP: {
- int64_t val = *(int64_t*)colDataGetNumData(pColData, ri);
+ int64_t val = *(const int64_t*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
break;
}
case TSDB_DATA_TYPE_FLOAT: {
- float val = *(float*)colDataGetNumData(pColData, ri);
+ float val = *(const float*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
- double val = *(double*)colDataGetNumData(pColData, ri);
+ double val = *(const double*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
break;
}
@@ -185,8 +275,8 @@ static int32_t jsonAddColumnField(const char* colName, const SColumnInfoData* pC
case TSDB_DATA_TYPE_NCHAR: {
// cJSON requires null-terminated strings, but this data is not null-terminated,
// so we need to manually copy the string and add null termination.
- const char* src = varDataVal(colDataGetVarData(pColData, ri));
- int32_t len = varDataLen(colDataGetVarData(pColData, ri));
+ const char* src = varDataVal(pData);
+ int32_t len = varDataLen(pData);
temp = cJSON_malloc(len + 1);
QUERY_CHECK_NULL(temp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
memcpy(temp, src, len);
@@ -202,25 +292,25 @@ static int32_t jsonAddColumnField(const char* colName, const SColumnInfoData* pC
}
case TSDB_DATA_TYPE_UTINYINT: {
- uint8_t val = *(uint8_t*)colDataGetNumData(pColData, ri);
+ uint8_t val = *(const uint8_t*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
break;
}
case TSDB_DATA_TYPE_USMALLINT: {
- uint16_t val = *(uint16_t*)colDataGetNumData(pColData, ri);
+ uint16_t val = *(const uint16_t*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
break;
}
case TSDB_DATA_TYPE_UINT: {
- uint32_t val = *(uint32_t*)colDataGetNumData(pColData, ri);
+ uint32_t val = *(const uint32_t*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
break;
}
case TSDB_DATA_TYPE_UBIGINT: {
- uint64_t val = *(uint64_t*)colDataGetNumData(pColData, ri);
+ uint64_t val = *(const uint64_t*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
break;
}
@@ -241,53 +331,91 @@ _end:
return code;
}
-int32_t addEventAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
- const SSDataBlock* pInputBlock, const SNodeList* pCondCols, int32_t ri,
- SStreamNotifyEventSupp* sup) {
- int32_t code = TSDB_CODE_SUCCESS;
- int32_t lino = 0;
- SNode* node = NULL;
- cJSON* event = NULL;
- cJSON* fields = NULL;
- cJSON* cond = NULL;
- SStreamNotifyEvent item = {0};
- char windowId[32];
+static cJSON* createBasicAggNotifyEvent(const char* windowType, EStreamNotifyEventType eventType,
+ const SSessionKey* pSessionKey) {
+ int32_t code = TSDB_CODE_SUCCESS;
+ int32_t lino = 0;
+ const char* eventTypeStr = NULL;
+ cJSON* event = NULL;
+ char windowId[32];
+ QUERY_CHECK_NULL(windowType, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(pSessionKey, code, lino, _end, TSDB_CODE_INVALID_PARA);
- QUERY_CHECK_NULL(pInputBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
- QUERY_CHECK_NULL(pInputBlock->pDataBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
- QUERY_CHECK_NULL(pCondCols, code, lino, _end, TSDB_CODE_INVALID_PARA);
- QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
- qDebug("add stream notify event from event window, type: %s, start: %" PRId64 ", end: %" PRId64,
- (eventType == SNOTIFY_EVENT_WINDOW_OPEN) ? "WINDOW_OPEN" : "WINDOW_CLOSE", pSessionKey->win.skey,
- pSessionKey->win.ekey);
+ if (eventType == SNOTIFY_EVENT_WINDOW_OPEN) {
+ eventTypeStr = "WINDOW_OPEN";
+ } else if (eventType == SNOTIFY_EVENT_WINDOW_CLOSE) {
+ eventTypeStr = "WINDOW_CLOSE";
+ } else if (eventType == SNOTIFY_EVENT_WINDOW_INVALIDATION) {
+ eventTypeStr = "WINDOW_INVALIDATION";
+ } else {
+ QUERY_CHECK_CONDITION(false, code, lino, _end, TSDB_CODE_INVALID_PARA);
+ }
+
+ qDebug("add stream notify event from %s Window, type: %s, start: %" PRId64 ", end: %" PRId64, windowType,
+ eventTypeStr, pSessionKey->win.skey, pSessionKey->win.ekey);
event = cJSON_CreateObject();
QUERY_CHECK_NULL(event, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
// add basic info
streamNotifyGetEventWindowId(pSessionKey, windowId);
- if (eventType == SNOTIFY_EVENT_WINDOW_OPEN) {
- JSON_CHECK_ADD_ITEM(event, "eventType", cJSON_CreateStringReference("WINDOW_OPEN"));
- } else if (eventType == SNOTIFY_EVENT_WINDOW_CLOSE) {
- JSON_CHECK_ADD_ITEM(event, "eventType", cJSON_CreateStringReference("WINDOW_CLOSE"));
- }
+ JSON_CHECK_ADD_ITEM(event, "eventType", cJSON_CreateStringReference(eventTypeStr));
JSON_CHECK_ADD_ITEM(event, "eventTime", cJSON_CreateNumber(taosGetTimestampMs()));
- JSON_CHECK_ADD_ITEM(event, "windowId", cJSON_CreateStringReference(windowId));
- JSON_CHECK_ADD_ITEM(event, "windowType", cJSON_CreateStringReference("Event"));
+ JSON_CHECK_ADD_ITEM(event, "windowId", cJSON_CreateString(windowId));
+ JSON_CHECK_ADD_ITEM(event, "windowType", cJSON_CreateStringReference(windowType));
JSON_CHECK_ADD_ITEM(event, "windowStart", cJSON_CreateNumber(pSessionKey->win.skey));
- if (eventType == SNOTIFY_EVENT_WINDOW_CLOSE) {
- JSON_CHECK_ADD_ITEM(event, "windowEnd", cJSON_CreateNumber(pSessionKey->win.ekey));
+ if (eventType != SNOTIFY_EVENT_WINDOW_OPEN) {
+ if (strcmp(windowType, "Time") == 0) {
+ JSON_CHECK_ADD_ITEM(event, "windowEnd", cJSON_CreateNumber(pSessionKey->win.ekey + 1));
+ } else {
+ JSON_CHECK_ADD_ITEM(event, "windowEnd", cJSON_CreateNumber(pSessionKey->win.ekey));
+ }
}
+_end:
+ if (code != TSDB_CODE_SUCCESS) {
+ qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
+ terrno = code;
+ cJSON_Delete(event);
+ event = NULL;
+ }
+ return event;
+}
+
+int32_t addEventAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
+ const SSDataBlock* pInputBlock, const SNodeList* pCondCols, int32_t ri,
+ SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
+ int32_t code = TSDB_CODE_SUCCESS;
+ int32_t lino = 0;
+ cJSON* event = NULL;
+ cJSON* fields = NULL;
+ cJSON* cond = NULL;
+ const SNode* pNode = NULL;
+ int32_t origSize = 0;
+ int64_t startTime = 0;
+ int64_t endTime = 0;
+ SStreamNotifyEvent item = {0};
+
+ QUERY_CHECK_NULL(pInputBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
+ QUERY_CHECK_NULL(pInputBlock->pDataBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
+ QUERY_CHECK_NULL(pCondCols, code, lino, _end, TSDB_CODE_INVALID_PARA);
+ QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
+ QUERY_CHECK_NULL(sup->windowType, code, lino, _end, TSDB_CODE_INVALID_PARA);
+ QUERY_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
+
+ startTime = taosGetMonoTimestampMs();
+ event = createBasicAggNotifyEvent(sup->windowType, eventType, pSessionKey);
+ QUERY_CHECK_NULL(event, code, lino, _end, terrno);
+
// create fields object to store matched column values
fields = cJSON_CreateObject();
QUERY_CHECK_NULL(fields, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
- FOREACH(node, pCondCols) {
- SColumnNode* pColDef = (SColumnNode*)node;
- SColumnInfoData* pColData = taosArrayGet(pInputBlock->pDataBlock, pColDef->slotId);
- code = jsonAddColumnField(pColDef->colName, pColData, ri, fields);
+ FOREACH(pNode, pCondCols) {
+ const SColumnNode* pColDef = (const SColumnNode*)pNode;
+ const SColumnInfoData* pColData = taosArrayGet(pInputBlock->pDataBlock, pColDef->slotId);
+ code = jsonAddColumnField(pColDef->colName, pColData->info.type, colDataIsNull_s(pColData, ri),
+ colDataGetData(pColData, ri), fields);
QUERY_CHECK_CODE(code, lino, _end);
}
@@ -297,16 +425,24 @@ int32_t addEventAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionK
JSON_CHECK_ADD_ITEM(cond, "conditionIndex", cJSON_CreateNumber(0));
JSON_CHECK_ADD_ITEM(cond, "fieldValues", fields);
fields = NULL;
- JSON_CHECK_ADD_ITEM(event, "triggerConditions", cond);
+ JSON_CHECK_ADD_ITEM(event, "triggerCondition", cond);
cond = NULL;
- // convert json object to string value
item.gid = pSessionKey->groupId;
- item.skey = pSessionKey->win.skey;
- item.isEnd = (eventType == SNOTIFY_EVENT_WINDOW_CLOSE);
- item.content = cJSON_PrintUnformatted(event);
- QUERY_CHECK_NULL(taosArrayPush(sup->pWindowEvents, &item), code, lino, _end, terrno);
- item.content = NULL;
+ item.win = pSessionKey->win;
+ item.eventType = eventType;
+ item.pJson = event;
+ event = NULL;
+ origSize = taosHashGetSize(sup->pWindowEventHashMap);
+ code = taosHashPut(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE, &item, sizeof(SStreamNotifyEvent));
+ QUERY_CHECK_CODE(code, lino, _end);
+ item.pJson = NULL;
+
+ endTime = taosGetMonoTimestampMs();
+ pNotifyEventStat->notifyEventAddTimes++;
+ pNotifyEventStat->notifyEventAddElems += taosHashGetSize(sup->pWindowEventHashMap) - origSize;
+ pNotifyEventStat->notifyEventAddCostSec += (endTime - startTime) / 1000.0;
+ pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
_end:
if (code != TSDB_CODE_SUCCESS) {
@@ -325,50 +461,204 @@ _end:
return code;
}
-int32_t addAggResultNotifyEvent(const SSDataBlock* pResultBlock, const SSchemaWrapper* pSchemaWrapper,
- SStreamNotifyEventSupp* sup) {
+int32_t addStateAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
+ const SStateKeys* pCurState, const SStateKeys* pAnotherState, bool onlyUpdate,
+ SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
- SNode * node = NULL;
cJSON* event = NULL;
- cJSON* result = NULL;
+ int32_t origSize = 0;
+ int64_t startTime = 0;
+ int64_t endTime = 0;
+ SStreamNotifyEvent item = {0};
+
+ QUERY_CHECK_NULL(pCurState, code, lino, _end, TSDB_CODE_INVALID_PARA);
+ QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
+ QUERY_CHECK_NULL(sup->windowType, code, lino, _end, TSDB_CODE_INVALID_PARA);
+ QUERY_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
+
+ item.gid = pSessionKey->groupId;
+ item.win = pSessionKey->win;
+ item.eventType = eventType;
+ // Check if the notify event exists for update
+ if (onlyUpdate && taosHashGet(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE) == NULL) {
+ goto _end;
+ }
+
+ startTime = taosGetMonoTimestampMs();
+ event = createBasicAggNotifyEvent(sup->windowType, eventType, pSessionKey);
+ QUERY_CHECK_NULL(event, code, lino, _end, terrno);
+
+ // add state value
+ if (eventType == SNOTIFY_EVENT_WINDOW_OPEN) {
+ if (pAnotherState) {
+ code = jsonAddColumnField("prevState", pAnotherState->type, pAnotherState->isNull, pAnotherState->pData, event);
+ QUERY_CHECK_CODE(code, lino, _end);
+ } else {
+ code = jsonAddColumnField("prevState", pCurState->type, true, NULL, event);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
+ }
+ code = jsonAddColumnField("curState", pCurState->type, pCurState->isNull, pCurState->pData, event);
+ QUERY_CHECK_CODE(code, lino, _end);
+ if (eventType == SNOTIFY_EVENT_WINDOW_CLOSE) {
+ if (pAnotherState) {
+ code = jsonAddColumnField("nextState", pAnotherState->type, pAnotherState->isNull, pAnotherState->pData, event);
+ QUERY_CHECK_CODE(code, lino, _end);
+ } else {
+ code = jsonAddColumnField("nextState", pCurState->type, true, NULL, event);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
+ }
+
+ item.pJson = event;
+ event = NULL;
+ origSize = taosHashGetSize(sup->pWindowEventHashMap);
+ code = taosHashPut(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE, &item, sizeof(SStreamNotifyEvent));
+ QUERY_CHECK_CODE(code, lino, _end);
+ item.pJson = NULL;
+
+ endTime = taosGetMonoTimestampMs();
+ pNotifyEventStat->notifyEventAddTimes++;
+ pNotifyEventStat->notifyEventAddElems += taosHashGetSize(sup->pWindowEventHashMap) - origSize;
+ pNotifyEventStat->notifyEventAddCostSec += (endTime - startTime) / 1000.0;
+ pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
+
+_end:
+ if (code != TSDB_CODE_SUCCESS) {
+ qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
+ }
+ destroyStreamWindowEvent(&item);
+ if (event != NULL) {
+ cJSON_Delete(event);
+ }
+ return code;
+}
+
+static int32_t addNormalAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
+ SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
+ int32_t code = TSDB_CODE_SUCCESS;
+ int32_t lino = 0;
+ cJSON* event = NULL;
+ int32_t origSize = 0;
+ int64_t startTime = 0;
+ int64_t endTime = 0;
+ SStreamNotifyEvent item = {0};
+
+ QUERY_CHECK_NULL(pSessionKey, code, lino, _end, TSDB_CODE_INVALID_PARA);
+ QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
+ QUERY_CHECK_NULL(sup->windowType, code, lino, _end, TSDB_CODE_INVALID_PARA);
+ QUERY_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
+
+ startTime = taosGetMonoTimestampMs();
+ event = createBasicAggNotifyEvent(sup->windowType, eventType, pSessionKey);
+ QUERY_CHECK_NULL(event, code, lino, _end, terrno);
+
+ item.gid = pSessionKey->groupId;
+ item.win = pSessionKey->win;
+ item.eventType = eventType;
+ item.pJson = event;
+ event = NULL;
+ origSize = taosHashGetSize(sup->pWindowEventHashMap);
+ code = taosHashPut(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE, &item, sizeof(SStreamNotifyEvent));
+ QUERY_CHECK_CODE(code, lino, _end);
+ item.pJson = NULL;
+
+ endTime = taosGetMonoTimestampMs();
+ pNotifyEventStat->notifyEventAddTimes++;
+ pNotifyEventStat->notifyEventAddElems += taosHashGetSize(sup->pWindowEventHashMap) - origSize;
+ pNotifyEventStat->notifyEventAddCostSec += (endTime - startTime) / 1000.0;
+ pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
+
+_end:
+ if (code != TSDB_CODE_SUCCESS) {
+ qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
+ }
+ destroyStreamWindowEvent(&item);
+ if (event != NULL) {
+ cJSON_Delete(event);
+ }
+ return code;
+}
+
+int32_t addIntervalAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
+ SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
+ return addNormalAggNotifyEvent(eventType, pSessionKey, sup, pNotifyEventStat);
+}
+
+int32_t addSessionAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
+ SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
+ return addNormalAggNotifyEvent(eventType, pSessionKey, sup, pNotifyEventStat);
+}
+
+int32_t addCountAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
+ SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
+ return addNormalAggNotifyEvent(eventType, pSessionKey, sup, pNotifyEventStat);
+}
+
+int32_t addAggResultNotifyEvent(const SSDataBlock* pResultBlock, const SArray* pSessionKeys,
+ const SSchemaWrapper* pSchemaWrapper, SStreamNotifyEventSupp* sup,
+ STaskNotifyEventStat* pNotifyEventStat) {
+ int32_t code = TSDB_CODE_SUCCESS;
+ int32_t lino = 0;
+ cJSON* result = NULL;
+ int32_t origSize = 0;
+ int64_t startTime = 0;
+ int64_t endTime = 0;
SStreamNotifyEvent item = {0};
- SColumnInfoData* pWstartCol = NULL;
QUERY_CHECK_NULL(pResultBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
+ QUERY_CHECK_NULL(pSessionKeys, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(pSchemaWrapper, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
+ QUERY_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
qDebug("add %" PRId64 " stream notify results from window agg", pResultBlock->info.rows);
+ startTime = taosGetMonoTimestampMs();
+ origSize = taosHashGetSize(sup->pWindowEventHashMap);
- pWstartCol = taosArrayGet(pResultBlock->pDataBlock, 0);
- for (int32_t i = 0; i< pResultBlock->info.rows; ++i) {
- event = cJSON_CreateObject();
- QUERY_CHECK_NULL(event, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
+ for (int32_t i = 0; i < pResultBlock->info.rows; ++i) {
+ const SSessionKey* pSessionKey = taosArrayGet(pSessionKeys, i);
+ item.gid = pSessionKey->groupId;
+ item.win = pSessionKey->win;
+ item.eventType = SNOTIFY_EVENT_WINDOW_CLOSE;
+ SStreamNotifyEvent* pItem = taosHashGet(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE);
+ if (pItem == NULL) {
+ item.pJson = createBasicAggNotifyEvent(sup->windowType, SNOTIFY_EVENT_WINDOW_CLOSE, pSessionKey);
+ QUERY_CHECK_NULL(item.pJson, code, lino, _end, terrno);
+ if (strcmp(sup->windowType, "Event") == 0) {
+ JSON_CHECK_ADD_ITEM(item.pJson, "triggerCondition", cJSON_CreateNull());
+ } else if (strcmp(sup->windowType, "State") == 0) {
+ JSON_CHECK_ADD_ITEM(item.pJson, "curState", cJSON_CreateNull());
+ JSON_CHECK_ADD_ITEM(item.pJson, "nextState", cJSON_CreateNull());
+ }
+ code = taosHashPut(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE, &item, sizeof(SStreamNotifyEvent));
+ QUERY_CHECK_CODE(code, lino, _end);
+ item.pJson = NULL;
+ pItem = taosHashGet(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE);
+ QUERY_CHECK_NULL(pItem, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
+ }
// convert the result row into json
result = cJSON_CreateObject();
QUERY_CHECK_NULL(result, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
for (int32_t j = 0; j < pSchemaWrapper->nCols; ++j) {
- SSchema *pCol = pSchemaWrapper->pSchema + j;
- SColumnInfoData *pColData = taosArrayGet(pResultBlock->pDataBlock, pCol->colId - 1);
- code = jsonAddColumnField(pCol->name, pColData, i, result);
+ const SSchema* pCol = pSchemaWrapper->pSchema + j;
+ const SColumnInfoData* pColData = taosArrayGet(pResultBlock->pDataBlock, pCol->colId - 1);
+ code = jsonAddColumnField(pCol->name, pColData->info.type, colDataIsNull_s(pColData, i),
+ colDataGetData(pColData, i), result);
QUERY_CHECK_CODE(code, lino, _end);
}
- JSON_CHECK_ADD_ITEM(event, "result", result);
+ JSON_CHECK_ADD_ITEM(pItem->pJson, "result", result);
result = NULL;
-
- item.gid = pResultBlock->info.id.groupId;
- item.skey = *(uint64_t*)colDataGetNumData(pWstartCol, i);
- item.content = cJSON_PrintUnformatted(event);
- code = taosHashPut(sup->pResultHashMap, &item.gid, sizeof(item.gid) + sizeof(item.skey), &item, sizeof(item));
- TSDB_CHECK_CODE(code, lino, _end);
- item.content = NULL;
-
- cJSON_Delete(event);
- event = NULL;
}
+ endTime = taosGetMonoTimestampMs();
+ pNotifyEventStat->notifyEventAddTimes++;
+ pNotifyEventStat->notifyEventAddElems += taosHashGetSize(sup->pWindowEventHashMap) - origSize;
+ pNotifyEventStat->notifyEventAddCostSec += (endTime - startTime) / 1000.0;
+ pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
+
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
@@ -377,8 +667,38 @@ _end:
if (result != NULL) {
cJSON_Delete(result);
}
- if (event != NULL) {
- cJSON_Delete(event);
+ return code;
+}
+
+int32_t addAggDeleteNotifyEvent(const SSDataBlock* pDeleteBlock, SStreamNotifyEventSupp* sup,
+ STaskNotifyEventStat* pNotifyEventStat) {
+ int32_t code = TSDB_CODE_SUCCESS;
+ int32_t lino = 0;
+ SSessionKey sessionKey = {0};
+ SColumnInfoData* pWstartCol = NULL;
+ SColumnInfoData* pWendCol = NULL;
+ SColumnInfoData* pGroupIdCol = NULL;
+
+ QUERY_CHECK_NULL(pDeleteBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
+ QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
+ QUERY_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
+
+ qDebug("add %" PRId64 " stream notify delete events from window agg", pDeleteBlock->info.rows);
+
+ pWstartCol = taosArrayGet(pDeleteBlock->pDataBlock, START_TS_COLUMN_INDEX);
+ pWendCol = taosArrayGet(pDeleteBlock->pDataBlock, END_TS_COLUMN_INDEX);
+ pGroupIdCol = taosArrayGet(pDeleteBlock->pDataBlock, GROUPID_COLUMN_INDEX);
+ for (int32_t i = 0; i < pDeleteBlock->info.rows; ++i) {
+ sessionKey.win.skey = *(int64_t*)colDataGetNumData(pWstartCol, i);
+ sessionKey.win.ekey = *(int64_t*)colDataGetNumData(pWendCol, i);
+ sessionKey.groupId = *(uint64_t*)colDataGetNumData(pGroupIdCol, i);
+ code = addNormalAggNotifyEvent(SNOTIFY_EVENT_WINDOW_INVALIDATION, &sessionKey, sup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
+
+_end:
+ if (code != TSDB_CODE_SUCCESS) {
+ qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
@@ -418,97 +738,42 @@ _end:
return code;
}
-static int32_t streamNotifyFillTableName(const char* tableName, const SStreamNotifyEvent* pEvent,
- const SStreamNotifyEvent* pResult, char** pVal) {
- int32_t code = TSDB_CODE_SUCCESS;
- int32_t lino = 0;
- static const char* prefix = "{\"tableName\":\"";
- uint64_t prefixLen = 0;
- uint64_t nameLen = 0;
- uint64_t eventLen = 0;
- uint64_t resultLen = 0;
- uint64_t valLen = 0;
- char* val = NULL;
- char* p = NULL;
-
- QUERY_CHECK_NULL(tableName, code, lino, _end, TSDB_CODE_INVALID_PARA);
- QUERY_CHECK_NULL(pEvent, code, lino , _end, TSDB_CODE_INVALID_PARA);
- QUERY_CHECK_NULL(pVal, code, lino , _end, TSDB_CODE_INVALID_PARA);
-
- *pVal = NULL;
- prefixLen = strlen(prefix);
- nameLen = strlen(tableName);
- eventLen = strlen(pEvent->content);
-
- if (pResult != NULL) {
- resultLen = strlen(pResult->content);
- valLen = VARSTR_HEADER_SIZE + prefixLen + nameLen + eventLen + resultLen;
- } else {
- valLen = VARSTR_HEADER_SIZE + prefixLen + nameLen + eventLen + 1;
- }
- val = taosMemoryMalloc(valLen);
- QUERY_CHECK_NULL(val, code, lino, _end, terrno);
- varDataSetLen(val, valLen - VARSTR_HEADER_SIZE);
-
- p = varDataVal(val);
- TAOS_STRNCPY(p, prefix, prefixLen);
- p += prefixLen;
- TAOS_STRNCPY(p, tableName, nameLen);
- p += nameLen;
- *(p++) = '\"';
- TAOS_STRNCPY(p, pEvent->content, eventLen);
- *p = ',';
-
- if (pResult != NULL) {
- p += eventLen - 1;
- TAOS_STRNCPY(p, pResult->content, resultLen);
- *p = ',';
- }
- *pVal = val;
- val = NULL;
-
-_end:
- if (code != TSDB_CODE_SUCCESS) {
- qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
- }
- if (val != NULL) {
- taosMemoryFreeClear(val);
- }
- return code;
-}
-
-int32_t buildNotifyEventBlock(const SExecTaskInfo* pTaskInfo, SStreamNotifyEventSupp* sup) {
+int32_t buildNotifyEventBlock(const SExecTaskInfo* pTaskInfo, SStreamNotifyEventSupp* sup,
+ STaskNotifyEventStat* pNotifyEventStat) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
- SColumnInfoData* pEventStrCol = NULL;
int32_t nWindowEvents = 0;
- int32_t nWindowResults = 0;
- char* val = NULL;
+ SColumnInfoData* pEventStrCol = NULL;
+ int64_t startTime = 0;
+ int64_t endTime = 0;
+ void* pIter = NULL;
- if (pTaskInfo == NULL || sup == NULL) {
+ if (pTaskInfo == NULL || sup == NULL || sup->pEventBlock == NULL || pNotifyEventStat == NULL) {
goto _end;
}
QUERY_CHECK_NULL(sup->pEventBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
- blockDataCleanup(sup->pEventBlock);
- nWindowEvents = taosArrayGetSize(sup->pWindowEvents);
- nWindowResults = taosHashGetSize(sup->pResultHashMap);
- qDebug("start to build stream notify event block, nWindowEvents: %d, nWindowResults: %d", nWindowEvents,
- nWindowResults);
- if (nWindowEvents == 0) {
- goto _end;
- }
- code = blockDataEnsureCapacity(sup->pEventBlock, nWindowEvents);
- QUERY_CHECK_CODE(code, lino, _end);
+ startTime = taosGetMonoTimestampMs();
+ blockDataCleanup(sup->pEventBlock);
+ nWindowEvents = taosHashGetSize(sup->pWindowEventHashMap);
+ qDebug("start to build stream notify event block, nWindowEvents: %d", nWindowEvents);
pEventStrCol = taosArrayGet(sup->pEventBlock->pDataBlock, NOTIFY_EVENT_STR_COLUMN_INDEX);
QUERY_CHECK_NULL(pEventStrCol, code, lino, _end, terrno);
- for (int32_t i = 0; i < nWindowEvents; ++i) {
- SStreamNotifyEvent* pResult = NULL;
- SStreamNotifyEvent* pEvent = taosArrayGet(sup->pWindowEvents, i);
- char* tableName = taosHashGet(sup->pTableNameHashMap, &pEvent->gid, sizeof(pEvent->gid));
+ // Append all events content into data block.
+ pIter = taosHashIterate(sup->pWindowEventHashMap, NULL);
+ while (pIter) {
+ const SStreamNotifyEvent* pEvent = (const SStreamNotifyEvent*)pIter;
+ pIter = taosHashIterate(sup->pWindowEventHashMap, pIter);
+ if (pEvent->eventType == SNOTIFY_EVENT_WINDOW_CLOSE && !cJSON_HasObjectItem(pEvent->pJson, "result")) {
+ // current WINDOW_CLOSE event cannot be pushed yet due to watermark
+ continue;
+ }
+
+ // get name of the dest child table
+ char* tableName = taosHashGet(sup->pTableNameHashMap, &pEvent->gid, sizeof(&pEvent->gid));
if (tableName == NULL) {
code = streamNotifyGetDestTableName(pTaskInfo, pEvent->gid, &tableName);
QUERY_CHECK_CODE(code, lino, _end);
@@ -518,32 +783,73 @@ int32_t buildNotifyEventBlock(const SExecTaskInfo* pTaskInfo, SStreamNotifyEvent
tableName = taosHashGet(sup->pTableNameHashMap, &pEvent->gid, sizeof(pEvent->gid));
QUERY_CHECK_NULL(tableName, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
}
- if (pEvent->isEnd) {
- pResult = taosHashGet(sup->pResultHashMap, &pEvent->gid, sizeof(pEvent->gid) + sizeof(pEvent->skey));
- QUERY_CHECK_NULL(pResult, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
- }
- code = streamNotifyFillTableName(tableName, pEvent, pResult, &val);
+ JSON_CHECK_ADD_ITEM(pEvent->pJson, "tableName", cJSON_CreateStringReference(tableName));
+
+ // convert the json object into string and append it into the block
+ char* str = cJSON_PrintUnformatted(pEvent->pJson);
+ QUERY_CHECK_NULL(str, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
+ int32_t len = strlen(str);
+ code = varColSetVarData(pEventStrCol, sup->pEventBlock->info.rows, str, len, false);
+ cJSON_free(str);
QUERY_CHECK_CODE(code, lino, _end);
- code = colDataSetVal(pEventStrCol, i, val, false);
- QUERY_CHECK_CODE(code, lino, _end);
- taosMemoryFreeClear(val);
sup->pEventBlock->info.rows++;
+ code = taosHashRemove(sup->pWindowEventHashMap, pEvent, NOTIFY_EVENT_KEY_SIZE);
+ if (code == TSDB_CODE_NOT_FOUND) {
+ code = TSDB_CODE_SUCCESS;
+ }
+ QUERY_CHECK_CODE(code, lino, _end);
+ if (sup->pEventBlock->info.rows >= sup->pEventBlock->info.capacity) {
+ break;
+ }
}
if (taosHashGetMemSize(sup->pTableNameHashMap) >= NOTIFY_EVENT_NAME_CACHE_LIMIT_MB * 1024 * 1024) {
taosHashClear(sup->pTableNameHashMap);
}
+ endTime = taosGetMonoTimestampMs();
+ if (sup->pEventBlock->info.rows > 0) {
+ pNotifyEventStat->notifyEventPushTimes++;
+ pNotifyEventStat->notifyEventPushElems += sup->pEventBlock->info.rows;
+ pNotifyEventStat->notifyEventPushCostSec += (endTime - startTime) / 1000.0;
+ }
+ pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
+
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
- if (val != NULL) {
- taosMemoryFreeClear(val);
- }
- if (sup != NULL) {
- taosArrayClearEx(sup->pWindowEvents, destroyStreamWindowEvent);
- taosHashClear(sup->pResultHashMap);
+ if (pIter) {
+ taosHashCancelIterate(sup->pWindowEventHashMap, pIter);
+ }
+ return code;
+}
+
+int32_t removeOutdatedNotifyEvents(STimeWindowAggSupp* pTwSup, SStreamNotifyEventSupp* sup,
+ STaskNotifyEventStat* pNotifyEventStat) {
+ int32_t code = TSDB_CODE_SUCCESS;
+ int32_t lino = 0;
+ void* pIter = NULL;
+
+ if (pTwSup || sup == NULL || pNotifyEventStat == NULL) {
+ goto _end;
+ }
+
+ pIter = taosHashIterate(sup->pWindowEventHashMap, NULL);
+ while (pIter) {
+ const SStreamNotifyEvent* pEvent = (const SStreamNotifyEvent*)pIter;
+ pIter = taosHashIterate(sup->pWindowEventHashMap, pIter);
+ if (isOverdue(pEvent->win.ekey, pTwSup)) {
+ code = taosHashRemove(sup->pWindowEventHashMap, pEvent, NOTIFY_EVENT_KEY_SIZE);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
+ }
+
+ pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
+
+_end:
+ if (code != TSDB_CODE_SUCCESS) {
+ qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
diff --git a/source/libs/executor/src/streamintervalsliceoperator.c b/source/libs/executor/src/streamintervalsliceoperator.c
index 44799f193b..cc06f5b693 100644
--- a/source/libs/executor/src/streamintervalsliceoperator.c
+++ b/source/libs/executor/src/streamintervalsliceoperator.c
@@ -87,23 +87,48 @@ static int32_t buildIntervalSliceResult(SOperatorInfo* pOperator, SSDataBlock**
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
uint16_t opType = pOperator->operatorType;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
-
+ SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
+ STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
+ bool addNotifyEvent = false;
+ addNotifyEvent = IS_NORMAL_INTERVAL_OP(pOperator) &&
+ BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
doBuildDeleteResultImpl(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pState, pInfo->pDelWins, &pInfo->delIndex,
pInfo->pDelRes);
if (pInfo->pDelRes->info.rows != 0) {
// process the rest of the data
printDataBlock(pInfo->pDelRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
+ if (addNotifyEvent) {
+ code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
(*ppRes) = pInfo->pDelRes;
return code;
}
- doBuildStreamIntervalResult(pOperator, pInfo->streamAggSup.pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
+ doBuildStreamIntervalResult(pOperator, pInfo->streamAggSup.pState, pInfo->binfo.pRes, &pInfo->groupResInfo,
+ addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
if (pInfo->binfo.pRes->info.rows != 0) {
printDataBlock(pInfo->binfo.pRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
+ if (addNotifyEvent) {
+ code = addAggResultNotifyEvent(pInfo->binfo.pRes, pNotifySup->pSessionKeys,
+ pTaskInfo->streamInfo.notifyResultSchema, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
(*ppRes) = pInfo->binfo.pRes;
goto _end;
}
+ code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
+ printDataBlock(pNotifySup->pEventBlock, getStreamOpName(opType), GET_TASKID(pTaskInfo));
+ (*ppRes) = pNotifySup->pEventBlock;
+ return code;
+ }
+
+ code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
@@ -316,6 +341,14 @@ static int32_t doStreamIntervalSliceAggImpl(SOperatorInfo* pOperator, SSDataBloc
code = setIntervalSliceOutputBuf(&curPoint, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset);
QUERY_CHECK_CODE(code, lino, _end);
+ if (winCode != TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator) &&
+ BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {
+ SSessionKey key = {.win = curWin, .groupId = groupId};
+ code = addIntervalAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &key, &pInfo->basic.notifyEventSup,
+ pTaskInfo->streamInfo.pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
+
resetIntervalSliceFunctionKey(pSup->pCtx, numOfOutput);
if (pInfo->hasInterpoFunc && IS_VALID_WIN_KEY(prevPoint.winKey.win.skey) && curPoint.winKey.win.skey != curTs) {
doStreamSliceInterpolation(prevPoint.pLastRow, curPoint.winKey.win.skey, curTs, pBlock, startPos, &pOperator->exprSupp, INTERVAL_SLICE_START, pInfo->pOffsetInfo);
@@ -652,8 +685,9 @@ int32_t createStreamIntervalSliceOperatorInfo(SOperatorInfo* downstream, SPhysiN
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamIntervalSliceReleaseState, streamIntervalSliceReloadState);
- code = initStreamBasicInfo(&pInfo->basic);
+ code = initStreamBasicInfo(&pInfo->basic, pOperator);
QUERY_CHECK_CODE(code, lino, _error);
+
if (downstream) {
code = initIntervalSliceDownStream(downstream, &pInfo->streamAggSup, pPhyNode->type, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic, &pInfo->interval, pInfo->hasInterpoFunc);
diff --git a/source/libs/executor/src/streamtimesliceoperator.c b/source/libs/executor/src/streamtimesliceoperator.c
index 4fe8efe397..681e07f452 100644
--- a/source/libs/executor/src/streamtimesliceoperator.c
+++ b/source/libs/executor/src/streamtimesliceoperator.c
@@ -150,7 +150,6 @@ void destroyStreamTimeSliceOperatorInfo(void* param) {
&pInfo->groupResInfo);
pInfo->pOperator = NULL;
}
- destroyStreamBasicInfo(&pInfo->basic);
colDataDestroy(&pInfo->twAggSup.timeWindowData);
destroyStreamAggSupporter(&pInfo->streamAggSup);
resetPrevAndNextWindow(pInfo->pFillSup);
@@ -2202,7 +2201,7 @@ int32_t createStreamTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamTimeSliceReleaseState, streamTimeSliceReloadState);
- code = initStreamBasicInfo(&pInfo->basic);
+ code = initStreamBasicInfo(&pInfo->basic, pOperator);
QUERY_CHECK_CODE(code, lino, _error);
if (downstream) {
code = initTimeSliceDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c
index 031d2e8bdc..fbb55301cd 100644
--- a/source/libs/executor/src/streamtimewindowoperator.c
+++ b/source/libs/executor/src/streamtimewindowoperator.c
@@ -32,11 +32,6 @@
#define IS_MID_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL)
#define IS_FINAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
-#define IS_NORMAL_SESSION_OP(op) \
- ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || \
- (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
-
-#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE)
#define DEAULT_DELETE_MARK INT64_MAX
#define STREAM_INTERVAL_OP_STATE_NAME "StreamIntervalHistoryState"
@@ -480,6 +475,8 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) {
false);
pInfo->pOperator = NULL;
}
+
+ destroyStreamBasicInfo(&pInfo->basic);
cleanupAggSup(&pInfo->aggSup);
clearGroupResInfo(&pInfo->groupResInfo);
taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos);
@@ -917,7 +914,7 @@ int32_t getOutputBuf(void* pState, SRowBuffPos* pPos, SResultRow** pResult, SSta
}
void buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,
- SGroupResInfo* pGroupResInfo) {
+ SGroupResInfo* pGroupResInfo, SArray* pSessionKeys) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@@ -991,6 +988,14 @@ void buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBl
}
}
+ if (pSessionKeys) {
+ SSessionKey key = {.groupId = groupId, .win = pRow->win};
+ for (int32_t j = 0; j < pRow->numOfRows; ++j) {
+ const void* px = taosArrayPush(pSessionKeys, &key);
+ QUERY_CHECK_NULL(px, code, lino, _end, terrno);
+ }
+ }
+
pBlock->info.rows += pRow->numOfRows;
}
@@ -1005,19 +1010,20 @@ _end:
}
void doBuildStreamIntervalResult(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,
- SGroupResInfo* pGroupResInfo) {
+ SGroupResInfo* pGroupResInfo, SArray* pSessionKeys) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
// set output datablock version
pBlock->info.version = pTaskInfo->version;
blockDataCleanup(pBlock);
+ taosArrayClear(pSessionKeys);
if (!hasRemainResults(pGroupResInfo)) {
return;
}
// clear the existed group id
pBlock->info.id.groupId = 0;
- buildDataBlockFromGroupRes(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo);
+ buildDataBlockFromGroupRes(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo, pSessionKeys);
}
static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
@@ -1150,6 +1156,14 @@ static int32_t doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS
pSup->rowEntryInfoOffset, &pInfo->aggSup, &pInfo->stateStore, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
+ if (winCode != TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator) &&
+ BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {
+ SSessionKey key = {.win = nextWin, .groupId = groupId};
+ code = addIntervalAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &key, &pInfo->basic.notifyEventSup,
+ pTaskInfo->streamInfo.pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
+
pResult = (SResultRow*)pResPos->pRowBuff;
if (IS_FINAL_INTERVAL_OP(pOperator)) {
@@ -1371,7 +1385,10 @@ int32_t doStreamIntervalEncodeOpState(void** buf, int32_t len, SOperatorInfo* pO
// 5.dataVersion
tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);
- // 6.checksum
+ // 6.basicInfo
+ tlen += encodeStreamBasicInfo(buf, &pInfo->basic);
+
+ // 7.checksum
if (buf) {
uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));
tlen += taosEncodeFixedU32(buf, cksum);
@@ -1387,18 +1404,20 @@ void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
int32_t lino = 0;
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ void* pDataEnd = POINTER_SHIFT(buf, len);
if (!pInfo) {
code = TSDB_CODE_FAILED;
QUERY_CHECK_CODE(code, lino, _end);
}
- // 6.checksum
+ // 7.checksum
int32_t dataLen = len - sizeof(uint32_t);
void* pCksum = POINTER_SHIFT(buf, dataLen);
if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {
code = TSDB_CODE_FAILED;
QUERY_CHECK_CODE(code, lino, _end);
}
+ pDataEnd = pCksum;
// 1.pResultRowHashTable
int32_t mapSize = 0;
@@ -1454,6 +1473,12 @@ void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
// 5.dataVersion
buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);
+ // 6.basicInfo
+ if (buf < pDataEnd) {
+ code = decodeStreamBasicInfo(&buf, &pInfo->basic);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
+
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
@@ -1503,8 +1528,12 @@ _end:
static int32_t buildIntervalResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
int32_t code = TSDB_CODE_SUCCESS;
+ int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
uint16_t opType = pOperator->operatorType;
+ SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
+ STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
+ bool addNotifyEvent = false;
// check if query task is closed or not
if (isTaskKilled(pTaskInfo)) {
@@ -1512,6 +1541,8 @@ static int32_t buildIntervalResult(SOperatorInfo* pOperator, SSDataBlock** ppRes
return code;
}
+ addNotifyEvent = IS_NORMAL_INTERVAL_OP(pOperator) &&
+ BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
if (IS_FINAL_INTERVAL_OP(pOperator)) {
doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
if (pInfo->pPullDataRes->info.rows != 0) {
@@ -1526,17 +1557,42 @@ static int32_t buildIntervalResult(SOperatorInfo* pOperator, SSDataBlock** ppRes
if (pInfo->pDelRes->info.rows != 0) {
// process the rest of the data
printDataBlock(pInfo->pDelRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
+ if (addNotifyEvent) {
+ code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
(*ppRes) = pInfo->pDelRes;
return code;
}
- doBuildStreamIntervalResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
+ doBuildStreamIntervalResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo,
+ addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
if (pInfo->binfo.pRes->info.rows != 0) {
printDataBlock(pInfo->binfo.pRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
+ if (addNotifyEvent) {
+ code = addAggResultNotifyEvent(pInfo->binfo.pRes, pNotifySup->pSessionKeys,
+ pTaskInfo->streamInfo.notifyResultSchema, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
(*ppRes) = pInfo->binfo.pRes;
return code;
}
+ code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
+ printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+ (*ppRes) = pNotifySup->pEventBlock;
+ return code;
+ }
+
+ code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+
+_end:
+ if (code != TSDB_CODE_SUCCESS) {
+ qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
+ }
(*ppRes) = NULL;
return code;
}
@@ -2023,7 +2079,7 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN
pInfo->pState->pResultRowStore.resultRowGet = getResultRowFromBuf;
pInfo->pState->pResultRowStore.resultRowPut = putResultRowToBuf;
pInfo->pState->pExprSupp = &pOperator->exprSupp;
-
+
code =
pAPI->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize,
compareTs, pInfo->pState, pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo),
@@ -2069,6 +2125,10 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
}
setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState);
+
+ code = initStreamBasicInfo(&pInfo->basic, pOperator);
+ QUERY_CHECK_CODE(code, lino, _error);
+
if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL ||
pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {
pInfo->basic.primaryPkIndex = -1;
@@ -2120,6 +2180,8 @@ void destroyStreamSessionAggOperatorInfo(void* param) {
&pInfo->groupResInfo);
pInfo->pOperator = NULL;
}
+
+ destroyStreamBasicInfo(&pInfo->basic);
destroyStreamAggSupporter(&pInfo->streamAggSup);
cleanupExprSupp(&pInfo->scalarSupp);
clearGroupResInfo(&pInfo->groupResInfo);
@@ -2248,8 +2310,8 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, in
}
if (stateType == STREAM_STATE_BUFF_SORT) {
- pSup->pState->pFileState = NULL;
- code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize,
+ pSup->pState->pFileState = NULL;
+ code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize,
funResSize, sesionTs, pSup->pState, pTwAggSup->deleteMark, taskIdStr,
pHandle->checkpointId, stateType, &pSup->pState->pFileState);
} else if (stateType == STREAM_STATE_BUFF_HASH_SORT || stateType == STREAM_STATE_BUFF_HASH_SEARCH) {
@@ -2309,24 +2371,23 @@ bool inWinRange(STimeWindow* range, STimeWindow* cur) {
void clearOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { pAPI->streamStateClearBuff(pState, pPos); }
int32_t setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
- SResultWindowInfo* pCurWin) {
+ SResultWindowInfo* pCurWin, int32_t* pWinCode) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
pCurWin->sessionWin.groupId = groupId;
pCurWin->sessionWin.win.skey = startTs;
pCurWin->sessionWin.win.ekey = endTs;
int32_t size = pAggSup->resultRowSize;
- int32_t winCode = TSDB_CODE_SUCCESS;
code = pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, pAggSup->gap,
- (void**)&pCurWin->pStatePos, &size, &winCode);
+ (void**)&pCurWin->pStatePos, &size, pWinCode);
QUERY_CHECK_CODE(code, lino, _end);
- if (winCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->sessionWin.win)) {
- winCode = TSDB_CODE_FAILED;
+ if (*pWinCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->sessionWin.win)) {
+ *pWinCode = TSDB_CODE_FAILED;
clearOutputBuf(pAggSup->pState, pCurWin->pStatePos, &pAggSup->pSessionAPI->stateStore);
}
- if (winCode == TSDB_CODE_SUCCESS) {
+ if (*pWinCode == TSDB_CODE_SUCCESS) {
pCurWin->isOutput = true;
if (pCurWin->pStatePos->needFree) {
pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->sessionWin);
@@ -2692,9 +2753,17 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
continue;
}
SResultWindowInfo winInfo = {0};
- code = setSessionOutputBuf(pAggSup, startTsCols[i], endTsCols[i], groupId, &winInfo);
+ int32_t winCode = TSDB_CODE_SUCCESS;
+ code = setSessionOutputBuf(pAggSup, startTsCols[i], endTsCols[i], groupId, &winInfo, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
+ if (winCode != TSDB_CODE_SUCCESS && IS_NORMAL_SESSION_OP(pOperator) &&
+ BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {
+ code = addSessionAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &winInfo.sessionWin, &pInfo->basic.notifyEventSup,
+ pTaskInfo->streamInfo.pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
+
// coverity scan error
if (!winInfo.pStatePos) {
continue;
@@ -2908,7 +2977,8 @@ static int32_t rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray,
if (winCode == TSDB_CODE_SUCCESS && inWinRange(&pWinKey->win, &childWin.sessionWin.win)) {
if (num == 0) {
- code = setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin);
+ code = setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin,
+ &winCode);
QUERY_CHECK_CODE(code, lino, _end);
parentWin.sessionWin = childWin.sessionWin;
@@ -3051,7 +3121,7 @@ void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayL
}
int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,
- SGroupResInfo* pGroupResInfo) {
+ SGroupResInfo* pGroupResInfo, SArray* pSessionKeys) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@@ -3131,6 +3201,13 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa
}
}
+ if (pSessionKeys) {
+ for (int32_t j = 0; j < pRow->numOfRows; ++j) {
+ const void* px = taosArrayPush(pSessionKeys, pKey);
+ QUERY_CHECK_NULL(px, code, lino, _end, terrno);
+ }
+ }
+
pBlock->info.dataLoad = 1;
pBlock->info.rows += pRow->numOfRows;
}
@@ -3144,7 +3221,8 @@ _end:
return code;
}
-void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock) {
+void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock,
+ SArray* pSessionKeys) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@@ -3152,6 +3230,7 @@ void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo*
pBlock->info.version = pTaskInfo->version;
blockDataCleanup(pBlock);
+ taosArrayClear(pSessionKeys);
if (!hasRemainResults(pGroupResInfo)) {
cleanupGroupResInfo(pGroupResInfo);
goto _end;
@@ -3159,7 +3238,7 @@ void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo*
// clear the existed group id
pBlock->info.id.groupId = 0;
- code = buildSessionResultDataBlock(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo);
+ code = buildSessionResultDataBlock(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo, pSessionKeys);
QUERY_CHECK_CODE(code, lino, _end);
if (pBlock->info.rows == 0) {
@@ -3174,23 +3253,60 @@ _end:
static int32_t buildSessionResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
int32_t code = TSDB_CODE_SUCCESS;
+ int32_t lino = 0;
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SOptrBasicInfo* pBInfo = &pInfo->binfo;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
+ STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
+ bool addNotifyEvent = false;
+ addNotifyEvent = IS_NORMAL_SESSION_OP(pOperator) &&
+ BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
if (pInfo->pDelRes->info.rows > 0) {
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+ if (addNotifyEvent) {
+ code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
(*ppRes) = pInfo->pDelRes;
return code;
}
- doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes);
+ doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes,
+ addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
if (pBInfo->pRes->info.rows > 0) {
printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+ if (addNotifyEvent) {
+ // Adjust the window end time based on the Session Window gap
+ for (int32_t i = 0; i < taosArrayGetSize(pNotifySup->pSessionKeys); ++i) {
+ SSessionKey* pKey = taosArrayGet(pNotifySup->pSessionKeys, i);
+ pKey->win.ekey += pAggSup->gap;
+ }
+ code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,
+ pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
(*ppRes) = pBInfo->pRes;
return code;
}
+
+ code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
+ printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+ (*ppRes) = pNotifySup->pEventBlock;
+ return code;
+ }
+
+ code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+
+_end:
+ if (code != TSDB_CODE_SUCCESS) {
+ qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
+ }
(*ppRes) = NULL;
return code;
}
@@ -3295,7 +3411,10 @@ int32_t doStreamSessionEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOp
// 4.dataVersion
tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);
- // 5.checksum
+ // 5.basicInfo
+ tlen += encodeStreamBasicInfo(buf, &pInfo->basic);
+
+ // 6.checksum
if (isParent) {
if (buf) {
uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));
@@ -3313,13 +3432,14 @@ int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpe
int32_t lino = 0;
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ void* pDataEnd = POINTER_SHIFT(buf, len);
if (!pInfo) {
code = TSDB_CODE_FAILED;
QUERY_CHECK_CODE(code, lino, _end);
}
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
- // 5.checksum
+ // 6.checksum
if (isParent) {
int32_t dataLen = len - sizeof(uint32_t);
void* pCksum = POINTER_SHIFT(buf, dataLen);
@@ -3328,6 +3448,7 @@ int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpe
code = TSDB_CODE_FAILED;
QUERY_CHECK_CODE(code, lino, _end);
}
+ pDataEnd = pCksum;
}
// 1.streamAggSup.pResultRows
@@ -3366,6 +3487,12 @@ int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpe
(*ppBuf) = buf;
}
+ // 5.basicInfo
+ if (buf < pDataEnd) {
+ code = decodeStreamBasicInfo(&buf, &pInfo->basic);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
+
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
@@ -3926,6 +4053,9 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState);
+ code = initStreamBasicInfo(&pInfo->basic, pOperator);
+ QUERY_CHECK_CODE(code, lino, _error);
+
if (downstream) {
pInfo->basic.primaryPkIndex = -1;
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
@@ -4162,6 +4292,10 @@ int32_t createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhys
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionSemiReloadState);
}
+
+ code = initStreamBasicInfo(&pInfo->basic, pOperator);
+ QUERY_CHECK_CODE(code, lino, _error);
+
setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), pPhyNode->type, false, OP_NOT_OPENED, pInfo,
pTaskInfo);
@@ -4230,6 +4364,8 @@ void destroyStreamStateOperatorInfo(void* param) {
&pInfo->groupResInfo);
pInfo->pOperator = NULL;
}
+
+ destroyStreamBasicInfo(&pInfo->basic);
destroyStreamAggSupporter(&pInfo->streamAggSup);
clearGroupResInfo(&pInfo->groupResInfo);
taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos);
@@ -4284,12 +4420,39 @@ bool compareWinStateKey(SStateKeys* left, SStateKeys* right) {
return compareVal(left->pData, right);
}
+static void getNextStateWin(const SStreamAggSupporter* pAggSup, SStateWindowInfo* pNextWin, bool asc) {
+ SStreamStateCur* pCur = NULL;
+
+ if (pAggSup == NULL || pNextWin == NULL) {
+ return;
+ }
+
+ if (asc)
+ pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin);
+ else
+ pCur = pAggSup->stateStore.streamStateSessionSeekKeyPrev(pAggSup->pState, &pNextWin->winInfo.sessionWin);
+ int32_t nextSize = pAggSup->resultRowSize;
+ int32_t winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin,
+ (void**)&pNextWin->winInfo.pStatePos, &nextSize);
+ if (winCode != TSDB_CODE_SUCCESS) {
+ SET_SESSION_WIN_INVALID(pNextWin->winInfo);
+ } else {
+ pNextWin->pStateKey =
+ (SStateKeys*)((char*)pNextWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));
+ pNextWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);
+ pNextWin->pStateKey->type = pAggSup->stateKeyType;
+ pNextWin->pStateKey->pData = (char*)pNextWin->pStateKey + sizeof(SStateKeys);
+ pNextWin->pStateKey->isNull = false;
+ pNextWin->winInfo.isOutput = true;
+ }
+ pAggSup->stateStore.streamStateFreeCur(pCur);
+}
+
int32_t getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SStateWindowInfo* pCurWin,
SStateWindowInfo* pNextWin) {
- int32_t code = TSDB_CODE_SUCCESS;
- int32_t lino = 0;
- SStreamStateCur* pCur = NULL;
- int32_t size = pAggSup->resultRowSize;
+ int32_t code = TSDB_CODE_SUCCESS;
+ int32_t lino = 0;
+ int32_t size = pAggSup->resultRowSize;
pCurWin->winInfo.sessionWin.groupId = pKey->groupId;
pCurWin->winInfo.sessionWin.win.skey = pKey->win.skey;
pCurWin->winInfo.sessionWin.win.ekey = pKey->win.ekey;
@@ -4313,24 +4476,9 @@ int32_t getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey,
pCurWin->winInfo.sessionWin.win.ekey);
pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin;
- pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin);
- int32_t nextSize = pAggSup->resultRowSize;
- int32_t winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin,
- (void**)&pNextWin->winInfo.pStatePos, &nextSize);
- if (winCode != TSDB_CODE_SUCCESS) {
- SET_SESSION_WIN_INVALID(pNextWin->winInfo);
- } else {
- pNextWin->pStateKey =
- (SStateKeys*)((char*)pNextWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));
- pNextWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);
- pNextWin->pStateKey->type = pAggSup->stateKeyType;
- pNextWin->pStateKey->pData = (char*)pNextWin->pStateKey + sizeof(SStateKeys);
- pNextWin->pStateKey->isNull = false;
- pNextWin->winInfo.isOutput = true;
- }
+ getNextStateWin(pAggSup, pNextWin, true);
_end:
- pAggSup->stateStore.streamStateFreeCur(pCur);
qDebug("===stream===get state next win buff. skey:%" PRId64 ", endkey:%" PRId64,
pNextWin->winInfo.sessionWin.win.skey, pNextWin->winInfo.sessionWin.win.ekey);
if (code != TSDB_CODE_SUCCESS) {
@@ -4340,9 +4488,8 @@ _end:
}
int32_t setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData,
- SStateWindowInfo* pCurWin, SStateWindowInfo* pNextWin) {
- int32_t size = pAggSup->resultRowSize;
- SStreamStateCur* pCur = NULL;
+ SStateWindowInfo* pCurWin, SStateWindowInfo* pNextWin, int32_t* pWinCode) {
+ int32_t size = pAggSup->resultRowSize;
pCurWin->winInfo.sessionWin.groupId = groupId;
pCurWin->winInfo.sessionWin.win.skey = ts;
pCurWin->winInfo.sessionWin.win.ekey = ts;
@@ -4390,29 +4537,16 @@ int32_t setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group
}
}
+ *pWinCode = winCode;
+
qDebug("===stream===set state cur win buff. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey,
pCurWin->winInfo.sessionWin.win.ekey);
pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin;
- pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin);
- int32_t nextSize = pAggSup->resultRowSize;
- winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin,
- (void**)&pNextWin->winInfo.pStatePos, &nextSize);
- if (winCode != TSDB_CODE_SUCCESS) {
- SET_SESSION_WIN_INVALID(pNextWin->winInfo);
- } else {
- pNextWin->pStateKey =
- (SStateKeys*)((char*)pNextWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));
- pNextWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);
- pNextWin->pStateKey->type = pAggSup->stateKeyType;
- pNextWin->pStateKey->pData = (char*)pNextWin->pStateKey + sizeof(SStateKeys);
- pNextWin->pStateKey->isNull = false;
- pNextWin->winInfo.isOutput = true;
- }
+ getNextStateWin(pAggSup, pNextWin, true);
qDebug("===stream===set state next win buff. skey:%" PRId64 ", endkey:%" PRId64,
pNextWin->winInfo.sessionWin.win.skey, pNextWin->winInfo.sessionWin.win.ekey);
_end:
- pAggSup->stateStore.streamStateFreeCur(pCur);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
@@ -4472,7 +4606,7 @@ static bool isWinResult(SSessionKey* pKey, SSHashObj* pSeUpdate, SSHashObj* pRes
if (tSimpleHashGet(pSeUpdate, &checkKey, sizeof(SSessionKey)) != NULL) {
return true;
}
-
+
if (tSimpleHashGet(pResults, &checkKey, sizeof(SSessionKey)) != NULL) {
return true;
}
@@ -4493,6 +4627,8 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
SResultRow* pResult = NULL;
int32_t winRows = 0;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
+ SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
+ STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);
pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;
@@ -4528,9 +4664,31 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
bool allEqual = true;
SStateWindowInfo curWin = {0};
SStateWindowInfo nextWin = {0};
- code = setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin);
+ int32_t winCode = TSDB_CODE_SUCCESS;
+ code = setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
+ if (winCode != TSDB_CODE_SUCCESS && pTaskInfo->streamInfo.eventTypes) {
+ SStateWindowInfo prevWin = {.winInfo.sessionWin = curWin.winInfo.sessionWin};
+ getNextStateWin(pAggSup, &prevWin, false);
+ qDebug("===stream===get state prev win buff. skey:%" PRId64 ", endkey:%" PRId64,
+ prevWin.winInfo.sessionWin.win.skey, prevWin.winInfo.sessionWin.win.ekey);
+ releaseOutputBuf(pAggSup->pState, prevWin.winInfo.pStatePos, &pAPI->stateStore);
+ // For ordered data, the previous window's closure did not record the corresponding state values, so they need to
+ // be added here.
+ if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE) &&
+ IS_VALID_SESSION_WIN(prevWin.winInfo)) {
+ code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_CLOSE, &prevWin.winInfo.sessionWin, prevWin.pStateKey,
+ curWin.pStateKey, true, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
+ if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {
+ code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &curWin.winInfo.sessionWin, curWin.pStateKey,
+ prevWin.pStateKey, false, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
+ }
+
if (isWinResult(&nextWin.winInfo.sessionWin, pSeUpdated, pAggSup->pResultRows) == false) {
releaseOutputBuf(pAggSup->pState, nextWin.winInfo.pStatePos, &pAPI->stateStore);
}
@@ -4578,6 +4736,14 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));
QUERY_CHECK_CODE(code, lino, _end);
}
+
+ // If this is a windown recalculation, add the corresponding state values here since the next window may not require
+ // recalculation.
+ if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE)) {
+ code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_CLOSE, &curWin.winInfo.sessionWin, curWin.pStateKey,
+ nextWin.pStateKey, false, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
}
_end:
@@ -4621,7 +4787,10 @@ int32_t doStreamStateEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper
// 4.dataVersion
tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);
- // 5.checksum
+ // 5.basicInfo
+ tlen += encodeStreamBasicInfo(buf, &pInfo->basic);
+
+ // 6.checksum
if (isParent) {
if (buf) {
uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));
@@ -4640,12 +4809,13 @@ int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
int32_t lino = 0;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ void* pDataEnd = POINTER_SHIFT(buf, len);
if (!pInfo) {
code = TSDB_CODE_FAILED;
QUERY_CHECK_CODE(code, lino, _end);
}
- // 5.checksum
+ // 6.checksum
if (isParent) {
int32_t dataLen = len - sizeof(uint32_t);
void* pCksum = POINTER_SHIFT(buf, dataLen);
@@ -4654,6 +4824,7 @@ int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
code = TSDB_CODE_FAILED;
QUERY_CHECK_CODE(code, lino, _end);
}
+ pDataEnd = pCksum;
}
// 1.streamAggSup.pResultRows
@@ -4693,6 +4864,12 @@ int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
(*ppBuf) = buf;
}
+ // 5.basicInfo
+ if (buf < pDataEnd) {
+ code = decodeStreamBasicInfo(&buf, &pInfo->basic);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
+
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
@@ -4720,23 +4897,53 @@ void doStreamStateSaveCheckpoint(SOperatorInfo* pOperator) {
static int32_t buildStateResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
int32_t code = TSDB_CODE_SUCCESS;
+ int32_t lino = 0;
SStreamStateAggOperatorInfo* pInfo = pOperator->info;
SOptrBasicInfo* pBInfo = &pInfo->binfo;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
-
+ SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
+ STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
+ bool addNotifyEvent = false;
+ addNotifyEvent = BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
if (pInfo->pDelRes->info.rows > 0) {
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+ if (addNotifyEvent) {
+ code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
(*ppRes) = pInfo->pDelRes;
return code;
}
- doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes);
+ doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes,
+ addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
if (pBInfo->pRes->info.rows > 0) {
printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+ if (addNotifyEvent) {
+ code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,
+ pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ }
(*ppRes) = pBInfo->pRes;
return code;
}
+
+ code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+ if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
+ printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+ (*ppRes) = pNotifySup->pEventBlock;
+ return code;
+ }
+
+ code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
+ QUERY_CHECK_CODE(code, lino, _end);
+
+_end:
+ if (code != TSDB_CODE_SUCCESS) {
+ qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
+ }
(*ppRes) = NULL;
return code;
}
@@ -5122,6 +5329,10 @@ int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAggNext, NULL, destroyStreamStateOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState);
+
+ code = initStreamBasicInfo(&pInfo->basic, pOperator);
+ QUERY_CHECK_CODE(code, lino, _error);
+
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
QUERY_CHECK_CODE(code, lino, _error);
@@ -5331,8 +5542,9 @@ _end:
return code;
}
-static int32_t createStreamSingleIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,
- SReadHandle* pHandle, SOperatorInfo** pOptrInfo) {
+static int32_t createStreamSingleIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
+ SExecTaskInfo* pTaskInfo, SReadHandle* pHandle,
+ SOperatorInfo** pOptrInfo) {
QRY_PARAM_CHECK(pOptrInfo);
int32_t code = TSDB_CODE_SUCCESS;
@@ -5455,6 +5667,9 @@ static int32_t createStreamSingleIntervalOperatorInfo(SOperatorInfo* downstream,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState);
+ code = initStreamBasicInfo(&pInfo->basic, pOperator);
+ QUERY_CHECK_CODE(code, lino, _error);
+
pInfo->recvGetAll = false;
code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);
diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c
index 161c5f7ca7..0e7a719c2c 100644
--- a/source/libs/nodes/src/nodesCloneFuncs.c
+++ b/source/libs/nodes/src/nodesCloneFuncs.c
@@ -510,6 +510,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) {
COPY_SCALAR_FIELD(paraTablesSort);
COPY_SCALAR_FIELD(smallDataTsSort);
COPY_SCALAR_FIELD(needSplit);
+ COPY_SCALAR_FIELD(noPseudoRefAfterGrp);
return TSDB_CODE_SUCCESS;
}
@@ -849,11 +850,6 @@ static int32_t physiDispatchCopy(const SDataDispatcherNode* pSrc, SDataDispatche
return TSDB_CODE_SUCCESS;
}
-static int32_t physiInserterCopy(const SDataInserterNode* pSrc, SDataInserterNode* pDst) {
- COPY_BASE_OBJECT_FIELD(sink, dataSinkNodeCopy);
- return TSDB_CODE_SUCCESS;
-}
-
static int32_t physiQueryInserterCopy(const SQueryInserterNode* pSrc, SQueryInserterNode* pDst) {
COPY_BASE_OBJECT_FIELD(sink, dataSinkNodeCopy);
CLONE_NODE_LIST_FIELD(pCols);
@@ -1135,9 +1131,6 @@ int32_t nodesCloneNode(const SNode* pNode, SNode** ppNode) {
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
code = physiDispatchCopy((const SDataDispatcherNode*)pNode, (SDataDispatcherNode*)pDst);
break;
- //case QUERY_NODE_PHYSICAL_PLAN_INSERT:
- // code = physiInserterCopy((const SDataInserterNode*)pNode, (SDataInserterNode*)pDst);
- // break;
case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT:
code = physiQueryInserterCopy((const SQueryInserterNode*)pNode, (SQueryInserterNode*)pDst);
break;
diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c
index 7006979216..9473e75642 100644
--- a/source/libs/nodes/src/nodesUtilFuncs.c
+++ b/source/libs/nodes/src/nodesUtilFuncs.c
@@ -2000,7 +2000,7 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_PHYSICAL_PLAN_INSERT: {
SDataInserterNode* pSink = (SDataInserterNode*)pNode;
destroyDataSinkNode((SDataSinkNode*)pSink);
- taosMemoryFreeClear(pSink->pData);
+ taosMemFreeClear(pSink->pData);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: {
diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c
index 676aed4464..bd463bfd9d 100644
--- a/source/libs/parser/src/parInsertSml.c
+++ b/source/libs/parser/src/parInsertSml.c
@@ -403,3 +403,18 @@ end:
}
return code;
}
+
+int32_t smlBuildOutputRaw(SQuery* handle, SHashObj* pVgHash) {
+ int32_t lino = 0;
+ int32_t code = 0;
+
+ SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)(handle)->pRoot;
+ code = insBuildVgDataBlocks(pVgHash, pStmt->pVgDataBlocks, &pStmt->pDataBlocks, false);
+ TSDB_CHECK_CODE(code, lino, end);
+
+ end:
+ if (code != 0) {
+ uError("%s failed at %d since %s", __func__, lino, tstrerror(code));
+ }
+ return code;
+}
diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c
index 74fac463f1..9f9077d1b6 100644
--- a/source/libs/parser/src/parInsertStmt.c
+++ b/source/libs/parser/src/parInsertStmt.c
@@ -834,6 +834,10 @@ int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind,
goto _return;
}
+ if(boundInfo->pColIndex[c]==0){
+ pCol->cflag |= COL_IS_KEY;
+ }
+
if (bind[c].num != rowNum) {
code = buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same");
goto _return;
@@ -863,7 +867,7 @@ int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind,
}
}
- qDebug("stmt all %d columns bind %d rows data", boundInfo->numOfBound, rowNum);
+ qDebug("stmt2 all %d columns bind %d rows data as col format", boundInfo->numOfBound, rowNum);
_return:
diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c
index ed1f498a32..1931f0803d 100644
--- a/source/libs/parser/src/parInsertUtil.c
+++ b/source/libs/parser/src/parInsertUtil.c
@@ -483,7 +483,7 @@ static int32_t fillVgroupDataCxt(STableDataCxt* pTableCxt, SVgroupDataCxt* pVgCx
return code;
}
-static int32_t createVgroupDataCxt(STableDataCxt* pTableCxt, SHashObj* pVgroupHash, SArray* pVgroupList,
+static int32_t createVgroupDataCxt(int32_t vgId, SHashObj* pVgroupHash, SArray* pVgroupList,
SVgroupDataCxt** pOutput) {
SVgroupDataCxt* pVgCxt = taosMemoryCalloc(1, sizeof(SVgroupDataCxt));
if (NULL == pVgCxt) {
@@ -495,7 +495,7 @@ static int32_t createVgroupDataCxt(STableDataCxt* pTableCxt, SHashObj* pVgroupHa
return terrno;
}
- pVgCxt->vgId = pTableCxt->pMeta->vgId;
+ pVgCxt->vgId = vgId;
int32_t code = taosHashPut(pVgroupHash, &pVgCxt->vgId, sizeof(pVgCxt->vgId), &pVgCxt, POINTER_BYTES);
if (TSDB_CODE_SUCCESS == code) {
if (NULL == taosArrayPush(pVgroupList, &pVgCxt)) {
@@ -642,7 +642,7 @@ int32_t insAppendStmtTableDataCxt(SHashObj* pAllVgHash, STableColsData* pTbData,
if (NULL == pp) {
pp = taosHashGet(pBuildInfo->pVgroupHash, &vgId, sizeof(vgId));
if (NULL == pp) {
- code = createVgroupDataCxt(pTbCtx, pBuildInfo->pVgroupHash, pBuildInfo->pVgroupList, &pVgCxt);
+ code = createVgroupDataCxt(vgId, pBuildInfo->pVgroupHash, pBuildInfo->pVgroupList, &pVgCxt);
} else {
pVgCxt = *(SVgroupDataCxt**)pp;
}
@@ -723,7 +723,7 @@ int32_t tbNum) { SHashObj* pVgroupHash = taosHashInit(128, taosGetDefaultHashFun
*/
int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks, bool isRebuild) {
- SHashObj* pVgroupHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
+ SHashObj* pVgroupHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
SArray* pVgroupList = taosArrayInit(8, POINTER_BYTES);
if (NULL == pVgroupHash || NULL == pVgroupList) {
taosHashCleanup(pVgroupHash);
@@ -777,7 +777,7 @@ int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks, bool
int32_t vgId = pTableCxt->pMeta->vgId;
void** pp = taosHashGet(pVgroupHash, &vgId, sizeof(vgId));
if (NULL == pp) {
- code = createVgroupDataCxt(pTableCxt, pVgroupHash, pVgroupList, &pVgCxt);
+ code = createVgroupDataCxt(vgId, pVgroupHash, pVgroupList, &pVgCxt);
} else {
pVgCxt = *(SVgroupDataCxt**)pp;
}
@@ -830,6 +830,7 @@ static int32_t buildSubmitReq(int32_t vgId, SSubmitReq2* pReq, void** pData, uin
}
static void destroyVgDataBlocks(void* p) {
+ if (p == NULL) return;
SVgDataBlocks* pVg = p;
taosMemoryFree(pVg->pData);
taosMemoryFree(pVg);
@@ -855,7 +856,6 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList,
if (TSDB_CODE_SUCCESS == code) {
dst->numOfTables = taosArrayGetSize(src->pData->aSubmitTbData);
code = taosHashGetDup(pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg);
- // uError("td23101 3vgId:%d, numEps:%d", src->vgId, dst->vg.epSet.numOfEps);
}
if (TSDB_CODE_SUCCESS == code) {
code = buildSubmitReq(src->vgId, src->pData, &dst->pData, &dst->size);
@@ -863,6 +863,9 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList,
if (TSDB_CODE_SUCCESS == code) {
code = (NULL == taosArrayPush(pDataBlocks, &dst) ? terrno : TSDB_CODE_SUCCESS);
}
+ if (TSDB_CODE_SUCCESS != code) {
+ destroyVgDataBlocks(dst);
+ }
}
if (append) {
@@ -1074,3 +1077,33 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
end:
return ret;
}
+
+int rawBlockBindRawData(SHashObj* pVgroupHash, SArray* pVgroupList, STableMeta* pTableMeta, void* data) {
+ transformRawSSubmitTbData(data, pTableMeta->suid, pTableMeta->uid, pTableMeta->sversion);
+ SVgroupDataCxt* pVgCxt = NULL;
+ void** pp = taosHashGet(pVgroupHash, &pTableMeta->vgId, sizeof(pTableMeta->vgId));
+ if (NULL == pp) {
+ int code = createVgroupDataCxt(pTableMeta->vgId, pVgroupHash, pVgroupList, &pVgCxt);
+ if (code != 0){
+ return code;
+ }
+ } else {
+ pVgCxt = *(SVgroupDataCxt**)pp;
+ }
+ if (NULL == pVgCxt->pData->aSubmitTbData) {
+ pVgCxt->pData->aSubmitTbData = taosArrayInit(0, POINTER_BYTES);
+ pVgCxt->pData->raw = true;
+ if (NULL == pVgCxt->pData->aSubmitTbData) {
+ return terrno;
+ }
+ }
+
+ // push data to submit, rebuild empty data for next submit
+ if (NULL == taosArrayPush(pVgCxt->pData->aSubmitTbData, &data)) {
+ return terrno;
+ }
+
+ uTrace("add raw data to vgId:%d, len:%d", pTableMeta->vgId, *(int32_t*)data);
+
+ return 0;
+}
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index b657272b91..dac6446402 100755
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -3911,7 +3911,7 @@ static EDealRes rewriteColsToSelectValFuncImpl(SNode** pNode, void* pContext) {
static int32_t rewriteColsToSelectValFunc(STranslateContext* pCxt, SSelectStmt* pSelect) {
nodesRewriteExprs(pSelect->pProjectionList, rewriteColsToSelectValFuncImpl, pCxt);
- if (TSDB_CODE_SUCCESS == pCxt->errCode && !pSelect->isDistinct) {
+ if (TSDB_CODE_SUCCESS == pCxt->errCode) {
nodesRewriteExprs(pSelect->pOrderByList, rewriteColsToSelectValFuncImpl, pCxt);
}
return pCxt->errCode;
@@ -11420,7 +11420,7 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm
!hasTbnameFunction(pSelect->pPartitionByList) && pSelect->pWindow != NULL &&
pSelect->pWindow->type == QUERY_NODE_EVENT_WINDOW) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
- "Event window for stream on super table must patitioned by table name");
+ "Event window for stream on super table must partitioned by table name");
}
if (pSelect->pWindow != NULL && pSelect->pWindow->type == QUERY_NODE_EVENT_WINDOW &&
diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c
index c3fd9cdcf2..3f064f2b66 100644
--- a/source/libs/planner/src/planLogicCreater.c
+++ b/source/libs/planner/src/planLogicCreater.c
@@ -406,6 +406,47 @@ static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealT
static bool needScanDefaultCol(EScanType scanType) { return SCAN_TYPE_TABLE_COUNT != scanType; }
+static int32_t updateScanNoPseudoRefAfterGrp(SSelectStmt* pSelect, SScanLogicNode* pScan, SRealTableNode* pRealTable) {
+ if (NULL == pScan->pScanPseudoCols || pScan->pScanPseudoCols->length <= 0 || NULL != pSelect->pTags || NULL != pSelect->pSubtable) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ SNodeList* pList = NULL;
+ int32_t code = 0;
+ if (NULL == pSelect->pPartitionByList || pSelect->pPartitionByList->length <= 0) {
+ if (NULL == pSelect->pGroupByList || pSelect->pGroupByList->length <= 0) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ code = nodesCollectColumns(pSelect, SQL_CLAUSE_GROUP_BY, pRealTable->table.tableAlias, COLLECT_COL_TYPE_TAG,
+ &pList);
+ if (TSDB_CODE_SUCCESS == code) {
+ code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, pRealTable->table.tableAlias, fmIsScanPseudoColumnFunc,
+ &pList);
+ }
+ if (TSDB_CODE_SUCCESS == code && (NULL == pList || pList->length <= 0)) {
+ pScan->noPseudoRefAfterGrp = true;
+ }
+ goto _return;
+ }
+
+ code = nodesCollectColumns(pSelect, SQL_CLAUSE_PARTITION_BY, pRealTable->table.tableAlias, COLLECT_COL_TYPE_TAG,
+ &pList);
+ if (TSDB_CODE_SUCCESS == code) {
+ code = nodesCollectFuncs(pSelect, SQL_CLAUSE_PARTITION_BY, pRealTable->table.tableAlias, fmIsScanPseudoColumnFunc,
+ &pList);
+ }
+
+ if (TSDB_CODE_SUCCESS == code && (NULL == pList || pList->length <= 0)) {
+ pScan->noPseudoRefAfterGrp = true;
+ }
+
+_return:
+
+ nodesDestroyList(pList);
+ return code;
+}
+
static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable,
SLogicNode** pLogicNode) {
SScanLogicNode* pScan = NULL;
@@ -437,7 +478,13 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
&pScan->pScanPseudoCols);
}
- pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->tableType, pSelect->tagScan);
+ if (TSDB_CODE_SUCCESS == code) {
+ code = updateScanNoPseudoRefAfterGrp(pSelect, pScan, pRealTable);
+ }
+
+ if (TSDB_CODE_SUCCESS == code) {
+ pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->tableType, pSelect->tagScan);
+ }
// rewrite the expression in subsequent clauses
if (TSDB_CODE_SUCCESS == code) {
diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c
index 38f2e5024f..e8a8e6889a 100644
--- a/source/libs/planner/src/planOptimizer.c
+++ b/source/libs/planner/src/planOptimizer.c
@@ -93,6 +93,12 @@ typedef struct SCpdCollectTableColCxt {
int32_t errCode;
} SCpdCollectTableColCxt;
+typedef struct SCollectColsCxt {
+ SHashObj* pColHash;
+ int32_t errCode;
+} SCollectColsCxt;
+
+
typedef enum ECondAction {
COND_ACTION_STAY = 1,
COND_ACTION_PUSH_JOIN,
@@ -3302,6 +3308,70 @@ static int32_t partTagsRewriteGroupTagsToFuncs(SNodeList* pGroupTags, int32_t st
return code;
}
+static EDealRes partTagsCollectColsNodes(SNode* pNode, void* pContext) {
+ SCollectColsCxt* pCxt = pContext;
+ if (QUERY_NODE_COLUMN == nodeType(pNode)) {
+ SColumnNode* pCol = (SColumnNode*)pNode;
+ if (NULL == taosHashGet(pCxt->pColHash, pCol->colName, strlen(pCol->colName))) {
+ pCxt->errCode = taosHashPut(pCxt->pColHash, pCol->colName, strlen(pCol->colName), NULL, 0);
+ }
+ }
+
+ return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR);
+}
+
+
+static bool partTagsIsScanPseudoColsInConds(SScanLogicNode* pScan) {
+ SCollectColsCxt cxt = {
+ .errCode = TSDB_CODE_SUCCESS,
+ .pColHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK)};
+ if (NULL == cxt.pColHash) {
+ return true;
+ }
+
+ nodesWalkExpr(pScan->node.pConditions, partTagsCollectColsNodes, &cxt);
+ if (cxt.errCode) {
+ taosHashCleanup(cxt.pColHash);
+ return true;
+ }
+
+ SNode* pNode = NULL;
+ FOREACH(pNode, pScan->pScanPseudoCols) {
+ if (taosHashGet(cxt.pColHash, ((SExprNode*)pNode)->aliasName, strlen(((SExprNode*)pNode)->aliasName))) {
+ taosHashCleanup(cxt.pColHash);
+ return true;
+ }
+ }
+
+ taosHashCleanup(cxt.pColHash);
+ return false;
+}
+
+static int32_t partTagsOptRemovePseudoCols(SScanLogicNode* pScan) {
+ if (!pScan->noPseudoRefAfterGrp || NULL == pScan->pScanPseudoCols || pScan->pScanPseudoCols->length <= 0) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ if (pScan->node.pConditions && partTagsIsScanPseudoColsInConds(pScan)) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ SNode* pNode = NULL, *pTarget = NULL;
+ FOREACH(pNode, pScan->pScanPseudoCols) {
+ FOREACH(pTarget, pScan->node.pTargets) {
+ if (0 == strcmp(((SExprNode*)pNode)->aliasName, ((SColumnNode*)pTarget)->colName)) {
+ ERASE_NODE(pScan->node.pTargets);
+ break;
+ }
+ }
+ }
+
+ nodesDestroyList(pScan->pScanPseudoCols);
+ pScan->pScanPseudoCols = NULL;
+
+ return TSDB_CODE_SUCCESS;
+}
+
static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
SLogicNode* pNode = optFindPossibleNode(pLogicSubplan->pNode, partTagsOptMayBeOptimized, NULL);
if (NULL == pNode) {
@@ -3362,6 +3432,9 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub
code = partTagsRewriteGroupTagsToFuncs(pScan->pGroupTags, start, pAgg);
}
}
+ if (TSDB_CODE_SUCCESS == code) {
+ code = partTagsOptRemovePseudoCols(pScan);
+ }
if (TSDB_CODE_SUCCESS == code) {
code = partTagsOptRebuildTbanme(pScan->pGroupTags);
}
diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c
index 9513e90c50..316699bba1 100644
--- a/source/libs/planner/src/planPhysiCreater.c
+++ b/source/libs/planner/src/planPhysiCreater.c
@@ -2839,7 +2839,8 @@ static int32_t createDataInserter(SPhysiPlanContext* pCxt, SVgDataBlocks* pBlock
pInserter->numOfTables = pBlocks->numOfTables;
pInserter->size = pBlocks->size;
- TSWAP(pInserter->pData, pBlocks->pData);
+ pInserter->pData = pBlocks->pData;
+ pBlocks->pData = NULL;
*pSink = (SDataSinkNode*)pInserter;
return TSDB_CODE_SUCCESS;
diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c
index 85b1c543c0..35f258c554 100644
--- a/source/libs/qcom/src/queryUtil.c
+++ b/source/libs/qcom/src/queryUtil.c
@@ -202,6 +202,18 @@ int32_t taosAsyncRecover() {
return taskQueue.wrokrerPool.pCb->afterRecoverFromBlocking(&taskQueue.wrokrerPool);
}
+int32_t taosStmt2AsyncBind(__async_exec_fn_t bindFn, void* bindParam) {
+ SSchedMsg* pSchedMsg;
+ int32_t rc = taosAllocateQitem(sizeof(SSchedMsg), DEF_QITEM, 0, (void **)&pSchedMsg);
+ if (rc) return rc;
+ pSchedMsg->fp = NULL;
+ pSchedMsg->ahandle = bindFn;
+ pSchedMsg->thandle = bindParam;
+ // pSchedMsg->msg = code;
+
+ return taosWriteQitem(taskQueue.pTaskQueue, pSchedMsg);
+}
+
void destroySendMsgInfo(SMsgSendInfo* pMsgBody) {
if (NULL == pMsgBody) {
return;
diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c
index a031bc08de..ec9913d6ac 100644
--- a/source/libs/scheduler/src/schRemote.c
+++ b/source/libs/scheduler/src/schRemote.c
@@ -1110,7 +1110,7 @@ _return:
}
int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType, void* param) {
- int32_t msgSize = 0;
+ int32_t msgSize = 0;
void *msg = NULL;
int32_t code = 0;
bool isCandidateAddr = false;
@@ -1136,13 +1136,8 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
case TDMT_VND_SUBMIT:
case TDMT_VND_COMMIT: {
msgSize = pTask->msgLen;
- msg = taosMemoryCalloc(1, msgSize);
- if (NULL == msg) {
- SCH_TASK_ELOG("calloc %d failed", msgSize);
- SCH_ERR_RET(terrno);
- }
-
- TAOS_MEMCPY(msg, pTask->msg, msgSize);
+ msg = pTask->msg;
+ pTask->msg = NULL;
break;
}
diff --git a/source/libs/scheduler/src/schStatus.c b/source/libs/scheduler/src/schStatus.c
index f24ee74101..9792af22f6 100644
--- a/source/libs/scheduler/src/schStatus.c
+++ b/source/libs/scheduler/src/schStatus.c
@@ -22,9 +22,11 @@
#include "trpc.h"
int32_t schSwitchJobStatus(SSchJob* pJob, int32_t status, void* param) {
- int32_t code = 0;
- SCH_ERR_JRET(schUpdateJobStatus(pJob, status));
-
+ int32_t code = schUpdateJobStatus(pJob, status);
+ if (TSDB_CODE_SUCCESS != code) {
+ SCH_ERR_JRET((param && *(int32_t*)param) ? *(int32_t*)param : code);
+ }
+
switch (status) {
case JOB_TASK_STATUS_INIT:
break;
diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h
index f922a5e03e..d9778a6a05 100644
--- a/source/libs/stream/inc/streamInt.h
+++ b/source/libs/stream/inc/streamInt.h
@@ -38,7 +38,7 @@ extern "C" {
#define META_HB_SEND_IDLE_COUNTER 25 // send hb every 5 sec
#define STREAM_TASK_KEY_LEN ((sizeof(int64_t)) << 1)
#define STREAM_TASK_QUEUE_CAPACITY 5120
-#define STREAM_TASK_QUEUE_CAPACITY_IN_SIZE (30)
+#define STREAM_TASK_QUEUE_CAPACITY_IN_SIZE (10)
// clang-format off
#define stFatal(...) do { if (stDebugFlag & DEBUG_FATAL) { taosPrintLog("STM FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0)
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index baf36d0453..41773ee42d 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -131,12 +131,12 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r
code = tmsgSendReq(&pEpInfo->epSet, &rpcMsg);
if (code != 0) {
- rpcFreeCont(buf);
- return code;
+ stError("s-task:%s (child %d) failed to send retrieve req to task:0x%x (vgId:%d) QID:0x%" PRIx64 " code:%s",
+ pTask->id.idStr, pTask->info.selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req->reqId, tstrerror(code));
+ } else {
+ stDebug("s-task:%s (child %d) send retrieve req to task:0x%x (vgId:%d),QID:0x%" PRIx64, pTask->id.idStr,
+ pTask->info.selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req->reqId);
}
-
- stDebug("s-task:%s (child %d) send retrieve req to task:0x%x (vgId:%d),QID:0x%" PRIx64, pTask->id.idStr,
- pTask->info.selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req->reqId);
}
return code;
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index 5e099712ca..ee34648a47 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -807,6 +807,8 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
return 0;
}
+ int64_t st = taosGetTimestampMs();
+
EExtractDataCode ret = streamTaskGetDataFromInputQ(pTask, &pInput, &numOfBlocks, &blockSize);
if (ret == EXEC_AFTER_IDLE) {
streamTaskSetIdleInfo(pTask, MIN_INVOKE_INTERVAL);
@@ -841,8 +843,6 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
continue;
}
- int64_t st = taosGetTimestampMs();
-
// here only handle the data block sink operation
if (type == STREAM_INPUT__DATA_BLOCK) {
pTask->execInfo.sink.dataSize += blockSize;
@@ -873,6 +873,13 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
if (code) {
return code;
}
+
+ double el = (taosGetTimestampMs() - st) / 1000.0;
+ if (el > 5.0) { // elapsed more than 5 sec, not occupy the CPU anymore
+ stDebug("s-task:%s occupy more than 5.0s, release the exec threads and idle for 500ms", id);
+ streamTaskSetIdleInfo(pTask, 500);
+ return code;
+ }
}
}
}
diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c
index d2d7c7b11b..9aabb30baa 100644
--- a/source/libs/stream/src/streamSessionState.c
+++ b/source/libs/stream/src/streamSessionState.c
@@ -562,6 +562,33 @@ SStreamStateCur* sessionWinStateSeekKeyCurrentPrev(SStreamFileState* pFileState,
pCur->pStreamFileState = pFileState;
return pCur;
}
+
+SStreamStateCur* sessionWinStateSeekKeyPrev(SStreamFileState *pFileState, const SSessionKey *pWinKey) {
+ SArray* pWinStates = NULL;
+ int32_t index = -1;
+ SStreamStateCur *pCur = seekKeyCurrentPrev_buff(pFileState, pWinKey, &pWinStates, &index);
+ if (pCur) {
+ int32_t cmpRes= sessionStateRangeKeyCompare(pWinKey, pWinStates, index);
+ if (cmpRes > 0) {
+ return pCur;
+ } else if (cmpRes == 0 && index > 0) {
+ sessionWinStateMoveToPrev(pCur);
+ return pCur;
+ }
+ streamStateFreeCur(pCur);
+ pCur = NULL;
+ }
+
+ void* pFileStore = getStateFileStore(pFileState);
+ pCur = streamStateSessionSeekKeyPrev_rocksdb(pFileStore, pWinKey);
+ if (!pCur) {
+ return NULL;
+ }
+ pCur->buffIndex = -1;
+ pCur->pStreamFileState = pFileState;
+ return pCur;
+}
+
static void transformCursor(SStreamFileState* pFileState, SStreamStateCur* pCur) {
if (!pCur) {
return;
@@ -747,6 +774,15 @@ void sessionWinStateMoveToNext(SStreamStateCur* pCur) {
}
}
+void sessionWinStateMoveToPrev(SStreamStateCur* pCur) {
+ qTrace("move cursor to prev");
+ if (pCur && pCur->buffIndex >= 1) {
+ pCur->buffIndex--;
+ } else {
+ streamStateCurPrev_rocksdb(pCur);
+ }
+}
+
int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey,
range_cmpr_fn cmpFn) {
SStreamStateCur* pCur = sessionWinStateSeekKeyCurrentPrev(pFileState, key);
diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c
index 7259c0e49a..dba02015ed 100644
--- a/source/libs/stream/src/streamState.c
+++ b/source/libs/stream/src/streamState.c
@@ -440,6 +440,10 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, cons
return sessionWinStateSeekKeyCurrentNext(pState->pFileState, key);
}
+SStreamStateCur *streamStateSessionSeekKeyPrev(SStreamState *pState, const SSessionKey *key) {
+ return sessionWinStateSeekKeyPrev(pState->pFileState, key);
+}
+
SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key) {
return sessionWinStateSeekKeyNext(pState->pFileState, key);
}
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index 5ee8bd43f5..7209b6434f 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -331,6 +331,8 @@ void tFreeStreamTask(void* pParam) {
taosMemoryFreeClear(pTask->notifyInfo.stbFullName);
tDeleteSchemaWrapper(pTask->notifyInfo.pSchemaWrapper);
+ pTask->notifyEventStat = (STaskNotifyEventStat){0};
+
taosMemoryFree(pTask);
stDebug("s-task:0x%x free task completed", taskId);
}
@@ -988,6 +990,7 @@ void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc)
pDst->startTime = pSrc->startTime;
pDst->hTaskId = pSrc->hTaskId;
+ pDst->notifyEventStat = pSrc->notifyEventStat;
}
STaskStatusEntry streamTaskGetStatusEntry(SStreamTask* pTask) {
@@ -1016,6 +1019,7 @@ STaskStatusEntry streamTaskGetStatusEntry(SStreamTask* pTask) {
.outputThroughput = SIZE_IN_KiB(pExecInfo->outputThroughput),
.startCheckpointId = pExecInfo->startCheckpointId,
.startCheckpointVer = pExecInfo->startCheckpointVer,
+ .notifyEventStat = pTask->notifyEventStat,
};
return entry;
}
diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index d1209d08ce..f9bb48e1e0 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -679,6 +679,14 @@ SSyncState syncGetState(int64_t rid) {
return state;
}
+void syncGetCommitIndex(int64_t rid, int64_t* syncCommitIndex) {
+ SSyncNode* pSyncNode = syncNodeAcquire(rid);
+ if (pSyncNode != NULL) {
+ *syncCommitIndex = pSyncNode->commitIndex;
+ syncNodeRelease(pSyncNode);
+ }
+}
+
int32_t syncGetArbToken(int64_t rid, char* outToken) {
int32_t code = 0;
SSyncNode* pSyncNode = syncNodeAcquire(rid);
diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c
index d7576005d7..b3b69d81c0 100644
--- a/source/libs/transport/src/trans.c
+++ b/source/libs/transport/src/trans.c
@@ -163,7 +163,7 @@ void* rpcMallocCont(int64_t contLen) {
tTrace("malloc mem:%p size:%" PRId64, start, size);
}
- return start + sizeof(STransMsgHead);
+ return start + TRANS_MSG_OVERHEAD;
}
void rpcFreeCont(void* cont) { transFreeMsg(cont); }
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index ba2d471ccf..f34b00bec5 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -859,6 +859,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT, "Replay is disabled
TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_NO_TABLE_QUALIFIED, "No table qualified for query")
TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_NO_NEED_REBALANCE, "No need rebalance")
TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_STATUS, "Invalid status, please subscribe topic first")
+TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_DATA, "Invalid data use here")
+TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_RAW_DATA_SPLIT, "Split submit data for rawdata")
// stream
TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_NOT_EXIST, "Stream task not exist")
diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c
index 0e3e236b36..2460c7c650 100644
--- a/source/util/src/tlog.c
+++ b/source/util/src/tlog.c
@@ -1502,31 +1502,46 @@ bool taosAssertRelease(bool condition) {
}
#endif
+#define NUM_BASE 100
+#define DIGIT_LENGTH 2
+#define MAX_DIGITS 24
+
char* u64toaFastLut(uint64_t val, char* buf) {
+ // Look-up table for 2-digit numbers
static const char* lut =
"0001020304050607080910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455"
"5657585960616263646566676869707172737475767778798081828384858687888990919293949596979899";
- char temp[24];
- char* p = temp;
+ char temp[MAX_DIGITS];
+ char* p = temp + tListLen(temp);
- while (val >= 100) {
- strncpy(p, lut + (val % 100) * 2, 2);
- val /= 100;
- p += 2;
+ // Process the digits greater than or equal to 100
+ while (val >= NUM_BASE) {
+ // Get the last 2 digits from the look-up table and add to the buffer
+ p -= DIGIT_LENGTH;
+ strncpy(p, lut + (val % NUM_BASE) * DIGIT_LENGTH, DIGIT_LENGTH);
+ val /= NUM_BASE;
}
+ // Process the remaining 1 or 2 digits
if (val >= 10) {
- strncpy(p, lut + val * 2, 2);
- p += 2;
+ // If the number is 10 or more, get the 2 digits from the look-up table
+ p -= DIGIT_LENGTH;
+ strncpy(p, lut + val * DIGIT_LENGTH, DIGIT_LENGTH);
} else if (val > 0 || p == temp) {
- *(p++) = val + '0';
+ // If the number is less than 10, add the single digit to the buffer
+ p -= 1;
+ *p = val + '0';
}
- while (p != temp) {
- *buf++ = *--p;
+ int64_t len = temp + tListLen(temp) - p;
+ if (len > 0) {
+ memcpy(buf, p, len);
+ } else {
+ buf[0] = '0';
+ len = 1;
}
+ buf[len] = '\0';
- *buf = '\0';
- return buf;
+ return buf + len;
}
diff --git a/source/util/test/log.cpp b/source/util/test/log.cpp
index 1899aac2c4..ae1be94e40 100644
--- a/source/util/test/log.cpp
+++ b/source/util/test/log.cpp
@@ -139,3 +139,28 @@ TEST(log, misc) {
taosCloseLog();
}
+
+TEST(log, test_u64toa) {
+ char buf[64] = {0};
+ char *p = buf;
+
+ p = u64toaFastLut(0, buf);
+ EXPECT_EQ(p, buf + 1);
+ EXPECT_EQ(strcmp(buf, "0"), 0);
+
+ p = u64toaFastLut(1, buf);
+ EXPECT_EQ(p, buf + 1);
+ EXPECT_EQ(strcmp(buf, "1"), 0);
+
+ p = u64toaFastLut(12, buf);
+ EXPECT_EQ(p, buf + 2);
+ EXPECT_EQ(strcmp(buf, "12"), 0);
+
+ p = u64toaFastLut(12345, buf);
+ EXPECT_EQ(p, buf + 5);
+ EXPECT_EQ(strcmp(buf, "12345"), 0);
+
+ p = u64toaFastLut(1234567890, buf);
+ EXPECT_EQ(p, buf + 10);
+ EXPECT_EQ(strcmp(buf, "1234567890"), 0);
+}
diff --git a/tests/README.md b/tests/README.md
index 13b5c3620d..5450e9072c 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -252,6 +252,10 @@ cd scripts/tsdbComp && \
./testTsbs.sh
```
+> [!NOTE]
+> 1. TSBS test is written in Golang, in order to run the test smoothly, a Go proxy in China is set in above script by default. If this is not what you want, please unset it with command `sed -i '/GOPROXY/d' /usr/local/src/tsbs/scripts/tsdbComp/installTsbsCommand.sh` before starting the test.
+> 2. To check your current Go proxy setting, please run `go env | grep GOPROXY`.
+
### 3.7.2 How to start client and server on different hosts?
By default, both client and server will be started on the local host. To start the client and server on separate hosts, please follow steps below to configure `test.ini` before starting the test:
diff --git a/tests/army/cluster/arbitrator.py b/tests/army/cluster/arbitrator.py
index 98881c5fc4..766181635f 100644
--- a/tests/army/cluster/arbitrator.py
+++ b/tests/army/cluster/arbitrator.py
@@ -61,11 +61,11 @@ class TDTestCase(TBase):
tdSql.query("show db.vgroups;")
- if(tdSql.getData(0, 4) == "follower") and (tdSql.getData(0, 6) == "leader"):
+ if(tdSql.getData(0, 4) == "follower") and (tdSql.getData(0, 7) == "leader"):
tdLog.info("stop dnode2")
sc.dnodeStop(2)
- if(tdSql.getData(0, 6) == "follower") and (tdSql.getData(0, 4) == "leader"):
+ if(tdSql.getData(0, 7) == "follower") and (tdSql.getData(0, 4) == "leader"):
tdLog.info("stop dnode 3")
sc.dnodeStop(3)
@@ -74,7 +74,7 @@ class TDTestCase(TBase):
while count < 100:
tdSql.query("show db.vgroups;")
- if(tdSql.getData(0, 4) == "assigned ") or (tdSql.getData(0, 6) == "assigned "):
+ if(tdSql.getData(0, 4) == "assigned ") or (tdSql.getData(0, 7) == "assigned "):
break
tdLog.info("wait %d seconds for set assigned"%count)
diff --git a/tests/army/frame/clusterCommonCheck.py b/tests/army/frame/clusterCommonCheck.py
index 9cbac776b9..75b82d2101 100644
--- a/tests/army/frame/clusterCommonCheck.py
+++ b/tests/army/frame/clusterCommonCheck.py
@@ -291,9 +291,9 @@ class ClusterComCheck:
return True
elif self.db_replica == 3 :
- vgroup_status_first=[tdSql.res[0][4],tdSql.res[0][6],tdSql.res[0][8]]
+ vgroup_status_first=[tdSql.res[0][4],tdSql.res[0][7],tdSql.res[0][10]]
- vgroup_status_last=[tdSql.res[last_number][4],tdSql.res[last_number][6],tdSql.res[last_number][8]]
+ vgroup_status_last=[tdSql.res[last_number][4],tdSql.res[last_number][7],tdSql.res[last_number][10]]
if vgroup_status_first.count('leader') == 1 and vgroup_status_first.count('follower') == 2:
if vgroup_status_last.count('leader') == 1 and vgroup_status_last.count('follower') == 2:
tdSql.query(f"select `replica` from information_schema.ins_databases where `name`='{db_name}';")
@@ -301,6 +301,18 @@ class ClusterComCheck:
if tdSql.res[0][0] == db_replica:
tdLog.success(f"elections of {db_name}.vgroups with replica {self.db_replica} are ready in {count} s")
return True
+
+ vgruop_apply_commit_first = [tdSql.res[0][5],tdSql.res[0][8],tdSql.res[0][11]]
+ vgruop_apply_commit_end = [tdSql.res[last_number][5],tdSql.res[last_number][8],tdSql.res[last_number][11]]
+ for i in range(3):
+ v = vgruop_apply_commit_first[i].split('/')
+ assert (int(v[0]) <= int(v[1])) ,f"apply {v[0]} > commit {v[1]}"
+
+ v = vgruop_apply_commit_end[i].split('/')
+ assert (int(v[0]) <= int(v[1])) ,f"apply {v[0]} > commit {v[1]}"
+
+
+
else:
tdLog.debug(tdSql.res)
tdLog.notice(f"elections of {db_name} all vgroups with replica {self.db_replica} are failed in {count} s ")
diff --git a/tests/army/stream/stream_notify.json b/tests/army/stream/stream_notify.json
new file mode 100644
index 0000000000..9dcbe4efb2
--- /dev/null
+++ b/tests/army/stream/stream_notify.json
@@ -0,0 +1,71 @@
+{
+ "filetype": "insert",
+ "cfgdir": "/etc/taos",
+ "host": "127.0.0.1",
+ "port": 6030,
+ "user": "root",
+ "password": "taosdata",
+ "thread_count": 4,
+ "create_table_thread_count": 4,
+ "result_file": "./insert_res.txt",
+ "confirm_parameter_prompt": "no",
+ "num_of_records_per_req": 10000,
+ "prepared_rand": 10000,
+ "chinese": "no",
+ "escape_character": "yes",
+ "continue_if_fail": "no",
+ "databases": [
+ {
+ "dbinfo": {
+ "name": "test",
+ "drop": "no",
+ "vgroups": 4,
+ "precision": "ms"
+ },
+ "super_tables": [
+ {
+ "name": "st",
+ "child_table_exists": "no",
+ "childtable_count": 5,
+ "childtable_prefix": "ct",
+ "auto_create_table": "yes",
+ "batch_create_tbl_num": 5,
+ "data_source": "rand",
+ "insert_mode": "taosc",
+ "non_stop_mode": "no",
+ "line_protocol": "line",
+ "insert_rows": 10000,
+ "childtable_limit": 0,
+ "childtable_offset": 0,
+ "interlace_rows": 50,
+ "insert_interval": 10,
+ "partial_col_num": 0,
+ "timestamp_step": 500,
+ "start_timestamp": "2025-01-13 17:30:00.000",
+ "sample_format": "csv",
+ "sample_file": "./sample.csv",
+ "use_sample_ts": "no",
+ "tags_file": "",
+ "columns": [
+ {"type": "TINYINT", "name": "c0"},
+ {"type": "SMALLINT", "name": "c1"},
+ {"type": "INT", "name": "c2"},
+ {"type": "BIGINT", "name": "c3"},
+ {"type": "DOUBLE", "name": "c4"},
+ {"type": "FLOAT", "name": "c5"},
+ {"type": "BOOL", "name": "c6"},
+ {"type": "VARCHAR", "name": "c7", "len": 10},
+ {"type": "NCHAR", "name": "c8", "len": 10},
+ {"type": "UTINYINT", "name": "c9"},
+ {"type": "USMALLINT", "name": "c10"},
+ {"type": "UINT", "name": "c11"},
+ {"type": "UBIGINT", "name": "c12"}
+ ],
+ "tags": [
+ {"type": "INT", "name": "groupid", "max": 100, "min": 1}
+ ]
+ }
+ ]
+ }
+ ]
+}
diff --git a/tests/army/stream/stream_notify_disorder.json b/tests/army/stream/stream_notify_disorder.json
new file mode 100644
index 0000000000..2f9f9bea69
--- /dev/null
+++ b/tests/army/stream/stream_notify_disorder.json
@@ -0,0 +1,72 @@
+{
+ "filetype": "insert",
+ "cfgdir": "/etc/taos",
+ "host": "127.0.0.1",
+ "port": 6030,
+ "user": "root",
+ "password": "taosdata",
+ "thread_count": 4,
+ "create_table_thread_count": 4,
+ "result_file": "./insert_res.txt",
+ "confirm_parameter_prompt": "no",
+ "num_of_records_per_req": 10000,
+ "prepared_rand": 10000,
+ "chinese": "no",
+ "escape_character": "yes",
+ "continue_if_fail": "no",
+ "databases": [
+ {
+ "dbinfo": {
+ "name": "test",
+ "drop": "no",
+ "vgroups": 4,
+ "precision": "ms"
+ },
+ "super_tables": [
+ {
+ "name": "st",
+ "child_table_exists": "no",
+ "childtable_count": 5,
+ "childtable_prefix": "ct",
+ "auto_create_table": "yes",
+ "batch_create_tbl_num": 5,
+ "data_source": "rand",
+ "insert_mode": "taosc",
+ "non_stop_mode": "no",
+ "line_protocol": "line",
+ "insert_rows": 10000,
+ "disorder_ratio": 10,
+ "childtable_limit": 0,
+ "childtable_offset": 0,
+ "interlace_rows": 50,
+ "insert_interval": 10,
+ "partial_col_num": 0,
+ "timestamp_step": 500,
+ "start_timestamp": "2025-01-13 17:30:00.000",
+ "sample_format": "csv",
+ "sample_file": "./sample.csv",
+ "use_sample_ts": "no",
+ "tags_file": "",
+ "columns": [
+ {"type": "TINYINT", "name": "c0"},
+ {"type": "SMALLINT", "name": "c1"},
+ {"type": "INT", "name": "c2"},
+ {"type": "BIGINT", "name": "c3"},
+ {"type": "DOUBLE", "name": "c4"},
+ {"type": "FLOAT", "name": "c5"},
+ {"type": "BOOL", "name": "c6"},
+ {"type": "VARCHAR", "name": "c7", "len": 10},
+ {"type": "NCHAR", "name": "c8", "len": 10},
+ {"type": "UTINYINT", "name": "c9"},
+ {"type": "USMALLINT", "name": "c10"},
+ {"type": "UINT", "name": "c11"},
+ {"type": "UBIGINT", "name": "c12"}
+ ],
+ "tags": [
+ {"type": "INT", "name": "groupid", "max": 100, "min": 1}
+ ]
+ }
+ ]
+ }
+ ]
+}
diff --git a/tests/army/stream/stream_notify_server.py b/tests/army/stream/stream_notify_server.py
new file mode 100644
index 0000000000..a105d55971
--- /dev/null
+++ b/tests/army/stream/stream_notify_server.py
@@ -0,0 +1,55 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import asyncio
+import signal
+import websockets
+import argparse
+
+stop_event = asyncio.Event()
+
+async def handle_websocket(websocket, log_file):
+ try:
+ # Write the message to the specified log file
+ if log_file != "":
+ with open(log_file, "a", encoding="utf-8") as f:
+ async for message in websocket:
+ f.write(message + "\n")
+ if stop_event.is_set():
+ break
+ except Exception as e:
+ print(f"Connection closed with error: {e}")
+
+async def listen(port, log_file):
+ async with websockets.serve(
+ lambda ws: handle_websocket(ws, log_file),
+ "0.0.0.0",
+ port,
+ ping_timeout = None,
+ max_size= 10 * 1024 * 1024 # 10MB,
+ ):
+ print(f"WebSocket server listening on port {port}...")
+ await stop_event.wait() # Run forever (until canceled)
+
+def signal_handler(sig, frame):
+ stop_event.set()
+
+if __name__ == '__main__':
+ signal.signal(signal.SIGINT, signal_handler)
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-d', '--log_file', type=str, default='stream_notify_server.log', help='log file')
+ parser.add_argument('-p', '--port', type=int, default=12345, help='port number')
+ args = parser.parse_args()
+
+ asyncio.run(listen(args.port, args.log_file))
diff --git a/tests/army/stream/test_stream_notify.py b/tests/army/stream/test_stream_notify.py
new file mode 100644
index 0000000000..0e6bbd5c01
--- /dev/null
+++ b/tests/army/stream/test_stream_notify.py
@@ -0,0 +1,473 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+from frame import etool
+from frame.etool import *
+from frame.log import *
+from frame.cases import *
+from frame.sql import *
+from frame.caseBase import *
+from frame.common import *
+import signal
+import subprocess
+
+class StreamNotifyServer:
+ def __init__(self):
+ self.log_file = ""
+ self.sub_process = None
+
+ def __del__(self):
+ self.stop()
+
+ def run(self, port, log_file):
+ tdLog.info(f"Start notify server: python3 {etool.curFile(__file__, 'stream_notify_server.py')} -p {port} -d {log_file}")
+ self.sub_process = subprocess.Popen(['python3', etool.curFile(__file__, 'stream_notify_server.py'), '-p', str(port), '-d', log_file])
+ self.log_file = log_file
+
+ def stop(self):
+ if self.sub_process is not None:
+ self.sub_process.send_signal(signal.SIGINT)
+ try:
+ self.sub_process.wait(60)
+ except subprocess.TimeoutExpired:
+ self.sub_process.kill()
+
+class TestStreamNotifySinglePass():
+ def __init__(self, num_addr_per_stream, trigger_mode, notify_event, disorder):
+ self.current_dir = os.path.dirname(os.path.abspath(__file__))
+ self.num_addr_per_stream = num_addr_per_stream
+ self.trigger_mode = trigger_mode
+ self.notify_event = notify_event
+ self.disorder = disorder
+ self.streams = []
+ self.id = ''.join(random.choices(string.ascii_letters + string.digits, k=8))
+
+ def is_port_in_use(self, port):
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+ return s.connect_ex(("127.0.0.1", port)) == 0
+
+ def gen_streams(self):
+ self.streams = [
+ {
+ "stream_name": "s_time_par",
+ "dest_table": "dst_time_par",
+ "window_clause": "interval(5s)",
+ "partitioned": True,
+ },
+ {
+ "stream_name": "s_time",
+ "dest_table": "dst_time",
+ "window_clause": "interval(5s)",
+ "partitioned": False,
+ },
+ {
+ "stream_name": "s_state_par",
+ "dest_table": "dst_state_par",
+ "window_clause": "state_window(c6)",
+ "partitioned": True,
+ },
+ {
+ "stream_name": "s_session_par",
+ "dest_table": "dst_session_par",
+ "window_clause": "session(ts, 50a)",
+ "partitioned": True,
+ },
+ {
+ "stream_name": "s_session",
+ "dest_table": "dst_session",
+ "window_clause": "session(ts, 50a)",
+ "partitioned": False,
+ },
+ {
+ "stream_name": "s_event_par",
+ "dest_table": "dst_event_par",
+ "window_clause": "event_window start with c6 = true end with c6 = false",
+ "partitioned": True,
+ },
+ {
+ "stream_name": "s_count_par",
+ "dest_table": "dst_count_par",
+ "window_clause": "count_window(10)",
+ "partitioned": True,
+ },
+ ]
+ # set port to random number between 10000 and 20000
+ port = random.randint(10000, 20000)
+ for stream in self.streams:
+ stream["notify_address"] = ""
+ stream["notify_server"] = []
+ if self.trigger_mode == "FORCE_WINDOW_CLOSE":
+ if stream["stream_name"] == "s_time" or stream["stream_name"] == "s_session":
+ continue
+ elif "MAX_DELAY" in self.trigger_mode or "AT_ONCE" in self.trigger_mode or self.disorder:
+ if stream["stream_name"] == "s_session_par" or stream["stream_name"] == "s_state_par":
+ continue
+ for i in range(self.num_addr_per_stream):
+ # Find an available port
+ while self.is_port_in_use(port):
+ port += 1
+ # Start the stream notify server and add the address to the stream
+ log_file = f"{self.current_dir}/{self.id}_{stream['stream_name']}_{i}.log"
+ if os.path.exists(log_file):
+ os.remove(log_file)
+ server = StreamNotifyServer()
+ server.run(port, log_file)
+ stream["notify_address"] += f"'ws://127.0.0.1:{port}',"
+ stream["notify_server"].append(server)
+ port += 1
+ stream["notify_address"] = stream["notify_address"][:-1]
+
+ def create_streams(self):
+ tdLog.info("==========step1:create table")
+ tdSql.execute("drop database if exists test;")
+ tdSql.execute("create database test keep 3650;")
+ tdSql.execute("use test;")
+ tdSql.execute(
+ f"""create stable if not exists test.st
+ (ts timestamp, c0 tinyint, c1 smallint, c2 int, c3 bigint, c4 double, c5 float, c6 bool, c7 varchar(10), c8 nchar(10), c9 tinyint unsigned, c10 smallint unsigned, c11 int unsigned, c12 bigint unsigned)
+ tags(groupid int);
+ """
+ )
+ for stream in self.streams:
+ if len(stream["notify_server"]) == 0:
+ continue
+ stream_option = f"TRIGGER {self.trigger_mode}"
+ if self.trigger_mode != "FORCE_WINDOW_CLOSE":
+ stream_option += " IGNORE UPDATE 0"
+ if not stream["stream_name"].startswith("s_count"):
+ stream_option += " IGNORE EXPIRED 0"
+ if stream["stream_name"].startswith("s_count"):
+ stream_option += " WATERMARK 1a"
+ stream_sql = f"""create stream {stream["stream_name"]} {stream_option} into {stream["dest_table"]} as
+ select _wstart, _wend, min(c0), max(c1), min(c2), max(c3), min(c4), max(c5), first(c6), first(c7), last(c8), min(c9), max(c10), min(c11), max(c12)
+ from test.st {stream["partitioned"] and "partition by tbname" or ""}
+ {stream["window_clause"]}
+ notify ({stream["notify_address"]}) on ({self.notify_event});
+ """
+ tdSql.execute(stream_sql, show=True)
+ # Wait for the stream tasks to be ready
+ for i in range(50):
+ tdLog.info(f"i={i} wait for stream tasks ready ...")
+ time.sleep(1)
+ rows = tdSql.query("select * from information_schema.ins_stream_tasks where status <> 'ready';")
+ if rows == 0:
+ break
+
+ def insert_data(self):
+ tdLog.info("insert stream notify test data.")
+ # taosBenchmark run
+ json_file = self.disorder and "stream_notify_disorder.json" or "stream_notify.json"
+ json = etool.curFile(__file__, json_file)
+ etool.benchMark(json=json)
+
+ def wait_all_streams_done(self):
+ while True:
+ tdLog.info("wait for all streams done ...")
+ time.sleep(10)
+ rows = tdSql.query("select stream_name, level, notify_event_stat from information_schema.ins_stream_tasks where notify_event_stat is not null;")
+ num_pushed = 0
+ num_sent = 0
+ for i in range(rows):
+ tdLog.printNoPrefix(f"{tdSql.getData(i, 0)}, {tdSql.getData(i, 1)}, {tdSql.getData(i, 2)}")
+ notify_event_stat = tdSql.getData(i, 2)
+ match = re.search(r"Push (\d+)x, (\d+) elems", notify_event_stat)
+ if match:
+ num_pushed += int(match.group(2))
+ match = re.search(r"Send (\d+)x, (\d+) elems", notify_event_stat)
+ if match:
+ num_sent += int(match.group(2))
+ if num_pushed == num_sent:
+ break
+ tdLog.info("wait for all notify servers stop ...")
+ for stream in self.streams:
+ for server in stream["notify_server"]:
+ server.stop()
+
+ def parse(self, log_file, out_file, stream_name):
+ message_ids = set()
+ events_map = {}
+ has_open = "window_open" in self.notify_event
+ has_close = "window_close" in self.notify_event
+ with open(log_file, "r", encoding="utf-8") as f:
+ for line in f:
+ data = json.loads(line)
+
+ # Check if the data has the required fields: messageId, timestamp, streams
+ if "messageId" not in data:
+ print(f"Error: Missing 'messageId' in data {data}")
+ return False
+ if "timestamp" not in data:
+ print(f"Error: Missing 'timestamp' in data {data}")
+ return False
+ if "streams" not in data:
+ print(f"Error: Missing 'streams' in data {data}")
+ return False
+
+ # Check if the message id is duplicated
+ if message_ids.__contains__(data["messageId"]):
+ print(f"Error: Duplicate message id {data['messageId']}")
+ return False
+ message_ids.add(data["messageId"])
+
+ # Check if the streams is correct
+ for stream in data["streams"]:
+ # Check if the stream has the required fields: streamName, events
+ if "streamName" not in stream:
+ print(f"Error: Missing 'streamName' in stream {stream}")
+ return False
+ if "events" not in stream:
+ print(f"Error: Missing 'events' in stream {stream}")
+ return False
+
+ # Check if the stream name is correct
+ if stream["streamName"] != stream_name:
+ print(f"Error: Incorrect stream name {stream['streamName']}")
+ return False
+
+ # Check if the events are correct
+ for event in stream["events"]:
+ # Check if the event has the required fields: tableName, eventType, eventTime, windowId, windowType
+ if "tableName" not in event:
+ print(f"Error: Missing 'tableName' in event {event}")
+ return False
+ if "eventType" not in event:
+ print(f"Error: Missing 'eventType' in event {event}")
+ return False
+ if "eventTime" not in event:
+ print(f"Error: Missing 'eventTime' in event {event}")
+ return False
+ if "windowId" not in event:
+ print(f"Error: Missing 'windowId' in event {event}")
+ return False
+ if "windowType" not in event:
+ print(f"Error: Missing 'windowType' in event {event}")
+ return False
+ if event["eventType"] not in [
+ "WINDOW_OPEN",
+ "WINDOW_CLOSE",
+ "WINDOW_INVALIDATION",
+ ]:
+ print(f"Error: Invalid event type {event['eventType']}")
+ return False
+ if event["windowType"] not in [
+ "Time",
+ "State",
+ "Session",
+ "Event",
+ "Count",
+ ]:
+ print(f"Error: Invalid window type {event['windowType']}")
+ return False
+
+ if event["eventType"] == "WINDOW_INVALIDATION":
+ if not has_close:
+ print(f"Error: WINDOW_INVALIDATION event is not allowed")
+ return False
+ # WINDOW_INVALIDATION must have fields: windowStart, windowEnd
+ if "windowStart" not in event:
+ print(f"Error: Missing 'windowStart' in event {event}")
+ return False
+ if "windowEnd" not in event:
+ print(f"Error: Missing 'windowEnd' in event {event}")
+ return False
+ events_map.pop(
+ (event["tableName"], event["windowId"]), None
+ )
+ continue
+
+ # Get the event from the event map; if it doesn't exist, create a new one
+ e = events_map.get((event["tableName"], event["windowId"]))
+ if e is None:
+ events_map[(event["tableName"], event["windowId"])] = {
+ "opened": False,
+ "closed": False,
+ "wstart": 0,
+ "wend": 0,
+ }
+ e = events_map.get((event["tableName"], event["windowId"]))
+
+ if event["eventType"] == "WINDOW_OPEN":
+ if not has_open:
+ print(f"Error: WINDOW_OPEN event is not allowed")
+ return False
+ # WINDOW_OPEN for all windows must have field: windowStart
+ if "windowStart" not in event:
+ print(f"Error: Missing 'windowStart' in event {event}")
+ return False
+ if event["windowType"] == "State":
+ # WINDOW_OPEN for State window must also have fields: prevState, curState
+ if "prevState" not in event:
+ print(
+ f"Error: Missing 'prevState' in event {event}"
+ )
+ return False
+ if "curState" not in event:
+ print(f"Error: Missing 'curState' in event {event}")
+ return False
+ elif event["windowType"] == "Event":
+ # WINDOW_OPEN for Event window must also have fields: triggerCondition
+ if "triggerCondition" not in event:
+ print(
+ f"Error: Missing 'triggerCondition' in event {event}"
+ )
+ return False
+ e["opened"] = True
+ e["wstart"] = event["windowStart"]
+ elif event["eventType"] == "WINDOW_CLOSE":
+ if not has_close:
+ print(f"Error: WINDOW_CLOSE event is not allowed")
+ return False
+ # WINDOW_CLOSE for all windows must have fields: windowStart, windowEnd, result
+ if "windowStart" not in event:
+ print(f"Error: Missing 'windowStart' in event {event}")
+ return False
+ if "windowEnd" not in event:
+ print(f"Error: Missing 'windowEnd' in event {event}")
+ return False
+ if "result" not in event:
+ print(f"Error: Missing 'result' in event {event}")
+ return False
+ if event["windowType"] == "State":
+ # WINDOW_CLOSE for State window must also have fields: curState, nextState
+ if "curState" not in event:
+ print(f"Error: Missing 'curState' in event {event}")
+ return False
+ if "nextState" not in event:
+ print(
+ f"Error: Missing 'nextState' in event {event}"
+ )
+ return False
+ elif event["windowType"] == "Event":
+ # WINDOW_CLOSE for Event window must also have fields: triggerCondition
+ if "triggerCondition" not in event:
+ print(
+ f"Error: Missing 'triggerCondition' in event {event}"
+ )
+ return False
+ e["closed"] = True
+ e["wstart"] = event["windowStart"]
+ e["wend"] = event["windowEnd"]
+
+ # Collect all the windows that closed
+ windows_map = {}
+ for k, v in events_map.items():
+ if not v["closed"]:
+ continue
+ e = windows_map.get(k[0])
+ if e is None:
+ windows_map[k[0]] = []
+ e = windows_map.get(k[0])
+ e.append((v["wstart"], v["wend"]))
+
+ # Sort the windows by start time
+ for k, v in windows_map.items():
+ v.sort(key=lambda x: x[0])
+
+ # Write all collected window info to the specified output file in sorted order as csv format
+ with open(out_file, "w", encoding="utf-8") as f:
+ f.write("wstart,wend,tbname\n")
+ for k, v in sorted(windows_map.items()):
+ for w in v:
+ f.write(f"{w[0]},{w[1]},\"{k}\"\n")
+ return True
+
+ def check_notify_result(self):
+ all_right = True
+ for stream in self.streams:
+ if len(stream["notify_server"]) == 0 or not "window_close" in self.notify_event:
+ continue
+ query_file = f"{self.current_dir}/{self.id}_{stream['stream_name']}.csv"
+ query_sql = f"select cast(_wstart as bigint) as wstart, cast(_wend as bigint) as wend, tbname from test.{stream['dest_table']} order by tbname, wstart >> {query_file};"
+ tdLog.info("query_sql: " + query_sql)
+ os.system(f"taos -c {tdCom.getClientCfgPath()} -s '{query_sql}'")
+ for i in range(self.num_addr_per_stream):
+ server = stream["notify_server"][i]
+ parse_file = f"{self.current_dir}/{self.id}_{stream['stream_name']}_{i}.csv"
+ if os.path.exists(parse_file):
+ os.remove(parse_file)
+ if not self.parse(f"{self.current_dir}/{self.id}_{stream['stream_name']}_{i}.log", parse_file, stream["stream_name"]):
+ tdLog.exit(f"Error: {stream['stream_name']}_{i} parse notify result failed")
+ # Compare the result using diff command
+ diff_file = f"{self.current_dir}/{self.id}_{stream['stream_name']}_{i}.diff"
+ if os.path.exists(diff_file):
+ os.remove(diff_file)
+ os.system(f"diff --strip-trailing-cr {query_file} {parse_file} > {diff_file}")
+ if os.path.getsize(diff_file) != 0:
+ tdLog.info(f"Error: {stream['stream_name']}_{i} notify result is not correct")
+ all_right = False
+ if not all_right:
+ raise Exception("Error: notify result is not correct")
+
+ def drop_all_streams(self):
+ for stream in self.streams:
+ tdSql.execute(f"drop stream if exists {stream['stream_name']};")
+ # Also remove all generaetd files
+ query_file = f"{self.current_dir}/{self.id}_{stream['stream_name']}.csv"
+ if os.path.exists(query_file):
+ os.remove(query_file)
+ for i in range(self.num_addr_per_stream):
+ log_file = f"{self.current_dir}/{self.id}_{stream['stream_name']}_{i}.log"
+ parse_file = f"{self.current_dir}/{self.id}_{stream['stream_name']}_{i}.csv"
+ diff_file = f"{self.current_dir}/{self.id}_{stream['stream_name']}_{i}.diff"
+ if os.path.exists(log_file):
+ os.remove(log_file)
+ if os.path.exists(parse_file):
+ os.remove(parse_file)
+ if os.path.exists(diff_file):
+ os.remove(diff_file)
+
+ def run(self):
+ tdLog.info(f"Start to execute TestStreamNotifySinglePass({self.num_addr_per_stream}, {self.trigger_mode}, {self.notify_event}, {self.disorder})")
+ self.gen_streams()
+ self.create_streams()
+ self.insert_data()
+ self.wait_all_streams_done()
+ self.check_notify_result()
+ self.drop_all_streams()
+ tdLog.info(f"TestStreamNotifySinglePass({self.num_addr_per_stream}, {self.trigger_mode}, {self.notify_event}, {self.disorder}) successfully executed")
+
+class TDTestCase(TBase):
+ def init(self, conn, logSql, replicaVar=1):
+ self.replicaVar = int(replicaVar)
+ tdLog.debug(f"start to excute {__file__}")
+ tdSql.init(conn.cursor(), True)
+
+ def run(self):
+ # Disable many tests due to long execution time
+
+ # TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="AT_ONCE", notify_event="'window_open', 'window_close'", disorder=False).run()
+ # TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="MAX_DELAY 10s", notify_event="'window_open', 'window_close'", disorder=False).run()
+ TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="WINDOW_CLOSE", notify_event="'window_open', 'window_close'", disorder=False).run()
+ # TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="FORCE_WINDOW_CLOSE", notify_event="'window_open', 'window_close'", disorder=False).run()
+
+ # TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="AT_ONCE", notify_event="'window_open', 'window_close'", disorder=True).run()
+ # TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="MAX_DELAY 10s", notify_event="'window_open', 'window_close'", disorder=True).run()
+ TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="WINDOW_CLOSE", notify_event="'window_open', 'window_close'", disorder=True).run()
+ # TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="FORCE_WINDOW_CLOSE", notify_event="'window_open', 'window_close'", disorder=True).run()
+
+ # TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="AT_ONCE", notify_event="'window_close'", disorder=False).run()
+ TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="MAX_DELAY 10s", notify_event="'window_close'", disorder=False).run()
+ # TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="WINDOW_CLOSE", notify_event="'window_close'", disorder=False).run()
+ # TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="FORCE_WINDOW_CLOSE", notify_event="'window_close'", disorder=False).run()
+
+ TestStreamNotifySinglePass(num_addr_per_stream=3, trigger_mode="AT_ONCE", notify_event="'window_open'", disorder=False).run()
+ # TestStreamNotifySinglePass(num_addr_per_stream=3, trigger_mode="MAX_DELAY 10s", notify_event="'window_open'", disorder=False).run()
+ # TestStreamNotifySinglePass(num_addr_per_stream=3, trigger_mode="WINDOW_CLOSE", notify_event="'window_open'", disorder=False).run()
+ # TestStreamNotifySinglePass(num_addr_per_stream=3, trigger_mode="FORCE_WINDOW_CLOSE", notify_event="'window_open'", disorder=False).run()
+
+ def stop(self):
+ tdLog.success(f"{__file__} successfully executed")
+
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165561.0.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165561.0.avro
deleted file mode 100644
index bd6659545a..0000000000
Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165561.0.avro and /dev/null differ
diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165563.2.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165563.2.avro
deleted file mode 100644
index 1dc4853c13..0000000000
Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165563.2.avro and /dev/null differ
diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165566.4.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165566.4.avro
deleted file mode 100644
index 6b9825f77e..0000000000
Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165566.4.avro and /dev/null differ
diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165569.5.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165569.5.avro
deleted file mode 100644
index 7465371600..0000000000
Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165569.5.avro and /dev/null differ
diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165572.6.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165572.6.avro
deleted file mode 100644
index f239d68fd8..0000000000
Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165572.6.avro and /dev/null differ
diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165574.7.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165574.7.avro
deleted file mode 100644
index 0d29e52cc5..0000000000
Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165574.7.avro and /dev/null differ
diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165576.8.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165576.8.avro
deleted file mode 100644
index 1112bc8610..0000000000
Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165576.8.avro and /dev/null differ
diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165578.9.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165578.9.avro
deleted file mode 100644
index e9adc51dd0..0000000000
Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165578.9.avro and /dev/null differ
diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165610.3.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165610.3.avro
deleted file mode 100644
index 4fd5995027..0000000000
Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165610.3.avro and /dev/null differ
diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165620.1.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165620.1.avro
deleted file mode 100644
index 59dafef4e6..0000000000
Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165620.1.avro and /dev/null differ
diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/dbs.sql b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/dbs.sql
deleted file mode 100644
index 4123f8c954..0000000000
--- a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/dbs.sql
+++ /dev/null
@@ -1,10 +0,0 @@
-#!server_ver: ver:3.1.0.0
-#!taosdump_ver: 2.5.2_cf16c4d
-#!os_id: LINUX
-#!escape_char: true
-#!loose_mode: false
-#!charset: UTF-8
-CREATE DATABASE IF NOT EXISTS test REPLICA 1 DURATION 14400m KEEP 5256000m,5256000m,5256000m PRECISION 'ms' MINROWS 100 MAXROWS 4096 COMP 2 ;
-
-CREATE TABLE IF NOT EXISTS test.`meters`(`ts` timestamp,`current` float,`voltage` int,`phase` float) TAGS(`groupid` int,`location` binary(24));
-
diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/test.3479004165475.avro-tbtags b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/test.3479004165475.avro-tbtags
deleted file mode 100644
index 96392475d9..0000000000
Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/test.3479004165475.avro-tbtags and /dev/null differ
diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479698875513/data0/test.3479698875538.0.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479698875513/data0/test.3479698875538.0.avro
new file mode 100644
index 0000000000..3c2ecc1e03
Binary files /dev/null and b/tests/army/tools/taosdump/native/compa/taosdump.3479698875513/data0/test.3479698875538.0.avro differ
diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479698875513/dbs.sql b/tests/army/tools/taosdump/native/compa/taosdump.3479698875513/dbs.sql
new file mode 100644
index 0000000000..c7e7296979
--- /dev/null
+++ b/tests/army/tools/taosdump/native/compa/taosdump.3479698875513/dbs.sql
@@ -0,0 +1,10 @@
+#!server_ver: ver:3.1.0.0
+#!taosdump_ver: 2.5.2_cf16c4d
+#!os_id: LINUX
+#!escape_char: true
+#!loose_mode: false
+#!charset: UTF-8
+CREATE DATABASE IF NOT EXISTS test REPLICA 1 DURATION 14400m KEEP 5256000m,5256000m,5256000m PRECISION 'ms' MINROWS 100 MAXROWS 4096 COMP 2 ;
+
+CREATE TABLE IF NOT EXISTS test.`meters`(`ts` timestamp,`bc` bool,`fc` float,`dc` double,`ti` tinyint,`si` smallint,`ic` int,`bi` bigint,`uti` tinyint unsigned,`usi` smallint unsigned,`ui` int unsigned,`ubi` bigint unsigned,`bin` binary(4),`nch` nchar(8)) TAGS(`tbc` bool,`tfc` float,`tdc` double,`tti` tinyint,`tsi` smallint,`tic` int,`tbi` bigint,`tuti` tinyint unsigned,`tusi` smallint unsigned,`tui` int unsigned,`tubi` bigint unsigned,`tbin` binary(4),`tnch` nchar(8));
+
diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479698875513/test.3479698875523.avro-tbtags b/tests/army/tools/taosdump/native/compa/taosdump.3479698875513/test.3479698875523.avro-tbtags
new file mode 100644
index 0000000000..eb1c6ac080
Binary files /dev/null and b/tests/army/tools/taosdump/native/compa/taosdump.3479698875513/test.3479698875523.avro-tbtags differ
diff --git a/tests/army/tools/taosdump/native/taosdumpCompa.py b/tests/army/tools/taosdump/native/taosdumpCompa.py
index 30088a1f6e..780cb303e7 100644
--- a/tests/army/tools/taosdump/native/taosdumpCompa.py
+++ b/tests/army/tools/taosdump/native/taosdumpCompa.py
@@ -59,21 +59,40 @@ class TDTestCase(TBase):
# sum pk db
sql = f"select {aggfun} from {db}.{stb}"
tdSql.query(sql)
- value = tdSql.getData(0,0)
-
- if value == expect:
- tdLog.info(f"{aggfun} not equal. real={value} expect={expect}")
- else:
- tdLog.info(f"{aggfun} equal. real={value} expect={expect}")
-
+ tdSql.checkData(0, 0, expect, show=True)
def verifyResult(self, db):
# compare sum(pk)
stb = "meters"
- self.checkSame(db, stb, "count(ts)", 100000)
- self.checkSame(db, stb, "sum(current)", 1005767.2491703)
- self.checkSame(db, stb, "avg(voltage)", 208.58818)
+ self.checkSame(db, stb, "count(ts)", 5000)
+ self.checkSame(db, stb, "last(ts)", "2023-11-15 07:36:39")
+ self.checkSame(db, stb, "last(bc)", False)
+ self.checkSame(db, stb, "sum(fc)", 2468.910999777726829)
+ self.checkSame(db, stb, "sum(dc)", 24811.172123999996984)
+ self.checkSame(db, stb, "sum(ti)", -411)
+ self.checkSame(db, stb, "sum(si)", 117073)
+ self.checkSame(db, stb, "sum(ic)", -39181)
+ self.checkSame(db, stb, "sum(bi)", -2231976)
+ self.checkSame(db, stb, "sum(uti)", 248825)
+ self.checkSame(db, stb, "sum(usi)", 248333)
+ self.checkSame(db, stb, "sum(ui)", 2484501)
+ self.checkSame(db, stb, "sum(ubi)", 25051956)
+ self.checkSame(db, stb, "last(bin)", "kwax")
+ self.checkSame(db, stb, "last(nch)", "0cYzPVcV")
+
+ self.checkSame(db, stb, "sum(tfc)", 3420.000076293945312)
+ self.checkSame(db, stb, "sum(tdc)", 3020.234999999780030)
+ self.checkSame(db, stb, "sum(tti)", -100000)
+ self.checkSame(db, stb, "sum(tsi)", -85000)
+ self.checkSame(db, stb, "sum(tic)", -4795000)
+ self.checkSame(db, stb, "sum(tbi)", -1125000)
+ self.checkSame(db, stb, "sum(tuti)", 475000)
+ self.checkSame(db, stb, "sum(tusi)", 460000)
+ self.checkSame(db, stb, "sum(tui)", 520000)
+ self.checkSame(db, stb, "sum(tubi)", 43155000)
+ self.checkSame(db, stb, "last(tbin)", "ywkc")
+ self.checkSame(db, stb, "last(tnch)", "kEoWzCBj")
def run(self):
# database
@@ -89,7 +108,6 @@ class TDTestCase(TBase):
# verify db
self.verifyResult(db)
-
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
diff --git a/tests/army/whole/checkErrorCode.py b/tests/army/whole/checkErrorCode.py
index b2aa1fce1b..1a14269f54 100644
--- a/tests/army/whole/checkErrorCode.py
+++ b/tests/army/whole/checkErrorCode.py
@@ -61,7 +61,8 @@ ignoreCodes = [
'0x80003107', '0x80003108', '0x80003109', '0x80003110', '0x80003111', '0x80003112', '0x80003250', '0x80004003', '0x80004004', '0x80004005',
'0x80004006', '0x80004007', '0x80004008', '0x80004009', '0x80004010', '0x80004011', '0x80004012', '0x80004013', '0x80004014', '0x80004015',
'0x80004016', '0x80004102', '0x80004103', '0x80004104', '0x80004105', '0x80004106', '0x80004107', '0x80004108', '0x80004109', '0x80005100',
- '0x80005101', '0x80006000', '0x80006100', '0x80006101', '0x80006102', '0x80000019', '0x80002639', '0x80002666', '0x80000237']
+ '0x80005101', '0x80006000', '0x80006100', '0x80006101', '0x80006102', '0x80000019', '0x80002639', '0x80002666', '0x80000237', '0x80004018',
+ '0x80004019']
class TDTestCase(TBase):
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index accd57d739..0d14722aaf 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -321,7 +321,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxGroupIds.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumeDiscontinuousData.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqOffset.py
-,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_primary_key.py
+#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_primary_key.py
,,n,system-test,python3 ./test.py -f 7-tmq/tmqDropConsumer.py
@@ -455,6 +455,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts5466.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_td33504.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts-5473.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts-5776.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts5906.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-32187.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-33225.py
diff --git a/tests/parallel_test/cases_tdengine.task b/tests/parallel_test/cases_tdengine.task
index 4ecfb7d919..e52fe68957 100644
--- a/tests/parallel_test/cases_tdengine.task
+++ b/tests/parallel_test/cases_tdengine.task
@@ -170,7 +170,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxGroupIds.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumeDiscontinuousData.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqOffset.py
-,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_primary_key.py
+#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_primary_key.py
,,n,system-test,python3 ./test.py -f 7-tmq/tmqDropConsumer.py
diff --git a/tests/parallel_test/longtimeruning_cases.task b/tests/parallel_test/longtimeruning_cases.task
index 1034771159..a4230882a9 100644
--- a/tests/parallel_test/longtimeruning_cases.task
+++ b/tests/parallel_test/longtimeruning_cases.task
@@ -10,6 +10,7 @@
# army-test
#,,y,army,./pytest.sh python3 ./test.py -f multi-level/mlevel_basic.py -N 3 -L 3 -D 2
+,,y,army,./pytest.sh python3 ./test.py -f stream/test_stream_notify.py
#tsim test
#,,y,script,./test.sh -f tsim/query/timeline.sim
diff --git a/tests/script/tsim/dnode/balance3.sim b/tests/script/tsim/dnode/balance3.sim
index d0235e096e..f2363fb0f3 100644
--- a/tests/script/tsim/dnode/balance3.sim
+++ b/tests/script/tsim/dnode/balance3.sim
@@ -70,10 +70,10 @@ endi
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -98,10 +98,10 @@ endi
if $data(3)[4] == leader then
$leaderExist = 1
endi
-if $data(3)[6] == leader then
+if $data(3)[7] == leader then
$leaderExist = 1
endi
-if $data(3)[8] == leader then
+if $data(3)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
diff --git a/tests/script/tsim/dnode/balance_replica3.sim b/tests/script/tsim/dnode/balance_replica3.sim
index afd7603b16..37b4a6beca 100644
--- a/tests/script/tsim/dnode/balance_replica3.sim
+++ b/tests/script/tsim/dnode/balance_replica3.sim
@@ -75,10 +75,10 @@ endi
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -105,10 +105,10 @@ endi
if $data(3)[4] == leader then
$leaderExist = 1
endi
-if $data(3)[6] == leader then
+if $data(3)[7] == leader then
$leaderExist = 1
endi
-if $data(3)[8] == leader then
+if $data(3)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -135,10 +135,10 @@ endi
if $data(4)[4] == leader then
$leaderExist = 1
endi
-if $data(4)[6] == leader then
+if $data(4)[7] == leader then
$leaderExist = 1
endi
-if $data(4)[8] == leader then
+if $data(4)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -165,10 +165,10 @@ endi
if $data(4)[4] == leader then
$leaderExist = 1
endi
-if $data(4)[6] == leader then
+if $data(4)[7] == leader then
$leaderExist = 1
endi
-if $data(4)[8] == leader then
+if $data(4)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -247,10 +247,10 @@ endi
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -307,10 +307,10 @@ endi
if $data(4)[4] == leader then
$leaderExist = 1
endi
-if $data(4)[6] == leader then
+if $data(4)[7] == leader then
$leaderExist = 1
endi
-if $data(4)[8] == leader then
+if $data(4)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -337,10 +337,10 @@ endi
if $data(4)[4] == leader then
$leaderExist = 1
endi
-if $data(4)[6] == leader then
+if $data(4)[7] == leader then
$leaderExist = 1
endi
-if $data(4)[8] == leader then
+if $data(4)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
diff --git a/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim b/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
index ef5001dcee..4425865bba 100644
--- a/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
+++ b/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
@@ -70,10 +70,10 @@ endi
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -100,10 +100,10 @@ endi
if $data(3)[4] == leader then
$leaderExist = 1
endi
-if $data(3)[6] == leader then
+if $data(3)[7] == leader then
$leaderExist = 1
endi
-if $data(3)[8] == leader then
+if $data(3)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -130,10 +130,10 @@ endi
if $data(4)[4] == leader then
$leaderExist = 1
endi
-if $data(4)[6] == leader then
+if $data(4)[7] == leader then
$leaderExist = 1
endi
-if $data(4)[8] == leader then
+if $data(4)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -160,10 +160,10 @@ endi
if $data(4)[4] == leader then
$leaderExist = 1
endi
-if $data(4)[6] == leader then
+if $data(4)[7] == leader then
$leaderExist = 1
endi
-if $data(4)[8] == leader then
+if $data(4)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -256,10 +256,10 @@ endi
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -286,10 +286,10 @@ endi
if $data(3)[4] == leader then
$leaderExist = 1
endi
-if $data(3)[6] == leader then
+if $data(3)[7] == leader then
$leaderExist = 1
endi
-if $data(3)[8] == leader then
+if $data(3)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -316,10 +316,10 @@ endi
if $data(4)[4] == leader then
$leaderExist = 1
endi
-if $data(4)[6] == leader then
+if $data(4)[7] == leader then
$leaderExist = 1
endi
-if $data(4)[8] == leader then
+if $data(4)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -346,10 +346,10 @@ endi
if $data(4)[4] == leader then
$leaderExist = 1
endi
-if $data(4)[6] == leader then
+if $data(4)[7] == leader then
$leaderExist = 1
endi
-if $data(4)[8] == leader then
+if $data(4)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
diff --git a/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim b/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim
index 2510692846..9c6abe536b 100644
--- a/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim
+++ b/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim
@@ -73,13 +73,13 @@ if $data(2)[4] == leader then
$follower1 = 3
$follower2 = 4
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
$leaderVnode = 3
$follower1 = 2
$follower2 = 4
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
$leaderVnode = 4
$follower1 = 2
@@ -109,10 +109,10 @@ endi
if $data(2)[3] != 2 then
return -1
endi
-if $data(2)[5] != 3 then
+if $data(2)[6] != 3 then
return -1
endi
-if $data(2)[7] != 4 then
+if $data(2)[9] != 4 then
return -1
endi
@@ -199,10 +199,10 @@ endi
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
index a576969697..ff34a8884e 100644
--- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
+++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
@@ -137,13 +137,13 @@ if $data(2)[4] == leader then
$follower1 = 3
$follower2 = 4
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
$leaderVnode = 3
$follower1 = 2
$follower2 = 4
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
$leaderVnode = 4
$follower1 = 2
diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim
index 739e3f2984..c6ee0dfeea 100644
--- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim
+++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim
@@ -116,13 +116,13 @@ if $data(2)[4] == leader then
$follower1 = 3
$follower2 = 4
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
$leaderVnode = 3
$follower1 = 2
$follower2 = 4
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
$leaderVnode = 4
$follower1 = 2
diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim
index 652f0c14b4..ae0b60c943 100644
--- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim
+++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim
@@ -122,13 +122,13 @@ if $data(2)[4] == leader then
$follower1 = 3
$follower2 = 4
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
$leaderVnode = 3
$follower1 = 2
$follower2 = 4
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
$leaderVnode = 4
$follower1 = 2
diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim
index 30e644f170..b3746dfcf4 100644
--- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim
+++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim
@@ -136,13 +136,13 @@ if $data(2)[4] == leader then
$follower1 = 3
$follower2 = 4
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
$leaderVnode = 3
$follower1 = 2
$follower2 = 4
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
$leaderVnode = 4
$follower1 = 2
diff --git a/tests/script/tsim/show/basic.sim b/tests/script/tsim/show/basic.sim
index 0acb97f1b6..595c386548 100644
--- a/tests/script/tsim/show/basic.sim
+++ b/tests/script/tsim/show/basic.sim
@@ -230,7 +230,7 @@ endi
sql_error show create stable t0;
sql show variables;
-if $rows != 88 then
+if $rows != 87 then
return -1
endi
diff --git a/tests/script/tsim/stream/checkpointInterval0.sim b/tests/script/tsim/stream/checkpointInterval0.sim
index a5e5c87704..d560edfab5 100644
--- a/tests/script/tsim/stream/checkpointInterval0.sim
+++ b/tests/script/tsim/stream/checkpointInterval0.sim
@@ -190,7 +190,7 @@ system sh/exec.sh -n dnode1 -s start
sql insert into t1 values(1648791223004,5,2,3,1.1);
loop4:
-sleep 1000
+run tsim/stream/checkTaskStatus.sim
sql select * from streamt;
diff --git a/tests/script/tsim/sync/3Replica1VgElect.sim b/tests/script/tsim/sync/3Replica1VgElect.sim
index aae1b25636..6ebee885a8 100644
--- a/tests/script/tsim/sync/3Replica1VgElect.sim
+++ b/tests/script/tsim/sync/3Replica1VgElect.sim
@@ -89,20 +89,20 @@ if $rows != $vgroups then
endi
if $data[0][4] == leader then
- if $data[0][6] == follower then
- if $data[0][8] == follower then
+ if $data[0][7] == follower then
+ if $data[0][10] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][3]
endi
endi
elif $data[0][6] == leader then
- if $data[0][4] == follower then
- if $data[0][8] == follower then
+ if $data[0][7] == follower then
+ if $data[0][10] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][5]
endi
endi
-elif $data[0][8] == leader then
+elif $data[0][10] == leader then
if $data[0][4] == follower then
- if $data[0][6] == follower then
+ if $data[0][7] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][7]
endi
endi
@@ -225,20 +225,20 @@ if $rows != $vgroups then
endi
if $data[0][4] == leader then
- if $data[0][6] == follower then
- if $data[0][8] == follower then
+ if $data[0][7] == follower then
+ if $data[0][10] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][3]
endi
endi
-elif $data[0][6] == leader then
+elif $data[0][7] == leader then
if $data[0][4] == follower then
- if $data[0][8] == follower then
+ if $data[0][10] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][5]
endi
endi
-elif $data[0][8] == leader then
+elif $data[0][10] == leader then
if $data[0][4] == follower then
- if $data[0][6] == follower then
+ if $data[0][7] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][7]
endi
endi
@@ -437,20 +437,20 @@ if $rows != $vgroups then
endi
if $data[0][4] == leader then
- if $data[0][6] == follower then
- if $data[0][8] == follower then
+ if $data[0][7] == follower then
+ if $data[0][10] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][3]
endi
endi
-elif $data[0][6] == leader then
+elif $data[0][7] == leader then
if $data[0][4] == follower then
- if $data[0][8] == follower then
+ if $data[0][10] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][5]
endi
endi
-elif $data[0][8] == leader then
+elif $data[0][10] == leader then
if $data[0][4] == follower then
- if $data[0][6] == follower then
+ if $data[0][7] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][7]
endi
endi
diff --git a/tests/script/tsim/sync/3Replica5VgElect.sim b/tests/script/tsim/sync/3Replica5VgElect.sim
index 37e4199e23..01bd41af42 100644
--- a/tests/script/tsim/sync/3Replica5VgElect.sim
+++ b/tests/script/tsim/sync/3Replica5VgElect.sim
@@ -92,20 +92,20 @@ if $rows != $vgroups then
endi
if $data[0][4] == leader then
- if $data[0][6] == follower then
- if $data[0][8] == follower then
+ if $data[0][7] == follower then
+ if $data[0][10] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][3]
endi
endi
-elif $data[0][6] == leader then
+elif $data[0][7] == leader then
if $data[0][4] == follower then
- if $data[0][8] == follower then
+ if $data[0][10] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][5]
endi
endi
-elif $data[0][8] == leader then
+elif $data[0][10] == leader then
if $data[0][4] == follower then
- if $data[0][6] == follower then
+ if $data[0][7] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][7]
endi
endi
@@ -114,20 +114,20 @@ else
endi
if $data[1][4] == leader then
- if $data[1][6] == follower then
- if $data[1][8] == follower then
+ if $data[1][7] == follower then
+ if $data[1][10] == follower then
print ---- vgroup $data[1][0] leader locate on dnode $data[1][3]
endi
endi
-elif $data[1][6] == leader then
+elif $data[1][7] == leader then
if $data[1][4] == follower then
- if $data[1][8] == follower then
+ if $data[1][10] == follower then
print ---- vgroup $data[1][0] leader locate on dnode $data[1][5]
endi
endi
-elif $data[1][8] == leader then
+elif $data[1][10] == leader then
if $data[1][4] == follower then
- if $data[1][6] == follower then
+ if $data[1][7] == follower then
print ---- vgroup $data[1][0] leader locate on dnode $data[1][7]
endi
endi
@@ -136,20 +136,20 @@ else
endi
if $data[2][4] == leader then
- if $data[2][6] == follower then
- if $data[2][8] == follower then
+ if $data[2][7] == follower then
+ if $data[2][10] == follower then
print ---- vgroup $data[2][0] leader locate on dnode $data[2][3]
endi
endi
-elif $data[2][6] == leader then
+elif $data[2][7] == leader then
if $data[2][4] == follower then
- if $data[2][8] == follower then
+ if $data[2][10] == follower then
print ---- vgroup $data[2][0] leader locate on dnode $data[2][5]
endi
endi
-elif $data[2][8] == leader then
+elif $data[2][10] == leader then
if $data[2][4] == follower then
- if $data[2][6] == follower then
+ if $data[2][7] == follower then
print ---- vgroup $data[2][0] leader locate on dnode $data[2][7]
endi
endi
@@ -158,20 +158,20 @@ else
endi
if $data[3][4] == leader then
- if $data[3][6] == follower then
- if $data[3][8] == follower then
+ if $data[3][7] == follower then
+ if $data[3][10] == follower then
print ---- vgroup $data[3][0] leader locate on dnode $data[3][3]
endi
endi
-elif $data[3][6] == leader then
+elif $data[3][7] == leader then
if $data[3][4] == follower then
- if $data[3][8] == follower then
+ if $data[3][10] == follower then
print ---- vgroup $data[3][0] leader locate on dnode $data[3][5]
endi
endi
-elif $data[3][8] == leader then
+elif $data[3][10] == leader then
if $data[3][4] == follower then
- if $data[3][6] == follower then
+ if $data[3][7] == follower then
print ---- vgroup $data[3][0] leader locate on dnode $data[3][7]
endi
endi
@@ -180,20 +180,20 @@ else
endi
if $data[4][4] == leader then
- if $data[4][6] == follower then
- if $data[4][8] == follower then
+ if $data[4][7] == follower then
+ if $data[4][10] == follower then
print ---- vgroup $data[4][0] leader locate on dnode $data[4][3]
endi
endi
-elif $data[4][6] == leader then
+elif $data[4][7] == leader then
if $data[4][4] == follower then
- if $data[4][8] == follower then
+ if $data[4][10] == follower then
print ---- vgroup $data[4][0] leader locate on dnode $data[4][5]
endi
endi
-elif $data[4][8] == leader then
+elif $data[4][10] == leader then
if $data[4][4] == follower then
- if $data[4][6] == follower then
+ if $data[4][7] == follower then
print ---- vgroup $data[4][0] leader locate on dnode $data[4][7]
endi
endi
@@ -305,20 +305,20 @@ if $rows != $vgroups then
endi
if $data[0][4] == leader then
- if $data[0][6] == follower then
- if $data[0][8] == follower then
+ if $data[0][7] == follower then
+ if $data[0][10] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][3]
endi
endi
-elif $data[0][6] == leader then
+elif $data[0][7] == leader then
if $data[0][4] == follower then
- if $data[0][8] == follower then
+ if $data[0][10] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][5]
endi
endi
-elif $data[0][8] == leader then
+elif $data[0][10] == leader then
if $data[0][4] == follower then
- if $data[0][6] == follower then
+ if $data[0][7] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][7]
endi
endi
@@ -327,20 +327,20 @@ else
endi
if $data[1][4] == leader then
- if $data[1][6] == follower then
- if $data[1][8] == follower then
+ if $data[1][7] == follower then
+ if $data[1][10] == follower then
print ---- vgroup $data[1][0] leader locate on dnode $data[1][3]
endi
endi
-elif $data[1][6] == leader then
+elif $data[1][7] == leader then
if $data[1][4] == follower then
- if $data[1][8] == follower then
+ if $data[1][10] == follower then
print ---- vgroup $data[1][0] leader locate on dnode $data[1][5]
endi
endi
-elif $data[1][8] == leader then
+elif $data[1][10] == leader then
if $data[1][4] == follower then
- if $data[1][6] == follower then
+ if $data[1][7] == follower then
print ---- vgroup $data[1][0] leader locate on dnode $data[1][7]
endi
endi
@@ -349,20 +349,20 @@ else
endi
if $data[2][4] == leader then
- if $data[2][6] == follower then
- if $data[2][8] == follower then
+ if $data[2][7] == follower then
+ if $data[2][10] == follower then
print ---- vgroup $data[2][0] leader locate on dnode $data[2][3]
endi
endi
-elif $data[2][6] == leader then
+elif $data[2][7] == leader then
if $data[2][4] == follower then
- if $data[2][8] == follower then
+ if $data[2][10] == follower then
print ---- vgroup $data[2][0] leader locate on dnode $data[2][5]
endi
endi
-elif $data[2][8] == leader then
+elif $data[2][10] == leader then
if $data[2][4] == follower then
- if $data[2][6] == follower then
+ if $data[2][7] == follower then
print ---- vgroup $data[2][0] leader locate on dnode $data[2][7]
endi
endi
@@ -371,20 +371,20 @@ else
endi
if $data[3][4] == leader then
- if $data[3][6] == follower then
- if $data[3][8] == follower then
+ if $data[3][7] == follower then
+ if $data[3][10] == follower then
print ---- vgroup $data[3][0] leader locate on dnode $data[3][3]
endi
endi
-elif $data[3][6] == leader then
+elif $data[3][7] == leader then
if $data[3][4] == follower then
- if $data[3][8] == follower then
+ if $data[3][10] == follower then
print ---- vgroup $data[3][0] leader locate on dnode $data[3][5]
endi
endi
-elif $data[3][8] == leader then
+elif $data[3][10] == leader then
if $data[3][4] == follower then
- if $data[3][6] == follower then
+ if $data[3][7] == follower then
print ---- vgroup $data[3][0] leader locate on dnode $data[3][7]
endi
endi
@@ -393,20 +393,20 @@ else
endi
if $data[4][4] == leader then
- if $data[4][6] == follower then
- if $data[4][8] == follower then
+ if $data[4][7] == follower then
+ if $data[4][10] == follower then
print ---- vgroup $data[4][0] leader locate on dnode $data[4][3]
endi
endi
-elif $data[4][6] == leader then
+elif $data[4][7] == leader then
if $data[4][4] == follower then
- if $data[4][8] == follower then
+ if $data[4][10] == follower then
print ---- vgroup $data[4][0] leader locate on dnode $data[4][5]
endi
endi
-elif $data[4][8] == leader then
+elif $data[4][10] == leader then
if $data[4][4] == follower then
- if $data[4][6] == follower then
+ if $data[4][7] == follower then
print ---- vgroup $data[4][0] leader locate on dnode $data[4][7]
endi
endi
@@ -607,20 +607,20 @@ if $rows != $vgroups then
endi
if $data[0][4] == leader then
- if $data[0][6] == follower then
- if $data[0][8] == follower then
+ if $data[0][7] == follower then
+ if $data[0][10] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][3]
endi
endi
-elif $data[0][6] == leader then
+elif $data[0][7] == leader then
if $data[0][4] == follower then
- if $data[0][8] == follower then
+ if $data[0][10] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][5]
endi
endi
-elif $data[0][8] == leader then
+elif $data[0][10] == leader then
if $data[0][4] == follower then
- if $data[0][6] == follower then
+ if $data[0][7] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][7]
endi
endi
@@ -629,20 +629,20 @@ else
endi
if $data[1][4] == leader then
- if $data[1][6] == follower then
- if $data[1][8] == follower then
+ if $data[1][7] == follower then
+ if $data[1][10] == follower then
print ---- vgroup $data[1][0] leader locate on dnode $data[1][3]
endi
endi
-elif $data[1][6] == leader then
+elif $data[1][7] == leader then
if $data[1][4] == follower then
- if $data[1][8] == follower then
+ if $data[1][10] == follower then
print ---- vgroup $data[1][0] leader locate on dnode $data[1][5]
endi
endi
-elif $data[1][8] == leader then
+elif $data[1][10] == leader then
if $data[1][4] == follower then
- if $data[1][6] == follower then
+ if $data[1][7] == follower then
print ---- vgroup $data[1][0] leader locate on dnode $data[1][7]
endi
endi
@@ -651,20 +651,20 @@ else
endi
if $data[2][4] == leader then
- if $data[2][6] == follower then
- if $data[2][8] == follower then
+ if $data[2][7] == follower then
+ if $data[2][10] == follower then
print ---- vgroup $data[2][0] leader locate on dnode $data[2][3]
endi
endi
-elif $data[2][6] == leader then
+elif $data[2][7] == leader then
if $data[2][4] == follower then
- if $data[2][8] == follower then
+ if $data[2][10] == follower then
print ---- vgroup $data[2][0] leader locate on dnode $data[2][5]
endi
endi
-elif $data[2][8] == leader then
+elif $data[2][10] == leader then
if $data[2][4] == follower then
- if $data[2][6] == follower then
+ if $data[2][7] == follower then
print ---- vgroup $data[2][0] leader locate on dnode $data[2][7]
endi
endi
@@ -673,20 +673,20 @@ else
endi
if $data[3][4] == leader then
- if $data[3][6] == follower then
- if $data[3][8] == follower then
+ if $data[3][7] == follower then
+ if $data[3][10] == follower then
print ---- vgroup $data[3][0] leader locate on dnode $data[3][3]
endi
endi
-elif $data[3][6] == leader then
+elif $data[3][7] == leader then
if $data[3][4] == follower then
- if $data[3][8] == follower then
+ if $data[3][10] == follower then
print ---- vgroup $data[3][0] leader locate on dnode $data[3][5]
endi
endi
-elif $data[3][8] == leader then
+elif $data[3][10] == leader then
if $data[3][4] == follower then
- if $data[3][6] == follower then
+ if $data[3][7] == follower then
print ---- vgroup $data[3][0] leader locate on dnode $data[3][7]
endi
endi
@@ -695,20 +695,20 @@ else
endi
if $data[4][4] == leader then
- if $data[4][6] == follower then
- if $data[4][8] == follower then
+ if $data[4][7] == follower then
+ if $data[4][10] == follower then
print ---- vgroup $data[4][0] leader locate on dnode $data[4][3]
endi
endi
-elif $data[4][6] == leader then
+elif $data[4][7] == leader then
if $data[4][4] == follower then
- if $data[4][8] == follower then
+ if $data[4][10] == follower then
print ---- vgroup $data[4][0] leader locate on dnode $data[4][5]
endi
endi
-elif $data[4][8] == leader then
+elif $data[4][10] == leader then
if $data[4][4] == follower then
- if $data[4][6] == follower then
+ if $data[4][7] == follower then
print ---- vgroup $data[4][0] leader locate on dnode $data[4][7]
endi
endi
diff --git a/tests/script/tsim/sync/vnodesnapshot-rsma-test.sim b/tests/script/tsim/sync/vnodesnapshot-rsma-test.sim
index 8b1720d213..d07c66bc53 100644
--- a/tests/script/tsim/sync/vnodesnapshot-rsma-test.sim
+++ b/tests/script/tsim/sync/vnodesnapshot-rsma-test.sim
@@ -90,20 +90,20 @@ if $rows != $vgroups then
endi
if $data[0][4] == leader then
- if $data[0][6] == follower then
- if $data[0][8] == follower then
+ if $data[0][7] == follower then
+ if $data[0][10] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][3]
endi
endi
-elif $data[0][6] == leader then
+elif $data[0][7] == leader then
if $data[0][4] == follower then
- if $data[0][8] == follower then
+ if $data[0][10] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][5]
endi
endi
-elif $data[0][8] == leader then
+elif $data[0][10] == leader then
if $data[0][4] == follower then
- if $data[0][6] == follower then
+ if $data[0][7] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][7]
endi
endi
diff --git a/tests/script/tsim/valgrind/checkError1.sim b/tests/script/tsim/valgrind/checkError1.sim
index 8ac43ebaf3..54427986a5 100644
--- a/tests/script/tsim/valgrind/checkError1.sim
+++ b/tests/script/tsim/valgrind/checkError1.sim
@@ -120,7 +120,7 @@ if $rows != 3 then
endi
sql show variables;
-if $rows != 88 then
+if $rows != 87 then
return -1
endi
diff --git a/tests/script/tsim/vnode/replica3_basic.sim b/tests/script/tsim/vnode/replica3_basic.sim
index 473e53e84b..5da9fe6b38 100644
--- a/tests/script/tsim/vnode/replica3_basic.sim
+++ b/tests/script/tsim/vnode/replica3_basic.sim
@@ -96,10 +96,10 @@ $leaderExist = 0
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -141,10 +141,10 @@ $leaderExist = 0
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -180,10 +180,10 @@ $leaderExist = 0
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -220,10 +220,10 @@ $leaderExist = 0
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -260,10 +260,10 @@ $leaderExist = 0
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -300,10 +300,10 @@ $leaderExist = 0
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -340,10 +340,10 @@ $leaderExist = 0
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
diff --git a/tests/script/tsim/vnode/replica3_import.sim b/tests/script/tsim/vnode/replica3_import.sim
index 99608c72e2..2840fd45a1 100644
--- a/tests/script/tsim/vnode/replica3_import.sim
+++ b/tests/script/tsim/vnode/replica3_import.sim
@@ -71,10 +71,10 @@ endi
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -157,10 +157,10 @@ $leaderExist = 0
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -227,10 +227,10 @@ $leaderExist = 0
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -275,10 +275,10 @@ $leaderExist = 0
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -313,10 +313,10 @@ $leaderExist = 0
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
diff --git a/tests/script/tsim/vnode/replica3_many.sim b/tests/script/tsim/vnode/replica3_many.sim
index e3c73b2018..930eebc688 100644
--- a/tests/script/tsim/vnode/replica3_many.sim
+++ b/tests/script/tsim/vnode/replica3_many.sim
@@ -68,10 +68,10 @@ endi
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -96,10 +96,10 @@ endi
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -121,10 +121,10 @@ $leaderExist = 0
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -146,10 +146,10 @@ $leaderExist = 0
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
diff --git a/tests/script/tsim/vnode/replica3_repeat.sim b/tests/script/tsim/vnode/replica3_repeat.sim
index 10e18d01d0..defee0fe0a 100644
--- a/tests/script/tsim/vnode/replica3_repeat.sim
+++ b/tests/script/tsim/vnode/replica3_repeat.sim
@@ -62,10 +62,10 @@ endi
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
diff --git a/tests/script/tsim/vnode/replica3_vgroup.sim b/tests/script/tsim/vnode/replica3_vgroup.sim
index 6643966344..a7a8b725b9 100644
--- a/tests/script/tsim/vnode/replica3_vgroup.sim
+++ b/tests/script/tsim/vnode/replica3_vgroup.sim
@@ -70,10 +70,10 @@ $leaderExist = 0
if $data(2)[4] == leader then
$leaderExist = 1
endi
-if $data(2)[6] == leader then
+if $data(2)[7] == leader then
$leaderExist = 1
endi
-if $data(2)[8] == leader then
+if $data(2)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
@@ -83,10 +83,10 @@ $leaderExist = 0
if $data(3)[4] == leader then
$leaderExist = 1
endi
-if $data(3)[6] == leader then
+if $data(3)[7] == leader then
$leaderExist = 1
endi
-if $data(3)[8] == leader then
+if $data(3)[10] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
diff --git a/tests/system-test/0-others/grant.py b/tests/system-test/0-others/grant.py
index 490541539f..25af6eb842 100644
--- a/tests/system-test/0-others/grant.py
+++ b/tests/system-test/0-others/grant.py
@@ -135,8 +135,38 @@ class TDTestCase:
port = dnode.cfgDict["serverPort"]
config_dir = dnode.cfgDir
return taos.connect(host=host, port=int(port), config=config_dir)
+
+ def getShowGrantsTimeSeries(self, maxRetry=10):
+ for nRetry in range(maxRetry):
+ tdSql.query("show grants")
+ timeseries = tdSql.queryResult[0][5]
+ tdSql.query("show grants full")
+ full_timeseries = tdSql.queryResult[1][3]
+ if timeseries == full_timeseries:
+ return int(timeseries.split('/')[0])
+ else:
+ tdLog.info(f"timeseries: {timeseries}, != full_timeseries: {full_timeseries}, retry: {nRetry}")
+ time.sleep(1)
+ raise Exception("Timeseries not equal within {maxRetry} seconds")
- def s1_check_alive(self):
+ def getTablesTimeSeries(self):
+ tdSql.query(f"select cast(sum(columns-1) as int) as tss from information_schema.ins_tables where db_name not in ('information_schema', 'performance_schema', 'audit')")
+ return int(tdSql.queryResult[0][0])
+
+ def checkGrantsTimeSeries(self, prompt="", nExpectedTimeSeries=0, maxRetry=10):
+ for nRetry in range(maxRetry):
+ tss_grant = self.getShowGrantsTimeSeries()
+ if tss_grant == nExpectedTimeSeries:
+ tss_table = self.getTablesTimeSeries()
+ if tss_grant == tss_table:
+ tdLog.info(f"{prompt}: tss_grant: {tss_grant} == tss_table: {tss_table}")
+ return
+ else:
+ raise Exception(f"{prompt}: tss_grant: {tss_grant} != tss_table: {tss_table}")
+ time.sleep(1)
+ raise Exception(f"{prompt}: tss_grant: {tss_grant} != nExpectedTimeSeries: {nExpectedTimeSeries}")
+
+ def s1_check_timeseries(self):
# check cluster alive
tdLog.printNoPrefix("======== test cluster alive: ")
tdSql.checkDataLoop(0, 0, 1, "show cluster alive;", 20, 0.5)
@@ -144,6 +174,46 @@ class TDTestCase:
tdSql.query("show db.alive;")
tdSql.checkData(0, 0, 1)
+ # check timeseries
+ tss_grant = 5
+ for i in range(0, 3):
+ tdLog.printNoPrefix(f"======== test timeseries: loop{i}")
+ self.checkGrantsTimeSeries("initial check", tss_grant)
+ tdSql.execute("create database if not exists db100")
+ tdSql.execute("create table db100.stb100(ts timestamp, c0 int,c1 bigint,c2 int,c3 float,c4 double) tags(t0 bigint unsigned)")
+ tdSql.execute("create table db100.ctb100 using db100.stb100 tags(100)")
+ tdSql.execute("create table db100.ctb101 using db100.stb100 tags(101)")
+ tdSql.execute("create table db100.ntb100 (ts timestamp, c0 int,c1 bigint,c2 int,c3 float,c4 double)")
+ tdSql.execute("create table db100.ntb101 (ts timestamp, c0 int,c1 bigint,c2 int,c3 float,c4 double)")
+ tss_grant += 20
+ self.checkGrantsTimeSeries("create tables and check", tss_grant)
+ tdSql.execute("alter table db100.stb100 add column c5 int")
+ tdSql.execute("alter stable db100.stb100 add column c6 int")
+ tdSql.execute("alter table db100.stb100 add tag t1 int")
+ tss_grant += 4
+ self.checkGrantsTimeSeries("add stable column and check", tss_grant)
+ tdSql.execute("create table db100.ctb102 using db100.stb100 tags(102, 102)")
+ tdSql.execute("alter table db100.ctb100 set tag t0=1000")
+ tdSql.execute("alter table db100.ntb100 add column c5 int")
+ tss_grant += 8
+ self.checkGrantsTimeSeries("add ntable column and check", tss_grant)
+ tdSql.execute("alter table db100.stb100 drop column c5")
+ tdSql.execute("alter table db100.stb100 drop tag t1")
+ tdSql.execute("alter table db100.ntb100 drop column c0")
+ tdSql.execute("alter table db100.stb100 drop column c0")
+ tss_grant -= 7
+ self.checkGrantsTimeSeries("drop stb/ntb column and check", tss_grant)
+ tdSql.execute("drop table db100.ctb100")
+ tdSql.execute("drop table db100.ntb100")
+ tss_grant -= 10
+ self.checkGrantsTimeSeries("drop ctb/ntb and check", tss_grant)
+ tdSql.execute("drop table db100.stb100")
+ tss_grant -= 10
+ self.checkGrantsTimeSeries("drop stb and check", tss_grant)
+ tdSql.execute("drop database db100")
+ tss_grant -= 5
+ self.checkGrantsTimeSeries("drop database and check", tss_grant)
+
def s2_check_show_grants_ungranted(self):
tdLog.printNoPrefix("======== test show grants ungranted: ")
self.infoPath = os.path.join(self.workPath, ".clusterInfo")
@@ -221,7 +291,7 @@ class TDTestCase:
# print(self.master_dnode.cfgDict)
# keep the order of following steps
self.s0_five_dnode_one_mnode()
- self.s1_check_alive()
+ self.s1_check_timeseries()
self.s2_check_show_grants_ungranted()
self.s3_check_show_grants_granted()
diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py
index 5bdb744f6c..96cd596a20 100644
--- a/tests/system-test/0-others/information_schema.py
+++ b/tests/system-test/0-others/information_schema.py
@@ -222,8 +222,7 @@ class TDTestCase:
tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'")
tdLog.info(len(tdSql.queryResult))
- tdSql.checkEqual(True, len(tdSql.queryResult) in range(313, 314))
-
+ tdSql.checkEqual(True, len(tdSql.queryResult) in range(320, 321))
tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'")
tdSql.checkEqual(61, len(tdSql.queryResult))
diff --git a/tests/system-test/1-insert/alter_replica.py b/tests/system-test/1-insert/alter_replica.py
index 900b64d943..6d364618ee 100644
--- a/tests/system-test/1-insert/alter_replica.py
+++ b/tests/system-test/1-insert/alter_replica.py
@@ -27,8 +27,8 @@ class TDTestCase:
flag = 0
for vgid in range (vgNum) :
v1_status = tdSql.queryResult[vgid][4]
- v2_status = tdSql.queryResult[vgid][6]
- v3_status = tdSql.queryResult[vgid][8]
+ v2_status = tdSql.queryResult[vgid][7]
+ v3_status = tdSql.queryResult[vgid][10]
if ((v1_status == 'leader') and (v2_status == 'follower') and (v3_status == 'follower')) \
or ((v2_status == 'leader') and (v1_status == 'follower') and (v3_status == 'follower')) \
or ((v3_status == 'leader') and (v2_status == 'follower') and (v1_status == 'follower')):
diff --git a/tests/system-test/2-query/db.py b/tests/system-test/2-query/db.py
index 895df852c7..f380fdf00b 100644
--- a/tests/system-test/2-query/db.py
+++ b/tests/system-test/2-query/db.py
@@ -47,7 +47,7 @@ class TDTestCase:
def case2(self):
tdSql.query("show variables")
- tdSql.checkRows(88)
+ tdSql.checkRows(87)
for i in range(self.replicaVar):
tdSql.query("show dnode %d variables like 'debugFlag'" % (i + 1))
diff --git a/tests/system-test/2-query/distinct.py b/tests/system-test/2-query/distinct.py
index 5025b39753..bef1c18ad8 100644
--- a/tests/system-test/2-query/distinct.py
+++ b/tests/system-test/2-query/distinct.py
@@ -255,7 +255,31 @@ class TDTestCase:
tdSql.error(f"select distinct t1, t0 from (select t1,t0 from {dbname}.stb1 where t0 > 2 group by ts) where t1 < 3")
tdSql.query(f"select distinct stb1.t1, stb1.t2 from {dbname}.stb1, {dbname}.stb2 where stb1.ts=stb2.ts and stb1.t2=stb2.t4")
tdSql.query(f"select distinct t1.t1, t1.t2 from {dbname}.t1, {dbname}.t2 where t1.ts=t2.ts ")
+
+ self.ts5971()
+ def ts5971(self):
+ dbname = "db"
+
+ tdSql.execute(f"DROP TABLE IF EXISTS {dbname}.t5971")
+ tdSql.execute(f"create table {dbname}.t5971 (time TIMESTAMP, c1 INT)")
+ tdSql.execute(f"INSERT INTO {dbname}.t5971(time, c1) VALUES (1641024000000, 1), (1641024005000, 2)")
+ tdSql.query(f"SELECT DISTINCT CSUM(c1), time FROM {dbname}.t5971 ORDER BY time")
+ tdSql.checkRows(2)
+ tdSql.checkData(0, 0, 1)
+ tdSql.checkData(0, 1, 1641024000000)
+ tdSql.checkData(1, 0, 3)
+ tdSql.checkData(1, 1, 1641024005000)
+
+ tdSql.query(f"SELECT DISTINCT CSUM(c1), time AS ref FROM {dbname}.t5971 ORDER BY ref")
+ tdSql.checkRows(2)
+ tdSql.checkData(0, 0, 1)
+ tdSql.checkData(0, 1, 1641024000000)
+ tdSql.checkData(1, 0, 3)
+ tdSql.checkData(1, 1, 1641024005000)
+
+ tdSql.query(f"SELECT DISTINCT CSUM(c1), time FROM {dbname}.t5971")
+ tdSql.checkRows(2)
def stop(self):
diff --git a/tests/system-test/3-enterprise/restore/restoreBasic.py b/tests/system-test/3-enterprise/restore/restoreBasic.py
index 74cf572018..9ca48ed7e8 100644
--- a/tests/system-test/3-enterprise/restore/restoreBasic.py
+++ b/tests/system-test/3-enterprise/restore/restoreBasic.py
@@ -80,7 +80,7 @@ class RestoreBasic:
for i in range(8):
leader = False
for j in range(3):
- status = tdSql.getData(i, 4 + j*2)
+ status = tdSql.getData(i, 4 + j*3)
if status == "leader":
leader = True
elif status == "follower":
diff --git a/tests/system-test/6-cluster/clusterCommonCheck.py b/tests/system-test/6-cluster/clusterCommonCheck.py
index f3e2b5d5bc..c42e3d918d 100644
--- a/tests/system-test/6-cluster/clusterCommonCheck.py
+++ b/tests/system-test/6-cluster/clusterCommonCheck.py
@@ -251,9 +251,9 @@ class ClusterComCheck:
return True
elif self.db_replica == 3 :
- vgroup_status_first=[tdSql.queryResult[0][4],tdSql.queryResult[0][6],tdSql.queryResult[0][8]]
+ vgroup_status_first=[tdSql.queryResult[0][4],tdSql.queryResult[0][7],tdSql.queryResult[0][10]]
- vgroup_status_last=[tdSql.queryResult[last_number][4],tdSql.queryResult[last_number][6],tdSql.queryResult[last_number][8]]
+ vgroup_status_last=[tdSql.queryResult[last_number][4],tdSql.queryResult[last_number][7],tdSql.queryResult[last_number][10]]
if vgroup_status_first.count('leader') == 1 and vgroup_status_first.count('follower') == 2:
if vgroup_status_last.count('leader') == 1 and vgroup_status_last.count('follower') == 2:
tdSql.query(f"select `replica` from information_schema.ins_databases where `name`='{db_name}';")
diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py
index 9eea490eba..cdc1b1d607 100644
--- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py
+++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py
@@ -135,11 +135,9 @@ class TDTestCase:
vgroup_id = vgroup_info[0]
vgroup_status = []
for ind , role in enumerate(vgroup_info[3:-4]):
-
- if ind%2==0:
- continue
- else:
+ if role in ['leader', 'leader*', 'leader**', 'follower']:
vgroup_status.append(role)
+
if vgroup_status.count("leader")!=1 or vgroup_status.count("follower")!=2:
status = False
return status
diff --git a/tests/system-test/7-tmq/taosx-performance.py b/tests/system-test/7-tmq/taosx-performance.py
new file mode 100755
index 0000000000..5f7c6c3e07
--- /dev/null
+++ b/tests/system-test/7-tmq/taosx-performance.py
@@ -0,0 +1,280 @@
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+
+sys.path.append("../../pytest")
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+from taos.tmq import *
+
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+tdDnodes1 = TDDnodes()
+tdDnodes2 = TDDnodes()
+vgroups = 1
+tables = 1
+rows_of_each_table = 1
+path = "./taosx_perf"
+dnode1 = "./dnode1"
+dnode2 = "./dnode2"
+cfg = "sim/dnode1/cfg"
+taosx = "taosx"
+taosd = "taosd"
+taosxLog = "taosx.log"
+taosxTimeout = 2
+timeCost = []
+speedupStr = []
+insertJson = '''{
+ "filetype": "insert",
+ "cfgdir": "/etc/taos",
+ "host": "127.0.0.1",
+ "port": 6030,
+ "user": "root",
+ "password": "taosdata",
+ "connection_pool_size": 20,
+ "thread_bind_vgroup": "yes",
+ "thread_count": 20,
+ "create_table_thread_count": 16,
+ "result_file": "./insert_res.txt",
+ "confirm_parameter_prompt": "no",
+ "num_of_records_per_req": 10,
+ "prepared_rand": 10000,
+ "chinese": "no",
+ "escape_character": "yes",
+ "continue_if_fail": "no",
+ "databases": [
+ {
+ "dbinfo": {
+ "name": "test",
+ "drop": "yes",
+ "vgroups": 8,
+ "precision": "ms",
+ "WAL_RETENTION_PERIOD": 864000
+ },
+ "super_tables": [
+ {
+ "name": "meters",
+ "child_table_exists": "no",
+ "childtable_count": 10000,
+ "childtable_prefix": "d",
+ "auto_create_table": "yes",
+ "batch_create_tbl_num": 500,
+ "data_source": "rand",
+ "insert_mode": "taosc",
+ "non_stop_mode": "no",
+ "line_protocol": "line",
+ "insert_rows": 1000,
+ "childtable_limit": 0,
+ "childtable_offset": 0,
+ "interlace_rows": 1,
+ "insert_interval": 0,
+ "partial_col_num": 0,
+ "timestamp_step": 10,
+ "start_timestamp": "2020-10-01 00:00:00.000",
+ "sample_format": "csv",
+ "sample_file": "./sample.csv",
+ "use_sample_ts": "no",
+ "tags_file": "",
+ "generate_row_rule": 2,
+ "columns": [
+ {"type": "TINYINT", "name": "current", "max": 128, "min": 1 },
+ { "type": "BOOL", "name": "phaseewe" },
+ { "type": "BINARY", "name": "str", "len":16374,
+ "values": [
+ "{kkey00000k: kvalue00000k, kkey00001k: kvalue00001k, kkey00002k: kvalue00002k, kkey00003k: kvalue00003k, kkey00004k: kvalue00004k, kkey00005k: kvalue00005k, kkey00006k: kvalue00006k, kkey00007k: kvalue00007k, kkey00008k: kvalue00008k, kkey00009k: kvalue00009k, kkey00010k: kvalue00010k, kkey00011k: kvalue00011k, kkey00012k: kvalue00012k, kkey00013k: kvalue00013k, kkey00014k: kvalue00014k, kkey00015k: kvalue00015k, kkey00016k: kvalue00016k, kkey00017k: kvalue00017k, kkey00018k: kvalue00018k, kkey00019k: kvalue00019k, kkey00020k: kvalue00020k, kkey00021k: kvalue00021k, kkey00022k: kvalue00022k, kkey00023k: kvalue00023k, kkey00024k: kvalue00024k, kkey00025k: kvalue00025k, kkey00026k: kvalue00026k, kkey00027k: kvalue00027k, kkey00028k: kvalue00028k, kkey00029k: kvalue00029k, kkey00030k: kvalue00030k, kkey00031k: kvalue00031k, kkey00032k: kvalue00032k, kkey00033k: kvalue00033k, kkey00034k: kvalue00034k, kkey00035k: kvalue00035k, kkey00036k: kvalue00036k, kkey00037k: kvalue00037k, kkey00038k: kvalue00038k, kkey00039k: kvalue00039k, kkey00040k: kvalue00040k, kkey00041k: kvalue00041k, kkey00042k: kvalue00042k, kkey00043k: kvalue00043k, kkey00044k: kvalue00044k, kkey00045k: kvalue00045k, kkey00046k: kvalue00046k, kkey00047k: kvalue00047k, kkey00048k: kvalue00048k, kkey00049k: kvalue00049k, kkey00050k: kvalue00050k, kkey00051k: kvalue00051k, kkey00052k: kvalue00052k, kkey00053k: kvalue00053k, kkey00054k: kvalue00054k, kkey00055k: kvalue00055k, kkey00056k: kvalue00056k, kkey00057k: kvalue00057k, kkey00058k: kvalue00058k, kkey00059k: kvalue00059k, kkey00060k: kvalue00060k, kkey00061k: kvalue00061k, kkey00062k: kvalue00062k, kkey00063k: kvalue00063k, kkey00064k: kvalue00064k, kkey00065k: kvalue00065k, kkey00066k: kvalue00066k, kkey00067k: kvalue00067k, kkey00068k: kvalue00068k, kkey00069k: kvalue00069k, kkey00070k: kvalue00070k, kkey00071k: kvalue00071k, kkey00072k: kvalue00072k, kkey00073k: kvalue00073k, kkey00074k: kvalue00074k, kkey00075k: kvalue00075k, kkey00076k: kvalue00076k, kkey00077k: kvalue00077k, kkey00078k: kvalue00078k, kkey00079k: kvalue00079k, kkey00080k: kvalue00080k, kkey00081k: kvalue00081k, kkey00082k: kvalue00082k, kkey00083k: kvalue00083k, kkey00084k: kvalue00084k, kkey00085k: kvalue00085k, kkey00086k: kvalue00086k, kkey00087k: kvalue00087k, kkey00088k: kvalue00088k, kkey00089k: kvalue00089k, kkey00090k: kvalue00090k, kkey00091k: kvalue00091k, kkey00092k: kvalue00092k, kkey00093k: kvalue00093k, kkey00094k: kvalue00094k, kkey00095k: kvalue00095k, kkey00096k: kvalue00096k, kkey00097k: kvalue00097k, kkey00098k: kvalue00098k, kkey00099k: kvalue00099k, kkey00100k: kvalue00100k, kkey00101k: kvalue00101k, kkey00102k: kvalue00102k, kkey00103k: kvalue00103k, kkey00104k: kvalue00104k, kkey00105k: kvalue00105k, kkey00106k: kvalue00106k, kkey00107k: kvalue00107k, kkey00108k: kvalue00108k, kkey00109k: kvalue00109k, kkey00110k: kvalue00110k, kkey00111k: kvalue00111k, kkey00112k: kvalue00112k, kkey00113k: kvalue00113k, kkey00114k: kvalue00114k, kkey00115k: kvalue00115k, kkey00116k: kvalue00116k, kkey00117k: kvalue00117k, kkey00118k: kvalue00118k, kkey00119k: kvalue00119k, kkey00120k: kvalue00120k, kkey00121k: kvalue00121k, kkey00122k: kvalue00122k, kkey00123k: kvalue00123k, kkey00124k: kvalue00124k, kkey00125k: kvalue00125k, kkey00126k: kvalue00126k, kkey00127k: kvalue00127k, kkey00128k: kvalue00128k, kkey00129k: kvalue00129k, kkey00130k: kvalue00130k, kkey00131k: kvalue00131k, kkey00132k: kvalue00132k, kkey00133k: kvalue00133k, kkey00134k: kvalue00134k, kkey00135k: kvalue00135k, kkey00136k: kvalue00136k, kkey00137k: kvalue00137k, kkey00138k: kvalue00138k, kkey00139k: kvalue00139k, kkey00140k: kvalue00140k, kkey00141k: kvalue00141k, kkey00142k: kvalue00142k, kkey00143k: kvalue00143k, kkey00144k: kvalue00144k, kkey00145k: kvalue00145k, kkey00146k: kvalue00146k, kkey00147k: kvalue00147k, kkey00148k: kvalue00148k, kkey00149k: kvalue00149k, kkey00150k: kvalue00150k, kkey00151k: kvalue00151k, kkey00152k: kvalue00152k, kkey00153k: kvalue00153k, kkey00154k: kvalue00154k, kkey00155k: kvalue00155k, kkey00156k: kvalue00156k, kkey00157k: kvalue00157k, kkey00158k: kvalue00158k, kkey00159k: kvalue00159k, kkey00160k: kvalue00160k, kkey00161k: kvalue00161k, kkey00162k: kvalue00162k, kkey00163k: kvalue00163k, kkey00164k: kvalue00164k, kkey00165k: kvalue00165k, kkey00166k: kvalue00166k, kkey00167k: kvalue00167k, kkey00168k: kvalue00168k, kkey00169k: kvalue00169k, kkey00170k: kvalue00170k, kkey00171k: kvalue00171k, kkey00172k: kvalue00172k, kkey00173k: kvalue00173k, kkey00174k: kvalue00174k, kkey00175k: kvalue00175k, kkey00176k: kvalue00176k, kkey00177k: kvalue00177k, kkey00178k: kvalue00178k, kkey00179k: kvalue00179k, kkey00180k: kvalue00180k, kkey00181k: kvalue00181k, kkey00182k: kvalue00182k, kkey00183k: kvalue00183k, kkey00184k: kvalue00184k, kkey00185k: kvalue00185k, kkey00186k: kvalue00186k, kkey00187k: kvalue00187k, kkey00188k: kvalue00188k, kkey00189k: kvalue00189k, kkey00190k: kvalue00190k, kkey00191k: kvalue00191k, kkey00192k: kvalue00192k, kkey00193k: kvalue00193k, kkey00194k: kvalue00194k, kkey00195k: kvalue00195k, kkey00196k: kvalue00196k, kkey00197k: kvalue00197k, kkey00198k: kvalue00198k, kkey00199k: kvalue00199k}",
+ "NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL",
+ "NULerL","NdrerULL","NU232LL","NUL23L","NU23LL","NU23LL","NUL23L","NUL32L"
+ ]
+ },
+ { "type": "BIGINT", "name": "cnt", "max" : 2563332323232, "min":1 },
+ { "type": "DOUBLE", "name": "phase", "max": 1000, "min": 0 }
+ ],
+ "tags": [
+ {"type": "TINYINT", "name": "groupid", "max": 10, "min": 1},
+ {"type": "BINARY", "name": "location", "len": 16,
+ "values": ["San Francisco", "Los Angles", "San Diego",
+ "San Jose", "Palo Alto", "Campbell", "Mountain View",
+ "Sunnyvale", "Santa Clara", "Cupertino"]
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}'''
+
+
+def initEnv():
+ updatecfgDict2 = {'debugFlag': 131, 'serverPort': 7030}
+ tdDnodes2.init(dnode2)
+ tdDnodes2.deploy(1, updatecfgDict2)
+ tdDnodes2.start(1)
+
+ updatecfgDict1 = {'debugFlag': 131, 'serverPort': 6030}
+ tdDnodes1.init(dnode1)
+ tdDnodes1.deploy(1, updatecfgDict1)
+ tdDnodes1.start(1)
+
+ with open('taosx-performance.json', 'w') as file:
+ file.write(insertJson)
+
+ changeVgroups = f"sed -i '/vgroups/c\ \"vgroups\": {vgroups},' taosx-performance.json"
+ os.system(changeVgroups)
+
+ changeTables = f"sed -i '/childtable_count/c\ \"childtable_count\": {tables},' taosx-performance.json"
+ os.system(changeTables)
+
+ changeRows = f"sed -i '/insert_rows/c\ \"insert_rows\": {rows_of_each_table},' taosx-performance.json"
+ os.system(changeRows)
+ os.system("taosBenchmark -f taosx-performance.json")
+
+def stopTaosd(str):
+ psCmd = f"ps -ef | grep -w taosd | grep -v grep | grep {str}" + " | awk '{print $2}' | xargs"
+ processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip()
+ print(f"kill taosd pid={processID}")
+ if processID:
+ cmd = f"kill -9 {processID}"
+ os.system(cmd)
+
+def cleanDb():
+ dropTopic = f"{taosd}/bin/taos -c {dnode1}/{cfg} -s \"drop topic if exists test\""
+ print("dropTopic:%s" % dropTopic)
+ os.system(dropTopic)
+
+ dropDb = f"{taosd}/bin/taos -c {dnode2}/{cfg} -s \"drop database if exists test\""
+ print("dropDb:%s" % dropDb)
+ os.system(dropDb)
+
+ createDb = f"{taosd}/bin/taos -c {dnode2}/{cfg} -s \"create database test vgroups {vgroups}\""
+ print("createDb:%s" % createDb)
+ os.system(createDb)
+
+
+def restartTaosd():
+ cmd1 = f"{taosd}/bin/taosd -c {dnode1}/{cfg} > /dev/null 2>&1 &"
+ cmd2 = f"{taosd}/bin/taosd -c {dnode2}/{cfg} > /dev/null 2>&1 &"
+ print("start taosd1 :%s" % cmd1)
+ print("start taosd2 :%s" % cmd2)
+ os.system(cmd1)
+ os.system(cmd2)
+
+
+def runTaosx():
+ cmd = f"{taosx} run -f \"tmq://root:taosdata@localhost:6030/test?group.id=taosx-new-`date +%s`&timeout={taosxTimeout}s&experimental.snapshot.enable=false&auto.offset.reset=earliest&prefer=raw&libraryPath={taosd}/lib/libtaos.so\" -t \"taos://root:taosdata@localhost:7030/test?libraryPath={taosd}/lib/libtaos.so\" > {taosxLog}"
+ print("run taosx:%s" % cmd)
+ os.system(cmd)
+
+
+def parseArgs(argv):
+ if len(argv) < 6:
+ print(
+ "Usage: python3 taosx-performance.py path_to_taosx path_to_taosd init vgroups tables rows_of_each_table [path]")
+ sys.exit(1)
+
+ global taosx
+ global taosd
+ global init
+ global vgroups
+ global tables
+ global rows_of_each_table
+ global path
+ global dnode1
+ global dnode2
+ taosx = argv[0]
+ taosd = argv[1]
+ init = argv[2]
+ vgroups = argv[3]
+ tables = argv[4]
+ rows_of_each_table = argv[5]
+ if len(argv) == 7:
+ path = argv[6]
+
+ if not os.path.isdir(path):
+ mkCmd = f"mkdir {path}"
+ os.system(mkCmd)
+ # os.system("git clone https://github.com/brendangregg/FlameGraph.git taosx-perf")
+def getCost():
+ costCmd = f"cat {taosxLog}| grep 'time cost'" + "| awk '{print $3}' | xargs"
+ cost = subprocess.check_output(costCmd, shell=True).decode("utf-8").strip()
+ return cost
+
+def runOnce(argv):
+ parseArgs(argv)
+
+ os.chdir(path)
+ print("current dir:" + os.getcwd())
+
+ stopTaosd("dnode2")
+ stopTaosd("dnode1")
+ time.sleep(2)
+
+ if init == "true":
+ initEnv()
+ else:
+ restartTaosd()
+
+ cleanDb()
+ runTaosx()
+
+ cost = getCost()
+ print("cost:%s" % cost)
+ timeCost.append(cost)
+
+'''
+python3 taosx-performance.py once path_to_taosx path_to_taosd init vgroups tables rows_of_each_table [path]
+'''
+paras = [
+ ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 1, 5000, 200, "/tmp/taosx_perf"),
+ ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 1, 5000, 200, "/tmp/taosx_perf"),
+
+ ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 1, 5000, 1000, "/tmp/taosx_perf"),
+ ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 1, 5000, 1000, "/tmp/taosx_perf"),
+
+ ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 1, 5000, 2000, "/tmp/taosx_perf"),
+ ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 1, 5000, 2000, "/tmp/taosx_perf"),
+
+ ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 2, 10000, 100, "/tmp/taosx_perf"),
+ ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 2, 10000, 100, "/tmp/taosx_perf"),
+
+ ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 2, 10000, 500, "/tmp/taosx_perf"),
+ ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 2, 10000, 500, "/tmp/taosx_perf"),
+
+ ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 2, 10000, 1000, "/tmp/taosx_perf"),
+ ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 2, 10000, 1000, "/tmp/taosx_perf"),
+
+ ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 4, 20000, 500, "/tmp/taosx_perf"),
+ ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 4, 20000, 500, "/tmp/taosx_perf"),
+
+ ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 8, 40000, 250, "/tmp/taosx_perf"),
+ ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 8, 40000, 250, "/tmp/taosx_perf"),
+
+ ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 16, 100000, 100, "/tmp/taosx_perf"),
+ ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 16, 100000, 100, "/tmp/taosx_perf"),
+]
+if __name__ == "__main__":
+ print("run performance start")
+
+ once = sys.argv[1]
+ if once == "once":
+ runOnce(sys.argv[2:])
+ else:
+ for i in range(len(paras)):
+ runOnce(paras[i])
+ if i % 2 == 1 :
+ print(f"opti cost:{float(timeCost[0]) - taosxTimeout}")
+ print(f"old cost:{float(timeCost[1]) - taosxTimeout}")
+ tmp = str(paras[i]) + f" speedup:{(float(timeCost[1]) - taosxTimeout)/(float(timeCost[0]) - taosxTimeout)}"
+ speedupStr.append(tmp)
+ print(tmp + "\n\n\n")
+ timeCost.clear()
+
+ print("performance result:\n" + str(speedupStr))
+ tdLog.info("run performance end")
diff --git a/tests/system-test/7-tmq/tmq_primary_key.py b/tests/system-test/7-tmq/tmq_primary_key.py
index 13d6bd565d..58f7bfb17d 100644
--- a/tests/system-test/7-tmq/tmq_primary_key.py
+++ b/tests/system-test/7-tmq/tmq_primary_key.py
@@ -18,10 +18,19 @@ import datetime
sys.path.append("./7-tmq")
from tmqCommon import *
+consumer_dict = {
+ "group.id": "g1",
+ "td.connect.user": "root",
+ "td.connect.pass": "taosdata",
+ "auto.offset.reset": "earliest",
+ "enable.auto.commit": "false",
+ "experimental.snapshot.enable": "true",
+ "min.poll.rows": "1"
+}
class TDTestCase:
clientCfgDict = {'debugFlag': 135}
- updatecfgDict = {'debugFlag': 135, 'asynclog': 0, 'tmqRowSize':1}
+ updatecfgDict = {'debugFlag': 135, 'asynclog': 0}
updatecfgDict["clientCfg"] = clientCfgDict
def init(self, conn, logSql, replicaVar=1):
@@ -44,16 +53,7 @@ class TDTestCase:
tdSql.execute(f'create topic topic_pk_query as select * from pk')
- consumer_dict = {
- "group.id": "g1",
- "td.connect.user": "root",
- "td.connect.pass": "taosdata",
- "auto.offset.reset": "earliest",
- "enable.auto.commit": "false",
- "experimental.snapshot.enable": "true",
- }
consumer = Consumer(consumer_dict)
-
try:
consumer.subscribe(["topic_pk_query"])
except TmqError:
@@ -156,14 +156,6 @@ class TDTestCase:
tdSql.execute(f'create topic topic_pk_stable as stable pks')
- consumer_dict = {
- "group.id": "g1",
- "td.connect.user": "root",
- "td.connect.pass": "taosdata",
- "auto.offset.reset": "earliest",
- "enable.auto.commit": "false",
- "experimental.snapshot.enable": "true",
- }
consumer = Consumer(consumer_dict)
try:
@@ -266,14 +258,6 @@ class TDTestCase:
tdSql.execute(f'create topic topic_in with meta as database abc1')
- consumer_dict = {
- "group.id": "g1",
- "td.connect.user": "root",
- "td.connect.pass": "taosdata",
- "auto.offset.reset": "earliest",
- "enable.auto.commit": "false",
- "experimental.snapshot.enable": "true",
- }
consumer = Consumer(consumer_dict)
try:
@@ -376,14 +360,6 @@ class TDTestCase:
tdSql.execute(f'create topic topic_pk_string with meta as database db_pk_string')
- consumer_dict = {
- "group.id": "g1",
- "td.connect.user": "root",
- "td.connect.pass": "taosdata",
- "auto.offset.reset": "earliest",
- "enable.auto.commit": "false",
- "experimental.snapshot.enable": "true",
- }
consumer = Consumer(consumer_dict)
try:
@@ -485,14 +461,6 @@ class TDTestCase:
tdSql.execute(f'create topic topic_pk_query_30755 as select * from pk')
- consumer_dict = {
- "group.id": "g1",
- "td.connect.user": "root",
- "td.connect.pass": "taosdata",
- "auto.offset.reset": "earliest",
- "enable.auto.commit": "false",
- "experimental.snapshot.enable": "true",
- }
consumer = Consumer(consumer_dict)
try:
diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py
index 5047ada1d1..845e7229a8 100644
--- a/tests/system-test/7-tmq/tmq_taosx.py
+++ b/tests/system-test/7-tmq/tmq_taosx.py
@@ -17,6 +17,8 @@ from tmqCommon import *
class TDTestCase:
updatecfgDict = {'debugFlag': 135, 'asynclog': 0}
+ clientCfgDict = {'debugFlag': 135, 'asynclog': 0}
+ updatecfgDict["clientCfg"] = clientCfgDict
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug(f"start to excute {__file__}")
@@ -65,7 +67,7 @@ class TDTestCase:
tdSql.checkData(1, 5, "sttb4")
tdSql.query("select * from stt order by ts")
- tdSql.checkRows(3)
+ tdSql.checkRows(5)
tdSql.checkData(0, 1, 1)
tdSql.checkData(2, 1, 21)
tdSql.checkData(0, 2, 2)
@@ -96,7 +98,7 @@ class TDTestCase:
tdSql.checkData(1, 5, "sttb4")
tdSql.query("select * from stt order by ts")
- tdSql.checkRows(3)
+ tdSql.checkRows(5)
tdSql.checkData(0, 1, 1)
tdSql.checkData(2, 1, 21)
tdSql.checkData(0, 2, 2)
@@ -256,6 +258,17 @@ class TDTestCase:
return
+ def checkWalMultiVgroupsRawData(self):
+ buildPath = tdCom.getBuildPath()
+ cmdStr = '%s/build/bin/tmq_taosx_ci -sv 3 -dv 5 -raw'%(buildPath)
+ tdLog.info(cmdStr)
+ os.system(cmdStr)
+
+ self.checkData()
+ self.checkDropData(False)
+
+ return
+
def checkWalMultiVgroupsWithDropTable(self):
buildPath = tdCom.getBuildPath()
cmdStr = '%s/build/bin/tmq_taosx_ci -sv 3 -dv 5 -d'%(buildPath)
@@ -681,6 +694,7 @@ class TDTestCase:
self.checkSnapshot1VgroupTable()
self.checkWalMultiVgroups()
+ self.checkWalMultiVgroupsRawData()
self.checkSnapshotMultiVgroups()
self.checkWalMultiVgroupsWithDropTable()
diff --git a/tests/system-test/7-tmq/tmq_ts-5776.py b/tests/system-test/7-tmq/tmq_ts-5776.py
new file mode 100644
index 0000000000..738d69701f
--- /dev/null
+++ b/tests/system-test/7-tmq/tmq_ts-5776.py
@@ -0,0 +1,39 @@
+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+from taos.tmq import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+ updatecfgDict = {'debugFlag': 135, 'asynclog': 0}
+ def init(self, conn, logSql, replicaVar=1):
+ self.replicaVar = int(replicaVar)
+ tdLog.debug(f"start to excute {__file__}")
+ tdSql.init(conn.cursor())
+ #tdSql.init(conn.cursor(), logSql) # output sql.txt file
+
+ def run(self):
+ buildPath = tdCom.getBuildPath()
+ cmdStr = '%s/build/bin/tmq_ts5776'%(buildPath)
+ tdLog.info(cmdStr)
+ os.system(cmdStr)
+
+ return
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/system-test/win-test-file b/tests/system-test/win-test-file
index 408d9e71c5..077113edaa 100644
--- a/tests/system-test/win-test-file
+++ b/tests/system-test/win-test-file
@@ -90,7 +90,6 @@ python3 ./test.py -f 7-tmq/tmqParamsTest.py -R
python3 ./test.py -f 7-tmq/tmqMaxGroupIds.py
python3 ./test.py -f 7-tmq/tmqConsumeDiscontinuousData.py
python3 ./test.py -f 7-tmq/tmqOffset.py
-python3 ./test.py -f 7-tmq/tmq_primary_key.py
python3 ./test.py -f 7-tmq/tmqDropConsumer.py
python3 ./test.py -f 1-insert/insert_stb.py
python3 ./test.py -f 1-insert/delete_stable.py
diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt
index 86b13ca723..1ee2bc4ce6 100644
--- a/tools/CMakeLists.txt
+++ b/tools/CMakeLists.txt
@@ -16,10 +16,6 @@ IF(TD_WEBSOCKET)
DEPENDS ${TAOS_LIB}
BUILD_IN_SOURCE 1
CONFIGURE_COMMAND cmake -E echo "taosws-rs no need cmake to config"
- UPDATE_COMMAND
- COMMAND echo "Starting to switch branch"
- COMMAND git fetch origin main && git checkout -f origin/main
- COMMAND echo "Switched to the main branch successfully"
PATCH_COMMAND
COMMAND git clean -f -d
BUILD_COMMAND
@@ -39,10 +35,6 @@ IF(TD_WEBSOCKET)
DEPENDS ${TAOS_LIB}
BUILD_IN_SOURCE 1
CONFIGURE_COMMAND cmake -E echo "taosws-rs no need cmake to config"
- UPDATE_COMMAND
- COMMAND echo "Starting to switch branch"
- COMMAND git fetch origin main && git checkout -f origin/main
- COMMAND echo "Switched to the main branch successfully"
PATCH_COMMAND
COMMAND git clean -f -d
BUILD_COMMAND
@@ -63,10 +55,6 @@ IF(TD_WEBSOCKET)
DEPENDS ${TAOS_LIB}
BUILD_IN_SOURCE 1
CONFIGURE_COMMAND cmake -E echo "taosws-rs no need cmake to config"
- UPDATE_COMMAND
- COMMAND echo "Starting to switch branch"
- COMMAND git fetch origin main && git checkout -f origin/main
- COMMAND echo "Switched to the main branch successfully"
PATCH_COMMAND
COMMAND git clean -f -d
BUILD_COMMAND
diff --git a/tools/keeper/README-CN.md b/tools/keeper/README-CN.md
index 770e9513c1..61ed631d8a 100644
--- a/tools/keeper/README-CN.md
+++ b/tools/keeper/README-CN.md
@@ -1,267 +1,123 @@
-# TaosKeeper
+
+# taosKeeper
-taosKeeper 是 TDengine 各项监控指标的导出工具，通过简单的几项配置即可获取 TDengine 的运行状态。并且 taosKeeper 企业版支持多种收集器，可以方便进行监控数据的展示。
+[](https://github.com/taosdata/TDengine/actions/workflows/taoskeeper-ci-build.yml)
+
+
+
+
+[](https://twitter.com/tdenginedb)
+[](https://www.youtube.com/@tdengine)
+[](https://discord.com/invite/VZdSuUg4pS)
+[](https://www.linkedin.com/company/tdengine)
+[](https://stackoverflow.com/questions/tagged/tdengine)
-taosKeeper 使用 TDengine RESTful 接口，所以不需要安装 TDengine 客户端即可使用。
+简体中文 | [English](./README.md)
-## 构建
+
+## 目录
-### 获取源码
+- [1. 简介](#1-简介)
+- [2. 文档](#2-文档)
+- [3. 前置条件](#3-前置条件)
+- [4. 构建](#4-构建)
+- [5. 测试](#5-测试)
+ - [5.1 运行测试](#51-运行测试)
+ - [5.2 添加用例](#52-添加用例)
+ - [5.3 性能测试](#53-性能测试)
+- [6. CI/CD](#6-cicd)
+- [7. 提交 Issues](#7-提交-issues)
+- [8. 提交 PR](#8-提交-pr)
+- [9. 引用](#9-引用)
+- [10. 许可证](#10-许可证)
-从 GitHub 克隆源码：
+## 1. 简介
-```sh
-git clone https://github.com/taosdata/TDengine
-cd TDengine/tools/keeper
-```
+taosKeeper 是 TDengine 3.0 版本全新引入的监控指标导出工具，旨在方便用户对 TDengine 的运行状态和性能指标进行实时监控。只需进行简单配置，TDengine 就能将自身的运行状态和各项指标等信息上报给 taosKeeper。taosKeeper 在接收到监控数据后，会利用 taosAdapter 提供的 RESTful 接口，将这些数据存储到 TDengine 中。
-### 编译
+taosKeeper 的一个重要价值在于，它能够将多个甚至一批 TDengine 集群的监控数据集中存储到一个统一的平台。如此一来，监控软件便能轻松获取这些数据，进而实现对 TDengine 集群的全面监控与实时分析。通过 taosKeeper，用户可以更加便捷地了解 TDengine 的运行状况，及时发现并解决潜在问题，确保系统的稳定性和高效性。
-taosKeeper 使用 `GO` 语言编写，在构建前需要配置好 `GO` 语言开发环境。
+## 2. 文档
-```sh
-go mod tidy
+- 使用 taosKeeper，请参考 [taosKeeper 参考手册](https://docs.taosdata.com/reference/components/taoskeeper/)，其中包括安装、配置、启动、数据收集与监控，以及集成 Prometheus 等方面的内容。
+- 本 README 主要面向希望自行贡献代码、编译和测试 taosKeeper 的开发者。如果想要学习 TDengine，可以浏览 [官方文档](https://docs.taosdata.com/)。
+
+## 3. 前置条件
+
+1. 已安装 Go 1.18 及以上版本。
+2. 本地已部署 TDengine，具体步骤请参考 [部署服务端](https://docs.taosdata.com/get-started/package/)，且已启动 taosd 与 taosAdapter。
+
+## 4. 构建
+
+在 `TDengine/tools/keeper` 目录下运行以下命令以构建项目：
+
+```bash
go build
```
-## 安装
+## 5. 测试
-如果是自行构建的项目，仅需要拷贝 `taoskeeper` 文件到你的 `PATH` 中。
+### 5.1 运行测试
-```sh
-sudo install taoskeeper /usr/bin/
+在 `TDengine/tools/keeper` 目录下执行以下命令运行测试：
+
+```bash
+sudo go test ./...
```
-## 启动
+测试用例将连接到本地的 TDengine 服务器和 taosAdapter 进行测试。测试完成后，你将看到类似如下的结果摘要。如果所有测试用例均通过，输出中将不会出现 `FAIL` 字样。
-在启动前，应该做好如下配置：
-在 `/etc/taos/taoskeeper.toml` 配置 TDengine 连接参数以及监控指标前缀等其他信息。
-
-```toml
-# gin 框架是否启用 debug
-debug = false
-
-# 服务监听端口, 默认为 6043
-port = 6043
-
-# 日志级别，包含 panic、error、info、debug、trace等
-loglevel = "info"
-
-# 程序中使用协程池的大小
-gopoolsize = 50000
-
-# 查询 TDengine 监控数据轮询间隔
-RotationInterval = "15s"
-
-[tdengine]
-host = "127.0.0.1"
-port = 6041
-username = "root"
-password = "taosdata"
-
-# 需要被监控的 taosAdapter
-[taosAdapter]
-address = ["127.0.0.1:6041"]
-
-[metrics]
-# 监控指标前缀
-prefix = "taos"
-
-# 存放监控数据的数据库
-database = "log"
-
-# 指定需要监控的普通表
-tables = []
-
-[environment]
-# 是否在容器中运行，影响 taosKeeper 自身的监控数据
-incgroup = false
+```text
+ok github.com/taosdata/taoskeeper/api 17.405s
+ok github.com/taosdata/taoskeeper/cmd 1.819s
+ok github.com/taosdata/taoskeeper/db 0.484s
+ok github.com/taosdata/taoskeeper/infrastructure/config 0.417s
+ok github.com/taosdata/taoskeeper/infrastructure/log 0.785s
+ok github.com/taosdata/taoskeeper/monitor 4.623s
+ok github.com/taosdata/taoskeeper/process 0.606s
+ok github.com/taosdata/taoskeeper/system 3.420s
+ok github.com/taosdata/taoskeeper/util 0.097s
+ok github.com/taosdata/taoskeeper/util/pool 0.146s
```
-现在可以启动服务，输入：
+### 5.2 添加用例
-```sh
-taoskeeper
-```
+在以 `_test.go` 结尾的文件中添加测试用例，并且确保新增代码都有对应的测试用例覆盖。
-如果你使用 `systemd`，复制 `taoskeeper.service` 到 `/lib/systemd/system/`，并启动服务。
+### 5.3 性能测试
-```sh
-sudo cp taoskeeper.service /lib/systemd/system/
-sudo systemctl daemon-reload
-sudo systemctl start taoskeeper
-```
+性能测试正在开发中。
-让 taosKeeper 随系统开机自启动。
+## 6. CI/CD
-```sh
-sudo systemctl enable taoskeeper
-```
+- [Build Workflow](https://github.com/taosdata/TDengine/actions/workflows/taoskeeper-ci-build.yml)
+- Code Coverage - TODO
-如果使用 `systemd`，你可以使用如下命令完成安装。
+## 7. 提交 Issues
-```sh
-go mod tidy
-go build
-sudo install taoskeeper /usr/bin/
-sudo cp taoskeeper.service /lib/systemd/system/
-sudo systemctl daemon-reload
-sudo systemctl start taoskeeper
-sudo systemctl enable taoskeeper
-```
+我们欢迎提交 [GitHub Issue](https://github.com/taosdata/TDengine/issues)。提交时请尽量提供以下信息，以便快速定位问题：
-## Docker
+- 问题描述：具体问题表现及是否必现，建议附上详细调用堆栈或日志信息。
+- taosKeeper 版本：可通过 `taoskeeper -V` 获取版本信息。
+- TDengine 服务端版本：可通过 `taos -V` 获取版本信息。
-如下介绍了如何在 docker 中构建 taosKeeper：
+如有其它相关信息（如环境配置、操作系统版本等），请一并补充，以便我们更全面地了解问题。
-在构建前请配置好 `./config/taoskeeper.toml` 中合适的参数，并编辑 Dockerfile ，示例如下。
+## 8. 提交 PR
-```dockerfile
-FROM golang:1.18.6-alpine as builder
+我们欢迎开发者共同参与本项目开发，提交 PR 时请按照以下步骤操作：
-WORKDIR /usr/src/taoskeeper
-COPY ./ /usr/src/taoskeeper/
-ENV GO111MODULE=on \
- GOPROXY=https://goproxy.cn,direct
-RUN go mod tidy && go build
+1. Fork 仓库：请先 Fork 本仓库，具体步骤请参考 [如何 Fork 仓库](https://docs.github.com/en/get-started/quickstart/fork-a-repo)。
+2. 创建新分支：基于 `main` 分支创建一个新分支，并使用有意义的分支名称（例如：`git checkout -b feature/my_feature`）。请勿直接在 main 分支上进行修改。
+3. 开发与测试：完成代码修改后，确保所有单元测试都能通过，并为新增功能或修复的 Bug 添加相应的测试用例。
+4. 提交代码：将修改提交到远程分支（例如：`git push origin feature/my_feature`）。
+5. 创建 Pull Request：在 GitHub 上发起 [Pull Request](https://github.com/taosdata/TDengine/pulls)，具体步骤请参考 [如何创建 Pull Request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request)。
+6. 检查 CI：提交 PR 后，可在 Pull Request 中找到自己提交的 PR，点击对应的链接，即可查看该 PR 的 CI 是否通过。若通过，会显示 `All checks have passed`。无论 CI 是否通过，均可点击 `Show all checks -> Details` 查看详细的测试用例日志。
-FROM alpine:3
-RUN mkdir -p /etc/taos
-COPY --from=builder /usr/src/taoskeeper/taoskeeper /usr/bin/
-COPY ./config/taoskeeper.toml /etc/taos/taoskeeper.toml
-EXPOSE 6043
-CMD ["taoskeeper"]
-```
+## 9. 引用
-如果已经有 taosKeeper 可执行文件，在配置好 `taoskeeper.toml` 后你可以使用如下方式构建:
+[TDengine 官网](https://www.taosdata.com/)
-```dockerfile
-FROM ubuntu:18.04
-RUN mkdir -p /etc/taos
-COPY ./taoskeeper /usr/bin/
-COPY ./taoskeeper.toml /etc/taos/taoskeeper.toml
-EXPOSE 6043
-CMD ["taoskeeper"]
-```
+## 10. 许可证
-## 使用（**企业版**）
-
-### Prometheus (by scrape)
-
-taosKeeper 可以像 `node-exporter` 一样向 Prometheus 提供监控指标。\
-在 `/etc/prometheus/prometheus.yml` 添加配置：
-
-```yml
-global:
- scrape_interval: 5s
-
-scrape_configs:
- - job_name: "taoskeeper"
- static_configs:
- - targets: ["taoskeeper:6043"]
-```
-
-现在使用 PromQL 查询即可以显示结果，比如要查看指定主机（通过 FQDN 正则匹配表达式筛选）硬盘使用百分比：
-
-```promql
-taos_dn_disk_used / taos_dn_disk_total {fqdn=~ "tdengine.*"}
-```
-
-你可以使用 `docker-compose` 测试完整的链路。
-`docker-compose.yml`示例：
-
-```yml
-version: "3.7"
-
-services:
- tdengine:
- image: tdengine/tdengine
- environment:
- TAOS_FQDN: tdengine
- volumes:
- - taosdata:/var/lib/taos
- taoskeeper:
- build: ./
- depends_on:
- - tdengine
- environment:
- TDENGINE_HOST: tdengine
- TDENGINE_PORT: 6041
- volumes:
- - ./config/taoskeeper.toml:/etc/taos/taoskeeper.toml
- ports:
- - 6043:6043
- prometheus:
- image: prom/prometheus
- volumes:
- - ./prometheus/:/etc/prometheus/
- ports:
- - 9090:9090
-volumes:
- taosdata:
-```
-
-启动：
-
-```sh
-docker-compose up -d
-```
-
-现在通过访问 来查询结果。访问[simple dashboard](https://grafana.com/grafana/dashboards/15164) 来查看TaosKeeper + Prometheus + Grafana 监控 TDengine 的快速启动实例。
-
-### Telegraf
-
-如果使用 telegraf 来收集各个指标，仅需要在配置中增加：
-
-```toml
-[[inputs.prometheus]]
-## An array of urls to scrape metrics from.
-urls = ["http://taoskeeper:6043/metrics"]
-```
-
-可以通过 `docker-compose` 来测试
-
-```sh
-docker-compose -f docker-compose.yml -f telegraf.yml up -d telegraf taoskeeper
-```
-
-由于可以在 `telegraf.conf` 设置日志为标准输出：
-
-```toml
-[[outputs.file]]
-files = ["stdout"]
-```
-
-所以你可以通过 `docker-compose logs` 在标准输出中追踪 TDengine 各项指标。
-
-```sh
-docker-compose -f docker-compose.yml -f telegraf.yml logs -f telegraf
-```
-
-### Zabbix
-
-1. 导入 zabbix 临时文件 `zbx_taos_keeper_templates.xml`。
-2. 使用 `TDengine` 模板来创建主机，修改宏 `{$TAOSKEEPER_HOST}` 和 `{$COLLECTION_INTERVAL}`。
-3. 等待并查看到自动创建的条目。
-
-### 常见问题
-
-* 启动报错，显示connection refused
-
- **解析**：taosKeeper 依赖 restful 接口查询数据，请检查 taosAdapter 是否正常运行或 taoskeeper.toml 中 taosAdapter 地址是否正确。
-
-* taosKeeper 监控不同 TDengine 显示的检测指标数目不一致？
-
- **解析**：如果 TDengine 中未创建某项指标，taoskeeper 不能获取对应的检测结果。
-
-* 不能接收到 TDengine 的监控日志。
-
- **解析**: 修改 `/etc/taos/taos.cfg` 文件并增加如下参数：
-
- ```cfg
- monitor 1 // 启用monitor
- monitorInterval 30 // 发送间隔 (s)
- monitorFqdn localhost // 接收消息的FQDN，默认为空
- monitorPort 6043 // 接收消息的端口号
- monitorMaxLogs 100 // 每个监控间隔缓存的最大日志数量
- ```
+[AGPL-3.0 License](../../LICENSE)
diff --git a/tools/keeper/README.md b/tools/keeper/README.md
index 18e351f160..c282611564 100644
--- a/tools/keeper/README.md
+++ b/tools/keeper/README.md
@@ -1,273 +1,123 @@
-# TaosKeeper
+
+# taosKeeper
-TDengine Metrics Exporter for Kinds of Collectors, you can obtain the running status of TDengine by performing several simple configurations.
+[](https://github.com/taosdata/TDengine/actions/workflows/taoskeeper-ci-build.yml)
+
+
+
+
+[](https://twitter.com/tdenginedb)
+[](https://www.youtube.com/@tdengine)
+[](https://discord.com/invite/VZdSuUg4pS)
+[](https://www.linkedin.com/company/tdengine)
+[](https://stackoverflow.com/questions/tagged/tdengine)
-This tool uses TDengine RESTful API, so you could just build it without TDengine client.
+English | [简体中文](./README-CN.md)
-## Build
+
+## Table of Contents
-### Get the source codes
+- [1. Introduction](#1-introduction)
+- [2. Documentation](#2-documentation)
+- [3. Prerequisites](#3-prerequisites)
+- [4. Build](#4-build)
+- [5. Testing](#5-testing)
+ - [5.1 Test Execution](#51-test-execution)
+ - [5.2 Test Case Addition](#52-test-case-addition)
+ - [5.3 Performance Testing](#53-performance-testing)
+- [6. CI/CD](#6-cicd)
+- [7. Submitting Issues](#7-submitting-issues)
+- [8. Submitting PR](#8-submitting-pr)
+- [9. References](#9-references)
+- [10. License](#10-license)
-```sh
-git clone https://github.com/taosdata/TDengine
-cd TDengine/tools/keeper
-```
+## 1. Introduction
-### compile
+taosKeeper is a new monitoring indicator export tool introduced in TDengine 3.0, which is designed to facilitate users to monitor the operating status and performance indicators of TDengine in real time. With simple configuration, TDengine can report its own operating status and various indicators to taosKeeper. After receiving the monitoring data, taosKeeper will use the RESTful interface provided by taosAdapter to store the data in TDengine.
-```sh
-go mod tidy
+An important value of taosKeeper is that it can store the monitoring data of multiple or even a batch of TDengine clusters in a unified platform. In this way, the monitoring software can easily obtain this data, and then realize comprehensive monitoring and real-time analysis of the TDengine cluster. Through taosKeeper, users can more easily understand the operation status of TDengine, discover and solve potential problems in a timely manner, and ensure the stability and efficiency of the system.
+
+## 2. Documentation
+
+- To use taosKeeper, please refer to the [taosKeeper Reference](https://docs.tdengine.com/tdengine-reference/components/taoskeeper/), which includes installation, configuration, startup, data collection and monitoring, and Prometheus integration.
+- This README is mainly for developers who want to contribute code, compile and test taosKeeper. If you want to learn TDengine, you can browse the [official documentation](https://docs.tdengine.com/).
+
+## 3. Prerequisites
+
+1. Go 1.18 or above has been installed.
+2. TDengine has been deployed locally. For specific steps, please refer to [Deploy TDengine](https://docs.tdengine.com/get-started/deploy-from-package/), and taosd and taosAdapter have been started.
+
+## 4. Build
+
+Run the following command in the `TDengine/tools/keeper` directory to build the project:
+
+```bash
go build
```
-## Install
+## 5. Testing
-If you build the tool by your self, just copy the `taoskeeper` binary to your `PATH`.
+### 5.1 Test Execution
-```sh
-sudo install taoskeeper /usr/bin/
+Run the test by executing the following command in the `TDengine/tools/keeper` directory:
+
+```bash
+sudo go test ./...
```
-## Start
+The test case will connect to the local TDengine server and taosAdapter for testing. After the test is completed, you will see a result summary similar to the following. If all test cases pass, there will be no `FAIL` in the output.
-Before start, you should configure some options like database ip, port or the prefix and others for exported metrics.
-
-in `/etc/taos/taoskeeper.toml`.
-
-```toml
-# Start with debug middleware for gin
-debug = false
-
-# Listen port, default is 6043
-port = 6043
-
-# log level
-loglevel = "info"
-
-# go pool size
-gopoolsize = 50000
-
-# interval for TDengine metrics
-RotationInterval = "15s"
-
-[tdengine]
-host = "127.0.0.1"
-port = 6041
-username = "root"
-password = "taosdata"
-
-# list of taosAdapter that need to be monitored
-[taosAdapter]
-address = ["127.0.0.1:6041"]
-
-[metrics]
-# metrics prefix in metrics names.
-prefix = "taos"
-
-# database for storing metrics data
-database = "log"
-
-# export some tables that are not super table
-tables = []
-
-[environment]
-# Whether running in cgroup.
-incgroup = false
+```text
+ok github.com/taosdata/taoskeeper/api 17.405s
+ok github.com/taosdata/taoskeeper/cmd 1.819s
+ok github.com/taosdata/taoskeeper/db 0.484s
+ok github.com/taosdata/taoskeeper/infrastructure/config 0.417s
+ok github.com/taosdata/taoskeeper/infrastructure/log 0.785s
+ok github.com/taosdata/taoskeeper/monitor 4.623s
+ok github.com/taosdata/taoskeeper/process 0.606s
+ok github.com/taosdata/taoskeeper/system 3.420s
+ok github.com/taosdata/taoskeeper/util 0.097s
+ok github.com/taosdata/taoskeeper/util/pool 0.146s
```
-Now you could run the tool:
+### 5.2 Test Case Addition
-```sh
-taoskeeper
-```
+Add test cases in files ending with `_test.go` and make sure the new code is covered by the corresponding test cases.
-If you use `systemd`, copy the `taoskeeper.service` to `/lib/systemd/system/` and start the service.
+### 5.3 Performance Testing
-```sh
-sudo cp taoskeeper.service /lib/systemd/system/
-sudo systemctl daemon-reload
-sudo systemctl start taoskeeper
-```
+Performance testing is under development.
-To start taoskeeper whenever os rebooted, you should enable the systemd service:
+## 6. CI/CD
-```sh
-sudo systemctl enable taoskeeper
-```
+- [Build Workflow](https://github.com/taosdata/TDengine/actions/workflows/taoskeeper-ci-build.yml)
+- Code Coverage - TODO
-So if use `systemd`, you'd better install it with these lines all-in-one:
+## 7. Submitting Issues
-```sh
-go mod tidy
-go build
-sudo install taoskeeper /usr/bin/
-sudo cp taoskeeper.service /lib/systemd/system/
-sudo systemctl daemon-reload
-sudo systemctl start taoskeeper
-sudo systemctl enable taoskeeper
-```
+We welcome submissions of [GitHub Issues](https://github.com/taosdata/TDengine/issues). Please provide the following information when submitting so that the problem can be quickly located:
-## Docker
+- Problem description: The specific problem manifestation and whether it must occur. It is recommended to attach detailed call stack or log information.
+- taosKeeper version: You can get the version information through `taoskeeper -V`.
+- TDengine server version: You can get the version information through `taos -V`.
-Here is an example to show how to build this tool in docker:
+If you have other relevant information (such as environment configuration, operating system version, etc.), please add it so that we can understand the problem more comprehensively.
-Before building, you should configure `./config/taoskeeper.toml` with proper parameters and edit Dockerfile. Take following as example.
+## 8. Submitting PR
-```dockerfile
-FROM golang:1.18.2 as builder
+We welcome developers to participate in the development of this project. Please follow the steps below when submitting a PR:
-WORKDIR /usr/src/taoskeeper
-COPY ./ /usr/src/taoskeeper/
-ENV GO111MODULE=on \
- GOPROXY=https://goproxy.cn,direct
-RUN go mod tidy && go build
+1. Fork the repository: Please fork this repository first. For specific steps, please refer to [How to Fork a Repository](https://docs.github.com/en/get-started/quickstart/fork-a-repo).
+2. Create a new branch: Create a new branch based on the `main` branch and use a meaningful branch name (for example: `git checkout -b feature/my_feature`). Do not modify it directly on the main branch.
+3. Development and testing: After completing the code modification, make sure that all unit tests pass, and add corresponding test cases for new features or fixed bugs.
+4. Submit code: Submit the changes to the remote branch (for example: `git push origin feature/my_feature`).
+5. Create a Pull Request: Initiate a [Pull Request](https://github.com/taosdata/TDengine/pulls) on GitHub. For specific steps, please refer to [How to Create a Pull Request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request).
+6. Check CI: After submitting the PR, you can find the PR you submitted in the Pull Request and click the corresponding link to check whether the CI of the PR has passed. If it has passed, it will show `All checks have passed`. Regardless of whether CI has passed or not, you can click `Show all checks/Details` to view detailed test case logs.
-FROM alpine:3
-RUN mkdir -p /etc/taos
-COPY --from=builder /usr/src/taoskeeper/taoskeeper /usr/bin/
-COPY ./config/taoskeeper.toml /etc/taos/taoskeeper.toml
-EXPOSE 6043
-CMD ["taoskeeper"]
-```
+## 9. References
-If you already have taosKeeper binary file, you can build this tool like:
+[TDengine Official Website](https://www.tdengine.com/)
-```dockerfile
-FROM ubuntu:18.04
-RUN mkdir -p /etc/taos
-COPY ./taoskeeper /usr/bin/
-COPY ./taoskeeper.toml /etc/taos/taoskeeper.toml
-EXPOSE 6043
-CMD ["taoskeeper"]
-```
+## 10. License
-## Usage (**Enterprise Edition**)
-
-### Prometheus (by scrape)
-
-It's now act as a prometheus exporter like `node-exporter`.
-
-Here's how to add this in scrape configs of `/etc/prometheus/prometheus.yml`:
-
-```yml
-global:
- scrape_interval: 5s
-
-scrape_configs:
- - job_name: "taoskeeper"
- static_configs:
- - targets: [ "taoskeeper:6043" ]
-```
-
-Now PromQL query will show the right result, for example, to show disk used percent in an specific host with FQDN regex
-match expression:
-
-```promql
-taos_dn_disk_used / taos_dn_disk_total {fqdn=~ "tdengine.*"}
-```
-
-You can use `docker-compose` with the current `docker-compose.yml` to test the whole stack.
-
-Here is the `docker-compose.yml`:
-
-```yml
-version: "3.7"
-
-services:
- tdengine:
- image: tdengine/tdengine
- environment:
- TAOS_FQDN: tdengine
- volumes:
- - taosdata:/var/lib/taos
- taoskeeper:
- build: ./
- depends_on:
- - tdengine
- environment:
- TDENGINE_HOST: tdengine
- TDENGINE_PORT: 6041
- volumes:
- - ./config/taoskeeper.toml:/etc/taos/taoskeeper.toml
- ports:
- - 6043:6043
- prometheus:
- image: prom/prometheus
- volumes:
- - ./prometheus/:/etc/prometheus/
- ports:
- - 9090:9090
-volumes:
- taosdata:
-
-```
-
-Start the stack:
-
-```sh
-docker-compose up -d
-```
-
-Now you point to (if you have not started a prometheus server by yourself) and query.
-
-For a quick demo with TaosKeeper + Prometheus + Grafana, we provide
-a [simple dashboard](https://grafana.com/grafana/dashboards/15164) to monitor TDengine.
-
-### Telegraf
-
-If you are using telegraf to collect metrics, just add inputs like this:
-
-```toml
-[[inputs.prometheus]]
- ## An array of urls to scrape metrics from.
- urls = ["http://taoskeeper:6043/metrics"]
-```
-
-You can test it with `docker-compose`:
-
-```sh
-docker-compose -f docker-compose.yml -f telegraf.yml up -d telegraf taoskeeper
-```
-
-Since we have set an stdout file output in `telegraf.conf`:
-
-```toml
-[[outputs.file]]
- files = ["stdout"]
-```
-
-So you can track with TDengine metrics in standard output with `docker-compose logs`:
-
-```sh
-docker-compose -f docker-compose.yml -f telegraf.yml logs -f telegraf
-```
-
-### Zabbix
-
-1. Import the zabbix template file `zbx_taos_keeper_templates.xml`.
-2. Use the template `TDengine` to create the host and modify the macros `{$TAOSKEEPER_HOST}`
- and `{$COLLECTION_INTERVAL}`.
-3. Waiting for monitoring items to be created automatically.
-
-### FAQ
-
-* Error occurred: Connection refused, while taosKeeper was starting
-
- **Answer**: taoskeeper relies on restful interfaces to query data. Check whether the taosAdapter is running or whether
- the taosAdapter address in taoskeeper.toml is correct.
-
-* Why detection metrics displayed by different TDengine's inconsistent with taoskeeper monitoring?
-
- **Answer**: If a metric is not created in TDengine, taoskeeper cannot get the corresponding test results.
-
-* Cannot receive log from TDengine server.
-
- **Answer**: Modify `/etc/taos/taos.cfg` file and add parameters like:
-
- ```cfg
- monitor 1 // start monitor
- monitorInterval 30 // send log interval (s)
- monitorFqdn localhost
- monitorPort 6043 // taosKeeper port
- monitorMaxLogs 100
- ```
+[AGPL-3.0 License](../../LICENSE)
diff --git a/tools/taos-tools/src/benchUtil.c b/tools/taos-tools/src/benchUtil.c
index ad59c4e37e..8dcca30f8c 100644
--- a/tools/taos-tools/src/benchUtil.c
+++ b/tools/taos-tools/src/benchUtil.c
@@ -1033,18 +1033,26 @@ int convertStringToDatatype(char *type, int length) {
return TSDB_DATA_TYPE_BOOL;
} else if (0 == strCompareN(type, "tinyint", length)) {
return TSDB_DATA_TYPE_TINYINT;
+ } else if (0 == strCompareN(type, "tinyint unsigned", length)) {
+ return TSDB_DATA_TYPE_UTINYINT;
} else if (0 == strCompareN(type, "utinyint", length)) {
return TSDB_DATA_TYPE_UTINYINT;
} else if (0 == strCompareN(type, "smallint", length)) {
return TSDB_DATA_TYPE_SMALLINT;
+ } else if (0 == strCompareN(type, "smallint unsigned", length)) {
+ return TSDB_DATA_TYPE_USMALLINT;
} else if (0 == strCompareN(type, "usmallint", length)) {
return TSDB_DATA_TYPE_USMALLINT;
} else if (0 == strCompareN(type, "int", length)) {
return TSDB_DATA_TYPE_INT;
+ } else if (0 == strCompareN(type, "int unsigned", length)) {
+ return TSDB_DATA_TYPE_UINT;
} else if (0 == strCompareN(type, "uint", length)) {
return TSDB_DATA_TYPE_UINT;
} else if (0 == strCompareN(type, "bigint", length)) {
return TSDB_DATA_TYPE_BIGINT;
+ } else if (0 == strCompareN(type, "bigint unsigned", length)) {
+ return TSDB_DATA_TYPE_UBIGINT;
} else if (0 == strCompareN(type, "ubigint", length)) {
return TSDB_DATA_TYPE_UBIGINT;
} else if (0 == strCompareN(type, "float", length)) {
diff --git a/utils/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt
index d1c049ef1e..d73f91aad3 100644
--- a/utils/test/c/CMakeLists.txt
+++ b/utils/test/c/CMakeLists.txt
@@ -6,6 +6,7 @@ add_executable(tmq_taosx_ci tmq_taosx_ci.c)
add_executable(tmq_ts5466 tmq_ts5466.c)
add_executable(tmq_td32526 tmq_td32526.c)
add_executable(tmq_td32187 tmq_td32187.c)
+add_executable(tmq_ts5776 tmq_ts5776.c)
add_executable(tmq_td32471 tmq_td32471.c)
add_executable(tmq_write_raw_test tmq_write_raw_test.c)
add_executable(write_raw_block_test write_raw_block_test.c)
@@ -87,6 +88,13 @@ target_link_libraries(
PUBLIC common
PUBLIC os
)
+target_link_libraries(
+ tmq_ts5776
+ PUBLIC ${TAOS_LIB}
+ PUBLIC util
+ PUBLIC common
+ PUBLIC os
+)
target_link_libraries(
tmq_taosx_ci
PUBLIC ${TAOS_LIB}
diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c
index 117f9fa2e1..bc4a79bf12 100644
--- a/utils/test/c/tmq_taosx_ci.c
+++ b/utils/test/c/tmq_taosx_ci.c
@@ -34,6 +34,7 @@ typedef struct {
int dstVgroups;
char dir[256];
bool btMeta;
+ bool rawData;
} Config;
Config g_conf = {0};
@@ -424,6 +425,15 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes) {
}
taos_free_result(pRes);
+ pRes =
+ taos_query(pConn,
+ "insert into stt1 values(now + 322s, 3, 2, 'stt1')");
+ if (taos_errno(pRes) != 0) {
+ printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
pRes =
taos_query(pConn,
"insert into stt1 values(now + 2s, 3, 2, 'stt1') stt3 using stt tags(23, \"stt3\", true) values(now + "
@@ -436,6 +446,15 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes) {
}
taos_free_result(pRes);
+ pRes =
+ taos_query(pConn,
+ "insert into stt1 values(now + 442s, 3, 2, 'stt1')");
+ if (taos_errno(pRes) != 0) {
+ printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
return 0;
}
@@ -633,6 +652,9 @@ tmq_t* build_consumer() {
tmq_conf_set(conf, "enable.auto.commit", "true");
tmq_conf_set(conf, "auto.offset.reset", "earliest");
tmq_conf_set(conf, "msg.consume.excluded", "1");
+ if (g_conf.rawData) {
+ tmq_conf_set(conf, "msg.consume.rawdata", "1");
+ }
// tmq_conf_set(conf, "session.timeout.ms", "1000000");
// tmq_conf_set(conf, "max.poll.interval.ms", "20000");
@@ -1238,6 +1260,8 @@ int main(int argc, char* argv[]) {
g_conf.meta = 1;
} else if (strcmp(argv[i], "-bt") == 0) {
g_conf.btMeta = true;
+ } else if (strcmp(argv[i], "-raw") == 0) {
+ g_conf.rawData = true;
}
}
diff --git a/utils/test/c/tmq_ts5776.c b/utils/test/c/tmq_ts5776.c
new file mode 100644
index 0000000000..c50809bccd
--- /dev/null
+++ b/utils/test/c/tmq_ts5776.c
@@ -0,0 +1,255 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include "cJSON.h"
+#include "taos.h"
+#include "tmsg.h"
+#include "types.h"
+
+static TAOS* use_db() {
+ TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+ if (pConn == NULL) {
+ return NULL;
+ }
+
+ TAOS_RES* pRes = taos_query(pConn, "use db_dst");
+ if (taos_errno(pRes) != 0) {
+ printf("error in use db_taosx, reason:%s\n", taos_errstr(pRes));
+ return NULL;
+ }
+ taos_free_result(pRes);
+ return pConn;
+}
+
+static void msg_process(TAOS_RES* msg) {
+ printf("-----------topic-------------: %s\n", tmq_get_topic_name(msg));
+ printf("db: %s\n", tmq_get_db_name(msg));
+ printf("vg: %d\n", tmq_get_vgroup_id(msg));
+ TAOS* pConn = use_db();
+ if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META || tmq_get_res_type(msg) == TMQ_RES_METADATA) {
+ char* result = tmq_get_json_meta(msg);
+ printf("meta result: %s\n", result);
+ tmq_free_json_meta(result);
+ }
+
+ tmq_raw_data raw = {0};
+ tmq_get_raw(msg, &raw);
+ printf("write raw data type: %d\n", raw.raw_type);
+ int32_t ret = tmq_write_raw(pConn, raw);
+ printf("write raw data: %s\n", tmq_err2str(ret));
+ ASSERT(ret == 0);
+
+ tmq_free_raw(raw);
+ taos_close(pConn);
+}
+
+int buildDatabase(TAOS* pConn, TAOS_RES* pRes) {
+ pRes = taos_query(pConn,
+ "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 "
+ "nchar(8), t4 bool)");
+ if (taos_errno(pRes) != 0) {
+ printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
+ pRes = taos_query(
+ pConn,
+ "insert into ct3 using st1(t1) tags(3000) values(1626006833600, 5, 6, 'c') ct1 using st1(t1) tags(2000) values(1626006833601, 2, 3, 'sds') (1626006833602, 4, 5, "
+ "'ddd') ct0 using st1 tags(1000, \"ttt\", true) values(1626006833603, 4, 3, 'hwj') ct1 using st1(t1) tags(2000) values(now+5s, 23, 32, 's21ds')");
+ if (taos_errno(pRes) != 0) {
+ printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
+ pRes = taos_query(
+ pConn,
+ "insert into ct3 values(1626006839602, 5, 6, 'c')");
+ if (taos_errno(pRes) != 0) {
+ printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
+ pRes = taos_query(
+ pConn,
+ "insert into ct3 using st1(t1) tags(3000) values(1626006833602, 5, 6, 'c') ct1 values(1626006833611, 2, 3, 'sds') (1626006833612, 4, 5, "
+ "'ddd') ct10 using st1 tags(1000, \"ttt\", true) values(1626006833603, 4, 3, 'hwj')");
+ if (taos_errno(pRes) != 0) {
+ printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
+ return 0;
+}
+
+int32_t init_env() {
+ TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+ if (pConn == NULL) {
+ return -1;
+ }
+
+ TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_dst");
+ if (taos_errno(pRes) != 0) {
+ printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
+ pRes = taos_query(pConn, "create database if not exists db_dst vgroups 1 wal_retention_period 3600");
+ if (taos_errno(pRes) != 0) {
+ printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
+ pRes = taos_query(pConn, "drop topic if exists topic_db");
+ if (taos_errno(pRes) != 0) {
+ printf("error in drop topic, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
+ pRes = taos_query(pConn, "drop database if exists db_src");
+ if (taos_errno(pRes) != 0) {
+ printf("error in drop db, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
+ pRes = taos_query(pConn, "create database if not exists db_src vgroups 1 wal_retention_period 3600");
+ if (taos_errno(pRes) != 0) {
+ printf("error in create db, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
+ pRes = taos_query(pConn, "use db_src");
+ if (taos_errno(pRes) != 0) {
+ printf("error in use db, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
+ buildDatabase(pConn, pRes);
+
+ taos_close(pConn);
+ return 0;
+}
+
+int32_t create_topic() {
+ printf("create topic\n");
+ TAOS_RES* pRes;
+ TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+ if (pConn == NULL) {
+ return -1;
+ }
+
+ pRes = taos_query(pConn, "create topic topic_db with meta as database db_src");
+ if (taos_errno(pRes) != 0) {
+ printf("failed to create topic topic_db, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
+ taos_close(pConn);
+ return 0;
+}
+
+void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) {
+ printf("commit %d tmq %p param %p\n", code, tmq, param);
+}
+
+tmq_t* build_consumer() {
+ tmq_conf_t* conf = tmq_conf_new();
+ tmq_conf_set(conf, "group.id", "tg2");
+ tmq_conf_set(conf, "client.id", "my app 1");
+ tmq_conf_set(conf, "td.connect.user", "root");
+ tmq_conf_set(conf, "td.connect.pass", "taosdata");
+ tmq_conf_set(conf, "msg.with.table.name", "true");
+ tmq_conf_set(conf, "enable.auto.commit", "true");
+ tmq_conf_set(conf, "auto.offset.reset", "earliest");
+ tmq_conf_set(conf, "msg.consume.rawdata", "1");
+
+ tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
+ tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
+ assert(tmq);
+ tmq_conf_destroy(conf);
+ return tmq;
+}
+
+tmq_list_t* build_topic_list() {
+ tmq_list_t* topic_list = tmq_list_new();
+ tmq_list_append(topic_list, "topic_db");
+ return topic_list;
+}
+
+void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
+ int32_t code;
+
+ if ((code = tmq_subscribe(tmq, topics))) {
+ fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(code));
+ printf("subscribe err\n");
+ return;
+ }
+ int32_t cnt = 0;
+ while (1) {
+ TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 5000);
+ if (tmqmessage) {
+ cnt++;
+ msg_process(tmqmessage);
+ taos_free_result(tmqmessage);
+ } else {
+ break;
+ }
+ }
+
+ code = tmq_consumer_close(tmq);
+ if (code)
+ fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code));
+ else
+ fprintf(stderr, "%% Consumer closed\n");
+}
+
+void check_result(){
+ TAOS* taos = use_db();
+ TAOS_RES *res = taos_query(taos, "select * from st1");
+ int code = taos_errno(res);
+ assert(code == 0);
+ int affectedRows = taos_affected_rows(res);
+ printf("affected rows %d\n", affectedRows);
+ assert(affectedRows == 10);
+ taos_free_result(res);
+ taos_close(taos);
+
+}
+int main(int argc, char* argv[]) {
+ if (init_env() < 0) {
+ return -1;
+ }
+ create_topic();
+
+ tmq_t* tmq = build_consumer();
+ tmq_list_t* topic_list = build_topic_list();
+ basic_consume_loop(tmq, topic_list);
+ tmq_list_destroy(topic_list);
+}
diff --git a/utils/test/c/tmq_write_raw_test.c b/utils/test/c/tmq_write_raw_test.c
index f33fac9a0a..e8d4c6abae 100644
--- a/utils/test/c/tmq_write_raw_test.c
+++ b/utils/test/c/tmq_write_raw_test.c
@@ -136,6 +136,13 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes) {
}
taos_free_result(pRes);
+ pRes = taos_query(pConn, "alter table ct0 set tag t4=false");
+ if (taos_errno(pRes) != 0) {
+ printf("alter table ct0 set tag t4=false, reason:%s\n", taos_errstr(pRes));
+ return -1;
+ }
+ taos_free_result(pRes);
+
return 0;
}
@@ -226,6 +233,15 @@ tmq_t* build_consumer() {
tmq_conf_set(conf, "enable.auto.commit", "true");
tmq_conf_set(conf, "auto.offset.reset", "earliest");
tmq_conf_set(conf, "msg.consume.excluded", "1");
+ tmq_conf_set(conf, "fetch.max.wait.ms", "1");
+ assert(tmq_conf_set(conf, "fetch.max.wait.ms", "100000000000") == TMQ_CONF_INVALID);
+ assert(tmq_conf_set(conf, "fetch.max.wait.ms", "-100000000000") == TMQ_CONF_INVALID);
+ assert(tmq_conf_set(conf, "fetch.max.wait.ms", "0") == TMQ_CONF_INVALID);
+ assert(tmq_conf_set(conf, "fetch.max.wait.ms", "1000") == TMQ_CONF_OK);
+ assert(tmq_conf_set(conf, "min.poll.rows", "100000000000") == TMQ_CONF_INVALID);
+ assert(tmq_conf_set(conf, "min.poll.rows", "-1") == TMQ_CONF_INVALID);
+ assert(tmq_conf_set(conf, "min.poll.rows", "0") == TMQ_CONF_INVALID);
+ assert(tmq_conf_set(conf, "min.poll.rows", "1") == TMQ_CONF_OK);
// tmq_conf_set(conf, "max.poll.interval.ms", "20000");
tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);