diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index adefacc6af..c273e9889f 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
# taos-tools
ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
- GIT_TAG 7d5c1c0
+ GIT_TAG 2dba49c
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in
index 15d3764876..04b1262caf 100644
--- a/cmake/taosws_CMakeLists.txt.in
+++ b/cmake/taosws_CMakeLists.txt.in
@@ -2,7 +2,7 @@
# taosws-rs
ExternalProject_Add(taosws-rs
GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git
- GIT_TAG 0609b50
+ GIT_TAG e771403
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
diff --git a/docs/en/04-concept/index.md b/docs/en/04-concept/index.md
index b0a0c25d85..0b1b226c17 100644
--- a/docs/en/04-concept/index.md
+++ b/docs/en/04-concept/index.md
@@ -6,101 +6,100 @@ In order to explain the basic concepts and provide some sample code, the TDengin
-
- Device ID
- Time Stamp
- Collected Metrics
- Tags
+
+
+ Device ID
+ Timestamp
+ Collected Metrics
+ Tags
-
-Device ID
-Time Stamp
-current
-voltage
-phase
-location
-groupId
-
-
-
-
-d1001
-1538548685000
-10.3
-219
-0.31
-California.SanFrancisco
-2
-
-
-d1002
-1538548684000
-10.2
-220
-0.23
-California.SanFrancisco
-3
-
-
-d1003
-1538548686500
-11.5
-221
-0.35
-California.LosAngeles
-3
-
-
-d1004
-1538548685500
-13.4
-223
-0.29
-California.LosAngeles
-2
-
-
-d1001
-1538548695000
-12.6
-218
-0.33
-California.SanFrancisco
-2
-
-
-d1004
-1538548696600
-11.8
-221
-0.28
-California.LosAngeles
-2
-
-
-d1002
-1538548696650
-10.3
-218
-0.25
-California.SanFrancisco
-3
-
-
-d1001
-1538548696800
-12.3
-221
-0.31
-California.SanFrancisco
-2
-
-
+
+ current
+ voltage
+ phase
+ location
+ groupid
+
+
+
+
+ d1001
+ 1538548685000
+ 10.3
+ 219
+ 0.31
+ California.SanFrancisco
+ 2
+
+
+ d1002
+ 1538548684000
+ 10.2
+ 220
+ 0.23
+ California.SanFrancisco
+ 3
+
+
+ d1003
+ 1538548686500
+ 11.5
+ 221
+ 0.35
+ California.LosAngeles
+ 3
+
+
+ d1004
+ 1538548685500
+ 13.4
+ 223
+ 0.29
+ California.LosAngeles
+ 2
+
+
+ d1001
+ 1538548695000
+ 12.6
+ 218
+ 0.33
+ California.SanFrancisco
+ 2
+
+
+ d1004
+ 1538548696600
+ 11.8
+ 221
+ 0.28
+ California.LosAngeles
+ 2
+
+
+ d1002
+ 1538548696650
+ 10.3
+ 218
+ 0.25
+ California.SanFrancisco
+ 3
+
+
+ d1001
+ 1538548696800
+ 12.3
+ 221
+ 0.31
+ California.SanFrancisco
+ 2
+
+
Table 1: Smart meter example data
-Each row contains the device ID, time stamp, collected metrics (current, voltage, phase as above), and static tags (location and groupId in Table 1) associated with the devices. Each smart meter generates a row (measurement) in a pre-defined time interval or triggered by an external event. The device produces a sequence of measurements with associated time stamps.
+Each row contains the device ID, timestamp, collected metrics (`current`, `voltage`, `phase` as above), and static tags (`location` and `groupid` in Table 1) associated with the devices. Each smart meter generates a row (measurement) in a pre-defined time interval or triggered by an external event. The device produces a sequence of measurements with associated timestamps.
## Metric
@@ -112,22 +111,22 @@ Label/Tag refers to the static properties of sensors, equipment or other types o
## Data Collection Point
-Data Collection Point (DCP) refers to hardware or software that collects metrics based on preset time periods or triggered by events. A data collection point can collect one or multiple metrics, but these metrics are collected at the same time and have the same time stamp. For some complex equipment, there are often multiple data collection points, and the sampling rate of each collection point may be different, and fully independent. For example, for a car, there could be a data collection point to collect GPS position metrics, a data collection point to collect engine status metrics, and a data collection point to collect the environment metrics inside the car. So in this example the car would have three data collection points. In the smart meters example, d1001, d1002, d1003, and d1004 are the data collection points.
+Data Collection Point (DCP) refers to hardware or software that collects metrics based on preset time periods or triggered by events. A data collection point can collect one or multiple metrics, but these metrics are collected at the same time and have the same timestamp. For some complex equipment, there are often multiple data collection points, and the sampling rate of each collection point may be different, and fully independent. For example, for a car, there could be a data collection point to collect GPS position metrics, a data collection point to collect engine status metrics, and a data collection point to collect the environment metrics inside the car. So in this example the car would have three data collection points. In the smart meters example, d1001, d1002, d1003, and d1004 are the data collection points.
## Table
Since time-series data is most likely to be structured data, TDengine adopts the traditional relational database model to process them with a short learning curve. You need to create a database, create tables, then insert data points and execute queries to explore the data.
-To make full use of time-series data characteristics, TDengine adopts a strategy of "**One Table for One Data Collection Point**". TDengine requires the user to create a table for each data collection point (DCP) to store collected time-series data. For example, if there are over 10 million smart meters, it means 10 million tables should be created. For the table above, 4 tables should be created for devices D1001, D1002, D1003, and D1004 to store the data collected. This design has several benefits:
+To make full use of time-series data characteristics, TDengine adopts a strategy of "**One Table for One Data Collection Point**". TDengine requires the user to create a table for each data collection point (DCP) to store collected time-series data. For example, if there are over 10 million smart meters, it means 10 million tables should be created. For the table above, 4 tables should be created for devices d1001, d1002, d1003, and d1004 to store the data collected. This design has several benefits:
1. Since the metric data from different DCP are fully independent, the data source of each DCP is unique, and a table has only one writer. In this way, data points can be written in a lock-free manner, and the writing speed can be greatly improved.
2. For a DCP, the metric data generated by DCP is ordered by timestamp, so the write operation can be implemented by simple appending, which further greatly improves the data writing speed.
3. The metric data from a DCP is continuously stored, block by block. If you read data for a period of time, it can greatly reduce random read operations and improve read and query performance by orders of magnitude.
4. Inside a data block for a DCP, columnar storage is used, and different compression algorithms are used for different data types. Metrics generally don't vary as significantly between themselves over a time range as compared to other metrics, which allows for a higher compression rate.
-If the metric data of multiple DCPs are traditionally written into a single table, due to uncontrollable network delays, the timing of the data from different DCPs arriving at the server cannot be guaranteed, write operations must be protected by locks, and metric data from one DCP cannot be guaranteed to be continuously stored together. ** One table for one data collection point can ensure the best performance of insert and query of a single data collection point to the greatest possible extent.**
+If the metric data of multiple DCPs are traditionally written into a single table, due to uncontrollable network delays, the timing of the data from different DCPs arriving at the server cannot be guaranteed, write operations must be protected by locks, and metric data from one DCP cannot be guaranteed to be continuously stored together. **One table for one data collection point can ensure the best performance of insert and query of a single data collection point to the greatest possible extent.**
-TDengine suggests using DCP ID as the table name (like D1001 in the above table). Each DCP may collect one or multiple metrics (like the current, voltage, phase as above). Each metric has a corresponding column in the table. The data type for a column can be int, float, string and others. In addition, the first column in the table must be a timestamp. TDengine uses the time stamp as the index, and won’t build the index on any metrics stored. Column wise storage is used.
+TDengine suggests using DCP ID as the table name (like d1001 in the above table). Each DCP may collect one or multiple metrics (like the `current`, `voltage`, `phase` as above). Each metric has a corresponding column in the table. The data type for a column can be int, float, string and others. In addition, the first column in the table must be a timestamp. TDengine uses the timestamp as the index, and won’t build the index on any metrics stored. Column wise storage is used.
Complex devices, such as connected cars, may have multiple DCPs. In this case, multiple tables are created for a single device, one table per DCP.
@@ -156,9 +155,16 @@ The relationship between a STable and the subtables created based on this STable
Queries can be executed on both a table (subtable) and a STable. For a query on a STable, TDengine will treat the data in all its subtables as a whole data set for processing. TDengine will first find the subtables that meet the tag filter conditions, then scan the time-series data of these subtables to perform aggregation operation, which reduces the number of data sets to be scanned which in turn greatly improves the performance of data aggregation across multiple DCPs. In essence, querying a supertable is a very efficient aggregate query on multiple DCPs of the same type.
-In TDengine, it is recommended to use a subtable instead of a regular table for a DCP. In the smart meters example, we can create subtables like d1001, d1002, d1003, and d1004 under super table meters.
+In TDengine, it is recommended to use a subtable instead of a regular table for a DCP. In the smart meters example, we can create subtables like d1001, d1002, d1003, and d1004 under super table `meters`.
-To better understand the data model using metri, tags, super table and subtable, please refer to the diagram below which demonstrates the data model of the smart meters example. 
+To better understand the data model using metrics, tags, super table and subtable, please refer to the diagram below which demonstrates the data model of the smart meters example.
+
+
+
+
+
+Figure 1. Meters Data Model Diagram
## Database
@@ -172,4 +178,4 @@ FQDN (Fully Qualified Domain Name) is the full domain name of a specific compute
Each node of a TDengine cluster is uniquely identified by an End Point, which consists of an FQDN and a Port, such as h1.tdengine.com:6030. In this way, when the IP changes, we can still use the FQDN to dynamically find the node without changing any configuration of the cluster. In addition, FQDN is used to facilitate unified access to the same cluster from the Intranet and the Internet.
-TDengine does not recommend using an IP address to access the cluster. FQDN is recommended for cluster management.
+TDengine does not recommend using an IP address to access the cluster. FQDN is recommended for cluster management.
diff --git a/docs/en/05-get-started/01-docker.md b/docs/en/05-get-started/01-docker.md
index de5b620a77..66f7d5d594 100644
--- a/docs/en/05-get-started/01-docker.md
+++ b/docs/en/05-get-started/01-docker.md
@@ -13,7 +13,7 @@ If Docker is already installed on your computer, run the following command:
docker run -d -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043-6049/udp tdengine/tdengine
```
-Note that TDengine Server uses TCP port 6030. Port 6041 is used by taosAdapter for the REST API service. Ports 6043 through 6049 are used by taosAdapter for other connectors. You can open these ports as needed.
+Note that TDengine Server 3.0 uses TCP port 6030. Port 6041 is used by taosAdapter for the REST API service. Ports 6043 through 6049 are used by taosAdapter for other connectors. You can open these ports as needed.
Run the following command to ensure that your container is running:
@@ -21,7 +21,7 @@ Run the following command to ensure that your container is running:
docker ps
```
-Enter the container and open the bash shell:
+Enter the container and open the `bash` shell:
```shell
docker exec -it bash
@@ -31,68 +31,68 @@ You can now access TDengine or run other Linux commands.
Note: For information about installing docker, see the [official documentation](https://docs.docker.com/get-docker/).
-## Insert Data into TDengine
-
-You can use the `taosBenchmark` tool included with TDengine to write test data into your deployment.
-
-To do so, run the following command:
-
- ```bash
- $ taosBenchmark
-
- ```
-
-This command creates the `meters` supertable in the `test` database. In the `meters` supertable, it then creates 10,000 subtables named `d0` to `d9999`. Each table has 10,000 rows and each row has four columns: `ts`, `current`, `voltage`, and `phase`. The timestamps of the data in these columns range from 2017-07-14 10:40:00 000 to 2017-07-14 10:40:09 999. Each table is randomly assigned a `groupId` tag from 1 to 10 and a `location` tag of either `Campbell`, `Cupertino`, `Los Angeles`, `Mountain View`, `Palo Alto`, `San Diego`, `San Francisco`, `San Jose`, `Santa Clara` or `Sunnyvale`.
-
- The `taosBenchmark` command creates a deployment with 100 million data points that you can use for testing purposes. The time required depends on the hardware specifications of the local system.
-
- You can customize the test deployment that taosBenchmark creates by specifying command-line parameters. For information about command-line parameters, run the `taosBenchmark --help` command. For more information about taosBenchmark, see [taosBenchmark](/reference/taosbenchmark).
-
## Open the TDengine CLI
-On the container, run the following command to open the TDengine CLI:
+On the container, run the following command to open the TDengine CLI:
```
$ taos
-taos>
+taos>
```
-## Query Data in TDengine
+## Test data insert performance
-After using taosBenchmark to create your test deployment, you can run queries in the TDengine CLI to test its performance. For example:
+After your TDengine Server is running normally, you can run the taosBenchmark utility to test its performance:
-From the TDengine CLI query the number of rows in the `meters` supertable:
+Start TDengine service and execute `taosBenchmark` (formerly named `taosdemo`) in a Linux or Windows terminal.
+
+```bash
+taosBenchmark
+```
+
+This command creates the `meters` supertable in the `test` database. In the `meters` supertable, it then creates 10,000 subtables named `d0` to `d9999`. Each table has 10,000 rows and each row has four columns: `ts`, `current`, `voltage`, and `phase`. The timestamps of the data in these columns range from 2017-07-14 10:40:00 000 to 2017-07-14 10:40:09 999. Each table is randomly assigned a `groupId` tag from 1 to 10 and a `location` tag of either `Campbell`, `Cupertino`, `Los Angeles`, `Mountain View`, `Palo Alto`, `San Diego`, `San Francisco`, `San Jose`, `Santa Clara` or `Sunnyvale`.
+
+The `taosBenchmark` command creates a deployment with 100 million data points that you can use for testing purposes. The time required to create the deployment depends on your hardware. On most modern servers, the deployment is created in ten to twenty seconds.
+
+You can customize the test deployment that taosBenchmark creates by specifying command-line parameters. For information about command-line parameters, run the `taosBenchmark --help` command. For more information about taosBenchmark, see [taosBenchmark](../../reference/taosbenchmark).
+
+## Test data query performance
+
+After using `taosBenchmark` to create your test deployment, you can run queries in the TDengine CLI to test its performance:
+
+From the TDengine CLI (taos) query the number of rows in the `meters` supertable:
```sql
-select count(*) from test.meters;
+SELECT COUNT(*) FROM test.meters;
```
Query the average, maximum, and minimum values of all 100 million rows of data:
```sql
-select avg(current), max(voltage), min(phase) from test.meters;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters;
```
Query the number of rows whose `location` tag is `San Francisco`:
```sql
-select count(*) from test.meters where location="San Francisco";
+SELECT COUNT(*) FROM test.meters WHERE location = "San Francisco";
```
Query the average, maximum, and minimum values of all rows whose `groupId` tag is `10`:
```sql
-select avg(current), max(voltage), min(phase) from test.meters where groupId=10;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 10;
```
-Query the average, maximum, and minimum values for table `d10` in 1 second intervals:
+Query the average, maximum, and minimum values for table `d10` in 10 second intervals:
```sql
-select first(ts), avg(current), max(voltage), min(phase) from test.d10 interval(1s);
+SELECT FIRST(ts), AVG(current), MAX(voltage), MIN(phase) FROM test.d10 INTERVAL(10s);
```
-In the query above you are selecting the first timestamp (ts) in the interval, another way of selecting this would be _wstart which will give the start of the time window. For more information about windowed queries, see [Time-Series Extensions](../../taos-sql/distinguished/).
+
+In the query above you are selecting the first timestamp (ts) in the interval, another way of selecting this would be `\_wstart` which will give the start of the time window. For more information about windowed queries, see [Time-Series Extensions](../../taos-sql/distinguished/).
## Additional Information
diff --git a/docs/en/05-get-started/03-package.md b/docs/en/05-get-started/03-package.md
index 88096a759c..7257fccc80 100644
--- a/docs/en/05-get-started/03-package.md
+++ b/docs/en/05-get-started/03-package.md
@@ -9,23 +9,24 @@ import PkgListV3 from "/components/PkgListV3";
For information about installing TDengine on Docker, see [Quick Install on Docker](../../get-started/docker). If you want to view the source code, build TDengine yourself, or contribute to the project, see the [TDengine GitHub repository](https://github.com/taosdata/TDengine).
-The full package of TDengine includes the TDengine Server (`taosd`), TDengine Client (`taosc`), taosAdapter for connecting with third-party systems and providing a RESTful interface, a command-line interface, and some tools. Note that taosAdapter supports Linux only. In addition to connectors for multiple languages, TDengine also provides a [REST API](../../reference/rest-api) through [taosAdapter](../../reference/taosadapter).
+The full package of TDengine includes the TDengine Server (`taosd`), TDengine Client (`taosc`), taosAdapter for connecting with third-party systems and providing a RESTful interface, a command-line interface (CLI, taos), and some tools. Note that taosAdapter supports Linux only. In addition to connectors for multiple languages, TDengine also provides a [REST API](../../reference/rest-api) through [taosAdapter](../../reference/taosadapter).
-The standard server installation package includes `taos`, `taosd`, `taosAdapter`, `taosBenchmark`, and sample code. You can also download a lite package that includes only `taosd` and the C/C++ connector.
+The standard server installation package includes `taos`, `taosd`, `taosAdapter`, `taosBenchmark`, and sample code. You can also download the Lite package that includes only `taosd` and the C/C++ connector.
-The TDengine Community Edition is released as .deb and .rpm packages. The .deb package can be installed on Debian, Ubuntu, and derivative systems. The .rpm package can be installed on CentOS, RHEL, SUSE, and derivative systems. A .tar.gz package is also provided for enterprise customers, and you can install TDengine over `apt-get` as well. The .tar.tz package includes `taosdump` and the TDinsight installation script. If you want to use these utilities with the .deb or .rpm package, download and install taosTools separately. TDengine can also be installed on 64-bit Windows servers.
+The TDengine Community Edition is released as Deb and RPM packages. The Deb package can be installed on Debian, Ubuntu, and derivative systems. The RPM package can be installed on CentOS, RHEL, SUSE, and derivative systems. A .tar.gz package is also provided for enterprise customers, and you can install TDengine over `apt-get` as well. The .tar.tz package includes `taosdump` and the TDinsight installation script. If you want to use these utilities with the Deb or RPM package, download and install taosTools separately. TDengine can also be installed on 64-bit Windows.
## Installation
-1. Download the .deb installation package.
-
+1. Download the Deb installation package.
+
2. In the directory where the package is located, use `dpkg` to install the package:
+> Please replace `` with the corresponding version of the package downloaded
+
```bash
-# Enter the name of the package that you downloaded.
sudo dpkg -i TDengine-server--Linux-x64.deb
```
@@ -34,11 +35,12 @@ sudo dpkg -i TDengine-server--Linux-x64.deb
1. Download the .rpm installation package.
-
+
2. In the directory where the package is located, use rpm to install the package:
+> Please replace `` with the corresponding version of the package downloaded
+
```bash
-# Enter the name of the package that you downloaded.
sudo rpm -ivh TDengine-server--Linux-x64.rpm
```
@@ -47,11 +49,12 @@ sudo rpm -ivh TDengine-server--Linux-x64.rpm
1. Download the .tar.gz installation package.
-
+
2. In the directory where the package is located, use `tar` to decompress the package:
+> Please replace `` with the corresponding version of the package downloaded
+
```bash
-# Enter the name of the package that you downloaded.
tar -zxvf TDengine-server--Linux-x64.tar.gz
```
@@ -96,23 +99,23 @@ sudo apt-get install tdengine
This installation method is supported only for Debian and Ubuntu.
::::
-
+
-Note: TDengine only supports Windows Server 2016/2019 and windows 10/11 system versions on the windows platform.
+Note: TDengine only supports Windows Server 2016/2019 and Windows 10/11 on the Windows platform.
1. Download the Windows installation package.
-
+
2. Run the downloaded package to install TDengine.
:::info
-For information about TDengine releases, see [Release History](../../releases).
+For information about TDengine releases, see [Release History](../../releases).
:::
:::note
-On the first node in your TDengine cluster, leave the `Enter FQDN:` prompt blank and press **Enter**. On subsequent nodes, you can enter the end point of the first dnode in the cluster. You can also configure this setting after you have finished installing TDengine.
+On the first node in your TDengine cluster, leave the `Enter FQDN:` prompt blank and press **Enter**. On subsequent nodes, you can enter the endpoint of the first dnode in the cluster. You can also configure this setting after you have finished installing TDengine.
:::
@@ -147,7 +150,7 @@ Active: inactive (dead)
After confirming that TDengine is running, run the `taos` command to access the TDengine CLI.
-The following `systemctl` commands can help you manage TDengine:
+The following `systemctl` commands can help you manage TDengine service:
- Start TDengine Server: `systemctl start taosd`
@@ -159,7 +162,7 @@ The following `systemctl` commands can help you manage TDengine:
:::info
-- The `systemctl` command requires _root_ privileges. If you are not logged in as the `root` user, use the `sudo` command.
+- The `systemctl` command requires _root_ privileges. If you are not logged in as the _root_ user, use the `sudo` command.
- The `systemctl stop taosd` command does not instantly stop TDengine Server. The server is stopped only after all data in memory is flushed to disk. The time required depends on the cache size.
- If your system does not include `systemd`, you can run `/usr/local/taos/bin/taosd` to start TDengine manually.
@@ -174,23 +177,9 @@ After the installation is complete, run `C:\TDengine\taosd.exe` to start TDengin
-## Test data insert performance
+## Command Line Interface (CLI)
-After your TDengine Server is running normally, you can run the taosBenchmark utility to test its performance:
-
-```bash
-taosBenchmark
-```
-
-This command creates the `meters` supertable in the `test` database. In the `meters` supertable, it then creates 10,000 subtables named `d0` to `d9999`. Each table has 10,000 rows and each row has four columns: `ts`, `current`, `voltage`, and `phase`. The timestamps of the data in these columns range from 2017-07-14 10:40:00 000 to 2017-07-14 10:40:09 999. Each table is randomly assigned a `groupId` tag from 1 to 10 and a `location` tag of either `Campbell`, `Cupertino`, `Los Angeles`, `Mountain View`, `Palo Alto`, `San Diego`, `San Francisco`, `San Jose`, `Santa Clara` or `Sunnyvale`.
-
-The `taosBenchmark` command creates a deployment with 100 million data points that you can use for testing purposes. The time required to create the deployment depends on your hardware. On most modern servers, the deployment is created in less than a minute.
-
-You can customize the test deployment that taosBenchmark creates by specifying command-line parameters. For information about command-line parameters, run the `taosBenchmark --help` command. For more information about taosBenchmark, see [taosBenchmark](../../reference/taosbenchmark).
-
-## Command Line Interface
-
-You can use the TDengine CLI to monitor your TDengine deployment and execute ad hoc queries. To open the CLI, run the following command:
+You can use the TDengine CLI to monitor your TDengine deployment and execute ad hoc queries. To open the CLI, you can execute `taos` in the Linux terminal where TDengine is installed, or you can run `taos.exe` in the `C:\TDengine` directory of the Windows terminal where TDengine is installed to start the TDengine command line.
```bash
taos
@@ -205,52 +194,71 @@ taos>
For example, you can create and delete databases and tables and run all types of queries. Each SQL command must be end with a semicolon (;). For example:
```sql
-create database demo;
-use demo;
-create table t (ts timestamp, speed int);
-insert into t values ('2019-07-15 00:00:00', 10);
-insert into t values ('2019-07-15 01:00:00', 20);
-select * from t;
+CREATE DATABASE demo;
+USE demo;
+CREATE TABLE t (ts TIMESTAMP, speed INT);
+INSERT INTO t VALUES ('2019-07-15 00:00:00', 10);
+INSERT INTO t VALUES ('2019-07-15 01:00:00', 20);
+SELECT * FROM t;
+
ts | speed |
========================================
2019-07-15 00:00:00.000 | 10 |
2019-07-15 01:00:00.000 | 20 |
+
Query OK, 2 row(s) in set (0.003128s)
```
You can also can monitor the deployment status, add and remove user accounts, and manage running instances. You can run the TDengine CLI on either Linux or Windows machines. For more information, see [TDengine CLI](../../reference/taos-shell/).
-
+
+## Test data insert performance
+
+After your TDengine Server is running normally, you can run the taosBenchmark utility to test its performance:
+
+Start TDengine service and execute `taosBenchmark` (formerly named `taosdemo`) in a Linux or Windows terminal.
+
+```bash
+taosBenchmark
+```
+
+This command creates the `meters` supertable in the `test` database. In the `meters` supertable, it then creates 10,000 subtables named `d0` to `d9999`. Each table has 10,000 rows and each row has four columns: `ts`, `current`, `voltage`, and `phase`. The timestamps of the data in these columns range from 2017-07-14 10:40:00 000 to 2017-07-14 10:40:09 999. Each table is randomly assigned a `groupId` tag from 1 to 10 and a `location` tag of either `Campbell`, `Cupertino`, `Los Angeles`, `Mountain View`, `Palo Alto`, `San Diego`, `San Francisco`, `San Jose`, `Santa Clara` or `Sunnyvale`.
+
+The `taosBenchmark` command creates a deployment with 100 million data points that you can use for testing purposes. The time required to create the deployment depends on your hardware. On most modern servers, the deployment is created in ten to twenty seconds.
+
+You can customize the test deployment that taosBenchmark creates by specifying command-line parameters. For information about command-line parameters, run the `taosBenchmark --help` command. For more information about taosBenchmark, see [taosBenchmark](../../reference/taosbenchmark).
+
## Test data query performance
-After using taosBenchmark to create your test deployment, you can run queries in the TDengine CLI to test its performance:
+After using `taosBenchmark` to create your test deployment, you can run queries in the TDengine CLI to test its performance:
-From the TDengine CLI query the number of rows in the `meters` supertable:
+From the TDengine CLI (taos) query the number of rows in the `meters` supertable:
```sql
-select count(*) from test.meters;
+SELECT COUNT(*) FROM test.meters;
```
Query the average, maximum, and minimum values of all 100 million rows of data:
```sql
-select avg(current), max(voltage), min(phase) from test.meters;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters;
```
Query the number of rows whose `location` tag is `San Francisco`:
```sql
-select count(*) from test.meters where location="San Francisco";
+SELECT COUNT(*) FROM test.meters WHERE location = "San Francisco";
```
Query the average, maximum, and minimum values of all rows whose `groupId` tag is `10`:
```sql
-select avg(current), max(voltage), min(phase) from test.meters where groupId=10;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 10;
```
-Query the average, maximum, and minimum values for table `d10` in 1 second intervals:
+Query the average, maximum, and minimum values for table `d10` in 10 second intervals:
```sql
-select first(ts), avg(current), max(voltage), min(phase) from test.d10 interval(1s);
+SELECT FIRST(ts), AVG(current), MAX(voltage), MIN(phase) FROM test.d10 INTERVAL(10s);
```
-In the query above you are selecting the first timestamp (ts) in the interval, another way of selecting this would be _wstart which will give the start of the time window. For more information about windowed queries, see [Time-Series Extensions](../../taos-sql/distinguished/).
+
+In the query above you are selecting the first timestamp (ts) in the interval, another way of selecting this would be `\_wstart` which will give the start of the time window. For more information about windowed queries, see [Time-Series Extensions](../../taos-sql/distinguished/).
diff --git a/docs/en/12-taos-sql/20-keywords.md b/docs/en/12-taos-sql/20-keywords.md
index 6f166c8034..d7b9c73c4f 100644
--- a/docs/en/12-taos-sql/20-keywords.md
+++ b/docs/en/12-taos-sql/20-keywords.md
@@ -238,6 +238,7 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam
- TOPICS
- TRIGGER
- TSERIES
+- TTL
### U
@@ -253,6 +254,7 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam
### V
+- VALUE
- VALUES
- VARIABLE
- VARIABLES
diff --git a/docs/en/14-reference/03-connector/index.mdx b/docs/en/14-reference/03-connector/index.mdx
index 49e2dceec5..5dc54f0934 100644
--- a/docs/en/14-reference/03-connector/index.mdx
+++ b/docs/en/14-reference/03-connector/index.mdx
@@ -39,14 +39,14 @@ Comparing the connector support for TDengine functional features as follows.
### Using the native interface (taosc)
-| **Functional Features** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** |
-| -------------- | -------- | ---------- | ------ | ------ | ----------- | -------- |
-| **Connection Management** | Support | Support | Support | Support | Support | Support |
-| **Regular Query** | Support | Support | Support | Support | Support | Support |
-| **Parameter Binding** | Support | Support | Support | Support | Support | Support |
-| ** TMQ ** | Support | Support | Support | Support | Support | Support |
-| **Schemaless** | Support | Support | Support | Support | Support | Support |
-| **DataFrame** | Not Supported | Support | Not Supported | Not Supported | Not Supported | Not Supported |
+| **Functional Features** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** |
+| ----------------------------- | ------------- | ---------- | ------------- | ------------- | ------------- | ------------- |
+| **Connection Management** | Support | Support | Support | Support | Support | Support |
+| **Regular Query** | Support | Support | Support | Support | Support | Support |
+| **Parameter Binding** | Support | Support | Support | Support | Support | Support |
+| **Subscription (TMQ)** | Support | Support | Support | Support | Support | Support |
+| **Schemaless** | Support | Support | Support | Support | Support | Support |
+| **DataFrame** | Not Supported | Support | Not Supported | Not Supported | Not Supported | Not Supported |
:::info
The different database framework specifications for various programming languages do not mean that all C/C++ interfaces need a wrapper.
@@ -54,16 +54,15 @@ The different database framework specifications for various programming language
### Use HTTP Interfaces (REST or WebSocket)
-| **Functional Features** | **Java** | **Python** | **Go** | **C# (not supported yet)** | **Node.js** | **Rust** |
-| ------------------------------ | -------- | ---------- | -------- | ------------------ | ----------- | -------- |
-| **Connection Management** | Support | Support | Support | N/A | Support | Support |
-| **Regular Query** | Support | Support | Support | N/A | Support | Support |
-| **Continous Query ** | Support | Support | Support | N/A | Support | Support |
-| **Parameter Binding** | Not supported | Not supported | Not supported | N/A | Not supported | Support |
-| ** TMQ ** | Not supported | Not supported | Not supported | N/A | Not supported | Support |
-| **Schemaless** | Not supported | Not supported | Not supported | N/A | Not supported | Not supported |
-| **Bulk Pulling (based on WebSocket) **| Support | Support | Not Supported | N/A | Not Supported | Supported |
-| **DataFrame** | Not supported | Support | Not supported | N/A | Not supported | Not supported |
+| **Functional Features** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** |
+| -------------------------------------- | ------------- | --------------- | ------------- | ------------- | ------------- | ------------- |
+| **Connection Management** | Support | Support | Support | Support | Support | Support |
+| **Regular Query** | Support | Support | Support | Support | Support | Support |
+| **Parameter Binding** | Not supported | Not supported | Not supported | Support | Not supported | Support |
+| **Subscription (TMQ) ** | Not supported | Not supported | Not supported | Not supported | Not supported | Support |
+| **Schemaless** | Not supported | Not supported | Not supported | Not supported | Not supported | Not supported |
+| **Bulk Pulling (based on WebSocket) ** | Support | Support | Not Supported | support | Not Supported | Supported |
+| **DataFrame** | Not supported | Support | Not supported | Not supported | Not supported | Not supported |
:::warning
diff --git a/docs/examples/csharp/SQLInsertExample.cs b/docs/examples/csharp/SQLInsertExample.cs
index 192ea96d57..3ce70fe914 100644
--- a/docs/examples/csharp/SQLInsertExample.cs
+++ b/docs/examples/csharp/SQLInsertExample.cs
@@ -23,6 +23,7 @@ namespace TDengineExample
CheckRes(conn, res, "failed to insert data");
int affectedRows = TDengine.AffectRows(res);
Console.WriteLine("affectedRows " + affectedRows);
+ TDengine.FreeResult(res);
ExitProgram(conn, 0);
}
diff --git a/docs/zh/02-intro.md b/docs/zh/02-intro.md
index 9a0a6fb547..47bfd3f96b 100644
--- a/docs/zh/02-intro.md
+++ b/docs/zh/02-intro.md
@@ -48,7 +48,7 @@ TDengine 的主要功能如下：
- 多种[数据导出](../operation/export)方式
9. 工具
- 提供[交互式命令行程序（CLI）](../reference/taos-shell)，便于管理集群，检查系统状态，做即席查询
- - 提供压力测试工具[taosBenchmark](../reference/taosbenchmark)，用于测试 TDengine 的性能
+ - 提供压力测试工具 [taosBenchmark](../reference/taosbenchmark)，用于测试 TDengine 的性能
10. 编程
- 提供各种语言的[连接器（Connector）](../connector): 如 [C/C++](../connector/cpp)、[Java](../connector/java)、[Go](../connector/go)、[Node.js](../connector/node)、[Rust](../connector/rust)、[Python](../connector/python)、[C#](../connector/csharp) 等
- 支持 [REST 接口](../connector/rest-api/)
diff --git a/docs/zh/04-concept/index.md b/docs/zh/04-concept/index.md
index 89d3df9c97..2cba68edcd 100644
--- a/docs/zh/04-concept/index.md
+++ b/docs/zh/04-concept/index.md
@@ -4,119 +4,118 @@ title: 数据模型和基本概念
description: TDengine 的数据模型和基本概念
---
-为了便于解释基本概念，便于撰写示例程序，整个 TDengine 文档以智能电表作为典型时序数据场景。假设每个智能电表采集电流、电压、相位三个量，有多个智能电表，每个电表有位置 location 和分组 group ID 的静态属性. 其采集的数据类似如下的表格：
+为了便于解释基本概念，便于撰写示例程序，整个 TDengine 文档以智能电表作为典型时序数据场景。假设每个智能电表采集电流、电压、相位三个量，有多个智能电表，每个电表有位置 Location 和分组 Group ID 的静态属性. 其采集的数据类似如下的表格：
-
- Device ID
- Time Stamp
- Collected Metrics
- Tags
+
+
+ Device ID
+ Timestamp
+ Collected Metrics
+ Tags
-
-Device ID
-Time Stamp
-current
-voltage
-phase
-location
-groupId
-
-
-
-
-d1001
-1538548685000
-10.3
-219
-0.31
-California.SanFrancisco
-2
-
-
-d1002
-1538548684000
-10.2
-220
-0.23
-California.SanFrancisco
-3
-
-
-d1003
-1538548686500
-11.5
-221
-0.35
-California.LosAngeles
-3
-
-
-d1004
-1538548685500
-13.4
-223
-0.29
-California.LosAngeles
-2
-
-
-d1001
-1538548695000
-12.6
-218
-0.33
-California.SanFrancisco
-2
-
-
-d1004
-1538548696600
-11.8
-221
-0.28
-California.LosAngeles
-2
-
-
-d1002
-1538548696650
-10.3
-218
-0.25
-California.SanFrancisco
-3
-
-
-d1001
-1538548696800
-12.3
-221
-0.31
-California.SanFrancisco
-2
-
-
+
+ current
+ voltage
+ phase
+ location
+ groupid
+
+
+
+
+ d1001
+ 1538548685000
+ 10.3
+ 219
+ 0.31
+ California.SanFrancisco
+ 2
+
+
+ d1002
+ 1538548684000
+ 10.2
+ 220
+ 0.23
+ California.SanFrancisco
+ 3
+
+
+ d1003
+ 1538548686500
+ 11.5
+ 221
+ 0.35
+ California.LosAngeles
+ 3
+
+
+ d1004
+ 1538548685500
+ 13.4
+ 223
+ 0.29
+ California.LosAngeles
+ 2
+
+
+ d1001
+ 1538548695000
+ 12.6
+ 218
+ 0.33
+ California.SanFrancisco
+ 2
+
+
+ d1004
+ 1538548696600
+ 11.8
+ 221
+ 0.28
+ California.LosAngeles
+ 2
+
+
+ d1002
+ 1538548696650
+ 10.3
+ 218
+ 0.25
+ California.SanFrancisco
+ 3
+
+
+ d1001
+ 1538548696800
+ 12.3
+ 221
+ 0.31
+ California.SanFrancisco
+ 2
+
+
-
表 1：智能电表数据示例
+
表 1. 智能电表数据示例
-每一条记录都有设备 ID，时间戳，采集的物理量以及每个设备相关的静态标签。每个设备是受外界的触发，或按照设定的周期采集数据。采集的数据点是时序的，是一个数据流。
+每一条记录都有设备 ID、时间戳、采集的物理量（如上表中的 `current`、`voltage` 和 `phase`）以及每个设备相关的静态标签（`location` 和 `groupid`）。每个设备是受外界的触发，或按照设定的周期采集数据。采集的数据点是时序的，是一个数据流。
-## 采集量 (Metric)
+## 采集量（Metric）
采集量是指传感器、设备或其他类型采集点采集的物理量，比如电流、电压、温度、压力、GPS 位置等，是随时间变化的，数据类型可以是整型、浮点型、布尔型，也可是字符串。随着时间的推移，存储的采集量的数据量越来越大。智能电表示例中的电流、电压、相位就是采集量。
-## 标签 (Label/Tag)
+## 标签（Label/Tag）
-标签是指传感器、设备或其他类型采集点的静态属性，不是随时间变化的，比如设备型号、颜色、设备的所在地等，数据类型可以是任何类型。虽然是静态的，但 TDengine 容许用户修改、删除或增加标签值。与采集量不一样的是，随时间的推移，存储的标签的数据量不会有什么变化。智能电表示例中的location与groupId就是标签。
+标签是指传感器、设备或其他类型采集点的静态属性，不是随时间变化的，比如设备型号、颜色、设备的所在地等，数据类型可以是任何类型。虽然是静态的，但 TDengine 容许用户修改、删除或增加标签值。与采集量不一样的是，随时间的推移，存储的标签的数据量不会有什么变化。智能电表示例中的 `location` 与 `groupid` 就是标签。
-## 数据采集点 (Data Collection Point)
+## 数据采集点（Data Collection Point）
-数据采集点是指按照预设时间周期或受事件触发采集物理量的硬件或软件。一个数据采集点可以采集一个或多个采集量，**但这些采集量都是同一时刻采集的，具有相同的时间戳**。对于复杂的设备，往往有多个数据采集点，每个数据采集点采集的周期都可能不一样，而且完全独立，不同步。比如对于一台汽车，有数据采集点专门采集 GPS 位置，有数据采集点专门采集发动机状态，有数据采集点专门采集车内的环境，这样一台汽车就有三个数据采集点。智能电表示例中的d1001, d1002, d1003, d1004等就是数据采集点。
+数据采集点是指按照预设时间周期或受事件触发采集物理量的硬件或软件。一个数据采集点可以采集一个或多个采集量，**但这些采集量都是同一时刻采集的，具有相同的时间戳**。对于复杂的设备，往往有多个数据采集点，每个数据采集点采集的周期都可能不一样，而且完全独立，不同步。比如对于一台汽车，有数据采集点专门采集 GPS 位置，有数据采集点专门采集发动机状态，有数据采集点专门采集车内的环境，这样一台汽车就有三个数据采集点。智能电表示例中的 d1001、d1002、d1003、d1004 等就是数据采集点。
-## 表 (Table)
+## 表（Table）
因为采集量一般是结构化数据，同时为降低学习门槛，TDengine 采用传统的关系型数据库模型管理数据。用户需要先创建库，然后创建表，之后才能插入或查询数据。
@@ -129,50 +128,56 @@ description: TDengine 的数据模型和基本概念
如果采用传统的方式，将多个数据采集点的数据写入一张表，由于网络延时不可控，不同数据采集点的数据到达服务器的时序是无法保证的，写入操作是要有锁保护的，而且一个数据采集点的数据是难以保证连续存储在一起的。**采用一个数据采集点一张表的方式，能最大程度的保证单个数据采集点的插入和查询的性能是最优的。**
-TDengine 建议用数据采集点的名字（如上表中的 D1001）来做表名。每个数据采集点可能同时采集多个采集量（如上表中的 current，voltage，phase），每个采集量对应一张表中的一列，数据类型可以是整型、浮点型、字符串等。除此之外，表的第一列必须是时间戳，即数据类型为 timestamp。对采集量，TDengine 将自动按照时间戳建立索引，但对采集量本身不建任何索引。数据用列式存储方式保存。
+TDengine 建议用数据采集点的名字（如上表中的 d1001）来做表名。每个数据采集点可能同时采集多个采集量（如上表中的 `current`、`voltage` 和 `phase`），每个采集量对应一张表中的一列，数据类型可以是整型、浮点型、字符串等。除此之外，表的第一列必须是时间戳，即数据类型为 Timestamp。对采集量，TDengine 将自动按照时间戳建立索引，但对采集量本身不建任何索引。数据用列式存储方式保存。
-对于复杂的设备，比如汽车，它有多个数据采集点，那么就需要为一台汽车建立多张表。
+对于复杂的设备，比如汽车，它有多个数据采集点，那么就需要为一辆汽车建立多张表。
-
-## 超级表 (STable)
+## 超级表（STable）
由于一个数据采集点一张表，导致表的数量巨增，难以管理，而且应用经常需要做采集点之间的聚合操作，聚合的操作也变得复杂起来。为解决这个问题，TDengine 引入超级表（Super Table，简称为 STable）的概念。
-超级表是指某一特定类型的数据采集点的集合。同一类型的数据采集点，其表的结构是完全一样的，但每个表（数据采集点）的静态属性（标签）是不一样的。描述一个超级表（某一特定类型的数据采集点的集合），除需要定义采集量的表结构之外，还需要定义其标签的 schema，标签的数据类型可以是整数、浮点数、字符串，标签可以有多个，可以事后增加、删除或修改。如果整个系统有 N 个不同类型的数据采集点，就需要建立 N 个超级表。
+超级表是指某一特定类型的数据采集点的集合。同一类型的数据采集点，其表的结构是完全一样的，但每个表（数据采集点）的静态属性（标签）是不一样的。描述一个超级表（某一特定类型的数据采集点的集合），除需要定义采集量的表结构之外，还需要定义其标签的 Schema，标签的数据类型可以是整数、浮点数、字符串、JSON，标签可以有多个，可以事后增加、删除或修改。如果整个系统有 N 个不同类型的数据采集点，就需要建立 N 个超级表。
-在 TDengine 的设计里，**表用来代表一个具体的数据采集点，超级表用来代表一组相同类型的数据采集点集合**。智能电表示例中，我们可以创建一个超级表meters.
+在 TDengine 的设计里，**表用来代表一个具体的数据采集点，超级表用来代表一组相同类型的数据采集点集合**。智能电表示例中，我们可以创建一个超级表 `meters`.
-## 子表 (Subtable)
+## 子表（Subtable）
当为某个具体数据采集点创建表时，用户可以使用超级表的定义做模板，同时指定该具体采集点（表）的具体标签值来创建该表。**通过超级表创建的表称之为子表**。正常的表与子表的差异在于：
-1. 子表就是表，因此所有正常表的SQL操作都可以在子表上执行。
+1. 子表就是表，因此所有正常表的 SQL 操作都可以在子表上执行。
2. 子表在正常表的基础上有扩展，它是带有静态标签的，而且这些标签可以事后增加、删除、修改，而正常的表没有。
3. 子表一定属于一张超级表，但普通表不属于任何超级表
4. 普通表无法转为子表，子表也无法转为普通表。
超级表与与基于超级表建立的子表之间的关系表现在：
-1. 一张超级表包含有多张子表，这些子表具有相同的采集量 schema，但带有不同的标签值。
+1. 一张超级表包含有多张子表，这些子表具有相同的采集量 Schema，但带有不同的标签值。
2. 不能通过子表调整数据或标签的模式，对于超级表的数据模式修改立即对所有的子表生效。
3. 超级表只定义一个模板，自身不存储任何数据或标签信息。因此，不能向一个超级表写入数据，只能将数据写入子表中。
查询既可以在表上进行，也可以在超级表上进行。针对超级表的查询，TDengine 将把所有子表中的数据视为一个整体数据集进行处理，会先把满足标签过滤条件的表从超级表中找出来，然后再扫描这些表的时序数据，进行聚合操作，这样需要扫描的数据集会大幅减少，从而显著提高查询的性能。本质上，TDengine 通过对超级表查询的支持，实现了多个同类数据采集点的高效聚合。
-TDengine系统建议给一个数据采集点建表，需要通过超级表建表，而不是建普通表。在智能电表的示例中，我们可以通过超级表meters创建子表d1001, d1002, d1003, d1004等。
+TDengine 系统建议给一个数据采集点建表，需要通过超级表建表，而不是建普通表。在智能电表的示例中，我们可以通过超级表 meters 创建子表 d1001、d1002、d1003、d1004 等。
-为了更好地理解超级与子表的关系，可以参考下面关于智能电表数据模型的示意图。 
+为了更好地理解采集量、标签、超级与子表的关系，可以参考下面关于智能电表数据模型的示意图。
-## 库 (database)
+
+
+
+
+图 1. 智能电表数据模型示意图
+
+## 库（Database）
库是指一组表的集合。TDengine 容许一个运行实例有多个库，而且每个库可以配置不同的存储策略。不同类型的数据采集点往往具有不同的数据特征，包括数据采集频率的高低，数据保留时间的长短，副本的数目，数据块的大小，是否允许更新数据等等。为了在各种场景下 TDengine 都能最大效率的工作，TDengine 建议将不同数据特征的超级表创建在不同的库里。
一个库里，可以有一到多个超级表，但一个超级表只属于一个库。一个超级表所拥有的子表全部存在一个库里。
-## FQDN & End Point
+## FQDN & Endpoint
-FQDN (fully qualified domain name, 完全限定域名)是 Internet 上特定计算机或主机的完整域名。FQDN 由两部分组成:主机名和域名。例如，假设邮件服务器的 FQDN 可能是 mail.tdengine.com。主机名是 mail，主机位于域名 tdengine.com 中。DNS(Domain Name System)，负责将 FQDN 翻译成 IP，是互联网应用的寻址方式。对于没有 DNS 的系统，可以通过配置 hosts 文件来解决。
+FQDN（Fully Qualified Domain Name，完全限定域名）是 Internet 上特定计算机或主机的完整域名。FQDN 由两部分组成:主机名和域名。例如，假设邮件服务器的 FQDN 可能是 mail.tdengine.com。主机名是 mail，主机位于域名 tdengine.com 中。DNS（Domain Name System），负责将 FQDN 翻译成 IP，是互联网应用的寻址方式。对于没有 DNS 的系统，可以通过配置 hosts 文件来解决。
-TDengine 集群的每个节点是由 End Point 来唯一标识的，End Point 是由 FQDN 外加 Port 组成，比如 h1.tdengine.com:6030。这样当 IP 发生变化的时候，我们依然可以使用 FQDN 来动态找到节点，不需要更改集群的任何配置。而且采用 FQDN，便于内网和外网对同一个集群的统一访问。
+TDengine 集群的每个节点是由 Endpoint 来唯一标识的，Endpoint 是由 FQDN 外加 Port 组成，比如 h1.tdengine.com:6030。这样当 IP 发生变化的时候，我们依然可以使用 FQDN 来动态找到节点，不需要更改集群的任何配置。而且采用 FQDN，便于内网和外网对同一个集群的统一访问。
TDengine 不建议采用直接的 IP 地址访问集群，不利于管理。不了解 FQDN 概念，请看博文[《一篇文章说清楚 TDengine 的 FQDN》](https://www.taosdata.com/blog/2020/09/11/1824.html)。
diff --git a/docs/zh/05-get-started/01-docker.md b/docs/zh/05-get-started/01-docker.md
index e2be419517..e3345fed96 100644
--- a/docs/zh/05-get-started/01-docker.md
+++ b/docs/zh/05-get-started/01-docker.md
@@ -4,11 +4,11 @@ title: 通过 Docker 快速体验 TDengine
description: 使用 Docker 快速体验 TDengine 的高效写入和查询
---
-本节首先介绍如何通过 Docker 快速体验 TDengine，然后介绍如何在 Docker 环境下体验 TDengine 的写入和查询功能。如果你不熟悉 Docker，请使用[安装包的方式快速体验](../../get-started/package/)。如果您希望为 TDengine 贡献代码或对内部技术实现感兴趣，请参考 [TDengine GitHub 主页](https://github.com/taosdata/TDengine) 下载源码构建和安装.
+本节首先介绍如何通过 Docker 快速体验 TDengine，然后介绍如何在 Docker 环境下体验 TDengine 的写入和查询功能。如果你不熟悉 Docker，请使用[安装包的方式快速体验](../../get-started/package/)。如果您希望为 TDengine 贡献代码或对内部技术实现感兴趣，请参考 [TDengine GitHub 主页](https://github.com/taosdata/TDengine)下载源码构建和安装。
## 启动 TDengine
-如果已经安装了 docker， 只需执行下面的命令。
+如果已经安装了 Docker，只需执行下面的命令：
```shell
docker run -d -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043-6049/udp tdengine/tdengine
@@ -16,84 +16,84 @@ docker run -d -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043
注意：TDengine 3.0 服务端仅使用 6030 TCP 端口。6041 为 taosAdapter 所使用提供 REST 服务端口。6043-6049 为 taosAdapter 提供第三方应用接入所使用端口，可根据需要选择是否打开。
-确定该容器已经启动并且在正常运行
+确定该容器已经启动并且在正常运行。
```shell
docker ps
```
-进入该容器并执行 bash
+进入该容器并执行 `bash`
```shell
docker exec -it bash
```
-然后就可以执行相关的 Linux 命令操作和访问 TDengine
+然后就可以执行相关的 Linux 命令操作和访问 TDengine。
-注: Docker 工具自身的下载和使用请参考 [Docker 官网文档](https://docs.docker.com/get-docker/)。
+注：Docker 工具自身的下载和使用请参考 [Docker 官网文档](https://docs.docker.com/get-docker/)。
## 运行 TDengine CLI
-进入容器，执行 taos
+进入容器，执行 `taos`：
```
$ taos
-taos>
-
+taos>
```
-## 写入数据
+## 使用 taosBenchmark 体验写入速度
-可以使用 TDengine 的自带工具 taosBenchmark 快速体验 TDengine 的写入。
+可以使用 TDengine 的自带工具 taosBenchmark 快速体验 TDengine 的写入速度。
-进入容器，启动 taosBenchmark：
+启动 TDengine 的服务，在 Linux 或 Windows 终端执行 `taosBenchmark`（曾命名为 `taosdemo`）：
- ```bash
- $ taosBenchmark
-
- ```
+```bash
+$ taosBenchmark
+```
- 该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "d0" 到 "d9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupId，groupId 被设置为 1 到 10， location 被设置为 "San Francisco" 或者 "Los Angeles"等城市名称。
+该命令将在数据库 `test` 下面自动创建一张超级表 `meters`，该超级表下有 1 万张表，表名为 `d0` 到 `d9999`，每张表有 1 万条记录，每条记录有 `ts`、`current`、`voltage`、`phase` 四个字段，时间戳从 2017-07-14 10:40:00 000 到 2017-07-14 10:40:09 999，每张表带有标签 `location` 和 `groupId`，groupId 被设置为 1 到 10，location 被设置为 `Campbell`、`Cupertino`、`Los Angeles`、`Mountain View`、`Palo Alto`、`San Diego`、`San Francisco`、`San Jose`、`Santa Clara` 或者 `Sunnyvale`。
- 这条命令很快完成 1 亿条记录的插入。具体时间取决于硬件性能。
+这条命令很快完成 1 亿条记录的插入。具体时间取决于硬件性能，即使在一台普通的 PC 服务器往往也仅需十几秒。
- taosBenchmark 命令本身带有很多选项，配置表的数目、记录条数等等，您可以设置不同参数进行体验，请执行 `taosBenchmark --help` 详细列出。taosBenchmark 详细使用方法请参照 [taosBenchmark 参考手册](../../reference/taosbenchmark)。
+taosBenchmark 命令本身带有很多选项，配置表的数目、记录条数等等，您可以设置不同参数进行体验，请执行 `taosBenchmark --help` 详细列出。taosBenchmark 详细使用方法请参照[如何使用 taosBenchmark 对 TDengine 进行性能测试](https://www.taosdata.com/2021/10/09/3111.html)和 [taosBenchmark 参考手册](../../reference/taosbenchmark)。
-## 体验查询
+## 使用 TDengine CLI 体验查询速度
-使用上述 taosBenchmark 插入数据后，可以在 TDengine CLI 输入查询命令，体验查询速度。。
+使用上述 `taosBenchmark` 插入数据后，可以在 TDengine CLI（taos）输入查询命令，体验查询速度。
-查询超级表下记录总条数：
+查询超级表 `meters` 下的记录总条数：
```sql
-taos> select count(*) from test.meters;
+SELECT COUNT(*) FROM test.meters;
```
查询 1 亿条记录的平均值、最大值、最小值等：
```sql
-taos> select avg(current), max(voltage), min(phase) from test.meters;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters;
```
-查询 location="San Francisco" 的记录总条数：
+查询 location = "San Francisco" 的记录总条数：
```sql
-taos> select count(*) from test.meters where location="San Francisco";
+SELECT COUNT(*) FROM test.meters WHERE location = "San Francisco";
```
-查询 groupId=10 的所有记录的平均值、最大值、最小值等：
+查询 groupId = 10 的所有记录的平均值、最大值、最小值等：
```sql
-taos> select avg(current), max(voltage), min(phase) from test.meters where groupId=10;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 10;
```
-对表 d10 按 10s 进行平均值、最大值和最小值聚合统计：
+对表 `d10` 按 10 每秒进行平均值、最大值和最小值聚合统计：
```sql
-taos> select avg(current), max(voltage), min(phase) from test.d10 interval(10s);
+SELECT FIRST(ts), AVG(current), MAX(voltage), MIN(phase) FROM test.d10 INTERVAL(10s);
```
+在上面的查询中，你选择的是区间内的第一个时间戳（ts），另一种选择方式是 `\_wstart`，它将给出时间窗口的开始。关于窗口查询的更多信息，参见[特色查询](../../taos-sql/distinguished/)。
+
## 其它
-更多关于在 Docker 环境下使用 TDengine 的细节，请参考 [在 Docker 下使用 TDengine](../../reference/docker)
+更多关于在 Docker 环境下使用 TDengine 的细节，请参考 [在 Docker 下使用 TDengine](../../reference/docker)。
diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md
index 3e0fb056a5..cb2553a0bf 100644
--- a/docs/zh/05-get-started/03-package.md
+++ b/docs/zh/05-get-started/03-package.md
@@ -10,23 +10,24 @@ import PkgListV3 from "/components/PkgListV3";
您可以[用 Docker 立即体验](../../get-started/docker/) TDengine。如果您希望对 TDengine 贡献代码或对内部实现感兴趣，请参考我们的 [TDengine GitHub 主页](https://github.com/taosdata/TDengine) 下载源码构建和安装.
-TDengine 完整的软件包包括服务端（taosd）、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、应用驱动（taosc）、命令行程序 (CLI，taos) 和一些工具软件。目前 taosAdapter 仅在 Linux 系统上安装和运行，后续将支持 Windows、macOS 等系统。TDengine 除了提供多种语言的连接器之外，还通过 [taosAdapter](../../reference/taosadapter/) 提供 [RESTful 接口](../../connector/rest-api/)。
+TDengine 完整的软件包包括服务端（taosd）、应用驱动（taosc）、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、命令行程序（CLI，taos）和一些工具软件。目前 taosAdapter 仅在 Linux 系统上安装和运行，后续将支持 Windows、macOS 等系统。TDengine 除了提供多种语言的连接器之外，还通过 [taosAdapter](../../reference/taosadapter/) 提供 [RESTful 接口](../../connector/rest-api/)。
-为方便使用，标准的服务端安装包包含了 taosd、taosAdapter、taosc、taos、taosdump、taosBenchmark、TDinsight 安装脚本和示例代码；如果您只需要用到服务端程序和客户端连接的 C/C++ 语言支持，也可以仅下载 lite 版本的安装包。
+为方便使用，标准的服务端安装包包含了 taosd、taosAdapter、taosc、taos、taosdump、taosBenchmark、TDinsight 安装脚本和示例代码；如果您只需要用到服务端程序和客户端连接的 C/C++ 语言支持，也可以仅下载 Lite 版本的安装包。
-在 Linux 系统上，TDengine 开源版本提供 deb 和 rpm 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 deb 支持 Debian/Ubuntu 及衍生系统，rpm 支持 CentOS/RHEL/SUSE 及衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包，也支持通过 `apt-get` 工具从线上进行安装。需要注意的是，rpm 和 deb 包不含 taosdump 和 TDinsight 安装脚本，这些工具需要通过安装 taosTool 包获得。TDengine 也提供 Windows x64 平台的安装包。
+在 Linux 系统上，TDengine 社区版提供 Deb 和 RPM 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 Deb 支持 Debian/Ubuntu 及其衍生系统，RPM 支持 CentOS/RHEL/SUSE 及其衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包，也支持通过 `apt-get` 工具从线上进行安装。需要注意的是，RPM 和 Deb 包不含 `taosdump` 和 TDinsight 安装脚本，这些工具需要通过安装 taosTool 包获得。TDengine 也提供 Windows x64 平台的安装包。
## 安装
-1. 从列表中下载获得 deb 安装包；
-
+1. 从列表中下载获得 Deb 安装包；
+
2. 进入到安装包所在目录，执行如下的安装命令：
+> 请将 `` 替换为下载的安装包版本
+
```bash
-# 替换为下载的安装包版本
sudo dpkg -i TDengine-server--Linux-x64.deb
```
@@ -34,12 +35,13 @@ sudo dpkg -i TDengine-server--Linux-x64.deb
-1. 从列表中下载获得 rpm 安装包；
-
+1. 从列表中下载获得 RPM 安装包；
+
2. 进入到安装包所在目录，执行如下的安装命令：
+> 请将 `` 替换为下载的安装包版本
+
```bash
-# 替换为下载的安装包版本
sudo rpm -ivh TDengine-server--Linux-x64.rpm
```
@@ -48,44 +50,46 @@ sudo rpm -ivh TDengine-server--Linux-x64.rpm
1. 从列表中下载获得 tar.gz 安装包；
-
-2. 进入到安装包所在目录，先解压文件后，进入子目录，执行其中的 install.sh 安装脚本：
+
+2. 进入到安装包所在目录，使用 `tar` 解压安装包；
+3. 进入到安装包所在目录，先解压文件后，进入子目录，执行其中的 install.sh 安装脚本。
+
+> 请将 `` 替换为下载的安装包版本
```bash
-# 替换为下载的安装包版本
tar -zxvf TDengine-server--Linux-x64.tar.gz
```
-解压后进入相应路径，执行
+解压文件后，进入相应子目录，执行其中的 `install.sh` 安装脚本：
```bash
sudo ./install.sh
```
:::info
-install.sh 安装脚本在执行过程中，会通过命令行交互界面询问一些配置信息。如果希望采取无交互安装方式，那么可以用 -e no 参数来执行 install.sh 脚本。运行 `./install.sh -h` 指令可以查看所有参数的详细说明信息。
+install.sh 安装脚本在执行过程中，会通过命令行交互界面询问一些配置信息。如果希望采取无交互安装方式，那么可以运行 `./install.sh -e no`。运行 `./install.sh -h` 指令可以查看所有参数的详细说明信息。
:::
-可以使用 apt-get 工具从官方仓库安装。
+可以使用 `apt-get` 工具从官方仓库安装。
-**安装包仓库**
+**配置包仓库**
```bash
wget -qO - http://repos.taosdata.com/tdengine.key | sudo apt-key add -
echo "deb [arch=amd64] http://repos.taosdata.com/tdengine-stable stable main" | sudo tee /etc/apt/sources.list.d/tdengine-stable.list
```
-如果安装 Beta 版需要安装包仓库
+如果安装 Beta 版需要安装包仓库：
```bash
wget -qO - http://repos.taosdata.com/tdengine.key | sudo apt-key add -
echo "deb [arch=amd64] http://repos.taosdata.com/tdengine-beta beta main" | sudo tee /etc/apt/sources.list.d/tdengine-beta.list
```
-**使用 apt-get 命令安装**
+**使用 `apt-get` 命令安装**
```bash
sudo apt-get update
@@ -94,26 +98,26 @@ sudo apt-get install tdengine
```
:::tip
-apt-get 方式只适用于 Debian 或 Ubuntu 系统
+apt-get 方式只适用于 Debian 或 Ubuntu 系统。
::::
-
+
-注意：目前 TDengine 在 Windows 平台上只支持 Windows server 2016/2019 和 Windows 10/11 系统版本。
+注意：目前 TDengine 在 Windows 平台上只支持 Windows Server 2016/2019 和 Windows 10/11。
1. 从列表中下载获得 exe 安装程序；
-
+
2. 运行可执行程序来安装 TDengine。
:::info
-下载其他组件、最新 Beta 版及之前版本的安装包，请点击[发布历史页面](../../releases/tdengine)
+下载其他组件、最新 Beta 版及之前版本的安装包，请点击[发布历史页面](../../releases/tdengine)。
:::
:::note
-当安装第一个节点时，出现 Enter FQDN：提示的时候，不需要输入任何内容。只有当安装第二个或以后更多的节点时，才需要输入已有集群中任何一个可用节点的 FQDN，支持该新节点加入集群。当然也可以不输入，而是在新节点启动前，配置到新节点的配置文件中。
+当安装第一个节点时，出现 `Enter FQDN:` 提示的时候，不需要输入任何内容。只有当安装第二个或以后更多的节点时，才需要输入已有集群中任何一个可用节点的 FQDN，支持该新节点加入集群。当然也可以不输入，而是在新节点启动前，配置到新节点的配置文件中。
:::
@@ -148,7 +152,7 @@ Active: inactive (dead)
如果 TDengine 服务正常工作，那么您可以通过 TDengine 的命令行程序 `taos` 来访问并体验 TDengine。
-systemctl 命令汇总：
+如下 `systemctl` 命令可以帮助你管理 TDengine 服务：
- 启动服务进程：`systemctl start taosd`
@@ -160,7 +164,7 @@ systemctl 命令汇总：
:::info
-- systemctl 命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 sudo 。
+- `systemctl` 命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 `sudo`。
- `systemctl stop taosd` 指令在执行后并不会马上停止 TDengine 服务，而是会等待系统中必要的落盘工作正常完成。在数据量很大的情况下，这可能会消耗较长时间。
- 如果系统中不支持 `systemd`，也可以用手动运行 `/usr/local/taos/bin/taosd` 方式启动 TDengine 服务。
@@ -170,87 +174,93 @@ systemctl 命令汇总：
-安装后，在 C:\TDengine 目录下，运行 taosd.exe 来启动 TDengine 服务进程。
+安装后，在 `C:\TDengine` 目录下，运行 `taosd.exe` 来启动 TDengine 服务进程。
-## TDengine 命令行 (CLI)
+## TDengine 命令行（CLI）
-为便于检查 TDengine 的状态，执行数据库 (Database) 的各种即席(Ad Hoc)查询，TDengine 提供一命令行应用程序(以下简称为 TDengine CLI) taos。要进入 TDengine 命令行，您只要在安装有 TDengine 的 Linux 终端执行 `taos` 即可，也可以在安装有 TDengine 的 Windows 终端的 C:\TDengine 目录下，运行 taos.exe 来启动 TDengine 命令行。
+为便于检查 TDengine 的状态，执行数据库（Database）的各种即席（Ad Hoc）查询，TDengine 提供一命令行应用程序（以下简称为 TDengine CLI）taos。要进入 TDengine 命令行，您只要在安装有 TDengine 的 Linux 终端执行 `taos` 即可，也可以在安装有 TDengine 的 Windows 终端的 C:\TDengine 目录下，运行 taos.exe 来启动 TDengine 命令行。
```bash
taos
```
-如果连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考 [FAQ](/train-faq/faq) 来解决终端连接服务端失败的问题）。 TDengine CLI 的提示符号如下：
+如果连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考 [FAQ](/train-faq/faq) 来解决终端连接服务端失败的问题）。TDengine CLI 的提示符号如下：
```cmd
taos>
```
-在 TDengine CLI 中，用户可以通过 SQL 命令来创建/删除数据库、表等，并进行数据库(database)插入查询操作。在终端中运行的 SQL 语句需要以分号结束来运行。示例：
+在 TDengine CLI 中，用户可以通过 SQL 命令来创建/删除数据库、表等，并进行数据库（Database）插入查询操作。在终端中运行的 SQL 语句需要以分号（;）结束来运行。示例：
```sql
-create database demo;
-use demo;
-create table t (ts timestamp, speed int);
-insert into t values ('2019-07-15 00:00:00', 10);
-insert into t values ('2019-07-15 01:00:00', 20);
-select * from t;
+CREATE DATABASE demo;
+USE demo;
+CREATE TABLE t (ts TIMESTAMP, speed INT);
+INSERT INTO t VALUES ('2019-07-15 00:00:00', 10);
+INSERT INTO t VALUES ('2019-07-15 01:00:00', 20);
+SELECT * FROM t;
+
ts | speed |
========================================
2019-07-15 00:00:00.000 | 10 |
2019-07-15 01:00:00.000 | 20 |
+
Query OK, 2 row(s) in set (0.003128s)
```
-除执行 SQL 语句外，系统管理员还可以从 TDengine CLI 进行检查系统运行状态、添加删除用户账号等操作。TDengine CLI 连同应用驱动也可以独立安装在 Linux 或 Windows 机器上运行，更多细节请参考 [这里](../../reference/taos-shell/)
+除执行 SQL 语句外，系统管理员还可以从 TDengine CLI 进行检查系统运行状态、添加删除用户账号等操作。TDengine CLI 连同应用驱动也可以独立安装在 Linux 或 Windows 机器上运行，更多细节请参考 [TDengine 命令行](../../reference/taos-shell/)。
## 使用 taosBenchmark 体验写入速度
-启动 TDengine 的服务，在 Linux 或 windows 终端执行 `taosBenchmark` （曾命名为 `taosdemo`）：
+可以使用 TDengine 的自带工具 taosBenchmark 快速体验 TDengine 的写入速度。
+
+启动 TDengine 的服务，在 Linux 或 Windows 终端执行 `taosBenchmark`（曾命名为 `taosdemo`）：
```bash
-taosBenchmark
+$ taosBenchmark
```
-该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "d0" 到 "d9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupId，groupId 被设置为 1 到 10， location 被设置为 "California.SanFrancisco" 或者 "California.LosAngeles"。
+该命令将在数据库 `test` 下面自动创建一张超级表 `meters`，该超级表下有 1 万张表，表名为 `d0` 到 `d9999`，每张表有 1 万条记录，每条记录有 `ts`、`current`、`voltage`、`phase` 四个字段，时间戳从 2017-07-14 10:40:00 000 到 2017-07-14 10:40:09 999，每张表带有标签 `location` 和 `groupId`，groupId 被设置为 1 到 10，location 被设置为 `Campbell`、`Cupertino`、`Los Angeles`、`Mountain View`、`Palo Alto`、`San Diego`、`San Francisco`、`San Jose`、`Santa Clara` 或者 `Sunnyvale`。
这条命令很快完成 1 亿条记录的插入。具体时间取决于硬件性能，即使在一台普通的 PC 服务器往往也仅需十几秒。
-taosBenchmark 命令本身带有很多选项，配置表的数目、记录条数等等，您可以设置不同参数进行体验，请执行 `taosBenchmark --help` 详细列出。taosBenchmark 详细使用方法请参照 [如何使用 taosBenchmark 对 TDengine 进行性能测试](https://www.taosdata.com/2021/10/09/3111.html)。
+taosBenchmark 命令本身带有很多选项，配置表的数目、记录条数等等，您可以设置不同参数进行体验，请执行 `taosBenchmark --help` 详细列出。taosBenchmark 详细使用方法请参照[如何使用 taosBenchmark 对 TDengine 进行性能测试](https://www.taosdata.com/2021/10/09/3111.html)和 [taosBenchmark 参考手册](../../reference/taosbenchmark)。
## 使用 TDengine CLI 体验查询速度
-使用上述 taosBenchmark 插入数据后，可以在 TDengine CLI 输入查询命令，体验查询速度。
+使用上述 `taosBenchmark` 插入数据后，可以在 TDengine CLI（taos）输入查询命令，体验查询速度。
-查询超级表下记录总条数：
+查询超级表 `meters` 下的记录总条数：
```sql
-taos> select count(*) from test.meters;
+SELECT COUNT(*) FROM test.meters;
```
查询 1 亿条记录的平均值、最大值、最小值等：
```sql
-taos> select avg(current), max(voltage), min(phase) from test.meters;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters;
```
-查询 location="California.SanFrancisco" 的记录总条数：
+查询 location = "San Francisco" 的记录总条数：
```sql
-taos> select count(*) from test.meters where location="California.SanFrancisco";
+SELECT COUNT(*) FROM test.meters WHERE location = "San Francisco";
```
-查询 groupId=10 的所有记录的平均值、最大值、最小值等：
+查询 groupId = 10 的所有记录的平均值、最大值、最小值等：
```sql
-taos> select avg(current), max(voltage), min(phase) from test.meters where groupId=10;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 10;
```
-对表 d10 按 10s 进行平均值、最大值和最小值聚合统计：
+对表 `d10` 按 10 每秒进行平均值、最大值和最小值聚合统计：
```sql
-taos> select avg(current), max(voltage), min(phase) from test.d10 interval(10s);
+SELECT FIRST(ts), AVG(current), MAX(voltage), MIN(phase) FROM test.d10 INTERVAL(10s);
```
+
+在上面的查询中，你选择的是区间内的第一个时间戳（ts），另一种选择方式是 `\_wstart`，它将给出时间窗口的开始。关于窗口查询的更多信息，参见[特色查询](../../taos-sql/distinguished/)。
diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md
index ef1cd4797a..3239eae49b 100644
--- a/docs/zh/07-develop/09-udf.md
+++ b/docs/zh/07-develop/09-udf.md
@@ -116,7 +116,7 @@ aggfn为函数名的占位符，需要修改为自己的函数名，如l2norm。
参数的具体含义是：
- inputDataBlock: 输入的数据块
- - resultColumn: 输出列。输出列
+ - resultColumn: 输出列
### 聚合接口函数
diff --git a/docs/zh/08-connector/index.md b/docs/zh/08-connector/index.md
index 17de8e926c..f54470f742 100644
--- a/docs/zh/08-connector/index.md
+++ b/docs/zh/08-connector/index.md
@@ -41,14 +41,14 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器
### 使用原生接口（taosc）
-| **功能特性** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** |
-| -------------- | -------- | ---------- | ------ | ------ | ----------- | -------- |
-| **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
-| **普通查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
-| **参数绑定** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
-| ** TMQ ** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
-| **Schemaless** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
-| **DataFrame** | 不支持 | 支持 | 不支持 | 不支持 | 不支持 | 不支持 |
+| **功能特性** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** |
+| ------------------- | -------- | ---------- | ------ | ------ | ----------- | -------- |
+| **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
+| **普通查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
+| **参数绑定** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
+| **数据订阅（TMQ）** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
+| **Schemaless** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
+| **DataFrame** | 不支持 | 支持 | 不支持 | 不支持 | 不支持 | 不支持 |
:::info
由于不同编程语言数据库框架规范不同，并不意味着所有 C/C++ 接口都需要对应封装支持。
@@ -56,16 +56,15 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器
### 使用 http (REST 或 WebSocket) 接口
-| **功能特性** | **Java** | **Python** | **Go** | **C#（暂不支持）** | **Node.js** | **Rust** |
-| ------------------------------ | -------- | ---------- | -------- | ------------------ | ----------- | -------- |
-| **连接管理** | 支持 | 支持 | 支持 | N/A | 支持 | 支持 |
-| **普通查询** | 支持 | 支持 | 支持 | N/A | 支持 | 支持 |
-| **连续查询** | 支持 | 支持 | 支持 | N/A | 支持 | 支持 |
-| **参数绑定** | 不支持 | 暂不支持 | 暂不支持 | N/A | 不支持 | 支持 |
-| ** TMQ ** | 不支持 | 暂不支持 | 暂不支持 | N/A | 不支持 | 支持 |
-| **Schemaless** | 暂不支持 | 暂不支持 | 暂不支持 | N/A | 不支持 | 暂不支持 |
-| **批量拉取（基于 WebSocket）** | 支持 | 支持 | 暂不支持 | N/A | 不支持 | 支持 |
-| **DataFrame** | 不支持 | 支持 | 不支持 | N/A | 不支持 | 不支持 |
+| **功能特性** | **Java** | **Python** | **Go** | **C# ** | **Node.js** | **Rust** |
+| ------------------------------ | -------- | ---------- | -------- | -------- | ----------- | -------- |
+| **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
+| **普通查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
+| **参数绑定** | 暂不支持 | 暂不支持 | 暂不支持 | 支持 | 暂不支持 | 支持 |
+| **数据订阅（TMQ）** | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 支持 |
+| **Schemaless** | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 |
+| **批量拉取（基于 WebSocket）** | 支持 | 支持 | 暂不支持 | 支持 | 暂不支持 | 支持 |
+| **DataFrame** | 不支持 | 支持 | 不支持 | 不支持 | 不支持 | 不支持 |
:::warning
diff --git a/docs/zh/12-taos-sql/20-keywords.md b/docs/zh/12-taos-sql/20-keywords.md
index 09fea45b7a..ee36dc44f0 100644
--- a/docs/zh/12-taos-sql/20-keywords.md
+++ b/docs/zh/12-taos-sql/20-keywords.md
@@ -256,6 +256,7 @@ description: TDengine 保留关键字的详细列表
### V
+- VALUE
- VALUES
- VARIABLE
- VARIABLES
diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md
index c8a99cb40f..9e8757cc4e 100644
--- a/docs/zh/28-releases/02-tools.md
+++ b/docs/zh/28-releases/02-tools.md
@@ -9,7 +9,3 @@ import Release from "/components/ReleaseV3";
## 2.1.3
- SArray* aSstBlk; // SArray
- SBlockIdx* pBlockIdx;
- SSttBlk* pSstBlk;
-
- int32_t iBlockIdx;
- int32_t iBlockL;
- SMapData mBlock; // SMapData
- int32_t iBlock;
- SBlockData oBlockData;
- SBlockData nBlockData;
+ SFDataIter* pIter;
+ SRBTree rbt;
+ SFDataIter aFDataIter[TSDB_MAX_STT_TRIGGER + 1];
+ SBlockData bData;
+ SSkmInfo skmTable;
// for del file
int8_t delDone;
SDelFReader* pDelFReader;
SArray* aDelIdx; // SArray
int32_t iDelIdx;
SArray* aDelData; // SArray
+ uint8_t* aBuf[5];
};
+extern int32_t tRowInfoCmprFn(const void* p1, const void* p2);
+extern int32_t tsdbReadDataBlockEx(SDataFReader* pReader, SDataBlk* pDataBlk, SBlockData* pBlockData);
+extern int32_t tsdbUpdateTableSchema(SMeta* pMeta, int64_t suid, int64_t uid, SSkmInfo* pSkmInfo);
+
+static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) {
+ int32_t code = 0;
+
+ SDFileSet dFileSet = {.fid = pReader->fid};
+ SDFileSet* pSet = taosArraySearch(pReader->fs.aDFileSet, &dFileSet, tDFileSetCmprFn, TD_GT);
+ if (pSet == NULL) return code;
+
+ pReader->fid = pSet->fid;
+ code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, pSet);
+ if (code) goto _err;
+
+ pReader->pIter = NULL;
+ tRBTreeCreate(&pReader->rbt, tRowInfoCmprFn);
+
+ // .data file
+ SFDataIter* pIter = &pReader->aFDataIter[0];
+ pIter->type = SNAP_DATA_FILE_ITER;
+
+ code = tsdbReadBlockIdx(pReader->pDataFReader, pIter->aBlockIdx);
+ if (code) goto _err;
+
+ for (pIter->iBlockIdx = 0; pIter->iBlockIdx < taosArrayGetSize(pIter->aBlockIdx); pIter->iBlockIdx++) {
+ pIter->pBlockIdx = (SBlockIdx*)taosArrayGet(pIter->aBlockIdx, pIter->iBlockIdx);
+
+ code = tsdbReadDataBlk(pReader->pDataFReader, pIter->pBlockIdx, &pIter->mBlock);
+ if (code) goto _err;
+
+ for (pIter->iBlock = 0; pIter->iBlock < pIter->mBlock.nItem; pIter->iBlock++) {
+ SDataBlk dataBlk;
+ tMapDataGetItemByIdx(&pIter->mBlock, pIter->iBlock, &dataBlk, tGetDataBlk);
+
+ if (dataBlk.minVer > pReader->ever || dataBlk.maxVer < pReader->sver) continue;
+
+ code = tsdbReadDataBlockEx(pReader->pDataFReader, &dataBlk, &pIter->bData);
+ if (code) goto _err;
+
+ ASSERT(pIter->pBlockIdx->suid == pIter->bData.suid);
+ ASSERT(pIter->pBlockIdx->uid == pIter->bData.uid);
+
+ for (pIter->iRow = 0; pIter->iRow < pIter->bData.nRow; pIter->iRow++) {
+ int64_t rowVer = pIter->bData.aVersion[pIter->iRow];
+
+ if (rowVer >= pReader->sver && rowVer <= pReader->ever) {
+ pIter->rInfo.suid = pIter->pBlockIdx->suid;
+ pIter->rInfo.uid = pIter->pBlockIdx->uid;
+ pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow);
+ goto _add_iter_and_break;
+ }
+ }
+ }
+
+ continue;
+
+ _add_iter_and_break:
+ tRBTreePut(&pReader->rbt, (SRBTreeNode*)pIter);
+ break;
+ }
+
+ // .stt file
+ pIter = &pReader->aFDataIter[1];
+ for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+ pIter->type = SNAP_STT_FILE_ITER;
+ pIter->iStt = iStt;
+
+ code = tsdbReadSttBlk(pReader->pDataFReader, iStt, pIter->aSttBlk);
+ if (code) goto _err;
+
+ for (pIter->iSttBlk = 0; pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk); pIter->iSttBlk++) {
+ SSttBlk* pSttBlk = (SSttBlk*)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk);
+
+ if (pSttBlk->minVer > pReader->ever) continue;
+ if (pSttBlk->maxVer < pReader->sver) continue;
+
+ code = tsdbReadSttBlock(pReader->pDataFReader, iStt, pSttBlk, &pIter->bData);
+ if (code) goto _err;
+
+ for (pIter->iRow = 0; pIter->iRow < pIter->bData.nRow; pIter->iRow++) {
+ int64_t rowVer = pIter->bData.aVersion[pIter->iRow];
+
+ if (rowVer >= pReader->sver && rowVer <= pReader->ever) {
+ pIter->rInfo.suid = pIter->bData.suid;
+ pIter->rInfo.uid = pIter->bData.uid;
+ pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow);
+ goto _add_iter;
+ }
+ }
+ }
+
+ continue;
+
+ _add_iter:
+ tRBTreePut(&pReader->rbt, (SRBTreeNode*)pIter);
+ pIter++;
+ }
+
+ tsdbInfo("vgId:%d, vnode snapshot tsdb open data file to read for %s, fid:%d", TD_VID(pReader->pTsdb->pVnode),
+ pReader->pTsdb->path, pReader->fid);
+ return code;
+
+_err:
+ tsdbError("vgId:%d vnode snapshot tsdb snap read open file failed since %s", TD_VID(pReader->pTsdb->pVnode),
+ tstrerror(code));
+ return code;
+}
+
+static SRowInfo* tsdbSnapGetRow(STsdbSnapReader* pReader) { return pReader->pIter ? &pReader->pIter->rInfo : NULL; }
+
+static int32_t tsdbSnapNextRow(STsdbSnapReader* pReader) {
+ int32_t code = 0;
+
+ if (pReader->pIter) {
+ SFDataIter* pIter = pReader->pIter;
+
+ while (true) {
+ _find_row:
+ for (pIter->iRow++; pIter->iRow < pIter->bData.nRow; pIter->iRow++) {
+ int64_t rowVer = pIter->bData.aVersion[pIter->iRow];
+
+ if (rowVer >= pReader->sver && rowVer <= pReader->ever) {
+ pIter->rInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow];
+ pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow);
+ goto _out;
+ }
+ }
+
+ if (pIter->type == SNAP_DATA_FILE_ITER) {
+ while (true) {
+ for (pIter->iBlock++; pIter->iBlock < pIter->mBlock.nItem; pIter->iBlock++) {
+ SDataBlk dataBlk;
+ tMapDataGetItemByIdx(&pIter->mBlock, pIter->iBlock, &dataBlk, tGetDataBlk);
+
+ if (dataBlk.minVer > pReader->ever || dataBlk.maxVer < pReader->sver) continue;
+
+ code = tsdbReadDataBlockEx(pReader->pDataFReader, &dataBlk, &pIter->bData);
+ if (code) goto _err;
+
+ pIter->iRow = -1;
+ goto _find_row;
+ }
+
+ pIter->iBlockIdx++;
+ if (pIter->iBlockIdx >= taosArrayGetSize(pIter->aBlockIdx)) break;
+
+ pIter->pBlockIdx = (SBlockIdx*)taosArrayGet(pIter->aBlockIdx, pIter->iBlockIdx);
+ code = tsdbReadDataBlk(pReader->pDataFReader, pIter->pBlockIdx, &pIter->mBlock);
+ if (code) goto _err;
+ pIter->iBlock = -1;
+ }
+
+ pReader->pIter = NULL;
+ } else if (pIter->type == SNAP_STT_FILE_ITER) {
+ for (pIter->iSttBlk++; pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk); pIter->iSttBlk++) {
+ SSttBlk* pSttBlk = (SSttBlk*)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk);
+
+ if (pSttBlk->minVer > pReader->ever || pSttBlk->maxVer < pReader->sver) continue;
+
+ code = tsdbReadSttBlock(pReader->pDataFReader, pIter->iStt, pSttBlk, &pIter->bData);
+ if (code) goto _err;
+
+ pIter->iRow = -1;
+ goto _find_row;
+ }
+
+ pReader->pIter = NULL;
+ } else {
+ ASSERT(0);
+ }
+ }
+
+ _out:
+ pIter = (SFDataIter*)tRBTreeMin(&pReader->rbt);
+ if (pReader->pIter && pIter) {
+ int32_t c = tRowInfoCmprFn(&pReader->pIter->rInfo, &pIter->rInfo);
+ if (c > 0) {
+ tRBTreePut(&pReader->rbt, (SRBTreeNode*)pReader->pIter);
+ pReader->pIter = NULL;
+ } else {
+ ASSERT(c);
+ }
+ }
+ }
+
+ if (pReader->pIter == NULL) {
+ pReader->pIter = (SFDataIter*)tRBTreeMin(&pReader->rbt);
+ if (pReader->pIter) {
+ tRBTreeDrop(&pReader->rbt, (SRBTreeNode*)pReader->pIter);
+ }
+ }
+
+ return code;
+
+_err:
+ return code;
+}
+
+static int32_t tsdbSnapCmprData(STsdbSnapReader* pReader, uint8_t** ppData) {
+ int32_t code = 0;
+
+ ASSERT(pReader->bData.nRow);
+
+ int32_t aBufN[5] = {0};
+ code = tCmprBlockData(&pReader->bData, TWO_STAGE_COMP, NULL, NULL, pReader->aBuf, aBufN);
+ if (code) goto _exit;
+
+ int32_t size = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3];
+ *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size);
+ if (*ppData == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _exit;
+ }
+
+ SSnapDataHdr* pHdr = (SSnapDataHdr*)*ppData;
+ pHdr->type = SNAP_DATA_TSDB;
+ pHdr->size = size;
+
+ memcpy(pHdr->data, pReader->aBuf[3], aBufN[3]);
+ memcpy(pHdr->data + aBufN[3], pReader->aBuf[2], aBufN[2]);
+ if (aBufN[1]) {
+ memcpy(pHdr->data + aBufN[3] + aBufN[2], pReader->aBuf[1], aBufN[1]);
+ }
+ if (aBufN[0]) {
+ memcpy(pHdr->data + aBufN[3] + aBufN[2] + aBufN[1], pReader->aBuf[0], aBufN[0]);
+ }
+
+_exit:
+ return code;
+}
+
static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
int32_t code = 0;
STsdb* pTsdb = pReader->pTsdb;
while (true) {
if (pReader->pDataFReader == NULL) {
- // next
- SDFileSet dFileSet = {.fid = pReader->fid};
- SDFileSet* pSet = taosArraySearch(pReader->fs.aDFileSet, &dFileSet, tDFileSetCmprFn, TD_GT);
- if (pSet == NULL) goto _exit;
- pReader->fid = pSet->fid;
-
- // load
- code = tsdbDataFReaderOpen(&pReader->pDataFReader, pTsdb, pSet);
+ code = tsdbSnapReadOpenFile(pReader);
if (code) goto _err;
-
- code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx);
- if (code) goto _err;
-
- code = tsdbReadSttBlk(pReader->pDataFReader, 0, pReader->aSstBlk);
- if (code) goto _err;
-
- // init
- pReader->iBlockIdx = 0;
- if (pReader->iBlockIdx < taosArrayGetSize(pReader->aBlockIdx)) {
- pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx);
-
- code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock);
- if (code) goto _err;
-
- pReader->iBlock = 0;
- } else {
- pReader->pBlockIdx = NULL;
- }
-
- pReader->iBlockL = 0;
- while (true) {
- if (pReader->iBlockL >= taosArrayGetSize(pReader->aSstBlk)) {
- pReader->pSstBlk = NULL;
- break;
- }
-
- pReader->pSstBlk = (SSttBlk*)taosArrayGet(pReader->aSstBlk, pReader->iBlockL);
- if (pReader->pSstBlk->minVer <= pReader->ever && pReader->pSstBlk->maxVer >= pReader->sver) {
- // TODO
- break;
- }
-
- pReader->iBlockL++;
- }
-
- tsdbInfo("vgId:%d, vnode snapshot tsdb open data file to read for %s, fid:%d", TD_VID(pTsdb->pVnode), pTsdb->path,
- pReader->fid);
}
- while (true) {
- if (pReader->pBlockIdx && pReader->pSstBlk) {
- TABLEID id = {.suid = pReader->pSstBlk->suid, .uid = pReader->pSstBlk->minUid};
+ if (pReader->pDataFReader == NULL) break;
- ASSERT(0);
+ SRowInfo* pRowInfo = tsdbSnapGetRow(pReader);
+ if (pRowInfo == NULL) {
+ tsdbDataFReaderClose(&pReader->pDataFReader);
+ continue;
+ }
- // if (tTABLEIDCmprFn(pReader->pBlockIdx, &minId) < 0) {
- // // TODO
- // } else if (tTABLEIDCmprFn(pReader->pBlockIdx, &maxId) < 0) {
- // // TODO
- // } else {
- // // TODO
- // }
- } else if (pReader->pBlockIdx) {
- while (pReader->iBlock < pReader->mBlock.nItem) {
- SDataBlk block;
- tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, &block, tGetDataBlk);
+ TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid};
+ SBlockData* pBlockData = &pReader->bData;
- if (block.minVer <= pReader->ever && block.maxVer >= pReader->sver) {
- // load data (todo)
- }
+ code = tsdbUpdateTableSchema(pTsdb->pVnode->pMeta, id.suid, id.uid, &pReader->skmTable);
+ if (code) goto _err;
- // next
- pReader->iBlock++;
- if (*ppData) break;
- }
+ code = tBlockDataInit(pBlockData, id.suid, id.uid, pReader->skmTable.pTSchema);
+ if (code) goto _err;
- if (pReader->iBlock >= pReader->mBlock.nItem) {
- pReader->iBlockIdx++;
- if (pReader->iBlockIdx < taosArrayGetSize(pReader->aBlockIdx)) {
- pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx);
+ while (pRowInfo->suid == id.suid && pRowInfo->uid == id.uid) {
+ code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL, pRowInfo->uid);
+ if (code) goto _err;
- code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock);
- if (code) goto _err;
+ code = tsdbSnapNextRow(pReader);
+ if (code) goto _err;
- pReader->iBlock = 0;
- } else {
- pReader->pBlockIdx = NULL;
- }
- }
-
- if (*ppData) goto _exit;
- } else if (pReader->pSstBlk) {
- while (pReader->pSstBlk) {
- if (pReader->pSstBlk->minVer <= pReader->ever && pReader->pSstBlk->maxVer >= pReader->sver) {
- // load data (todo)
- }
-
- // next
- pReader->iBlockL++;
- if (pReader->iBlockL < taosArrayGetSize(pReader->aSstBlk)) {
- pReader->pSstBlk = (SSttBlk*)taosArrayGetSize(pReader->aSstBlk);
- } else {
- pReader->pSstBlk = NULL;
- }
-
- if (*ppData) goto _exit;
- }
- } else {
+ pRowInfo = tsdbSnapGetRow(pReader);
+ if (pRowInfo == NULL) {
tsdbDataFReaderClose(&pReader->pDataFReader);
break;
}
+
+ if (pBlockData->nRow >= 4096) break;
}
+
+ code = tsdbSnapCmprData(pReader, ppData);
+ if (code) goto _err;
+
+ break;
}
-_exit:
return code;
_err:
@@ -216,7 +394,6 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
size += tPutDelData(NULL, pDelData);
}
}
-
if (size == 0) continue;
// org data
@@ -292,23 +469,33 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type
goto _err;
}
+ // data
pReader->fid = INT32_MIN;
- pReader->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
- if (pReader->aBlockIdx == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ for (int32_t iIter = 0; iIter < sizeof(pReader->aFDataIter) / sizeof(pReader->aFDataIter[0]); iIter++) {
+ SFDataIter* pIter = &pReader->aFDataIter[iIter];
+
+ if (iIter == 0) {
+ pIter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
+ if (pIter->aBlockIdx == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
+ } else {
+ pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
+ if (pIter->aSttBlk == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
+ }
+
+ code = tBlockDataCreate(&pIter->bData);
+ if (code) goto _err;
}
- pReader->aSstBlk = taosArrayInit(0, sizeof(SSttBlk));
- if (pReader->aSstBlk == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
- pReader->mBlock = tMapDataInit();
- code = tBlockDataCreate(&pReader->oBlockData);
- if (code) goto _err;
- code = tBlockDataCreate(&pReader->nBlockData);
+
+ code = tBlockDataCreate(&pReader->bData);
if (code) goto _err;
+ // del
pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx));
if (pReader->aDelIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
@@ -335,18 +522,26 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
int32_t code = 0;
STsdbSnapReader* pReader = *ppReader;
- if (pReader->pDataFReader) {
- tsdbDataFReaderClose(&pReader->pDataFReader);
- }
- taosArrayDestroy(pReader->aSstBlk);
- taosArrayDestroy(pReader->aBlockIdx);
- tMapDataClear(&pReader->mBlock);
- tBlockDataDestroy(&pReader->oBlockData, 1);
- tBlockDataDestroy(&pReader->nBlockData, 1);
+ // data
+ if (pReader->pDataFReader) tsdbDataFReaderClose(&pReader->pDataFReader);
+ for (int32_t iIter = 0; iIter < sizeof(pReader->aFDataIter) / sizeof(pReader->aFDataIter[0]); iIter++) {
+ SFDataIter* pIter = &pReader->aFDataIter[iIter];
- if (pReader->pDelFReader) {
- tsdbDelFReaderClose(&pReader->pDelFReader);
+ if (iIter == 0) {
+ taosArrayDestroy(pIter->aBlockIdx);
+ tMapDataClear(&pIter->mBlock);
+ } else {
+ taosArrayDestroy(pIter->aSttBlk);
+ }
+
+ tBlockDataDestroy(&pIter->bData, 1);
}
+
+ tBlockDataDestroy(&pReader->bData, 1);
+ tTSchemaDestroy(pReader->skmTable.pTSchema);
+
+ // del
+ if (pReader->pDelFReader) tsdbDelFReaderClose(&pReader->pDelFReader);
taosArrayDestroy(pReader->aDelIdx);
taosArrayDestroy(pReader->aDelData);
@@ -354,6 +549,10 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
tsdbInfo("vgId:%d, vnode snapshot tsdb reader closed for %s", TD_VID(pReader->pTsdb->pVnode), pReader->pTsdb->path);
+ for (int32_t iBuf = 0; iBuf < sizeof(pReader->aBuf) / sizeof(pReader->aBuf[0]); iBuf++) {
+ tFree(pReader->aBuf[iBuf]);
+ }
+
taosMemoryFree(pReader);
*ppReader = NULL;
return code;
@@ -410,40 +609,37 @@ struct STsdbSnapWriter {
STsdbFS fs;
// config
- int32_t minutes;
- int8_t precision;
- int32_t minRow;
- int32_t maxRow;
- int8_t cmprAlg;
- int64_t commitID;
-
+ int32_t minutes;
+ int8_t precision;
+ int32_t minRow;
+ int32_t maxRow;
+ int8_t cmprAlg;
+ int64_t commitID;
uint8_t* aBuf[5];
+
// for data file
SBlockData bData;
-
- int32_t fid;
- SDataFReader* pDataFReader;
- SArray* aBlockIdx; // SArray
- int32_t iBlockIdx;
- SBlockIdx* pBlockIdx;
- SMapData mBlock; // SMapData
- int32_t iBlock;
- SBlockData* pBlockData;
- int32_t iRow;
- SBlockData bDataR;
- SArray* aSstBlk; // SArray
- int32_t iBlockL;
- SBlockData lDataR;
-
- SDataFWriter* pDataFWriter;
- SBlockIdx* pBlockIdxW; // NULL when no committing table
- SDataBlk blockW;
- SBlockData bDataW;
- SBlockIdx blockIdxW;
-
- SMapData mBlockW; // SMapData
- SArray* aBlockIdxW; // SArray
- SArray* aBlockLW; // SArray
+ int32_t fid;
+ TABLEID id;
+ SSkmInfo skmTable;
+ struct {
+ SDataFReader* pReader;
+ SArray* aBlockIdx;
+ int32_t iBlockIdx;
+ SBlockIdx* pBlockIdx;
+ SMapData mDataBlk;
+ int32_t iDataBlk;
+ SBlockData bData;
+ int32_t iRow;
+ } dReader;
+ struct {
+ SDataFWriter* pWriter;
+ SArray* aBlockIdx;
+ SMapData mDataBlk;
+ SArray* aSttBlk;
+ SBlockData bData;
+ SBlockData sData;
+ } dWriter;
// for del file
SDelFReader* pDelFReader;
@@ -454,520 +650,447 @@ struct STsdbSnapWriter {
SArray* aDelIdxW;
};
+// SNAP_DATA_TSDB
+extern int32_t tsdbWriteDataBlock(SDataFWriter* pWriter, SBlockData* pBlockData, SMapData* mDataBlk, int8_t cmprAlg);
+extern int32_t tsdbWriteSttBlock(SDataFWriter* pWriter, SBlockData* pBlockData, SArray* aSttBlk, int8_t cmprAlg);
+
+static int32_t tsdbSnapNextTableData(STsdbSnapWriter* pWriter) {
+ int32_t code = 0;
+
+ ASSERT(pWriter->dReader.iRow >= pWriter->dReader.bData.nRow);
+
+ if (pWriter->dReader.iBlockIdx < taosArrayGetSize(pWriter->dReader.aBlockIdx)) {
+ pWriter->dReader.pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->dReader.aBlockIdx, pWriter->dReader.iBlockIdx);
+
+ code = tsdbReadDataBlk(pWriter->dReader.pReader, pWriter->dReader.pBlockIdx, &pWriter->dReader.mDataBlk);
+ if (code) goto _exit;
+
+ pWriter->dReader.iBlockIdx++;
+ } else {
+ pWriter->dReader.pBlockIdx = NULL;
+ tMapDataReset(&pWriter->dReader.mDataBlk);
+ }
+ pWriter->dReader.iDataBlk = 0; // point to the next one
+ tBlockDataReset(&pWriter->dReader.bData);
+ pWriter->dReader.iRow = 0;
+
+_exit:
+ return code;
+}
+
+static int32_t tsdbSnapWriteCopyData(STsdbSnapWriter* pWriter, TABLEID* pId) {
+ int32_t code = 0;
+
+ while (true) {
+ if (pWriter->dReader.pBlockIdx == NULL) break;
+ if (tTABLEIDCmprFn(pWriter->dReader.pBlockIdx, pId) >= 0) break;
+
+ SBlockIdx blkIdx = *pWriter->dReader.pBlockIdx;
+ code = tsdbWriteDataBlk(pWriter->dWriter.pWriter, &pWriter->dReader.mDataBlk, &blkIdx);
+ if (code) goto _exit;
+
+ if (taosArrayPush(pWriter->dWriter.aBlockIdx, &blkIdx) == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _exit;
+ }
+
+ code = tsdbSnapNextTableData(pWriter);
+ if (code) goto _exit;
+ }
+
+_exit:
+ return code;
+}
+
+static int32_t tsdbSnapWriteTableDataStart(STsdbSnapWriter* pWriter, TABLEID* pId) {
+ int32_t code = 0;
+
+ code = tsdbSnapWriteCopyData(pWriter, pId);
+ if (code) goto _err;
+
+ pWriter->id.suid = pId->suid;
+ pWriter->id.uid = pId->uid;
+
+ code = tsdbUpdateTableSchema(pWriter->pTsdb->pVnode->pMeta, pId->suid, pId->uid, &pWriter->skmTable);
+ if (code) goto _err;
+
+ tMapDataReset(&pWriter->dWriter.mDataBlk);
+ code = tBlockDataInit(&pWriter->dWriter.bData, pId->suid, pId->uid, pWriter->skmTable.pTSchema);
+ if (code) goto _err;
+
+ return code;
+
+_err:
+ tsdbError("vgId:%d %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, tstrerror(code));
+ return code;
+}
+
static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) {
int32_t code = 0;
- ASSERT(pWriter->pDataFWriter);
+ if (pWriter->id.suid == 0 && pWriter->id.uid == 0) return code;
- if (pWriter->pBlockIdxW == NULL) goto _exit;
+ int32_t c = 1;
+ if (pWriter->dReader.pBlockIdx) {
+ c = tTABLEIDCmprFn(pWriter->dReader.pBlockIdx, &pWriter->id);
+ ASSERT(c >= 0);
+ }
- // consume remain rows
- if (pWriter->pBlockData) {
- ASSERT(pWriter->iRow < pWriter->pBlockData->nRow);
- while (pWriter->iRow < pWriter->pBlockData->nRow) {
- code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL,
- 0); // todo
+ if (c == 0) {
+ SBlockData* pBData = &pWriter->dWriter.bData;
+
+ for (; pWriter->dReader.iRow < pWriter->dReader.bData.nRow; pWriter->dReader.iRow++) {
+ TSDBROW row = tsdbRowFromBlockData(&pWriter->dReader.bData, pWriter->dReader.iRow);
+
+ code = tBlockDataAppendRow(pBData, &row, NULL, pWriter->id.uid);
if (code) goto _err;
- if (pWriter->bDataW.nRow >= pWriter->maxRow * 4 / 5) {
- // pWriter->blockW.last = 0;
- // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
- // &pWriter->blockW, pWriter->cmprAlg);
+ if (pBData->nRow >= pWriter->maxRow) {
+ code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, pBData, &pWriter->dWriter.mDataBlk, pWriter->cmprAlg);
if (code) goto _err;
-
- code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk);
- if (code) goto _err;
-
- tDataBlkReset(&pWriter->blockW);
- tBlockDataClear(&pWriter->bDataW);
- }
-
- pWriter->iRow++;
- }
- }
-
- // write remain data if has
- if (pWriter->bDataW.nRow > 0) {
- // pWriter->blockW.last = 0;
- if (pWriter->bDataW.nRow < pWriter->minRow) {
- if (pWriter->iBlock > pWriter->mBlock.nItem) {
- // pWriter->blockW.last = 1;
}
}
- // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
- // &pWriter->blockW, pWriter->cmprAlg);
- // if (code) goto _err;
-
- code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk);
- if (code) goto _err;
- }
-
- while (true) {
- if (pWriter->iBlock >= pWriter->mBlock.nItem) break;
-
- SDataBlk block;
- tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetDataBlk);
-
- // if (block.last) {
- // code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, &pWriter->bDataR, NULL, NULL);
- // if (code) goto _err;
-
- // tBlockReset(&block);
- // block.last = 1;
- // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pWriter->pBlockIdxW, &block,
- // pWriter->cmprAlg);
- // if (code) goto _err;
- // }
-
- code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutDataBlk);
+ code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, pBData, &pWriter->dWriter.mDataBlk, pWriter->cmprAlg);
if (code) goto _err;
- pWriter->iBlock++;
- }
+ for (; pWriter->dReader.iDataBlk < pWriter->dReader.mDataBlk.nItem; pWriter->dReader.iDataBlk++) {
+ SDataBlk dataBlk;
+ tMapDataGetItemByIdx(&pWriter->dReader.mDataBlk, pWriter->dReader.iDataBlk, &dataBlk, tGetDataBlk);
- // SDataBlk
- // code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, pWriter->pBlockIdxW);
- // if (code) goto _err;
-
- // SBlockIdx
- if (taosArrayPush(pWriter->aBlockIdxW, pWriter->pBlockIdxW) == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
-
-_exit:
- tsdbInfo("vgId:%d, tsdb snapshot write table data end for %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path);
- return code;
-
-_err:
- tsdbError("vgId:%d, tsdb snapshot write table data end for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode),
- pWriter->pTsdb->path, tstrerror(code));
- return code;
-}
-
-static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* pBlockIdx) {
- int32_t code = 0;
-
- code = tsdbReadBlock(pWriter->pDataFReader, pBlockIdx, &pWriter->mBlock);
- if (code) goto _err;
-
- // SBlockData
- SDataBlk block;
- tMapDataReset(&pWriter->mBlockW);
- for (int32_t iBlock = 0; iBlock < pWriter->mBlock.nItem; iBlock++) {
- tMapDataGetItemByIdx(&pWriter->mBlock, iBlock, &block, tGetDataBlk);
-
- // if (block.last) {
- // code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL);
- // if (code) goto _err;
-
- // tBlockReset(&block);
- // block.last = 1;
- // code =
- // tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pBlockIdx, &block,
- // pWriter->cmprAlg);
- // if (code) goto _err;
- // }
-
- code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutDataBlk);
- if (code) goto _err;
- }
-
- // SDataBlk
- SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid};
- code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, &blockIdx);
- if (code) goto _err;
-
- // SBlockIdx
- if (taosArrayPush(pWriter->aBlockIdxW, &blockIdx) == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
-
-_exit:
- return code;
-
-_err:
- tsdbError("vgId:%d, tsdb snapshot move write table data for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode),
- pWriter->pTsdb->path, tstrerror(code));
- return code;
-}
-
-static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) {
- int32_t code = 0;
- SBlockData* pBlockData = &pWriter->bData;
- int32_t iRow = 0;
- TSDBROW row;
- TSDBROW* pRow = &row;
-
- // // correct schema
- // code = tBlockDataCorrectSchema(&pWriter->bDataW, pBlockData);
- // if (code) goto _err;
-
- // loop to merge
- *pRow = tsdbRowFromBlockData(pBlockData, iRow);
- while (true) {
- if (pRow == NULL) break;
-
- if (pWriter->pBlockData) {
- ASSERT(pWriter->iRow < pWriter->pBlockData->nRow);
-
- int32_t c = tsdbRowCmprFn(pRow, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow));
-
- ASSERT(c);
-
- if (c < 0) {
- // code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL);
- // if (code) goto _err;
-
- iRow++;
- if (iRow < pWriter->pBlockData->nRow) {
- *pRow = tsdbRowFromBlockData(pBlockData, iRow);
- } else {
- pRow = NULL;
- }
- } else if (c > 0) {
- // code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow),
- // NULL); if (code) goto _err;
-
- pWriter->iRow++;
- if (pWriter->iRow >= pWriter->pBlockData->nRow) {
- pWriter->pBlockData = NULL;
- }
- }
- } else {
- TSDBKEY key = TSDBROW_KEY(pRow);
-
- while (true) {
- if (pWriter->iBlock >= pWriter->mBlock.nItem) break;
-
- SDataBlk block;
- int32_t c;
-
- tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetDataBlk);
-
- // if (block.last) {
- // pWriter->pBlockData = &pWriter->bDataR;
-
- // code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL,
- // NULL); if (code) goto _err; pWriter->iRow = 0;
-
- // pWriter->iBlock++;
- // break;
- // }
-
- c = tsdbKeyCmprFn(&block.maxKey, &key);
-
- ASSERT(c);
-
- if (c < 0) {
- if (pWriter->bDataW.nRow) {
- // pWriter->blockW.last = 0;
- // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
- // &pWriter->blockW, pWriter->cmprAlg);
- // if (code) goto _err;
-
- code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk);
- if (code) goto _err;
-
- tDataBlkReset(&pWriter->blockW);
- tBlockDataClear(&pWriter->bDataW);
- }
-
- code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutDataBlk);
- if (code) goto _err;
-
- pWriter->iBlock++;
- } else {
- c = tsdbKeyCmprFn(&tBlockDataLastKey(pBlockData), &block.minKey);
-
- ASSERT(c);
-
- if (c > 0) {
- pWriter->pBlockData = &pWriter->bDataR;
- // code =
- // tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL,
- // NULL);
- // if (code) goto _err;
- pWriter->iRow = 0;
-
- pWriter->iBlock++;
- }
- break;
- }
- }
-
- if (pWriter->pBlockData) continue;
-
- // code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL);
- // if (code) goto _err;
-
- iRow++;
- if (iRow < pBlockData->nRow) {
- *pRow = tsdbRowFromBlockData(pBlockData, iRow);
- } else {
- pRow = NULL;
- }
- }
-
- _check_write:
- if (pWriter->bDataW.nRow < pWriter->maxRow * 4 / 5) continue;
-
- _write_block:
- // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
- // &pWriter->blockW, pWriter->cmprAlg);
- // if (code) goto _err;
-
- code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk);
- if (code) goto _err;
-
- tDataBlkReset(&pWriter->blockW);
- tBlockDataClear(&pWriter->bDataW);
- }
-
- return code;
-
-_err:
- tsdbError("vgId:%d, vnode snapshot tsdb write table data impl for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode),
- pWriter->pTsdb->path, tstrerror(code));
- return code;
-}
-
-static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, TABLEID id) {
- int32_t code = 0;
- SBlockData* pBlockData = &pWriter->bData;
- TSDBKEY keyFirst = tBlockDataFirstKey(pBlockData);
- TSDBKEY keyLast = tBlockDataLastKey(pBlockData);
-
- // end last table write if should
- if (pWriter->pBlockIdxW) {
- int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdxW, &id);
- if (c < 0) {
- // end
- code = tsdbSnapWriteTableDataEnd(pWriter);
+ code = tMapDataPutItem(&pWriter->dWriter.mDataBlk, &dataBlk, tPutDataBlk);
if (code) goto _err;
+ }
- // reset
- pWriter->pBlockIdxW = NULL;
- } else if (c > 0) {
- ASSERT(0);
+ code = tsdbSnapNextTableData(pWriter);
+ if (code) goto _err;
+ }
+
+ if (pWriter->dWriter.mDataBlk.nItem) {
+ SBlockIdx blockIdx = {.suid = pWriter->id.suid, .uid = pWriter->id.uid};
+ code = tsdbWriteDataBlk(pWriter->dWriter.pWriter, &pWriter->dWriter.mDataBlk, &blockIdx);
+
+ if (taosArrayPush(pWriter->dWriter.aBlockIdx, &blockIdx) == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
}
}
- // start new table data write if need
- if (pWriter->pBlockIdxW == NULL) {
- // write table data ahead
- while (true) {
- if (pWriter->iBlockIdx >= taosArrayGetSize(pWriter->aBlockIdx)) break;
+ pWriter->id.suid = 0;
+ pWriter->id.uid = 0;
- SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx);
- int32_t c = tTABLEIDCmprFn(pBlockIdx, &id);
-
- if (c >= 0) break;
-
- code = tsdbSnapMoveWriteTableData(pWriter, pBlockIdx);
- if (code) goto _err;
-
- pWriter->iBlockIdx++;
- }
-
- // reader
- pWriter->pBlockIdx = NULL;
- if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) {
- ASSERT(pWriter->pDataFReader);
-
- SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx);
- int32_t c = tTABLEIDCmprFn(pBlockIdx, &id);
-
- ASSERT(c >= 0);
-
- if (c == 0) {
- pWriter->pBlockIdx = pBlockIdx;
- pWriter->iBlockIdx++;
- }
- }
-
- if (pWriter->pBlockIdx) {
- code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlock);
- if (code) goto _err;
- } else {
- tMapDataReset(&pWriter->mBlock);
- }
- pWriter->iBlock = 0;
- pWriter->pBlockData = NULL;
- pWriter->iRow = 0;
-
- // writer
- pWriter->pBlockIdxW = &pWriter->blockIdxW;
- pWriter->pBlockIdxW->suid = id.suid;
- pWriter->pBlockIdxW->uid = id.uid;
-
- tDataBlkReset(&pWriter->blockW);
- tBlockDataReset(&pWriter->bDataW);
- tMapDataReset(&pWriter->mBlockW);
- }
-
- ASSERT(pWriter->pBlockIdxW && pWriter->pBlockIdxW->suid == id.suid && pWriter->pBlockIdxW->uid == id.uid);
- ASSERT(pWriter->pBlockIdx == NULL || (pWriter->pBlockIdx->suid == id.suid && pWriter->pBlockIdx->uid == id.uid));
-
- code = tsdbSnapWriteTableDataImpl(pWriter);
- if (code) goto _err;
-
-_exit:
- tsdbDebug("vgId:%d, vnode snapshot tsdb write data impl for %s", TD_VID(pWriter->pTsdb->pVnode),
- pWriter->pTsdb->path);
return code;
_err:
- tsdbError("vgId:%d, vnode snapshot tsdb write data impl for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode),
- pWriter->pTsdb->path, tstrerror(code));
return code;
}
-static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) {
+static int32_t tsdbSnapWriteOpenFile(STsdbSnapWriter* pWriter, int32_t fid) {
int32_t code = 0;
STsdb* pTsdb = pWriter->pTsdb;
- if (pWriter->pDataFWriter == NULL) goto _exit;
+ ASSERT(pWriter->dWriter.pWriter == NULL);
+
+ pWriter->fid = fid;
+ pWriter->id = (TABLEID){0};
+ SDFileSet* pSet = taosArraySearch(pWriter->fs.aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, TD_EQ);
+
+ // Reader
+ if (pSet) {
+ code = tsdbDataFReaderOpen(&pWriter->dReader.pReader, pWriter->pTsdb, pSet);
+ if (code) goto _err;
+
+ code = tsdbReadBlockIdx(pWriter->dReader.pReader, pWriter->dReader.aBlockIdx);
+ if (code) goto _err;
+ } else {
+ ASSERT(pWriter->dReader.pReader == NULL);
+ taosArrayClear(pWriter->dReader.aBlockIdx);
+ }
+ pWriter->dReader.iBlockIdx = 0; // point to the next one
+ code = tsdbSnapNextTableData(pWriter);
+ if (code) goto _err;
+
+ // Writer
+ SHeadFile fHead = {.commitID = pWriter->commitID};
+ SDataFile fData = {.commitID = pWriter->commitID};
+ SSmaFile fSma = {.commitID = pWriter->commitID};
+ SSttFile fStt = {.commitID = pWriter->commitID};
+ SDFileSet wSet = {.fid = pWriter->fid, .pHeadF = &fHead, .pDataF = &fData, .pSmaF = &fSma};
+ if (pSet) {
+ wSet.diskId = pSet->diskId;
+ fData = *pSet->pDataF;
+ fSma = *pSet->pSmaF;
+ for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+ wSet.aSttF[iStt] = pSet->aSttF[iStt];
+ }
+ wSet.nSttF = pSet->nSttF + 1; // TODO: fix pSet->nSttF == pTsdb->maxFile
+ } else {
+ SDiskID did = {0};
+ tfsAllocDisk(pTsdb->pVnode->pTfs, 0, &did);
+ tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did);
+ wSet.diskId = did;
+ wSet.nSttF = 1;
+ }
+ wSet.aSttF[wSet.nSttF - 1] = &fStt;
+
+ code = tsdbDataFWriterOpen(&pWriter->dWriter.pWriter, pWriter->pTsdb, &wSet);
+ if (code) goto _err;
+ taosArrayClear(pWriter->dWriter.aBlockIdx);
+ tMapDataReset(&pWriter->dWriter.mDataBlk);
+ taosArrayClear(pWriter->dWriter.aSttBlk);
+ tBlockDataReset(&pWriter->dWriter.bData);
+ tBlockDataReset(&pWriter->dWriter.sData);
+
+ return code;
+
+_err:
+ return code;
+}
+
+static int32_t tsdbSnapWriteCloseFile(STsdbSnapWriter* pWriter) {
+ int32_t code = 0;
+
+ ASSERT(pWriter->dWriter.pWriter);
- // finish current table
code = tsdbSnapWriteTableDataEnd(pWriter);
if (code) goto _err;
- // move remain table
- while (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) {
- code = tsdbSnapMoveWriteTableData(pWriter, (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx));
- if (code) goto _err;
-
- pWriter->iBlockIdx++;
- }
-
- // write remain stuff
- if (taosArrayGetSize(pWriter->aBlockLW) > 0) {
- code = tsdbWriteSttBlk(pWriter->pDataFWriter, pWriter->aBlockIdxW);
- if (code) goto _err;
- }
-
- if (taosArrayGetSize(pWriter->aBlockIdx) > 0) {
- code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW);
- if (code) goto _err;
- }
-
- code = tsdbFSUpsertFSet(&pWriter->fs, &pWriter->pDataFWriter->wSet);
+ // copy remain table data
+ TABLEID id = {.suid = INT64_MAX, .uid = INT64_MAX};
+ code = tsdbSnapWriteCopyData(pWriter, &id);
if (code) goto _err;
- code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 1);
+ code =
+ tsdbWriteSttBlock(pWriter->dWriter.pWriter, &pWriter->dWriter.sData, pWriter->dWriter.aSttBlk, pWriter->cmprAlg);
if (code) goto _err;
- if (pWriter->pDataFReader) {
- code = tsdbDataFReaderClose(&pWriter->pDataFReader);
+ // Indices
+ code = tsdbWriteBlockIdx(pWriter->dWriter.pWriter, pWriter->dWriter.aBlockIdx);
+ if (code) goto _err;
+
+ code = tsdbWriteSttBlk(pWriter->dWriter.pWriter, pWriter->dWriter.aSttBlk);
+ if (code) goto _err;
+
+ code = tsdbUpdateDFileSetHeader(pWriter->dWriter.pWriter);
+ if (code) goto _err;
+
+ code = tsdbFSUpsertFSet(&pWriter->fs, &pWriter->dWriter.pWriter->wSet);
+ if (code) goto _err;
+
+ code = tsdbDataFWriterClose(&pWriter->dWriter.pWriter, 1);
+ if (code) goto _err;
+
+ if (pWriter->dReader.pReader) {
+ code = tsdbDataFReaderClose(&pWriter->dReader.pReader);
if (code) goto _err;
}
_exit:
- tsdbInfo("vgId:%d, vnode snapshot tsdb writer data end for %s", TD_VID(pTsdb->pVnode), pTsdb->path);
return code;
_err:
- tsdbError("vgId:%d, vnode snapshot tsdb writer data end for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path,
- tstrerror(code));
+ return code;
+}
+
+static int32_t tsdbSnapWriteToDataFile(STsdbSnapWriter* pWriter, int32_t iRow, int8_t* done) {
+ int32_t code = 0;
+
+ SBlockData* pBData = &pWriter->bData;
+ TABLEID id = {.suid = pBData->suid, .uid = pBData->uid ? pBData->uid : pBData->aUid[iRow]};
+ TSDBROW row = tsdbRowFromBlockData(pBData, iRow);
+ TSDBKEY key = TSDBROW_KEY(&row);
+
+ *done = 0;
+ while (pWriter->dReader.iRow < pWriter->dReader.bData.nRow ||
+ pWriter->dReader.iDataBlk < pWriter->dReader.mDataBlk.nItem) {
+ // Merge row by row
+ for (; pWriter->dReader.iRow < pWriter->dReader.bData.nRow; pWriter->dReader.iRow++) {
+ TSDBROW trow = tsdbRowFromBlockData(&pWriter->dReader.bData, pWriter->dReader.iRow);
+ TSDBKEY tKey = TSDBROW_KEY(&trow);
+
+ ASSERT(pWriter->dReader.bData.suid == id.suid && pWriter->dReader.bData.uid == id.uid);
+
+ int32_t c = tsdbKeyCmprFn(&key, &tKey);
+ if (c < 0) {
+ code = tBlockDataAppendRow(&pWriter->dWriter.bData, &row, NULL, id.uid);
+ if (code) goto _err;
+ } else if (c > 0) {
+ code = tBlockDataAppendRow(&pWriter->dWriter.bData, &trow, NULL, id.uid);
+ if (code) goto _err;
+ } else {
+ ASSERT(0);
+ }
+
+ if (pWriter->dWriter.bData.nRow >= pWriter->maxRow) {
+ code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, &pWriter->dWriter.bData, &pWriter->dWriter.mDataBlk,
+ pWriter->cmprAlg);
+ if (code) goto _err;
+ }
+
+ if (c < 0) {
+ *done = 1;
+ goto _exit;
+ }
+ }
+
+ // Merge row by block
+ SDataBlk tDataBlk = {.minKey = key, .maxKey = key};
+ for (; pWriter->dReader.iDataBlk < pWriter->dReader.mDataBlk.nItem; pWriter->dReader.iDataBlk++) {
+ SDataBlk dataBlk;
+ tMapDataGetItemByIdx(&pWriter->dReader.mDataBlk, pWriter->dReader.iDataBlk, &dataBlk, tGetDataBlk);
+
+ int32_t c = tDataBlkCmprFn(&dataBlk, &tDataBlk);
+ if (c < 0) {
+ code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, &pWriter->dWriter.bData, &pWriter->dWriter.mDataBlk,
+ pWriter->cmprAlg);
+ if (code) goto _err;
+
+ code = tMapDataPutItem(&pWriter->dWriter.mDataBlk, &dataBlk, tPutDataBlk);
+ if (code) goto _err;
+ } else if (c > 0) {
+ code = tBlockDataAppendRow(&pWriter->dWriter.bData, &row, NULL, id.uid);
+ if (code) goto _err;
+
+ if (pWriter->dWriter.bData.nRow >= pWriter->maxRow) {
+ code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, &pWriter->dWriter.bData, &pWriter->dWriter.mDataBlk,
+ pWriter->cmprAlg);
+ if (code) goto _err;
+ }
+
+ *done = 1;
+ goto _exit;
+ } else {
+ code = tsdbReadDataBlockEx(pWriter->dReader.pReader, &dataBlk, &pWriter->dReader.bData);
+ if (code) goto _err;
+ pWriter->dReader.iRow = 0;
+
+ pWriter->dReader.iDataBlk++;
+ break;
+ }
+ }
+ }
+
+_exit:
+ return code;
+
+_err:
+ tsdbError("vgId:%d %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, tstrerror(code));
+ return code;
+}
+
+static int32_t tsdbSnapWriteToSttFile(STsdbSnapWriter* pWriter, int32_t iRow) {
+ int32_t code = 0;
+
+ TABLEID id = {.suid = pWriter->bData.suid,
+ .uid = pWriter->bData.uid ? pWriter->bData.uid : pWriter->bData.aUid[iRow]};
+ TSDBROW row = tsdbRowFromBlockData(&pWriter->bData, iRow);
+ SBlockData* pBData = &pWriter->dWriter.sData;
+
+ if (pBData->suid || pBData->uid) {
+ if (!TABLE_SAME_SCHEMA(pBData->suid, pBData->uid, id.suid, id.uid)) {
+ code = tsdbWriteSttBlock(pWriter->dWriter.pWriter, pBData, pWriter->dWriter.aSttBlk, pWriter->cmprAlg);
+ if (code) goto _err;
+
+ pBData->suid = 0;
+ pBData->uid = 0;
+ }
+ }
+
+ if (pBData->suid == 0 && pBData->uid == 0) {
+ code = tsdbUpdateTableSchema(pWriter->pTsdb->pVnode->pMeta, pWriter->id.suid, pWriter->id.uid, &pWriter->skmTable);
+ if (code) goto _err;
+
+ code = tBlockDataInit(pBData, pWriter->id.suid, pWriter->id.suid ? 0 : pWriter->id.uid, pWriter->skmTable.pTSchema);
+ if (code) goto _err;
+ }
+
+ code = tBlockDataAppendRow(pBData, &row, NULL, id.uid);
+ if (code) goto _err;
+
+ if (pBData->nRow >= pWriter->maxRow) {
+ code = tsdbWriteSttBlock(pWriter->dWriter.pWriter, pBData, pWriter->dWriter.aSttBlk, pWriter->cmprAlg);
+ if (code) goto _err;
+ }
+
+_exit:
+ return code;
+
+_err:
+ return code;
+}
+
+static int32_t tsdbSnapWriteRowData(STsdbSnapWriter* pWriter, int32_t iRow) {
+ int32_t code = 0;
+
+ SBlockData* pBlockData = &pWriter->bData;
+ TABLEID id = {.suid = pBlockData->suid, .uid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[iRow]};
+
+ // End last table data write if need
+ if (tTABLEIDCmprFn(&pWriter->id, &id) != 0) {
+ code = tsdbSnapWriteTableDataEnd(pWriter);
+ if (code) goto _err;
+ }
+
+ // Start new table data write if need
+ if (pWriter->id.suid == 0 && pWriter->id.uid == 0) {
+ code = tsdbSnapWriteTableDataStart(pWriter, &id);
+ if (code) goto _err;
+ }
+
+ // Merge with .data file data
+ int8_t done = 0;
+ if (pWriter->dReader.pBlockIdx && tTABLEIDCmprFn(pWriter->dReader.pBlockIdx, &id) == 0) {
+ code = tsdbSnapWriteToDataFile(pWriter, iRow, &done);
+ if (code) goto _err;
+ }
+
+ // Append to the .stt data block (todo: check if need to set/reload sst block)
+ if (!done) {
+ code = tsdbSnapWriteToSttFile(pWriter, iRow);
+ if (code) goto _err;
+ }
+
+_exit:
+ return code;
+
+_err:
+ tsdbError("vgId:%d %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, tstrerror(code));
return code;
}
static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
- int32_t code = 0;
- STsdb* pTsdb = pWriter->pTsdb;
- SSnapDataHdr* pHdr = (SSnapDataHdr*)pData;
- TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr));
- int64_t n;
-
- // decode
+ int32_t code = 0;
+ STsdb* pTsdb = pWriter->pTsdb;
SBlockData* pBlockData = &pWriter->bData;
- code = tDecmprBlockData(pData + sizeof(SSnapDataHdr) + sizeof(TABLEID), pHdr->size - sizeof(TABLEID), pBlockData,
- pWriter->aBuf);
+
+ // Decode data
+ SSnapDataHdr* pHdr = (SSnapDataHdr*)pData;
+ code = tDecmprBlockData(pHdr->data, pHdr->size, pBlockData, pWriter->aBuf);
if (code) goto _err;
- // open file
- TSDBKEY keyFirst = {.version = pBlockData->aVersion[0], .ts = pBlockData->aTSKEY[0]};
- TSDBKEY keyLast = {.version = pBlockData->aVersion[pBlockData->nRow - 1],
- .ts = pBlockData->aTSKEY[pBlockData->nRow - 1]};
+ ASSERT(pBlockData->nRow > 0);
- int32_t fid = tsdbKeyFid(keyFirst.ts, pWriter->minutes, pWriter->precision);
- ASSERT(fid == tsdbKeyFid(keyLast.ts, pWriter->minutes, pWriter->precision));
- if (pWriter->pDataFWriter == NULL || pWriter->fid != fid) {
- // end last file data write if need
- code = tsdbSnapWriteDataEnd(pWriter);
- if (code) goto _err;
+ // Loop to handle each row
+ for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) {
+ TSKEY ts = pBlockData->aTSKEY[iRow];
+ int32_t fid = tsdbKeyFid(ts, pWriter->minutes, pWriter->precision);
- pWriter->fid = fid;
+ if (pWriter->dWriter.pWriter == NULL || pWriter->fid != fid) {
+ if (pWriter->dWriter.pWriter) {
+ ASSERT(fid > pWriter->fid);
- // read
- SDFileSet* pSet = taosArraySearch(pWriter->fs.aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, TD_EQ);
- if (pSet) {
- code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet);
+ code = tsdbSnapWriteCloseFile(pWriter);
+ if (code) goto _err;
+ }
+
+ code = tsdbSnapWriteOpenFile(pWriter, fid);
if (code) goto _err;
-
- code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx);
- if (code) goto _err;
-
- code = tsdbReadSttBlk(pWriter->pDataFReader, 0, pWriter->aSstBlk);
- if (code) goto _err;
- } else {
- ASSERT(pWriter->pDataFReader == NULL);
- taosArrayClear(pWriter->aBlockIdx);
- taosArrayClear(pWriter->aSstBlk);
- }
- pWriter->iBlockIdx = 0;
- pWriter->pBlockIdx = NULL;
- tMapDataReset(&pWriter->mBlock);
- pWriter->iBlock = 0;
- pWriter->pBlockData = NULL;
- pWriter->iRow = 0;
- pWriter->iBlockL = 0;
- tBlockDataReset(&pWriter->bDataR);
- tBlockDataReset(&pWriter->lDataR);
-
- // write
- SHeadFile fHead;
- SDataFile fData;
- SSttFile fLast;
- SSmaFile fSma;
- SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .aSttF[0] = &fLast, .pSmaF = &fSma};
-
- if (pSet) {
- wSet.diskId = pSet->diskId;
- wSet.fid = fid;
- wSet.nSttF = 1;
- fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0};
- fData = *pSet->pDataF;
- fLast = (SSttFile){.commitID = pWriter->commitID, .size = 0};
- fSma = *pSet->pSmaF;
- } else {
- wSet.diskId = (SDiskID){.level = 0, .id = 0};
- wSet.fid = fid;
- wSet.nSttF = 1;
- fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0};
- fData = (SDataFile){.commitID = pWriter->commitID, .size = 0};
- fLast = (SSttFile){.commitID = pWriter->commitID, .size = 0, .offset = 0};
- fSma = (SSmaFile){.commitID = pWriter->commitID, .size = 0};
}
- code = tsdbDataFWriterOpen(&pWriter->pDataFWriter, pTsdb, &wSet);
+ code = tsdbSnapWriteRowData(pWriter, iRow);
if (code) goto _err;
-
- taosArrayClear(pWriter->aBlockIdxW);
- taosArrayClear(pWriter->aBlockLW);
- tMapDataReset(&pWriter->mBlockW);
- pWriter->pBlockIdxW = NULL;
- tBlockDataReset(&pWriter->bDataW);
}
- code = tsdbSnapWriteTableData(pWriter, id);
- if (code) goto _err;
-
- tsdbInfo("vgId:%d, vnode snapshot tsdb write data for %s, fid:%d suid:%" PRId64 " uid:%" PRId64 " nRow:%d",
- TD_VID(pTsdb->pVnode), pTsdb->path, fid, id.suid, id.suid, pBlockData->nRow);
return code;
_err:
@@ -976,10 +1099,41 @@ _err:
return code;
}
+// SNAP_DATA_DEL
+static int32_t tsdbSnapMoveWriteDelData(STsdbSnapWriter* pWriter, TABLEID* pId) {
+ int32_t code = 0;
+
+ while (true) {
+ if (pWriter->iDelIdx >= taosArrayGetSize(pWriter->aDelIdxR)) break;
+
+ SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx);
+
+ if (tTABLEIDCmprFn(pDelIdx, pId) >= 0) break;
+
+ code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData);
+ if (code) goto _exit;
+
+ SDelIdx delIdx = *pDelIdx;
+ code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx);
+ if (code) goto _exit;
+
+ if (taosArrayPush(pWriter->aDelIdxW, &delIdx) == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _exit;
+ }
+
+ pWriter->iDelIdx++;
+ }
+
+_exit:
+ return code;
+}
+
static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
int32_t code = 0;
STsdb* pTsdb = pWriter->pTsdb;
+ // Open del file if not opened yet
if (pWriter->pDelFWriter == NULL) {
SDelFile* pDelFile = pWriter->fs.pDelFile;
@@ -990,38 +1144,28 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32
code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdxR);
if (code) goto _err;
+ } else {
+ taosArrayClear(pWriter->aDelIdxR);
}
+ pWriter->iDelIdx = 0;
// writer
- SDelFile delFile = {.commitID = pWriter->commitID, .offset = 0, .size = 0};
+ SDelFile delFile = {.commitID = pWriter->commitID};
code = tsdbDelFWriterOpen(&pWriter->pDelFWriter, &delFile, pTsdb);
if (code) goto _err;
+ taosArrayClear(pWriter->aDelIdxW);
}
- // process the del data
- TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr));
+ SSnapDataHdr* pHdr = (SSnapDataHdr*)pData;
+ TABLEID id = *(TABLEID*)pHdr->data;
- while (true) {
- if (pWriter->iDelIdx >= taosArrayGetSize(pWriter->aDelIdxR)) break;
- if (tTABLEIDCmprFn(taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx), &id) >= 0) break;
+ ASSERT(pHdr->size + sizeof(SSnapDataHdr) == nData);
- SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx);
-
- code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData);
- if (code) goto _err;
-
- SDelIdx delIdx = *pDelIdx;
- code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx);
- if (code) goto _err;
-
- if (taosArrayPush(pWriter->aDelIdxW, &delIdx) == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
-
- pWriter->iDelIdx++;
- }
+ // Move write data < id
+ code = tsdbSnapMoveWriteDelData(pWriter, &id);
+ if (code) goto _err;
+ // Merge incoming data with current
if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR) &&
tTABLEIDCmprFn(taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx), &id) == 0) {
SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx);
@@ -1055,7 +1199,6 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32
goto _err;
}
-_exit:
return code;
_err:
@@ -1068,23 +1211,14 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) {
int32_t code = 0;
STsdb* pTsdb = pWriter->pTsdb;
- if (pWriter->pDelFWriter == NULL) goto _exit;
+ if (pWriter->pDelFWriter == NULL) return code;
- for (; pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR); pWriter->iDelIdx++) {
- SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx);
+ TABLEID id = {.suid = INT64_MAX, .uid = INT64_MAX};
+ code = tsdbSnapMoveWriteDelData(pWriter, &id);
+ if (code) goto _err;
- code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData);
- if (code) goto _err;
-
- SDelIdx delIdx = *pDelIdx;
- code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx);
- if (code) goto _err;
-
- if (taosArrayPush(pWriter->aDelIdxR, &delIdx) == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
- }
+ code = tsdbWriteDelIdx(pWriter->pDelFWriter, pWriter->aDelIdxW);
+ if (code) goto _err;
code = tsdbUpdateDelFileHdr(pWriter->pDelFWriter);
if (code) goto _err;
@@ -1100,7 +1234,6 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) {
if (code) goto _err;
}
-_exit:
tsdbInfo("vgId:%d, vnode snapshot tsdb write del for %s end", TD_VID(pTsdb->pVnode), pTsdb->path);
return code;
@@ -1110,6 +1243,7 @@ _err:
return code;
}
+// APIs
int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) {
int32_t code = 0;
STsdbSnapWriter* pWriter = NULL;
@@ -1135,39 +1269,38 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
pWriter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression;
pWriter->commitID = pTsdb->pVnode->state.commitID;
- // for data file
+ // SNAP_DATA_TSDB
code = tBlockDataCreate(&pWriter->bData);
-
- if (code) goto _err;
- pWriter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
- if (pWriter->aBlockIdx == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
- code = tBlockDataCreate(&pWriter->bDataR);
if (code) goto _err;
- pWriter->aSstBlk = taosArrayInit(0, sizeof(SSttBlk));
- if (pWriter->aSstBlk == NULL) {
+ pWriter->fid = INT32_MIN;
+ pWriter->id = (TABLEID){0};
+ // Reader
+ pWriter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
+ if (pWriter->dReader.aBlockIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
-
- pWriter->aBlockIdxW = taosArrayInit(0, sizeof(SBlockIdx));
- if (pWriter->aBlockIdxW == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
- code = tBlockDataCreate(&pWriter->bDataW);
+ code = tBlockDataCreate(&pWriter->dReader.bData);
if (code) goto _err;
- pWriter->aBlockLW = taosArrayInit(0, sizeof(SSttBlk));
- if (pWriter->aBlockLW == NULL) {
+ // Writer
+ pWriter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
+ if (pWriter->dWriter.aBlockIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
+ pWriter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
+ if (pWriter->dWriter.aSttBlk == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
+ code = tBlockDataCreate(&pWriter->dWriter.bData);
+ if (code) goto _err;
+ code = tBlockDataCreate(&pWriter->dWriter.sData);
+ if (code) goto _err;
- // for del file
+ // SNAP_DATA_DEL
pWriter->aDelIdxR = taosArrayInit(0, sizeof(SDelIdx));
if (pWriter->aDelIdxR == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
@@ -1188,6 +1321,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
tsdbInfo("vgId:%d, tsdb snapshot writer open for %s succeed", TD_VID(pTsdb->pVnode), pTsdb->path);
return code;
+
_err:
tsdbError("vgId:%d, tsdb snapshot writer open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path,
tstrerror(code));
@@ -1198,14 +1332,17 @@ _err:
int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
int32_t code = 0;
STsdbSnapWriter* pWriter = *ppWriter;
+ STsdb* pTsdb = pWriter->pTsdb;
if (rollback) {
ASSERT(0);
// code = tsdbFSRollback(pWriter->pTsdb->pFS);
// if (code) goto _err;
} else {
- code = tsdbSnapWriteDataEnd(pWriter);
- if (code) goto _err;
+ if (pWriter->dWriter.pWriter) {
+ code = tsdbSnapWriteCloseFile(pWriter);
+ if (code) goto _err;
+ }
code = tsdbSnapWriteDelEnd(pWriter);
if (code) goto _err;
@@ -1213,14 +1350,44 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
code = tsdbFSCommit1(pWriter->pTsdb, &pWriter->fs);
if (code) goto _err;
+ // lock
+ taosThreadRwlockWrlock(&pTsdb->rwLock);
+
code = tsdbFSCommit2(pWriter->pTsdb, &pWriter->fs);
- if (code) goto _err;
+ if (code) {
+ taosThreadRwlockUnlock(&pTsdb->rwLock);
+ goto _err;
+ }
+
+ // unlock
+ taosThreadRwlockUnlock(&pTsdb->rwLock);
}
+ // SNAP_DATA_DEL
+ taosArrayDestroy(pWriter->aDelIdxW);
+ taosArrayDestroy(pWriter->aDelData);
+ taosArrayDestroy(pWriter->aDelIdxR);
+
+ // SNAP_DATA_TSDB
+
+ // Writer
+ tBlockDataDestroy(&pWriter->dWriter.sData, 1);
+ tBlockDataDestroy(&pWriter->dWriter.bData, 1);
+ taosArrayDestroy(pWriter->dWriter.aSttBlk);
+ tMapDataClear(&pWriter->dWriter.mDataBlk);
+ taosArrayDestroy(pWriter->dWriter.aBlockIdx);
+
+ // Reader
+ tBlockDataDestroy(&pWriter->dReader.bData, 1);
+ tMapDataClear(&pWriter->dReader.mDataBlk);
+ taosArrayDestroy(pWriter->dReader.aBlockIdx);
+
+ tBlockDataDestroy(&pWriter->bData, 1);
+ tTSchemaDestroy(pWriter->skmTable.pTSchema);
+
for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) {
tFree(pWriter->aBuf[iBuf]);
}
-
tsdbInfo("vgId:%d, vnode snapshot tsdb writer close for %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path);
taosMemoryFree(pWriter);
*ppWriter = NULL;
@@ -1245,8 +1412,8 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData)
goto _exit;
} else {
- if (pWriter->pDataFWriter) {
- code = tsdbSnapWriteDataEnd(pWriter);
+ if (pWriter->dWriter.pWriter) {
+ code = tsdbSnapWriteCloseFile(pWriter);
if (code) goto _err;
}
}
@@ -1259,7 +1426,6 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData)
_exit:
tsdbDebug("vgId:%d, tsdb snapshot write for %s succeed", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path);
-
return code;
_err:
diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c
index f379d9f71a..5adb2eb359 100644
--- a/source/dnode/vnode/src/vnd/vnodeCfg.c
+++ b/source/dnode/vnode/src/vnd/vnodeCfg.c
@@ -13,8 +13,8 @@
* along with this program. If not, see
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see
+#include
+#include
+#include
+#include
+#include "index.h"
+#include "indexCache.h"
+#include "indexFst.h"
+#include "indexFstUtil.h"
+#include "indexInt.h"
+#include "indexTfile.h"
+#include "indexUtil.h"
+#include "tskiplist.h"
+#include "tutil.h"
+using namespace std;
+
+static std::string logDir = TD_TMP_DIR_PATH "log";
+
+static void initLog() {
+ const char *defaultLogFileNamePrefix = "taoslog";
+ const int32_t maxLogFileNum = 10;
+
+ tsAsyncLog = 0;
+ idxDebugFlag = 143;
+ strcpy(tsLogDir, logDir.c_str());
+ taosRemoveDir(tsLogDir);
+ taosMkDir(tsLogDir);
+
+ if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
+ printf("failed to open log file in directory:%s\n", tsLogDir);
+ }
+}
+
+struct WriteBatch {
+ SIndexMultiTerm *terms;
+};
+class Idx {
+ public:
+ Idx(int _cacheSize = 1024 * 1024 * 4, const char *_path = "tindex") {
+ opts.cacheSize = _cacheSize;
+ path += TD_TMP_DIR_PATH;
+ path += _path;
+ }
+ int SetUp(bool remove) {
+ initLog();
+
+ if (remove) taosRemoveDir(path.c_str());
+
+ int ret = indexJsonOpen(&opts, path.c_str(), &index);
+ return ret;
+ }
+ int Write(WriteBatch *batch, uint64_t uid) {
+ // write batch
+ indexJsonPut(index, batch->terms, uid);
+ return 0;
+ }
+ int Read(const char *json, void *key, int64_t *id) {
+ // read batch
+ return 0;
+ }
+
+ void TearDown() { indexJsonClose(index); }
+
+ std::string path;
+
+ SIndexOpts opts;
+ SIndex *index;
+};
+
+SIndexTerm *indexTermCreateT(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName,
+ int32_t nColName, const char *colVal, int32_t nColVal) {
+ char buf[256] = {0};
+ int16_t sz = nColVal;
+ memcpy(buf, (uint16_t *)&sz, 2);
+ memcpy(buf + 2, colVal, nColVal);
+ if (colType == TSDB_DATA_TYPE_BINARY) {
+ return indexTermCreate(suid, oper, colType, colName, nColName, buf, sizeof(buf));
+ } else {
+ return indexTermCreate(suid, oper, colType, colName, nColName, colVal, nColVal);
+ }
+ return NULL;
+}
+int initWriteBatch(WriteBatch *wb, int batchSize) {
+ SIndexMultiTerm *terms = indexMultiTermCreate();
+
+ std::string colName;
+ std::string colVal;
+
+ for (int i = 0; i < 64; i++) {
+ colName += '0' + i;
+ colVal += '0' + i;
+ }
+
+ for (int i = 0; i < batchSize; i++) {
+ colVal[i % colVal.size()] = '0' + i % 128;
+ colName[i % colName.size()] = '0' + i % 128;
+ SIndexTerm *term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ colVal.c_str(), colVal.size());
+ indexMultiTermAdd(terms, term);
+ }
+
+ wb->terms = terms;
+ return 0;
+}
+
+int BenchWrite(Idx *idx, int batchSize, int limit) {
+ for (int i = 0; i < limit; i += batchSize) {
+ WriteBatch wb;
+ idx->Write(&wb, i);
+ }
+ return 0;
+}
+
+int BenchRead(Idx *idx) { return 0; }
+
+int main() {
+ // Idx *idx = new Idx;
+ // if (idx->SetUp(true) != 0) {
+ // std::cout << "failed to setup index" << std::endl;
+ // return 0;
+ // } else {
+ // std::cout << "succ to setup index" << std::endl;
+ // }
+ // BenchWrite(idx, 100, 10000);
+ return 1;
+}
diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc
index 5b76de2ef8..08bf84ff60 100644
--- a/source/libs/index/test/indexTests.cc
+++ b/source/libs/index/test/indexTests.cc
@@ -271,20 +271,20 @@ void validateFst() {
}
delete m;
}
+
static std::string logDir = TD_TMP_DIR_PATH "log";
+static void initLog() {
+ const char* defaultLogFileNamePrefix = "taoslog";
+ const int32_t maxLogFileNum = 10;
-static void initLog() {
- const char* defaultLogFileNamePrefix = "taoslog";
- const int32_t maxLogFileNum = 10;
+ tsAsyncLog = 0;
+ idxDebugFlag = 143;
+ strcpy(tsLogDir, logDir.c_str());
+ taosRemoveDir(tsLogDir);
+ taosMkDir(tsLogDir);
- tsAsyncLog = 0;
- idxDebugFlag = 143;
- strcpy(tsLogDir, logDir.c_str());
- taosRemoveDir(tsLogDir);
- taosMkDir(tsLogDir);
-
- if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
- printf("failed to open log file in directory:%s\n", tsLogDir);
+ if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
+ printf("failed to open log file in directory:%s\n", tsLogDir);
}
}
class IndexEnv : public ::testing::Test {
diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c
index 8976daadbd..a41462e3fd 100644
--- a/source/libs/nodes/src/nodesCodeFuncs.c
+++ b/source/libs/nodes/src/nodesCodeFuncs.c
@@ -362,18 +362,14 @@ static int32_t jsonToTableComInfo(const SJson* pJson, void* pObj) {
int32_t code;
tjsonGetNumberValue(pJson, jkTableComInfoNumOfTags, pNode->numOfTags, code);
- ;
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableComInfoPrecision, pNode->precision, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableComInfoNumOfColumns, pNode->numOfColumns, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableComInfoRowSize, pNode->rowSize, code);
- ;
}
return code;
@@ -406,14 +402,11 @@ static int32_t jsonToSchema(const SJson* pJson, void* pObj) {
int32_t code;
tjsonGetNumberValue(pJson, jkSchemaType, pNode->type, code);
- ;
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkSchemaColId, pNode->colId, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkSchemaBytes, pNode->bytes, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkSchemaName, pNode->name);
@@ -466,26 +459,20 @@ static int32_t jsonToTableMeta(const SJson* pJson, void* pObj) {
int32_t code;
tjsonGetNumberValue(pJson, jkTableMetaVgId, pNode->vgId, code);
- ;
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableMetaTableType, pNode->tableType, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableMetaUid, pNode->uid, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableMetaSuid, pNode->suid, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableMetaSversion, pNode->sversion, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableMetaTversion, pNode->tversion, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonToObject(pJson, jkTableMetaComInfo, jsonToTableComInfo, &pNode->tableInfo);
@@ -926,7 +913,6 @@ static int32_t jsonToLogicFillNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToLogicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkFillLogicPlanMode, pNode->mode, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkFillLogicPlanWStartTs, &pNode->pWStartTs);
@@ -2815,7 +2801,6 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) {
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkColumnColType, pNode->colType, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkColumnDbName, pNode->dbName);
@@ -3118,7 +3103,6 @@ static int32_t jsonToOperatorNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToExprNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkOperatorType, pNode->opType, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkOperatorLeft, &pNode->pLeft);
@@ -3153,7 +3137,6 @@ static int32_t jsonToLogicConditionNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToExprNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkLogicCondType, pNode->condType, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkLogicCondParameters, &pNode->pParameterList);
@@ -3442,11 +3425,9 @@ static int32_t jsonToOrderByExprNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToNodeObject(pJson, jkOrderByExprExpr, &pNode->pExpr);
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkOrderByExprOrder, pNode->order, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkOrderByExprNullOrder, pNode->nullOrder, code);
- ;
}
return code;
@@ -3624,7 +3605,6 @@ static int32_t jsonToFillNode(const SJson* pJson, void* pObj) {
int32_t code;
tjsonGetNumberValue(pJson, jkFillMode, pNode->mode, code);
- ;
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkFillValues, &pNode->pValues);
}
diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c
new file mode 100644
index 0000000000..5fe31ed78e
--- /dev/null
+++ b/source/libs/nodes/src/nodesMsgFuncs.c
@@ -0,0 +1,3108 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see
#include
+#include
#include "cmdnodes.h"
#include "mockCatalogService.h"
@@ -251,6 +252,7 @@ class PlannerTestBaseImpl {
string splitLogicPlan_;
string scaledLogicPlan_;
string physiPlan_;
+ string physiPlanMsg_;
vector physiSubplans_;
};
@@ -274,6 +276,7 @@ class PlannerTestBaseImpl {
res_.splitLogicPlan_.clear();
res_.scaledLogicPlan_.clear();
res_.physiPlan_.clear();
+ res_.physiPlanMsg_.clear();
res_.physiSubplans_.clear();
}
@@ -408,6 +411,8 @@ class PlannerTestBaseImpl {
SNode* pSubplan;
FOREACH(pSubplan, ((SNodeListNode*)pNode)->pNodeList) { res_.physiSubplans_.push_back(toString(pSubplan)); }
}
+ res_.physiPlanMsg_ = toMsg((SNode*)(*pPlan));
+ cout << "json len: " << res_.physiPlan_.length() << ", msg len: " << res_.physiPlanMsg_.length() << endl;
}
void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) {
@@ -446,12 +451,45 @@ class PlannerTestBaseImpl {
string toString(const SNode* pRoot) {
char* pStr = NULL;
int32_t len = 0;
+
+ auto start = chrono::steady_clock::now();
DO_WITH_THROW(nodesNodeToString, pRoot, false, &pStr, &len)
+ if (QUERY_NODE_PHYSICAL_PLAN == nodeType(pRoot)) {
+ cout << "nodesNodeToString: "
+ << chrono::duration_cast(chrono::steady_clock::now() - start).count() << "us" << endl;
+ }
+
string str(pStr);
taosMemoryFreeClear(pStr);
return str;
}
+ string toMsg(const SNode* pRoot) {
+ char* pStr = NULL;
+ int32_t len = 0;
+
+ auto start = chrono::steady_clock::now();
+ DO_WITH_THROW(nodesNodeToMsg, pRoot, &pStr, &len)
+ cout << "nodesNodeToMsg: "
+ << chrono::duration_cast(chrono::steady_clock::now() - start).count() << "us" << endl;
+
+ SNode* pNode = NULL;
+ char* pNewStr = NULL;
+ int32_t newlen = 0;
+ DO_WITH_THROW(nodesMsgToNode, pStr, len, &pNode)
+ DO_WITH_THROW(nodesNodeToMsg, pNode, &pNewStr, &newlen)
+ if (newlen != len || 0 != memcmp(pStr, pNewStr, len)) {
+ cout << "nodesNodeToMsg error!!!!!!!!!!!!!! len = " << len << ", newlen = " << newlen << endl;
+ DO_WITH_THROW(nodesNodeToString, pNode, false, &pNewStr, &newlen)
+ cout << "nodesNodeToString " << pNewStr << endl;
+ }
+ taosMemoryFreeClear(pNewStr);
+
+ string str(pStr, len);
+ taosMemoryFreeClear(pStr);
+ return str;
+ }
+
caseEnv caseEnv_;
stmtEnv stmtEnv_;
stmtRes res_;
diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c
index 9cab39c301..c5f161b66a 100644
--- a/source/libs/scheduler/src/schTask.c
+++ b/source/libs/scheduler/src/schTask.c
@@ -52,7 +52,7 @@ void schInitTaskRetryTimes(SSchJob *pJob, SSchTask *pTask, SSchLevel *pLevel) {
int32_t nodeNum = taosArrayGetSize(pJob->nodeList);
pTask->maxRetryTimes = TMAX(nodeNum, SCH_DEFAULT_MAX_RETRY_NUM);
}
-
+
pTask->maxExecTimes = pTask->maxRetryTimes * (pLevel->level + 1);
}
@@ -139,13 +139,15 @@ int32_t schUpdateTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle, int3
}
if ((execId != pTask->execId) || pTask->waitRetry) { // ignore it
- SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId, pTask->execId, pTask->waitRetry);
+ SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId,
+ pTask->execId, pTask->waitRetry);
return TSDB_CODE_SUCCESS;
}
SSchNodeInfo *nodeInfo = taosHashGet(pTask->execNodes, &execId, sizeof(execId));
if (NULL == nodeInfo) { // ignore it
- SCH_TASK_DLOG("handle not updated since execId %d already not exist, current execId %d, waitRetry %d", execId, pTask->execId, pTask->waitRetry);
+ SCH_TASK_DLOG("handle not updated since execId %d already not exist, current execId %d, waitRetry %d", execId,
+ pTask->execId, pTask->waitRetry);
return TSDB_CODE_SUCCESS;
}
@@ -314,7 +316,7 @@ int32_t schRescheduleTask(SSchJob *pJob, SSchTask *pTask) {
if (!schMgmt.cfg.enableReSchedule) {
return TSDB_CODE_SUCCESS;
}
-
+
if (SCH_IS_DATA_BIND_TASK(pTask)) {
return TSDB_CODE_SUCCESS;
}
@@ -341,7 +343,8 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32
}
if (((pTask->execId + 1) >= pTask->maxExecTimes) || ((pTask->retryTimes + 1) > pTask->maxRetryTimes)) {
- SCH_TASK_DLOG("task no more retry since reach max times %d:%d, execId %d", pTask->maxRetryTimes, pTask->maxExecTimes, pTask->execId);
+ SCH_TASK_DLOG("task no more retry since reach max times %d:%d, execId %d", pTask->maxRetryTimes,
+ pTask->maxExecTimes, pTask->execId);
schHandleJobFailure(pJob, rspCode);
return TSDB_CODE_SUCCESS;
}
@@ -548,7 +551,8 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo
if ((pTask->retryTimes + 1) > pTask->maxRetryTimes) {
*needRetry = false;
- SCH_TASK_DLOG("task no more retry since reach max retry times, retryTimes:%d/%d", pTask->retryTimes, pTask->maxRetryTimes);
+ SCH_TASK_DLOG("task no more retry since reach max retry times, retryTimes:%d/%d", pTask->retryTimes,
+ pTask->maxRetryTimes);
return TSDB_CODE_SUCCESS;
}
@@ -564,25 +568,25 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo
return TSDB_CODE_SUCCESS;
}
-/*
- if (SCH_IS_DATA_BIND_TASK(pTask)) {
- if ((pTask->execId + 1) >= SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)) {
- *needRetry = false;
- SCH_TASK_DLOG("task no more retry since all ep tried, execId:%d, epNum:%d", pTask->execId,
- SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode));
- return TSDB_CODE_SUCCESS;
- }
- } else {
- int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs);
+ /*
+ if (SCH_IS_DATA_BIND_TASK(pTask)) {
+ if ((pTask->execId + 1) >= SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)) {
+ *needRetry = false;
+ SCH_TASK_DLOG("task no more retry since all ep tried, execId:%d, epNum:%d", pTask->execId,
+ SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode));
+ return TSDB_CODE_SUCCESS;
+ }
+ } else {
+ int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs);
- if ((pTask->candidateIdx + 1) >= candidateNum && (TSDB_CODE_SCH_TIMEOUT_ERROR != errCode)) {
- *needRetry = false;
- SCH_TASK_DLOG("task no more retry since all candiates tried, candidateIdx:%d, candidateNum:%d",
- pTask->candidateIdx, candidateNum);
- return TSDB_CODE_SUCCESS;
+ if ((pTask->candidateIdx + 1) >= candidateNum && (TSDB_CODE_SCH_TIMEOUT_ERROR != errCode)) {
+ *needRetry = false;
+ SCH_TASK_DLOG("task no more retry since all candiates tried, candidateIdx:%d, candidateNum:%d",
+ pTask->candidateIdx, candidateNum);
+ return TSDB_CODE_SUCCESS;
+ }
}
- }
-*/
+ */
*needRetry = true;
SCH_TASK_DLOG("task need the %dth retry, errCode:%x - %s", pTask->execId + 1, errCode, tstrerror(errCode));
@@ -630,8 +634,9 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) {
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
- SCH_TASK_TLOG("set %dth candidate addr, id %d, inUse:%d/%d, fqdn:%s, port:%d", i, naddr->nodeId, naddr->epSet.inUse, naddr->epSet.numOfEps,
- SCH_GET_CUR_EP(naddr)->fqdn, SCH_GET_CUR_EP(naddr)->port);
+ SCH_TASK_TLOG("set %dth candidate addr, id %d, inUse:%d/%d, fqdn:%s, port:%d", i, naddr->nodeId,
+ naddr->epSet.inUse, naddr->epSet.numOfEps, SCH_GET_CUR_EP(naddr)->fqdn,
+ SCH_GET_CUR_EP(naddr)->port);
++addNum;
}
@@ -711,10 +716,10 @@ int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask) {
if (candidateNum <= 1) {
goto _return;
}
-
+
switch (schMgmt.cfg.schPolicy) {
case SCH_LOAD_SEQ:
- case SCH_ALL:
+ case SCH_ALL:
default:
if (++pTask->candidateIdx >= candidateNum) {
pTask->candidateIdx = 0;
@@ -732,7 +737,7 @@ int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask) {
_return:
SCH_TASK_DLOG("switch task candiateIdx to %d/%d", pTask->candidateIdx, candidateNum);
-
+
return TSDB_CODE_SUCCESS;
}
@@ -759,7 +764,7 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) {
return;
}
- int32_t i = 0;
+ int32_t i = 0;
SSchNodeInfo *nodeInfo = taosHashIterate(pTask->execNodes, NULL);
while (nodeInfo) {
if (nodeInfo->handle) {
@@ -821,16 +826,16 @@ int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList) {
int32_t schLaunchTaskImpl(void *param) {
SSchTaskCtx *pCtx = (SSchTaskCtx *)param;
- SSchJob *pJob = schAcquireJob(pCtx->jobRid);
+ SSchJob *pJob = schAcquireJob(pCtx->jobRid);
if (NULL == pJob) {
- taosMemoryFree(param);
qDebug("job refId 0x%" PRIx64 " already not exist", pCtx->jobRid);
+ taosMemoryFree(param);
SCH_RET(TSDB_CODE_SCH_JOB_IS_DROPPING);
}
-
+
SSchTask *pTask = pCtx->pTask;
- int8_t status = 0;
- int32_t code = 0;
+ int8_t status = 0;
+ int32_t code = 0;
atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1);
pTask->execId++;
@@ -891,13 +896,12 @@ _return:
SCH_RET(code);
}
-int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) {
-
+int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) {
SSchTaskCtx *param = taosMemoryCalloc(1, sizeof(SSchTaskCtx));
if (NULL == param) {
SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
-
+
param->jobRid = pJob->refId;
param->pTask = pTask;
@@ -906,7 +910,7 @@ int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) {
} else {
SCH_ERR_RET(schLaunchTaskImpl(param));
}
-
+
return TSDB_CODE_SUCCESS;
}
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index a2a45938e4..7cdb7c0db9 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -248,9 +248,12 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
char* ctbName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, groupId);
SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
- // TODO: get hash function by hashMethod
- uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));
+ /*uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));*/
+ SUseDbRsp* pDbInfo = &pTask->shuffleDispatcher.dbInfo;
+ uint32_t hashValue =
+ taosGetTbHashVal(ctbName, strlen(ctbName), pDbInfo->hashMethod, pDbInfo->hashPrefix, pDbInfo->hashSuffix);
taosMemoryFree(ctbName);
+
bool found = false;
// TODO: optimize search
int32_t j;
diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c
index bec07b935e..bb2151ed9d 100644
--- a/source/libs/tdb/src/db/tdbPager.c
+++ b/source/libs/tdb/src/db/tdbPager.c
@@ -34,6 +34,22 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage
static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage);
static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage);
+static FORCE_INLINE int32_t pageCmpFn(const void *lhs, const void *rhs) {
+ SPage *pPageL = (SPage *)(((uint8_t *)lhs) - sizeof(SRBTreeNode));
+ SPage *pPageR = (SPage *)(((uint8_t *)rhs) - sizeof(SRBTreeNode));
+
+ SPgno pgnoL = TDB_PAGE_PGNO(pPageL);
+ SPgno pgnoR = TDB_PAGE_PGNO(pPageR);
+
+ if (pgnoL < pgnoR) {
+ return -1;
+ } else if (pgnoL > pgnoR) {
+ return 1;
+ } else {
+ return 0;
+ }
+}
+
int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
uint8_t *pPtr;
SPager *pPager;
@@ -83,6 +99,8 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
ret = tdbGetFileSize(pPager->fd, pPager->pageSize, &(pPager->dbOrigSize));
pPager->dbFileSize = pPager->dbOrigSize;
+ tRBTreeCreate(&pPager->rbt, pageCmpFn);
+
*ppPager = pPager;
return 0;
}
@@ -167,7 +185,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
// ref page one more time so the page will not be release
tdbRefPage(pPage);
tdbDebug("pcache/mdirty page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id);
-
+ /*
// Set page as dirty
pPage->isDirty = 1;
@@ -185,6 +203,8 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
ASSERT(*ppPage == NULL || TDB_PAGE_PGNO(*ppPage) > TDB_PAGE_PGNO(pPage));
pPage->pDirtyNext = *ppPage;
*ppPage = pPage;
+ */
+ tRBTreePut(&pPager->rbt, (SRBTreeNode *)pPage);
// Write page to journal if neccessary
if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize) {
@@ -228,6 +248,23 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
return 0;
}
+ SRBTreeIter iter = tRBTreeIterCreate(&pPager->rbt, 1);
+ SRBTreeNode *pNode = NULL;
+ while ((pNode = tRBTreeIterNext(&iter)) != NULL) {
+ pPage = (SPage *)pNode;
+ ret = tdbPagerWritePageToDB(pPager, pPage);
+ if (ret < 0) {
+ ASSERT(0);
+ return -1;
+ }
+
+ pPage->isDirty = 0;
+
+ tdbPCacheRelease(pPager->pCache, pPage, pTxn);
+ }
+
+ tRBTreeCreate(&pPager->rbt, pageCmpFn);
+ /*
// loop to write the dirty pages to file
for (pPage = pPager->pDirty; pPage; pPage = pPage->pDirtyNext) {
// TODO: update the page footer
@@ -238,9 +275,6 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
}
}
- tdbTrace("tdbttl commit:%p, %d", pPager, pPager->dbOrigSize);
- pPager->dbOrigSize = pPager->dbFileSize;
-
// release the page
for (pPage = pPager->pDirty; pPage; pPage = pPager->pDirty) {
pPager->pDirty = pPage->pDirtyNext;
@@ -250,6 +284,9 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
tdbPCacheRelease(pPager->pCache, pPage, pTxn);
}
+ */
+ tdbTrace("tdbttl commit:%p, %d", pPager, pPager->dbOrigSize);
+ pPager->dbOrigSize = pPager->dbFileSize;
// sync the db file
tdbOsFSync(pPager->fd);
@@ -497,7 +534,14 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) {
return 0;
}
-
+/*
+struct TdFile {
+ TdThreadRwlock rwlock;
+ int refId;
+ int fd;
+ FILE *fp;
+} TdFile;
+*/
static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) {
i64 offset;
int ret;
@@ -514,6 +558,7 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) {
return -1;
}
+ // pwrite(pPager->fd->fd, pPage->pData, pPage->pageSize, offset);
return 0;
}
diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h
index 6a694cf8f1..df6ba8b35f 100644
--- a/source/libs/tdb/src/inc/tdbInt.h
+++ b/source/libs/tdb/src/inc/tdbInt.h
@@ -19,6 +19,7 @@
#include "tdb.h"
#include "tlog.h"
+#include "trbtree.h"
#ifdef __cplusplus
extern "C" {
@@ -256,6 +257,7 @@ typedef struct {
#pragma pack(pop)
struct SPage {
+ SRBTreeNode node; // must be the first field for pageCmpFn to work
tdb_spinlock_t lock;
int pageSize;
u8 *pData;
@@ -280,13 +282,13 @@ struct SPage {
static inline i32 tdbRefPage(SPage *pPage) {
i32 nRef = atomic_add_fetch_32(&((pPage)->nRef), 1);
- tdbTrace("ref page %p/%d, nRef %d", pPage, pPage->id, nRef);
+ // tdbTrace("ref page %p/%d, nRef %d", pPage, pPage->id, nRef);
return nRef;
}
static inline i32 tdbUnrefPage(SPage *pPage) {
i32 nRef = atomic_sub_fetch_32(&((pPage)->nRef), 1);
- tdbTrace("unref page %p/%d, nRef %d", pPage, pPage->id, nRef);
+ // tdbTrace("unref page %p/%d, nRef %d", pPage, pPage->id, nRef);
return nRef;
}
@@ -389,6 +391,7 @@ struct SPager {
SPgno dbFileSize;
SPgno dbOrigSize;
SPage *pDirty;
+ SRBTree rbt;
u8 inTran;
SPager *pNext; // used by TDB
SPager *pHashNext; // used by TDB
diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index 62cdf4e784..4a0008b5ff 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -162,7 +162,7 @@ static void cliWalkCb(uv_handle_t* handle, void* arg);
static void cliReleaseUnfinishedMsg(SCliConn* conn) {
for (int i = 0; i < transQueueSize(&conn->cliMsgs); i++) {
SCliMsg* msg = transQueueGet(&conn->cliMsgs, i);
- if (msg != NULL && msg->ctx != NULL) {
+ if (msg != NULL && msg->ctx != NULL && msg->ctx->ahandle != (void*)0x9527) {
if (conn->ctx.freeFunc != NULL && msg->ctx->ahandle != NULL) {
conn->ctx.freeFunc(msg->ctx->ahandle);
}
@@ -196,22 +196,22 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) {
#define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL)
#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrd*)(conn)->hostThrd)->pTransInst))->label)
-#define CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle) \
- do { \
- int i = 0, sz = transQueueSize(&conn->cliMsgs); \
- for (; i < sz; i++) { \
- pMsg = transQueueGet(&conn->cliMsgs, i); \
- if (pMsg != NULL && pMsg->ctx != NULL && (uint64_t)pMsg->ctx->ahandle == ahandle) { \
- break; \
- } \
- } \
- if (i == sz) { \
- pMsg = NULL; \
- tDebug("msg not found, %" PRIu64 "", ahandle); \
- } else { \
- pMsg = transQueueRm(&conn->cliMsgs, i); \
- tDebug("msg found, %" PRIu64 "", ahandle); \
- } \
+#define CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle) \
+ do { \
+ int i = 0, sz = transQueueSize(&conn->cliMsgs); \
+ for (; i < sz; i++) { \
+ pMsg = transQueueGet(&conn->cliMsgs, i); \
+ if (pMsg->ctx != NULL && (uint64_t)pMsg->ctx->ahandle == ahandle) { \
+ break; \
+ } \
+ } \
+ if (i == sz) { \
+ pMsg = NULL; \
+ tDebug("msg not found, %" PRIu64 "", ahandle); \
+ } else { \
+ pMsg = transQueueRm(&conn->cliMsgs, i); \
+ tDebug("msg found, %" PRIu64 "", ahandle); \
+ } \
} while (0)
#define CONN_GET_NEXT_SENDMSG(conn) \
do { \
@@ -289,7 +289,12 @@ bool cliMaySendCachedMsg(SCliConn* conn) {
if (!transQueueEmpty(&conn->cliMsgs)) {
SCliMsg* pCliMsg = NULL;
CONN_GET_NEXT_SENDMSG(conn);
- cliSend(conn);
+ if (pCliMsg == NULL)
+ return false;
+ else {
+ cliSend(conn);
+ return true;
+ }
}
return false;
_RETURN:
@@ -376,8 +381,10 @@ void cliHandleResp(SCliConn* conn) {
return;
}
- if (cliAppCb(conn, &transMsg, pMsg) != 0) {
- return;
+ if (pMsg == NULL || (pMsg && pMsg->type != Release)) {
+ if (cliAppCb(conn, &transMsg, pMsg) != 0) {
+ return;
+ }
}
destroyCmsg(pMsg);
@@ -425,18 +432,20 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) {
transMsg.info.ahandle);
}
} else {
- transMsg.info.ahandle = pCtx ? pCtx->ahandle : NULL;
+ transMsg.info.ahandle = (pMsg->type != Release && pCtx) ? pCtx->ahandle : NULL;
}
if (pCtx == NULL || pCtx->pSem == NULL) {
if (transMsg.info.ahandle == NULL) {
- if (REQUEST_NO_RESP(&pMsg->msg)) destroyCmsg(pMsg);
+ if (REQUEST_NO_RESP(&pMsg->msg) || pMsg->type == Release) destroyCmsg(pMsg);
once = true;
continue;
}
}
- if (cliAppCb(pConn, &transMsg, pMsg) != 0) {
- return;
+ if (pMsg == NULL || (pMsg && pMsg->type != Release)) {
+ if (cliAppCb(pConn, &transMsg, pMsg) != 0) {
+ return;
+ }
}
destroyCmsg(pMsg);
tTrace("%s conn %p start to destroy, ref:%d", CONN_GET_INST_LABEL(pConn), pConn, T_REF_VAL_GET(pConn));
@@ -702,6 +711,9 @@ static bool cliHandleNoResp(SCliConn* conn) {
if (cliMaySendCachedMsg(conn) == false) {
SCliThrd* thrd = conn->hostThrd;
addConnToPool(thrd->pool, conn);
+ res = false;
+ } else {
+ res = true;
}
}
}
@@ -779,7 +791,13 @@ void cliSend(SCliConn* pConn) {
uv_buf_t wb = uv_buf_init((char*)pHead, msgLen);
uv_write_t* req = transReqQueuePush(&pConn->wreqQueue);
- uv_write(req, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb);
+
+ int status = uv_write(req, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb);
+ if (status != 0) {
+ tGError("%s conn %p failed to sent msg:%s, errmsg:%s", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pMsg->msgType),
+ uv_err_name(status));
+ cliHandleExcept(pConn);
+ }
return;
_RETURN:
return;
@@ -928,7 +946,9 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) {
// persist conn already release by server
STransMsg resp;
cliBuildExceptResp(pMsg, &resp);
- pTransInst->cfp(pTransInst->parent, &resp, NULL);
+ if (pMsg->type != Release) {
+ pTransInst->cfp(pTransInst->parent, &resp, NULL);
+ }
destroyCmsg(pMsg);
return;
}
@@ -1399,53 +1419,57 @@ void transUnrefCliHandle(void* handle) {
cliDestroyConn((SCliConn*)handle, true);
}
}
-static FORCE_INLINE SCliThrd* transGetWorkThrdFromHandle(int64_t handle, bool* validHandle) {
+static FORCE_INLINE SCliThrd* transGetWorkThrdFromHandle(STrans* trans, int64_t handle) {
SCliThrd* pThrd = NULL;
SExHandle* exh = transAcquireExHandle(transGetRefMgt(), handle);
if (exh == NULL) {
return NULL;
}
- *validHandle = true;
+ if (exh->pThrd == NULL && trans != NULL) {
+ int idx = cliRBChoseIdx(trans);
+ if (idx < 0) return NULL;
+ exh->pThrd = ((SCliObj*)trans->tcphandle)->pThreadObj[idx];
+ }
+
pThrd = exh->pThrd;
transReleaseExHandle(transGetRefMgt(), handle);
return pThrd;
}
-SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle, bool* validHandle) {
+SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle) {
if (handle == 0) {
int idx = cliRBChoseIdx(trans);
if (idx < 0) return NULL;
return ((SCliObj*)trans->tcphandle)->pThreadObj[idx];
}
- SCliThrd* pThrd = transGetWorkThrdFromHandle(handle, validHandle);
- if (*validHandle == true && pThrd == NULL) {
- int idx = cliRBChoseIdx(trans);
- if (idx < 0) return NULL;
- pThrd = ((SCliObj*)trans->tcphandle)->pThreadObj[idx];
- }
+ SCliThrd* pThrd = transGetWorkThrdFromHandle(trans, handle);
return pThrd;
}
int transReleaseCliHandle(void* handle) {
int idx = -1;
bool valid = false;
- SCliThrd* pThrd = transGetWorkThrdFromHandle((int64_t)handle, &valid);
+ SCliThrd* pThrd = transGetWorkThrdFromHandle(NULL, (int64_t)handle);
if (pThrd == NULL) {
return -1;
}
- STransMsg tmsg = {.info.handle = handle};
+ STransMsg tmsg = {.info.handle = handle, .info.ahandle = (void*)0x9527};
TRACE_SET_MSGID(&tmsg.info.traceId, tGenIdPI64());
+ STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx));
+ pCtx->ahandle = tmsg.info.ahandle;
+
SCliMsg* cmsg = taosMemoryCalloc(1, sizeof(SCliMsg));
cmsg->msg = tmsg;
cmsg->type = Release;
+ cmsg->ctx = pCtx;
STraceId* trace = &tmsg.info.traceId;
tGDebug("send release request at thread:%08" PRId64 "", pThrd->pid);
if (0 != transAsyncSend(pThrd->asyncPool, &cmsg->q)) {
- taosMemoryFree(cmsg);
+ destroyCmsg(cmsg);
return -1;
}
return 0;
@@ -1458,9 +1482,8 @@ int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STran
return -1;
}
- bool valid = false;
- SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle, &valid);
- if (pThrd == NULL && valid == false) {
+ SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle);
+ if (pThrd == NULL) {
transFreeMsg(pReq->pCont);
transReleaseExHandle(transGetInstMgt(), (int64_t)shandle);
return TSDB_CODE_RPC_BROKEN_LINK;
@@ -1503,9 +1526,8 @@ int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMs
return -1;
}
- bool valid = false;
- SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle, &valid);
- if (pThrd == NULL && valid == false) {
+ SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle);
+ if (pThrd == NULL) {
transFreeMsg(pReq->pCont);
transReleaseExHandle(transGetInstMgt(), (int64_t)shandle);
return TSDB_CODE_RPC_BROKEN_LINK;
@@ -1589,6 +1611,7 @@ int64_t transAllocHandle() {
SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle));
exh->refId = transAddExHandle(transGetRefMgt(), exh);
tDebug("pre alloc refId %" PRId64 "", exh->refId);
+
return exh->refId;
}
#endif
diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c
index c00cafb708..d3277d1cc1 100644
--- a/source/libs/transport/src/transSvr.c
+++ b/source/libs/transport/src/transSvr.c
@@ -907,7 +907,7 @@ static void uvDestroyConn(uv_handle_t* handle) {
}
static void uvPipeListenCb(uv_stream_t* handle, int status) {
if (status != 0) {
- tError("server failed to init pipe");
+ tError("server failed to init pipe, errmsg: %s", uv_err_name(status));
return;
}
@@ -945,7 +945,10 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
uv_loop_init(srv->loop);
int ret = uv_pipe_init(srv->loop, &srv->pipeListen, 0);
- assert(ret == 0);
+ if (ret != 0) {
+ tError("failed to init pipe, errmsg: %s", uv_err_name(ret));
+ goto End;
+ }
#ifdef WINDOWS
char pipeName[64];
@@ -956,10 +959,16 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
taosGetSelfPthreadId());
#endif
ret = uv_pipe_bind(&srv->pipeListen, pipeName);
- assert(ret == 0);
+ if (ret != 0) {
+ tError("failed to bind pipe, errmsg: %s", uv_err_name(ret));
+ goto End;
+ }
ret = uv_listen((uv_stream_t*)&srv->pipeListen, SOMAXCONN, uvPipeListenCb);
- assert(ret == 0);
+ if (ret != 0) {
+ tError("failed to listen pipe, errmsg: %s", uv_err_name(ret));
+ goto End;
+ }
for (int i = 0; i < srv->numOfThreads; i++) {
SWorkThrd* thrd = (SWorkThrd*)taosMemoryCalloc(1, sizeof(SWorkThrd));
@@ -1082,12 +1091,12 @@ void transCloseServer(void* arg) {
if (srv->inited) {
uv_async_send(srv->pAcceptAsync);
taosThreadJoin(srv->thread, NULL);
- }
- SRV_RELEASE_UV(srv->loop);
+ SRV_RELEASE_UV(srv->loop);
- for (int i = 0; i < srv->numOfThreads; i++) {
- sendQuitToWorkThrd(srv->pThreadObj[i]);
- destroyWorkThrd(srv->pThreadObj[i]);
+ for (int i = 0; i < srv->numOfThreads; i++) {
+ sendQuitToWorkThrd(srv->pThreadObj[i]);
+ destroyWorkThrd(srv->pThreadObj[i]);
+ }
}
taosMemoryFree(srv->pThreadObj);
diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c
index 9db7d6c455..5c437e6f7a 100644
--- a/source/libs/wal/src/walRead.c
+++ b/source/libs/wal/src/walRead.c
@@ -168,6 +168,9 @@ static int32_t walReadChangeFile(SWalReader *pReader, int64_t fileFirstVer) {
}
pReader->pIdxFile = pIdxFile;
+
+ pReader->curFileFirstVer = fileFirstVer;
+
return 0;
}
@@ -372,7 +375,7 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) {
int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead) {
int64_t code;
-// ASSERT(pRead->curVersion == pHead->head.version);
+ // ASSERT(pRead->curVersion == pHead->head.version);
code = taosLSeekFile(pRead->pLogFile, pHead->head.bodyLen, SEEK_CUR);
if (code < 0) {
@@ -415,7 +418,8 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) {
}
if (walValidBodyCksum(*ppHead) != 0) {
- wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, ver);
+ wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId,
+ ver);
pRead->curInvalid = 1;
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
diff --git a/source/util/src/tuuid.c b/source/util/src/tuuid.c
index 9101aec949..7460ccbc82 100644
--- a/source/util/src/tuuid.c
+++ b/source/util/src/tuuid.c
@@ -51,11 +51,11 @@ int64_t tGenIdPI64(void) {
int64_t id;
while (true) {
- int64_t ts = taosGetTimestampMs();
+ int64_t ts = taosGetTimestampMs() >> 8;
uint64_t pid = taosGetPId();
int32_t val = atomic_add_fetch_32(&tUUIDSerialNo, 1);
- id = ((tUUIDHashId & 0x07FF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF);
+ id = ((tUUIDHashId & 0x07FF) << 52) | ((pid & 0x0F) << 48) | ((ts & 0x3FFFFFF) << 20) | (val & 0xFFFFF);
if (id) {
break;
}
diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt
index c06535fc36..161c878440 100644
--- a/tests/script/jenkins/basic.txt
+++ b/tests/script/jenkins/basic.txt
@@ -249,6 +249,12 @@
./test.sh -f tsim/stream/windowClose.sim
./test.sh -f tsim/stream/ignoreExpiredData.sim
./test.sh -f tsim/stream/sliding.sim
+#./test.sh -f tsim/stream/partitionbyColumnInterval.sim
+#./test.sh -f tsim/stream/partitionbyColumnSession.sim
+#./test.sh -f tsim/stream/partitionbyColumnState.sim
+#./test.sh -f tsim/stream/deleteInterval.sim
+#./test.sh -f tsim/stream/deleteSession.sim
+#./test.sh -f tsim/stream/deleteState.sim
# ---- transaction ----
./test.sh -f tsim/trans/lossdata1.sim
diff --git a/tests/script/sh/abs_max.c b/tests/script/sh/abs_max.c
deleted file mode 100644
index d623adacf9..0000000000
--- a/tests/script/sh/abs_max.c
+++ /dev/null
@@ -1,88 +0,0 @@
-#include
-#include
-#include
-
-typedef struct SUdfInit{
- int maybe_null; /* 1 if function can return NULL */
- int decimals; /* for real functions */
- long long length; /* For string functions */
- char *ptr; /* free pointer for function data */
- int const_item; /* 0 if result is independent of arguments */
-} SUdfInit;
-
-
-#define TSDB_DATA_INT_NULL 0x80000000LL
-#define TSDB_DATA_BIGINT_NULL 0x8000000000000000LL
-
-void abs_max(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput,
- int* numOfOutput, short otype, short obytes, SUdfInit* buf) {
- int i;
- int r = 0;
- printf("abs_max input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf);
- if (itype == 5) {
- r=*(long *)dataOutput;
- *numOfOutput=0;
-
- for(i=0;i r) {
- r = v;
- }
- }
-
- *(long *)dataOutput=r;
-
- printf("abs_max out, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput);
- }
-}
-
-
-
-void abs_max_finalize(char* dataOutput, char* interBuf, int* numOfOutput, SUdfInit* buf) {
- int i;
- int r = 0;
- printf("abs_max_finalize dataoutput:%p:%d, numOfOutput:%d, buf:%p\n", dataOutput, *dataOutput, *numOfOutput, buf);
- *numOfOutput=1;
- printf("abs_max finalize, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput);
-}
-
-void abs_max_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf) {
- int r = 0;
-
- if (numOfRows > 0) {
- r = *((long *)data);
- }
- printf("abs_max_merge numOfRows:%d, dataoutput:%p, buf:%p\n", numOfRows, dataOutput, buf);
- for (int i = 1; i < numOfRows; ++i) {
- printf("abs_max_merge %d - %ld\n", i, *((long *)data + i));
- if (*((long*)data + i) > r) {
- r= *((long*)data + i);
- }
- }
-
- *(long*)dataOutput=r;
- if (numOfRows > 0) {
- *numOfOutput=1;
- } else {
- *numOfOutput=0;
- }
-
- printf("abs_max_merge, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput);
-}
-
-
-int abs_max_init(SUdfInit* buf) {
- printf("abs_max init\n");
- return 0;
-}
-
-
-void abs_max_destroy(SUdfInit* buf) {
- printf("abs_max destroy\n");
-}
-
diff --git a/tests/script/sh/add_one.c b/tests/script/sh/add_one.c
deleted file mode 100644
index e12cf8f26f..0000000000
--- a/tests/script/sh/add_one.c
+++ /dev/null
@@ -1,33 +0,0 @@
-#include
-#include
-#include
-
-typedef struct SUdfInit{
- int maybe_null; /* 1 if function can return NULL */
- int decimals; /* for real functions */
- long long length; /* For string functions */
- char *ptr; /* free pointer for function data */
- int const_item; /* 0 if result is independent of arguments */
-} SUdfInit;
-
-void add_one(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBUf, char* tsOutput,
- int* numOfOutput, short otype, short obytes, SUdfInit* buf) {
- int i;
- int r = 0;
- printf("add_one input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf);
- if (itype == 4) {
- for(i=0;i
-#include
-#include
-
-typedef struct SUdfInit{
- int maybe_null; /* 1 if function can return NULL */
- int decimals; /* for real functions */
- long long length; /* For string functions */
- char *ptr; /* free pointer for function data */
- int const_item; /* 0 if result is independent of arguments */
-} SUdfInit;
-
-typedef struct SDemo{
- double sum;
- int num;
- short otype;
-}SDemo;
-
-#define FLOAT_NULL 0x7FF00000 // it is an NAN
-#define DOUBLE_NULL 0x7FFFFF0000000000LL // it is an NAN
-
-
-void demo(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput,
- int* numOfOutput, short otype, short obytes, SUdfInit* buf) {
- int i;
- double r = 0;
- SDemo *p = (SDemo *)interBuf;
- SDemo *q = (SDemo *)dataOutput;
- printf("demo input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, interBUf:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, interBuf, tsOutput, numOfOutput, buf);
-
- for(i=0;isum += r*r;
- }
-
- p->otype = otype;
- p->num += numOfRows;
-
- q->sum = p->sum;
- q->num = p->num;
- q->otype = p->otype;
-
- *numOfOutput=1;
-
- printf("demo out, sum:%f, num:%d, numOfOutput:%d\n", p->sum, p->num, *numOfOutput);
-}
-
-
-void demo_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf) {
- int i;
- SDemo *p = (SDemo *)data;
- SDemo res = {0};
- printf("demo_merge input data:%p, rows:%d, dataoutput:%p, numOfOutput:%p, buf:%p\n", data, numOfRows, dataOutput, numOfOutput, buf);
-
- for(i=0;isum * p->sum;
- res.num += p->num;
- p++;
- }
-
- p->sum = res.sum;
- p->num = res.num;
-
- *numOfOutput=1;
-
- printf("demo out, sum:%f, num:%d, numOfOutput:%d\n", p->sum, p->num, *numOfOutput);
-}
-
-
-
-void demo_finalize(char* dataOutput, char* interBuf, int* numOfOutput, SUdfInit* buf) {
- SDemo *p = (SDemo *)interBuf;
- printf("demo_finalize interbuf:%p, numOfOutput:%p, buf:%p, sum:%f, num:%d\n", interBuf, numOfOutput, buf, p->sum, p->num);
- if (p->otype == 6) {
- if (p->num != 30000) {
- *(unsigned int *)dataOutput = FLOAT_NULL;
- } else {
- *(float *)dataOutput = (float)(p->sum / p->num);
- }
- printf("finalize values:%f\n", *(float *)dataOutput);
- } else if (p->otype == 7) {
- if (p->num != 30000) {
- *(unsigned long long *)dataOutput = DOUBLE_NULL;
- } else {
- *(double *)dataOutput = (double)(p->sum / p->num);
- }
- printf("finalize values:%f\n", *(double *)dataOutput);
- }
-
- *numOfOutput=1;
-
- printf("demo finalize, numOfOutput:%d\n", *numOfOutput);
-}
-
-
-int demo_init(SUdfInit* buf) {
- printf("demo init\n");
- return 0;
-}
-
-
-void demo_destroy(SUdfInit* buf) {
- printf("demo destroy\n");
-}
-
diff --git a/tests/script/sh/demo.lua b/tests/script/sh/demo.lua
deleted file mode 100644
index c5e5582fc3..0000000000
--- a/tests/script/sh/demo.lua
+++ /dev/null
@@ -1,43 +0,0 @@
-funcName = "test"
-
-global = {}
-
-function test_init()
- return global
-end
-
-function test_add(rows, ans, key)
- t = {}
- t["sum"] = 0.0
- t["num"] = 0
- for i=1, #rows do
- t["sum"] = t["sum"] + rows[i] * rows[i]
- end
- t["num"] = #rows
-
-
- if (ans[key] ~= nil)
- then
- ans[key]["sum"] = ans[key]["sum"] + t["sum"]
- ans[key]["num"] = ans[key]["num"] + t["num"]
- else
- ans[key] = t
- end
-
- return ans;
-end
-
-function test_finalize(ans, key)
- local ret = 0.0
-
- if (ans[key] ~= nil and ans[key]["num"] == 30000)
- then
- ret = ans[key]["sum"]/ans[key]["num"]
- ans[key]["sum"] = 0.0
- ans[key]["num"] = 0
- else
- ret = inf
- end
-
- return ret, ans
-end
diff --git a/tests/script/sh/sum_double.c b/tests/script/sh/sum_double.c
deleted file mode 100644
index d6eea5d291..0000000000
--- a/tests/script/sh/sum_double.c
+++ /dev/null
@@ -1,84 +0,0 @@
-#include
-#include
-#include
-
-typedef struct SUdfInit{
- int maybe_null; /* 1 if function can return NULL */
- int decimals; /* for real functions */
- long long length; /* For string functions */
- char *ptr; /* free pointer for function data */
- int const_item; /* 0 if result is independent of arguments */
-} SUdfInit;
-
-#define TSDB_DATA_INT_NULL 0x80000000LL
-
-
-void sum_double(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput,
- int* numOfOutput, short otype, short obytes, SUdfInit* buf) {
- int i;
- int r = 0;
- printf("sum_double input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf);
- if (itype == 4) {
- r=*(int *)dataOutput;
- *numOfOutput=0;
-
- for(i=0;iptr)=*(int*)dataOutput*2;
- *(int*)dataOutput=*(int*)(buf->ptr);
- printf("sum_double finalize, dataoutput:%d, numOfOutput:%d\n", *(int *)dataOutput, *numOfOutput);
-}
-
-void sum_double_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf) {
- int r = 0;
- int sum = 0;
-
- printf("sum_double_merge numOfRows:%d, dataoutput:%p, buf:%p\n", numOfRows, dataOutput, buf);
- for (int i = 0; i < numOfRows; ++i) {
- printf("sum_double_merge %d - %d\n", i, *((int*)data + i));
- sum +=*((int*)data + i);
- }
-
- *(int*)dataOutput+=sum;
- if (numOfRows > 0) {
- *numOfOutput=1;
- } else {
- *numOfOutput=0;
- }
-
- printf("sum_double_merge, dataoutput:%d, numOfOutput:%d\n", *(int *)dataOutput, *numOfOutput);
-}
-
-
-int sum_double_init(SUdfInit* buf) {
- buf->maybe_null=1;
- buf->ptr = taosMemoryMalloc(sizeof(int));
- printf("sum_double init\n");
- return 0;
-}
-
-
-void sum_double_destroy(SUdfInit* buf) {
- taosMemoryFree(buf->ptr);
- printf("sum_double destroy\n");
-}
-
diff --git a/tests/script/tsim/column/table.sim b/tests/script/tsim/column/table.sim
index 4f1d32c373..03c4799681 100644
--- a/tests/script/tsim/column/table.sim
+++ b/tests/script/tsim/column/table.sim
@@ -159,6 +159,7 @@ if $data01 != 10 then
return -1
endi
if $data02 != 4.500000000 then
+ print expect 4.500000000, actual: $data02
return -1
endi
if $data03 != 4.500000000 then
diff --git a/tests/script/tsim/stream/deleteInterval.sim b/tests/script/tsim/stream/deleteInterval.sim
index dfd0ddc9d3..00d10afad9 100644
--- a/tests/script/tsim/stream/deleteInterval.sim
+++ b/tests/script/tsim/stream/deleteInterval.sim
@@ -186,7 +186,9 @@ endi
sql drop stream if exists streams2;
sql drop database if exists test2;
+sql drop database if exists test;
sql create database test2 vgroups 4;
+sql create database test vgroups 1;
sql use test2;
sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
@@ -411,6 +413,80 @@ if $data12 != 3 then
goto loop14
endi
+return 1
+
+sql drop stream if exists streams3;
+sql drop database if exists test3;
+sql drop database if exists test;
+sql create database test3 vgroups 4;
+sql create database test vgroups 1;
+sql use test3;
+sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
+sql create table t1 using st tags(1,1,1);
+sql create table t2 using st tags(2,2,2);
+sql create stream streams3 trigger at_once into test.streamt3 as select _wstart c1, count(*) c2, max(a) c3 from st interval(10s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+sql delete from t1;
+
+loop15:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 1 then
+ print =====rows=$rows
+ goto loop15
+endi
+
+$loop_count = 0
+
+sql delete from t1 where ts > 100;
+
+loop16:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 1 then
+ print =====rows=$rows
+ goto loop16
+endi
+
+$loop_count = 0
+
+sql delete from st;
+
+loop17:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 0 then
+ print =====rows=$rows
+ goto loop17
+endi
+
+
+
+
+
$loop_all = $loop_all + 1
print ============loop_all=$loop_all
diff --git a/tests/script/tsim/stream/deleteSession.sim b/tests/script/tsim/stream/deleteSession.sim
new file mode 100644
index 0000000000..541609633b
--- /dev/null
+++ b/tests/script/tsim/stream/deleteSession.sim
@@ -0,0 +1,532 @@
+$loop_all = 0
+looptest:
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 200
+sql connect
+
+sql drop stream if exists streams0;
+sql drop stream if exists streams1;
+sql drop stream if exists streams2;
+sql drop stream if exists streams3;
+sql drop stream if exists streams4;
+sql drop database if exists test;
+sql create database test vgroups 1;
+sql use test;
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3 from t1 session(ts, 5s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sleep 200
+sql delete from t1 where ts = 1648791213000;
+
+$loop_count = 0
+
+loop0:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 0 then
+ print =====rows=$rows
+ goto loop0
+endi
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop1:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop1
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop1
+endi
+
+sql insert into t1 values(1648791213000,1,1,1,1.0);
+sql insert into t1 values(1648791213001,2,2,2,2.0);
+sql insert into t1 values(1648791213002,3,3,3,3.0);
+sql insert into t1 values(1648791213003,4,4,4,4.0);
+
+sleep 200
+sql delete from t1 where ts >= 1648791213001 and ts <= 1648791213002;
+
+$loop_count = 0
+
+loop3:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop3
+endi
+
+if $data02 != 4 then
+ print =====data02=$data02
+ goto loop3
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+
+$loop_count = 0
+
+loop4:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop4
+endi
+
+sleep 200
+
+sql delete from t1 where ts >= 1648791223000 and ts <= 1648791223003;
+
+$loop_count = 0
+
+loop5:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop5
+endi
+
+if $data02 != 4 then
+ print =====data02=$data02
+ goto loop5
+endi
+
+sql insert into t1 values(1648791213000,1,1,1,1.0);
+sql insert into t1 values(1648791213005,2,2,2,2.0);
+sql insert into t1 values(1648791213006,3,3,3,3.0);
+sql insert into t1 values(1648791213007,4,4,4,4.0);
+
+sql insert into t1 values(1648791223000,1,1,1,1.0);
+sql insert into t1 values(1648791223001,2,2,2,2.0);
+sql insert into t1 values(1648791223002,3,3,3,3.0);
+sql insert into t1 values(1648791223003,4,4,4,4.0);
+
+sql insert into t1 values(1648791233000,1,1,1,1.0);
+sql insert into t1 values(1648791233001,2,2,2,2.0);
+sql insert into t1 values(1648791233008,3,3,3,3.0);
+sql insert into t1 values(1648791233009,4,4,4,4.0);
+
+sql delete from t1 where ts >= 1648791213001 and ts <= 1648791233005;
+
+$loop_count = 0
+
+loop6:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop6
+endi
+
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop6
+endi
+
+if $data11 != 2 then
+ print =====data11=$data11
+ goto loop6
+endi
+
+if $data12 != 4 then
+ print =====data12=$data12
+ goto loop6
+endi
+
+sql drop stream if exists streams2;
+sql drop database if exists test2;
+sql create database test2 vgroups 4;
+sql use test2;
+sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
+sql create table t1 using st tags(1,1,1);
+sql create table t2 using st tags(2,2,2);
+sql create stream streams2 trigger at_once into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop7:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 1 then
+ print =====rows=$rows
+ goto loop7
+endi
+
+sleep 200
+
+sql delete from t1 where ts = 1648791213000;
+
+$loop_count = 0
+
+loop8:
+sleep 200
+
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop8
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop8
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+sql insert into t2 values(1648791223000,1,2,3,1.0);
+sql insert into t2 values(1648791223001,1,2,3,1.0);
+sql insert into t2 values(1648791223002,3,2,3,1.0);
+sql insert into t2 values(1648791223003,3,2,3,1.0);
+
+sleep 200
+
+sql delete from t2 where ts >= 1648791223000 and ts <= 1648791223001;
+
+$loop_count = 0
+
+loop11:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop11
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop11
+endi
+
+if $data11 != 6 then
+ print =====data11=$data11
+ goto loop11
+endi
+
+if $data12 != 3 then
+ print =====data12=$data12
+ goto loop11
+endi
+
+sleep 200
+
+sql delete from st where ts >= 1648791223000 and ts <= 1648791223003;
+
+$loop_count = 0
+
+loop12:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 1 then
+ print =====rows=$rows
+ goto loop12
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop12
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop12
+endi
+
+sql insert into t1 values(1648791213004,3,2,3,1.0);
+sql insert into t1 values(1648791213005,3,2,3,1.0);
+sql insert into t1 values(1648791213006,3,2,3,1.0);
+sql insert into t1 values(1648791223004,1,2,3,1.0);
+sql insert into t2 values(1648791213004,3,2,3,1.0);
+sql insert into t2 values(1648791213005,3,2,3,1.0);
+sql insert into t2 values(1648791213006,3,2,3,1.0);
+sql insert into t2 values(1648791223004,1,2,3,1.0);
+
+sleep 200
+
+sql delete from t2 where ts >= 1648791213004 and ts <= 1648791213006;
+
+$loop_count = 0
+
+loop13:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop13
+endi
+
+if $data01 != 4 then
+ print =====data01=$data01
+ goto loop13
+endi
+
+if $data02 != 3 then
+ print =====data02=$data02
+ goto loop13
+endi
+
+if $data11 != 2 then
+ print =====data11=$data11
+ goto loop13
+endi
+
+if $data12 != 1 then
+ print =====data12=$data12
+ goto loop13
+endi
+
+sql insert into t1 values(1648791223005,1,2,3,1.0);
+sql insert into t1 values(1648791223006,1,2,3,1.0);
+sql insert into t2 values(1648791223005,1,2,3,1.0);
+sql insert into t2 values(1648791223006,1,2,3,1.0);
+
+sql insert into t1 values(1648791233005,4,2,3,1.0);
+sql insert into t1 values(1648791233006,2,2,3,1.0);
+sql insert into t2 values(1648791233005,5,2,3,1.0);
+sql insert into t2 values(1648791233006,3,2,3,1.0);
+
+sleep 200
+
+sql delete from st where ts >= 1648791213001 and ts <= 1648791233005;
+
+$loop_count = 0
+
+loop14:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop14
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop14
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop14
+endi
+
+if $data11 != 2 then
+ print =====data11=$data11
+ goto loop14
+endi
+
+if $data12 != 3 then
+ print =====data12=$data12
+ goto loop14
+endi
+
+sql drop stream if exists streams1;
+sql drop stream if exists streams2;
+sql drop stream if exists streams3;
+sql drop database if exists test3;
+sql drop database if exists test;
+sql create database test3 vgroups 4;
+sql create database test vgroups 1;
+sql use test3;
+sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
+sql create table t1 using st tags(1,1,1);
+sql create table t2 using st tags(2,2,2);
+sql create stream streams3 trigger at_once into test.streamt3 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s);
+
+sql insert into t1 values(1648791210000,1,1,1,NULL);
+sql insert into t1 values(1648791210001,2,2,2,NULL);
+sql insert into t2 values(1648791213001,3,3,3,NULL);
+sql insert into t2 values(1648791213003,4,4,4,NULL);
+sql insert into t1 values(1648791216000,5,5,5,NULL);
+sql insert into t1 values(1648791216002,6,6,6,NULL);
+sql insert into t1 values(1648791216004,7,7,7,NULL);
+sql insert into t2 values(1648791218001,8,8,8,NULL);
+sql insert into t2 values(1648791218003,9,9,9,NULL);
+sql insert into t1 values(1648791222000,10,10,10,NULL);
+sql insert into t1 values(1648791222003,11,11,11,NULL);
+sql insert into t1 values(1648791222005,12,12,12,NULL);
+
+sql insert into t1 values(1648791232005,13,13,13,NULL);
+sql insert into t2 values(1648791242005,14,14,14,NULL);
+
+$loop_count = 0
+
+loop19:
+sleep 200
+sql select * from test.streamt3 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 3 then
+ print =====rows=$rows
+ goto loop19
+endi
+
+sql delete from t2 where ts >= 1648791213001 and ts <= 1648791218003;
+
+$loop_count = 0
+
+loop20:
+sleep 200
+sql select * from test.streamt3 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 5 then
+ print =====rows=$rows
+ goto loop20
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop20
+endi
+
+if $data02 != 2 then
+ print =====data02=$data02
+ goto loop20
+endi
+
+if $data11 != 3 then
+ print =====data11=$data11
+ goto loop20
+endi
+
+if $data12 != 7 then
+ print =====data12=$data12
+ goto loop20
+endi
+
+if $data21 != 3 then
+ print =====data21=$data21
+ goto loop20
+endi
+
+if $data22 != 12 then
+ print =====data22=$data22
+ goto loop20
+endi
+
+if $data31 != 1 then
+ print =====data31=$data31
+ goto loop20
+endi
+
+if $data32 != 13 then
+ print =====data32=$data32
+ goto loop20
+endi
+
+if $data41 != 1 then
+ print =====data41=$data41
+ goto loop20
+endi
+
+if $data42 != 14 then
+ print =====data42=$data42
+ goto loop20
+endi
+
+$loop_all = $loop_all + 1
+print ============loop_all=$loop_all
+
+system sh/stop_dnodes.sh
+
+#goto looptest
\ No newline at end of file
diff --git a/tests/script/tsim/stream/deleteState.sim b/tests/script/tsim/stream/deleteState.sim
new file mode 100644
index 0000000000..ecd9f55340
--- /dev/null
+++ b/tests/script/tsim/stream/deleteState.sim
@@ -0,0 +1,198 @@
+$loop_all = 0
+looptest:
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 200
+sql connect
+
+sql drop stream if exists streams0;
+sql drop stream if exists streams1;
+sql drop stream if exists streams2;
+sql drop stream if exists streams3;
+sql drop stream if exists streams4;
+sql drop database if exists test;
+sql create database test vgroups 1;
+sql use test;
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(b) c3 from t1 state_window(a);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sleep 200
+sql delete from t1 where ts = 1648791213000;
+
+$loop_count = 0
+
+loop0:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 0 then
+ print =====rows=$rows
+ goto loop0
+endi
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop1:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop1
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop1
+endi
+
+sql insert into t1 values(1648791213000,1,1,1,1.0);
+sql insert into t1 values(1648791213001,1,2,2,2.0);
+sql insert into t1 values(1648791213002,1,3,3,3.0);
+sql insert into t1 values(1648791213003,1,4,4,4.0);
+
+sleep 200
+sql delete from t1 where ts >= 1648791213001 and ts <= 1648791213002;
+
+$loop_count = 0
+
+loop3:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop3
+endi
+
+if $data02 != 4 then
+ print =====data02=$data02
+ goto loop3
+endi
+
+sql insert into t1 values(1648791223000,2,2,3,1.0);
+sql insert into t1 values(1648791223001,2,2,3,1.0);
+sql insert into t1 values(1648791223002,2,2,3,1.0);
+sql insert into t1 values(1648791223003,2,2,3,1.0);
+
+$loop_count = 0
+
+loop4:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop4
+endi
+
+sleep 200
+
+sql delete from t1 where ts >= 1648791223000 and ts <= 1648791223003;
+
+$loop_count = 0
+
+loop5:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop5
+endi
+
+if $data02 != 4 then
+ print =====data02=$data02
+ goto loop5
+endi
+
+sql insert into t1 values(1648791213000,1,1,1,1.0);
+sql insert into t1 values(1648791213005,1,2,2,2.0);
+sql insert into t1 values(1648791213006,1,3,3,3.0);
+sql insert into t1 values(1648791213007,1,4,4,4.0);
+
+sql insert into t1 values(1648791223000,2,1,1,1.0);
+sql insert into t1 values(1648791223001,2,2,2,2.0);
+sql insert into t1 values(1648791223002,2,3,3,3.0);
+sql insert into t1 values(1648791223003,2,4,4,4.0);
+
+sql insert into t1 values(1648791233000,3,1,1,1.0);
+sql insert into t1 values(1648791233001,3,2,2,2.0);
+sql insert into t1 values(1648791233008,3,3,3,3.0);
+sql insert into t1 values(1648791233009,3,4,4,4.0);
+
+sql delete from t1 where ts >= 1648791213001 and ts <= 1648791233005;
+
+$loop_count = 0
+
+loop6:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop6
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop6
+endi
+
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop6
+endi
+
+if $data11 != 2 then
+ print =====data11=$data11
+ goto loop6
+endi
+
+if $data12 != 4 then
+ print =====data12=$data12
+ goto loop6
+endi
+
+
+$loop_all = $loop_all + 1
+print ============loop_all=$loop_all
+
+system sh/stop_dnodes.sh
+
+#goto looptest
\ No newline at end of file
diff --git a/tests/script/tsim/stream/partitionbyColumn0.sim b/tests/script/tsim/stream/partitionbyColumnInterval.sim
similarity index 100%
rename from tests/script/tsim/stream/partitionbyColumn0.sim
rename to tests/script/tsim/stream/partitionbyColumnInterval.sim
diff --git a/tests/script/tsim/stream/partitionbyColumn1.sim b/tests/script/tsim/stream/partitionbyColumnSession.sim
similarity index 100%
rename from tests/script/tsim/stream/partitionbyColumn1.sim
rename to tests/script/tsim/stream/partitionbyColumnSession.sim
diff --git a/tests/script/tsim/stream/partitionbyColumn2.sim b/tests/script/tsim/stream/partitionbyColumnState.sim
similarity index 100%
rename from tests/script/tsim/stream/partitionbyColumn2.sim
rename to tests/script/tsim/stream/partitionbyColumnState.sim
diff --git a/tests/system-test/2-query/ttl_comment.py b/tests/system-test/2-query/ttl_comment.py
index c26393158c..187df2e2d3 100644
--- a/tests/system-test/2-query/ttl_comment.py
+++ b/tests/system-test/2-query/ttl_comment.py
@@ -137,7 +137,8 @@ class TDTestCase:
tdSql.execute(f"create table {dbname}.child_table2 using {dbname}.super_table1 tags(1) comment ''")
tdSql.execute(f"create table {dbname}.child_table3 using {dbname}.super_table1 tags(1) comment 'child'")
tdSql.execute(f"insert into {dbname}.child_table4 using {dbname}.super_table1 tags(1) values(now, 1)")
-
+ tdSql.execute(f"insert into {dbname}.child_table5 using {dbname}.super_table1 tags(1) ttl 23 comment '' values(now, 1)")
+ tdSql.error(f"insert into {dbname}.child_table6 using {dbname}.super_table1 tags(1) ttl -23 comment '' values(now, 1)")
tdSql.query("select * from information_schema.ins_tables where table_name like 'child_table1'")
tdSql.checkData(0, 0, 'child_table1')
@@ -160,6 +161,11 @@ class TDTestCase:
tdSql.checkData(0, 7, 0)
tdSql.checkData(0, 8, None)
+ tdSql.query("select * from information_schema.ins_tables where table_name like 'child_table5'")
+ tdSql.checkData(0, 0, 'child_table5')
+ tdSql.checkData(0, 7, 23)
+ tdSql.checkData(0, 8, '')
+
tdSql.execute(f"alter table {dbname}.child_table1 comment 'nihao'")
tdSql.query("select * from information_schema.ins_tables where table_name like 'child_table1'")
diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh
index 8987ba3bbd..0b45b24258 100755
--- a/tests/system-test/fulltest.sh
+++ b/tests/system-test/fulltest.sh
@@ -2,7 +2,7 @@
set -e
set -x
-python3 ./test.py -f 0-others/taosShell.py
+#python3 ./test.py -f 0-others/taosShell.py
python3 ./test.py -f 0-others/taosShellError.py
python3 ./test.py -f 0-others/taosShellNetChk.py
python3 ./test.py -f 0-others/telemetry.py