diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0c7e31bfbb..90e841d5e0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -34,7 +34,7 @@ endif(${BUILD_TEST})
add_subdirectory(source)
add_subdirectory(tools)
-add_subdirectory(tests)
+add_subdirectory(utils)
add_subdirectory(examples/c)
# docs
diff --git a/cmake/cmake.platform b/cmake/cmake.platform
index 3aa1ffc07e..e4d440d76e 100644
--- a/cmake/cmake.platform
+++ b/cmake/cmake.platform
@@ -87,7 +87,7 @@ IF ("${CPUTYPE}" STREQUAL "")
SET(TD_ARM_32 TRUE)
ADD_DEFINITIONS("-D_TD_ARM_")
ADD_DEFINITIONS("-D_TD_ARM_32")
- ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
+ ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)")
MESSAGE(STATUS "The current platform is aarch64")
SET(PLATFORM_ARCH_STR "arm64")
SET(TD_ARM_64 TRUE)
diff --git a/cmake/cmake.version b/cmake/cmake.version
index db29644b38..7c895b12ff 100644
--- a/cmake/cmake.version
+++ b/cmake/cmake.version
@@ -2,7 +2,7 @@
IF (DEFINED VERNUMBER)
SET(TD_VER_NUMBER ${VERNUMBER})
ELSE ()
- SET(TD_VER_NUMBER "3.0.0.1")
+ SET(TD_VER_NUMBER "3.0.1.0")
ENDIF ()
IF (DEFINED VERCOMPATIBLE)
diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in
index f182beed33..eb0faf6d5d 100644
--- a/cmake/taosadapter_CMakeLists.txt.in
+++ b/cmake/taosadapter_CMakeLists.txt.in
@@ -2,7 +2,7 @@
# taosadapter
ExternalProject_Add(taosadapter
GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
- GIT_TAG abed566
+ GIT_TAG 71e7ccf
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index 3c7067a9f8..c273e9889f 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
# taos-tools
ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
- GIT_TAG f169c0f
+ GIT_TAG 2dba49c
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in
index 54e898ecc0..04b1262caf 100644
--- a/cmake/taosws_CMakeLists.txt.in
+++ b/cmake/taosws_CMakeLists.txt.in
@@ -2,7 +2,7 @@
# taosws-rs
ExternalProject_Add(taosws-rs
GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git
- GIT_TAG 6fc47d7
+ GIT_TAG e771403
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
diff --git a/docs/en/04-concept/index.md b/docs/en/04-concept/index.md
index b0a0c25d85..0b1b226c17 100644
--- a/docs/en/04-concept/index.md
+++ b/docs/en/04-concept/index.md
@@ -6,101 +6,100 @@ In order to explain the basic concepts and provide some sample code, the TDengin
-
- Device ID
- Time Stamp
- Collected Metrics
- Tags
+
+
+ Device ID
+ Timestamp
+ Collected Metrics
+ Tags
-
-Device ID
-Time Stamp
-current
-voltage
-phase
-location
-groupId
-
-
-
-
-d1001
-1538548685000
-10.3
-219
-0.31
-California.SanFrancisco
-2
-
-
-d1002
-1538548684000
-10.2
-220
-0.23
-California.SanFrancisco
-3
-
-
-d1003
-1538548686500
-11.5
-221
-0.35
-California.LosAngeles
-3
-
-
-d1004
-1538548685500
-13.4
-223
-0.29
-California.LosAngeles
-2
-
-
-d1001
-1538548695000
-12.6
-218
-0.33
-California.SanFrancisco
-2
-
-
-d1004
-1538548696600
-11.8
-221
-0.28
-California.LosAngeles
-2
-
-
-d1002
-1538548696650
-10.3
-218
-0.25
-California.SanFrancisco
-3
-
-
-d1001
-1538548696800
-12.3
-221
-0.31
-California.SanFrancisco
-2
-
-
+
+ current
+ voltage
+ phase
+ location
+ groupid
+
+
+
+
+ d1001
+ 1538548685000
+ 10.3
+ 219
+ 0.31
+ California.SanFrancisco
+ 2
+
+
+ d1002
+ 1538548684000
+ 10.2
+ 220
+ 0.23
+ California.SanFrancisco
+ 3
+
+
+ d1003
+ 1538548686500
+ 11.5
+ 221
+ 0.35
+ California.LosAngeles
+ 3
+
+
+ d1004
+ 1538548685500
+ 13.4
+ 223
+ 0.29
+ California.LosAngeles
+ 2
+
+
+ d1001
+ 1538548695000
+ 12.6
+ 218
+ 0.33
+ California.SanFrancisco
+ 2
+
+
+ d1004
+ 1538548696600
+ 11.8
+ 221
+ 0.28
+ California.LosAngeles
+ 2
+
+
+ d1002
+ 1538548696650
+ 10.3
+ 218
+ 0.25
+ California.SanFrancisco
+ 3
+
+
+ d1001
+ 1538548696800
+ 12.3
+ 221
+ 0.31
+ California.SanFrancisco
+ 2
+
+
Table 1: Smart meter example data
-Each row contains the device ID, time stamp, collected metrics (current, voltage, phase as above), and static tags (location and groupId in Table 1) associated with the devices. Each smart meter generates a row (measurement) in a pre-defined time interval or triggered by an external event. The device produces a sequence of measurements with associated time stamps.
+Each row contains the device ID, timestamp, collected metrics (`current`, `voltage`, `phase` as above), and static tags (`location` and `groupid` in Table 1) associated with the devices. Each smart meter generates a row (measurement) in a pre-defined time interval or triggered by an external event. The device produces a sequence of measurements with associated timestamps.
## Metric
@@ -112,22 +111,22 @@ Label/Tag refers to the static properties of sensors, equipment or other types o
## Data Collection Point
-Data Collection Point (DCP) refers to hardware or software that collects metrics based on preset time periods or triggered by events. A data collection point can collect one or multiple metrics, but these metrics are collected at the same time and have the same time stamp. For some complex equipment, there are often multiple data collection points, and the sampling rate of each collection point may be different, and fully independent. For example, for a car, there could be a data collection point to collect GPS position metrics, a data collection point to collect engine status metrics, and a data collection point to collect the environment metrics inside the car. So in this example the car would have three data collection points. In the smart meters example, d1001, d1002, d1003, and d1004 are the data collection points.
+Data Collection Point (DCP) refers to hardware or software that collects metrics based on preset time periods or triggered by events. A data collection point can collect one or multiple metrics, but these metrics are collected at the same time and have the same timestamp. For some complex equipment, there are often multiple data collection points, and the sampling rate of each collection point may be different, and fully independent. For example, for a car, there could be a data collection point to collect GPS position metrics, a data collection point to collect engine status metrics, and a data collection point to collect the environment metrics inside the car. So in this example the car would have three data collection points. In the smart meters example, d1001, d1002, d1003, and d1004 are the data collection points.
## Table
Since time-series data is most likely to be structured data, TDengine adopts the traditional relational database model to process them with a short learning curve. You need to create a database, create tables, then insert data points and execute queries to explore the data.
-To make full use of time-series data characteristics, TDengine adopts a strategy of "**One Table for One Data Collection Point**". TDengine requires the user to create a table for each data collection point (DCP) to store collected time-series data. For example, if there are over 10 million smart meters, it means 10 million tables should be created. For the table above, 4 tables should be created for devices D1001, D1002, D1003, and D1004 to store the data collected. This design has several benefits:
+To make full use of time-series data characteristics, TDengine adopts a strategy of "**One Table for One Data Collection Point**". TDengine requires the user to create a table for each data collection point (DCP) to store collected time-series data. For example, if there are over 10 million smart meters, it means 10 million tables should be created. For the table above, 4 tables should be created for devices d1001, d1002, d1003, and d1004 to store the data collected. This design has several benefits:
1. Since the metric data from different DCP are fully independent, the data source of each DCP is unique, and a table has only one writer. In this way, data points can be written in a lock-free manner, and the writing speed can be greatly improved.
2. For a DCP, the metric data generated by DCP is ordered by timestamp, so the write operation can be implemented by simple appending, which further greatly improves the data writing speed.
3. The metric data from a DCP is continuously stored, block by block. If you read data for a period of time, it can greatly reduce random read operations and improve read and query performance by orders of magnitude.
4. Inside a data block for a DCP, columnar storage is used, and different compression algorithms are used for different data types. Metrics generally don't vary as significantly between themselves over a time range as compared to other metrics, which allows for a higher compression rate.
-If the metric data of multiple DCPs are traditionally written into a single table, due to uncontrollable network delays, the timing of the data from different DCPs arriving at the server cannot be guaranteed, write operations must be protected by locks, and metric data from one DCP cannot be guaranteed to be continuously stored together. ** One table for one data collection point can ensure the best performance of insert and query of a single data collection point to the greatest possible extent.**
+If the metric data of multiple DCPs are traditionally written into a single table, due to uncontrollable network delays, the timing of the data from different DCPs arriving at the server cannot be guaranteed, write operations must be protected by locks, and metric data from one DCP cannot be guaranteed to be continuously stored together. **One table for one data collection point can ensure the best performance of insert and query of a single data collection point to the greatest possible extent.**
-TDengine suggests using DCP ID as the table name (like D1001 in the above table). Each DCP may collect one or multiple metrics (like the current, voltage, phase as above). Each metric has a corresponding column in the table. The data type for a column can be int, float, string and others. In addition, the first column in the table must be a timestamp. TDengine uses the time stamp as the index, and won’t build the index on any metrics stored. Column wise storage is used.
+TDengine suggests using DCP ID as the table name (like d1001 in the above table). Each DCP may collect one or multiple metrics (like the `current`, `voltage`, `phase` as above). Each metric has a corresponding column in the table. The data type for a column can be int, float, string and others. In addition, the first column in the table must be a timestamp. TDengine uses the timestamp as the index, and won’t build the index on any metrics stored. Column wise storage is used.
Complex devices, such as connected cars, may have multiple DCPs. In this case, multiple tables are created for a single device, one table per DCP.
@@ -156,9 +155,16 @@ The relationship between a STable and the subtables created based on this STable
Queries can be executed on both a table (subtable) and a STable. For a query on a STable, TDengine will treat the data in all its subtables as a whole data set for processing. TDengine will first find the subtables that meet the tag filter conditions, then scan the time-series data of these subtables to perform aggregation operation, which reduces the number of data sets to be scanned which in turn greatly improves the performance of data aggregation across multiple DCPs. In essence, querying a supertable is a very efficient aggregate query on multiple DCPs of the same type.
-In TDengine, it is recommended to use a subtable instead of a regular table for a DCP. In the smart meters example, we can create subtables like d1001, d1002, d1003, and d1004 under super table meters.
+In TDengine, it is recommended to use a subtable instead of a regular table for a DCP. In the smart meters example, we can create subtables like d1001, d1002, d1003, and d1004 under super table `meters`.
-To better understand the data model using metri, tags, super table and subtable, please refer to the diagram below which demonstrates the data model of the smart meters example. 
+To better understand the data model using metrics, tags, super table and subtable, please refer to the diagram below which demonstrates the data model of the smart meters example.
+
+
+
+
+
+Figure 1. Meters Data Model Diagram
## Database
@@ -172,4 +178,4 @@ FQDN (Fully Qualified Domain Name) is the full domain name of a specific compute
Each node of a TDengine cluster is uniquely identified by an End Point, which consists of an FQDN and a Port, such as h1.tdengine.com:6030. In this way, when the IP changes, we can still use the FQDN to dynamically find the node without changing any configuration of the cluster. In addition, FQDN is used to facilitate unified access to the same cluster from the Intranet and the Internet.
-TDengine does not recommend using an IP address to access the cluster. FQDN is recommended for cluster management.
+TDengine does not recommend using an IP address to access the cluster. FQDN is recommended for cluster management.
diff --git a/docs/en/05-get-started/01-docker.md b/docs/en/05-get-started/01-docker.md
index de5b620a77..66f7d5d594 100644
--- a/docs/en/05-get-started/01-docker.md
+++ b/docs/en/05-get-started/01-docker.md
@@ -13,7 +13,7 @@ If Docker is already installed on your computer, run the following command:
docker run -d -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043-6049/udp tdengine/tdengine
```
-Note that TDengine Server uses TCP port 6030. Port 6041 is used by taosAdapter for the REST API service. Ports 6043 through 6049 are used by taosAdapter for other connectors. You can open these ports as needed.
+Note that TDengine Server 3.0 uses TCP port 6030. Port 6041 is used by taosAdapter for the REST API service. Ports 6043 through 6049 are used by taosAdapter for other connectors. You can open these ports as needed.
Run the following command to ensure that your container is running:
@@ -21,7 +21,7 @@ Run the following command to ensure that your container is running:
docker ps
```
-Enter the container and open the bash shell:
+Enter the container and open the `bash` shell:
```shell
docker exec -it bash
@@ -31,68 +31,68 @@ You can now access TDengine or run other Linux commands.
Note: For information about installing docker, see the [official documentation](https://docs.docker.com/get-docker/).
-## Insert Data into TDengine
-
-You can use the `taosBenchmark` tool included with TDengine to write test data into your deployment.
-
-To do so, run the following command:
-
- ```bash
- $ taosBenchmark
-
- ```
-
-This command creates the `meters` supertable in the `test` database. In the `meters` supertable, it then creates 10,000 subtables named `d0` to `d9999`. Each table has 10,000 rows and each row has four columns: `ts`, `current`, `voltage`, and `phase`. The timestamps of the data in these columns range from 2017-07-14 10:40:00 000 to 2017-07-14 10:40:09 999. Each table is randomly assigned a `groupId` tag from 1 to 10 and a `location` tag of either `Campbell`, `Cupertino`, `Los Angeles`, `Mountain View`, `Palo Alto`, `San Diego`, `San Francisco`, `San Jose`, `Santa Clara` or `Sunnyvale`.
-
- The `taosBenchmark` command creates a deployment with 100 million data points that you can use for testing purposes. The time required depends on the hardware specifications of the local system.
-
- You can customize the test deployment that taosBenchmark creates by specifying command-line parameters. For information about command-line parameters, run the `taosBenchmark --help` command. For more information about taosBenchmark, see [taosBenchmark](/reference/taosbenchmark).
-
## Open the TDengine CLI
-On the container, run the following command to open the TDengine CLI:
+On the container, run the following command to open the TDengine CLI:
```
$ taos
-taos>
+taos>
```
-## Query Data in TDengine
+## Test data insert performance
-After using taosBenchmark to create your test deployment, you can run queries in the TDengine CLI to test its performance. For example:
+After your TDengine Server is running normally, you can run the taosBenchmark utility to test its performance:
-From the TDengine CLI query the number of rows in the `meters` supertable:
+Start TDengine service and execute `taosBenchmark` (formerly named `taosdemo`) in a Linux or Windows terminal.
+
+```bash
+taosBenchmark
+```
+
+This command creates the `meters` supertable in the `test` database. In the `meters` supertable, it then creates 10,000 subtables named `d0` to `d9999`. Each table has 10,000 rows and each row has four columns: `ts`, `current`, `voltage`, and `phase`. The timestamps of the data in these columns range from 2017-07-14 10:40:00 000 to 2017-07-14 10:40:09 999. Each table is randomly assigned a `groupId` tag from 1 to 10 and a `location` tag of either `Campbell`, `Cupertino`, `Los Angeles`, `Mountain View`, `Palo Alto`, `San Diego`, `San Francisco`, `San Jose`, `Santa Clara` or `Sunnyvale`.
+
+The `taosBenchmark` command creates a deployment with 100 million data points that you can use for testing purposes. The time required to create the deployment depends on your hardware. On most modern servers, the deployment is created in ten to twenty seconds.
+
+You can customize the test deployment that taosBenchmark creates by specifying command-line parameters. For information about command-line parameters, run the `taosBenchmark --help` command. For more information about taosBenchmark, see [taosBenchmark](../../reference/taosbenchmark).
+
+## Test data query performance
+
+After using `taosBenchmark` to create your test deployment, you can run queries in the TDengine CLI to test its performance:
+
+From the TDengine CLI (taos) query the number of rows in the `meters` supertable:
```sql
-select count(*) from test.meters;
+SELECT COUNT(*) FROM test.meters;
```
Query the average, maximum, and minimum values of all 100 million rows of data:
```sql
-select avg(current), max(voltage), min(phase) from test.meters;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters;
```
Query the number of rows whose `location` tag is `San Francisco`:
```sql
-select count(*) from test.meters where location="San Francisco";
+SELECT COUNT(*) FROM test.meters WHERE location = "San Francisco";
```
Query the average, maximum, and minimum values of all rows whose `groupId` tag is `10`:
```sql
-select avg(current), max(voltage), min(phase) from test.meters where groupId=10;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 10;
```
-Query the average, maximum, and minimum values for table `d10` in 1 second intervals:
+Query the average, maximum, and minimum values for table `d10` in 10 second intervals:
```sql
-select first(ts), avg(current), max(voltage), min(phase) from test.d10 interval(1s);
+SELECT FIRST(ts), AVG(current), MAX(voltage), MIN(phase) FROM test.d10 INTERVAL(10s);
```
-In the query above you are selecting the first timestamp (ts) in the interval, another way of selecting this would be _wstart which will give the start of the time window. For more information about windowed queries, see [Time-Series Extensions](../../taos-sql/distinguished/).
+
+In the query above you are selecting the first timestamp (ts) in the interval, another way of selecting this would be `\_wstart` which will give the start of the time window. For more information about windowed queries, see [Time-Series Extensions](../../taos-sql/distinguished/).
## Additional Information
diff --git a/docs/en/05-get-started/03-package.md b/docs/en/05-get-started/03-package.md
index 88096a759c..7257fccc80 100644
--- a/docs/en/05-get-started/03-package.md
+++ b/docs/en/05-get-started/03-package.md
@@ -9,23 +9,24 @@ import PkgListV3 from "/components/PkgListV3";
For information about installing TDengine on Docker, see [Quick Install on Docker](../../get-started/docker). If you want to view the source code, build TDengine yourself, or contribute to the project, see the [TDengine GitHub repository](https://github.com/taosdata/TDengine).
-The full package of TDengine includes the TDengine Server (`taosd`), TDengine Client (`taosc`), taosAdapter for connecting with third-party systems and providing a RESTful interface, a command-line interface, and some tools. Note that taosAdapter supports Linux only. In addition to connectors for multiple languages, TDengine also provides a [REST API](../../reference/rest-api) through [taosAdapter](../../reference/taosadapter).
+The full package of TDengine includes the TDengine Server (`taosd`), TDengine Client (`taosc`), taosAdapter for connecting with third-party systems and providing a RESTful interface, a command-line interface (CLI, taos), and some tools. Note that taosAdapter supports Linux only. In addition to connectors for multiple languages, TDengine also provides a [REST API](../../reference/rest-api) through [taosAdapter](../../reference/taosadapter).
-The standard server installation package includes `taos`, `taosd`, `taosAdapter`, `taosBenchmark`, and sample code. You can also download a lite package that includes only `taosd` and the C/C++ connector.
+The standard server installation package includes `taos`, `taosd`, `taosAdapter`, `taosBenchmark`, and sample code. You can also download the Lite package that includes only `taosd` and the C/C++ connector.
-The TDengine Community Edition is released as .deb and .rpm packages. The .deb package can be installed on Debian, Ubuntu, and derivative systems. The .rpm package can be installed on CentOS, RHEL, SUSE, and derivative systems. A .tar.gz package is also provided for enterprise customers, and you can install TDengine over `apt-get` as well. The .tar.tz package includes `taosdump` and the TDinsight installation script. If you want to use these utilities with the .deb or .rpm package, download and install taosTools separately. TDengine can also be installed on 64-bit Windows servers.
+The TDengine Community Edition is released as Deb and RPM packages. The Deb package can be installed on Debian, Ubuntu, and derivative systems. The RPM package can be installed on CentOS, RHEL, SUSE, and derivative systems. A .tar.gz package is also provided for enterprise customers, and you can install TDengine over `apt-get` as well. The .tar.tz package includes `taosdump` and the TDinsight installation script. If you want to use these utilities with the Deb or RPM package, download and install taosTools separately. TDengine can also be installed on 64-bit Windows.
## Installation
-1. Download the .deb installation package.
-
+1. Download the Deb installation package.
+
2. In the directory where the package is located, use `dpkg` to install the package:
+> Please replace `` with the corresponding version of the package downloaded
+
```bash
-# Enter the name of the package that you downloaded.
sudo dpkg -i TDengine-server--Linux-x64.deb
```
@@ -34,11 +35,12 @@ sudo dpkg -i TDengine-server--Linux-x64.deb
1. Download the .rpm installation package.
-
+
2. In the directory where the package is located, use rpm to install the package:
+> Please replace `` with the corresponding version of the package downloaded
+
```bash
-# Enter the name of the package that you downloaded.
sudo rpm -ivh TDengine-server--Linux-x64.rpm
```
@@ -47,11 +49,12 @@ sudo rpm -ivh TDengine-server--Linux-x64.rpm
1. Download the .tar.gz installation package.
-
+
2. In the directory where the package is located, use `tar` to decompress the package:
+> Please replace `` with the corresponding version of the package downloaded
+
```bash
-# Enter the name of the package that you downloaded.
tar -zxvf TDengine-server--Linux-x64.tar.gz
```
@@ -96,23 +99,23 @@ sudo apt-get install tdengine
This installation method is supported only for Debian and Ubuntu.
::::
-
+
-Note: TDengine only supports Windows Server 2016/2019 and windows 10/11 system versions on the windows platform.
+Note: TDengine only supports Windows Server 2016/2019 and Windows 10/11 on the Windows platform.
1. Download the Windows installation package.
-
+
2. Run the downloaded package to install TDengine.
:::info
-For information about TDengine releases, see [Release History](../../releases).
+For information about TDengine releases, see [Release History](../../releases).
:::
:::note
-On the first node in your TDengine cluster, leave the `Enter FQDN:` prompt blank and press **Enter**. On subsequent nodes, you can enter the end point of the first dnode in the cluster. You can also configure this setting after you have finished installing TDengine.
+On the first node in your TDengine cluster, leave the `Enter FQDN:` prompt blank and press **Enter**. On subsequent nodes, you can enter the endpoint of the first dnode in the cluster. You can also configure this setting after you have finished installing TDengine.
:::
@@ -147,7 +150,7 @@ Active: inactive (dead)
After confirming that TDengine is running, run the `taos` command to access the TDengine CLI.
-The following `systemctl` commands can help you manage TDengine:
+The following `systemctl` commands can help you manage TDengine service:
- Start TDengine Server: `systemctl start taosd`
@@ -159,7 +162,7 @@ The following `systemctl` commands can help you manage TDengine:
:::info
-- The `systemctl` command requires _root_ privileges. If you are not logged in as the `root` user, use the `sudo` command.
+- The `systemctl` command requires _root_ privileges. If you are not logged in as the _root_ user, use the `sudo` command.
- The `systemctl stop taosd` command does not instantly stop TDengine Server. The server is stopped only after all data in memory is flushed to disk. The time required depends on the cache size.
- If your system does not include `systemd`, you can run `/usr/local/taos/bin/taosd` to start TDengine manually.
@@ -174,23 +177,9 @@ After the installation is complete, run `C:\TDengine\taosd.exe` to start TDengin
-## Test data insert performance
+## Command Line Interface (CLI)
-After your TDengine Server is running normally, you can run the taosBenchmark utility to test its performance:
-
-```bash
-taosBenchmark
-```
-
-This command creates the `meters` supertable in the `test` database. In the `meters` supertable, it then creates 10,000 subtables named `d0` to `d9999`. Each table has 10,000 rows and each row has four columns: `ts`, `current`, `voltage`, and `phase`. The timestamps of the data in these columns range from 2017-07-14 10:40:00 000 to 2017-07-14 10:40:09 999. Each table is randomly assigned a `groupId` tag from 1 to 10 and a `location` tag of either `Campbell`, `Cupertino`, `Los Angeles`, `Mountain View`, `Palo Alto`, `San Diego`, `San Francisco`, `San Jose`, `Santa Clara` or `Sunnyvale`.
-
-The `taosBenchmark` command creates a deployment with 100 million data points that you can use for testing purposes. The time required to create the deployment depends on your hardware. On most modern servers, the deployment is created in less than a minute.
-
-You can customize the test deployment that taosBenchmark creates by specifying command-line parameters. For information about command-line parameters, run the `taosBenchmark --help` command. For more information about taosBenchmark, see [taosBenchmark](../../reference/taosbenchmark).
-
-## Command Line Interface
-
-You can use the TDengine CLI to monitor your TDengine deployment and execute ad hoc queries. To open the CLI, run the following command:
+You can use the TDengine CLI to monitor your TDengine deployment and execute ad hoc queries. To open the CLI, you can execute `taos` in the Linux terminal where TDengine is installed, or you can run `taos.exe` in the `C:\TDengine` directory of the Windows terminal where TDengine is installed to start the TDengine command line.
```bash
taos
@@ -205,52 +194,71 @@ taos>
For example, you can create and delete databases and tables and run all types of queries. Each SQL command must be end with a semicolon (;). For example:
```sql
-create database demo;
-use demo;
-create table t (ts timestamp, speed int);
-insert into t values ('2019-07-15 00:00:00', 10);
-insert into t values ('2019-07-15 01:00:00', 20);
-select * from t;
+CREATE DATABASE demo;
+USE demo;
+CREATE TABLE t (ts TIMESTAMP, speed INT);
+INSERT INTO t VALUES ('2019-07-15 00:00:00', 10);
+INSERT INTO t VALUES ('2019-07-15 01:00:00', 20);
+SELECT * FROM t;
+
ts | speed |
========================================
2019-07-15 00:00:00.000 | 10 |
2019-07-15 01:00:00.000 | 20 |
+
Query OK, 2 row(s) in set (0.003128s)
```
You can also can monitor the deployment status, add and remove user accounts, and manage running instances. You can run the TDengine CLI on either Linux or Windows machines. For more information, see [TDengine CLI](../../reference/taos-shell/).
-
+
+## Test data insert performance
+
+After your TDengine Server is running normally, you can run the taosBenchmark utility to test its performance:
+
+Start TDengine service and execute `taosBenchmark` (formerly named `taosdemo`) in a Linux or Windows terminal.
+
+```bash
+taosBenchmark
+```
+
+This command creates the `meters` supertable in the `test` database. In the `meters` supertable, it then creates 10,000 subtables named `d0` to `d9999`. Each table has 10,000 rows and each row has four columns: `ts`, `current`, `voltage`, and `phase`. The timestamps of the data in these columns range from 2017-07-14 10:40:00 000 to 2017-07-14 10:40:09 999. Each table is randomly assigned a `groupId` tag from 1 to 10 and a `location` tag of either `Campbell`, `Cupertino`, `Los Angeles`, `Mountain View`, `Palo Alto`, `San Diego`, `San Francisco`, `San Jose`, `Santa Clara` or `Sunnyvale`.
+
+The `taosBenchmark` command creates a deployment with 100 million data points that you can use for testing purposes. The time required to create the deployment depends on your hardware. On most modern servers, the deployment is created in ten to twenty seconds.
+
+You can customize the test deployment that taosBenchmark creates by specifying command-line parameters. For information about command-line parameters, run the `taosBenchmark --help` command. For more information about taosBenchmark, see [taosBenchmark](../../reference/taosbenchmark).
+
## Test data query performance
-After using taosBenchmark to create your test deployment, you can run queries in the TDengine CLI to test its performance:
+After using `taosBenchmark` to create your test deployment, you can run queries in the TDengine CLI to test its performance:
-From the TDengine CLI query the number of rows in the `meters` supertable:
+From the TDengine CLI (taos) query the number of rows in the `meters` supertable:
```sql
-select count(*) from test.meters;
+SELECT COUNT(*) FROM test.meters;
```
Query the average, maximum, and minimum values of all 100 million rows of data:
```sql
-select avg(current), max(voltage), min(phase) from test.meters;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters;
```
Query the number of rows whose `location` tag is `San Francisco`:
```sql
-select count(*) from test.meters where location="San Francisco";
+SELECT COUNT(*) FROM test.meters WHERE location = "San Francisco";
```
Query the average, maximum, and minimum values of all rows whose `groupId` tag is `10`:
```sql
-select avg(current), max(voltage), min(phase) from test.meters where groupId=10;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 10;
```
-Query the average, maximum, and minimum values for table `d10` in 1 second intervals:
+Query the average, maximum, and minimum values for table `d10` in 10 second intervals:
```sql
-select first(ts), avg(current), max(voltage), min(phase) from test.d10 interval(1s);
+SELECT FIRST(ts), AVG(current), MAX(voltage), MIN(phase) FROM test.d10 INTERVAL(10s);
```
-In the query above you are selecting the first timestamp (ts) in the interval, another way of selecting this would be _wstart which will give the start of the time window. For more information about windowed queries, see [Time-Series Extensions](../../taos-sql/distinguished/).
+
+In the query above you are selecting the first timestamp (ts) in the interval, another way of selecting this would be `\_wstart` which will give the start of the time window. For more information about windowed queries, see [Time-Series Extensions](../../taos-sql/distinguished/).
diff --git a/docs/examples/csharp/SQLInsertExample.cs b/docs/examples/csharp/SQLInsertExample.cs
index 192ea96d57..3ce70fe914 100644
--- a/docs/examples/csharp/SQLInsertExample.cs
+++ b/docs/examples/csharp/SQLInsertExample.cs
@@ -23,6 +23,7 @@ namespace TDengineExample
CheckRes(conn, res, "failed to insert data");
int affectedRows = TDengine.AffectRows(res);
Console.WriteLine("affectedRows " + affectedRows);
+ TDengine.FreeResult(res);
ExitProgram(conn, 0);
}
diff --git a/docs/zh/02-intro.md b/docs/zh/02-intro.md
index 9a0a6fb547..47bfd3f96b 100644
--- a/docs/zh/02-intro.md
+++ b/docs/zh/02-intro.md
@@ -48,7 +48,7 @@ TDengine 的主要功能如下：
- 多种[数据导出](../operation/export)方式
9. 工具
- 提供[交互式命令行程序（CLI）](../reference/taos-shell)，便于管理集群，检查系统状态，做即席查询
- - 提供压力测试工具[taosBenchmark](../reference/taosbenchmark)，用于测试 TDengine 的性能
+ - 提供压力测试工具 [taosBenchmark](../reference/taosbenchmark)，用于测试 TDengine 的性能
10. 编程
- 提供各种语言的[连接器（Connector）](../connector): 如 [C/C++](../connector/cpp)、[Java](../connector/java)、[Go](../connector/go)、[Node.js](../connector/node)、[Rust](../connector/rust)、[Python](../connector/python)、[C#](../connector/csharp) 等
- 支持 [REST 接口](../connector/rest-api/)
diff --git a/docs/zh/04-concept/index.md b/docs/zh/04-concept/index.md
index 89d3df9c97..2cba68edcd 100644
--- a/docs/zh/04-concept/index.md
+++ b/docs/zh/04-concept/index.md
@@ -4,119 +4,118 @@ title: 数据模型和基本概念
description: TDengine 的数据模型和基本概念
---
-为了便于解释基本概念，便于撰写示例程序，整个 TDengine 文档以智能电表作为典型时序数据场景。假设每个智能电表采集电流、电压、相位三个量，有多个智能电表，每个电表有位置 location 和分组 group ID 的静态属性. 其采集的数据类似如下的表格：
+为了便于解释基本概念，便于撰写示例程序，整个 TDengine 文档以智能电表作为典型时序数据场景。假设每个智能电表采集电流、电压、相位三个量，有多个智能电表，每个电表有位置 Location 和分组 Group ID 的静态属性. 其采集的数据类似如下的表格：
-
- Device ID
- Time Stamp
- Collected Metrics
- Tags
+
+
+ Device ID
+ Timestamp
+ Collected Metrics
+ Tags
-
-Device ID
-Time Stamp
-current
-voltage
-phase
-location
-groupId
-
-
-
-
-d1001
-1538548685000
-10.3
-219
-0.31
-California.SanFrancisco
-2
-
-
-d1002
-1538548684000
-10.2
-220
-0.23
-California.SanFrancisco
-3
-
-
-d1003
-1538548686500
-11.5
-221
-0.35
-California.LosAngeles
-3
-
-
-d1004
-1538548685500
-13.4
-223
-0.29
-California.LosAngeles
-2
-
-
-d1001
-1538548695000
-12.6
-218
-0.33
-California.SanFrancisco
-2
-
-
-d1004
-1538548696600
-11.8
-221
-0.28
-California.LosAngeles
-2
-
-
-d1002
-1538548696650
-10.3
-218
-0.25
-California.SanFrancisco
-3
-
-
-d1001
-1538548696800
-12.3
-221
-0.31
-California.SanFrancisco
-2
-
-
+
+ current
+ voltage
+ phase
+ location
+ groupid
+
+
+
+
+ d1001
+ 1538548685000
+ 10.3
+ 219
+ 0.31
+ California.SanFrancisco
+ 2
+
+
+ d1002
+ 1538548684000
+ 10.2
+ 220
+ 0.23
+ California.SanFrancisco
+ 3
+
+
+ d1003
+ 1538548686500
+ 11.5
+ 221
+ 0.35
+ California.LosAngeles
+ 3
+
+
+ d1004
+ 1538548685500
+ 13.4
+ 223
+ 0.29
+ California.LosAngeles
+ 2
+
+
+ d1001
+ 1538548695000
+ 12.6
+ 218
+ 0.33
+ California.SanFrancisco
+ 2
+
+
+ d1004
+ 1538548696600
+ 11.8
+ 221
+ 0.28
+ California.LosAngeles
+ 2
+
+
+ d1002
+ 1538548696650
+ 10.3
+ 218
+ 0.25
+ California.SanFrancisco
+ 3
+
+
+ d1001
+ 1538548696800
+ 12.3
+ 221
+ 0.31
+ California.SanFrancisco
+ 2
+
+
-
表 1：智能电表数据示例
+
表 1. 智能电表数据示例
-每一条记录都有设备 ID，时间戳，采集的物理量以及每个设备相关的静态标签。每个设备是受外界的触发，或按照设定的周期采集数据。采集的数据点是时序的，是一个数据流。
+每一条记录都有设备 ID、时间戳、采集的物理量（如上表中的 `current`、`voltage` 和 `phase`）以及每个设备相关的静态标签（`location` 和 `groupid`）。每个设备是受外界的触发，或按照设定的周期采集数据。采集的数据点是时序的，是一个数据流。
-## 采集量 (Metric)
+## 采集量（Metric）
采集量是指传感器、设备或其他类型采集点采集的物理量，比如电流、电压、温度、压力、GPS 位置等，是随时间变化的，数据类型可以是整型、浮点型、布尔型，也可是字符串。随着时间的推移，存储的采集量的数据量越来越大。智能电表示例中的电流、电压、相位就是采集量。
-## 标签 (Label/Tag)
+## 标签（Label/Tag）
-标签是指传感器、设备或其他类型采集点的静态属性，不是随时间变化的，比如设备型号、颜色、设备的所在地等，数据类型可以是任何类型。虽然是静态的，但 TDengine 容许用户修改、删除或增加标签值。与采集量不一样的是，随时间的推移，存储的标签的数据量不会有什么变化。智能电表示例中的location与groupId就是标签。
+标签是指传感器、设备或其他类型采集点的静态属性，不是随时间变化的，比如设备型号、颜色、设备的所在地等，数据类型可以是任何类型。虽然是静态的，但 TDengine 容许用户修改、删除或增加标签值。与采集量不一样的是，随时间的推移，存储的标签的数据量不会有什么变化。智能电表示例中的 `location` 与 `groupid` 就是标签。
-## 数据采集点 (Data Collection Point)
+## 数据采集点（Data Collection Point）
-数据采集点是指按照预设时间周期或受事件触发采集物理量的硬件或软件。一个数据采集点可以采集一个或多个采集量，**但这些采集量都是同一时刻采集的，具有相同的时间戳**。对于复杂的设备，往往有多个数据采集点，每个数据采集点采集的周期都可能不一样，而且完全独立，不同步。比如对于一台汽车，有数据采集点专门采集 GPS 位置，有数据采集点专门采集发动机状态，有数据采集点专门采集车内的环境，这样一台汽车就有三个数据采集点。智能电表示例中的d1001, d1002, d1003, d1004等就是数据采集点。
+数据采集点是指按照预设时间周期或受事件触发采集物理量的硬件或软件。一个数据采集点可以采集一个或多个采集量，**但这些采集量都是同一时刻采集的，具有相同的时间戳**。对于复杂的设备，往往有多个数据采集点，每个数据采集点采集的周期都可能不一样，而且完全独立，不同步。比如对于一台汽车，有数据采集点专门采集 GPS 位置，有数据采集点专门采集发动机状态，有数据采集点专门采集车内的环境，这样一台汽车就有三个数据采集点。智能电表示例中的 d1001、d1002、d1003、d1004 等就是数据采集点。
-## 表 (Table)
+## 表（Table）
因为采集量一般是结构化数据，同时为降低学习门槛，TDengine 采用传统的关系型数据库模型管理数据。用户需要先创建库，然后创建表，之后才能插入或查询数据。
@@ -129,50 +128,56 @@ description: TDengine 的数据模型和基本概念
如果采用传统的方式，将多个数据采集点的数据写入一张表，由于网络延时不可控，不同数据采集点的数据到达服务器的时序是无法保证的，写入操作是要有锁保护的，而且一个数据采集点的数据是难以保证连续存储在一起的。**采用一个数据采集点一张表的方式，能最大程度的保证单个数据采集点的插入和查询的性能是最优的。**
-TDengine 建议用数据采集点的名字（如上表中的 D1001）来做表名。每个数据采集点可能同时采集多个采集量（如上表中的 current，voltage，phase），每个采集量对应一张表中的一列，数据类型可以是整型、浮点型、字符串等。除此之外，表的第一列必须是时间戳，即数据类型为 timestamp。对采集量，TDengine 将自动按照时间戳建立索引，但对采集量本身不建任何索引。数据用列式存储方式保存。
+TDengine 建议用数据采集点的名字（如上表中的 d1001）来做表名。每个数据采集点可能同时采集多个采集量（如上表中的 `current`、`voltage` 和 `phase`），每个采集量对应一张表中的一列，数据类型可以是整型、浮点型、字符串等。除此之外，表的第一列必须是时间戳，即数据类型为 Timestamp。对采集量，TDengine 将自动按照时间戳建立索引，但对采集量本身不建任何索引。数据用列式存储方式保存。
-对于复杂的设备，比如汽车，它有多个数据采集点，那么就需要为一台汽车建立多张表。
+对于复杂的设备，比如汽车，它有多个数据采集点，那么就需要为一辆汽车建立多张表。
-
-## 超级表 (STable)
+## 超级表（STable）
由于一个数据采集点一张表，导致表的数量巨增，难以管理，而且应用经常需要做采集点之间的聚合操作，聚合的操作也变得复杂起来。为解决这个问题，TDengine 引入超级表（Super Table，简称为 STable）的概念。
-超级表是指某一特定类型的数据采集点的集合。同一类型的数据采集点，其表的结构是完全一样的，但每个表（数据采集点）的静态属性（标签）是不一样的。描述一个超级表（某一特定类型的数据采集点的集合），除需要定义采集量的表结构之外，还需要定义其标签的 schema，标签的数据类型可以是整数、浮点数、字符串，标签可以有多个，可以事后增加、删除或修改。如果整个系统有 N 个不同类型的数据采集点，就需要建立 N 个超级表。
+超级表是指某一特定类型的数据采集点的集合。同一类型的数据采集点，其表的结构是完全一样的，但每个表（数据采集点）的静态属性（标签）是不一样的。描述一个超级表（某一特定类型的数据采集点的集合），除需要定义采集量的表结构之外，还需要定义其标签的 Schema，标签的数据类型可以是整数、浮点数、字符串、JSON，标签可以有多个，可以事后增加、删除或修改。如果整个系统有 N 个不同类型的数据采集点，就需要建立 N 个超级表。
-在 TDengine 的设计里，**表用来代表一个具体的数据采集点，超级表用来代表一组相同类型的数据采集点集合**。智能电表示例中，我们可以创建一个超级表meters.
+在 TDengine 的设计里，**表用来代表一个具体的数据采集点，超级表用来代表一组相同类型的数据采集点集合**。智能电表示例中，我们可以创建一个超级表 `meters`.
-## 子表 (Subtable)
+## 子表（Subtable）
当为某个具体数据采集点创建表时，用户可以使用超级表的定义做模板，同时指定该具体采集点（表）的具体标签值来创建该表。**通过超级表创建的表称之为子表**。正常的表与子表的差异在于：
-1. 子表就是表，因此所有正常表的SQL操作都可以在子表上执行。
+1. 子表就是表，因此所有正常表的 SQL 操作都可以在子表上执行。
2. 子表在正常表的基础上有扩展，它是带有静态标签的，而且这些标签可以事后增加、删除、修改，而正常的表没有。
3. 子表一定属于一张超级表，但普通表不属于任何超级表
4. 普通表无法转为子表，子表也无法转为普通表。
超级表与与基于超级表建立的子表之间的关系表现在：
-1. 一张超级表包含有多张子表，这些子表具有相同的采集量 schema，但带有不同的标签值。
+1. 一张超级表包含有多张子表，这些子表具有相同的采集量 Schema，但带有不同的标签值。
2. 不能通过子表调整数据或标签的模式，对于超级表的数据模式修改立即对所有的子表生效。
3. 超级表只定义一个模板，自身不存储任何数据或标签信息。因此，不能向一个超级表写入数据，只能将数据写入子表中。
查询既可以在表上进行，也可以在超级表上进行。针对超级表的查询，TDengine 将把所有子表中的数据视为一个整体数据集进行处理，会先把满足标签过滤条件的表从超级表中找出来，然后再扫描这些表的时序数据，进行聚合操作，这样需要扫描的数据集会大幅减少，从而显著提高查询的性能。本质上，TDengine 通过对超级表查询的支持，实现了多个同类数据采集点的高效聚合。
-TDengine系统建议给一个数据采集点建表，需要通过超级表建表，而不是建普通表。在智能电表的示例中，我们可以通过超级表meters创建子表d1001, d1002, d1003, d1004等。
+TDengine 系统建议给一个数据采集点建表，需要通过超级表建表，而不是建普通表。在智能电表的示例中，我们可以通过超级表 meters 创建子表 d1001、d1002、d1003、d1004 等。
-为了更好地理解超级与子表的关系，可以参考下面关于智能电表数据模型的示意图。 
+为了更好地理解采集量、标签、超级与子表的关系，可以参考下面关于智能电表数据模型的示意图。
-## 库 (database)
+
+
+
+
+图 1. 智能电表数据模型示意图
+
+## 库（Database）
库是指一组表的集合。TDengine 容许一个运行实例有多个库，而且每个库可以配置不同的存储策略。不同类型的数据采集点往往具有不同的数据特征，包括数据采集频率的高低，数据保留时间的长短，副本的数目，数据块的大小，是否允许更新数据等等。为了在各种场景下 TDengine 都能最大效率的工作，TDengine 建议将不同数据特征的超级表创建在不同的库里。
一个库里，可以有一到多个超级表，但一个超级表只属于一个库。一个超级表所拥有的子表全部存在一个库里。
-## FQDN & End Point
+## FQDN & Endpoint
-FQDN (fully qualified domain name, 完全限定域名)是 Internet 上特定计算机或主机的完整域名。FQDN 由两部分组成:主机名和域名。例如，假设邮件服务器的 FQDN 可能是 mail.tdengine.com。主机名是 mail，主机位于域名 tdengine.com 中。DNS(Domain Name System)，负责将 FQDN 翻译成 IP，是互联网应用的寻址方式。对于没有 DNS 的系统，可以通过配置 hosts 文件来解决。
+FQDN（Fully Qualified Domain Name，完全限定域名）是 Internet 上特定计算机或主机的完整域名。FQDN 由两部分组成:主机名和域名。例如，假设邮件服务器的 FQDN 可能是 mail.tdengine.com。主机名是 mail，主机位于域名 tdengine.com 中。DNS（Domain Name System），负责将 FQDN 翻译成 IP，是互联网应用的寻址方式。对于没有 DNS 的系统，可以通过配置 hosts 文件来解决。
-TDengine 集群的每个节点是由 End Point 来唯一标识的，End Point 是由 FQDN 外加 Port 组成，比如 h1.tdengine.com:6030。这样当 IP 发生变化的时候，我们依然可以使用 FQDN 来动态找到节点，不需要更改集群的任何配置。而且采用 FQDN，便于内网和外网对同一个集群的统一访问。
+TDengine 集群的每个节点是由 Endpoint 来唯一标识的，Endpoint 是由 FQDN 外加 Port 组成，比如 h1.tdengine.com:6030。这样当 IP 发生变化的时候，我们依然可以使用 FQDN 来动态找到节点，不需要更改集群的任何配置。而且采用 FQDN，便于内网和外网对同一个集群的统一访问。
TDengine 不建议采用直接的 IP 地址访问集群，不利于管理。不了解 FQDN 概念，请看博文[《一篇文章说清楚 TDengine 的 FQDN》](https://www.taosdata.com/blog/2020/09/11/1824.html)。
diff --git a/docs/zh/05-get-started/01-docker.md b/docs/zh/05-get-started/01-docker.md
index e2be419517..e3345fed96 100644
--- a/docs/zh/05-get-started/01-docker.md
+++ b/docs/zh/05-get-started/01-docker.md
@@ -4,11 +4,11 @@ title: 通过 Docker 快速体验 TDengine
description: 使用 Docker 快速体验 TDengine 的高效写入和查询
---
-本节首先介绍如何通过 Docker 快速体验 TDengine，然后介绍如何在 Docker 环境下体验 TDengine 的写入和查询功能。如果你不熟悉 Docker，请使用[安装包的方式快速体验](../../get-started/package/)。如果您希望为 TDengine 贡献代码或对内部技术实现感兴趣，请参考 [TDengine GitHub 主页](https://github.com/taosdata/TDengine) 下载源码构建和安装.
+本节首先介绍如何通过 Docker 快速体验 TDengine，然后介绍如何在 Docker 环境下体验 TDengine 的写入和查询功能。如果你不熟悉 Docker，请使用[安装包的方式快速体验](../../get-started/package/)。如果您希望为 TDengine 贡献代码或对内部技术实现感兴趣，请参考 [TDengine GitHub 主页](https://github.com/taosdata/TDengine)下载源码构建和安装。
## 启动 TDengine
-如果已经安装了 docker， 只需执行下面的命令。
+如果已经安装了 Docker，只需执行下面的命令：
```shell
docker run -d -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043-6049/udp tdengine/tdengine
@@ -16,84 +16,84 @@ docker run -d -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043
注意：TDengine 3.0 服务端仅使用 6030 TCP 端口。6041 为 taosAdapter 所使用提供 REST 服务端口。6043-6049 为 taosAdapter 提供第三方应用接入所使用端口，可根据需要选择是否打开。
-确定该容器已经启动并且在正常运行
+确定该容器已经启动并且在正常运行。
```shell
docker ps
```
-进入该容器并执行 bash
+进入该容器并执行 `bash`
```shell
docker exec -it bash
```
-然后就可以执行相关的 Linux 命令操作和访问 TDengine
+然后就可以执行相关的 Linux 命令操作和访问 TDengine。
-注: Docker 工具自身的下载和使用请参考 [Docker 官网文档](https://docs.docker.com/get-docker/)。
+注：Docker 工具自身的下载和使用请参考 [Docker 官网文档](https://docs.docker.com/get-docker/)。
## 运行 TDengine CLI
-进入容器，执行 taos
+进入容器，执行 `taos`：
```
$ taos
-taos>
-
+taos>
```
-## 写入数据
+## 使用 taosBenchmark 体验写入速度
-可以使用 TDengine 的自带工具 taosBenchmark 快速体验 TDengine 的写入。
+可以使用 TDengine 的自带工具 taosBenchmark 快速体验 TDengine 的写入速度。
-进入容器，启动 taosBenchmark：
+启动 TDengine 的服务，在 Linux 或 Windows 终端执行 `taosBenchmark`（曾命名为 `taosdemo`）：
- ```bash
- $ taosBenchmark
-
- ```
+```bash
+$ taosBenchmark
+```
- 该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "d0" 到 "d9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupId，groupId 被设置为 1 到 10， location 被设置为 "San Francisco" 或者 "Los Angeles"等城市名称。
+该命令将在数据库 `test` 下面自动创建一张超级表 `meters`，该超级表下有 1 万张表，表名为 `d0` 到 `d9999`，每张表有 1 万条记录，每条记录有 `ts`、`current`、`voltage`、`phase` 四个字段，时间戳从 2017-07-14 10:40:00 000 到 2017-07-14 10:40:09 999，每张表带有标签 `location` 和 `groupId`，groupId 被设置为 1 到 10，location 被设置为 `Campbell`、`Cupertino`、`Los Angeles`、`Mountain View`、`Palo Alto`、`San Diego`、`San Francisco`、`San Jose`、`Santa Clara` 或者 `Sunnyvale`。
- 这条命令很快完成 1 亿条记录的插入。具体时间取决于硬件性能。
+这条命令很快完成 1 亿条记录的插入。具体时间取决于硬件性能，即使在一台普通的 PC 服务器往往也仅需十几秒。
- taosBenchmark 命令本身带有很多选项，配置表的数目、记录条数等等，您可以设置不同参数进行体验，请执行 `taosBenchmark --help` 详细列出。taosBenchmark 详细使用方法请参照 [taosBenchmark 参考手册](../../reference/taosbenchmark)。
+taosBenchmark 命令本身带有很多选项，配置表的数目、记录条数等等，您可以设置不同参数进行体验，请执行 `taosBenchmark --help` 详细列出。taosBenchmark 详细使用方法请参照[如何使用 taosBenchmark 对 TDengine 进行性能测试](https://www.taosdata.com/2021/10/09/3111.html)和 [taosBenchmark 参考手册](../../reference/taosbenchmark)。
-## 体验查询
+## 使用 TDengine CLI 体验查询速度
-使用上述 taosBenchmark 插入数据后，可以在 TDengine CLI 输入查询命令，体验查询速度。。
+使用上述 `taosBenchmark` 插入数据后，可以在 TDengine CLI（taos）输入查询命令，体验查询速度。
-查询超级表下记录总条数：
+查询超级表 `meters` 下的记录总条数：
```sql
-taos> select count(*) from test.meters;
+SELECT COUNT(*) FROM test.meters;
```
查询 1 亿条记录的平均值、最大值、最小值等：
```sql
-taos> select avg(current), max(voltage), min(phase) from test.meters;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters;
```
-查询 location="San Francisco" 的记录总条数：
+查询 location = "San Francisco" 的记录总条数：
```sql
-taos> select count(*) from test.meters where location="San Francisco";
+SELECT COUNT(*) FROM test.meters WHERE location = "San Francisco";
```
-查询 groupId=10 的所有记录的平均值、最大值、最小值等：
+查询 groupId = 10 的所有记录的平均值、最大值、最小值等：
```sql
-taos> select avg(current), max(voltage), min(phase) from test.meters where groupId=10;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 10;
```
-对表 d10 按 10s 进行平均值、最大值和最小值聚合统计：
+对表 `d10` 按 10 每秒进行平均值、最大值和最小值聚合统计：
```sql
-taos> select avg(current), max(voltage), min(phase) from test.d10 interval(10s);
+SELECT FIRST(ts), AVG(current), MAX(voltage), MIN(phase) FROM test.d10 INTERVAL(10s);
```
+在上面的查询中，你选择的是区间内的第一个时间戳（ts），另一种选择方式是 `\_wstart`，它将给出时间窗口的开始。关于窗口查询的更多信息，参见[特色查询](../../taos-sql/distinguished/)。
+
## 其它
-更多关于在 Docker 环境下使用 TDengine 的细节，请参考 [在 Docker 下使用 TDengine](../../reference/docker)
+更多关于在 Docker 环境下使用 TDengine 的细节，请参考 [在 Docker 下使用 TDengine](../../reference/docker)。
diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md
index 3e0fb056a5..cb2553a0bf 100644
--- a/docs/zh/05-get-started/03-package.md
+++ b/docs/zh/05-get-started/03-package.md
@@ -10,23 +10,24 @@ import PkgListV3 from "/components/PkgListV3";
您可以[用 Docker 立即体验](../../get-started/docker/) TDengine。如果您希望对 TDengine 贡献代码或对内部实现感兴趣，请参考我们的 [TDengine GitHub 主页](https://github.com/taosdata/TDengine) 下载源码构建和安装.
-TDengine 完整的软件包包括服务端（taosd）、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、应用驱动（taosc）、命令行程序 (CLI，taos) 和一些工具软件。目前 taosAdapter 仅在 Linux 系统上安装和运行，后续将支持 Windows、macOS 等系统。TDengine 除了提供多种语言的连接器之外，还通过 [taosAdapter](../../reference/taosadapter/) 提供 [RESTful 接口](../../connector/rest-api/)。
+TDengine 完整的软件包包括服务端（taosd）、应用驱动（taosc）、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、命令行程序（CLI，taos）和一些工具软件。目前 taosAdapter 仅在 Linux 系统上安装和运行，后续将支持 Windows、macOS 等系统。TDengine 除了提供多种语言的连接器之外，还通过 [taosAdapter](../../reference/taosadapter/) 提供 [RESTful 接口](../../connector/rest-api/)。
-为方便使用，标准的服务端安装包包含了 taosd、taosAdapter、taosc、taos、taosdump、taosBenchmark、TDinsight 安装脚本和示例代码；如果您只需要用到服务端程序和客户端连接的 C/C++ 语言支持，也可以仅下载 lite 版本的安装包。
+为方便使用，标准的服务端安装包包含了 taosd、taosAdapter、taosc、taos、taosdump、taosBenchmark、TDinsight 安装脚本和示例代码；如果您只需要用到服务端程序和客户端连接的 C/C++ 语言支持，也可以仅下载 Lite 版本的安装包。
-在 Linux 系统上，TDengine 开源版本提供 deb 和 rpm 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 deb 支持 Debian/Ubuntu 及衍生系统，rpm 支持 CentOS/RHEL/SUSE 及衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包，也支持通过 `apt-get` 工具从线上进行安装。需要注意的是，rpm 和 deb 包不含 taosdump 和 TDinsight 安装脚本，这些工具需要通过安装 taosTool 包获得。TDengine 也提供 Windows x64 平台的安装包。
+在 Linux 系统上，TDengine 社区版提供 Deb 和 RPM 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 Deb 支持 Debian/Ubuntu 及其衍生系统，RPM 支持 CentOS/RHEL/SUSE 及其衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包，也支持通过 `apt-get` 工具从线上进行安装。需要注意的是，RPM 和 Deb 包不含 `taosdump` 和 TDinsight 安装脚本，这些工具需要通过安装 taosTool 包获得。TDengine 也提供 Windows x64 平台的安装包。
## 安装
-1. 从列表中下载获得 deb 安装包；
-
+1. 从列表中下载获得 Deb 安装包；
+
2. 进入到安装包所在目录，执行如下的安装命令：
+> 请将 `` 替换为下载的安装包版本
+
```bash
-# 替换为下载的安装包版本
sudo dpkg -i TDengine-server--Linux-x64.deb
```
@@ -34,12 +35,13 @@ sudo dpkg -i TDengine-server--Linux-x64.deb
-1. 从列表中下载获得 rpm 安装包；
-
+1. 从列表中下载获得 RPM 安装包；
+
2. 进入到安装包所在目录，执行如下的安装命令：
+> 请将 `` 替换为下载的安装包版本
+
```bash
-# 替换为下载的安装包版本
sudo rpm -ivh TDengine-server--Linux-x64.rpm
```
@@ -48,44 +50,46 @@ sudo rpm -ivh TDengine-server--Linux-x64.rpm
1. 从列表中下载获得 tar.gz 安装包；
-
-2. 进入到安装包所在目录，先解压文件后，进入子目录，执行其中的 install.sh 安装脚本：
+
+2. 进入到安装包所在目录，使用 `tar` 解压安装包；
+3. 进入到安装包所在目录，先解压文件后，进入子目录，执行其中的 install.sh 安装脚本。
+
+> 请将 `` 替换为下载的安装包版本
```bash
-# 替换为下载的安装包版本
tar -zxvf TDengine-server--Linux-x64.tar.gz
```
-解压后进入相应路径，执行
+解压文件后，进入相应子目录，执行其中的 `install.sh` 安装脚本：
```bash
sudo ./install.sh
```
:::info
-install.sh 安装脚本在执行过程中，会通过命令行交互界面询问一些配置信息。如果希望采取无交互安装方式，那么可以用 -e no 参数来执行 install.sh 脚本。运行 `./install.sh -h` 指令可以查看所有参数的详细说明信息。
+install.sh 安装脚本在执行过程中，会通过命令行交互界面询问一些配置信息。如果希望采取无交互安装方式，那么可以运行 `./install.sh -e no`。运行 `./install.sh -h` 指令可以查看所有参数的详细说明信息。
:::
-可以使用 apt-get 工具从官方仓库安装。
+可以使用 `apt-get` 工具从官方仓库安装。
-**安装包仓库**
+**配置包仓库**
```bash
wget -qO - http://repos.taosdata.com/tdengine.key | sudo apt-key add -
echo "deb [arch=amd64] http://repos.taosdata.com/tdengine-stable stable main" | sudo tee /etc/apt/sources.list.d/tdengine-stable.list
```
-如果安装 Beta 版需要安装包仓库
+如果安装 Beta 版需要安装包仓库：
```bash
wget -qO - http://repos.taosdata.com/tdengine.key | sudo apt-key add -
echo "deb [arch=amd64] http://repos.taosdata.com/tdengine-beta beta main" | sudo tee /etc/apt/sources.list.d/tdengine-beta.list
```
-**使用 apt-get 命令安装**
+**使用 `apt-get` 命令安装**
```bash
sudo apt-get update
@@ -94,26 +98,26 @@ sudo apt-get install tdengine
```
:::tip
-apt-get 方式只适用于 Debian 或 Ubuntu 系统
+apt-get 方式只适用于 Debian 或 Ubuntu 系统。
::::
-
+
-注意：目前 TDengine 在 Windows 平台上只支持 Windows server 2016/2019 和 Windows 10/11 系统版本。
+注意：目前 TDengine 在 Windows 平台上只支持 Windows Server 2016/2019 和 Windows 10/11。
1. 从列表中下载获得 exe 安装程序；
-
+
2. 运行可执行程序来安装 TDengine。
:::info
-下载其他组件、最新 Beta 版及之前版本的安装包，请点击[发布历史页面](../../releases/tdengine)
+下载其他组件、最新 Beta 版及之前版本的安装包，请点击[发布历史页面](../../releases/tdengine)。
:::
:::note
-当安装第一个节点时，出现 Enter FQDN：提示的时候，不需要输入任何内容。只有当安装第二个或以后更多的节点时，才需要输入已有集群中任何一个可用节点的 FQDN，支持该新节点加入集群。当然也可以不输入，而是在新节点启动前，配置到新节点的配置文件中。
+当安装第一个节点时，出现 `Enter FQDN:` 提示的时候，不需要输入任何内容。只有当安装第二个或以后更多的节点时，才需要输入已有集群中任何一个可用节点的 FQDN，支持该新节点加入集群。当然也可以不输入，而是在新节点启动前，配置到新节点的配置文件中。
:::
@@ -148,7 +152,7 @@ Active: inactive (dead)
如果 TDengine 服务正常工作，那么您可以通过 TDengine 的命令行程序 `taos` 来访问并体验 TDengine。
-systemctl 命令汇总：
+如下 `systemctl` 命令可以帮助你管理 TDengine 服务：
- 启动服务进程：`systemctl start taosd`
@@ -160,7 +164,7 @@ systemctl 命令汇总：
:::info
-- systemctl 命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 sudo 。
+- `systemctl` 命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 `sudo`。
- `systemctl stop taosd` 指令在执行后并不会马上停止 TDengine 服务，而是会等待系统中必要的落盘工作正常完成。在数据量很大的情况下，这可能会消耗较长时间。
- 如果系统中不支持 `systemd`，也可以用手动运行 `/usr/local/taos/bin/taosd` 方式启动 TDengine 服务。
@@ -170,87 +174,93 @@ systemctl 命令汇总：
-安装后，在 C:\TDengine 目录下，运行 taosd.exe 来启动 TDengine 服务进程。
+安装后，在 `C:\TDengine` 目录下，运行 `taosd.exe` 来启动 TDengine 服务进程。
-## TDengine 命令行 (CLI)
+## TDengine 命令行（CLI）
-为便于检查 TDengine 的状态，执行数据库 (Database) 的各种即席(Ad Hoc)查询，TDengine 提供一命令行应用程序(以下简称为 TDengine CLI) taos。要进入 TDengine 命令行，您只要在安装有 TDengine 的 Linux 终端执行 `taos` 即可，也可以在安装有 TDengine 的 Windows 终端的 C:\TDengine 目录下，运行 taos.exe 来启动 TDengine 命令行。
+为便于检查 TDengine 的状态，执行数据库（Database）的各种即席（Ad Hoc）查询，TDengine 提供一命令行应用程序（以下简称为 TDengine CLI）taos。要进入 TDengine 命令行，您只要在安装有 TDengine 的 Linux 终端执行 `taos` 即可，也可以在安装有 TDengine 的 Windows 终端的 C:\TDengine 目录下，运行 taos.exe 来启动 TDengine 命令行。
```bash
taos
```
-如果连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考 [FAQ](/train-faq/faq) 来解决终端连接服务端失败的问题）。 TDengine CLI 的提示符号如下：
+如果连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考 [FAQ](/train-faq/faq) 来解决终端连接服务端失败的问题）。TDengine CLI 的提示符号如下：
```cmd
taos>
```
-在 TDengine CLI 中，用户可以通过 SQL 命令来创建/删除数据库、表等，并进行数据库(database)插入查询操作。在终端中运行的 SQL 语句需要以分号结束来运行。示例：
+在 TDengine CLI 中，用户可以通过 SQL 命令来创建/删除数据库、表等，并进行数据库（Database）插入查询操作。在终端中运行的 SQL 语句需要以分号（;）结束来运行。示例：
```sql
-create database demo;
-use demo;
-create table t (ts timestamp, speed int);
-insert into t values ('2019-07-15 00:00:00', 10);
-insert into t values ('2019-07-15 01:00:00', 20);
-select * from t;
+CREATE DATABASE demo;
+USE demo;
+CREATE TABLE t (ts TIMESTAMP, speed INT);
+INSERT INTO t VALUES ('2019-07-15 00:00:00', 10);
+INSERT INTO t VALUES ('2019-07-15 01:00:00', 20);
+SELECT * FROM t;
+
ts | speed |
========================================
2019-07-15 00:00:00.000 | 10 |
2019-07-15 01:00:00.000 | 20 |
+
Query OK, 2 row(s) in set (0.003128s)
```
-除执行 SQL 语句外，系统管理员还可以从 TDengine CLI 进行检查系统运行状态、添加删除用户账号等操作。TDengine CLI 连同应用驱动也可以独立安装在 Linux 或 Windows 机器上运行，更多细节请参考 [这里](../../reference/taos-shell/)
+除执行 SQL 语句外，系统管理员还可以从 TDengine CLI 进行检查系统运行状态、添加删除用户账号等操作。TDengine CLI 连同应用驱动也可以独立安装在 Linux 或 Windows 机器上运行，更多细节请参考 [TDengine 命令行](../../reference/taos-shell/)。
## 使用 taosBenchmark 体验写入速度
-启动 TDengine 的服务，在 Linux 或 windows 终端执行 `taosBenchmark` （曾命名为 `taosdemo`）：
+可以使用 TDengine 的自带工具 taosBenchmark 快速体验 TDengine 的写入速度。
+
+启动 TDengine 的服务，在 Linux 或 Windows 终端执行 `taosBenchmark`（曾命名为 `taosdemo`）：
```bash
-taosBenchmark
+$ taosBenchmark
```
-该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "d0" 到 "d9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupId，groupId 被设置为 1 到 10， location 被设置为 "California.SanFrancisco" 或者 "California.LosAngeles"。
+该命令将在数据库 `test` 下面自动创建一张超级表 `meters`，该超级表下有 1 万张表，表名为 `d0` 到 `d9999`，每张表有 1 万条记录，每条记录有 `ts`、`current`、`voltage`、`phase` 四个字段，时间戳从 2017-07-14 10:40:00 000 到 2017-07-14 10:40:09 999，每张表带有标签 `location` 和 `groupId`，groupId 被设置为 1 到 10，location 被设置为 `Campbell`、`Cupertino`、`Los Angeles`、`Mountain View`、`Palo Alto`、`San Diego`、`San Francisco`、`San Jose`、`Santa Clara` 或者 `Sunnyvale`。
这条命令很快完成 1 亿条记录的插入。具体时间取决于硬件性能，即使在一台普通的 PC 服务器往往也仅需十几秒。
-taosBenchmark 命令本身带有很多选项，配置表的数目、记录条数等等，您可以设置不同参数进行体验，请执行 `taosBenchmark --help` 详细列出。taosBenchmark 详细使用方法请参照 [如何使用 taosBenchmark 对 TDengine 进行性能测试](https://www.taosdata.com/2021/10/09/3111.html)。
+taosBenchmark 命令本身带有很多选项，配置表的数目、记录条数等等，您可以设置不同参数进行体验，请执行 `taosBenchmark --help` 详细列出。taosBenchmark 详细使用方法请参照[如何使用 taosBenchmark 对 TDengine 进行性能测试](https://www.taosdata.com/2021/10/09/3111.html)和 [taosBenchmark 参考手册](../../reference/taosbenchmark)。
## 使用 TDengine CLI 体验查询速度
-使用上述 taosBenchmark 插入数据后，可以在 TDengine CLI 输入查询命令，体验查询速度。
+使用上述 `taosBenchmark` 插入数据后，可以在 TDengine CLI（taos）输入查询命令，体验查询速度。
-查询超级表下记录总条数：
+查询超级表 `meters` 下的记录总条数：
```sql
-taos> select count(*) from test.meters;
+SELECT COUNT(*) FROM test.meters;
```
查询 1 亿条记录的平均值、最大值、最小值等：
```sql
-taos> select avg(current), max(voltage), min(phase) from test.meters;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters;
```
-查询 location="California.SanFrancisco" 的记录总条数：
+查询 location = "San Francisco" 的记录总条数：
```sql
-taos> select count(*) from test.meters where location="California.SanFrancisco";
+SELECT COUNT(*) FROM test.meters WHERE location = "San Francisco";
```
-查询 groupId=10 的所有记录的平均值、最大值、最小值等：
+查询 groupId = 10 的所有记录的平均值、最大值、最小值等：
```sql
-taos> select avg(current), max(voltage), min(phase) from test.meters where groupId=10;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 10;
```
-对表 d10 按 10s 进行平均值、最大值和最小值聚合统计：
+对表 `d10` 按 10 每秒进行平均值、最大值和最小值聚合统计：
```sql
-taos> select avg(current), max(voltage), min(phase) from test.d10 interval(10s);
+SELECT FIRST(ts), AVG(current), MAX(voltage), MIN(phase) FROM test.d10 INTERVAL(10s);
```
+
+在上面的查询中，你选择的是区间内的第一个时间戳（ts），另一种选择方式是 `\_wstart`，它将给出时间窗口的开始。关于窗口查询的更多信息，参见[特色查询](../../taos-sql/distinguished/)。
diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md
index ef1cd4797a..3239eae49b 100644
--- a/docs/zh/07-develop/09-udf.md
+++ b/docs/zh/07-develop/09-udf.md
@@ -116,7 +116,7 @@ aggfn为函数名的占位符，需要修改为自己的函数名，如l2norm。
参数的具体含义是：
- inputDataBlock: 输入的数据块
- - resultColumn: 输出列。输出列
+ - resultColumn: 输出列
### 聚合接口函数
diff --git a/docs/zh/07-develop/_sub_java.mdx b/docs/zh/07-develop/_sub_java.mdx
index d14b5fd609..e7de158cc8 100644
--- a/docs/zh/07-develop/_sub_java.mdx
+++ b/docs/zh/07-develop/_sub_java.mdx
@@ -1,7 +1,5 @@
```java
{{#include docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java}}
-{{#include docs/examples/java/src/main/java/com/taos/example/MetersDeserializer.java}}
-{{#include docs/examples/java/src/main/java/com/taos/example/Meters.java}}
```
```java
{{#include docs/examples/java/src/main/java/com/taos/example/MetersDeserializer.java}}
diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md
index 2337f2a272..f6790e3c69 100644
--- a/docs/zh/12-taos-sql/03-table.md
+++ b/docs/zh/12-taos-sql/03-table.md
@@ -23,10 +23,7 @@ create_subtable_clause: {
}
create_definition:
- col_name column_definition
-
-column_definition:
- type_name [comment 'string_value']
+ col_name column_type
table_options:
table_option ...
diff --git a/docs/zh/12-taos-sql/20-keywords.md b/docs/zh/12-taos-sql/20-keywords.md
index 92c814067e..09fea45b7a 100644
--- a/docs/zh/12-taos-sql/20-keywords.md
+++ b/docs/zh/12-taos-sql/20-keywords.md
@@ -6,7 +6,8 @@ description: TDengine 保留关键字的详细列表
## 保留关键字
-目前 TDengine 有将近 200 个内部保留关键字，这些关键字无论大小写均不可以用作库名、表名、STable 名、数据列名及标签列名等。这些关键字列表如下：
+目前 TDengine 有将近 200 个内部保留关键字，这些关键字无论大小写如果需要用作库名、表名、STable 名、数据列名及标签列名等，需要使用符合``将关键字括起来使用，例如`ADD`。
+关键字列表如下：
### A
@@ -239,6 +240,7 @@ description: TDengine 保留关键字的详细列表
- TOPICS
- TRIGGER
- TSERIES
+- TTL
### U
diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md
index e3e1463131..a6ec560d3c 100644
--- a/docs/zh/28-releases/01-tdengine.md
+++ b/docs/zh/28-releases/01-tdengine.md
@@ -6,11 +6,7 @@ description: TDengine 发布历史、Release Notes 及下载链接
import Release from "/components/ReleaseV3";
-## 3.0.0.1
+## 3.0.1.0
-
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see
};
-// ================== TSDB global config
-extern bool tsdbForceKeepFile;
-
-#define TSDB_FS_ITER_FORWARD TSDB_ORDER_ASC
-#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC
-
struct TABLEID {
tb_uid_t suid;
tb_uid_t uid;
@@ -536,7 +560,7 @@ struct SDataFile {
int64_t size;
};
-struct SLastFile {
+struct SSttFile {
volatile int32_t nRef;
int64_t commitID;
@@ -556,8 +580,9 @@ struct SDFileSet {
int32_t fid;
SHeadFile *pHeadF;
SDataFile *pDataF;
- SLastFile *pLastF;
SSmaFile *pSmaF;
+ uint8_t nSttF;
+ SSttFile *aSttF[TSDB_MAX_STT_TRIGGER];
};
struct SRowIter {
@@ -572,37 +597,99 @@ struct SRowMerger {
SArray *pArray; // SArray
};
+typedef struct {
+ char *path;
+ int32_t szPage;
+ int32_t flag;
+ TdFilePtr pFD;
+ int64_t pgno;
+ uint8_t *pBuf;
+ int64_t szFile;
+} STsdbFD;
+
struct SDelFWriter {
- STsdb *pTsdb;
- SDelFile fDel;
- TdFilePtr pWriteH;
-
+ STsdb *pTsdb;
+ SDelFile fDel;
+ STsdbFD *pWriteH;
uint8_t *aBuf[1];
};
-struct SDataFWriter {
- STsdb *pTsdb;
- SDFileSet wSet;
-
- TdFilePtr pHeadFD;
- TdFilePtr pDataFD;
- TdFilePtr pLastFD;
- TdFilePtr pSmaFD;
-
- SHeadFile fHead;
- SDataFile fData;
- SLastFile fLast;
- SSmaFile fSma;
-
- uint8_t *aBuf[4];
-};
-
struct STsdbReadSnap {
SMemTable *pMem;
SMemTable *pIMem;
STsdbFS fs;
};
+struct SDataFWriter {
+ STsdb *pTsdb;
+ SDFileSet wSet;
+
+ STsdbFD *pHeadFD;
+ STsdbFD *pDataFD;
+ STsdbFD *pSmaFD;
+ STsdbFD *pSttFD;
+
+ SHeadFile fHead;
+ SDataFile fData;
+ SSmaFile fSma;
+ SSttFile fStt[TSDB_MAX_STT_TRIGGER];
+
+ uint8_t *aBuf[4];
+};
+
+struct SDataFReader {
+ STsdb *pTsdb;
+ SDFileSet *pSet;
+ STsdbFD *pHeadFD;
+ STsdbFD *pDataFD;
+ STsdbFD *pSmaFD;
+ STsdbFD *aSttFD[TSDB_MAX_STT_TRIGGER];
+ uint8_t *aBuf[3];
+};
+
+typedef struct {
+ int64_t suid;
+ int64_t uid;
+ TSDBROW row;
+} SRowInfo;
+
+typedef struct SSttBlockLoadInfo {
+ SBlockData blockData[2];
+ SArray *aSttBlk;
+ int32_t blockIndex[2]; // to denote the loaded block in the corresponding position.
+ int32_t currentLoadBlockIndex;
+ int32_t loadBlocks;
+ double elapsedTime;
+} SSttBlockLoadInfo;
+
+typedef struct SMergeTree {
+ int8_t backward;
+ SRBTree rbt;
+ SArray *pIterList;
+ SLDataIter *pIter;
+ bool destroyLoadInfo;
+ SSttBlockLoadInfo *pLoadInfo;
+ const char *idStr;
+} SMergeTree;
+
+typedef struct {
+ int64_t suid;
+ int64_t uid;
+ STSchema *pTSchema;
+} SSkmInfo;
+
+int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
+ STimeWindow *pTimeWindow, SVersionRange *pVerRange, void *pLoadInfo, const char *idStr);
+void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter);
+bool tMergeTreeNext(SMergeTree *pMTree);
+TSDBROW tMergeTreeGetRow(SMergeTree *pMTree);
+void tMergeTreeClose(SMergeTree *pMTree);
+
+SSttBlockLoadInfo *tCreateLastBlockLoadInfo();
+void resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
+void getLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, int64_t *blocks, double *el);
+void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
+
// ========== inline functions ==========
static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) {
TSDBKEY *pKey1 = (TSDBKEY *)p1;
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 9b252df58b..0e85e7bfb6 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -36,6 +36,7 @@
#include "tlosertree.h"
#include "tlrucache.h"
#include "tmsgcb.h"
+#include "trbtree.h"
#include "tref.h"
#include "tskiplist.h"
#include "tstream.h"
@@ -70,8 +71,8 @@ typedef struct SStreamTaskReader SStreamTaskReader;
typedef struct SStreamTaskWriter SStreamTaskWriter;
typedef struct SStreamStateReader SStreamStateReader;
typedef struct SStreamStateWriter SStreamStateWriter;
-typedef struct SRsmaSnapReader SRsmaSnapReader;
-typedef struct SRsmaSnapWriter SRsmaSnapWriter;
+typedef struct SRSmaSnapReader SRSmaSnapReader;
+typedef struct SRSmaSnapWriter SRSmaSnapWriter;
typedef struct SSnapDataHdr SSnapDataHdr;
#define VNODE_META_DIR "meta"
@@ -102,8 +103,8 @@ int metaCommit(SMeta* pMeta);
int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq, SArray* tbUidList);
-int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp **pMetaRsp);
-int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids);
+int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** pMetaRsp);
+int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids, int64_t* tbUid);
int metaTtlDropTable(SMeta* pMeta, int64_t ttl, SArray* tbUids);
int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp);
SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline);
@@ -173,7 +174,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg);
// tq-stream
int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen);
int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen);
-int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* data, int64_t ver);
+int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* data, int64_t ver);
+int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver);
int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec);
int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg);
@@ -206,7 +208,7 @@ int32_t tdProcessRSmaCreate(SSma* pSma, SVCreateStbReq* pReq);
int32_t tdProcessRSmaSubmit(SSma* pSma, void* pMsg, int32_t inputType);
int32_t tdProcessRSmaDrop(SSma* pSma, SVDropStbReq* pReq);
int32_t tdFetchTbUidList(SSma* pSma, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid);
-int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore);
+int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore, bool isAdd);
void tdUidStoreDestory(STbUidStore* pStore);
void* tdUidStoreFree(STbUidStore* pStore);
@@ -246,14 +248,14 @@ int32_t tqOffsetSnapWrite(STqOffsetWriter* pWriter, uint8_t* pData, uint32_t nDa
// SStreamTaskReader ======================================
// SStreamStateWriter =====================================
// SStreamStateReader =====================================
-// SRsmaSnapReader ========================================
-int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRsmaSnapReader** ppReader);
-int32_t rsmaSnapReaderClose(SRsmaSnapReader** ppReader);
-int32_t rsmaSnapRead(SRsmaSnapReader* pReader, uint8_t** ppData);
-// SRsmaSnapWriter ========================================
-int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRsmaSnapWriter** ppWriter);
-int32_t rsmaSnapWrite(SRsmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
-int32_t rsmaSnapWriterClose(SRsmaSnapWriter** ppWriter, int8_t rollback);
+// SRSmaSnapReader ========================================
+int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapReader** ppReader);
+int32_t rsmaSnapReaderClose(SRSmaSnapReader** ppReader);
+int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData);
+// SRSmaSnapWriter ========================================
+int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWriter** ppWriter);
+int32_t rsmaSnapWrite(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
+int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback);
typedef struct {
int8_t streamType; // sma or other
diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c
index c282f17116..a34569b08e 100644
--- a/source/dnode/vnode/src/meta/metaQuery.c
+++ b/source/dnode/vnode/src/meta/metaQuery.c
@@ -129,10 +129,16 @@ _err:
bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid) {
// query uid.idx
+ metaRLock(pMeta);
+
if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), NULL, NULL) < 0) {
+ metaULock(pMeta);
+
return false;
}
+ metaULock(pMeta);
+
return true;
}
@@ -182,9 +188,14 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) {
}
int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName) {
+ int code = 0;
SMetaReader mr = {0};
metaReaderInit(&mr, (SMeta *)meta, 0);
- metaGetTableEntryByUid(&mr, uid);
+ code = metaGetTableEntryByUid(&mr, uid);
+ if (code < 0) {
+ metaReaderClear(&mr);
+ return -1;
+ }
STR_TO_VARSTR(tbName, mr.me.name);
metaReaderClear(&mr);
diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index 583a2e098f..92cf90d328 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -416,20 +416,22 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMe
me.ctbEntry.pTags = pReq->ctb.pTag;
#ifdef TAG_FILTER_DEBUG
- SArray* pTagVals = NULL;
- int32_t code = tTagToValArray((STag*)pReq->ctb.pTag, &pTagVals);
+ SArray *pTagVals = NULL;
+ int32_t code = tTagToValArray((STag *)pReq->ctb.pTag, &pTagVals);
for (int i = 0; i < taosArrayGetSize(pTagVals); i++) {
- STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, i);
+ STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
if (IS_VAR_DATA_TYPE(pTagVal->type)) {
- char* buf = taosMemoryCalloc(pTagVal->nData + 1, 1);
+ char *buf = taosMemoryCalloc(pTagVal->nData + 1, 1);
memcpy(buf, pTagVal->pData, pTagVal->nData);
- metaDebug("metaTag table:%s varchar index:%d cid:%d type:%d value:%s", pReq->name, i, pTagVal->cid, pTagVal->type, buf);
+ metaDebug("metaTag table:%s varchar index:%d cid:%d type:%d value:%s", pReq->name, i, pTagVal->cid,
+ pTagVal->type, buf);
taosMemoryFree(buf);
} else {
double val = 0;
GET_TYPED_DATA(val, double, pTagVal->type, &pTagVal->i64);
- metaDebug("metaTag table:%s number index:%d cid:%d type:%d value:%f", pReq->name, i, pTagVal->cid, pTagVal->type, val);
+ metaDebug("metaTag table:%s number index:%d cid:%d type:%d value:%f", pReq->name, i, pTagVal->cid,
+ pTagVal->type, val);
}
}
#endif
@@ -474,7 +476,7 @@ _err:
return -1;
}
-int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids) {
+int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids, tb_uid_t *tbUid) {
void *pData = NULL;
int nData = 0;
int rc = 0;
@@ -496,6 +498,10 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi
taosArrayPush(tbUids, &uid);
}
+ if ((type == TSDB_CHILD_TABLE) && tbUid) {
+ *tbUid = uid;
+ }
+
tdbFree(pData);
return 0;
}
diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c
index 3cf50a035a..07ec7d0694 100644
--- a/source/dnode/vnode/src/sma/smaCommit.c
+++ b/source/dnode/vnode/src/sma/smaCommit.c
@@ -15,13 +15,15 @@
#include "sma.h"
+extern SSmaMgmt smaMgmt;
+
static int32_t tdProcessRSmaSyncPreCommitImpl(SSma *pSma);
static int32_t tdProcessRSmaSyncCommitImpl(SSma *pSma);
static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma);
static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma);
static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma);
static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma);
-static int32_t tdCleanupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat);
+static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat);
/**
* @brief Only applicable to Rollup SMA
@@ -166,114 +168,65 @@ static int32_t tdProcessRSmaSyncCommitImpl(SSma *pSma) {
return TSDB_CODE_SUCCESS;
}
-static int32_t tdCleanupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat) {
- SVnode *pVnode = pSma->pVnode;
- int64_t committed = pRSmaStat->commitAppliedVer;
- TdDirPtr pDir = NULL;
- TdDirEntryPtr pDirEntry = NULL;
- char dir[TSDB_FILENAME_LEN];
- const char *pattern = "v[0-9]+qinf\\.v([0-9]+)?$";
- regex_t regex;
- int code = 0;
-
- tdGetVndDirName(TD_VID(pVnode), tfsGetPrimaryPath(pVnode->pTfs), VNODE_RSMA_DIR, true, dir);
-
- // Resource allocation and init
- if ((code = regcomp(®ex, pattern, REG_EXTENDED)) != 0) {
- char errbuf[128];
- regerror(code, ®ex, errbuf, sizeof(errbuf));
- smaWarn("vgId:%d, rsma post commit, regcomp for %s failed since %s", TD_VID(pVnode), dir, errbuf);
- return TSDB_CODE_FAILED;
- }
-
- if ((pDir = taosOpenDir(dir)) == NULL) {
- regfree(®ex);
- terrno = TAOS_SYSTEM_ERROR(errno);
- smaDebug("vgId:%d, rsma post commit, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr());
- return TSDB_CODE_FAILED;
- }
-
- int32_t dirLen = strlen(dir);
- char *dirEnd = POINTER_SHIFT(dir, dirLen);
- regmatch_t regMatch[2];
- while ((pDirEntry = taosReadDir(pDir)) != NULL) {
- char *entryName = taosGetDirEntryName(pDirEntry);
- if (!entryName) {
- continue;
- }
-
- code = regexec(®ex, entryName, 2, regMatch, 0);
-
- if (code == 0) {
- // match
- int64_t version = -1;
- sscanf((const char *)POINTER_SHIFT(entryName, regMatch[1].rm_so), "%" PRIi64, &version);
- if ((version < committed) && (version > -1)) {
- strncpy(dirEnd, entryName, TSDB_FILENAME_LEN - dirLen);
- if (taosRemoveFile(dir) != 0) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- smaWarn("vgId:%d, committed version:%" PRIi64 ", failed to remove %s since %s", TD_VID(pVnode), committed,
- dir, terrstr());
- } else {
- smaDebug("vgId:%d, committed version:%" PRIi64 ", success to remove %s", TD_VID(pVnode), committed, dir);
- }
- }
- } else if (code == REG_NOMATCH) {
- // not match
- smaTrace("vgId:%d, rsma post commit, not match %s", TD_VID(pVnode), entryName);
- continue;
- } else {
- // has other error
- char errbuf[128];
- regerror(code, ®ex, errbuf, sizeof(errbuf));
- smaWarn("vgId:%d, rsma post commit, regexec failed since %s", TD_VID(pVnode), errbuf);
-
- taosCloseDir(&pDir);
- regfree(®ex);
- return TSDB_CODE_FAILED;
- }
- }
-
- taosCloseDir(&pDir);
- regfree(®ex);
-
- return TSDB_CODE_SUCCESS;
-}
-
// SQTaskFile ======================================================
-// int32_t tCmprQTaskFile(void const *lhs, void const *rhs) {
-// int64_t *lCommitted = *(int64_t *)lhs;
-// SQTaskFile *rQTaskF = (SQTaskFile *)rhs;
-// if (lCommitted < rQTaskF->commitID) {
-// return -1;
-// } else if (lCommitted > rQTaskF->commitID) {
-// return 1;
-// }
-
-// return 0;
-// }
-
-#if 0
/**
* @brief At most time, there is only one qtaskinfo file committed latest in aTaskFile. Sometimes, there would be
* multiple qtaskinfo files supporting snapshot replication.
*
* @param pSma
- * @param pRSmaStat
+ * @param pStat
* @return int32_t
*/
-static int32_t tdCleanupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat) {
- SVnode *pVnode = pSma->pVnode;
- int64_t committed = pRSmaStat->commitAppliedVer;
- SArray *aTaskFile = pRSmaStat->aTaskFile;
+static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pStat) {
+ SVnode *pVnode = pSma->pVnode;
+ SRSmaFS *pFS = RSMA_FS(pStat);
+ int64_t committed = pStat->commitAppliedVer;
+ int64_t fsMaxVer = -1;
+ char qTaskInfoFullName[TSDB_FILENAME_LEN];
- void *qTaskFile = taosArraySearch(aTaskFile, committed, tCmprQTaskFile, TD_LE);
-
+ taosWLockLatch(RSMA_FS_LOCK(pStat));
+ for (int32_t i = 0; i < taosArrayGetSize(pFS->aQTaskInf);) {
+ SQTaskFile *pTaskF = taosArrayGet(pFS->aQTaskInf, i);
+ int32_t oldVal = atomic_fetch_sub_32(&pTaskF->nRef, 1);
+ if ((oldVal <= 1) && (pTaskF->version < committed)) {
+ tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->version, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName);
+ if (taosRemoveFile(qTaskInfoFullName) < 0) {
+ smaWarn("vgId:%d, cleanup qinf, committed %" PRIi64 ", failed to remove %s since %s", TD_VID(pVnode), committed,
+ qTaskInfoFullName, tstrerror(TAOS_SYSTEM_ERROR(errno)));
+ } else {
+ smaDebug("vgId:%d, cleanup qinf, committed %" PRIi64 ", success to remove %s", TD_VID(pVnode), committed,
+ qTaskInfoFullName);
+ }
+ taosArrayRemove(pFS->aQTaskInf, i);
+ continue;
+ }
+ ++i;
+ }
+
+ if (taosArrayGetSize(pFS->aQTaskInf) > 0) {
+ fsMaxVer = ((SQTaskFile *)taosArrayGetLast(pFS->aQTaskInf))->version;
+ }
+
+ if (fsMaxVer < committed) {
+ tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), committed, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName);
+ if (taosCheckExistFile(qTaskInfoFullName)) {
+ SQTaskFile qFile = {.nRef = 1, .padding = 0, .version = committed, .size = 0};
+ if (taosArrayPush(pFS->aQTaskInf, &qFile) < 0) {
+ taosWUnLockLatch(RSMA_FS_LOCK(pStat));
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return TSDB_CODE_FAILED;
+ }
+ }
+ } else {
+ smaDebug("vgId:%d, update qinf, no need as committed %" PRIi64 " not larger than fsMaxVer %" PRIi64, TD_VID(pVnode),
+ committed, fsMaxVer);
+ }
+
+ taosWUnLockLatch(RSMA_FS_LOCK(pStat));
return TSDB_CODE_SUCCESS;
}
-#endif
/**
* @brief post-commit for rollup sma
@@ -290,8 +243,7 @@ static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma) {
SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSma);
- // cleanup outdated qtaskinfo files
- tdCleanupQTaskInfoFiles(pSma, pRSmaStat);
+ tdUpdateQTaskInfoFiles(pSma, pRSmaStat);
return TSDB_CODE_SUCCESS;
}
@@ -365,12 +317,12 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) {
if (tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)) < 0) {
return TSDB_CODE_FAILED;
}
- smaInfo("vgId:%d, rsma commit, operator state commited, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId());
+ smaInfo("vgId:%d, rsma commit, operator state committed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId());
#if 0 // consuming task of qTaskInfo clone
// step 4: swap queue/qall and iQueue/iQall
// lock
- // taosWLockLatch(SMA_ENV_LOCK(pEnv));
+ taosWLockLatch(SMA_ENV_LOCK(pEnv));
ASSERT(RSMA_INFO_HASH(pRSmaStat));
@@ -386,7 +338,7 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) {
}
// unlock
- // taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
+ taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
#endif
return TSDB_CODE_SUCCESS;
@@ -428,33 +380,29 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) {
}
SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
- SArray *rsmaDeleted = NULL;
// step 1: merge qTaskInfo and iQTaskInfo
// lock
- // taosWLockLatch(SMA_ENV_LOCK(pEnv));
+ if (1 == atomic_val_compare_exchange_8(&pRSmaStat->delFlag, 1, 0)) {
+ taosWLockLatch(SMA_ENV_LOCK(pEnv));
- void *pIter = NULL;
- while ((pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), pIter))) {
- tb_uid_t *pSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL);
- SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pIter;
- if (RSMA_INFO_IS_DEL(pRSmaInfo)) {
- int32_t refVal = T_REF_VAL_GET(pRSmaInfo);
- if (refVal == 0) {
- if (!rsmaDeleted) {
- if ((rsmaDeleted = taosArrayInit(1, sizeof(tb_uid_t)))) {
- taosArrayPush(rsmaDeleted, pSuid);
- }
+ void *pIter = NULL;
+ while ((pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), pIter))) {
+ tb_uid_t *pSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL);
+ SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pIter;
+ if (RSMA_INFO_IS_DEL(pRSmaInfo)) {
+ int32_t refVal = T_REF_VAL_GET(pRSmaInfo);
+ if (refVal == 0) {
+ taosHashRemove(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(*pSuid));
+ } else {
+ smaDebug(
+ "vgId:%d, rsma async post commit, not free rsma info since ref is %d although already deleted for "
+ "table:%" PRIi64,
+ SMA_VID(pSma), refVal, *pSuid);
}
- } else {
- smaDebug(
- "vgId:%d, rsma async post commit, not free rsma info since ref is %d although already deleted for "
- "table:%" PRIi64,
- SMA_VID(pSma), refVal, *pSuid);
- }
- continue;
- }
+ continue;
+ }
#if 0
if (pRSmaInfo->taskInfo[0]) {
if (pRSmaInfo->iTaskInfo[0]) {
@@ -469,27 +417,13 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) {
taosHashPut(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t), pIter, sizeof(pIter));
smaDebug("vgId:%d, rsma async post commit, migrated from iRsmaInfoHash for table:%" PRIi64, SMA_VID(pSma), *pSuid);
#endif
- }
-
- for (int32_t i = 0; i < taosArrayGetSize(rsmaDeleted); ++i) {
- tb_uid_t *pSuid = taosArrayGet(rsmaDeleted, i);
- void *pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t));
- if ((pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
- tdFreeRSmaInfo(pSma, pRSmaInfo, true);
- smaDebug(
- "vgId:%d, rsma async post commit, free rsma info since already deleted and ref is 0 for "
- "table:%" PRIi64,
- SMA_VID(pSma), *pSuid);
}
- taosHashRemove(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t));
+
+ // unlock
+ taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
}
- taosArrayDestroy(rsmaDeleted);
- // unlock
- // taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
-
- // step 2: cleanup outdated qtaskinfo files
- tdCleanupQTaskInfoFiles(pSma, pRSmaStat);
+ tdUpdateQTaskInfoFiles(pSma, pRSmaStat);
atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 0);
diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c
index 32a419022a..b870ea1b62 100644
--- a/source/dnode/vnode/src/sma/smaEnv.c
+++ b/source/dnode/vnode/src/sma/smaEnv.c
@@ -28,6 +28,8 @@ static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, SSmaEnv **ppEnv);
static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pSma);
static int32_t tdRsmaStartExecutor(const SSma *pSma);
static int32_t tdRsmaStopExecutor(const SSma *pSma);
+static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType);
+static void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType);
static void *tdFreeTSmaStat(STSmaStat *pStat);
static void tdDestroyRSmaStat(void *pRSmaStat);
@@ -59,12 +61,23 @@ int32_t smaInit() {
return TSDB_CODE_FAILED;
}
+ int32_t type = (8 == POINTER_BYTES) ? TSDB_DATA_TYPE_UBIGINT : TSDB_DATA_TYPE_UINT;
+ smaMgmt.refHash = taosHashInit(64, taosGetDefaultHashFunction(type), true, HASH_ENTRY_LOCK);
+ if (!smaMgmt.refHash) {
+ taosCloseRef(smaMgmt.rsetId);
+ atomic_store_8(&smaMgmt.inited, 0);
+ smaError("failed to init sma tmr hanle since %s", terrstr());
+ return TSDB_CODE_FAILED;
+ }
+
// init fetch timer handle
smaMgmt.tmrHandle = taosTmrInit(10000, 100, 10000, "RSMA");
if (!smaMgmt.tmrHandle) {
taosCloseRef(smaMgmt.rsetId);
+ taosHashCleanup(smaMgmt.refHash);
+ smaMgmt.refHash = NULL;
atomic_store_8(&smaMgmt.inited, 0);
- smaError("failed to init sma tmr hanle since %s", terrstr());
+ smaError("failed to init sma tmr handle since %s", terrstr());
return TSDB_CODE_FAILED;
}
@@ -93,6 +106,8 @@ void smaCleanUp() {
if (old == 1) {
taosCloseRef(smaMgmt.rsetId);
+ taosHashCleanup(smaMgmt.refHash);
+ smaMgmt.refHash = NULL;
taosTmrCleanUp(smaMgmt.tmrHandle);
smaInfo("sma mgmt env is cleaned up, rsetId:%d, tmrHandle:%p", smaMgmt.rsetId, smaMgmt.tmrHandle);
atomic_store_8(&smaMgmt.inited, 0);
@@ -162,37 +177,19 @@ void *tdFreeSmaEnv(SSmaEnv *pSmaEnv) {
return NULL;
}
-int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat) {
- if (!pStat) return 0;
+static void tRSmaInfoHashFreeNode(void *data) {
+ SRSmaInfo *pRSmaInfo = NULL;
+ SRSmaInfoItem *pItem = NULL;
- int ref = T_REF_INC(pStat);
- smaDebug("vgId:%d, ref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref);
- return 0;
-}
-
-int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) {
- if (!pStat) return 0;
-
- int ref = T_REF_DEC(pStat);
- smaDebug("vgId:%d, unref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref);
- return 0;
-}
-
-int32_t tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) {
- if (!pRSmaInfo) return 0;
-
- int ref = T_REF_INC(pRSmaInfo);
- smaDebug("vgId:%d, ref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref);
- return 0;
-}
-
-int32_t tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) {
- if (!pRSmaInfo) return 0;
-
- int ref = T_REF_DEC(pRSmaInfo);
- smaDebug("vgId:%d, unref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref);
-
- return 0;
+ if ((pRSmaInfo = *(SRSmaInfo **)data)) {
+ if ((pItem = RSMA_INFO_ITEM((SRSmaInfo *)pRSmaInfo, 0)) && pItem->level) {
+ taosHashRemove(smaMgmt.refHash, &pItem, POINTER_BYTES);
+ }
+ if ((pItem = RSMA_INFO_ITEM((SRSmaInfo *)pRSmaInfo, 1)) && pItem->level) {
+ taosHashRemove(smaMgmt.refHash, &pItem, POINTER_BYTES);
+ }
+ tdFreeRSmaInfo(pRSmaInfo->pSma, pRSmaInfo, true);
+ }
}
static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pSma) {
@@ -240,10 +237,16 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS
if (!RSMA_INFO_HASH(pRSmaStat)) {
return TSDB_CODE_FAILED;
}
+ taosHashSetFreeFp(RSMA_INFO_HASH(pRSmaStat), tRSmaInfoHashFreeNode);
if (tdRsmaStartExecutor(pSma) < 0) {
return TSDB_CODE_FAILED;
}
+
+ if (!(RSMA_FS(pRSmaStat)->aQTaskInf = taosArrayInit(1, sizeof(SQTaskFile)))) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return TSDB_CODE_FAILED;
+ }
} else if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
// TODO
} else {
@@ -278,14 +281,6 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
tsem_destroy(&(pStat->notEmpty));
// step 2: destroy the rsma info and associated fetch tasks
- if (taosHashGetSize(RSMA_INFO_HASH(pStat)) > 0) {
- void *infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), NULL);
- while (infoHash) {
- SRSmaInfo *pSmaInfo = *(SRSmaInfo **)infoHash;
- tdFreeRSmaInfo(pSma, pSmaInfo, true);
- infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), infoHash);
- }
- }
taosHashCleanup(RSMA_INFO_HASH(pStat));
// step 3: wait for all triggered fetch tasks to finish
@@ -307,12 +302,15 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
// step 4:
tdRsmaStopExecutor(pSma);
- // step 5: free pStat
+ // step 5:
+ tdRSmaFSClose(RSMA_FS(pStat));
+
+ // step 6: free pStat
taosMemoryFreeClear(pStat);
}
}
-void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) {
+static void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) {
tdDestroySmaState(pSmaStat, smaType);
if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
taosMemoryFreeClear(pSmaStat);
@@ -329,7 +327,7 @@ void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) {
* @return int32_t
*/
-int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) {
+static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) {
if (pSmaStat) {
if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
tdDestroyTSmaStat(SMA_STAT_TSMA(pSmaStat));
@@ -337,7 +335,7 @@ int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) {
SRSmaStat *pRSmaStat = &pSmaStat->rsmaStat;
int32_t vid = SMA_VID(pRSmaStat->pSma);
int64_t refId = RSMA_REF_ID(pRSmaStat);
- if (taosRemoveRef(smaMgmt.rsetId, RSMA_REF_ID(pRSmaStat)) < 0) {
+ if (taosRemoveRef(smaMgmt.rsetId, refId) < 0) {
smaError("vgId:%d, remove refId:%" PRIi64 " from rsmaRef:%" PRIi32 " failed since %s", vid, refId,
smaMgmt.rsetId, terrstr());
} else {
@@ -461,6 +459,8 @@ static int32_t tdRsmaStopExecutor(const SSma *pSma) {
taosThreadJoin(pthread[i], NULL);
}
}
+
+ smaInfo("vgId:%d, rsma executor stopped, number:%d", SMA_VID(pSma), tsNumOfVnodeRsmaThreads);
}
return 0;
}
\ No newline at end of file
diff --git a/source/dnode/vnode/src/sma/smaFS.c b/source/dnode/vnode/src/sma/smaFS.c
new file mode 100644
index 0000000000..8e8611f0e8
--- /dev/null
+++ b/source/dnode/vnode/src/sma/smaFS.c
@@ -0,0 +1,264 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see
- int32_t iBlockIdx;
- SBlockIdx *pBlockIdx;
- SMapData mBlock; // SMapData
- SBlockData bData;
- // last
- SArray *aBlockL; // SArray
- int32_t iBlockL;
- SBlockData bDatal;
- int32_t iRow;
- SRowInfo *pRowInfo;
- SRowInfo rowInfo;
+ SArray *aBlockIdx; // SArray
+ int32_t iBlockIdx;
+ SBlockIdx *pBlockIdx;
+ SMapData mBlock; // SMapData
+ SBlockData bData;
} dReader;
+ struct {
+ SDataIter *pIter;
+ SRBTree rbt;
+ SDataIter dataIter;
+ SDataIter aDataIter[TSDB_MAX_STT_TRIGGER];
+ int8_t toLastOnly;
+ };
struct {
SDataFWriter *pWriter;
SArray *aBlockIdx; // SArray
- SArray *aBlockL; // SArray
- SMapData mBlock; // SMapData
+ SArray *aSttBlk; // SArray
+ SMapData mBlock; // SMapData
SBlockData bData;
SBlockData bDatal;
} dWriter;
@@ -82,6 +92,26 @@ static int32_t tsdbCommitData(SCommitter *pCommitter);
static int32_t tsdbCommitDel(SCommitter *pCommitter);
static int32_t tsdbCommitCache(SCommitter *pCommitter);
static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno);
+static int32_t tsdbNextCommitRow(SCommitter *pCommitter);
+
+int32_t tRowInfoCmprFn(const void *p1, const void *p2) {
+ SRowInfo *pInfo1 = (SRowInfo *)p1;
+ SRowInfo *pInfo2 = (SRowInfo *)p2;
+
+ if (pInfo1->suid < pInfo2->suid) {
+ return -1;
+ } else if (pInfo1->suid > pInfo2->suid) {
+ return 1;
+ }
+
+ if (pInfo1->uid < pInfo2->uid) {
+ return -1;
+ } else if (pInfo1->uid > pInfo2->uid) {
+ return 1;
+ }
+
+ return tsdbRowCmprFn(&pInfo1->row, &pInfo2->row);
+}
int32_t tsdbBegin(STsdb *pTsdb) {
int32_t code = 0;
@@ -290,19 +320,22 @@ _err:
return code;
}
-static int32_t tsdbCommitterUpdateTableSchema(SCommitter *pCommitter, int64_t suid, int64_t uid) {
+int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo *pSkmInfo) {
int32_t code = 0;
if (suid) {
- if (pCommitter->skmTable.suid == suid) goto _exit;
+ if (pSkmInfo->suid == suid) {
+ pSkmInfo->uid = uid;
+ goto _exit;
+ }
} else {
- if (pCommitter->skmTable.uid == uid) goto _exit;
+ if (pSkmInfo->uid == uid) goto _exit;
}
- pCommitter->skmTable.suid = suid;
- pCommitter->skmTable.uid = uid;
- tTSchemaDestroy(pCommitter->skmTable.pTSchema);
- code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, -1, &pCommitter->skmTable.pTSchema);
+ pSkmInfo->suid = suid;
+ pSkmInfo->uid = uid;
+ tTSchemaDestroy(pSkmInfo->pTSchema);
+ code = metaGetTbTSchemaEx(pMeta, suid, uid, -1, &pSkmInfo->pTSchema);
if (code) goto _exit;
_exit:
@@ -334,54 +367,6 @@ _exit:
return code;
}
-static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) {
- int32_t code = 0;
-
- ASSERT(pCommitter->dReader.pReader);
- ASSERT(pCommitter->dReader.pRowInfo);
-
- SBlockData *pBlockDatal = &pCommitter->dReader.bDatal;
- pCommitter->dReader.iRow++;
- if (pCommitter->dReader.iRow < pBlockDatal->nRow) {
- if (pBlockDatal->uid) {
- pCommitter->dReader.pRowInfo->uid = pBlockDatal->uid;
- } else {
- pCommitter->dReader.pRowInfo->uid = pBlockDatal->aUid[pCommitter->dReader.iRow];
- }
- pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(pBlockDatal, pCommitter->dReader.iRow);
- } else {
- pCommitter->dReader.iBlockL++;
- if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) {
- SBlockL *pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL);
- int64_t suid = pBlockL->suid;
- int64_t uid = pBlockL->maxUid;
-
- code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid);
- if (code) goto _exit;
-
- code = tBlockDataInit(pBlockDatal, suid, suid ? 0 : uid, pCommitter->skmTable.pTSchema);
- if (code) goto _exit;
-
- code = tsdbReadLastBlock(pCommitter->dReader.pReader, pBlockL, pBlockDatal);
- if (code) goto _exit;
-
- pCommitter->dReader.iRow = 0;
- pCommitter->dReader.pRowInfo->suid = pBlockDatal->suid;
- if (pBlockDatal->uid) {
- pCommitter->dReader.pRowInfo->uid = pBlockDatal->uid;
- } else {
- pCommitter->dReader.pRowInfo->uid = pBlockDatal->aUid[0];
- }
- pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(pBlockDatal, pCommitter->dReader.iRow);
- } else {
- pCommitter->dReader.pRowInfo = NULL;
- }
- }
-
-_exit:
- return code;
-}
-
static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) {
int32_t code = 0;
@@ -392,7 +377,7 @@ static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) {
pCommitter->dReader.pBlockIdx =
(SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx);
- code = tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock);
+ code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock);
if (code) goto _exit;
ASSERT(pCommitter->dReader.mBlock.nItem > 0);
@@ -404,6 +389,85 @@ _exit:
return code;
}
+static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) {
+ int32_t code = 0;
+
+ pCommitter->pIter = NULL;
+ tRBTreeCreate(&pCommitter->rbt, tRowInfoCmprFn);
+
+ // memory
+ TSDBKEY tKey = {.ts = pCommitter->minKey, .version = VERSION_MIN};
+ SDataIter *pIter = &pCommitter->dataIter;
+ pIter->type = MEMORY_DATA_ITER;
+ pIter->iTbDataP = 0;
+ for (; pIter->iTbDataP < taosArrayGetSize(pCommitter->aTbDataP); pIter->iTbDataP++) {
+ STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, pIter->iTbDataP);
+ tsdbTbDataIterOpen(pTbData, &tKey, 0, &pIter->iter);
+ TSDBROW *pRow = tsdbTbDataIterGet(&pIter->iter);
+ if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) {
+ pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow));
+ pRow = NULL;
+ }
+
+ if (pRow == NULL) continue;
+
+ pIter->r.suid = pTbData->suid;
+ pIter->r.uid = pTbData->uid;
+ pIter->r.row = *pRow;
+ break;
+ }
+ ASSERT(pIter->iTbDataP < taosArrayGetSize(pCommitter->aTbDataP));
+ tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pIter);
+
+ // disk
+ pCommitter->toLastOnly = 0;
+ SDataFReader *pReader = pCommitter->dReader.pReader;
+ if (pReader) {
+ if (pReader->pSet->nSttF >= pCommitter->sttTrigger) {
+ int8_t iIter = 0;
+ for (int32_t iStt = 0; iStt < pReader->pSet->nSttF; iStt++) {
+ pIter = &pCommitter->aDataIter[iIter];
+ pIter->type = STT_DATA_ITER;
+ pIter->iStt = iStt;
+
+ code = tsdbReadSttBlk(pCommitter->dReader.pReader, iStt, pIter->aSttBlk);
+ if (code) goto _err;
+
+ if (taosArrayGetSize(pIter->aSttBlk) == 0) continue;
+
+ pIter->iSttBlk = 0;
+ SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, 0);
+ code = tsdbReadSttBlock(pCommitter->dReader.pReader, iStt, pSttBlk, &pIter->bData);
+ if (code) goto _err;
+
+ pIter->iRow = 0;
+ pIter->r.suid = pIter->bData.suid;
+ pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0];
+ pIter->r.row = tsdbRowFromBlockData(&pIter->bData, 0);
+
+ tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pIter);
+ iIter++;
+ }
+ } else {
+ for (int32_t iStt = 0; iStt < pReader->pSet->nSttF; iStt++) {
+ SSttFile *pSttFile = pReader->pSet->aSttF[iStt];
+ if (pSttFile->size > pSttFile->offset) {
+ pCommitter->toLastOnly = 1;
+ break;
+ }
+ }
+ }
+ }
+
+ code = tsdbNextCommitRow(pCommitter);
+ if (code) goto _err;
+
+ return code;
+
+_err:
+ return code;
+}
+
static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
int32_t code = 0;
STsdb *pTsdb = pCommitter->pTsdb;
@@ -416,8 +480,8 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
pCommitter->nextKey = TSKEY_MAX;
// Reader
- pRSet = (SDFileSet *)taosArraySearch(pCommitter->fs.aDFileSet, &(SDFileSet){.fid = pCommitter->commitFid},
- tDFileSetCmprFn, TD_EQ);
+ SDFileSet tDFileSet = {.fid = pCommitter->commitFid};
+ pRSet = (SDFileSet *)taosArraySearch(pCommitter->fs.aDFileSet, &tDFileSet, tDFileSetCmprFn, TD_EQ);
if (pRSet) {
code = tsdbDataFReaderOpen(&pCommitter->dReader.pReader, pTsdb, pRSet);
if (code) goto _err;
@@ -427,68 +491,58 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
if (code) goto _err;
pCommitter->dReader.iBlockIdx = 0;
- if (pCommitter->dReader.iBlockIdx < taosArrayGetSize(pCommitter->dReader.aBlockIdx)) {
- pCommitter->dReader.pBlockIdx =
- (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx);
-
- code = tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock);
+ if (taosArrayGetSize(pCommitter->dReader.aBlockIdx) > 0) {
+ pCommitter->dReader.pBlockIdx = (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, 0);
+ code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock);
if (code) goto _err;
} else {
pCommitter->dReader.pBlockIdx = NULL;
}
tBlockDataReset(&pCommitter->dReader.bData);
-
- // last
- code = tsdbReadBlockL(pCommitter->dReader.pReader, pCommitter->dReader.aBlockL);
- if (code) goto _err;
-
- pCommitter->dReader.iBlockL = -1;
- pCommitter->dReader.iRow = -1;
- pCommitter->dReader.pRowInfo = &pCommitter->dReader.rowInfo;
- tBlockDataReset(&pCommitter->dReader.bDatal);
- code = tsdbCommitterNextLastRow(pCommitter);
- if (code) goto _err;
} else {
pCommitter->dReader.pBlockIdx = NULL;
- pCommitter->dReader.pRowInfo = NULL;
}
// Writer
- SHeadFile fHead;
- SDataFile fData;
- SLastFile fLast;
- SSmaFile fSma;
- SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .pLastF = &fLast, .pSmaF = &fSma};
+ SHeadFile fHead = {.commitID = pCommitter->commitID};
+ SDataFile fData = {.commitID = pCommitter->commitID};
+ SSmaFile fSma = {.commitID = pCommitter->commitID};
+ SSttFile fStt = {.commitID = pCommitter->commitID};
+ SDFileSet wSet = {.fid = pCommitter->commitFid, .pHeadF = &fHead, .pDataF = &fData, .pSmaF = &fSma};
if (pRSet) {
- wSet.diskId = pRSet->diskId;
- wSet.fid = pCommitter->commitFid;
- fHead = (SHeadFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0};
+ ASSERT(pRSet->nSttF <= pCommitter->sttTrigger);
fData = *pRSet->pDataF;
- fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0};
fSma = *pRSet->pSmaF;
+ wSet.diskId = pRSet->diskId;
+ if (pRSet->nSttF < pCommitter->sttTrigger) {
+ for (int32_t iStt = 0; iStt < pRSet->nSttF; iStt++) {
+ wSet.aSttF[iStt] = pRSet->aSttF[iStt];
+ }
+ wSet.nSttF = pRSet->nSttF + 1;
+ } else {
+ wSet.nSttF = 1;
+ }
} else {
SDiskID did = {0};
-
tfsAllocDisk(pTsdb->pVnode->pTfs, 0, &did);
-
tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did);
-
wSet.diskId = did;
- wSet.fid = pCommitter->commitFid;
- fHead = (SHeadFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0};
- fData = (SDataFile){.commitID = pCommitter->commitID, .size = 0};
- fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0};
- fSma = (SSmaFile){.commitID = pCommitter->commitID, .size = 0};
+ wSet.nSttF = 1;
}
+ wSet.aSttF[wSet.nSttF - 1] = &fStt;
code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet);
if (code) goto _err;
taosArrayClear(pCommitter->dWriter.aBlockIdx);
- taosArrayClear(pCommitter->dWriter.aBlockL);
+ taosArrayClear(pCommitter->dWriter.aSttBlk);
tMapDataReset(&pCommitter->dWriter.mBlock);
tBlockDataReset(&pCommitter->dWriter.bData);
tBlockDataReset(&pCommitter->dWriter.bDatal);
+ // open iter
+ code = tsdbOpenCommitIter(pCommitter);
+ if (code) goto _err;
+
_exit:
return code;
@@ -497,50 +551,45 @@ _err:
return code;
}
-static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) {
- int32_t code = 0;
- SBlockData *pBlockData = &pCommitter->dWriter.bData;
- SBlock block;
+int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapData *mDataBlk, int8_t cmprAlg) {
+ int32_t code = 0;
- ASSERT(pBlockData->nRow > 0);
+ if (pBlockData->nRow == 0) return code;
- if (pBlock) {
- block = *pBlock; // as a subblock
- } else {
- tBlockReset(&block); // as a new block
- }
+ SDataBlk dataBlk;
+ tDataBlkReset(&dataBlk);
// info
- block.nRow += pBlockData->nRow;
+ dataBlk.nRow += pBlockData->nRow;
for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) {
TSDBKEY key = {.ts = pBlockData->aTSKEY[iRow], .version = pBlockData->aVersion[iRow]};
if (iRow == 0) {
- if (tsdbKeyCmprFn(&block.minKey, &key) > 0) {
- block.minKey = key;
+ if (tsdbKeyCmprFn(&dataBlk.minKey, &key) > 0) {
+ dataBlk.minKey = key;
}
} else {
if (pBlockData->aTSKEY[iRow] == pBlockData->aTSKEY[iRow - 1]) {
- block.hasDup = 1;
+ dataBlk.hasDup = 1;
}
}
- if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&block.maxKey, &key) < 0) {
- block.maxKey = key;
+ if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&dataBlk.maxKey, &key) < 0) {
+ dataBlk.maxKey = key;
}
- block.minVer = TMIN(block.minVer, key.version);
- block.maxVer = TMAX(block.maxVer, key.version);
+ dataBlk.minVer = TMIN(dataBlk.minVer, key.version);
+ dataBlk.maxVer = TMAX(dataBlk.maxVer, key.version);
}
// write
- block.nSubBlock++;
- code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &block.aSubBlock[block.nSubBlock - 1],
- ((block.nSubBlock == 1) && !block.hasDup) ? &block.smaInfo : NULL, pCommitter->cmprAlg, 0);
+ dataBlk.nSubBlock++;
+ code = tsdbWriteBlockData(pWriter, pBlockData, &dataBlk.aSubBlock[dataBlk.nSubBlock - 1],
+ ((dataBlk.nSubBlock == 1) && !dataBlk.hasDup) ? &dataBlk.smaInfo : NULL, cmprAlg, 0);
if (code) goto _err;
- // put SBlock
- code = tMapDataPutItem(&pCommitter->dWriter.mBlock, &block, tPutBlock);
+ // put SDataBlk
+ code = tMapDataPutItem(mDataBlk, &dataBlk, tPutDataBlk);
if (code) goto _err;
// clear
@@ -549,39 +598,38 @@ static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) {
return code;
_err:
- tsdbError("vgId:%d tsdb commit data block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
+ tsdbError("vgId:%d tsdb commit data block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
return code;
}
-static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) {
- int32_t code = 0;
- SBlockL blockL;
- SBlockData *pBlockData = &pCommitter->dWriter.bDatal;
+int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray *aSttBlk, int8_t cmprAlg) {
+ int32_t code = 0;
+ SSttBlk sstBlk;
- ASSERT(pBlockData->nRow > 0);
+ if (pBlockData->nRow == 0) return code;
// info
- blockL.suid = pBlockData->suid;
- blockL.nRow = pBlockData->nRow;
- blockL.minKey = TSKEY_MAX;
- blockL.maxKey = TSKEY_MIN;
- blockL.minVer = VERSION_MAX;
- blockL.maxVer = VERSION_MIN;
+ sstBlk.suid = pBlockData->suid;
+ sstBlk.nRow = pBlockData->nRow;
+ sstBlk.minKey = TSKEY_MAX;
+ sstBlk.maxKey = TSKEY_MIN;
+ sstBlk.minVer = VERSION_MAX;
+ sstBlk.maxVer = VERSION_MIN;
for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) {
- blockL.minKey = TMIN(blockL.minKey, pBlockData->aTSKEY[iRow]);
- blockL.maxKey = TMAX(blockL.maxKey, pBlockData->aTSKEY[iRow]);
- blockL.minVer = TMIN(blockL.minVer, pBlockData->aVersion[iRow]);
- blockL.maxVer = TMAX(blockL.maxVer, pBlockData->aVersion[iRow]);
+ sstBlk.minKey = TMIN(sstBlk.minKey, pBlockData->aTSKEY[iRow]);
+ sstBlk.maxKey = TMAX(sstBlk.maxKey, pBlockData->aTSKEY[iRow]);
+ sstBlk.minVer = TMIN(sstBlk.minVer, pBlockData->aVersion[iRow]);
+ sstBlk.maxVer = TMAX(sstBlk.maxVer, pBlockData->aVersion[iRow]);
}
- blockL.minUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[0];
- blockL.maxUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[pBlockData->nRow - 1];
+ sstBlk.minUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[0];
+ sstBlk.maxUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[pBlockData->nRow - 1];
// write
- code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &blockL.bInfo, NULL, pCommitter->cmprAlg, 1);
+ code = tsdbWriteBlockData(pWriter, pBlockData, &sstBlk.bInfo, NULL, cmprAlg, 1);
if (code) goto _err;
- // push SBlockL
- if (taosArrayPush(pCommitter->dWriter.aBlockL, &blockL) == NULL) {
+ // push SSttBlk
+ if (taosArrayPush(aSttBlk, &sstBlk) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
@@ -592,502 +640,7 @@ static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) {
return code;
_err:
- tsdbError("vgId:%d tsdb commit last block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
- return code;
-}
-
-static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter, SBlock *pBlock) {
- int32_t code = 0;
- STbData *pTbData = pIter->pTbData;
- SBlockData *pBlockDataR = &pCommitter->dReader.bData;
- SBlockData *pBlockDataW = &pCommitter->dWriter.bData;
-
- code = tsdbReadDataBlock(pCommitter->dReader.pReader, pBlock, pBlockDataR);
- if (code) goto _err;
-
- tBlockDataClear(pBlockDataW);
- int32_t iRow = 0;
- TSDBROW row;
- TSDBROW *pRow1 = tsdbTbDataIterGet(pIter);
- TSDBROW *pRow2 = &row;
- *pRow2 = tsdbRowFromBlockData(pBlockDataR, iRow);
- while (pRow1 && pRow2) {
- int32_t c = tsdbRowCmprFn(pRow1, pRow2);
-
- if (c < 0) {
- code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow1));
- if (code) goto _err;
-
- code = tBlockDataAppendRow(pBlockDataW, pRow1, pCommitter->skmRow.pTSchema, pTbData->uid);
- if (code) goto _err;
-
- // next
- tsdbTbDataIterNext(pIter);
- pRow1 = tsdbTbDataIterGet(pIter);
- } else if (c > 0) {
- code = tBlockDataAppendRow(pBlockDataW, pRow2, NULL, pTbData->uid);
- if (code) goto _err;
-
- iRow++;
- if (iRow < pBlockDataR->nRow) {
- *pRow2 = tsdbRowFromBlockData(pBlockDataR, iRow);
- } else {
- pRow2 = NULL;
- }
- } else {
- ASSERT(0);
- }
-
- // check
- if (pBlockDataW->nRow >= pCommitter->maxRow * 4 / 5) {
- code = tsdbCommitDataBlock(pCommitter, NULL);
- if (code) goto _err;
- }
- }
-
- while (pRow2) {
- code = tBlockDataAppendRow(pBlockDataW, pRow2, NULL, pTbData->uid);
- if (code) goto _err;
-
- iRow++;
- if (iRow < pBlockDataR->nRow) {
- *pRow2 = tsdbRowFromBlockData(pBlockDataR, iRow);
- } else {
- pRow2 = NULL;
- }
-
- // check
- if (pBlockDataW->nRow >= pCommitter->maxRow * 4 / 5) {
- code = tsdbCommitDataBlock(pCommitter, NULL);
- if (code) goto _err;
- }
- }
-
- // check
- if (pBlockDataW->nRow > 0) {
- code = tsdbCommitDataBlock(pCommitter, NULL);
- if (code) goto _err;
- }
-
- return code;
-
-_err:
- tsdbError("vgId:%d, tsdb merge commit data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
- return code;
-}
-
-static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter, TSDBKEY toKey) {
- int32_t code = 0;
- STbData *pTbData = pIter->pTbData;
- SBlockData *pBlockData = &pCommitter->dWriter.bData;
-
- tBlockDataClear(pBlockData);
- TSDBROW *pRow = tsdbTbDataIterGet(pIter);
- while (true) {
- if (pRow == NULL) {
- if (pBlockData->nRow > 0) {
- goto _write_block;
- } else {
- break;
- }
- }
-
- // update schema
- code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow));
- if (code) goto _err;
-
- // append
- code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid);
- if (code) goto _err;
-
- tsdbTbDataIterNext(pIter);
- pRow = tsdbTbDataIterGet(pIter);
- if (pRow) {
- TSDBKEY rowKey = TSDBROW_KEY(pRow);
- if (tsdbKeyCmprFn(&rowKey, &toKey) >= 0) {
- pRow = NULL;
- }
- }
-
- if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) {
- _write_block:
- code = tsdbCommitDataBlock(pCommitter, NULL);
- if (code) goto _err;
- }
- }
-
- return code;
-
-_err:
- tsdbError("vgId:%d, tsdb commit table mem data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
- return code;
-}
-
-static int32_t tsdbGetNumOfRowsLessThan(STbDataIter *pIter, TSDBKEY key) {
- int32_t nRow = 0;
-
- STbDataIter iter = *pIter;
- while (true) {
- TSDBROW *pRow = tsdbTbDataIterGet(&iter);
- if (pRow == NULL) break;
-
- int32_t c = tsdbKeyCmprFn(&TSDBROW_KEY(pRow), &key);
- if (c < 0) {
- nRow++;
- tsdbTbDataIterNext(&iter);
- } else if (c > 0) {
- break;
- } else {
- ASSERT(0);
- }
- }
-
- return nRow;
-}
-
-static int32_t tsdbMergeAsSubBlock(SCommitter *pCommitter, STbDataIter *pIter, SBlock *pBlock) {
- int32_t code = 0;
- STbData *pTbData = pIter->pTbData;
- SBlockData *pBlockData = &pCommitter->dWriter.bData;
-
- tBlockDataClear(pBlockData);
- TSDBROW *pRow = tsdbTbDataIterGet(pIter);
- while (true) {
- if (pRow == NULL) break;
-
- code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow));
- if (code) goto _err;
-
- code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid);
- if (code) goto _err;
-
- tsdbTbDataIterNext(pIter);
- pRow = tsdbTbDataIterGet(pIter);
- if (pRow) {
- TSDBKEY rowKey = TSDBROW_KEY(pRow);
- if (tsdbKeyCmprFn(&rowKey, &pBlock->maxKey) > 0) {
- pRow = NULL;
- }
- }
- }
-
- ASSERT(pBlockData->nRow > 0 && pBlock->nRow + pBlockData->nRow <= pCommitter->maxRow);
-
- code = tsdbCommitDataBlock(pCommitter, pBlock);
- if (code) goto _err;
-
- return code;
-
-_err:
- tsdbError("vgId:%d, tsdb merge as subblock failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
- return code;
-}
-
-static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) {
- int32_t code = 0;
- STbData *pTbData = pIter->pTbData;
- int32_t nRow = tsdbGetNumOfRowsLessThan(pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN});
-
- if (pCommitter->dReader.pRowInfo && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pRowInfo) == 0) {
- if (pCommitter->dReader.pRowInfo->suid) { // super table
- for (int32_t iRow = pCommitter->dReader.iRow; iRow < pCommitter->dReader.bDatal.nRow; iRow++) {
- if (pTbData->uid != pCommitter->dReader.bDatal.aUid[iRow]) break;
- nRow++;
- }
- } else { // normal table
- ASSERT(pCommitter->dReader.iRow == 0);
- nRow += pCommitter->dReader.bDatal.nRow;
- }
- }
-
- if (nRow == 0) goto _exit;
-
- TSDBROW *pRow = tsdbTbDataIterGet(pIter);
- if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) {
- pRow = NULL;
- }
-
- SRowInfo *pRowInfo = pCommitter->dReader.pRowInfo;
- if (pRowInfo && pRowInfo->uid != pTbData->uid) {
- pRowInfo = NULL;
- }
-
- while (nRow) {
- SBlockData *pBlockData;
- int8_t toData;
-
- if (nRow < pCommitter->minRow) { // to .last
- toData = 0;
- pBlockData = &pCommitter->dWriter.bDatal;
-
- // commit and reset block data schema if need
- // QUESTION: Is there a case that pBlockData->nRow == 0 but need to change schema ?
- if (pBlockData->suid || pBlockData->uid) {
- if (pBlockData->suid != pTbData->suid || pBlockData->suid == 0) {
- if (pBlockData->nRow > 0) {
- code = tsdbCommitLastBlock(pCommitter);
- if (code) goto _err;
- }
-
- tBlockDataReset(pBlockData);
- }
- }
-
- // set block data schema if need
- if (pBlockData->suid == 0 && pBlockData->uid == 0) {
- code = tsdbCommitterUpdateTableSchema(pCommitter, pTbData->suid, pTbData->uid);
- if (code) goto _err;
-
- code =
- tBlockDataInit(pBlockData, pTbData->suid, pTbData->suid ? 0 : pTbData->uid, pCommitter->skmTable.pTSchema);
- if (code) goto _err;
- }
-
- if (pBlockData->nRow + nRow > pCommitter->maxRow) {
- code = tsdbCommitLastBlock(pCommitter);
- if (code) goto _err;
- }
- } else { // to .data
- toData = 1;
- pBlockData = &pCommitter->dWriter.bData;
- ASSERT(pBlockData->nRow == 0);
- }
-
- while (pRow && pRowInfo) {
- int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row);
- if (c < 0) {
- code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow));
- if (code) goto _err;
-
- code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid);
- if (code) goto _err;
-
- tsdbTbDataIterNext(pIter);
- pRow = tsdbTbDataIterGet(pIter);
- if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) {
- pRow = NULL;
- }
- } else if (c > 0) {
- code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL, pTbData->uid);
- if (code) goto _err;
-
- code = tsdbCommitterNextLastRow(pCommitter);
- if (code) goto _err;
-
- pRowInfo = pCommitter->dReader.pRowInfo;
- if (pRowInfo && pRowInfo->uid != pTbData->uid) {
- pRowInfo = NULL;
- }
- } else {
- ASSERT(0);
- }
-
- nRow--;
- if (toData) {
- if (nRow == 0 || pBlockData->nRow >= pCommitter->maxRow * 4 / 5) {
- code = tsdbCommitDataBlock(pCommitter, NULL);
- if (code) goto _err;
- goto _outer_break;
- }
- }
- }
-
- while (pRow) {
- code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow));
- if (code) goto _err;
-
- code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid);
- if (code) goto _err;
-
- tsdbTbDataIterNext(pIter);
- pRow = tsdbTbDataIterGet(pIter);
- if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) {
- pRow = NULL;
- }
-
- nRow--;
- if (toData) {
- if (nRow == 0 || pBlockData->nRow >= pCommitter->maxRow * 4 / 5) {
- code = tsdbCommitDataBlock(pCommitter, NULL);
- if (code) goto _err;
- goto _outer_break;
- }
- }
- }
-
- while (pRowInfo) {
- code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL, pTbData->uid);
- if (code) goto _err;
-
- code = tsdbCommitterNextLastRow(pCommitter);
- if (code) goto _err;
-
- pRowInfo = pCommitter->dReader.pRowInfo;
- if (pRowInfo && pRowInfo->uid != pTbData->uid) {
- pRowInfo = NULL;
- }
-
- nRow--;
- if (toData) {
- if (nRow == 0 || pBlockData->nRow >= pCommitter->maxRow * 4 / 5) {
- code = tsdbCommitDataBlock(pCommitter, NULL);
- if (code) goto _err;
- goto _outer_break;
- }
- }
- }
-
- _outer_break:
- ASSERT(nRow >= 0);
- }
-
-_exit:
- return code;
-
-_err:
- tsdbError("vgId:%d tsdb merge commit last failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
- return code;
-}
-
-static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) {
- int32_t code = 0;
-
- ASSERT(pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, pTbData) >= 0);
- ASSERT(pCommitter->dReader.pRowInfo == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRowInfo, pTbData) >= 0);
-
- // merge commit table data
- STbDataIter iter = {0};
- STbDataIter *pIter = &iter;
- TSDBROW *pRow;
-
- tsdbTbDataIterOpen(pTbData, &(TSDBKEY){.ts = pCommitter->minKey, .version = VERSION_MIN}, 0, pIter);
- pRow = tsdbTbDataIterGet(pIter);
- if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) {
- pRow = NULL;
- }
-
- if (pRow == NULL) {
- if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, pTbData) == 0) {
- SBlockIdx blockIdx = {.suid = pTbData->suid, .uid = pTbData->uid};
- code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx);
- if (code) goto _err;
-
- if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
- }
-
- goto _exit;
- }
-
- int32_t iBlock = 0;
- SBlock block;
- SBlock *pBlock = █
- if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pBlockIdx) == 0) {
- tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock);
- } else {
- pBlock = NULL;
- }
-
- code = tsdbCommitterUpdateTableSchema(pCommitter, pTbData->suid, pTbData->uid);
- if (code) goto _err;
-
- tMapDataReset(&pCommitter->dWriter.mBlock);
- code = tBlockDataInit(&pCommitter->dReader.bData, pTbData->suid, pTbData->uid, pCommitter->skmTable.pTSchema);
- if (code) goto _err;
- code = tBlockDataInit(&pCommitter->dWriter.bData, pTbData->suid, pTbData->uid, pCommitter->skmTable.pTSchema);
- if (code) goto _err;
-
- // .data merge
- while (pBlock && pRow) {
- int32_t c = tBlockCmprFn(pBlock, &(SBlock){.minKey = TSDBROW_KEY(pRow), .maxKey = TSDBROW_KEY(pRow)});
- if (c < 0) { // disk
- code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock);
- if (code) goto _err;
-
- // next
- iBlock++;
- if (iBlock < pCommitter->dReader.mBlock.nItem) {
- tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock);
- } else {
- pBlock = NULL;
- }
- } else if (c > 0) { // memory
- code = tsdbCommitTableMemData(pCommitter, pIter, pBlock->minKey);
- if (code) goto _err;
-
- // next
- pRow = tsdbTbDataIterGet(pIter);
- if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) {
- pRow = NULL;
- }
- } else { // merge
- int32_t nOvlp = tsdbGetNumOfRowsLessThan(pIter, pBlock->maxKey);
-
- ASSERT(nOvlp > 0);
-
- if (pBlock->nRow + nOvlp <= pCommitter->maxRow && pBlock->nSubBlock < TSDB_MAX_SUBBLOCKS) {
- code = tsdbMergeAsSubBlock(pCommitter, pIter, pBlock);
- if (code) goto _err;
- } else {
- code = tsdbMergeCommitData(pCommitter, pIter, pBlock);
- if (code) goto _err;
- }
-
- // next
- pRow = tsdbTbDataIterGet(pIter);
- if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) {
- pRow = NULL;
- }
- iBlock++;
- if (iBlock < pCommitter->dReader.mBlock.nItem) {
- tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock);
- } else {
- pBlock = NULL;
- }
- }
- }
-
- while (pBlock) {
- code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock);
- if (code) goto _err;
-
- // next
- iBlock++;
- if (iBlock < pCommitter->dReader.mBlock.nItem) {
- tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock);
- } else {
- pBlock = NULL;
- }
- }
-
- // .data append and .last merge
- code = tsdbMergeCommitLast(pCommitter, pIter);
- if (code) goto _err;
-
- // end
- if (pCommitter->dWriter.mBlock.nItem > 0) {
- SBlockIdx blockIdx = {.suid = pTbData->suid, .uid = pTbData->uid};
- code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx);
- if (code) goto _err;
-
- if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
- }
-
-_exit:
- pRow = tsdbTbDataIterGet(pIter);
- if (pRow) {
- pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow));
- }
-
- return code;
-
-_err:
- tsdbError("vgId:%d tsdb commit table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
+ tsdbError("vgId:%d tsdb commit last block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
return code;
}
@@ -1098,8 +651,8 @@ static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) {
code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx);
if (code) goto _err;
- // write aBlockL
- code = tsdbWriteBlockL(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockL);
+ // write aSttBlk
+ code = tsdbWriteSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.aSttBlk);
if (code) goto _err;
// update file header
@@ -1130,12 +683,9 @@ _err:
static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) {
int32_t code = 0;
- // .data
- while (true) {
- if (pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) >= 0) break;
-
+ while (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) < 0) {
SBlockIdx blockIdx = *pCommitter->dReader.pBlockIdx;
- code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx);
+ code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx);
if (code) goto _err;
if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) {
@@ -1147,71 +697,6 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) {
if (code) goto _err;
}
- // .last
- while (true) {
- if (pCommitter->dReader.pRowInfo == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRowInfo, &toTable) >= 0) break;
-
- SBlockData *pBlockDataR = &pCommitter->dReader.bDatal;
- SBlockData *pBlockDataW = &pCommitter->dWriter.bDatal;
- tb_uid_t suid = pCommitter->dReader.pRowInfo->suid;
- tb_uid_t uid = pCommitter->dReader.pRowInfo->uid;
-
- ASSERT((pBlockDataR->suid && !pBlockDataR->uid) || (!pBlockDataR->suid && pBlockDataR->uid));
- ASSERT(pBlockDataR->nRow > 0);
-
- // commit and reset block data schema if need
- if (pBlockDataW->suid || pBlockDataW->uid) {
- if (pBlockDataW->suid != suid || pBlockDataW->suid == 0) {
- if (pBlockDataW->nRow > 0) {
- code = tsdbCommitLastBlock(pCommitter);
- if (code) goto _err;
- }
- tBlockDataReset(pBlockDataW);
- }
- }
-
- // set block data schema if need
- if (pBlockDataW->suid == 0 && pBlockDataW->uid == 0) {
- code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid);
- if (code) goto _err;
-
- code = tBlockDataInit(pBlockDataW, suid, suid ? 0 : uid, pCommitter->skmTable.pTSchema);
- if (code) goto _err;
- }
-
- // check if it can make sure that one table data in one block
- int32_t nRow = 0;
- if (pBlockDataR->suid) {
- int32_t iRow = pCommitter->dReader.iRow;
- while ((iRow < pBlockDataR->nRow) && (pBlockDataR->aUid[iRow] == uid)) {
- nRow++;
- iRow++;
- }
- } else {
- ASSERT(pCommitter->dReader.iRow == 0);
- nRow = pBlockDataR->nRow;
- }
-
- ASSERT(nRow > 0 && nRow < pCommitter->minRow);
-
- if (pBlockDataW->nRow + nRow > pCommitter->maxRow) {
- ASSERT(pBlockDataW->nRow > 0);
-
- code = tsdbCommitLastBlock(pCommitter);
- if (code) goto _err;
- }
-
- while (nRow > 0) {
- code = tBlockDataAppendRow(pBlockDataW, &pCommitter->dReader.pRowInfo->row, NULL, uid);
- if (code) goto _err;
-
- code = tsdbCommitterNextLastRow(pCommitter);
- if (code) goto _err;
-
- nRow--;
- }
- }
-
return code;
_err:
@@ -1219,6 +704,7 @@ _err:
return code;
}
+static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter);
static int32_t tsdbCommitFileData(SCommitter *pCommitter) {
int32_t code = 0;
STsdb *pTsdb = pCommitter->pTsdb;
@@ -1228,33 +714,10 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) {
code = tsdbCommitFileDataStart(pCommitter);
if (code) goto _err;
- // commit file data impl
- for (int32_t iTbData = 0; iTbData < taosArrayGetSize(pCommitter->aTbDataP); iTbData++) {
- STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData);
-
- // move commit until current (suid, uid)
- code = tsdbMoveCommitData(pCommitter, *(TABLEID *)pTbData);
- if (code) goto _err;
-
- // commit current table data
- code = tsdbCommitTableData(pCommitter, pTbData);
- if (code) goto _err;
-
- // move next reader table data if need
- if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pBlockIdx) == 0) {
- code = tsdbCommitterNextTableData(pCommitter);
- if (code) goto _err;
- }
- }
-
- code = tsdbMoveCommitData(pCommitter, (TABLEID){.suid = INT64_MAX, .uid = INT64_MAX});
+ // impl
+ code = tsdbCommitFileDataImpl(pCommitter);
if (code) goto _err;
- if (pCommitter->dWriter.bDatal.nRow > 0) {
- code = tsdbCommitLastBlock(pCommitter);
- if (code) goto _err;
- }
-
// commit file data end
code = tsdbCommitFileDataEnd(pCommitter);
if (code) goto _err;
@@ -1287,12 +750,12 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) {
pCommitter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows;
pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows;
pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression;
+ pCommitter->sttTrigger = pTsdb->pVnode->config.sttTrigger;
pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem);
if (pCommitter->aTbDataP == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
-
code = tsdbFSCopy(pTsdb, &pCommitter->fs);
if (code) goto _err;
@@ -1306,7 +769,7 @@ _err:
static int32_t tsdbCommitDataStart(SCommitter *pCommitter) {
int32_t code = 0;
- // Reader
+ // reader
pCommitter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
if (pCommitter->dReader.aBlockIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
@@ -1316,24 +779,28 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) {
code = tBlockDataCreate(&pCommitter->dReader.bData);
if (code) goto _exit;
- pCommitter->dReader.aBlockL = taosArrayInit(0, sizeof(SBlockL));
- if (pCommitter->dReader.aBlockL == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _exit;
+ // merger
+ for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) {
+ SDataIter *pIter = &pCommitter->aDataIter[iStt];
+ pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
+ if (pIter->aSttBlk == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _exit;
+ }
+
+ code = tBlockDataCreate(&pIter->bData);
+ if (code) goto _exit;
}
- code = tBlockDataCreate(&pCommitter->dReader.bDatal);
- if (code) goto _exit;
-
- // Writer
+ // writer
pCommitter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
if (pCommitter->dWriter.aBlockIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
- pCommitter->dWriter.aBlockL = taosArrayInit(0, sizeof(SBlockL));
- if (pCommitter->dWriter.aBlockL == NULL) {
+ pCommitter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
+ if (pCommitter->dWriter.aSttBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
@@ -1349,16 +816,21 @@ _exit:
}
static void tsdbCommitDataEnd(SCommitter *pCommitter) {
- // Reader
+ // reader
taosArrayDestroy(pCommitter->dReader.aBlockIdx);
tMapDataClear(&pCommitter->dReader.mBlock);
tBlockDataDestroy(&pCommitter->dReader.bData, 1);
- taosArrayDestroy(pCommitter->dReader.aBlockL);
- tBlockDataDestroy(&pCommitter->dReader.bDatal, 1);
- // Writer
+ // merger
+ for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) {
+ SDataIter *pIter = &pCommitter->aDataIter[iStt];
+ taosArrayDestroy(pIter->aSttBlk);
+ tBlockDataDestroy(&pIter->bData, 1);
+ }
+
+ // writer
taosArrayDestroy(pCommitter->dWriter.aBlockIdx);
- taosArrayDestroy(pCommitter->dWriter.aBlockL);
+ taosArrayDestroy(pCommitter->dWriter.aSttBlk);
tMapDataClear(&pCommitter->dWriter.mBlock);
tBlockDataDestroy(&pCommitter->dWriter.bData, 1);
tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1);
@@ -1389,7 +861,7 @@ static int32_t tsdbCommitData(SCommitter *pCommitter) {
tsdbCommitDataEnd(pCommitter);
_exit:
- tsdbDebug("vgId:%d, commit data done, nRow:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nRow);
+ tsdbInfo("vgId:%d, commit data done, nRow:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nRow);
return code;
_err:
@@ -1515,6 +987,11 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) {
tsdbFSDestroy(&pCommitter->fs);
taosArrayDestroy(pCommitter->aTbDataP);
+ // if (pCommitter->toMerge) {
+ // code = tsdbMerge(pTsdb);
+ // if (code) goto _err;
+ // }
+
tsdbInfo("vgId:%d, tsdb end commit", TD_VID(pTsdb->pVnode));
return code;
@@ -1522,3 +999,473 @@ _err:
tsdbError("vgId:%d, tsdb end commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code;
}
+
+// ================================================================================
+
+static FORCE_INLINE SRowInfo *tsdbGetCommitRow(SCommitter *pCommitter) {
+ return (pCommitter->pIter) ? &pCommitter->pIter->r : NULL;
+}
+
+static int32_t tsdbNextCommitRow(SCommitter *pCommitter) {
+ int32_t code = 0;
+
+ if (pCommitter->pIter) {
+ SDataIter *pIter = pCommitter->pIter;
+ if (pCommitter->pIter->type == MEMORY_DATA_ITER) { // memory
+ tsdbTbDataIterNext(&pIter->iter);
+ TSDBROW *pRow = tsdbTbDataIterGet(&pIter->iter);
+ while (true) {
+ if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) {
+ pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow));
+ pRow = NULL;
+ }
+
+ if (pRow) {
+ pIter->r.suid = pIter->iter.pTbData->suid;
+ pIter->r.uid = pIter->iter.pTbData->uid;
+ pIter->r.row = *pRow;
+ break;
+ }
+
+ pIter->iTbDataP++;
+ if (pIter->iTbDataP < taosArrayGetSize(pCommitter->aTbDataP)) {
+ STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, pIter->iTbDataP);
+ TSDBKEY keyFrom = {.ts = pCommitter->minKey, .version = VERSION_MIN};
+ tsdbTbDataIterOpen(pTbData, &keyFrom, 0, &pIter->iter);
+ pRow = tsdbTbDataIterGet(&pIter->iter);
+ continue;
+ } else {
+ pCommitter->pIter = NULL;
+ break;
+ }
+ }
+ } else if (pCommitter->pIter->type == STT_DATA_ITER) { // last file
+ pIter->iRow++;
+ if (pIter->iRow < pIter->bData.nRow) {
+ pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow];
+ pIter->r.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow);
+ } else {
+ pIter->iSttBlk++;
+ if (pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk)) {
+ SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk);
+
+ code = tsdbReadSttBlock(pCommitter->dReader.pReader, pIter->iStt, pSttBlk, &pIter->bData);
+ if (code) goto _exit;
+
+ pIter->iRow = 0;
+ pIter->r.suid = pIter->bData.suid;
+ pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0];
+ pIter->r.row = tsdbRowFromBlockData(&pIter->bData, 0);
+ } else {
+ pCommitter->pIter = NULL;
+ }
+ }
+ } else {
+ ASSERT(0);
+ }
+
+ // compare with min in RB Tree
+ pIter = (SDataIter *)tRBTreeMin(&pCommitter->rbt);
+ if (pCommitter->pIter && pIter) {
+ int32_t c = tRowInfoCmprFn(&pCommitter->pIter->r, &pIter->r);
+ if (c > 0) {
+ tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pCommitter->pIter);
+ pCommitter->pIter = NULL;
+ } else {
+ ASSERT(c);
+ }
+ }
+ }
+
+ if (pCommitter->pIter == NULL) {
+ pCommitter->pIter = (SDataIter *)tRBTreeMin(&pCommitter->rbt);
+ if (pCommitter->pIter) {
+ tRBTreeDrop(&pCommitter->rbt, (SRBTreeNode *)pCommitter->pIter);
+ }
+ }
+
+_exit:
+ return code;
+}
+
+static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) {
+ int32_t code = 0;
+ SBlockData *pBlockData = &pCommitter->dWriter.bData;
+ SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter);
+ TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid};
+
+ tBlockDataClear(pBlockData);
+ while (pRowInfo) {
+ ASSERT(pRowInfo->row.type == 0);
+ code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row));
+ if (code) goto _err;
+
+ code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid);
+ if (code) goto _err;
+
+ code = tsdbNextCommitRow(pCommitter);
+ if (code) goto _err;
+
+ pRowInfo = tsdbGetCommitRow(pCommitter);
+ if (pRowInfo) {
+ if (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid) {
+ pRowInfo = NULL;
+ } else {
+ TSDBKEY tKey = TSDBROW_KEY(&pRowInfo->row);
+ if (tsdbKeyCmprFn(&tKey, &pDataBlk->minKey) >= 0) pRowInfo = NULL;
+ }
+ }
+
+ if (pBlockData->nRow >= pCommitter->maxRow) {
+ code =
+ tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
+ if (code) goto _err;
+ }
+ }
+
+ code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
+ if (code) goto _err;
+
+ return code;
+
+_err:
+ tsdbError("vgId:%d, tsdb commit ahead block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
+ return code;
+}
+
+static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) {
+ int32_t code = 0;
+ SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter);
+ TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid};
+ SBlockData *pBDataR = &pCommitter->dReader.bData;
+ SBlockData *pBDataW = &pCommitter->dWriter.bData;
+
+ code = tsdbReadDataBlock(pCommitter->dReader.pReader, pDataBlk, pBDataR);
+ if (code) goto _err;
+
+ tBlockDataClear(pBDataW);
+ int32_t iRow = 0;
+ TSDBROW row = tsdbRowFromBlockData(pBDataR, 0);
+ TSDBROW *pRow = &row;
+
+ while (pRow && pRowInfo) {
+ int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row);
+ if (c < 0) {
+ code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid);
+ if (code) goto _err;
+
+ iRow++;
+ if (iRow < pBDataR->nRow) {
+ row = tsdbRowFromBlockData(pBDataR, iRow);
+ } else {
+ pRow = NULL;
+ }
+ } else if (c > 0) {
+ ASSERT(pRowInfo->row.type == 0);
+ code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row));
+ if (code) goto _err;
+
+ code = tBlockDataAppendRow(pBDataW, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid);
+ if (code) goto _err;
+
+ code = tsdbNextCommitRow(pCommitter);
+ if (code) goto _err;
+
+ pRowInfo = tsdbGetCommitRow(pCommitter);
+ if (pRowInfo) {
+ if (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid) {
+ pRowInfo = NULL;
+ } else {
+ TSDBKEY tKey = TSDBROW_KEY(&pRowInfo->row);
+ if (tsdbKeyCmprFn(&tKey, &pDataBlk->maxKey) > 0) pRowInfo = NULL;
+ }
+ }
+ } else {
+ ASSERT(0);
+ }
+
+ if (pBDataW->nRow >= pCommitter->maxRow) {
+ code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
+ if (code) goto _err;
+ }
+ }
+
+ while (pRow) {
+ code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid);
+ if (code) goto _err;
+
+ iRow++;
+ if (iRow < pBDataR->nRow) {
+ row = tsdbRowFromBlockData(pBDataR, iRow);
+ } else {
+ pRow = NULL;
+ }
+
+ if (pBDataW->nRow >= pCommitter->maxRow) {
+ code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
+ if (code) goto _err;
+ }
+ }
+
+ code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
+ if (code) goto _err;
+
+ return code;
+
+_err:
+ tsdbError("vgId:%d, tsdb commit merge block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
+ return code;
+}
+
+static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) {
+ int32_t code = 0;
+ SBlockIdx *pBlockIdx = pCommitter->dReader.pBlockIdx;
+
+ ASSERT(pBlockIdx == NULL || tTABLEIDCmprFn(pBlockIdx, &id) >= 0);
+ if (pBlockIdx && pBlockIdx->suid == id.suid && pBlockIdx->uid == id.uid) {
+ int32_t iBlock = 0;
+ SDataBlk block;
+ SDataBlk *pDataBlk = █
+ SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter);
+
+ ASSERT(pRowInfo->suid == id.suid && pRowInfo->uid == id.uid);
+
+ tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk);
+ while (pDataBlk && pRowInfo) {
+ SDataBlk tBlock = {.minKey = TSDBROW_KEY(&pRowInfo->row), .maxKey = TSDBROW_KEY(&pRowInfo->row)};
+ int32_t c = tDataBlkCmprFn(pDataBlk, &tBlock);
+
+ if (c < 0) {
+ code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk);
+ if (code) goto _err;
+
+ iBlock++;
+ if (iBlock < pCommitter->dReader.mBlock.nItem) {
+ tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk);
+ } else {
+ pDataBlk = NULL;
+ }
+ } else if (c > 0) {
+ code = tsdbCommitAheadBlock(pCommitter, pDataBlk);
+ if (code) goto _err;
+
+ pRowInfo = tsdbGetCommitRow(pCommitter);
+ if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL;
+ } else {
+ code = tsdbCommitMergeBlock(pCommitter, pDataBlk);
+ if (code) goto _err;
+
+ iBlock++;
+ if (iBlock < pCommitter->dReader.mBlock.nItem) {
+ tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk);
+ } else {
+ pDataBlk = NULL;
+ }
+ pRowInfo = tsdbGetCommitRow(pCommitter);
+ if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL;
+ }
+ }
+
+ while (pDataBlk) {
+ code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk);
+ if (code) goto _err;
+
+ iBlock++;
+ if (iBlock < pCommitter->dReader.mBlock.nItem) {
+ tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk);
+ } else {
+ pDataBlk = NULL;
+ }
+ }
+
+ code = tsdbCommitterNextTableData(pCommitter);
+ if (code) goto _err;
+ }
+
+_exit:
+ return code;
+
+_err:
+ tsdbError("vgId:%d tsdb merge table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
+ return code;
+}
+
+static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) {
+ int32_t code = 0;
+
+ SBlockData *pBDatal = &pCommitter->dWriter.bDatal;
+ if (pBDatal->suid || pBDatal->uid) {
+ if ((pBDatal->suid != id.suid) || (id.suid == 0)) {
+ code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg);
+ if (code) goto _exit;
+ tBlockDataReset(pBDatal);
+ }
+ }
+
+ if (!pBDatal->suid && !pBDatal->uid) {
+ ASSERT(pCommitter->skmTable.suid == id.suid);
+ ASSERT(pCommitter->skmTable.uid == id.uid);
+ code = tBlockDataInit(pBDatal, id.suid, id.suid ? 0 : id.uid, pCommitter->skmTable.pTSchema);
+ if (code) goto _exit;
+ }
+
+_exit:
+ return code;
+}
+
+static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) {
+ int32_t code = 0;
+
+ SBlockData *pBData = &pCommitter->dWriter.bData;
+ SBlockData *pBDatal = &pCommitter->dWriter.bDatal;
+
+ TABLEID id = {.suid = pBData->suid, .uid = pBData->uid};
+ code = tsdbInitLastBlockIfNeed(pCommitter, id);
+ if (code) goto _err;
+
+ for (int32_t iRow = 0; iRow < pBData->nRow; iRow++) {
+ TSDBROW row = tsdbRowFromBlockData(pBData, iRow);
+ code = tBlockDataAppendRow(pBDatal, &row, NULL, pBData->uid);
+ if (code) goto _err;
+
+ if (pBDatal->nRow >= pCommitter->maxRow) {
+ code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg);
+ if (code) goto _err;
+ }
+ }
+
+ return code;
+
+_err:
+ return code;
+}
+
+static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) {
+ int32_t code = 0;
+
+ SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter);
+ if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) {
+ pRowInfo = NULL;
+ }
+
+ if (pRowInfo == NULL) goto _exit;
+
+ SBlockData *pBData;
+ if (pCommitter->toLastOnly) {
+ pBData = &pCommitter->dWriter.bDatal;
+ code = tsdbInitLastBlockIfNeed(pCommitter, id);
+ if (code) goto _err;
+ } else {
+ pBData = &pCommitter->dWriter.bData;
+ ASSERT(pBData->nRow == 0);
+ }
+
+ while (pRowInfo) {
+ STSchema *pTSchema = NULL;
+ if (pRowInfo->row.type == 0) {
+ code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row));
+ if (code) goto _err;
+ pTSchema = pCommitter->skmRow.pTSchema;
+ }
+
+ code = tBlockDataAppendRow(pBData, &pRowInfo->row, pTSchema, id.uid);
+ if (code) goto _err;
+
+ code = tsdbNextCommitRow(pCommitter);
+ if (code) goto _err;
+
+ pRowInfo = tsdbGetCommitRow(pCommitter);
+ if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) {
+ pRowInfo = NULL;
+ }
+
+ if (pBData->nRow >= pCommitter->maxRow) {
+ if (pCommitter->toLastOnly) {
+ code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg);
+ if (code) goto _err;
+ } else {
+ code =
+ tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
+ if (code) goto _err;
+ }
+ }
+ }
+
+ if (!pCommitter->toLastOnly && pBData->nRow) {
+ if (pBData->nRow > pCommitter->minRow) {
+ code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
+ if (code) goto _err;
+ } else {
+ code = tsdbAppendLastBlock(pCommitter);
+ if (code) goto _err;
+ }
+ }
+
+_exit:
+ return code;
+
+_err:
+ tsdbError("vgId:%d tsdb commit table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
+ return code;
+}
+
+static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) {
+ int32_t code = 0;
+
+ SRowInfo *pRowInfo;
+ TABLEID id = {0};
+ while ((pRowInfo = tsdbGetCommitRow(pCommitter)) != NULL) {
+ ASSERT(pRowInfo->suid != id.suid || pRowInfo->uid != id.uid);
+ id.suid = pRowInfo->suid;
+ id.uid = pRowInfo->uid;
+
+ code = tsdbMoveCommitData(pCommitter, id);
+ if (code) goto _err;
+
+ // start
+ tMapDataReset(&pCommitter->dWriter.mBlock);
+
+ // impl
+ code = tsdbUpdateTableSchema(pCommitter->pTsdb->pVnode->pMeta, id.suid, id.uid, &pCommitter->skmTable);
+ if (code) goto _err;
+ code = tBlockDataInit(&pCommitter->dReader.bData, id.suid, id.uid, pCommitter->skmTable.pTSchema);
+ if (code) goto _err;
+ code = tBlockDataInit(&pCommitter->dWriter.bData, id.suid, id.uid, pCommitter->skmTable.pTSchema);
+ if (code) goto _err;
+
+ /* merge with data in .data file */
+ code = tsdbMergeTableData(pCommitter, id);
+ if (code) goto _err;
+
+ /* handle remain table data */
+ code = tsdbCommitTableData(pCommitter, id);
+ if (code) goto _err;
+
+ // end
+ if (pCommitter->dWriter.mBlock.nItem > 0) {
+ SBlockIdx blockIdx = {.suid = id.suid, .uid = id.uid};
+ code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx);
+ if (code) goto _err;
+
+ if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
+ }
+ }
+
+ id.suid = INT64_MAX;
+ id.uid = INT64_MAX;
+ code = tsdbMoveCommitData(pCommitter, id);
+ if (code) goto _err;
+
+ code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk,
+ pCommitter->cmprAlg);
+ if (code) goto _err;
+
+ return code;
+
+_err:
+ tsdbError("vgId:%d tsdb commit file data impl failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
+ return code;
+}
diff --git a/source/dnode/vnode/src/tsdb/tsdbCompact.c b/source/dnode/vnode/src/tsdb/tsdbCompact.c
new file mode 100644
index 0000000000..fb3917be64
--- /dev/null
+++ b/source/dnode/vnode/src/tsdb/tsdbCompact.c
@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see
+};
+
+int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder) {
+ int32_t code = 0;
+ // TODO
+ return code;
+}
+
+void tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) {
+ // TODO
+}
+
+void tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, int64_t suid, int64_t uid, STSchema *pTSchema, int8_t cmprAlg) {
+ pBuilder->hdr = (SDiskDataHdr){.delimiter = TSDB_FILE_DLMT, //
+ .fmtVer = 0,
+ .suid = suid,
+ .uid = uid,
+ .cmprAlg = cmprAlg};
+}
+
+void tDiskDataBuilderReset(SDiskDataBuilder *pBuilder) {
+ // TODO
+}
+
+int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, int64_t uid) {
+ int32_t code = 0;
+
+ // uid (todo)
+
+ // version (todo)
+
+ // TSKEY (todo)
+
+ SRowIter iter = {0};
+ tRowIterInit(&iter, pRow, pTSchema);
+
+ for (int32_t iDiskCol = 0; iDiskCol < 0; iDiskCol++) {
+ }
+
+ return code;
+}
+
+int32_t tDiskDataBuilderGet(SDiskDataBuilder *pBuilder, uint8_t **ppData) {
+ int32_t code = 0;
+ // TODO
+ return code;
+}
\ No newline at end of file
diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c
index 247de99338..6b4134f416 100644
--- a/source/dnode/vnode/src/tsdb/tsdbFS.c
+++ b/source/dnode/vnode/src/tsdb/tsdbFS.c
@@ -21,6 +21,9 @@ static int32_t tsdbEncodeFS(uint8_t *p, STsdbFS *pFS) {
int8_t hasDel = pFS->pDelFile ? 1 : 0;
uint32_t nSet = taosArrayGetSize(pFS->aDFileSet);
+ // version
+ n += tPutI8(p ? p + n : p, 0);
+
// SDelFile
n += tPutI8(p ? p + n : p, hasDel);
if (hasDel) {
@@ -110,7 +113,7 @@ _err:
// taosRemoveFile(fname);
// }
-// // last
+// // stt
// if (isSameDisk && pFrom->pLastF->commitID == pTo->pLastF->commitID) {
// if (pFrom->pLastF->size > pTo->pLastF->size) {
// code = tsdbDFileRollback(pFS->pTsdb, pTo, TSDB_LAST_FILE);
@@ -140,7 +143,7 @@ _err:
// tsdbDataFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pDataF, fname);
// taosRemoveFile(fname);
-// // last
+// // stt
// tsdbLastFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pLastF, fname);
// taosRemoveFile(fname);
@@ -254,8 +257,10 @@ void tsdbFSDestroy(STsdbFS *pFS) {
SDFileSet *pSet = (SDFileSet *)taosArrayGet(pFS->aDFileSet, iSet);
taosMemoryFree(pSet->pHeadF);
taosMemoryFree(pSet->pDataF);
- taosMemoryFree(pSet->pLastF);
taosMemoryFree(pSet->pSmaF);
+ for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+ taosMemoryFree(pSet->aSttF[iStt]);
+ }
}
taosArrayDestroy(pFS->aDFileSet);
@@ -290,7 +295,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
- if (size != pSet->pHeadF->size) {
+ if (size != tsdbLogicToFileSize(pSet->pHeadF->size, pTsdb->pVnode->config.tsdbPageSize)) {
code = TSDB_CODE_FILE_CORRUPTED;
goto _err;
}
@@ -301,38 +306,40 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
- if (size < pSet->pDataF->size) {
+ if (size < tsdbLogicToFileSize(pSet->pDataF->size, pTsdb->pVnode->config.tsdbPageSize)) {
code = TSDB_CODE_FILE_CORRUPTED;
goto _err;
- } else if (size > pSet->pDataF->size) {
+ } else if (size > tsdbLogicToFileSize(pSet->pDataF->size, pTsdb->pVnode->config.tsdbPageSize)) {
code = tsdbDFileRollback(pTsdb, pSet, TSDB_DATA_FILE);
if (code) goto _err;
}
- // last ===========
- tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pLastF, fname);
- if (taosStatFile(fname, &size, NULL)) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
- if (size != pSet->pLastF->size) {
- code = TSDB_CODE_FILE_CORRUPTED;
- goto _err;
- }
-
// sma =============
tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
if (taosStatFile(fname, &size, NULL)) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
- if (size < pSet->pSmaF->size) {
+ if (size < tsdbLogicToFileSize(pSet->pSmaF->size, pTsdb->pVnode->config.tsdbPageSize)) {
code = TSDB_CODE_FILE_CORRUPTED;
goto _err;
- } else if (size > pSet->pSmaF->size) {
+ } else if (size > tsdbLogicToFileSize(pSet->pSmaF->size, pTsdb->pVnode->config.tsdbPageSize)) {
code = tsdbDFileRollback(pTsdb, pSet, TSDB_SMA_FILE);
if (code) goto _err;
}
+
+ // stt ===========
+ for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+ tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname);
+ if (taosStatFile(fname, &size, NULL)) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ if (size != tsdbLogicToFileSize(pSet->aSttF[iStt]->size, pTsdb->pVnode->config.tsdbPageSize)) {
+ code = TSDB_CODE_FILE_CORRUPTED;
+ goto _err;
+ }
+ }
}
{
@@ -360,10 +367,12 @@ static int32_t tsdbRecoverFS(STsdb *pTsdb, uint8_t *pData, int64_t nData) {
int32_t code = 0;
int8_t hasDel;
uint32_t nSet;
- int32_t n;
+ int32_t n = 0;
+
+ // version
+ n += tGetI8(pData + n, NULL);
// SDelFile
- n = 0;
n += tGetI8(pData + n, &hasDel);
if (hasDel) {
pTsdb->fs.pDelFile = (SDelFile *)taosMemoryMalloc(sizeof(SDelFile));
@@ -382,41 +391,15 @@ static int32_t tsdbRecoverFS(STsdb *pTsdb, uint8_t *pData, int64_t nData) {
taosArrayClear(pTsdb->fs.aDFileSet);
n += tGetU32v(pData + n, &nSet);
for (uint32_t iSet = 0; iSet < nSet; iSet++) {
- SDFileSet fSet;
+ SDFileSet fSet = {0};
- // head
- fSet.pHeadF = (SHeadFile *)taosMemoryCalloc(1, sizeof(SHeadFile));
- if (fSet.pHeadF == NULL) {
+ int32_t nt = tGetDFileSet(pData + n, &fSet);
+ if (nt < 0) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
- fSet.pHeadF->nRef = 1;
- // data
- fSet.pDataF = (SDataFile *)taosMemoryCalloc(1, sizeof(SDataFile));
- if (fSet.pDataF == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
- fSet.pDataF->nRef = 1;
-
- // last
- fSet.pLastF = (SLastFile *)taosMemoryCalloc(1, sizeof(SLastFile));
- if (fSet.pLastF == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
- fSet.pLastF->nRef = 1;
-
- // sma
- fSet.pSmaF = (SSmaFile *)taosMemoryCalloc(1, sizeof(SSmaFile));
- if (fSet.pSmaF == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
- fSet.pSmaF->nRef = 1;
-
- n += tGetDFileSet(pData + n, &fSet);
+ n += nt;
if (taosArrayPush(pTsdb->fs.aDFileSet, &fSet) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
@@ -532,13 +515,15 @@ int32_t tsdbFSClose(STsdb *pTsdb) {
ASSERT(pSet->pDataF->nRef == 1);
taosMemoryFree(pSet->pDataF);
- // last
- ASSERT(pSet->pLastF->nRef == 1);
- taosMemoryFree(pSet->pLastF);
-
// sma
ASSERT(pSet->pSmaF->nRef == 1);
taosMemoryFree(pSet->pSmaF);
+
+ // stt
+ for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+ ASSERT(pSet->aSttF[iStt]->nRef == 1);
+ taosMemoryFree(pSet->aSttF[iStt]);
+ }
}
taosArrayDestroy(pTsdb->fs.aDFileSet);
@@ -586,15 +571,7 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) {
}
*fSet.pDataF = *pSet->pDataF;
- // data
- fSet.pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile));
- if (fSet.pLastF == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _exit;
- }
- *fSet.pLastF = *pSet->pLastF;
-
- // last
+ // sma
fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile));
if (fSet.pSmaF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
@@ -602,6 +579,16 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) {
}
*fSet.pSmaF = *pSet->pSmaF;
+ // stt
+ for (fSet.nSttF = 0; fSet.nSttF < pSet->nSttF; fSet.nSttF++) {
+ fSet.aSttF[fSet.nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile));
+ if (fSet.aSttF[fSet.nSttF] == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _exit;
+ }
+ *fSet.aSttF[fSet.nSttF] = *pSet->aSttF[fSet.nSttF];
+ }
+
if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
@@ -651,14 +638,38 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) {
if (c == 0) {
*pDFileSet->pHeadF = *pSet->pHeadF;
*pDFileSet->pDataF = *pSet->pDataF;
- *pDFileSet->pLastF = *pSet->pLastF;
*pDFileSet->pSmaF = *pSet->pSmaF;
+ // stt
+ if (pSet->nSttF > pDFileSet->nSttF) {
+ ASSERT(pSet->nSttF == pDFileSet->nSttF + 1);
+
+ pDFileSet->aSttF[pDFileSet->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile));
+ if (pDFileSet->aSttF[pDFileSet->nSttF] == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _exit;
+ }
+ *pDFileSet->aSttF[pDFileSet->nSttF] = *pSet->aSttF[pSet->nSttF - 1];
+ pDFileSet->nSttF++;
+ } else if (pSet->nSttF < pDFileSet->nSttF) {
+ ASSERT(pSet->nSttF == 1);
+ for (int32_t iStt = 1; iStt < pDFileSet->nSttF; iStt++) {
+ taosMemoryFree(pDFileSet->aSttF[iStt]);
+ }
+
+ *pDFileSet->aSttF[0] = *pSet->aSttF[0];
+ pDFileSet->nSttF = 1;
+ } else {
+ for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+ *pDFileSet->aSttF[iStt] = *pSet->aSttF[iStt];
+ }
+ }
goto _exit;
}
}
- SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid};
+ ASSERT(pSet->nSttF == 1);
+ SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid, .nSttF = 1};
// head
fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile));
@@ -676,15 +687,7 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) {
}
*fSet.pDataF = *pSet->pDataF;
- // data
- fSet.pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile));
- if (fSet.pLastF == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _exit;
- }
- *fSet.pLastF = *pSet->pLastF;
-
- // last
+ // sma
fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile));
if (fSet.pSmaF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
@@ -692,6 +695,14 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) {
}
*fSet.pSmaF = *pSet->pSmaF;
+ // stt
+ fSet.aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile));
+ if (fSet.aSttF[0] == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _exit;
+ }
+ *fSet.aSttF[0] = *pSet->aSttF[0];
+
if (taosArrayInsert(pFS->aDFileSet, idx, &fSet) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
@@ -836,27 +847,6 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
pSetOld->pDataF->size = pSetNew->pDataF->size;
}
- // last
- fSet.pLastF = pSetOld->pLastF;
- if ((!sameDisk) || (pSetOld->pLastF->commitID != pSetNew->pLastF->commitID)) {
- pSetOld->pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile));
- if (pSetOld->pLastF == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
- *pSetOld->pLastF = *pSetNew->pLastF;
- pSetOld->pLastF->nRef = 1;
-
- nRef = atomic_sub_fetch_32(&fSet.pLastF->nRef, 1);
- if (nRef == 0) {
- tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pLastF, fname);
- taosRemoveFile(fname);
- taosMemoryFree(fSet.pLastF);
- }
- } else {
- ASSERT(pSetOld->pLastF->size == pSetNew->pLastF->size);
- }
-
// sma
fSet.pSmaF = pSetOld->pSmaF;
if ((!sameDisk) || (pSetOld->pSmaF->commitID != pSetNew->pSmaF->commitID)) {
@@ -879,6 +869,84 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
pSetOld->pSmaF->size = pSetNew->pSmaF->size;
}
+ // stt
+ if (sameDisk) {
+ if (pSetNew->nSttF > pSetOld->nSttF) {
+ ASSERT(pSetNew->nSttF = pSetOld->nSttF + 1);
+ pSetOld->aSttF[pSetOld->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile));
+ if (pSetOld->aSttF[pSetOld->nSttF] == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
+ *pSetOld->aSttF[pSetOld->nSttF] = *pSetNew->aSttF[pSetOld->nSttF];
+ pSetOld->aSttF[pSetOld->nSttF]->nRef = 1;
+ pSetOld->nSttF++;
+ } else if (pSetNew->nSttF < pSetOld->nSttF) {
+ ASSERT(pSetNew->nSttF == 1);
+ for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) {
+ SSttFile *pSttFile = pSetOld->aSttF[iStt];
+ nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1);
+ if (nRef == 0) {
+ tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname);
+ taosRemoveFile(fname);
+ taosMemoryFree(pSttFile);
+ }
+ pSetOld->aSttF[iStt] = NULL;
+ }
+
+ pSetOld->nSttF = 1;
+ pSetOld->aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile));
+ if (pSetOld->aSttF[0] == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
+ *pSetOld->aSttF[0] = *pSetNew->aSttF[0];
+ pSetOld->aSttF[0]->nRef = 1;
+ } else {
+ for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) {
+ if (pSetOld->aSttF[iStt]->commitID != pSetNew->aSttF[iStt]->commitID) {
+ SSttFile *pSttFile = pSetOld->aSttF[iStt];
+ nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1);
+ if (nRef == 0) {
+ tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname);
+ taosRemoveFile(fname);
+ taosMemoryFree(pSttFile);
+ }
+
+ pSetOld->aSttF[iStt] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile));
+ if (pSetOld->aSttF[iStt] == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
+ *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt];
+ pSetOld->aSttF[iStt]->nRef = 1;
+ } else {
+ ASSERT(pSetOld->aSttF[iStt]->size == pSetOld->aSttF[iStt]->size);
+ ASSERT(pSetOld->aSttF[iStt]->offset == pSetOld->aSttF[iStt]->offset);
+ }
+ }
+ }
+ } else {
+ ASSERT(pSetOld->nSttF == pSetNew->nSttF);
+ for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) {
+ SSttFile *pSttFile = pSetOld->aSttF[iStt];
+ nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1);
+ if (nRef == 0) {
+ tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname);
+ taosRemoveFile(fname);
+ taosMemoryFree(pSttFile);
+ }
+
+ pSetOld->aSttF[iStt] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile));
+ if (pSetOld->aSttF[iStt] == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
+ *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt];
+ pSetOld->aSttF[iStt]->nRef = 1;
+ }
+ }
+
if (!sameDisk) {
pSetOld->diskId = pSetNew->diskId;
}
@@ -902,13 +970,6 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
taosMemoryFree(pSetOld->pDataF);
}
- nRef = atomic_sub_fetch_32(&pSetOld->pLastF->nRef, 1);
- if (nRef == 0) {
- tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pLastF, fname);
- taosRemoveFile(fname);
- taosMemoryFree(pSetOld->pLastF);
- }
-
nRef = atomic_sub_fetch_32(&pSetOld->pSmaF->nRef, 1);
if (nRef == 0) {
tsdbSmaFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pSmaF, fname);
@@ -916,12 +977,20 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
taosMemoryFree(pSetOld->pSmaF);
}
+ for (int8_t iStt = 0; iStt < pSetOld->nSttF; iStt++) {
+ nRef = atomic_sub_fetch_32(&pSetOld->aSttF[iStt]->nRef, 1);
+ if (nRef == 0) {
+ tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->aSttF[iStt], fname);
+ taosRemoveFile(fname);
+ taosMemoryFree(pSetOld->aSttF[iStt]);
+ }
+ }
+
taosArrayRemove(pTsdb->fs.aDFileSet, iOld);
continue;
_add_new:
- fSet.diskId = pSetNew->diskId;
- fSet.fid = pSetNew->fid;
+ fSet = (SDFileSet){.diskId = pSetNew->diskId, .fid = pSetNew->fid, .nSttF = 1};
// head
fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile));
@@ -941,15 +1010,6 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
*fSet.pDataF = *pSetNew->pDataF;
fSet.pDataF->nRef = 1;
- // last
- fSet.pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile));
- if (fSet.pLastF == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
- *fSet.pLastF = *pSetNew->pLastF;
- fSet.pLastF->nRef = 1;
-
// sma
fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile));
if (fSet.pSmaF == NULL) {
@@ -959,6 +1019,16 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
*fSet.pSmaF = *pSetNew->pSmaF;
fSet.pSmaF->nRef = 1;
+ // stt
+ ASSERT(pSetNew->nSttF == 1);
+ fSet.aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile));
+ if (fSet.aSttF[0] == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
+ *fSet.aSttF[0] = *pSetNew->aSttF[0];
+ fSet.aSttF[0]->nRef = 1;
+
if (taosArrayInsert(pTsdb->fs.aDFileSet, iOld, &fSet) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
@@ -1002,12 +1072,14 @@ int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS) {
nRef = atomic_fetch_add_32(&pSet->pDataF->nRef, 1);
ASSERT(nRef > 0);
- nRef = atomic_fetch_add_32(&pSet->pLastF->nRef, 1);
- ASSERT(nRef > 0);
-
nRef = atomic_fetch_add_32(&pSet->pSmaF->nRef, 1);
ASSERT(nRef > 0);
+ for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+ nRef = atomic_fetch_add_32(&pSet->aSttF[iStt]->nRef, 1);
+ ASSERT(nRef > 0);
+ }
+
if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
@@ -1053,15 +1125,6 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) {
taosMemoryFree(pSet->pDataF);
}
- // last
- nRef = atomic_sub_fetch_32(&pSet->pLastF->nRef, 1);
- ASSERT(nRef >= 0);
- if (nRef == 0) {
- tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pLastF, fname);
- taosRemoveFile(fname);
- taosMemoryFree(pSet->pLastF);
- }
-
// sma
nRef = atomic_sub_fetch_32(&pSet->pSmaF->nRef, 1);
ASSERT(nRef >= 0);
@@ -1070,6 +1133,18 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) {
taosRemoveFile(fname);
taosMemoryFree(pSet->pSmaF);
}
+
+ // stt
+ for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+ nRef = atomic_sub_fetch_32(&pSet->aSttF[iStt]->nRef, 1);
+ ASSERT(nRef >= 0);
+ if (nRef == 0) {
+ tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname);
+ taosRemoveFile(fname);
+ taosMemoryFree(pSet->aSttF[iStt]);
+ /* code */
+ }
+ }
}
taosArrayDestroy(pFS->aDFileSet);
diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c
index 00d2ac848f..3c944584de 100644
--- a/source/dnode/vnode/src/tsdb/tsdbFile.c
+++ b/source/dnode/vnode/src/tsdb/tsdbFile.c
@@ -53,22 +53,22 @@ static int32_t tGetDataFile(uint8_t *p, SDataFile *pDataFile) {
return n;
}
-int32_t tPutLastFile(uint8_t *p, SLastFile *pLastFile) {
+int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile) {
int32_t n = 0;
- n += tPutI64v(p ? p + n : p, pLastFile->commitID);
- n += tPutI64v(p ? p + n : p, pLastFile->size);
- n += tPutI64v(p ? p + n : p, pLastFile->offset);
+ n += tPutI64v(p ? p + n : p, pSttFile->commitID);
+ n += tPutI64v(p ? p + n : p, pSttFile->size);
+ n += tPutI64v(p ? p + n : p, pSttFile->offset);
return n;
}
-static int32_t tGetLastFile(uint8_t *p, SLastFile *pLastFile) {
+static int32_t tGetSttFile(uint8_t *p, SSttFile *pSttFile) {
int32_t n = 0;
- n += tGetI64v(p + n, &pLastFile->commitID);
- n += tGetI64v(p + n, &pLastFile->size);
- n += tGetI64v(p + n, &pLastFile->offset);
+ n += tGetI64v(p + n, &pSttFile->commitID);
+ n += tGetI64v(p + n, &pSttFile->size);
+ n += tGetI64v(p + n, &pSttFile->offset);
return n;
}
@@ -102,9 +102,9 @@ void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF,
TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pDataF->commitID, ".data");
}
-void tsdbLastFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SLastFile *pLastF, char fname[]) {
+void tsdbSttFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSttFile *pSttF, char fname[]) {
snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%df%dver%" PRId64 "%s", tfsGetDiskPath(pTsdb->pVnode->pTfs, did),
- TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pLastF->commitID, ".last");
+ TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pSttF->commitID, ".stt");
}
void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]) {
@@ -148,7 +148,7 @@ int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype) {
}
// ftruncate
- if (taosFtruncateFile(pFD, size) < 0) {
+ if (taosFtruncateFile(pFD, tsdbLogicToFileSize(size, pTsdb->pVnode->config.tsdbPageSize)) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
@@ -194,9 +194,11 @@ int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) {
n += tPutDataFile(p ? p + n : p, pSet->pDataF);
n += tPutSmaFile(p ? p + n : p, pSet->pSmaF);
- // last
- n += tPutU8(p ? p + n : p, 1); // for future compatibility
- n += tPutLastFile(p ? p + n : p, pSet->pLastF);
+ // stt
+ n += tPutU8(p ? p + n : p, pSet->nSttF);
+ for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+ n += tPutSttFile(p ? p + n : p, pSet->aSttF[iStt]);
+ }
return n;
}
@@ -208,15 +210,40 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) {
n += tGetI32v(p + n, &pSet->diskId.id);
n += tGetI32v(p + n, &pSet->fid);
- // data
+ // head
+ pSet->pHeadF = (SHeadFile *)taosMemoryCalloc(1, sizeof(SHeadFile));
+ if (pSet->pHeadF == NULL) {
+ return -1;
+ }
+ pSet->pHeadF->nRef = 1;
n += tGetHeadFile(p + n, pSet->pHeadF);
+
+ // data
+ pSet->pDataF = (SDataFile *)taosMemoryCalloc(1, sizeof(SDataFile));
+ if (pSet->pDataF == NULL) {
+ return -1;
+ }
+ pSet->pDataF->nRef = 1;
n += tGetDataFile(p + n, pSet->pDataF);
+
+ // sma
+ pSet->pSmaF = (SSmaFile *)taosMemoryCalloc(1, sizeof(SSmaFile));
+ if (pSet->pSmaF == NULL) {
+ return -1;
+ }
+ pSet->pSmaF->nRef = 1;
n += tGetSmaFile(p + n, pSet->pSmaF);
- // last
- uint8_t nLast;
- n += tGetU8(p + n, &nLast);
- n += tGetLastFile(p + n, pSet->pLastF);
+ // stt
+ n += tGetU8(p + n, &pSet->nSttF);
+ for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+ pSet->aSttF[iStt] = (SSttFile *)taosMemoryCalloc(1, sizeof(SSttFile));
+ if (pSet->aSttF[iStt] == NULL) {
+ return -1;
+ }
+ pSet->aSttF[iStt]->nRef = 1;
+ n += tGetSttFile(p + n, pSet->aSttF[iStt]);
+ }
return n;
}
diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
new file mode 100644
index 0000000000..45fe29f0fa
--- /dev/null
+++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
@@ -0,0 +1,561 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see
+ SArray* blockList; // SArray
int32_t order;
- SBlock block; // current SBlock data
+ SDataBlk block; // current SDataBlk data
SHashObj* pTableMap;
} SDataBlockIter;
@@ -130,8 +122,8 @@ typedef struct SFileBlockDumpInfo {
} SFileBlockDumpInfo;
typedef struct SUidOrderCheckInfo {
- uint64_t* tableUidList; // access table uid list in uid ascending order list
- int32_t currentIndex; // index in table uid list
+ uint64_t* tableUidList; // access table uid list in uid ascending order list
+ int32_t currentIndex; // index in table uid list
} SUidOrderCheckInfo;
typedef struct SReaderStatus {
@@ -139,9 +131,9 @@ typedef struct SReaderStatus {
bool composedDataBlock; // the returned data block is a composed block or not
SHashObj* pTableMap; // SHash
STableBlockScanInfo* pTableIter; // table iterator used in building in-memory buffer data blocks.
- SUidOrderCheckInfo uidCheckInfo; // check all table in uid order
+ SUidOrderCheckInfo uidCheckInfo; // check all table in uid order
SFileBlockDumpInfo fBlockDumpInfo;
- SDFileSet* pCurrentFileset; // current opened file set
+ SDFileSet* pCurrentFileset; // current opened file set
SBlockData fileBlockData;
SFilesetIter fileIter;
SDataBlockIter blockIter;
@@ -175,29 +167,34 @@ static int buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, i
static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader);
static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader,
SRowMerger* pMerger);
-static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, SRowMerger* pMerger);
+static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts,
+ SRowMerger* pMerger);
static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger,
STsdbReader* pReader);
static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid);
static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData,
- int32_t rowIndex);
+ int32_t rowIndex);
static void setComposedBlockFlag(STsdbReader* pReader, bool composed);
static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order);
-static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow,
- STsdbReader* pReader, bool* freeTSRow);
-static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader,
- STSRow** pTSRow);
-static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader);
+static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList,
+ STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow);
+static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo,
+ STsdbReader* pReader, STSRow** pTSRow);
+static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key,
+ STsdbReader* pReader);
static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STbData* pMemTbData,
STbData* piMemTbData);
static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idstr,
int8_t* pLevel);
static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level);
-static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader);
-static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader);
-static int32_t doBuildDataBlock(STsdbReader* pReader);
+static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader);
+static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader);
+static int32_t doBuildDataBlock(STsdbReader* pReader);
+static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader);
+
+static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); }
static int32_t setColumnIdSlotList(STsdbReader* pReader, SSDataBlock* pBlock) {
SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
@@ -234,15 +231,13 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK
}
for (int32_t j = 0; j < numOfTables; ++j) {
- STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid, .indexInBlockL = INITIAL_ROW_INDEX_VAL};
+ STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid};
if (ASCENDING_TRAVERSE(pTsdbReader->order)) {
- if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReader->window.skey) {
- info.lastKey = pTsdbReader->window.skey;
- }
-
- ASSERT(info.lastKey >= pTsdbReader->window.skey && info.lastKey <= pTsdbReader->window.ekey);
+ int64_t skey = pTsdbReader->window.skey;
+ info.lastKey = (skey > INT64_MIN) ? (skey - 1) : skey;
} else {
- info.lastKey = pTsdbReader->window.skey;
+ int64_t ekey = pTsdbReader->window.ekey;
+ info.lastKey = (ekey < INT64_MAX) ? (ekey + 1) : ekey;
}
taosHashPut(pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info));
@@ -256,7 +251,7 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK
return pTableMap;
}
-static void resetDataBlockScanInfo(SHashObj* pTableMap) {
+static void resetDataBlockScanInfo(SHashObj* pTableMap, int64_t ts) {
STableBlockScanInfo* p = NULL;
while ((p = taosHashIterate(pTableMap, p)) != NULL) {
@@ -266,9 +261,8 @@ static void resetDataBlockScanInfo(SHashObj* pTableMap) {
p->iter.iter = tsdbTbDataIterDestroy(p->iter.iter);
}
- p->fileDelIndex = -1;
- p->delSkyline = taosArrayDestroy(p->delSkyline);
- p->lastBlockDelIndex = INITIAL_ROW_INDEX_VAL;
+ p->delSkyline = taosArrayDestroy(p->delSkyline);
+ p->lastKey = ts;
}
}
@@ -330,7 +324,7 @@ static void limitOutputBufferSize(const SQueryTableDataCond* pCond, int32_t* cap
}
// init file iterator
-static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdbReader* pReader/*int32_t order, const char* idstr*/) {
+static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdbReader* pReader) {
size_t numOfFileset = taosArrayGetSize(aDFileSet);
pIter->index = ASCENDING_TRAVERSE(pReader->order) ? -1 : numOfFileset;
@@ -345,17 +339,21 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb
tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), pReader->idStr);
return code;
}
+ }
- SLastBlockReader* pLReader = pIter->pLastBlockReader;
- pLReader->pBlockL = taosArrayInit(4, sizeof(SBlockL));
- pLReader->order = pReader->order;
- pLReader->window = pReader->window;
- pLReader->verRange = pReader->verRange;
- pLReader->currentBlockIndex = -1;
+ SLastBlockReader* pLReader = pIter->pLastBlockReader;
+ pLReader->order = pReader->order;
+ pLReader->window = pReader->window;
+ pLReader->verRange = pReader->verRange;
- int32_t code = tBlockDataCreate(&pLReader->lastBlockData);
- if (code != TSDB_CODE_SUCCESS) {
- return code;
+ pLReader->uid = 0;
+ tMergeTreeClose(&pLReader->mergeTree);
+
+ if (pLReader->pInfo == NULL) {
+ pLReader->pInfo = tCreateLastBlockLoadInfo();
+ if (pLReader->pInfo == NULL) {
+ tsdbDebug("init fileset iterator failed, code:%s %s", tstrerror(terrno), pReader->idStr);
+ return terrno;
}
}
@@ -372,6 +370,13 @@ static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) {
return false;
}
+ SIOCostSummary* pSum = &pReader->cost;
+ getLastBlockLoadInfo(pIter->pLastBlockReader->pInfo, &pSum->lastBlockLoad, &pReader->cost.lastBlockLoadTime);
+
+ pIter->pLastBlockReader->uid = 0;
+ tMergeTreeClose(&pIter->pLastBlockReader->mergeTree);
+ resetLastBlockLoadInfo(pIter->pLastBlockReader->pInfo);
+
// check file the time range of coverage
STimeWindow win = {0};
@@ -580,14 +585,12 @@ static void cleanupTableScanInfo(SHashObj* pTableMap) {
}
// reset the index in last block when handing a new file
- px->indexInBlockL = INITIAL_ROW_INDEX_VAL;
tMapDataClear(&px->mapData);
taosArrayClear(px->pBlockList);
}
}
-static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* pLastBlockIndex,
- SBlockNumber * pBlockNum, SArray* pQualifiedLastBlock) {
+static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum) {
int32_t numOfQTable = 0;
size_t sizeInDisk = 0;
size_t numOfTables = taosArrayGetSize(pIndexList);
@@ -601,12 +604,12 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray*
STableBlockScanInfo* pScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(int64_t));
tMapDataReset(&pScanInfo->mapData);
- tsdbReadBlock(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData);
+ tsdbReadDataBlk(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData);
sizeInDisk += pScanInfo->mapData.nData;
for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) {
- SBlock block = {0};
- tMapDataGetItemByIdx(&pScanInfo->mapData, j, &block, tGetBlock);
+ SDataBlk block = {0};
+ tMapDataGetItemByIdx(&pScanInfo->mapData, j, &block, tGetDataBlk);
// 1. time range check
if (block.minKey.ts > pReader->window.ekey || block.maxKey.ts < pReader->window.skey) {
@@ -632,36 +635,14 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray*
}
}
- size_t numOfLast = taosArrayGetSize(pLastBlockIndex);
- for(int32_t i = 0; i < numOfLast; ++i) {
- SBlockL* pLastBlock = taosArrayGet(pLastBlockIndex, i);
- if (pLastBlock->suid != pReader->suid) {
- continue;
- }
-
- {
- // 1. time range check
- if (pLastBlock->minKey > pReader->window.ekey || pLastBlock->maxKey < pReader->window.skey) {
- continue;
- }
-
- // 2. version range check
- if (pLastBlock->minVer > pReader->verRange.maxVer || pLastBlock->maxVer < pReader->verRange.minVer) {
- continue;
- }
-
- pBlockNum->numOfLastBlocks += 1;
- taosArrayPush(pQualifiedLastBlock, pLastBlock);
- }
- }
-
- int32_t total = pBlockNum->numOfLastBlocks + pBlockNum->numOfBlocks;
+ pBlockNum->numOfLastFiles = pReader->pFileReader->pSet->nSttF;
+ int32_t total = pBlockNum->numOfLastFiles + pBlockNum->numOfBlocks;
double el = (taosGetTimestampUs() - st) / 1000.0;
tsdbDebug(
- "load block of %d tables completed, blocks:%d in %d tables, lastBlock:%d, block-info-size:%.2f Kb, elapsed "
+ "load block of %d tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed "
"time:%.2f ms %s",
- numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastBlocks, sizeInDisk / 1000.0, el,
+ numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastFiles, sizeInDisk / 1000.0, el,
pReader->idStr);
pReader->cost.numOfBlocks += total;
@@ -702,15 +683,156 @@ static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) {
return pBlockInfo;
}
-static SBlock* getCurrentBlock(SDataBlockIter* pBlockIter) { return &pBlockIter->block; }
+static SDataBlk* getCurrentBlock(SDataBlockIter* pBlockIter) { return &pBlockIter->block; }
+
+int32_t binarySearchForTs(char* pValue, int num, TSKEY key, int order) {
+ int32_t midPos = -1;
+ int32_t numOfRows;
+
+ ASSERT(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);
+
+ TSKEY* keyList = (TSKEY*)pValue;
+ int32_t firstPos = 0;
+ int32_t lastPos = num - 1;
+
+ if (order == TSDB_ORDER_DESC) {
+ // find the first position which is smaller than the key
+ while (1) {
+ if (key >= keyList[firstPos]) return firstPos;
+ if (key == keyList[lastPos]) return lastPos;
+
+ if (key < keyList[lastPos]) {
+ lastPos += 1;
+ if (lastPos >= num) {
+ return -1;
+ } else {
+ return lastPos;
+ }
+ }
+
+ numOfRows = lastPos - firstPos + 1;
+ midPos = (numOfRows >> 1) + firstPos;
+
+ if (key < keyList[midPos]) {
+ firstPos = midPos + 1;
+ } else if (key > keyList[midPos]) {
+ lastPos = midPos - 1;
+ } else {
+ break;
+ }
+ }
+
+ } else {
+ // find the first position which is bigger than the key
+ while (1) {
+ if (key <= keyList[firstPos]) return firstPos;
+ if (key == keyList[lastPos]) return lastPos;
+
+ if (key > keyList[lastPos]) {
+ lastPos = lastPos + 1;
+ if (lastPos >= num)
+ return -1;
+ else
+ return lastPos;
+ }
+
+ numOfRows = lastPos - firstPos + 1;
+ midPos = (numOfRows >> 1u) + firstPos;
+
+ if (key < keyList[midPos]) {
+ lastPos = midPos - 1;
+ } else if (key > keyList[midPos]) {
+ firstPos = midPos + 1;
+ } else {
+ break;
+ }
+ }
+ }
+
+ return midPos;
+}
+
+static int doBinarySearchKey(TSKEY* keyList, int num, int pos, TSKEY key, int order) {
+ // start end position
+ int s, e;
+ s = pos;
+
+ // check
+ assert(pos >=0 && pos < num);
+ assert(num > 0);
+
+ if (order == TSDB_ORDER_ASC) {
+ // find the first position which is smaller than the key
+ e = num - 1;
+ if (key < keyList[pos])
+ return -1;
+ while (1) {
+ // check can return
+ if (key >= keyList[e])
+ return e;
+ if (key <= keyList[s])
+ return s;
+ if (e - s <= 1)
+ return s;
+
+ // change start or end position
+ int mid = s + (e - s + 1)/2;
+ if (keyList[mid] > key)
+ e = mid;
+ else if(keyList[mid] < key)
+ s = mid;
+ else
+ return mid;
+ }
+ } else { // DESC
+ // find the first position which is bigger than the key
+ e = 0;
+ if (key > keyList[pos])
+ return -1;
+ while (1) {
+ // check can return
+ if (key <= keyList[e])
+ return e;
+ if (key >= keyList[s])
+ return s;
+ if (s - e <= 1)
+ return s;
+
+ // change start or end position
+ int mid = s - (s - e + 1)/2;
+ if (keyList[mid] < key)
+ e = mid;
+ else if(keyList[mid] > key)
+ s = mid;
+ else
+ return mid;
+ }
+ }
+}
+
+int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData, SDataBlk* pBlock, int32_t pos) {
+ // NOTE: reverse the order to find the end position in data block
+ int32_t endPos = -1;
+ bool asc = ASCENDING_TRAVERSE(pReader->order);
+
+ if (asc && pReader->window.ekey >= pBlock->maxKey.ts) {
+ endPos = pBlock->nRow - 1;
+ } else if (!asc && pReader->window.skey <= pBlock->minKey.ts) {
+ endPos = 0;
+ } else {
+ endPos = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, pReader->window.ekey, pReader->order);
+ }
+
+ return endPos;
+}
static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) {
SReaderStatus* pStatus = &pReader->status;
SDataBlockIter* pBlockIter = &pStatus->blockIter;
SBlockData* pBlockData = &pStatus->fileBlockData;
- SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter);
- SBlock* pBlock = getCurrentBlock(pBlockIter);
+ SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter);
+ SDataBlk* pBlock = getCurrentBlock(pBlockIter);
SSDataBlock* pResBlock = pReader->pResBlock;
int32_t numOfOutputCols = blockDataGetNumOfCols(pResBlock);
@@ -722,23 +844,42 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
bool asc = ASCENDING_TRAVERSE(pReader->order);
int32_t step = asc ? 1 : -1;
- int32_t rowIndex = 0;
- int32_t remain = asc ? (pBlockData->nRow - pDumpInfo->rowIndex) : (pDumpInfo->rowIndex + 1);
-
- int32_t endIndex = 0;
- if (remain <= pReader->capacity) {
- endIndex = pBlockData->nRow;
+ if (asc && pReader->window.skey <= pBlock->minKey.ts) {
+ pDumpInfo->rowIndex = 0;
+ } else if (!asc && pReader->window.ekey >= pBlock->maxKey.ts) {
+ pDumpInfo->rowIndex = pBlock->nRow - 1;
} else {
- endIndex = pDumpInfo->rowIndex + step * pReader->capacity;
+ int32_t pos = asc? pBlock->nRow-1:0;
+ int32_t order = (pReader->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC;
+ pDumpInfo->rowIndex = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, pReader->window.skey, order);
+ }
+
+ // time window check
+ int32_t endIndex = getEndPosInDataBlock(pReader, pBlockData, pBlock, pDumpInfo->rowIndex);
+ if (endIndex == -1) {
+ setBlockAllDumped(pDumpInfo, pReader->window.ekey, pReader->order);
+ return TSDB_CODE_SUCCESS;
+ }
+
+ endIndex += step;
+ int32_t remain = asc ? (endIndex - pDumpInfo->rowIndex) : (pDumpInfo->rowIndex - endIndex);
+ if (remain > pReader->capacity) { // output buffer check
remain = pReader->capacity;
}
- int32_t i = 0;
+ int32_t rowIndex = 0;
+
+ int32_t i = 0;
SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i);
if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
- for (int32_t j = pDumpInfo->rowIndex; j < endIndex && j >= 0; j += step) {
- colDataAppend(pColData, rowIndex++, (const char*)&pBlockData->aTSKEY[j], false);
+ if (asc) {
+ memcpy(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], remain * sizeof(int64_t));
+ } else {
+ for (int32_t j = pDumpInfo->rowIndex; rowIndex < remain; j += step) {
+ colDataAppendInt64(pColData, rowIndex++, &pBlockData->aTSKEY[j]);
+ }
}
+
i += 1;
}
@@ -752,13 +893,32 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
if (pData->cid < pColData->info.colId) {
colIndex += 1;
} else if (pData->cid == pColData->info.colId) {
- for (int32_t j = pDumpInfo->rowIndex; j < endIndex && j >= 0; j += step) {
- tColDataGetValue(pData, j, &cv);
- doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo);
+ if (pData->flag == HAS_NONE || pData->flag == HAS_NULL) {
+ colDataAppendNNULL(pColData, 0, remain);
+ } else {
+ if (IS_NUMERIC_TYPE(pColData->info.type) && asc) {
+ uint8_t* p = pData->pData + tDataTypes[pData->type].bytes * pDumpInfo->rowIndex;
+ memcpy(pColData->pData, p, remain * tDataTypes[pData->type].bytes);
+
+ // null value exists, check one-by-one
+ if (pData->flag != HAS_VALUE) {
+ for (int32_t j = pDumpInfo->rowIndex; rowIndex < remain; j += step, rowIndex++) {
+ uint8_t v = GET_BIT2(pData->pBitMap, j);
+ if (v == 0 || v == 1) {
+ colDataSetNull_f(pColData->nullbitmap, rowIndex);
+ }
+ }
+ }
+ } else {
+ for (int32_t j = pDumpInfo->rowIndex; rowIndex < remain; j += step) {
+ tColDataGetValue(pData, j, &cv);
+ doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo);
+ }
+ }
}
+
colIndex += 1;
i += 1;
- ASSERT(rowIndex == remain);
} else { // the specified column does not exist in file block, fill with null data
colDataAppendNNULL(pColData, 0, remain);
i += 1;
@@ -774,7 +934,13 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
pResBlock->info.rows = remain;
pDumpInfo->rowIndex += step * remain;
- setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order);
+ if (pDumpInfo->rowIndex >= 0 && pDumpInfo->rowIndex < pBlock->nRow) {
+ int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex];
+ setBlockAllDumped(pDumpInfo, ts, pReader->order);
+ } else {
+ int64_t k = asc? pBlock->maxKey.ts:pBlock->minKey.ts;
+ setBlockAllDumped(pDumpInfo, k, pReader->order);
+ }
double elapsedTime = (taosGetTimestampUs() - st) / 1000.0;
pReader->cost.blockLoadTime += elapsedTime;
@@ -782,7 +948,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
int32_t unDumpedRows = asc ? pBlock->nRow - pDumpInfo->rowIndex : pDumpInfo->rowIndex + 1;
tsdbDebug("%p copy file block to sdatablock, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64
", rows:%d, remain:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s",
- pReader, pBlockIter->index, pFBlock->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, remain, unDumpedRows,
+ pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, remain, unDumpedRows,
pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr);
return TSDB_CODE_SUCCESS;
@@ -795,8 +961,8 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
ASSERT(pBlockInfo != NULL);
- SBlock* pBlock = getCurrentBlock(pBlockIter);
- int32_t code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData);
+ SDataBlk* pBlock = getCurrentBlock(pBlockIter);
+ int32_t code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData);
if (code != TSDB_CODE_SUCCESS) {
tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64
", rows:%d, code:%s %s",
@@ -872,8 +1038,8 @@ static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter) {
SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter);
if (pBlockInfo != NULL) {
STableBlockScanInfo* pScanInfo = taosHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid));
- int32_t* mapDataIndex = taosArrayGet(pScanInfo->pBlockList, pBlockInfo->tbBlockIdx);
- tMapDataGetItemByIdx(&pScanInfo->mapData, *mapDataIndex, &pBlockIter->block, tGetBlock);
+ int32_t* mapDataIndex = taosArrayGet(pScanInfo->pBlockList, pBlockInfo->tbBlockIdx);
+ tMapDataGetItemByIdx(&pScanInfo->mapData, *mapDataIndex, &pBlockIter->block, tGetDataBlk);
}
#if 0
@@ -924,12 +1090,12 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte
}
sup.pDataBlockInfo[sup.numOfTables] = (SBlockOrderWrapper*)buf;
- SBlock block = {0};
+ SDataBlk block = {0};
for (int32_t k = 0; k < num; ++k) {
SBlockOrderWrapper wrapper = {0};
int32_t* mapDataIndex = taosArrayGet(pTableScanInfo->pBlockList, k);
- tMapDataGetItemByIdx(&pTableScanInfo->mapData, *mapDataIndex, &block, tGetBlock);
+ tMapDataGetItemByIdx(&pTableScanInfo->mapData, *mapDataIndex, &block, tGetDataBlk);
wrapper.uid = pTableScanInfo->uid;
wrapper.offset = block.aSubBlock[0].offset;
@@ -990,8 +1156,8 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte
}
int64_t et = taosGetTimestampUs();
- tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, numOfBlocks, (et - st) / 1000.0,
- pReader->idStr);
+ tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, numOfBlocks,
+ (et - st) / 1000.0, pReader->idStr);
cleanupBlockOrderSupporter(&sup);
taosMemoryFree(pTree);
@@ -1018,15 +1184,15 @@ static bool blockIteratorNext(SDataBlockIter* pBlockIter) {
/**
* This is an two rectangles overlap cases.
*/
-static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* pVerRange, SBlock* pBlock) {
+static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* pVerRange, SDataBlk* pBlock) {
return (pWindow->ekey < pBlock->maxKey.ts && pWindow->ekey >= pBlock->minKey.ts) ||
(pWindow->skey > pBlock->minKey.ts && pWindow->skey <= pBlock->maxKey.ts) ||
(pVerRange->minVer > pBlock->minVer && pVerRange->minVer <= pBlock->maxVer) ||
(pVerRange->maxVer < pBlock->maxVer && pVerRange->maxVer >= pBlock->minVer);
}
-static SBlock* getNeighborBlockOfSameTable(SFileDataBlockInfo* pFBlockInfo, STableBlockScanInfo* pTableBlockScanInfo,
- int32_t* nextIndex, int32_t order) {
+static SDataBlk* getNeighborBlockOfSameTable(SFileDataBlockInfo* pFBlockInfo, STableBlockScanInfo* pTableBlockScanInfo,
+ int32_t* nextIndex, int32_t order) {
bool asc = ASCENDING_TRAVERSE(order);
if (asc && pFBlockInfo->tbBlockIdx >= taosArrayGetSize(pTableBlockScanInfo->pBlockList) - 1) {
return NULL;
@@ -1039,10 +1205,10 @@ static SBlock* getNeighborBlockOfSameTable(SFileDataBlockInfo* pFBlockInfo, STab
int32_t step = asc ? 1 : -1;
*nextIndex = pFBlockInfo->tbBlockIdx + step;
- SBlock* pBlock = taosMemoryCalloc(1, sizeof(SBlock));
- int32_t* indexInMapdata = taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex);
+ SDataBlk* pBlock = taosMemoryCalloc(1, sizeof(SDataBlk));
+ int32_t* indexInMapdata = taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex);
- tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, *indexInMapdata, pBlock, tGetBlock);
+ tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, *indexInMapdata, pBlock, tGetDataBlk);
return pBlock;
}
@@ -1085,7 +1251,7 @@ static int32_t setFileBlockActiveInBlockIter(SDataBlockIter* pBlockIter, int32_t
return TSDB_CODE_SUCCESS;
}
-static bool overlapWithNeighborBlock(SBlock* pBlock, SBlock* pNeighbor, int32_t order) {
+static bool overlapWithNeighborBlock(SDataBlk* pBlock, SDataBlk* pNeighbor, int32_t order) {
// it is the last block in current file, no chance to overlap with neighbor blocks.
if (ASCENDING_TRAVERSE(order)) {
return pBlock->maxKey.ts == pNeighbor->minKey.ts;
@@ -1094,19 +1260,19 @@ static bool overlapWithNeighborBlock(SBlock* pBlock, SBlock* pNeighbor, int32_t
}
}
-static bool bufferDataInFileBlockGap(int32_t order, TSDBKEY key, SBlock* pBlock) {
+static bool bufferDataInFileBlockGap(int32_t order, TSDBKEY key, SDataBlk* pBlock) {
bool ascScan = ASCENDING_TRAVERSE(order);
return (ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts <= pBlock->minKey.ts)) ||
(!ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts >= pBlock->maxKey.ts));
}
-static bool keyOverlapFileBlock(TSDBKEY key, SBlock* pBlock, SVersionRange* pVerRange) {
+static bool keyOverlapFileBlock(TSDBKEY key, SDataBlk* pBlock, SVersionRange* pVerRange) {
return (key.ts >= pBlock->minKey.ts && key.ts <= pBlock->maxKey.ts) && (pBlock->maxVer >= pVerRange->minVer) &&
(pBlock->minVer <= pVerRange->maxVer);
}
-static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock) {
+static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SDataBlk* pBlock) {
size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline);
for (int32_t i = pBlockScanInfo->fileDelIndex; i < num; i += 1) {
@@ -1140,7 +1306,7 @@ static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, cons
return false;
}
-static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock, int32_t order) {
+static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SDataBlk* pBlock, int32_t order) {
if (pBlockScanInfo->delSkyline == NULL) {
return false;
}
@@ -1170,53 +1336,84 @@ static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBl
}
}
+typedef struct {
+ bool overlapWithNeighborBlock;
+ bool hasDupTs;
+ bool overlapWithDelInfo;
+ bool overlapWithLastBlock;
+ bool overlapWithKeyInBuf;
+ bool partiallyRequired;
+ bool moreThanCapcity;
+} SDataBlockToLoadInfo;
+
+static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pBlockInfo, SDataBlk* pBlock,
+ STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader,
+ STsdbReader* pReader) {
+ int32_t neighborIndex = 0;
+ SDataBlk* pNeighbor = getNeighborBlockOfSameTable(pBlockInfo, pScanInfo, &neighborIndex, pReader->order);
+
+ // overlap with neighbor
+ if (pNeighbor) {
+ pInfo->overlapWithNeighborBlock = overlapWithNeighborBlock(pBlock, pNeighbor, pReader->order);
+ taosMemoryFree(pNeighbor);
+ }
+
+ // has duplicated ts of different version in this block
+ pInfo->hasDupTs = (pBlock->nSubBlock == 1) ? pBlock->hasDup : true;
+ pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, pBlock, pReader->order);
+
+ // todo here we need to each key in the last files to identify if it is really overlapped with last block
+ // todo
+ bool overlapWithlastBlock = false;
+#if 0
+ if (taosArrayGetSize(pLastBlockReader->pSstBlk) > 0 && (pLastBlockReader->currentBlockIndex != -1)) {
+ SSttBlk* pSstBlk = taosArrayGet(pLastBlockReader->pSstBlk, pLastBlockReader->currentBlockIndex);
+ overlapWithlastBlock = !(pBlock->maxKey.ts < pSstBlk->minKey || pBlock->minKey.ts > pSstBlk->maxKey);
+ }
+#endif
+
+ pInfo->moreThanCapcity = pBlock->nRow > pReader->capacity;
+ pInfo->partiallyRequired = dataBlockPartiallyRequired(&pReader->window, &pReader->verRange, pBlock);
+ pInfo->overlapWithKeyInBuf = keyOverlapFileBlock(keyInBuf, pBlock, &pReader->verRange);
+}
+
// 1. the version of all rows should be less than the endVersion
// 2. current block should not overlap with next neighbor block
// 3. current timestamp should not be overlap with each other
// 4. output buffer should be large enough to hold all rows in current block
// 5. delete info should not overlap with current block data
-static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBlock, SBlock* pBlock,
- STableBlockScanInfo* pScanInfo, TSDBKEY key, SLastBlockReader* pLastBlockReader) {
- int32_t neighborIndex = 0;
- SBlock* pNeighbor = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &neighborIndex, pReader->order);
+// 6. current block should not contain the duplicated ts
+static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo, SDataBlk* pBlock,
+ STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader) {
+ SDataBlockToLoadInfo info = {0};
+ getBlockToLoadInfo(&info, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader, pReader);
- // overlap with neighbor
- bool overlapWithNeighbor = false;
- if (pNeighbor) {
- overlapWithNeighbor = overlapWithNeighborBlock(pBlock, pNeighbor, pReader->order);
- taosMemoryFree(pNeighbor);
- }
-
- // has duplicated ts of different version in this block
- bool hasDup = (pBlock->nSubBlock == 1) ? pBlock->hasDup : true;
- bool overlapWithDel = overlapWithDelSkyline(pScanInfo, pBlock, pReader->order);
-
- // todo here we need to each key in the last files to identify if it is really overlapped with last block
- bool overlapWithlastBlock = false;
- if (taosArrayGetSize(pLastBlockReader->pBlockL) > 0 && (pLastBlockReader->currentBlockIndex != -1)) {
- SBlockL *pBlockL = taosArrayGet(pLastBlockReader->pBlockL, pLastBlockReader->currentBlockIndex);
- overlapWithlastBlock = !(pBlock->maxKey.ts < pBlockL->minKey || pBlock->minKey.ts > pBlockL->maxKey);
- }
-
- bool moreThanOutputCapacity = pBlock->nRow > pReader->capacity;
- bool partiallyRequired = dataBlockPartiallyRequired(&pReader->window, &pReader->verRange, pBlock);
- bool overlapWithKey = keyOverlapFileBlock(key, pBlock, &pReader->verRange);
-
- bool loadDataBlock = (overlapWithNeighbor || hasDup || partiallyRequired || overlapWithKey ||
- moreThanOutputCapacity || overlapWithDel || overlapWithlastBlock);
+ bool loadDataBlock =
+ (info.overlapWithNeighborBlock || info.hasDupTs || info.partiallyRequired || info.overlapWithKeyInBuf ||
+ info.moreThanCapcity || info.overlapWithDelInfo || info.overlapWithLastBlock);
// log the reason why load the datablock for profile
if (loadDataBlock) {
tsdbDebug("%p uid:%" PRIu64
" need to load the datablock, overlapwithneighborblock:%d, hasDup:%d, partiallyRequired:%d, "
"overlapWithKey:%d, greaterThanBuf:%d, overlapWithDel:%d, overlapWithlastBlock:%d, %s",
- pReader, pFBlock->uid, overlapWithNeighbor, hasDup, partiallyRequired, overlapWithKey,
- moreThanOutputCapacity, overlapWithDel, overlapWithlastBlock, pReader->idStr);
+ pReader, pBlockInfo->uid, info.overlapWithNeighborBlock, info.hasDupTs, info.partiallyRequired,
+ info.overlapWithKeyInBuf, info.moreThanCapcity, info.overlapWithDelInfo, info.overlapWithLastBlock,
+ pReader->idStr);
}
return loadDataBlock;
}
+static bool isCleanFileDataBlock(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo, SDataBlk* pBlock,
+ STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader) {
+ SDataBlockToLoadInfo info = {0};
+ getBlockToLoadInfo(&info, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader, pReader);
+ bool isCleanFileBlock = !(info.overlapWithNeighborBlock || info.hasDupTs || info.overlapWithKeyInBuf ||
+ info.overlapWithDelInfo || info.overlapWithLastBlock);
+ return isCleanFileBlock;
+}
+
static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, int64_t endKey) {
if (!(pBlockScanInfo->iiter.hasVal || pBlockScanInfo->iter.hasVal)) {
return TSDB_CODE_SUCCESS;
@@ -1262,6 +1459,38 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB
return false;
}
+static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) {
+ while (1) {
+ bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree);
+ if (!hasVal) {
+ return false;
+ }
+
+ TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
+ TSDBKEY k = TSDBROW_KEY(&row);
+ if (!hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order)) {
+ return true;
+ }
+ }
+}
+
+static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLastBlockReader,
+ STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader) {
+ bool hasVal = nextRowFromLastBlocks(pLastBlockReader, pScanInfo);
+ if (hasVal) {
+ int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader);
+ if (next1 != ts) {
+ doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow);
+ return true;
+ }
+ } else {
+ doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow);
+ return true;
+ }
+
+ return false;
+}
+
static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* pReader, uint64_t uid) {
// always set the newest schema version in pReader->pSchema
if (pReader->pSchema == NULL) {
@@ -1295,18 +1524,16 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
int64_t tsLast = INT64_MIN;
- if ((pLastBlockReader->lastBlockData.nRow > 0) && hasDataInLastBlock(pLastBlockReader)) {
+ if (hasDataInLastBlock(pLastBlockReader)) {
tsLast = getCurrentKeyInLastBlock(pLastBlockReader);
}
- TSDBKEY k = TSDBROW_KEY(pRow);
- TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
-
- SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData;
+ TSDBKEY k = TSDBROW_KEY(pRow);
+ TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
int64_t minKey = 0;
if (pReader->order == TSDB_ORDER_ASC) {
- minKey = INT64_MAX; // chosen the minimum value
+ minKey = INT64_MAX; // chosen the minimum value
if (minKey > tsLast && hasDataInLastBlock(pLastBlockReader)) {
minKey = tsLast;
}
@@ -1336,7 +1563,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
bool init = false;
// ASC: file block ---> last block -----> imem -----> mem
- //DESC: mem -----> imem -----> last block -----> file block
+ // DESC: mem -----> imem -----> last block -----> file block
if (pReader->order == TSDB_ORDER_ASC) {
if (minKey == key) {
init = true;
@@ -1345,7 +1572,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
}
if (minKey == tsLast) {
- TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex);
+ TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
if (init) {
tRowMerge(&merge, &fRow1);
} else {
@@ -1374,7 +1601,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
}
if (minKey == tsLast) {
- TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex);
+ TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
if (init) {
tRowMerge(&merge, &fRow1);
} else {
@@ -1410,31 +1637,55 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, STsdbReader* pReader,
STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData,
bool mergeBlockData) {
- SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData;
- int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader);
+ SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
+ int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader);
STSRow* pTSRow = NULL;
SRowMerger merge = {0};
+ TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
- TSDBROW fRow = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex);
+ // only last block exists
+ if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) {
+ if (tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader)) {
+ return TSDB_CODE_SUCCESS;
+ } else {
+ tRowMergerInit(&merge, &fRow, pReader->pSchema);
- tRowMergerInit(&merge, &fRow, pReader->pSchema);
- doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge);
+ TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
+ tRowMerge(&merge, &fRow1);
+ doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge);
- // merge with block data if ts == key
- if (mergeBlockData) {
- doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge);
+ int32_t code = tRowMergerGetRow(&merge, &pTSRow);
+ if (code != TSDB_CODE_SUCCESS) {
+ return code;
+ }
+
+ doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid);
+
+ taosMemoryFree(pTSRow);
+ tRowMergerClear(&merge);
+ }
+ } else { // not merge block data
+ tRowMergerInit(&merge, &fRow, pReader->pSchema);
+ doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge);
+ ASSERT(mergeBlockData);
+
+ // merge with block data if ts == key
+ if (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex]) {
+ doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge);
+ }
+
+ int32_t code = tRowMergerGetRow(&merge, &pTSRow);
+ if (code != TSDB_CODE_SUCCESS) {
+ return code;
+ }
+
+ doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid);
+
+ taosMemoryFree(pTSRow);
+ tRowMergerClear(&merge);
}
- int32_t code = tRowMergerGetRow(&merge, &pTSRow);
- if (code != TSDB_CODE_SUCCESS) {
- return code;
- }
-
- doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid);
-
- taosMemoryFree(pTSRow);
- tRowMergerClear(&merge);
return TSDB_CODE_SUCCESS;
}
@@ -1444,7 +1695,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader
if (pBlockData->nRow > 0) {
// no last block available, only data block exists
- if (pLastBlockReader->lastBlockData.nRow == 0 || (!hasDataInLastBlock(pLastBlockReader))) {
+ if (!hasDataInLastBlock(pLastBlockReader)) {
return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader);
}
@@ -1454,7 +1705,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader
ASSERT(ts >= key);
if (ASCENDING_TRAVERSE(pReader->order)) {
- if (key < ts) { // imem, mem are all empty, file blocks (data blocks and last block) exist
+ if (key < ts) { // imem, mem are all empty, file blocks (data blocks and last block) exist
return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader);
} else if (key == ts) {
STSRow* pTSRow = NULL;
@@ -1462,6 +1713,10 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader
tRowMergerInit(&merge, &fRow, pReader->pSchema);
doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge);
+
+ TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
+ tRowMerge(&merge, &fRow1);
+
doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge);
int32_t code = tRowMergerGetRow(&merge, &pTSRow);
@@ -1498,7 +1753,6 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader);
ASSERT(pRow != NULL && piRow != NULL);
- SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData;
int64_t tsLast = INT64_MIN;
if (hasDataInLastBlock(pLastBlockReader)) {
tsLast = getCurrentKeyInLastBlock(pLastBlockReader);
@@ -1528,7 +1782,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
minKey = tsLast;
}
} else {
- minKey = INT64_MIN; // let find the maximum ts value
+ minKey = INT64_MIN; // let find the maximum ts value
if (minKey < k.ts) {
minKey = k.ts;
}
@@ -1559,7 +1813,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
}
if (minKey == tsLast) {
- TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex);
+ TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
if (init) {
tRowMerge(&merge, &fRow1);
} else {
@@ -1609,7 +1863,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
}
if (minKey == tsLast) {
- TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex);
+ TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
if (init) {
tRowMerge(&merge, &fRow1);
} else {
@@ -1787,261 +2041,6 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo*
}
#endif
-static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo,
- STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) {
- // it is an multi-table data block
- if (pBlockData->aUid != NULL) {
- uint64_t uid = pBlockData->aUid[pDumpInfo->rowIndex];
- if (uid != pBlockScanInfo->uid) { // move to next row
- return false;
- }
- }
-
- // check for version and time range
- int64_t ver = pBlockData->aVersion[pDumpInfo->rowIndex];
- if (ver > pReader->verRange.maxVer || ver < pReader->verRange.minVer) {
- return false;
- }
-
- int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex];
- if (ts > pReader->window.ekey || ts < pReader->window.skey) {
- return false;
- }
-
- TSDBKEY k = {.ts = ts, .version = ver};
- if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, &k, pReader->order)) {
- return false;
- }
-
- return true;
-}
-
-static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); }
-
-static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid, int16_t* startPos) {
- pLastBlockReader->uid = uid;
- pLastBlockReader->rowIndex = startPos;
-
- if (*startPos == -1) {
- if (ASCENDING_TRAVERSE(pLastBlockReader->order)) {
- // do nothing
- } else {
- *startPos = pLastBlockReader->lastBlockData.nRow;
- }
- }
-}
-
-static void setAllRowsChecked(SLastBlockReader *pLastBlockReader) {
- *pLastBlockReader->rowIndex = ALL_ROWS_CHECKED_INDEX;
-}
-
-static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) {
- bool asc = ASCENDING_TRAVERSE(pLastBlockReader->order);
- int32_t step = (asc) ? 1 : -1;
- if (*pLastBlockReader->rowIndex == ALL_ROWS_CHECKED_INDEX) {
- return false;
- }
-
- *(pLastBlockReader->rowIndex) += step;
-
- SBlockData* pBlockData = &pLastBlockReader->lastBlockData;
- for(int32_t i = *(pLastBlockReader->rowIndex); i < pBlockData->nRow && i >= 0; i += step) {
- if (pBlockData->aUid != NULL) {
- if (asc) {
- if (pBlockData->aUid[i] < pLastBlockReader->uid) {
- continue;
- } else if (pBlockData->aUid[i] > pLastBlockReader->uid) {
- break;
- }
- } else {
- if (pBlockData->aUid[i] > pLastBlockReader->uid) {
- continue;
- } else if (pBlockData->aUid[i] < pLastBlockReader->uid) {
- break;
- }
- }
- }
-
- int64_t ts = pBlockData->aTSKEY[i];
- if (ts < pLastBlockReader->window.skey) {
- continue;
- }
-
- int64_t ver = pBlockData->aVersion[i];
- if (ver < pLastBlockReader->verRange.minVer) {
- continue;
- }
-
- // no data any more, todo opt handle desc case
- if (ts > pLastBlockReader->window.ekey) {
- continue;
- }
-
- // todo opt handle desc case
- if (ver > pLastBlockReader->verRange.maxVer) {
- continue;
- }
-
- TSDBKEY k = {.ts = ts, .version = ver};
- if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order)) {
- continue;
- }
-
- *(pLastBlockReader->rowIndex) = i;
- return true;
- }
-
- // set all data is consumed in last block
- setAllRowsChecked(pLastBlockReader);
- return false;
-}
-
-static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) {
- SBlockData* pBlockData = &pLastBlockReader->lastBlockData;
- return pBlockData->aTSKEY[*pLastBlockReader->rowIndex];
-}
-
-static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) {
- if (*pLastBlockReader->rowIndex == ALL_ROWS_CHECKED_INDEX) {
- return false;
- }
-
- ASSERT(pLastBlockReader->lastBlockData.nRow > 0);
- return true;
-}
-
-int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader) {
- SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
-
- TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
-
- if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) {
- return TSDB_CODE_SUCCESS;
- } else {
- STSRow* pTSRow = NULL;
- SRowMerger merge = {0};
-
- tRowMergerInit(&merge, &fRow, pReader->pSchema);
- doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge);
- int32_t code = tRowMergerGetRow(&merge, &pTSRow);
- if (code != TSDB_CODE_SUCCESS) {
- return code;
- }
-
- doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid);
-
- taosMemoryFree(pTSRow);
- tRowMergerClear(&merge);
- return TSDB_CODE_SUCCESS;
- }
-
- return TSDB_CODE_SUCCESS;
-}
-
-static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo,
- SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) {
- SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
-
- int64_t key = (pBlockData->nRow > 0)? pBlockData->aTSKEY[pDumpInfo->rowIndex]:INT64_MIN;
- TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader);
- TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader);
-
- if (pBlockScanInfo->iter.hasVal && pBlockScanInfo->iiter.hasVal) {
- return doMergeMultiLevelRows(pReader, pBlockScanInfo, pBlockData, pLastBlockReader);
- } else {
- // imem + file + last block
- if (pBlockScanInfo->iiter.hasVal) {
- return doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, key, pLastBlockReader);
- }
-
- // mem + file + last block
- if (pBlockScanInfo->iter.hasVal) {
- return doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, key, pLastBlockReader);
- }
-
- // files data blocks + last block
- return mergeFileBlockAndLastBlock(pReader, pLastBlockReader, key, pBlockScanInfo, pBlockData);
- }
-}
-
-static int32_t buildComposedDataBlock(STsdbReader* pReader) {
- SSDataBlock* pResBlock = pReader->pResBlock;
-
- SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter);
-
- STableBlockScanInfo* pBlockScanInfo = NULL;
- if (pBlockInfo != NULL) {
- pBlockScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid));
- } else {
- pBlockScanInfo = pReader->status.pTableIter;
- }
-
- SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader;
- SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
- SBlockData* pBlockData = &pReader->status.fileBlockData;
- int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1;
-
- int64_t st = taosGetTimestampUs();
-
- while (1) {
- // todo check the validate of row in file block
- bool hasBlockData = false;
- {
- while (pBlockData->nRow > 0) { // find the first qualified row in data block
- if (isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) {
- hasBlockData = true;
- break;
- }
-
- pDumpInfo->rowIndex += step;
-
- SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter);
- if (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0) {
- setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order);
- break;
- }
- }
- }
-
- bool hasBlockLData = hasDataInLastBlock(pLastBlockReader);
-
- // no data in last block and block, no need to proceed.
- if ((hasBlockData == false) && (hasBlockLData == false)) {
- break;
- }
-
- buildComposedDataBlockImpl(pReader, pBlockScanInfo, pBlockData, pLastBlockReader);
-
- // currently loaded file data block is consumed
- if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) {
- SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter);
- setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order);
- break;
- }
-
- if (pResBlock->info.rows >= pReader->capacity) {
- break;
- }
- }
-
- pResBlock->info.uid = pBlockScanInfo->uid;
- blockDataUpdateTsWindow(pResBlock, 0);
-
- setComposedBlockFlag(pReader, true);
- int64_t et = taosGetTimestampUs();
-
- if (pResBlock->info.rows > 0) {
- tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64
- " rows:%d, elapsed time:%.2f ms %s",
- pReader, pBlockScanInfo->uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,
- pResBlock->info.rows, (et - st) / 1000.0, pReader->idStr);
- }
-
- return TSDB_CODE_SUCCESS;
-}
-
-void setComposedBlockFlag(STsdbReader* pReader, bool composed) { pReader->status.composedDataBlock = composed; }
-
static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) {
if (pBlockScanInfo->iterInit) {
return TSDB_CODE_SUCCESS;
@@ -2106,6 +2105,222 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea
return TSDB_CODE_SUCCESS;
}
+static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo,
+ STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) {
+ // it is an multi-table data block
+ if (pBlockData->aUid != NULL) {
+ uint64_t uid = pBlockData->aUid[pDumpInfo->rowIndex];
+ if (uid != pBlockScanInfo->uid) { // move to next row
+ return false;
+ }
+ }
+
+ // check for version and time range
+ int64_t ver = pBlockData->aVersion[pDumpInfo->rowIndex];
+ if (ver > pReader->verRange.maxVer || ver < pReader->verRange.minVer) {
+ return false;
+ }
+
+ int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex];
+ if (ts > pReader->window.ekey || ts < pReader->window.skey) {
+ return false;
+ }
+
+ TSDBKEY k = {.ts = ts, .version = ver};
+ if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, &k, pReader->order)) {
+ return false;
+ }
+
+ return true;
+}
+
+static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) {
+ // the last block reader has been initialized for this table.
+ if (pLBlockReader->uid == pScanInfo->uid) {
+ return true;
+ }
+
+ if (pLBlockReader->uid != 0) {
+ tMergeTreeClose(&pLBlockReader->mergeTree);
+ }
+
+ initMemDataIterator(pScanInfo, pReader);
+ pLBlockReader->uid = pScanInfo->uid;
+
+ int32_t step = ASCENDING_TRAVERSE(pLBlockReader->order) ? 1 : -1;
+ STimeWindow w = pLBlockReader->window;
+ if (ASCENDING_TRAVERSE(pLBlockReader->order)) {
+ w.skey = pScanInfo->lastKey + step;
+ } else {
+ w.ekey = pScanInfo->lastKey + step;
+ }
+
+ int32_t code =
+ tMergeTreeOpen(&pLBlockReader->mergeTree, (pLBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader,
+ pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange, pLBlockReader->pInfo, pReader->idStr);
+ if (code != TSDB_CODE_SUCCESS) {
+ return false;
+ }
+
+ return nextRowFromLastBlocks(pLBlockReader, pScanInfo);
+}
+
+static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) {
+ TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
+ return TSDBROW_TS(&row);
+}
+
+static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->mergeTree.pIter != NULL; }
+
+int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key,
+ STsdbReader* pReader) {
+ SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
+ if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) {
+ return TSDB_CODE_SUCCESS;
+ } else {
+ TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
+
+ STSRow* pTSRow = NULL;
+ SRowMerger merge = {0};
+
+ tRowMergerInit(&merge, &fRow, pReader->pSchema);
+ doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge);
+ int32_t code = tRowMergerGetRow(&merge, &pTSRow);
+ if (code != TSDB_CODE_SUCCESS) {
+ return code;
+ }
+
+ doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid);
+
+ taosMemoryFree(pTSRow);
+ tRowMergerClear(&merge);
+ return TSDB_CODE_SUCCESS;
+ }
+}
+
+static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo,
+ SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) {
+ SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
+
+ int64_t key = (pBlockData->nRow > 0) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : INT64_MIN;
+ if (pBlockScanInfo->iter.hasVal && pBlockScanInfo->iiter.hasVal) {
+ return doMergeMultiLevelRows(pReader, pBlockScanInfo, pBlockData, pLastBlockReader);
+ } else {
+ TSDBROW *pRow = NULL, *piRow = NULL;
+ if (pBlockScanInfo->iter.hasVal) {
+ pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader);
+ }
+
+ if (pBlockScanInfo->iiter.hasVal) {
+ piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader);
+ }
+
+ // imem + file + last block
+ if (pBlockScanInfo->iiter.hasVal) {
+ return doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, key, pLastBlockReader);
+ }
+
+ // mem + file + last block
+ if (pBlockScanInfo->iter.hasVal) {
+ return doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, key, pLastBlockReader);
+ }
+
+ // files data blocks + last block
+ return mergeFileBlockAndLastBlock(pReader, pLastBlockReader, key, pBlockScanInfo, pBlockData);
+ }
+}
+
+static int32_t buildComposedDataBlock(STsdbReader* pReader) {
+ SSDataBlock* pResBlock = pReader->pResBlock;
+
+ SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter);
+ SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader;
+
+ int64_t st = taosGetTimestampUs();
+
+ STableBlockScanInfo* pBlockScanInfo = NULL;
+ if (pBlockInfo != NULL) {
+ pBlockScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid));
+ SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
+ TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader);
+
+ // it is a clean block, load it directly
+ if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader)) {
+ copyBlockDataToSDataBlock(pReader, pBlockScanInfo);
+ goto _end;
+ }
+ } else { // file blocks not exist
+ pBlockScanInfo = pReader->status.pTableIter;
+ }
+
+ SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
+ SBlockData* pBlockData = &pReader->status.fileBlockData;
+ int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1;
+
+
+ while (1) {
+ // todo check the validate of row in file block
+ bool hasBlockData = false;
+ {
+ while (pBlockData->nRow > 0) { // find the first qualified row in data block
+ if (isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) {
+ hasBlockData = true;
+ break;
+ }
+
+ pDumpInfo->rowIndex += step;
+
+ SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
+ if (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0) {
+ setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order);
+ break;
+ }
+ }
+ }
+
+ bool hasBlockLData = hasDataInLastBlock(pLastBlockReader);
+
+ // no data in last block and block, no need to proceed.
+ if ((hasBlockData == false) && (hasBlockLData == false)) {
+ break;
+ }
+
+ buildComposedDataBlockImpl(pReader, pBlockScanInfo, pBlockData, pLastBlockReader);
+
+ // currently loaded file data block is consumed
+ if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) {
+ SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
+ setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order);
+ break;
+ }
+
+ if (pResBlock->info.rows >= pReader->capacity) {
+ break;
+ }
+ }
+
+ _end:
+ pResBlock->info.uid = pBlockScanInfo->uid;
+ blockDataUpdateTsWindow(pResBlock, 0);
+
+ setComposedBlockFlag(pReader, true);
+ double el = (taosGetTimestampUs() - st)/1000.0;
+
+ pReader->cost.composedBlocks += 1;
+ pReader->cost.buildComposedBlockTime += el;
+
+ if (pResBlock->info.rows > 0) {
+ tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64
+ " rows:%d, elapsed time:%.2f ms %s",
+ pReader, pBlockScanInfo->uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,
+ pResBlock->info.rows, el, pReader->idStr);
+ }
+
+ return TSDB_CODE_SUCCESS;
+}
+
+void setComposedBlockFlag(STsdbReader* pReader, bool composed) { pReader->status.composedDataBlock = composed; }
+
int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STbData* pMemTbData,
STbData* piMemTbData) {
if (pBlockScanInfo->delSkyline != NULL) {
@@ -2188,10 +2403,8 @@ _err:
return code;
}
-static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) {
- TSDBKEY key = {.ts = TSKEY_INITIAL_VAL};
-
- initMemDataIterator(pScanInfo, pReader);
+TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) {
+ TSDBKEY key = {.ts = TSKEY_INITIAL_VAL};
TSDBROW* pRow = getValidMemRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader);
if (pRow != NULL) {
key = TSDBROW_KEY(pRow);
@@ -2211,12 +2424,10 @@ static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* p
static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) {
SReaderStatus* pStatus = &pReader->status;
pBlockNum->numOfBlocks = 0;
- pBlockNum->numOfLastBlocks = 0;
+ pBlockNum->numOfLastFiles = 0;
size_t numOfTables = taosHashGetSize(pReader->status.pTableMap);
SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBlockIdx));
- SArray* pLastBlocks = pStatus->fileIter.pLastBlockReader->pBlockL;
- taosArrayClear(pLastBlocks);
while (1) {
bool hasNext = filesetIteratorNext(&pStatus->fileIter, pReader);
@@ -2231,32 +2442,16 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) {
return code;
}
- code = tsdbReadBlockL(pReader->pFileReader, pLastBlocks);
- if (code != TSDB_CODE_SUCCESS) {
- taosArrayDestroy(pIndexList);
- return code;
- }
-
- if (taosArrayGetSize(pIndexList) > 0 || taosArrayGetSize(pLastBlocks) > 0) {
- SArray* pQLastBlock = taosArrayInit(4, sizeof(SBlockL));
-
- code = doLoadFileBlock(pReader, pIndexList, pLastBlocks, pBlockNum, pQLastBlock);
+ if (taosArrayGetSize(pIndexList) > 0 || pReader->pFileReader->pSet->nSttF > 0) {
+ code = doLoadFileBlock(pReader, pIndexList, pBlockNum);
if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(pIndexList);
- taosArrayDestroy(pQLastBlock);
return code;
}
- if (pBlockNum->numOfBlocks + pBlockNum->numOfLastBlocks > 0) {
- ASSERT(taosArrayGetSize(pQLastBlock) == pBlockNum->numOfLastBlocks);
- taosArrayClear(pLastBlocks);
- taosArrayAddAll(pLastBlocks, pQLastBlock);
-
- taosArrayDestroy(pQLastBlock);
+ if (pBlockNum->numOfBlocks + pBlockNum->numOfLastFiles > 0) {
break;
}
-
- taosArrayDestroy(pQLastBlock);
}
// no blocks in current file, try next files
@@ -2266,81 +2461,22 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) {
return TSDB_CODE_SUCCESS;
}
-static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo *pBlockScanInfo, STsdbReader* pReader) {
- SArray* pBlocks = pLastBlockReader->pBlockL;
- SBlockL* pBlock = NULL;
-
- uint64_t uid = pBlockScanInfo->uid;
- int32_t totalLastBlocks = (int32_t)taosArrayGetSize(pBlocks);
-
- initMemDataIterator(pBlockScanInfo, pReader);
-
- // find the correct SBlockL. todo binary search
- int32_t index = -1;
- for (int32_t i = 0; i < totalLastBlocks; ++i) {
- SBlockL* p = taosArrayGet(pBlocks, i);
- if (p->minUid <= uid && p->maxUid >= uid) {
- index = i;
- pBlock = p;
- break;
- }
- }
-
- if (index == -1) {
- pLastBlockReader->currentBlockIndex = index;
- tBlockDataReset(&pLastBlockReader->lastBlockData);
- return TSDB_CODE_SUCCESS;
- }
-
- // the required last datablock has already loaded
- if (index == pLastBlockReader->currentBlockIndex) {
- return TSDB_CODE_SUCCESS;
- }
-
- int64_t st = taosGetTimestampUs();
- int32_t code = tBlockDataInit(&pLastBlockReader->lastBlockData, pReader->suid, pReader->suid ? 0 : uid, pReader->pSchema);
- if (code != TSDB_CODE_SUCCESS) {
- tsdbError("%p init block data failed, code:%s %s", pReader, tstrerror(code), pReader->idStr);
- return code;
- }
-
- code = tsdbReadLastBlock(pReader->pFileReader, pBlock, &pLastBlockReader->lastBlockData);
-
- double el = (taosGetTimestampUs() - st) / 1000.0;
- if (code != TSDB_CODE_SUCCESS) {
- tsdbError("%p error occurs in loading last block into buffer, last block index:%d, total:%d code:%s %s", pReader,
- pLastBlockReader->currentBlockIndex, totalLastBlocks, tstrerror(code), pReader->idStr);
- } else {
- tsdbDebug("%p load last block completed, uid:%" PRIu64
- " last block index:%d, total:%d rows:%d, minVer:%d, maxVer:%d, brange:%" PRId64 "-%" PRId64
- " elapsed time:%.2f ms, %s",
- pReader, uid, index, totalLastBlocks, pBlock->nRow, pBlock->minVer, pBlock->maxVer, pBlock->minKey,
- pBlock->maxKey, el, pReader->idStr);
- }
-
- pLastBlockReader->currentBlockIndex = index;
- pReader->cost.lastBlockLoad += 1;
- pReader->cost.lastBlockLoadTime += el;
-
- return TSDB_CODE_SUCCESS;
-}
-
static int32_t uidComparFunc(const void* p1, const void* p2) {
- uint64_t pu1 = *(uint64_t*) p1;
- uint64_t pu2 = *(uint64_t*) p2;
+ uint64_t pu1 = *(uint64_t*)p1;
+ uint64_t pu2 = *(uint64_t*)p2;
if (pu1 == pu2) {
return 0;
} else {
- return (pu1 < pu2)? -1:1;
+ return (pu1 < pu2) ? -1 : 1;
}
}
-static void extractOrderedTableUidList(SUidOrderCheckInfo *pOrderCheckInfo, SReaderStatus* pStatus) {
+static void extractOrderedTableUidList(SUidOrderCheckInfo* pOrderCheckInfo, SReaderStatus* pStatus) {
int32_t index = 0;
int32_t total = taosHashGetSize(pStatus->pTableMap);
void* p = taosHashIterate(pStatus->pTableMap, NULL);
- while(p != NULL) {
+ while (p != NULL) {
STableBlockScanInfo* pScanInfo = p;
pOrderCheckInfo->tableUidList[index++] = pScanInfo->uid;
p = taosHashIterate(pStatus->pTableMap, p);
@@ -2350,9 +2486,12 @@ static void extractOrderedTableUidList(SUidOrderCheckInfo *pOrderCheckInfo, SRea
}
static int32_t initOrderCheckInfo(SUidOrderCheckInfo* pOrderCheckInfo, SReaderStatus* pStatus) {
- if (pOrderCheckInfo->tableUidList == NULL) {
- int32_t total = taosHashGetSize(pStatus->pTableMap);
+ int32_t total = taosHashGetSize(pStatus->pTableMap);
+ if (total == 0) {
+ return TSDB_CODE_SUCCESS;
+ }
+ if (pOrderCheckInfo->tableUidList == NULL) {
pOrderCheckInfo->currentIndex = 0;
pOrderCheckInfo->tableUidList = taosMemoryMalloc(total * sizeof(uint64_t));
if (pOrderCheckInfo->tableUidList == NULL) {
@@ -2360,20 +2499,17 @@ static int32_t initOrderCheckInfo(SUidOrderCheckInfo* pOrderCheckInfo, SReaderSt
}
extractOrderedTableUidList(pOrderCheckInfo, pStatus);
-
uint64_t uid = pOrderCheckInfo->tableUidList[0];
pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid));
} else {
if (pStatus->pTableIter == NULL) { // it is the last block of a new file
-// ASSERT(pOrderCheckInfo->currentIndex == taosHashGetSize(pStatus->pTableMap));
-
pOrderCheckInfo->currentIndex = 0;
uint64_t uid = pOrderCheckInfo->tableUidList[pOrderCheckInfo->currentIndex];
pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid));
// the tableMap has already updated
if (pStatus->pTableIter == NULL) {
- void* p = taosMemoryRealloc(pOrderCheckInfo->tableUidList, taosHashGetSize(pStatus->pTableMap)*sizeof(uint64_t));
+ void* p = taosMemoryRealloc(pOrderCheckInfo->tableUidList, total * sizeof(uint64_t));
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
@@ -2390,7 +2526,7 @@ static int32_t initOrderCheckInfo(SUidOrderCheckInfo* pOrderCheckInfo, SReaderSt
return TSDB_CODE_SUCCESS;
}
-static bool moveToNextTable(SUidOrderCheckInfo *pOrderedCheckInfo, SReaderStatus* pStatus) {
+static bool moveToNextTable(SUidOrderCheckInfo* pOrderedCheckInfo, SReaderStatus* pStatus) {
pOrderedCheckInfo->currentIndex += 1;
if (pOrderedCheckInfo->currentIndex >= taosHashGetSize(pStatus->pTableMap)) {
pStatus->pTableIter = NULL;
@@ -2404,38 +2540,20 @@ static bool moveToNextTable(SUidOrderCheckInfo *pOrderedCheckInfo, SReaderStatus
}
static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) {
- SReaderStatus* pStatus = &pReader->status;
+ SReaderStatus* pStatus = &pReader->status;
SLastBlockReader* pLastBlockReader = pStatus->fileIter.pLastBlockReader;
- SUidOrderCheckInfo *pOrderedCheckInfo = &pStatus->uidCheckInfo;
- int32_t code = initOrderCheckInfo(pOrderedCheckInfo, pStatus);
+ SUidOrderCheckInfo* pOrderedCheckInfo = &pStatus->uidCheckInfo;
+ int32_t code = initOrderCheckInfo(pOrderedCheckInfo, pStatus);
if (code != TSDB_CODE_SUCCESS || (taosHashGetSize(pStatus->pTableMap) == 0)) {
return code;
}
- while(1) {
+ while (1) {
// load the last data block of current table
STableBlockScanInfo* pScanInfo = pStatus->pTableIter;
- code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo, pReader);
- if (code != TSDB_CODE_SUCCESS) {
- return code;
- }
-
- if (pLastBlockReader->currentBlockIndex != -1) {
- initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL);
- int32_t index = pScanInfo->indexInBlockL;
-
- if (index == INITIAL_ROW_INDEX_VAL || index == pLastBlockReader->lastBlockData.nRow) {
- bool hasData = nextRowInLastBlock(pLastBlockReader, pScanInfo);
- if (!hasData) { // current table does not have rows in last block, try next table
- bool hasNexTable = moveToNextTable(pOrderedCheckInfo, pStatus);
- if (!hasNexTable) {
- return TSDB_CODE_SUCCESS;
- }
- continue;
- }
- }
- } else { // no data in last block, try next table
+ bool hasVal = initLastBlockReader(pLastBlockReader, pScanInfo, pReader);
+ if (!hasVal) {
bool hasNexTable = moveToNextTable(pOrderedCheckInfo, pStatus);
if (!hasNexTable) {
return TSDB_CODE_SUCCESS;
@@ -2461,9 +2579,8 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) {
}
static int32_t doBuildDataBlock(STsdbReader* pReader) {
- TSDBKEY key = {0};
- int32_t code = TSDB_CODE_SUCCESS;
- SBlock* pBlock = NULL;
+ int32_t code = TSDB_CODE_SUCCESS;
+ SDataBlk* pBlock = NULL;
SReaderStatus* pStatus = &pReader->status;
SDataBlockIter* pBlockIter = &pStatus->blockIter;
@@ -2481,26 +2598,13 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
pBlock = getCurrentBlock(pBlockIter);
}
- {
- key = getCurrentKeyInBuf(pScanInfo, pReader);
-
- // load the last data block of current table
- code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo, pReader);
- if (code != TSDB_CODE_SUCCESS) {
- return code;
- }
-
- // note: the lastblock may be null here
- initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL);
- if (pScanInfo->indexInBlockL == INITIAL_ROW_INDEX_VAL || pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) {
- bool hasData = nextRowInLastBlock(pLastBlockReader, pScanInfo);
- }
- }
+ initLastBlockReader(pLastBlockReader, pScanInfo, pReader);
+ TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader);
if (pBlockInfo == NULL) { // build data block from last data file
ASSERT(pBlockIter->numOfBlocks == 0);
code = buildComposedDataBlock(pReader);
- } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, key, pLastBlockReader)) {
+ } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) {
tBlockDataReset(&pStatus->fileBlockData);
code = tBlockDataInit(&pStatus->fileBlockData, pReader->suid, pScanInfo->uid, pReader->pSchema);
if (code != TSDB_CODE_SUCCESS) {
@@ -2514,7 +2618,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
// build composed data block
code = buildComposedDataBlock(pReader);
- } else if (bufferDataInFileBlockGap(pReader->order, key, pBlock)) {
+ } else if (bufferDataInFileBlockGap(pReader->order, keyInBuf, pBlock)) {
// data in memory that are earlier than current file block
// todo rows in buffer should be less than the file block in asc, greater than file block in desc
int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts;
@@ -2523,11 +2627,11 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
if (hasDataInLastBlock(pLastBlockReader) && !ASCENDING_TRAVERSE(pReader->order)) {
// only return the rows in last block
int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader);
- ASSERT (tsLast >= pBlock->maxKey.ts);
+ ASSERT(tsLast >= pBlock->maxKey.ts);
tBlockDataReset(&pReader->status.fileBlockData);
code = buildComposedDataBlock(pReader);
- } else { // whole block is required, return it directly
+ } else { // whole block is required, return it directly
SDataBlockInfo* pInfo = &pReader->pResBlock->info;
pInfo->rows = pBlock->nRow;
pInfo->uid = pScanInfo->uid;
@@ -2574,7 +2678,7 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) {
// set the correct start position in case of the first/last file block, according to the query time window
static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) {
- SBlock* pBlock = getCurrentBlock(pBlockIter);
+ SDataBlk* pBlock = getCurrentBlock(pBlockIter);
SReaderStatus* pStatus = &pReader->status;
@@ -2594,7 +2698,7 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl
}
// all data files are consumed, try data in buffer
- if (num.numOfBlocks + num.numOfLastBlocks == 0) {
+ if (num.numOfBlocks + num.numOfLastFiles == 0) {
pReader->status.loadFromFile = false;
return code;
}
@@ -2602,14 +2706,11 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl
// initialize the block iterator for a new fileset
if (num.numOfBlocks > 0) {
code = initBlockIterator(pReader, pBlockIter, num.numOfBlocks);
- } else { // no block data, only last block exists
+ } else { // no block data, only last block exists
tBlockDataReset(&pReader->status.fileBlockData);
resetDataBlockIterator(pBlockIter, pReader->order);
}
- SLastBlockReader* pLReader = pReader->status.fileIter.pLastBlockReader;
- pLReader->currentBlockIndex = -1;
-
// set the correct start position according to the query time window
initBlockDumpInfo(pReader, pBlockIter);
return code;
@@ -2627,7 +2728,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) {
SDataBlockIter* pBlockIter = &pReader->status.blockIter;
if (pBlockIter->numOfBlocks == 0) {
- _begin:
+ _begin:
code = doLoadLastBlockSequentially(pReader);
if (code != TSDB_CODE_SUCCESS) {
return code;
@@ -2674,21 +2775,24 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) {
bool hasNext = blockIteratorNext(&pReader->status.blockIter);
if (hasNext) { // check for the next block in the block accessed order list
initBlockDumpInfo(pReader, pBlockIter);
- } else if (taosArrayGetSize(pReader->status.fileIter.pLastBlockReader->pBlockL) > 0) { // data blocks in current file are exhausted, let's try the next file now
- tBlockDataReset(&pReader->status.fileBlockData);
- resetDataBlockIterator(pBlockIter, pReader->order);
- goto _begin;
} else {
- code = initForFirstBlockInFile(pReader, pBlockIter);
-
- // error happens or all the data files are completely checked
- if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) {
- return code;
- }
-
- // this file does not have blocks, let's start check the last block file
- if (pBlockIter->numOfBlocks == 0) {
+ if (pReader->status.pCurrentFileset->nSttF > 0) {
+ // data blocks in current file are exhausted, let's try the next file now
+ tBlockDataReset(&pReader->status.fileBlockData);
+ resetDataBlockIterator(pBlockIter, pReader->order);
goto _begin;
+ } else {
+ code = initForFirstBlockInFile(pReader, pBlockIter);
+
+ // error happens or all the data files are completely checked
+ if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) {
+ return code;
+ }
+
+ // this file does not have blocks, let's start check the last block file
+ if (pBlockIter->numOfBlocks == 0) {
+ goto _begin;
+ }
}
}
}
@@ -2711,6 +2815,7 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret
if (VND_IS_RSMA(pVnode)) {
int8_t level = 0;
int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision);
+ int64_t offset = TSDB_TICK_PER_SECOND(pVnode->config.tsdbCfg.precision);
for (int8_t i = 0; i < TSDB_RETENTION_MAX; ++i) {
SRetention* pRetention = retentions + level;
@@ -2720,7 +2825,7 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret
}
break;
}
- if ((now - pRetention->keep) <= winSKey) {
+ if ((now - pRetention->keep) <= (winSKey + offset)) {
break;
}
++level;
@@ -2947,7 +3052,7 @@ typedef enum {
CHECK_FILEBLOCK_QUIT = 0x2,
} CHECK_FILEBLOCK_STATE;
-static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanInfo* pScanInfo, SBlock* pBlock,
+static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanInfo* pScanInfo, SDataBlk* pBlock,
SFileDataBlockInfo* pFBlock, SRowMerger* pMerger, int64_t key,
CHECK_FILEBLOCK_STATE* state) {
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
@@ -2956,8 +3061,8 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn
*state = CHECK_FILEBLOCK_QUIT;
int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1;
- int32_t nextIndex = -1;
- SBlock* pNeighborBlock = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &nextIndex, pReader->order);
+ int32_t nextIndex = -1;
+ SDataBlk* pNeighborBlock = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &nextIndex, pReader->order);
if (pNeighborBlock == NULL) { // do nothing
return 0;
}
@@ -3021,7 +3126,7 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc
CHECK_FILEBLOCK_STATE st;
SFileDataBlockInfo* pFileBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter);
- SBlock* pCurrentBlock = getCurrentBlock(&pReader->status.blockIter);
+ SDataBlk* pCurrentBlock = getCurrentBlock(&pReader->status.blockIter);
checkForNeighborFileBlock(pReader, pScanInfo, pCurrentBlock, pFileBlockInfo, pMerger, key, &st);
if (st == CHECK_FILEBLOCK_QUIT) {
break;
@@ -3032,11 +3137,13 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc
return TSDB_CODE_SUCCESS;
}
-int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, SRowMerger* pMerger) {
- while(nextRowInLastBlock(pLastBlockReader, pScanInfo)) {
+int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts,
+ SRowMerger* pMerger) {
+ pScanInfo->lastKey = ts;
+ while (nextRowFromLastBlocks(pLastBlockReader, pScanInfo)) {
int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader);
if (next1 == ts) {
- TSDBROW fRow1 = tsdbRowFromBlockData(&pLastBlockReader->lastBlockData, *pLastBlockReader->rowIndex);
+ TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
tRowMerge(pMerger, &fRow1);
} else {
break;
@@ -3100,7 +3207,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter,
}
int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader,
- STSRow** pTSRow) {
+ STSRow** pTSRow) {
SRowMerger merge = {0};
TSDBKEY k = TSDBROW_KEY(pRow);
@@ -3174,7 +3281,8 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR
}
if (pBlockScanInfo->iter.hasVal && pRow != NULL) {
- return doMergeMemTableMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, freeTSRow);
+ return doMergeMemTableMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader,
+ freeTSRow);
}
if (pBlockScanInfo->iiter.hasVal && piRow != NULL) {
@@ -3228,7 +3336,8 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
return TSDB_CODE_SUCCESS;
}
-int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex) {
+int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData,
+ int32_t rowIndex) {
int32_t i = 0, j = 0;
int32_t outputRowIndex = pResBlock->info.rows;
@@ -3241,18 +3350,23 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
}
SColVal cv = {0};
- int32_t numOfInputCols = taosArrayGetSize(pBlockData->aIdx);
- int32_t numOfOutputCols = blockDataGetNumOfCols(pResBlock);
+ int32_t numOfInputCols = pBlockData->aIdx->size;
+ int32_t numOfOutputCols = pResBlock->pDataBlock->size;
while (i < numOfOutputCols && j < numOfInputCols) {
- SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i);
+ SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, i);
SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j);
+ if (pData->cid < pCol->info.colId) {
+ j += 1;
+ continue;
+ }
+
if (pData->cid == pCol->info.colId) {
tColDataGetValue(pData, rowIndex, &cv);
doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo);
j += 1;
- } else { // the specified column does not exist in file block, fill with null data
+ } else if (pData->cid > pCol->info.colId) { // the specified column does not exist in file block, fill with null data
colDataAppendNULL(pCol, outputRowIndex);
}
@@ -3305,7 +3419,7 @@ int32_t tsdbSetTableId(STsdbReader* pReader, int64_t uid) {
ASSERT(pReader != NULL);
taosHashClear(pReader->status.pTableMap);
- STableBlockScanInfo info = {.lastKey = 0, .uid = uid, .indexInBlockL = INITIAL_ROW_INDEX_VAL};
+ STableBlockScanInfo info = {.lastKey = 0, .uid = uid};
taosHashPut(pReader->status.pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info));
return TDB_CODE_SUCCESS;
}
@@ -3326,7 +3440,6 @@ void* tsdbGetIvtIdx(SMeta* pMeta) {
uint64_t getReaderMaxVersion(STsdbReader* pReader) { return pReader->verRange.maxVer; }
-
// ====================================== EXPOSED APIs ======================================
int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTableList, STsdbReader** ppReader,
const char* idstr) {
@@ -3375,11 +3488,18 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
}
}
+ // NOTE: the endVersion in pCond is the data version not schema version, so pCond->endVersion is not correct here.
if (pCond->suid != 0) {
- pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, pCond->endVersion);
+ pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, /*pCond->endVersion*/ -1);
+ if (pReader->pSchema == NULL) {
+ tsdbError("failed to get table schema, suid:%"PRIu64", ver:%"PRId64" , %s", pReader->suid, -1, pReader->idStr);
+ }
} else if (taosArrayGetSize(pTableList) > 0) {
STableKeyInfo* pKey = taosArrayGet(pTableList, 0);
- pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, pCond->endVersion);
+ pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, /*pCond->endVersion*/ -1);
+ if (pReader->pSchema == NULL) {
+ tsdbError("failed to get table schema, uid:%"PRIu64", ver:%"PRId64" , %s", pKey->uid, -1, pReader->idStr);
+ }
}
int32_t numOfTables = taosArrayGetSize(pTableList);
@@ -3449,7 +3569,6 @@ void tsdbReaderClose(STsdbReader* pReader) {
}
SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
- tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap);
taosMemoryFreeClear(pSupInfo->plist);
taosMemoryFree(pSupInfo->colIds);
@@ -3460,6 +3579,7 @@ void tsdbReaderClose(STsdbReader* pReader) {
taosMemoryFreeClear(pSupInfo->buildBuf[i]);
}
}
+
taosMemoryFree(pSupInfo->buildBuf);
tBlockDataDestroy(&pReader->status.fileBlockData, true);
@@ -3473,25 +3593,30 @@ void tsdbReaderClose(STsdbReader* pReader) {
tsdbDataFReaderClose(&pReader->pFileReader);
}
+ tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap);
+
taosMemoryFree(pReader->status.uidCheckInfo.tableUidList);
+ SIOCostSummary* pCost = &pReader->cost;
SFilesetIter* pFilesetIter = &pReader->status.fileIter;
if (pFilesetIter->pLastBlockReader != NULL) {
- tBlockDataDestroy(&pFilesetIter->pLastBlockReader->lastBlockData, true);
- taosArrayDestroy(pFilesetIter->pLastBlockReader->pBlockL);
- taosMemoryFree(pFilesetIter->pLastBlockReader);
+ SLastBlockReader* pLReader = pFilesetIter->pLastBlockReader;
+ tMergeTreeClose(&pLReader->mergeTree);
+
+ getLastBlockLoadInfo(pLReader->pInfo, &pCost->lastBlockLoad, &pCost->lastBlockLoadTime);
+
+ pLReader->pInfo = destroyLastBlockLoadInfo(pLReader->pInfo);
+ taosMemoryFree(pLReader);
}
- SIOCostSummary* pCost = &pReader->cost;
-
- tsdbDebug("%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64
- " SMA-time:%.2f ms, fileBlocks:%" PRId64
- ", fileBlocks-time:%.2f ms, "
- "build in-memory-block-time:%.2f ms, lastBlocks:%" PRId64
- ", lastBlocks-time:%.2f ms, STableBlockScanInfo size:%.2f Kb %s",
- pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime,
- pCost->numOfBlocks, pCost->blockLoadTime, pCost->buildmemBlock, pCost->lastBlockLoad,
- pCost->lastBlockLoadTime, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pReader->idStr);
+ tsdbDebug(
+ "%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64
+ " SMA-time:%.2f ms, fileBlocks:%" PRId64 ", fileBlocks-load-time:%.2f ms, "
+ "build in-memory-block-time:%.2f ms, lastBlocks:%" PRId64 ", lastBlocks-time:%.2f ms, composed-blocks:%" PRId64
+ ", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb %s",
+ pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, pCost->numOfBlocks,
+ pCost->blockLoadTime, pCost->buildmemBlock, pCost->lastBlockLoad, pCost->lastBlockLoadTime, pCost->composedBlocks,
+ pCost->buildComposedBlockTime, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pReader->idStr);
taosMemoryFree(pReader->idStr);
taosMemoryFree(pReader->pSchema);
@@ -3602,12 +3727,12 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS
SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter);
- SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter);
- int64_t stime = taosGetTimestampUs();
+ SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
+ int64_t stime = taosGetTimestampUs();
SBlockLoadSuppInfo* pSup = &pReader->suppInfo;
- if (tBlockHasSma(pBlock)) {
+ if (tDataBlkHasSma(pBlock)) {
code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg);
if (code != TSDB_CODE_SUCCESS) {
tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64 ", code:%s, %s", 0, pFBlock->uid, tstrerror(code),
@@ -3722,11 +3847,12 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
tsdbDataFReaderClose(&pReader->pFileReader);
int32_t numOfTables = taosHashGetSize(pReader->status.pTableMap);
- tsdbDataFReaderClose(&pReader->pFileReader);
initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader);
resetDataBlockIterator(&pReader->status.blockIter, pReader->order);
- resetDataBlockScanInfo(pReader->status.pTableMap);
+
+ int64_t ts = ASCENDING_TRAVERSE(pReader->order) ? pReader->window.skey - 1 : pReader->window.ekey + 1;
+ resetDataBlockScanInfo(pReader->status.pTableMap, ts);
int32_t code = 0;
SDataBlockIter* pBlockIter = &pReader->status.blockIter;
@@ -3784,7 +3910,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa
while (true) {
if (hasNext) {
- SBlock* pBlock = getCurrentBlock(pBlockIter);
+ SDataBlk* pBlock = getCurrentBlock(pBlockIter);
int32_t numOfRows = pBlock->nRow;
pTableBlockInfo->totalRows += numOfRows;
@@ -3952,4 +4078,4 @@ void tsdbUntakeReadSnap(STsdb* pTsdb, STsdbReadSnap* pSnap) {
}
tsdbTrace("vgId:%d, untake read snapshot", TD_VID(pTsdb->pVnode));
-}
+}
\ No newline at end of file
diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
index c8f3862071..5fe0b408b1 100644
--- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
+++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
@@ -15,399 +15,706 @@
#include "tsdb.h"
-// SDelFWriter ====================================================
-int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) {
- int32_t code = 0;
- char fname[TSDB_FILENAME_LEN];
- char hdr[TSDB_FHDR_SIZE] = {0};
- SDelFWriter *pDelFWriter;
- int64_t n;
-
- // alloc
- pDelFWriter = (SDelFWriter *)taosMemoryCalloc(1, sizeof(*pDelFWriter));
- if (pDelFWriter == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
- pDelFWriter->pTsdb = pTsdb;
- pDelFWriter->fDel = *pFile;
-
- tsdbDelFileName(pTsdb, pFile, fname);
- pDelFWriter->pWriteH = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE);
- if (pDelFWriter->pWriteH == NULL) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- // update header
- n = taosWriteFile(pDelFWriter->pWriteH, &hdr, TSDB_FHDR_SIZE);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- pDelFWriter->fDel.size = TSDB_FHDR_SIZE;
- pDelFWriter->fDel.offset = 0;
-
- *ppWriter = pDelFWriter;
- return code;
-
-_err:
- tsdbError("vgId:%d, failed to open del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
- *ppWriter = NULL;
- return code;
-}
-
-int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync) {
- int32_t code = 0;
- SDelFWriter *pWriter = *ppWriter;
- STsdb *pTsdb = pWriter->pTsdb;
-
- // sync
- if (sync && taosFsyncFile(pWriter->pWriteH) < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- // close
- if (taosCloseFile(&pWriter->pWriteH) < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t *); iBuf++) {
- tFree(pWriter->aBuf[iBuf]);
- }
- taosMemoryFree(pWriter);
-
- *ppWriter = NULL;
- return code;
-
-_err:
- tsdbError("vgId:%d, failed to close del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
- return code;
-}
-
-int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelIdx) {
- int32_t code = 0;
- int64_t size;
- int64_t n;
-
- // prepare
- size = sizeof(uint32_t);
- for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) {
- size += tPutDelData(NULL, taosArrayGet(aDelData, iDelData));
- }
- size += sizeof(TSCKSUM);
-
- // alloc
- code = tRealloc(&pWriter->aBuf[0], size);
- if (code) goto _err;
-
- // build
- n = 0;
- n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT);
- for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) {
- n += tPutDelData(pWriter->aBuf[0] + n, taosArrayGet(aDelData, iDelData));
- }
- taosCalcChecksumAppend(0, pWriter->aBuf[0], size);
-
- ASSERT(n + sizeof(TSCKSUM) == size);
-
- // write
- n = taosWriteFile(pWriter->pWriteH, pWriter->aBuf[0], size);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- ASSERT(n == size);
-
- // update
- pDelIdx->offset = pWriter->fDel.size;
- pDelIdx->size = size;
- pWriter->fDel.size += size;
-
- return code;
-
-_err:
- tsdbError("vgId:%d, failed to write del data since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
- return code;
-}
-
-int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx) {
+// =============== PAGE-WISE FILE ===============
+static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **ppFD) {
int32_t code = 0;
- int64_t size;
- int64_t n;
- SDelIdx *pDelIdx;
+ STsdbFD *pFD;
- // prepare
- size = sizeof(uint32_t);
- for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) {
- size += tPutDelIdx(NULL, taosArrayGet(aDelIdx, iDelIdx));
- }
- size += sizeof(TSCKSUM);
+ *ppFD = NULL;
- // alloc
- code = tRealloc(&pWriter->aBuf[0], size);
- if (code) goto _err;
-
- // build
- n = 0;
- n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT);
- for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) {
- n += tPutDelIdx(pWriter->aBuf[0] + n, taosArrayGet(aDelIdx, iDelIdx));
- }
- taosCalcChecksumAppend(0, pWriter->aBuf[0], size);
-
- ASSERT(n + sizeof(TSCKSUM) == size);
-
- // write
- n = taosWriteFile(pWriter->pWriteH, pWriter->aBuf[0], size);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- // update
- pWriter->fDel.offset = pWriter->fDel.size;
- pWriter->fDel.size += size;
-
- return code;
-
-_err:
- tsdbError("vgId:%d, write del idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
- return code;
-}
-
-int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter) {
- int32_t code = 0;
- char hdr[TSDB_FHDR_SIZE];
- int64_t size = TSDB_FHDR_SIZE;
- int64_t n;
-
- // build
- memset(hdr, 0, size);
- tPutDelFile(hdr, &pWriter->fDel);
- taosCalcChecksumAppend(0, hdr, size);
-
- // seek
- if (taosLSeekFile(pWriter->pWriteH, 0, SEEK_SET) < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- // write
- n = taosWriteFile(pWriter->pWriteH, hdr, size);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- return code;
-
-_err:
- tsdbError("vgId:%d, update del file hdr failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
- return code;
-}
-
-// SDelFReader ====================================================
-struct SDelFReader {
- STsdb *pTsdb;
- SDelFile fDel;
- TdFilePtr pReadH;
-
- uint8_t *aBuf[1];
-};
-
-int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb) {
- int32_t code = 0;
- char fname[TSDB_FILENAME_LEN];
- SDelFReader *pDelFReader;
- int64_t n;
-
- // alloc
- pDelFReader = (SDelFReader *)taosMemoryCalloc(1, sizeof(*pDelFReader));
- if (pDelFReader == NULL) {
+ pFD = (STsdbFD *)taosMemoryCalloc(1, sizeof(*pFD) + strlen(path) + 1);
+ if (pFD == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ goto _exit;
}
- // open impl
- pDelFReader->pTsdb = pTsdb;
- pDelFReader->fDel = *pFile;
-
- tsdbDelFileName(pTsdb, pFile, fname);
- pDelFReader->pReadH = taosOpenFile(fname, TD_FILE_READ);
- if (pDelFReader->pReadH == NULL) {
+ pFD->path = (char *)&pFD[1];
+ strcpy(pFD->path, path);
+ pFD->szPage = szPage;
+ pFD->flag = flag;
+ pFD->pFD = taosOpenFile(path, flag);
+ if (pFD->pFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
- taosMemoryFree(pDelFReader);
- goto _err;
+ goto _exit;
}
+ pFD->szPage = szPage;
+ pFD->pgno = 0;
+ pFD->pBuf = taosMemoryCalloc(1, szPage);
+ if (pFD->pBuf == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ taosMemoryFree(pFD);
+ goto _exit;
+ }
+ if (taosStatFile(path, &pFD->szFile, NULL) < 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _exit;
+ }
+ ASSERT(pFD->szFile % szPage == 0);
+ pFD->szFile = pFD->szFile / szPage;
+ *ppFD = pFD;
_exit:
- *ppReader = pDelFReader;
- return code;
-
-_err:
- tsdbError("vgId:%d, del file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
- *ppReader = NULL;
return code;
}
-int32_t tsdbDelFReaderClose(SDelFReader **ppReader) {
- int32_t code = 0;
- SDelFReader *pReader = *ppReader;
+static void tsdbCloseFile(STsdbFD **ppFD) {
+ STsdbFD *pFD = *ppFD;
+ taosMemoryFree(pFD->pBuf);
+ taosCloseFile(&pFD->pFD);
+ taosMemoryFree(pFD);
+ *ppFD = NULL;
+}
- if (pReader) {
- if (taosCloseFile(&pReader->pReadH) < 0) {
+static int32_t tsdbWriteFilePage(STsdbFD *pFD) {
+ int32_t code = 0;
+
+ if (pFD->pgno > 0) {
+ int64_t n = taosLSeekFile(pFD->pFD, PAGE_OFFSET(pFD->pgno, pFD->szPage), SEEK_SET);
+ if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
}
- for (int32_t iBuf = 0; iBuf < sizeof(pReader->aBuf) / sizeof(uint8_t *); iBuf++) {
- tFree(pReader->aBuf[iBuf]);
+
+ taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage);
+
+ n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage);
+ if (n < 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _exit;
+ }
+
+ if (pFD->szFile < pFD->pgno) {
+ pFD->szFile = pFD->pgno;
}
- taosMemoryFree(pReader);
}
- *ppReader = NULL;
+ pFD->pgno = 0;
_exit:
return code;
}
-int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData) {
+static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) {
int32_t code = 0;
- int64_t offset = pDelIdx->offset;
- int64_t size = pDelIdx->size;
- int64_t n;
- taosArrayClear(aDelData);
+ ASSERT(pgno <= pFD->szFile);
// seek
- if (taosLSeekFile(pReader->pReadH, offset, SEEK_SET) < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- // alloc
- code = tRealloc(&pReader->aBuf[0], size);
- if (code) goto _err;
-
- // read
- n = taosReadFile(pReader->pReadH, pReader->aBuf[0], size);
+ int64_t offset = PAGE_OFFSET(pgno, pFD->szPage);
+ int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- } else if (n < size) {
+ goto _exit;
+ }
+
+ // read
+ n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage);
+ if (n < 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _exit;
+ } else if (n < pFD->szPage) {
code = TSDB_CODE_FILE_CORRUPTED;
- goto _err;
+ goto _exit;
}
// check
- if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) {
+ if (!taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) {
code = TSDB_CODE_FILE_CORRUPTED;
- goto _err;
+ goto _exit;
}
- // // decode
- n = 0;
+ pFD->pgno = pgno;
- uint32_t delimiter;
- n += tGetU32(pReader->aBuf[0] + n, &delimiter);
- while (n < size - sizeof(TSCKSUM)) {
- SDelData delData;
- n += tGetDelData(pReader->aBuf[0] + n, &delData);
-
- if (taosArrayPush(aDelData, &delData) == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
- }
-
- ASSERT(n == size - sizeof(TSCKSUM));
-
- return code;
-
-_err:
- tsdbError("vgId:%d, read del data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+_exit:
return code;
}
-int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) {
+static int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) {
int32_t code = 0;
- int32_t n;
- int64_t offset = pReader->fDel.offset;
- int64_t size = pReader->fDel.size - offset;
+ int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
+ int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage);
+ int64_t bOffset = fOffset % pFD->szPage;
+ int64_t n = 0;
- taosArrayClear(aDelIdx);
+ do {
+ if (pFD->pgno != pgno) {
+ code = tsdbWriteFilePage(pFD);
+ if (code) goto _exit;
- // seek
- if (taosLSeekFile(pReader->pReadH, offset, SEEK_SET) < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
+ if (pgno <= pFD->szFile) {
+ code = tsdbReadFilePage(pFD, pgno);
+ if (code) goto _exit;
+ } else {
+ pFD->pgno = pgno;
+ }
+ }
+
+ int64_t nWrite = TMIN(PAGE_CONTENT_SIZE(pFD->szPage) - bOffset, size - n);
+ memcpy(pFD->pBuf + bOffset, pBuf + n, nWrite);
+
+ pgno++;
+ bOffset = 0;
+ n += nWrite;
+ } while (n < size);
+
+_exit:
+ return code;
+}
+
+static int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) {
+ int32_t code = 0;
+ int64_t n = 0;
+ int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
+ int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage);
+ int32_t szPgCont = PAGE_CONTENT_SIZE(pFD->szPage);
+ int64_t bOffset = fOffset % pFD->szPage;
+
+ ASSERT(pgno && pgno <= pFD->szFile);
+ ASSERT(bOffset < szPgCont);
+
+ while (n < size) {
+ if (pFD->pgno != pgno) {
+ code = tsdbReadFilePage(pFD, pgno);
+ if (code) goto _exit;
+ }
+
+ int64_t nRead = TMIN(szPgCont - bOffset, size - n);
+ memcpy(pBuf + n, pFD->pBuf + bOffset, nRead);
+
+ n += nRead;
+ pgno++;
+ bOffset = 0;
}
+_exit:
+ return code;
+}
+
+static int32_t tsdbFsyncFile(STsdbFD *pFD) {
+ int32_t code = 0;
+
+ code = tsdbWriteFilePage(pFD);
+ if (code) goto _exit;
+
+ if (taosFsyncFile(pFD->pFD) < 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _exit;
+ }
+
+_exit:
+ return code;
+}
+
+// SDataFWriter ====================================================
+int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) {
+ int32_t code = 0;
+ int32_t flag;
+ int64_t n;
+ int32_t szPage = pTsdb->pVnode->config.tsdbPageSize;
+ SDataFWriter *pWriter = NULL;
+ char fname[TSDB_FILENAME_LEN];
+ char hdr[TSDB_FHDR_SIZE] = {0};
+
// alloc
- code = tRealloc(&pReader->aBuf[0], size);
+ pWriter = taosMemoryCalloc(1, sizeof(*pWriter));
+ if (pWriter == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
+ pWriter->pTsdb = pTsdb;
+ pWriter->wSet = (SDFileSet){.diskId = pSet->diskId,
+ .fid = pSet->fid,
+ .pHeadF = &pWriter->fHead,
+ .pDataF = &pWriter->fData,
+ .pSmaF = &pWriter->fSma,
+ .nSttF = pSet->nSttF};
+ pWriter->fHead = *pSet->pHeadF;
+ pWriter->fData = *pSet->pDataF;
+ pWriter->fSma = *pSet->pSmaF;
+ for (int8_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+ pWriter->wSet.aSttF[iStt] = &pWriter->fStt[iStt];
+ pWriter->fStt[iStt] = *pSet->aSttF[iStt];
+ }
+
+ // head
+ flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
+ tsdbHeadFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fHead, fname);
+ code = tsdbOpenFile(fname, szPage, flag, &pWriter->pHeadFD);
if (code) goto _err;
- // read
- n = taosReadFile(pReader->pReadH, pReader->aBuf[0], size);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- } else if (n < size) {
- code = TSDB_CODE_FILE_CORRUPTED;
- goto _err;
+ code = tsdbWriteFile(pWriter->pHeadFD, 0, hdr, TSDB_FHDR_SIZE);
+ if (code) goto _err;
+ pWriter->fHead.size += TSDB_FHDR_SIZE;
+
+ // data
+ if (pWriter->fData.size == 0) {
+ flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
+ } else {
+ flag = TD_FILE_READ | TD_FILE_WRITE;
+ }
+ tsdbDataFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fData, fname);
+ code = tsdbOpenFile(fname, szPage, flag, &pWriter->pDataFD);
+ if (code) goto _err;
+ if (pWriter->fData.size == 0) {
+ code = tsdbWriteFile(pWriter->pDataFD, 0, hdr, TSDB_FHDR_SIZE);
+ if (code) goto _err;
+ pWriter->fData.size += TSDB_FHDR_SIZE;
}
- // check
- if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) {
- code = TSDB_CODE_FILE_CORRUPTED;
- goto _err;
+ // sma
+ if (pWriter->fSma.size == 0) {
+ flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
+ } else {
+ flag = TD_FILE_READ | TD_FILE_WRITE;
+ }
+ tsdbSmaFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSma, fname);
+ code = tsdbOpenFile(fname, szPage, flag, &pWriter->pSmaFD);
+ if (code) goto _err;
+ if (pWriter->fSma.size == 0) {
+ code = tsdbWriteFile(pWriter->pSmaFD, 0, hdr, TSDB_FHDR_SIZE);
+ if (code) goto _err;
+
+ pWriter->fSma.size += TSDB_FHDR_SIZE;
}
- // decode
- n = 0;
- uint32_t delimiter;
- n += tGetU32(pReader->aBuf[0] + n, &delimiter);
- ASSERT(delimiter == TSDB_FILE_DLMT);
+ // stt
+ ASSERT(pWriter->fStt[pSet->nSttF - 1].size == 0);
+ flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
+ tsdbSttFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fStt[pSet->nSttF - 1], fname);
+ code = tsdbOpenFile(fname, szPage, flag, &pWriter->pSttFD);
+ if (code) goto _err;
+ code = tsdbWriteFile(pWriter->pSttFD, 0, hdr, TSDB_FHDR_SIZE);
+ if (code) goto _err;
+ pWriter->fStt[pWriter->wSet.nSttF - 1].size += TSDB_FHDR_SIZE;
- while (n < size - sizeof(TSCKSUM)) {
- SDelIdx delIdx;
+ *ppWriter = pWriter;
+ return code;
- n += tGetDelIdx(pReader->aBuf[0] + n, &delIdx);
+_err:
+ tsdbError("vgId:%d, tsdb data file writer open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+ *ppWriter = NULL;
+ return code;
+}
- if (taosArrayPush(aDelIdx, &delIdx) == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
+int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) {
+ int32_t code = 0;
+ STsdb *pTsdb = NULL;
+
+ if (*ppWriter == NULL) goto _exit;
+
+ pTsdb = (*ppWriter)->pTsdb;
+ if (sync) {
+ code = tsdbFsyncFile((*ppWriter)->pHeadFD);
+ if (code) goto _err;
+
+ code = tsdbFsyncFile((*ppWriter)->pDataFD);
+ if (code) goto _err;
+
+ code = tsdbFsyncFile((*ppWriter)->pSmaFD);
+ if (code) goto _err;
+
+ code = tsdbFsyncFile((*ppWriter)->pSttFD);
+ if (code) goto _err;
}
- ASSERT(n == size - sizeof(TSCKSUM));
+ tsdbCloseFile(&(*ppWriter)->pHeadFD);
+ tsdbCloseFile(&(*ppWriter)->pDataFD);
+ tsdbCloseFile(&(*ppWriter)->pSmaFD);
+ tsdbCloseFile(&(*ppWriter)->pSttFD);
+
+ for (int32_t iBuf = 0; iBuf < sizeof((*ppWriter)->aBuf) / sizeof(uint8_t *); iBuf++) {
+ tFree((*ppWriter)->aBuf[iBuf]);
+ }
+ taosMemoryFree(*ppWriter);
+_exit:
+ *ppWriter = NULL;
+ return code;
+
+_err:
+ tsdbError("vgId:%d, data file writer close failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+ return code;
+}
+
+int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) {
+ int32_t code = 0;
+ int64_t n;
+ char hdr[TSDB_FHDR_SIZE];
+
+ // head ==============
+ memset(hdr, 0, TSDB_FHDR_SIZE);
+ tPutHeadFile(hdr, &pWriter->fHead);
+ code = tsdbWriteFile(pWriter->pHeadFD, 0, hdr, TSDB_FHDR_SIZE);
+ if (code) goto _err;
+
+ // data ==============
+ memset(hdr, 0, TSDB_FHDR_SIZE);
+ tPutDataFile(hdr, &pWriter->fData);
+ code = tsdbWriteFile(pWriter->pDataFD, 0, hdr, TSDB_FHDR_SIZE);
+ if (code) goto _err;
+
+ // sma ==============
+ memset(hdr, 0, TSDB_FHDR_SIZE);
+ tPutSmaFile(hdr, &pWriter->fSma);
+ code = tsdbWriteFile(pWriter->pSmaFD, 0, hdr, TSDB_FHDR_SIZE);
+ if (code) goto _err;
+
+ // stt ==============
+ memset(hdr, 0, TSDB_FHDR_SIZE);
+ tPutSttFile(hdr, &pWriter->fStt[pWriter->wSet.nSttF - 1]);
+ code = tsdbWriteFile(pWriter->pSttFD, 0, hdr, TSDB_FHDR_SIZE);
+ if (code) goto _err;
return code;
_err:
- tsdbError("vgId:%d, read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+ tsdbError("vgId:%d, update DFileSet header failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+ return code;
+}
+
+int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) {
+ int32_t code = 0;
+ SHeadFile *pHeadFile = &pWriter->fHead;
+ int64_t size;
+ int64_t n;
+
+ // check
+ if (taosArrayGetSize(aBlockIdx) == 0) {
+ pHeadFile->offset = pHeadFile->size;
+ goto _exit;
+ }
+
+ // prepare
+ size = 0;
+ for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) {
+ size += tPutBlockIdx(NULL, taosArrayGet(aBlockIdx, iBlockIdx));
+ }
+
+ // alloc
+ code = tRealloc(&pWriter->aBuf[0], size);
+ if (code) goto _err;
+
+ // build
+ n = 0;
+ for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) {
+ n += tPutBlockIdx(pWriter->aBuf[0] + n, taosArrayGet(aBlockIdx, iBlockIdx));
+ }
+ ASSERT(n == size);
+
+ // write
+ code = tsdbWriteFile(pWriter->pHeadFD, pHeadFile->size, pWriter->aBuf[0], size);
+ if (code) goto _err;
+
+ // update
+ pHeadFile->offset = pHeadFile->size;
+ pHeadFile->size += size;
+
+_exit:
+ // tsdbTrace("vgId:%d write block idx, offset:%" PRId64 " size:%" PRId64 " nBlockIdx:%d",
+ // TD_VID(pWriter->pTsdb->pVnode),
+ // pHeadFile->offset, size, taosArrayGetSize(aBlockIdx));
+ return code;
+
+_err:
+ tsdbError("vgId:%d, write block idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+ return code;
+}
+
+int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *pBlockIdx) {
+ int32_t code = 0;
+ SHeadFile *pHeadFile = &pWriter->fHead;
+ int64_t size;
+ int64_t n;
+
+ ASSERT(mDataBlk->nItem > 0);
+
+ // alloc
+ size = tPutMapData(NULL, mDataBlk);
+ code = tRealloc(&pWriter->aBuf[0], size);
+ if (code) goto _err;
+
+ // build
+ n = tPutMapData(pWriter->aBuf[0], mDataBlk);
+
+ // write
+ code = tsdbWriteFile(pWriter->pHeadFD, pHeadFile->size, pWriter->aBuf[0], size);
+ if (code) goto _err;
+
+ // update
+ pBlockIdx->offset = pHeadFile->size;
+ pBlockIdx->size = size;
+ pHeadFile->size += size;
+
+ tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%d suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64
+ " size:%" PRId64 " nItem:%d",
+ TD_VID(pWriter->pTsdb->pVnode), pWriter->wSet.fid, pHeadFile->commitID, pBlockIdx->suid, pBlockIdx->uid,
+ pBlockIdx->offset, pBlockIdx->size, mDataBlk->nItem);
+ return code;
+
+_err:
+ tsdbError("vgId:%d, write block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+ return code;
+}
+
+int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk) {
+ int32_t code = 0;
+ SSttFile *pSttFile = &pWriter->fStt[pWriter->wSet.nSttF - 1];
+ int64_t size;
+ int64_t n;
+
+ // check
+ if (taosArrayGetSize(aSttBlk) == 0) {
+ pSttFile->offset = pSttFile->size;
+ goto _exit;
+ }
+
+ // size
+ size = 0;
+ for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSttBlk); iBlockL++) {
+ size += tPutSttBlk(NULL, taosArrayGet(aSttBlk, iBlockL));
+ }
+
+ // alloc
+ code = tRealloc(&pWriter->aBuf[0], size);
+ if (code) goto _err;
+
+ // encode
+ n = 0;
+ for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSttBlk); iBlockL++) {
+ n += tPutSttBlk(pWriter->aBuf[0] + n, taosArrayGet(aSttBlk, iBlockL));
+ }
+
+ // write
+ code = tsdbWriteFile(pWriter->pSttFD, pSttFile->size, pWriter->aBuf[0], size);
+ if (code) goto _err;
+
+ // update
+ pSttFile->offset = pSttFile->size;
+ pSttFile->size += size;
+
+_exit:
+ tsdbTrace("vgId:%d tsdb write stt block, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode),
+ pSttFile->offset, size);
+ return code;
+
+_err:
+ tsdbError("vgId:%d tsdb write blockl failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+ return code;
+}
+
+static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, SSmaInfo *pSmaInfo) {
+ int32_t code = 0;
+
+ pSmaInfo->offset = 0;
+ pSmaInfo->size = 0;
+
+ // encode
+ for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) {
+ SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
+
+ if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type)) continue;
+
+ SColumnDataAgg sma;
+ tsdbCalcColDataSMA(pColData, &sma);
+
+ code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &sma));
+ if (code) goto _err;
+ pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &sma);
+ }
+
+ // write
+ if (pSmaInfo->size) {
+ code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size);
+ if (code) goto _err;
+
+ code = tsdbWriteFile(pWriter->pSmaFD, pWriter->fSma.size, pWriter->aBuf[0], pSmaInfo->size);
+ if (code) goto _err;
+
+ pSmaInfo->offset = pWriter->fSma.size;
+ pWriter->fSma.size += pSmaInfo->size;
+ }
+
+ return code;
+
+_err:
+ tsdbError("vgId:%d tsdb write block sma failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+ return code;
+}
+
+int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo,
+ int8_t cmprAlg, int8_t toLast) {
+ int32_t code = 0;
+
+ ASSERT(pBlockData->nRow > 0);
+
+ if (toLast) {
+ pBlkInfo->offset = pWriter->fStt[pWriter->wSet.nSttF - 1].size;
+ } else {
+ pBlkInfo->offset = pWriter->fData.size;
+ }
+ pBlkInfo->szBlock = 0;
+ pBlkInfo->szKey = 0;
+
+ int32_t aBufN[4] = {0};
+ code = tCmprBlockData(pBlockData, cmprAlg, NULL, NULL, pWriter->aBuf, aBufN);
+ if (code) goto _err;
+
+ // write =================
+ STsdbFD *pFD = toLast ? pWriter->pSttFD : pWriter->pDataFD;
+
+ pBlkInfo->szKey = aBufN[3] + aBufN[2];
+ pBlkInfo->szBlock = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3];
+
+ int64_t offset = pBlkInfo->offset;
+ code = tsdbWriteFile(pFD, offset, pWriter->aBuf[3], aBufN[3]);
+ if (code) goto _err;
+ offset += aBufN[3];
+
+ code = tsdbWriteFile(pFD, offset, pWriter->aBuf[2], aBufN[2]);
+ if (code) goto _err;
+ offset += aBufN[2];
+
+ if (aBufN[1]) {
+ code = tsdbWriteFile(pFD, offset, pWriter->aBuf[1], aBufN[1]);
+ if (code) goto _err;
+ offset += aBufN[1];
+ }
+
+ if (aBufN[0]) {
+ code = tsdbWriteFile(pFD, offset, pWriter->aBuf[0], aBufN[0]);
+ if (code) goto _err;
+ }
+
+ // update info
+ if (toLast) {
+ pWriter->fStt[pWriter->wSet.nSttF - 1].size += pBlkInfo->szBlock;
+ } else {
+ pWriter->fData.size += pBlkInfo->szBlock;
+ }
+
+ // ================= SMA ====================
+ if (pSmaInfo) {
+ code = tsdbWriteBlockSma(pWriter, pBlockData, pSmaInfo);
+ if (code) goto _err;
+ }
+
+_exit:
+ tsdbTrace("vgId:%d tsdb write block data, suid:%" PRId64 " uid:%" PRId64 " nRow:%d, offset:%" PRId64 " size:%d",
+ TD_VID(pWriter->pTsdb->pVnode), pBlockData->suid, pBlockData->uid, pBlockData->nRow, pBlkInfo->offset,
+ pBlkInfo->szBlock);
+ return code;
+
+_err:
+ tsdbError("vgId:%d tsdb write block data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+ return code;
+}
+
+int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
+ int32_t code = 0;
+ int64_t n;
+ int64_t size;
+ TdFilePtr pOutFD = NULL;
+ TdFilePtr PInFD = NULL;
+ int32_t szPage = pTsdb->pVnode->config.szPage;
+ char fNameFrom[TSDB_FILENAME_LEN];
+ char fNameTo[TSDB_FILENAME_LEN];
+
+ // head
+ tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom);
+ tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo);
+ pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
+ if (pOutFD == NULL) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ PInFD = taosOpenFile(fNameFrom, TD_FILE_READ);
+ if (PInFD == NULL) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pHeadF->size, szPage));
+ if (n < 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ taosCloseFile(&pOutFD);
+ taosCloseFile(&PInFD);
+
+ // data
+ tsdbDataFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pDataF, fNameFrom);
+ tsdbDataFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pDataF, fNameTo);
+ pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
+ if (pOutFD == NULL) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ PInFD = taosOpenFile(fNameFrom, TD_FILE_READ);
+ if (PInFD == NULL) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ n = taosFSendFile(pOutFD, PInFD, 0, LOGIC_TO_FILE_OFFSET(pSetFrom->pDataF->size, szPage));
+ if (n < 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ taosCloseFile(&pOutFD);
+ taosCloseFile(&PInFD);
+
+ // sma
+ tsdbSmaFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pSmaF, fNameFrom);
+ tsdbSmaFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pSmaF, fNameTo);
+ pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
+ if (pOutFD == NULL) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ PInFD = taosOpenFile(fNameFrom, TD_FILE_READ);
+ if (PInFD == NULL) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pSmaF->size, szPage));
+ if (n < 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ taosCloseFile(&pOutFD);
+ taosCloseFile(&PInFD);
+
+ // stt
+ for (int8_t iStt = 0; iStt < pSetFrom->nSttF; iStt++) {
+ tsdbSttFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSttF[iStt], fNameFrom);
+ tsdbSttFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSttF[iStt], fNameTo);
+ pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
+ if (pOutFD == NULL) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ PInFD = taosOpenFile(fNameFrom, TD_FILE_READ);
+ if (PInFD == NULL) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->aSttF[iStt]->size, szPage));
+ if (n < 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ taosCloseFile(&pOutFD);
+ taosCloseFile(&PInFD);
+ }
+
+ return code;
+
+_err:
+ tsdbError("vgId:%d, tsdb DFileSet copy failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code;
}
// SDataFReader ====================================================
-struct SDataFReader {
- STsdb *pTsdb;
- SDFileSet *pSet;
- TdFilePtr pHeadFD;
- TdFilePtr pDataFD;
- TdFilePtr pLastFD;
- TdFilePtr pSmaFD;
-
- uint8_t *aBuf[3];
-};
-
int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) {
int32_t code = 0;
SDataFReader *pReader;
+ int32_t szPage = pTsdb->pVnode->config.tsdbPageSize;
char fname[TSDB_FILENAME_LEN];
// alloc
@@ -419,37 +726,26 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
pReader->pTsdb = pTsdb;
pReader->pSet = pSet;
- // open impl
// head
tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
- pReader->pHeadFD = taosOpenFile(fname, TD_FILE_READ);
- if (pReader->pHeadFD == NULL) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
+ code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pHeadFD);
+ if (code) goto _err;
// data
tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname);
- pReader->pDataFD = taosOpenFile(fname, TD_FILE_READ);
- if (pReader->pDataFD == NULL) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- // last
- tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pLastF, fname);
- pReader->pLastFD = taosOpenFile(fname, TD_FILE_READ);
- if (pReader->pLastFD == NULL) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
+ code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pDataFD);
+ if (code) goto _err;
// sma
tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
- pReader->pSmaFD = taosOpenFile(fname, TD_FILE_READ);
- if (pReader->pSmaFD == NULL) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
+ code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pSmaFD);
+ if (code) goto _err;
+
+ // stt
+ for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+ tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname);
+ code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->aSttFD[iStt]);
+ if (code) goto _err;
}
*ppReader = pReader;
@@ -463,35 +759,28 @@ _err:
int32_t tsdbDataFReaderClose(SDataFReader **ppReader) {
int32_t code = 0;
- if (*ppReader == NULL) goto _exit;
+ if (*ppReader == NULL) return code;
- if (taosCloseFile(&(*ppReader)->pHeadFD) < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
+ // head
+ tsdbCloseFile(&(*ppReader)->pHeadFD);
- if (taosCloseFile(&(*ppReader)->pDataFD) < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
+ // data
+ tsdbCloseFile(&(*ppReader)->pDataFD);
- if (taosCloseFile(&(*ppReader)->pLastFD) < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
+ // sma
+ tsdbCloseFile(&(*ppReader)->pSmaFD);
- if (taosCloseFile(&(*ppReader)->pSmaFD) < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
+ // stt
+ for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) {
+ if ((*ppReader)->aSttFD[iStt]) {
+ tsdbCloseFile(&(*ppReader)->aSttFD[iStt]);
+ }
}
for (int32_t iBuf = 0; iBuf < sizeof((*ppReader)->aBuf) / sizeof(uint8_t *); iBuf++) {
tFree((*ppReader)->aBuf[iBuf]);
}
-
taosMemoryFree(*ppReader);
-
-_exit:
*ppReader = NULL;
return code;
@@ -501,49 +790,25 @@ _err:
}
int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) {
- int32_t code = 0;
- int64_t offset = pReader->pSet->pHeadF->offset;
- int64_t size = pReader->pSet->pHeadF->size - offset;
- int64_t n;
- uint32_t delimiter;
+ int32_t code = 0;
+ SHeadFile *pHeadFile = pReader->pSet->pHeadF;
+ int64_t offset = pHeadFile->offset;
+ int64_t size = pHeadFile->size - offset;
taosArrayClear(aBlockIdx);
- if (size == 0) {
- goto _exit;
- }
+ if (size == 0) return code;
// alloc
code = tRealloc(&pReader->aBuf[0], size);
if (code) goto _err;
- // seek
- if (taosLSeekFile(pReader->pHeadFD, offset, SEEK_SET) < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
// read
- n = taosReadFile(pReader->pHeadFD, pReader->aBuf[0], size);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- } else if (n < size) {
- code = TSDB_CODE_FILE_CORRUPTED;
- goto _err;
- }
-
- // check
- if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) {
- code = TSDB_CODE_FILE_CORRUPTED;
- goto _err;
- }
+ code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size);
+ if (code) goto _err;
// decode
- n = 0;
- n = tGetU32(pReader->aBuf[0] + n, &delimiter);
- ASSERT(delimiter == TSDB_FILE_DLMT);
-
- while (n < size - sizeof(TSCKSUM)) {
+ int64_t n = 0;
+ while (n < size) {
SBlockIdx blockIdx;
n += tGetBlockIdx(pReader->aBuf[0] + n, &blockIdx);
@@ -552,10 +817,8 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) {
goto _err;
}
}
+ ASSERT(n == size);
- ASSERT(n + sizeof(TSCKSUM) == size);
-
-_exit:
return code;
_err:
@@ -563,116 +826,63 @@ _err:
return code;
}
-int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) {
- int32_t code = 0;
- int64_t offset = pReader->pSet->pLastF->offset;
- int64_t size = pReader->pSet->pLastF->size - offset;
- int64_t n;
- uint32_t delimiter;
+int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk) {
+ int32_t code = 0;
+ SSttFile *pSttFile = pReader->pSet->aSttF[iStt];
+ int64_t offset = pSttFile->offset;
+ int64_t size = pSttFile->size - offset;
- taosArrayClear(aBlockL);
- if (size == 0) {
- goto _exit;
- }
+ taosArrayClear(aSttBlk);
+ if (size == 0) return code;
// alloc
code = tRealloc(&pReader->aBuf[0], size);
if (code) goto _err;
- // seek
- if (taosLSeekFile(pReader->pLastFD, offset, SEEK_SET) < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
// read
- n = taosReadFile(pReader->pLastFD, pReader->aBuf[0], size);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- } else if (n < size) {
- code = TSDB_CODE_FILE_CORRUPTED;
- goto _err;
- }
-
- // check
- if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) {
- code = TSDB_CODE_FILE_CORRUPTED;
- goto _err;
- }
+ code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size);
+ if (code) goto _err;
// decode
- n = 0;
- n = tGetU32(pReader->aBuf[0] + n, &delimiter);
- ASSERT(delimiter == TSDB_FILE_DLMT);
+ int64_t n = 0;
+ while (n < size) {
+ SSttBlk sttBlk;
+ n += tGetSttBlk(pReader->aBuf[0] + n, &sttBlk);
- while (n < size - sizeof(TSCKSUM)) {
- SBlockL blockl;
- n += tGetBlockL(pReader->aBuf[0] + n, &blockl);
-
- if (taosArrayPush(aBlockL, &blockl) == NULL) {
+ if (taosArrayPush(aSttBlk, &sttBlk) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
}
+ ASSERT(n == size);
- ASSERT(n + sizeof(TSCKSUM) == size);
-
-_exit:
return code;
_err:
- tsdbError("vgId:%d read blockl failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+ tsdbError("vgId:%d read stt blk failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
return code;
}
-int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBlock) {
+int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mDataBlk) {
int32_t code = 0;
int64_t offset = pBlockIdx->offset;
int64_t size = pBlockIdx->size;
- int64_t n;
- int64_t tn;
// alloc
code = tRealloc(&pReader->aBuf[0], size);
if (code) goto _err;
- // seek
- if (taosLSeekFile(pReader->pHeadFD, offset, SEEK_SET) < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
// read
- n = taosReadFile(pReader->pHeadFD, pReader->aBuf[0], size);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- } else if (n < size) {
- code = TSDB_CODE_FILE_CORRUPTED;
- goto _err;
- }
-
- // check
- if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) {
- code = TSDB_CODE_FILE_CORRUPTED;
- goto _err;
- }
+ code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size);
+ if (code) goto _err;
// decode
- n = 0;
-
- uint32_t delimiter;
- n += tGetU32(pReader->aBuf[0] + n, &delimiter);
- ASSERT(delimiter == TSDB_FILE_DLMT);
-
- tn = tGetMapData(pReader->aBuf[0] + n, mBlock);
- if (tn < 0) {
+ int64_t n = tGetMapData(pReader->aBuf[0], mDataBlk);
+ if (n < 0) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
- n += tn;
- ASSERT(n + sizeof(TSCKSUM) == size);
+ ASSERT(n == size);
return code;
@@ -681,57 +891,34 @@ _err:
return code;
}
-int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg) {
+int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pDataBlk, SArray *aColumnDataAgg) {
int32_t code = 0;
- SSmaInfo *pSmaInfo = &pBlock->smaInfo;
+ SSmaInfo *pSmaInfo = &pDataBlk->smaInfo;
ASSERT(pSmaInfo->size > 0);
taosArrayClear(aColumnDataAgg);
// alloc
- int32_t size = pSmaInfo->size + sizeof(TSCKSUM);
- code = tRealloc(&pReader->aBuf[0], size);
+ code = tRealloc(&pReader->aBuf[0], pSmaInfo->size);
if (code) goto _err;
- // seek
- int64_t n = taosLSeekFile(pReader->pSmaFD, pSmaInfo->offset, SEEK_SET);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- } else if (n < pSmaInfo->offset) {
- code = TSDB_CODE_FILE_CORRUPTED;
- goto _err;
- }
-
// read
- n = taosReadFile(pReader->pSmaFD, pReader->aBuf[0], size);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- } else if (n < size) {
- code = TSDB_CODE_FILE_CORRUPTED;
- goto _err;
- }
-
- // check
- if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) {
- code = TSDB_CODE_FILE_CORRUPTED;
- goto _err;
- }
+ code = tsdbReadFile(pReader->pSmaFD, pSmaInfo->offset, pReader->aBuf[0], pSmaInfo->size);
+ if (code) goto _err;
// decode
- n = 0;
+ int32_t n = 0;
while (n < pSmaInfo->size) {
SColumnDataAgg sma;
-
n += tGetColumnDataAgg(pReader->aBuf[0] + n, &sma);
+
if (taosArrayPush(aColumnDataAgg, &sma) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
}
-
+ ASSERT(n == pSmaInfo->size);
return code;
_err:
@@ -739,17 +926,20 @@ _err:
return code;
}
-static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo, int8_t fromLast,
- SBlockData *pBlockData) {
+static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo, SBlockData *pBlockData) {
int32_t code = 0;
tBlockDataClear(pBlockData);
- TdFilePtr pFD = fromLast ? pReader->pLastFD : pReader->pDataFD;
+ STsdbFD *pFD = pReader->pDataFD;
// uid + version + tskey
- code = tsdbReadAndCheck(pFD, pBlkInfo->offset, &pReader->aBuf[0], pBlkInfo->szKey, 1);
+ code = tRealloc(&pReader->aBuf[0], pBlkInfo->szKey);
if (code) goto _err;
+
+ code = tsdbReadFile(pFD, pBlkInfo->offset, pReader->aBuf[0], pBlkInfo->szKey);
+ if (code) goto _err;
+
SDiskDataHdr hdr;
uint8_t *p = pReader->aBuf[0] + tGetDiskDataHdr(pReader->aBuf[0], &hdr);
@@ -782,14 +972,18 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo
if (code) goto _err;
p += hdr.szKey;
- ASSERT(p - pReader->aBuf[0] == pBlkInfo->szKey - sizeof(TSCKSUM));
+ ASSERT(p - pReader->aBuf[0] == pBlkInfo->szKey);
// read and decode columns
if (taosArrayGetSize(pBlockData->aIdx) == 0) goto _exit;
if (hdr.szBlkCol > 0) {
int64_t offset = pBlkInfo->offset + pBlkInfo->szKey;
- code = tsdbReadAndCheck(pFD, offset, &pReader->aBuf[0], hdr.szBlkCol + sizeof(TSCKSUM), 1);
+
+ code = tRealloc(&pReader->aBuf[0], hdr.szBlkCol);
+ if (code) goto _err;
+
+ code = tsdbReadFile(pFD, offset, pReader->aBuf[0], hdr.szBlkCol);
if (code) goto _err;
}
@@ -827,10 +1021,13 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo
}
} else {
// decode from binary
- int64_t offset = pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + sizeof(TSCKSUM) + pBlockCol->offset;
- int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM);
+ int64_t offset = pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + pBlockCol->offset;
+ int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue;
- code = tsdbReadAndCheck(pFD, offset, &pReader->aBuf[1], size, 0);
+ code = tRealloc(&pReader->aBuf[1], size);
+ if (code) goto _err;
+
+ code = tsdbReadFile(pFD, offset, pReader->aBuf[1], size);
if (code) goto _err;
code = tsdbDecmprColData(pReader->aBuf[1], pBlockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->aBuf[2]);
@@ -847,13 +1044,36 @@ _err:
return code;
}
-int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData) {
- int32_t code = 0;
+int32_t tsdbReadDataBlockEx(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData) {
+ int32_t code = 0;
+ SBlockInfo *pBlockInfo = &pDataBlk->aSubBlock[0];
- code = tsdbReadBlockDataImpl(pReader, &pBlock->aSubBlock[0], 0, pBlockData);
+ // alloc
+ code = tRealloc(&pReader->aBuf[0], pBlockInfo->szBlock);
if (code) goto _err;
- if (pBlock->nSubBlock > 1) {
+ // read
+ code = tsdbReadFile(pReader->pDataFD, pBlockInfo->offset, pReader->aBuf[0], pBlockInfo->szBlock);
+ if (code) goto _err;
+
+ // decmpr
+ code = tDecmprBlockData(pReader->aBuf[0], pBlockInfo->szBlock, pBlockData, &pReader->aBuf[1]);
+ if (code) goto _err;
+
+ return code;
+
+_err:
+ tsdbError("vgId:%d tsdb read data block ex failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+ return code;
+}
+
+int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData) {
+ int32_t code = 0;
+
+ code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[0], pBlockData);
+ if (code) goto _err;
+
+ if (pDataBlk->nSubBlock > 1) {
SBlockData bData1;
SBlockData bData2;
@@ -867,8 +1087,8 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBl
tBlockDataInitEx(&bData1, pBlockData);
tBlockDataInitEx(&bData2, pBlockData);
- for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) {
- code = tsdbReadBlockDataImpl(pReader, &pBlock->aSubBlock[iSubBlock], 0, &bData1);
+ for (int32_t iSubBlock = 1; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) {
+ code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[iSubBlock], &bData1);
if (code) {
tBlockDataDestroy(&bData1, 1);
tBlockDataDestroy(&bData2, 1);
@@ -901,325 +1121,103 @@ _err:
return code;
}
-int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData) {
+int32_t tsdbReadSttBlock(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData) {
int32_t code = 0;
- code = tsdbReadBlockDataImpl(pReader, &pBlockL->bInfo, 1, pBlockData);
+ // alloc
+ code = tRealloc(&pReader->aBuf[0], pSttBlk->bInfo.szBlock);
+ if (code) goto _err;
+
+ // read
+ code = tsdbReadFile(pReader->aSttFD[iStt], pSttBlk->bInfo.offset, pReader->aBuf[0], pSttBlk->bInfo.szBlock);
+ if (code) goto _err;
+
+ // decmpr
+ code = tDecmprBlockData(pReader->aBuf[0], pSttBlk->bInfo.szBlock, pBlockData, &pReader->aBuf[1]);
if (code) goto _err;
return code;
_err:
- tsdbError("vgId:%d tsdb read last block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+ tsdbError("vgId:%d tsdb read stt block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
return code;
}
-// SDataFWriter ====================================================
-int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) {
- int32_t code = 0;
- int32_t flag;
- int64_t n;
- SDataFWriter *pWriter = NULL;
- char fname[TSDB_FILENAME_LEN];
- char hdr[TSDB_FHDR_SIZE] = {0};
+// SDelFWriter ====================================================
+int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) {
+ int32_t code = 0;
+ char fname[TSDB_FILENAME_LEN];
+ uint8_t hdr[TSDB_FHDR_SIZE] = {0};
+ SDelFWriter *pDelFWriter;
+ int64_t n;
// alloc
- pWriter = taosMemoryCalloc(1, sizeof(*pWriter));
- if (pWriter == NULL) {
+ pDelFWriter = (SDelFWriter *)taosMemoryCalloc(1, sizeof(*pDelFWriter));
+ if (pDelFWriter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
+ pDelFWriter->pTsdb = pTsdb;
+ pDelFWriter->fDel = *pFile;
+
+ tsdbDelFileName(pTsdb, pFile, fname);
+ code = tsdbOpenFile(fname, pTsdb->pVnode->config.tsdbPageSize, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE,
+ &pDelFWriter->pWriteH);
if (code) goto _err;
- pWriter->pTsdb = pTsdb;
- pWriter->wSet = (SDFileSet){.diskId = pSet->diskId,
- .fid = pSet->fid,
- .pHeadF = &pWriter->fHead,
- .pDataF = &pWriter->fData,
- .pLastF = &pWriter->fLast,
- .pSmaF = &pWriter->fSma};
- pWriter->fHead = *pSet->pHeadF;
- pWriter->fData = *pSet->pDataF;
- pWriter->fLast = *pSet->pLastF;
- pWriter->fSma = *pSet->pSmaF;
- // head
- flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
- tsdbHeadFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fHead, fname);
- pWriter->pHeadFD = taosOpenFile(fname, flag);
- if (pWriter->pHeadFD == NULL) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
+ // update header
+ code = tsdbWriteFile(pDelFWriter->pWriteH, 0, hdr, TSDB_FHDR_SIZE);
+ if (code) goto _err;
- n = taosWriteFile(pWriter->pHeadFD, hdr, TSDB_FHDR_SIZE);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
+ pDelFWriter->fDel.size = TSDB_FHDR_SIZE;
+ pDelFWriter->fDel.offset = 0;
- ASSERT(n == TSDB_FHDR_SIZE);
-
- pWriter->fHead.size += TSDB_FHDR_SIZE;
-
- // data
- if (pWriter->fData.size == 0) {
- flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
- } else {
- flag = TD_FILE_WRITE;
- }
- tsdbDataFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fData, fname);
- pWriter->pDataFD = taosOpenFile(fname, flag);
- if (pWriter->pDataFD == NULL) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
- if (pWriter->fData.size == 0) {
- n = taosWriteFile(pWriter->pDataFD, hdr, TSDB_FHDR_SIZE);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- pWriter->fData.size += TSDB_FHDR_SIZE;
- } else {
- n = taosLSeekFile(pWriter->pDataFD, 0, SEEK_END);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- ASSERT(n == pWriter->fData.size);
- }
-
- // last
- if (pWriter->fLast.size == 0) {
- flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
- } else {
- flag = TD_FILE_WRITE;
- }
- tsdbLastFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fLast, fname);
- pWriter->pLastFD = taosOpenFile(fname, flag);
- if (pWriter->pLastFD == NULL) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
- if (pWriter->fLast.size == 0) {
- n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- pWriter->fLast.size += TSDB_FHDR_SIZE;
- } else {
- n = taosLSeekFile(pWriter->pLastFD, 0, SEEK_END);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- ASSERT(n == pWriter->fLast.size);
- }
-
- // sma
- if (pWriter->fSma.size == 0) {
- flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
- } else {
- flag = TD_FILE_WRITE;
- }
- tsdbSmaFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSma, fname);
- pWriter->pSmaFD = taosOpenFile(fname, flag);
- if (pWriter->pSmaFD == NULL) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
- if (pWriter->fSma.size == 0) {
- n = taosWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- pWriter->fSma.size += TSDB_FHDR_SIZE;
- } else {
- n = taosLSeekFile(pWriter->pSmaFD, 0, SEEK_END);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- ASSERT(n == pWriter->fSma.size);
- }
-
- *ppWriter = pWriter;
+ *ppWriter = pDelFWriter;
return code;
_err:
- tsdbError("vgId:%d, tsdb data file writer open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+ tsdbError("vgId:%d, failed to open del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
*ppWriter = NULL;
return code;
}
-int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) {
- int32_t code = 0;
- STsdb *pTsdb = NULL;
+int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync) {
+ int32_t code = 0;
+ SDelFWriter *pWriter = *ppWriter;
+ STsdb *pTsdb = pWriter->pTsdb;
- if (*ppWriter == NULL) goto _exit;
-
- pTsdb = (*ppWriter)->pTsdb;
+ // sync
if (sync) {
- if (taosFsyncFile((*ppWriter)->pHeadFD) < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- if (taosFsyncFile((*ppWriter)->pDataFD) < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- if (taosFsyncFile((*ppWriter)->pLastFD) < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- if (taosFsyncFile((*ppWriter)->pSmaFD) < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
+ code = tsdbFsyncFile(pWriter->pWriteH);
+ if (code) goto _err;
}
- if (taosCloseFile(&(*ppWriter)->pHeadFD) < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
+ // close
+ tsdbCloseFile(&pWriter->pWriteH);
- if (taosCloseFile(&(*ppWriter)->pDataFD) < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
+ for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t *); iBuf++) {
+ tFree(pWriter->aBuf[iBuf]);
}
+ taosMemoryFree(pWriter);
- if (taosCloseFile(&(*ppWriter)->pLastFD) < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- if (taosCloseFile(&(*ppWriter)->pSmaFD) < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- for (int32_t iBuf = 0; iBuf < sizeof((*ppWriter)->aBuf) / sizeof(uint8_t *); iBuf++) {
- tFree((*ppWriter)->aBuf[iBuf]);
- }
- taosMemoryFree(*ppWriter);
-_exit:
*ppWriter = NULL;
return code;
_err:
- tsdbError("vgId:%d, data file writer close failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+ tsdbError("vgId:%d, failed to close del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code;
}
-int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) {
+int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelIdx) {
int32_t code = 0;
+ int64_t size;
int64_t n;
- char hdr[TSDB_FHDR_SIZE];
-
- // head ==============
- memset(hdr, 0, TSDB_FHDR_SIZE);
- tPutHeadFile(hdr, &pWriter->fHead);
- taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE);
-
- n = taosLSeekFile(pWriter->pHeadFD, 0, SEEK_SET);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- n = taosWriteFile(pWriter->pHeadFD, hdr, TSDB_FHDR_SIZE);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- // data ==============
- memset(hdr, 0, TSDB_FHDR_SIZE);
- tPutDataFile(hdr, &pWriter->fData);
- taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE);
-
- n = taosLSeekFile(pWriter->pDataFD, 0, SEEK_SET);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- n = taosWriteFile(pWriter->pDataFD, hdr, TSDB_FHDR_SIZE);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- // last ==============
- memset(hdr, 0, TSDB_FHDR_SIZE);
- tPutLastFile(hdr, &pWriter->fLast);
- taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE);
-
- n = taosLSeekFile(pWriter->pLastFD, 0, SEEK_SET);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- // sma ==============
- memset(hdr, 0, TSDB_FHDR_SIZE);
- tPutSmaFile(hdr, &pWriter->fSma);
- taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE);
-
- n = taosLSeekFile(pWriter->pSmaFD, 0, SEEK_SET);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- n = taosWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- return code;
-
-_err:
- tsdbError("vgId:%d, update DFileSet header failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
- return code;
-}
-
-int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) {
- int32_t code = 0;
- SHeadFile *pHeadFile = &pWriter->fHead;
- int64_t size = 0;
- int64_t n;
-
- // check
- if (taosArrayGetSize(aBlockIdx) == 0) {
- pHeadFile->offset = pHeadFile->size;
- goto _exit;
- }
// prepare
- size = sizeof(uint32_t);
- for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) {
- size += tPutBlockIdx(NULL, taosArrayGet(aBlockIdx, iBlockIdx));
+ size = 0;
+ for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) {
+ size += tPutDelData(NULL, taosArrayGet(aDelData, iDelData));
}
- size += sizeof(TSCKSUM);
// alloc
code = tRealloc(&pWriter->aBuf[0], size);
@@ -1227,383 +1225,209 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) {
// build
n = 0;
- n = tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT);
- for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) {
- n += tPutBlockIdx(pWriter->aBuf[0] + n, taosArrayGet(aBlockIdx, iBlockIdx));
+ for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) {
+ n += tPutDelData(pWriter->aBuf[0] + n, taosArrayGet(aDelData, iDelData));
}
- taosCalcChecksumAppend(0, pWriter->aBuf[0], size);
-
- ASSERT(n + sizeof(TSCKSUM) == size);
+ ASSERT(n == size);
// write
- n = taosWriteFile(pWriter->pHeadFD, pWriter->aBuf[0], size);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
+ code = tsdbWriteFile(pWriter->pWriteH, pWriter->fDel.size, pWriter->aBuf[0], size);
+ if (code) goto _err;
// update
- pHeadFile->offset = pHeadFile->size;
- pHeadFile->size += size;
+ pDelIdx->offset = pWriter->fDel.size;
+ pDelIdx->size = size;
+ pWriter->fDel.size += size;
-_exit:
- tsdbTrace("vgId:%d write block idx, offset:%" PRId64 " size:%" PRId64 " nBlockIdx:%d", TD_VID(pWriter->pTsdb->pVnode),
- pHeadFile->offset, size, taosArrayGetSize(aBlockIdx));
return code;
_err:
- tsdbError("vgId:%d, write block idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+ tsdbError("vgId:%d, failed to write del data since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
return code;
}
-int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, SBlockIdx *pBlockIdx) {
- int32_t code = 0;
- SHeadFile *pHeadFile = &pWriter->fHead;
- int64_t size;
- int64_t n;
+int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx) {
+ int32_t code = 0;
+ int64_t size;
+ int64_t n;
+ SDelIdx *pDelIdx;
- ASSERT(mBlock->nItem > 0);
+ // prepare
+ size = 0;
+ for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) {
+ size += tPutDelIdx(NULL, taosArrayGet(aDelIdx, iDelIdx));
+ }
// alloc
- size = sizeof(uint32_t) + tPutMapData(NULL, mBlock) + sizeof(TSCKSUM);
code = tRealloc(&pWriter->aBuf[0], size);
if (code) goto _err;
// build
n = 0;
- n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT);
- n += tPutMapData(pWriter->aBuf[0] + n, mBlock);
- taosCalcChecksumAppend(0, pWriter->aBuf[0], size);
-
- ASSERT(n + sizeof(TSCKSUM) == size);
+ for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) {
+ n += tPutDelIdx(pWriter->aBuf[0] + n, taosArrayGet(aDelIdx, iDelIdx));
+ }
+ ASSERT(n == size);
// write
- n = taosWriteFile(pWriter->pHeadFD, pWriter->aBuf[0], size);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
+ code = tsdbWriteFile(pWriter->pWriteH, pWriter->fDel.size, pWriter->aBuf[0], size);
+ if (code) goto _err;
// update
- pBlockIdx->offset = pHeadFile->size;
- pBlockIdx->size = size;
- pHeadFile->size += size;
+ pWriter->fDel.offset = pWriter->fDel.size;
+ pWriter->fDel.size += size;
- tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%d suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64
- " size:%" PRId64 " nItem:%d",
- TD_VID(pWriter->pTsdb->pVnode), pWriter->wSet.fid, pHeadFile->commitID, pBlockIdx->suid, pBlockIdx->uid,
- pBlockIdx->offset, pBlockIdx->size, mBlock->nItem);
return code;
_err:
- tsdbError("vgId:%d, write block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+ tsdbError("vgId:%d, write del idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
return code;
}
-int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) {
- int32_t code = 0;
- SLastFile *pLastFile = &pWriter->fLast;
- int64_t size;
- int64_t n;
+int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter) {
+ int32_t code = 0;
+ char hdr[TSDB_FHDR_SIZE] = {0};
+ int64_t size = TSDB_FHDR_SIZE;
+ int64_t n;
- // check
- if (taosArrayGetSize(aBlockL) == 0) {
- pLastFile->offset = pLastFile->size;
- goto _exit;
- }
+ // build
+ tPutDelFile(hdr, &pWriter->fDel);
- // size
- size = sizeof(uint32_t); // TSDB_FILE_DLMT
- for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) {
- size += tPutBlockL(NULL, taosArrayGet(aBlockL, iBlockL));
- }
- size += sizeof(TSCKSUM);
+ // write
+ code = tsdbWriteFile(pWriter->pWriteH, 0, hdr, size);
+ if (code) goto _err;
+
+ return code;
+
+_err:
+ tsdbError("vgId:%d, update del file hdr failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+ return code;
+}
+// SDelFReader ====================================================
+struct SDelFReader {
+ STsdb *pTsdb;
+ SDelFile fDel;
+ STsdbFD *pReadH;
+ uint8_t *aBuf[1];
+};
+
+int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb) {
+ int32_t code = 0;
+ char fname[TSDB_FILENAME_LEN];
+ SDelFReader *pDelFReader;
+ int64_t n;
// alloc
- code = tRealloc(&pWriter->aBuf[0], size);
+ pDelFReader = (SDelFReader *)taosMemoryCalloc(1, sizeof(*pDelFReader));
+ if (pDelFReader == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
+
+ // open impl
+ pDelFReader->pTsdb = pTsdb;
+ pDelFReader->fDel = *pFile;
+
+ tsdbDelFileName(pTsdb, pFile, fname);
+ code = tsdbOpenFile(fname, pTsdb->pVnode->config.tsdbPageSize, TD_FILE_READ, &pDelFReader->pReadH);
if (code) goto _err;
- // encode
+ *ppReader = pDelFReader;
+ return code;
+
+_err:
+ tsdbError("vgId:%d, del file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+ *ppReader = NULL;
+ return code;
+}
+
+int32_t tsdbDelFReaderClose(SDelFReader **ppReader) {
+ int32_t code = 0;
+ SDelFReader *pReader = *ppReader;
+
+ if (pReader) {
+ tsdbCloseFile(&pReader->pReadH);
+ for (int32_t iBuf = 0; iBuf < sizeof(pReader->aBuf) / sizeof(uint8_t *); iBuf++) {
+ tFree(pReader->aBuf[iBuf]);
+ }
+ taosMemoryFree(pReader);
+ }
+ *ppReader = NULL;
+
+_exit:
+ return code;
+}
+
+int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData) {
+ int32_t code = 0;
+ int64_t offset = pDelIdx->offset;
+ int64_t size = pDelIdx->size;
+ int64_t n;
+
+ taosArrayClear(aDelData);
+
+ // alloc
+ code = tRealloc(&pReader->aBuf[0], size);
+ if (code) goto _err;
+
+ // read
+ code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size);
+ if (code) goto _err;
+
+ // // decode
n = 0;
- n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT);
- for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) {
- n += tPutBlockL(pWriter->aBuf[0] + n, taosArrayGet(aBlockL, iBlockL));
- }
- taosCalcChecksumAppend(0, pWriter->aBuf[0], size);
+ while (n < size) {
+ SDelData delData;
+ n += tGetDelData(pReader->aBuf[0] + n, &delData);
- ASSERT(n + sizeof(TSCKSUM) == size);
-
- // write
- n = taosWriteFile(pWriter->pLastFD, pWriter->aBuf[0], size);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- // update
- pLastFile->offset = pLastFile->size;
- pLastFile->size += size;
-
-_exit:
- tsdbTrace("vgId:%d tsdb write blockl, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode),
- pLastFile->offset, size);
- return code;
-
-_err:
- tsdbError("vgId:%d tsdb write blockl failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
- return code;
-}
-
-static void tsdbUpdateBlockInfo(SBlockData *pBlockData, SBlock *pBlock) {
- for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) {
- TSDBKEY key = {.ts = pBlockData->aTSKEY[iRow], .version = pBlockData->aVersion[iRow]};
-
- if (iRow == 0) {
- if (tsdbKeyCmprFn(&pBlock->minKey, &key) > 0) {
- pBlock->minKey = key;
- }
- } else {
- if (pBlockData->aTSKEY[iRow] == pBlockData->aTSKEY[iRow - 1]) {
- pBlock->hasDup = 1;
- }
- }
-
- if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&pBlock->maxKey, &key) < 0) {
- pBlock->maxKey = key;
- }
-
- pBlock->minVer = TMIN(pBlock->minVer, key.version);
- pBlock->maxVer = TMAX(pBlock->maxVer, key.version);
- }
- pBlock->nRow += pBlockData->nRow;
-}
-
-static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, SSmaInfo *pSmaInfo) {
- int32_t code = 0;
-
- pSmaInfo->offset = 0;
- pSmaInfo->size = 0;
-
- // encode
- for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) {
- SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
-
- if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type)) continue;
-
- SColumnDataAgg sma;
- tsdbCalcColDataSMA(pColData, &sma);
-
- code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &sma));
- if (code) goto _err;
- pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &sma);
- }
-
- // write
- if (pSmaInfo->size) {
- int32_t size = pSmaInfo->size + sizeof(TSCKSUM);
-
- code = tRealloc(&pWriter->aBuf[0], size);
- if (code) goto _err;
-
- taosCalcChecksumAppend(0, pWriter->aBuf[0], size);
-
- int64_t n = taosWriteFile(pWriter->pSmaFD, pWriter->aBuf[0], size);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
+ if (taosArrayPush(aDelData, &delData) == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
-
- pSmaInfo->offset = pWriter->fSma.size;
- pWriter->fSma.size += size;
}
+ ASSERT(n == size);
return code;
_err:
- tsdbError("vgId:%d tsdb write block sma failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+ tsdbError("vgId:%d, read del data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
return code;
}
-int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo,
- int8_t cmprAlg, int8_t toLast) {
+int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) {
int32_t code = 0;
+ int32_t n;
+ int64_t offset = pReader->fDel.offset;
+ int64_t size = pReader->fDel.size - offset;
- ASSERT(pBlockData->nRow > 0);
+ taosArrayClear(aDelIdx);
- pBlkInfo->offset = toLast ? pWriter->fLast.size : pWriter->fData.size;
- pBlkInfo->szBlock = 0;
- pBlkInfo->szKey = 0;
-
- int32_t aBufN[4] = {0};
- code = tCmprBlockData(pBlockData, cmprAlg, NULL, NULL, pWriter->aBuf, aBufN);
+ // alloc
+ code = tRealloc(&pReader->aBuf[0], size);
if (code) goto _err;
- // write =================
- TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD;
+ // read
+ code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size);
+ if (code) goto _err;
- pBlkInfo->szKey = aBufN[3] + aBufN[2];
- pBlkInfo->szBlock = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3];
+ // decode
+ n = 0;
+ while (n < size) {
+ SDelIdx delIdx;
- int64_t n = taosWriteFile(pFD, pWriter->aBuf[3], aBufN[3]);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
+ n += tGetDelIdx(pReader->aBuf[0] + n, &delIdx);
- n = taosWriteFile(pFD, pWriter->aBuf[2], aBufN[2]);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- if (aBufN[1]) {
- n = taosWriteFile(pFD, pWriter->aBuf[1], aBufN[1]);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
+ if (taosArrayPush(aDelIdx, &delIdx) == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
}
- if (aBufN[0]) {
- n = taosWriteFile(pFD, pWriter->aBuf[0], aBufN[0]);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
- }
-
- // update info
- if (toLast) {
- pWriter->fLast.size += pBlkInfo->szBlock;
- } else {
- pWriter->fData.size += pBlkInfo->szBlock;
- }
-
- // ================= SMA ====================
- if (pSmaInfo) {
- code = tsdbWriteBlockSma(pWriter, pBlockData, pSmaInfo);
- if (code) goto _err;
- }
-
-_exit:
- tsdbTrace("vgId:%d tsdb write block data, suid:%" PRId64 " uid:%" PRId64 " nRow:%d, offset:%" PRId64 " size:%d",
- TD_VID(pWriter->pTsdb->pVnode), pBlockData->suid, pBlockData->uid, pBlockData->nRow, pBlkInfo->offset,
- pBlkInfo->szBlock);
- return code;
-
-_err:
- tsdbError("vgId:%d tsdb write block data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
- return code;
-}
-
-int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
- int32_t code = 0;
- int64_t n;
- int64_t size;
- TdFilePtr pOutFD = NULL; // TODO
- TdFilePtr PInFD = NULL; // TODO
- char fNameFrom[TSDB_FILENAME_LEN];
- char fNameTo[TSDB_FILENAME_LEN];
-
- // head
- tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom);
- tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo);
-
- pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
- if (pOutFD == NULL) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- PInFD = taosOpenFile(fNameFrom, TD_FILE_READ);
- if (PInFD == NULL) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pHeadF->size);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
- taosCloseFile(&pOutFD);
- taosCloseFile(&PInFD);
-
- // data
- tsdbDataFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pDataF, fNameFrom);
- tsdbDataFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pDataF, fNameTo);
-
- pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
- if (pOutFD == NULL) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- PInFD = taosOpenFile(fNameFrom, TD_FILE_READ);
- if (PInFD == NULL) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pDataF->size);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
- taosCloseFile(&pOutFD);
- taosCloseFile(&PInFD);
-
- // last
- tsdbLastFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pLastF, fNameFrom);
- tsdbLastFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pLastF, fNameTo);
-
- pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
- if (pOutFD == NULL) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- PInFD = taosOpenFile(fNameFrom, TD_FILE_READ);
- if (PInFD == NULL) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pLastF->size);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
- taosCloseFile(&pOutFD);
- taosCloseFile(&PInFD);
-
- // sma
- tsdbSmaFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pSmaF, fNameFrom);
- tsdbSmaFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pSmaF, fNameTo);
-
- pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
- if (pOutFD == NULL) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- PInFD = taosOpenFile(fNameFrom, TD_FILE_READ);
- if (PInFD == NULL) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
-
- n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pSmaF->size);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _err;
- }
- taosCloseFile(&pOutFD);
- taosCloseFile(&PInFD);
+ ASSERT(n == size);
return code;
_err:
- tsdbError("vgId:%d, tsdb DFileSet copy failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+ tsdbError("vgId:%d, read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
return code;
-}
+}
\ No newline at end of file
diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c
index a30b9154ab..d99bf2aa5c 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRetention.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c
@@ -60,7 +60,7 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) {
if (expLevel < 0) {
taosMemoryFree(pSet->pHeadF);
taosMemoryFree(pSet->pDataF);
- taosMemoryFree(pSet->pLastF);
+ taosMemoryFree(pSet->aSttF[0]);
taosMemoryFree(pSet->pSmaF);
taosArrayRemove(fs.aDFileSet, iSet);
iSet--;
@@ -82,8 +82,6 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) {
code = tsdbFSUpsertFSet(&fs, &fSet);
if (code) goto _err;
}
-
- /* code */
}
// do change fs
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index ab2b2b617a..99e88a442c 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -16,6 +16,29 @@
#include "tsdb.h"
// STsdbSnapReader ========================================
+typedef enum { SNAP_DATA_FILE_ITER = 0, SNAP_STT_FILE_ITER } EFIterT;
+typedef struct {
+ SRBTreeNode n;
+ SRowInfo rInfo;
+ EFIterT type;
+ union {
+ struct {
+ SArray* aBlockIdx;
+ int32_t iBlockIdx;
+ SBlockIdx* pBlockIdx;
+ SMapData mBlock;
+ int32_t iBlock;
+ }; // .data file
+ struct {
+ int32_t iStt;
+ SArray* aSttBlk;
+ int32_t iSttBlk;
+ }; // .stt file
+ };
+ SBlockData bData;
+ int32_t iRow;
+} SFDataIter;
+
struct STsdbSnapReader {
STsdb* pTsdb;
int64_t sver;
@@ -26,146 +49,301 @@ struct STsdbSnapReader {
int8_t dataDone;
int32_t fid;
SDataFReader* pDataFReader;
- SArray* aBlockIdx; // SArray
- SArray* aBlockL; // SArray
- SBlockIdx* pBlockIdx;
- SBlockL* pBlockL;
-
- int32_t iBlockIdx;
- int32_t iBlockL;
- SMapData mBlock; // SMapData
- int32_t iBlock;
- SBlockData oBlockData;
- SBlockData nBlockData;
+ SFDataIter* pIter;
+ SRBTree rbt;
+ SFDataIter aFDataIter[TSDB_MAX_STT_TRIGGER + 1];
+ SBlockData bData;
+ SSkmInfo skmTable;
// for del file
int8_t delDone;
SDelFReader* pDelFReader;
SArray* aDelIdx; // SArray
int32_t iDelIdx;
SArray* aDelData; // SArray
+ uint8_t* aBuf[5];
};
+extern int32_t tRowInfoCmprFn(const void* p1, const void* p2);
+extern int32_t tsdbReadDataBlockEx(SDataFReader* pReader, SDataBlk* pDataBlk, SBlockData* pBlockData);
+extern int32_t tsdbUpdateTableSchema(SMeta* pMeta, int64_t suid, int64_t uid, SSkmInfo* pSkmInfo);
+
+static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) {
+ int32_t code = 0;
+
+ SDFileSet dFileSet = {.fid = pReader->fid};
+ SDFileSet* pSet = taosArraySearch(pReader->fs.aDFileSet, &dFileSet, tDFileSetCmprFn, TD_GT);
+ if (pSet == NULL) return code;
+
+ pReader->fid = pSet->fid;
+ code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, pSet);
+ if (code) goto _err;
+
+ pReader->pIter = NULL;
+ tRBTreeCreate(&pReader->rbt, tRowInfoCmprFn);
+
+ // .data file
+ SFDataIter* pIter = &pReader->aFDataIter[0];
+ pIter->type = SNAP_DATA_FILE_ITER;
+
+ code = tsdbReadBlockIdx(pReader->pDataFReader, pIter->aBlockIdx);
+ if (code) goto _err;
+
+ for (pIter->iBlockIdx = 0; pIter->iBlockIdx < taosArrayGetSize(pIter->aBlockIdx); pIter->iBlockIdx++) {
+ pIter->pBlockIdx = (SBlockIdx*)taosArrayGet(pIter->aBlockIdx, pIter->iBlockIdx);
+
+ code = tsdbReadDataBlk(pReader->pDataFReader, pIter->pBlockIdx, &pIter->mBlock);
+ if (code) goto _err;
+
+ for (pIter->iBlock = 0; pIter->iBlock < pIter->mBlock.nItem; pIter->iBlock++) {
+ SDataBlk dataBlk;
+ tMapDataGetItemByIdx(&pIter->mBlock, pIter->iBlock, &dataBlk, tGetDataBlk);
+
+ if (dataBlk.minVer > pReader->ever || dataBlk.maxVer < pReader->sver) continue;
+
+ code = tsdbReadDataBlockEx(pReader->pDataFReader, &dataBlk, &pIter->bData);
+ if (code) goto _err;
+
+ ASSERT(pIter->pBlockIdx->suid == pIter->bData.suid);
+ ASSERT(pIter->pBlockIdx->uid == pIter->bData.uid);
+
+ for (pIter->iRow = 0; pIter->iRow < pIter->bData.nRow; pIter->iRow++) {
+ int64_t rowVer = pIter->bData.aVersion[pIter->iRow];
+
+ if (rowVer >= pReader->sver && rowVer <= pReader->ever) {
+ pIter->rInfo.suid = pIter->pBlockIdx->suid;
+ pIter->rInfo.uid = pIter->pBlockIdx->uid;
+ pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow);
+ goto _add_iter_and_break;
+ }
+ }
+ }
+
+ continue;
+
+ _add_iter_and_break:
+ tRBTreePut(&pReader->rbt, (SRBTreeNode*)pIter);
+ break;
+ }
+
+ // .stt file
+ pIter = &pReader->aFDataIter[1];
+ for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+ pIter->type = SNAP_STT_FILE_ITER;
+ pIter->iStt = iStt;
+
+ code = tsdbReadSttBlk(pReader->pDataFReader, iStt, pIter->aSttBlk);
+ if (code) goto _err;
+
+ for (pIter->iSttBlk = 0; pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk); pIter->iSttBlk++) {
+ SSttBlk* pSttBlk = (SSttBlk*)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk);
+
+ if (pSttBlk->minVer > pReader->ever) continue;
+ if (pSttBlk->maxVer < pReader->sver) continue;
+
+ code = tsdbReadSttBlock(pReader->pDataFReader, iStt, pSttBlk, &pIter->bData);
+ if (code) goto _err;
+
+ for (pIter->iRow = 0; pIter->iRow < pIter->bData.nRow; pIter->iRow++) {
+ int64_t rowVer = pIter->bData.aVersion[pIter->iRow];
+
+ if (rowVer >= pReader->sver && rowVer <= pReader->ever) {
+ pIter->rInfo.suid = pIter->bData.suid;
+ pIter->rInfo.uid = pIter->bData.uid;
+ pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow);
+ goto _add_iter;
+ }
+ }
+ }
+
+ continue;
+
+ _add_iter:
+ tRBTreePut(&pReader->rbt, (SRBTreeNode*)pIter);
+ pIter++;
+ }
+
+ tsdbInfo("vgId:%d, vnode snapshot tsdb open data file to read for %s, fid:%d", TD_VID(pReader->pTsdb->pVnode),
+ pReader->pTsdb->path, pReader->fid);
+ return code;
+
+_err:
+ tsdbError("vgId:%d vnode snapshot tsdb snap read open file failed since %s", TD_VID(pReader->pTsdb->pVnode),
+ tstrerror(code));
+ return code;
+}
+
+static SRowInfo* tsdbSnapGetRow(STsdbSnapReader* pReader) { return pReader->pIter ? &pReader->pIter->rInfo : NULL; }
+
+static int32_t tsdbSnapNextRow(STsdbSnapReader* pReader) {
+ int32_t code = 0;
+
+ if (pReader->pIter) {
+ SFDataIter* pIter = pReader->pIter;
+
+ while (true) {
+ _find_row:
+ for (pIter->iRow++; pIter->iRow < pIter->bData.nRow; pIter->iRow++) {
+ int64_t rowVer = pIter->bData.aVersion[pIter->iRow];
+
+ if (rowVer >= pReader->sver && rowVer <= pReader->ever) {
+ pIter->rInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow];
+ pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow);
+ goto _out;
+ }
+ }
+
+ if (pIter->type == SNAP_DATA_FILE_ITER) {
+ while (true) {
+ for (pIter->iBlock++; pIter->iBlock < pIter->mBlock.nItem; pIter->iBlock++) {
+ SDataBlk dataBlk;
+ tMapDataGetItemByIdx(&pIter->mBlock, pIter->iBlock, &dataBlk, tGetDataBlk);
+
+ if (dataBlk.minVer > pReader->ever || dataBlk.maxVer < pReader->sver) continue;
+
+ code = tsdbReadDataBlockEx(pReader->pDataFReader, &dataBlk, &pIter->bData);
+ if (code) goto _err;
+
+ pIter->iRow = -1;
+ goto _find_row;
+ }
+
+ pIter->iBlockIdx++;
+ if (pIter->iBlockIdx >= taosArrayGetSize(pIter->aBlockIdx)) break;
+
+ pIter->pBlockIdx = (SBlockIdx*)taosArrayGet(pIter->aBlockIdx, pIter->iBlockIdx);
+ code = tsdbReadDataBlk(pReader->pDataFReader, pIter->pBlockIdx, &pIter->mBlock);
+ if (code) goto _err;
+ pIter->iBlock = -1;
+ }
+
+ pReader->pIter = NULL;
+ } else if (pIter->type == SNAP_STT_FILE_ITER) {
+ for (pIter->iSttBlk++; pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk); pIter->iSttBlk++) {
+ SSttBlk* pSttBlk = (SSttBlk*)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk);
+
+ if (pSttBlk->minVer > pReader->ever || pSttBlk->maxVer < pReader->sver) continue;
+
+ code = tsdbReadSttBlock(pReader->pDataFReader, pIter->iStt, pSttBlk, &pIter->bData);
+ if (code) goto _err;
+
+ pIter->iRow = -1;
+ goto _find_row;
+ }
+
+ pReader->pIter = NULL;
+ } else {
+ ASSERT(0);
+ }
+ }
+
+ _out:
+ pIter = (SFDataIter*)tRBTreeMin(&pReader->rbt);
+ if (pReader->pIter && pIter) {
+ int32_t c = tRowInfoCmprFn(&pReader->pIter->rInfo, &pIter->rInfo);
+ if (c > 0) {
+ tRBTreePut(&pReader->rbt, (SRBTreeNode*)pReader->pIter);
+ pReader->pIter = NULL;
+ } else {
+ ASSERT(c);
+ }
+ }
+ }
+
+ if (pReader->pIter == NULL) {
+ pReader->pIter = (SFDataIter*)tRBTreeMin(&pReader->rbt);
+ if (pReader->pIter) {
+ tRBTreeDrop(&pReader->rbt, (SRBTreeNode*)pReader->pIter);
+ }
+ }
+
+ return code;
+
+_err:
+ return code;
+}
+
+static int32_t tsdbSnapCmprData(STsdbSnapReader* pReader, uint8_t** ppData) {
+ int32_t code = 0;
+
+ ASSERT(pReader->bData.nRow);
+
+ int32_t aBufN[5] = {0};
+ code = tCmprBlockData(&pReader->bData, TWO_STAGE_COMP, NULL, NULL, pReader->aBuf, aBufN);
+ if (code) goto _exit;
+
+ int32_t size = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3];
+ *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size);
+ if (*ppData == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _exit;
+ }
+
+ SSnapDataHdr* pHdr = (SSnapDataHdr*)*ppData;
+ pHdr->type = SNAP_DATA_TSDB;
+ pHdr->size = size;
+
+ memcpy(pHdr->data, pReader->aBuf[3], aBufN[3]);
+ memcpy(pHdr->data + aBufN[3], pReader->aBuf[2], aBufN[2]);
+ if (aBufN[1]) {
+ memcpy(pHdr->data + aBufN[3] + aBufN[2], pReader->aBuf[1], aBufN[1]);
+ }
+ if (aBufN[0]) {
+ memcpy(pHdr->data + aBufN[3] + aBufN[2] + aBufN[1], pReader->aBuf[0], aBufN[0]);
+ }
+
+_exit:
+ return code;
+}
+
static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
int32_t code = 0;
STsdb* pTsdb = pReader->pTsdb;
while (true) {
if (pReader->pDataFReader == NULL) {
- // next
- SDFileSet dFileSet = {.fid = pReader->fid};
- SDFileSet* pSet = taosArraySearch(pReader->fs.aDFileSet, &dFileSet, tDFileSetCmprFn, TD_GT);
- if (pSet == NULL) goto _exit;
- pReader->fid = pSet->fid;
-
- // load
- code = tsdbDataFReaderOpen(&pReader->pDataFReader, pTsdb, pSet);
+ code = tsdbSnapReadOpenFile(pReader);
if (code) goto _err;
-
- code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx);
- if (code) goto _err;
-
- code = tsdbReadBlockL(pReader->pDataFReader, pReader->aBlockL);
- if (code) goto _err;
-
- // init
- pReader->iBlockIdx = 0;
- if (pReader->iBlockIdx < taosArrayGetSize(pReader->aBlockIdx)) {
- pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx);
-
- code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock);
- if (code) goto _err;
-
- pReader->iBlock = 0;
- } else {
- pReader->pBlockIdx = NULL;
- }
-
- pReader->iBlockL = 0;
- while (true) {
- if (pReader->iBlockL >= taosArrayGetSize(pReader->aBlockL)) {
- pReader->pBlockL = NULL;
- break;
- }
-
- pReader->pBlockL = (SBlockL*)taosArrayGet(pReader->aBlockL, pReader->iBlockL);
- if (pReader->pBlockL->minVer <= pReader->ever && pReader->pBlockL->maxVer >= pReader->sver) {
- // TODO
- break;
- }
-
- pReader->iBlockL++;
- }
-
- tsdbInfo("vgId:%d, vnode snapshot tsdb open data file to read for %s, fid:%d", TD_VID(pTsdb->pVnode), pTsdb->path,
- pReader->fid);
}
- while (true) {
- if (pReader->pBlockIdx && pReader->pBlockL) {
- TABLEID id = {.suid = pReader->pBlockL->suid, .uid = pReader->pBlockL->minUid};
+ if (pReader->pDataFReader == NULL) break;
- ASSERT(0);
+ SRowInfo* pRowInfo = tsdbSnapGetRow(pReader);
+ if (pRowInfo == NULL) {
+ tsdbDataFReaderClose(&pReader->pDataFReader);
+ continue;
+ }
- // if (tTABLEIDCmprFn(pReader->pBlockIdx, &minId) < 0) {
- // // TODO
- // } else if (tTABLEIDCmprFn(pReader->pBlockIdx, &maxId) < 0) {
- // // TODO
- // } else {
- // // TODO
- // }
- } else if (pReader->pBlockIdx) {
- while (pReader->iBlock < pReader->mBlock.nItem) {
- SBlock block;
- tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, &block, tGetBlock);
+ TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid};
+ SBlockData* pBlockData = &pReader->bData;
- if (block.minVer <= pReader->ever && block.maxVer >= pReader->sver) {
- // load data (todo)
- }
+ code = tsdbUpdateTableSchema(pTsdb->pVnode->pMeta, id.suid, id.uid, &pReader->skmTable);
+ if (code) goto _err;
- // next
- pReader->iBlock++;
- if (*ppData) break;
- }
+ code = tBlockDataInit(pBlockData, id.suid, id.uid, pReader->skmTable.pTSchema);
+ if (code) goto _err;
- if (pReader->iBlock >= pReader->mBlock.nItem) {
- pReader->iBlockIdx++;
- if (pReader->iBlockIdx < taosArrayGetSize(pReader->aBlockIdx)) {
- pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx);
+ while (pRowInfo->suid == id.suid && pRowInfo->uid == id.uid) {
+ code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL, pRowInfo->uid);
+ if (code) goto _err;
- code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock);
- if (code) goto _err;
+ code = tsdbSnapNextRow(pReader);
+ if (code) goto _err;
- pReader->iBlock = 0;
- } else {
- pReader->pBlockIdx = NULL;
- }
- }
-
- if (*ppData) goto _exit;
- } else if (pReader->pBlockL) {
- while (pReader->pBlockL) {
- if (pReader->pBlockL->minVer <= pReader->ever && pReader->pBlockL->maxVer >= pReader->sver) {
- // load data (todo)
- }
-
- // next
- pReader->iBlockL++;
- if (pReader->iBlockL < taosArrayGetSize(pReader->aBlockL)) {
- pReader->pBlockL = (SBlockL*)taosArrayGetSize(pReader->aBlockL);
- } else {
- pReader->pBlockL = NULL;
- }
-
- if (*ppData) goto _exit;
- }
- } else {
+ pRowInfo = tsdbSnapGetRow(pReader);
+ if (pRowInfo == NULL) {
tsdbDataFReaderClose(&pReader->pDataFReader);
break;
}
+
+ if (pBlockData->nRow >= 4096) break;
}
+
+ code = tsdbSnapCmprData(pReader, ppData);
+ if (code) goto _err;
+
+ break;
}
-_exit:
return code;
_err:
@@ -216,7 +394,6 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
size += tPutDelData(NULL, pDelData);
}
}
-
if (size == 0) continue;
// org data
@@ -292,23 +469,33 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type
goto _err;
}
+ // data
pReader->fid = INT32_MIN;
- pReader->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
- if (pReader->aBlockIdx == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ for (int32_t iIter = 0; iIter < sizeof(pReader->aFDataIter) / sizeof(pReader->aFDataIter[0]); iIter++) {
+ SFDataIter* pIter = &pReader->aFDataIter[iIter];
+
+ if (iIter == 0) {
+ pIter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
+ if (pIter->aBlockIdx == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
+ } else {
+ pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
+ if (pIter->aSttBlk == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
+ }
+
+ code = tBlockDataCreate(&pIter->bData);
+ if (code) goto _err;
}
- pReader->aBlockL = taosArrayInit(0, sizeof(SBlockL));
- if (pReader->aBlockL == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
- pReader->mBlock = tMapDataInit();
- code = tBlockDataCreate(&pReader->oBlockData);
- if (code) goto _err;
- code = tBlockDataCreate(&pReader->nBlockData);
+
+ code = tBlockDataCreate(&pReader->bData);
if (code) goto _err;
+ // del
pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx));
if (pReader->aDelIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
@@ -335,18 +522,26 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
int32_t code = 0;
STsdbSnapReader* pReader = *ppReader;
- if (pReader->pDataFReader) {
- tsdbDataFReaderClose(&pReader->pDataFReader);
- }
- taosArrayDestroy(pReader->aBlockL);
- taosArrayDestroy(pReader->aBlockIdx);
- tMapDataClear(&pReader->mBlock);
- tBlockDataDestroy(&pReader->oBlockData, 1);
- tBlockDataDestroy(&pReader->nBlockData, 1);
+ // data
+ if (pReader->pDataFReader) tsdbDataFReaderClose(&pReader->pDataFReader);
+ for (int32_t iIter = 0; iIter < sizeof(pReader->aFDataIter) / sizeof(pReader->aFDataIter[0]); iIter++) {
+ SFDataIter* pIter = &pReader->aFDataIter[iIter];
- if (pReader->pDelFReader) {
- tsdbDelFReaderClose(&pReader->pDelFReader);
+ if (iIter == 0) {
+ taosArrayDestroy(pIter->aBlockIdx);
+ tMapDataClear(&pIter->mBlock);
+ } else {
+ taosArrayDestroy(pIter->aSttBlk);
+ }
+
+ tBlockDataDestroy(&pIter->bData, 1);
}
+
+ tBlockDataDestroy(&pReader->bData, 1);
+ tTSchemaDestroy(pReader->skmTable.pTSchema);
+
+ // del
+ if (pReader->pDelFReader) tsdbDelFReaderClose(&pReader->pDelFReader);
taosArrayDestroy(pReader->aDelIdx);
taosArrayDestroy(pReader->aDelData);
@@ -354,6 +549,10 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
tsdbInfo("vgId:%d, vnode snapshot tsdb reader closed for %s", TD_VID(pReader->pTsdb->pVnode), pReader->pTsdb->path);
+ for (int32_t iBuf = 0; iBuf < sizeof(pReader->aBuf) / sizeof(pReader->aBuf[0]); iBuf++) {
+ tFree(pReader->aBuf[iBuf]);
+ }
+
taosMemoryFree(pReader);
*ppReader = NULL;
return code;
@@ -410,40 +609,37 @@ struct STsdbSnapWriter {
STsdbFS fs;
// config
- int32_t minutes;
- int8_t precision;
- int32_t minRow;
- int32_t maxRow;
- int8_t cmprAlg;
- int64_t commitID;
-
+ int32_t minutes;
+ int8_t precision;
+ int32_t minRow;
+ int32_t maxRow;
+ int8_t cmprAlg;
+ int64_t commitID;
uint8_t* aBuf[5];
+
// for data file
SBlockData bData;
-
- int32_t fid;
- SDataFReader* pDataFReader;
- SArray* aBlockIdx; // SArray
- int32_t iBlockIdx;
- SBlockIdx* pBlockIdx;
- SMapData mBlock; // SMapData
- int32_t iBlock;
- SBlockData* pBlockData;
- int32_t iRow;
- SBlockData bDataR;
- SArray* aBlockL; // SArray
- int32_t iBlockL;
- SBlockData lDataR;
-
- SDataFWriter* pDataFWriter;
- SBlockIdx* pBlockIdxW; // NULL when no committing table
- SBlock blockW;
- SBlockData bDataW;
- SBlockIdx blockIdxW;
-
- SMapData mBlockW; // SMapData
- SArray* aBlockIdxW; // SArray
- SArray* aBlockLW; // SArray
+ int32_t fid;
+ TABLEID id;
+ SSkmInfo skmTable;
+ struct {
+ SDataFReader* pReader;
+ SArray* aBlockIdx;
+ int32_t iBlockIdx;
+ SBlockIdx* pBlockIdx;
+ SMapData mDataBlk;
+ int32_t iDataBlk;
+ SBlockData bData;
+ int32_t iRow;
+ } dReader;
+ struct {
+ SDataFWriter* pWriter;
+ SArray* aBlockIdx;
+ SMapData mDataBlk;
+ SArray* aSttBlk;
+ SBlockData bData;
+ SBlockData sData;
+ } dWriter;
// for del file
SDelFReader* pDelFReader;
@@ -454,518 +650,447 @@ struct STsdbSnapWriter {
SArray* aDelIdxW;
};
+// SNAP_DATA_TSDB
+extern int32_t tsdbWriteDataBlock(SDataFWriter* pWriter, SBlockData* pBlockData, SMapData* mDataBlk, int8_t cmprAlg);
+extern int32_t tsdbWriteSttBlock(SDataFWriter* pWriter, SBlockData* pBlockData, SArray* aSttBlk, int8_t cmprAlg);
+
+static int32_t tsdbSnapNextTableData(STsdbSnapWriter* pWriter) {
+ int32_t code = 0;
+
+ ASSERT(pWriter->dReader.iRow >= pWriter->dReader.bData.nRow);
+
+ if (pWriter->dReader.iBlockIdx < taosArrayGetSize(pWriter->dReader.aBlockIdx)) {
+ pWriter->dReader.pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->dReader.aBlockIdx, pWriter->dReader.iBlockIdx);
+
+ code = tsdbReadDataBlk(pWriter->dReader.pReader, pWriter->dReader.pBlockIdx, &pWriter->dReader.mDataBlk);
+ if (code) goto _exit;
+
+ pWriter->dReader.iBlockIdx++;
+ } else {
+ pWriter->dReader.pBlockIdx = NULL;
+ tMapDataReset(&pWriter->dReader.mDataBlk);
+ }
+ pWriter->dReader.iDataBlk = 0; // point to the next one
+ tBlockDataReset(&pWriter->dReader.bData);
+ pWriter->dReader.iRow = 0;
+
+_exit:
+ return code;
+}
+
+static int32_t tsdbSnapWriteCopyData(STsdbSnapWriter* pWriter, TABLEID* pId) {
+ int32_t code = 0;
+
+ while (true) {
+ if (pWriter->dReader.pBlockIdx == NULL) break;
+ if (tTABLEIDCmprFn(pWriter->dReader.pBlockIdx, pId) >= 0) break;
+
+ SBlockIdx blkIdx = *pWriter->dReader.pBlockIdx;
+ code = tsdbWriteDataBlk(pWriter->dWriter.pWriter, &pWriter->dReader.mDataBlk, &blkIdx);
+ if (code) goto _exit;
+
+ if (taosArrayPush(pWriter->dWriter.aBlockIdx, &blkIdx) == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _exit;
+ }
+
+ code = tsdbSnapNextTableData(pWriter);
+ if (code) goto _exit;
+ }
+
+_exit:
+ return code;
+}
+
+static int32_t tsdbSnapWriteTableDataStart(STsdbSnapWriter* pWriter, TABLEID* pId) {
+ int32_t code = 0;
+
+ code = tsdbSnapWriteCopyData(pWriter, pId);
+ if (code) goto _err;
+
+ pWriter->id.suid = pId->suid;
+ pWriter->id.uid = pId->uid;
+
+ code = tsdbUpdateTableSchema(pWriter->pTsdb->pVnode->pMeta, pId->suid, pId->uid, &pWriter->skmTable);
+ if (code) goto _err;
+
+ tMapDataReset(&pWriter->dWriter.mDataBlk);
+ code = tBlockDataInit(&pWriter->dWriter.bData, pId->suid, pId->uid, pWriter->skmTable.pTSchema);
+ if (code) goto _err;
+
+ return code;
+
+_err:
+ tsdbError("vgId:%d %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, tstrerror(code));
+ return code;
+}
+
static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) {
int32_t code = 0;
- ASSERT(pWriter->pDataFWriter);
+ if (pWriter->id.suid == 0 && pWriter->id.uid == 0) return code;
- if (pWriter->pBlockIdxW == NULL) goto _exit;
+ int32_t c = 1;
+ if (pWriter->dReader.pBlockIdx) {
+ c = tTABLEIDCmprFn(pWriter->dReader.pBlockIdx, &pWriter->id);
+ ASSERT(c >= 0);
+ }
- // consume remain rows
- if (pWriter->pBlockData) {
- ASSERT(pWriter->iRow < pWriter->pBlockData->nRow);
- while (pWriter->iRow < pWriter->pBlockData->nRow) {
- code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL,
- 0); // todo
+ if (c == 0) {
+ SBlockData* pBData = &pWriter->dWriter.bData;
+
+ for (; pWriter->dReader.iRow < pWriter->dReader.bData.nRow; pWriter->dReader.iRow++) {
+ TSDBROW row = tsdbRowFromBlockData(&pWriter->dReader.bData, pWriter->dReader.iRow);
+
+ code = tBlockDataAppendRow(pBData, &row, NULL, pWriter->id.uid);
if (code) goto _err;
- if (pWriter->bDataW.nRow >= pWriter->maxRow * 4 / 5) {
- // pWriter->blockW.last = 0;
- // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
- // &pWriter->blockW, pWriter->cmprAlg);
+ if (pBData->nRow >= pWriter->maxRow) {
+ code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, pBData, &pWriter->dWriter.mDataBlk, pWriter->cmprAlg);
if (code) goto _err;
-
- code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock);
- if (code) goto _err;
-
- tBlockReset(&pWriter->blockW);
- tBlockDataClear(&pWriter->bDataW);
- }
-
- pWriter->iRow++;
- }
- }
-
- // write remain data if has
- if (pWriter->bDataW.nRow > 0) {
- // pWriter->blockW.last = 0;
- if (pWriter->bDataW.nRow < pWriter->minRow) {
- if (pWriter->iBlock > pWriter->mBlock.nItem) {
- // pWriter->blockW.last = 1;
}
}
- // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
- // &pWriter->blockW, pWriter->cmprAlg);
- // if (code) goto _err;
-
- code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock);
- if (code) goto _err;
- }
-
- while (true) {
- if (pWriter->iBlock >= pWriter->mBlock.nItem) break;
-
- SBlock block;
- tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock);
-
- // if (block.last) {
- // code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, &pWriter->bDataR, NULL, NULL);
- // if (code) goto _err;
-
- // tBlockReset(&block);
- // block.last = 1;
- // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pWriter->pBlockIdxW, &block,
- // pWriter->cmprAlg);
- // if (code) goto _err;
- // }
-
- code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock);
+ code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, pBData, &pWriter->dWriter.mDataBlk, pWriter->cmprAlg);
if (code) goto _err;
- pWriter->iBlock++;
- }
+ for (; pWriter->dReader.iDataBlk < pWriter->dReader.mDataBlk.nItem; pWriter->dReader.iDataBlk++) {
+ SDataBlk dataBlk;
+ tMapDataGetItemByIdx(&pWriter->dReader.mDataBlk, pWriter->dReader.iDataBlk, &dataBlk, tGetDataBlk);
- // SBlock
- // code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, pWriter->pBlockIdxW);
- // if (code) goto _err;
-
- // SBlockIdx
- if (taosArrayPush(pWriter->aBlockIdxW, pWriter->pBlockIdxW) == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
-
-_exit:
- tsdbInfo("vgId:%d, tsdb snapshot write table data end for %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path);
- return code;
-
-_err:
- tsdbError("vgId:%d, tsdb snapshot write table data end for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode),
- pWriter->pTsdb->path, tstrerror(code));
- return code;
-}
-
-static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* pBlockIdx) {
- int32_t code = 0;
-
- code = tsdbReadBlock(pWriter->pDataFReader, pBlockIdx, &pWriter->mBlock);
- if (code) goto _err;
-
- // SBlockData
- SBlock block;
- tMapDataReset(&pWriter->mBlockW);
- for (int32_t iBlock = 0; iBlock < pWriter->mBlock.nItem; iBlock++) {
- tMapDataGetItemByIdx(&pWriter->mBlock, iBlock, &block, tGetBlock);
-
- // if (block.last) {
- // code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL);
- // if (code) goto _err;
-
- // tBlockReset(&block);
- // block.last = 1;
- // code =
- // tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pBlockIdx, &block,
- // pWriter->cmprAlg);
- // if (code) goto _err;
- // }
-
- code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock);
- if (code) goto _err;
- }
-
- // SBlock
- SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid};
- code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, &blockIdx);
- if (code) goto _err;
-
- // SBlockIdx
- if (taosArrayPush(pWriter->aBlockIdxW, &blockIdx) == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
-
-_exit:
- return code;
-
-_err:
- tsdbError("vgId:%d, tsdb snapshot move write table data for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode),
- pWriter->pTsdb->path, tstrerror(code));
- return code;
-}
-
-static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) {
- int32_t code = 0;
- SBlockData* pBlockData = &pWriter->bData;
- int32_t iRow = 0;
- TSDBROW row;
- TSDBROW* pRow = &row;
-
- // // correct schema
- // code = tBlockDataCorrectSchema(&pWriter->bDataW, pBlockData);
- // if (code) goto _err;
-
- // loop to merge
- *pRow = tsdbRowFromBlockData(pBlockData, iRow);
- while (true) {
- if (pRow == NULL) break;
-
- if (pWriter->pBlockData) {
- ASSERT(pWriter->iRow < pWriter->pBlockData->nRow);
-
- int32_t c = tsdbRowCmprFn(pRow, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow));
-
- ASSERT(c);
-
- if (c < 0) {
- // code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL);
- // if (code) goto _err;
-
- iRow++;
- if (iRow < pWriter->pBlockData->nRow) {
- *pRow = tsdbRowFromBlockData(pBlockData, iRow);
- } else {
- pRow = NULL;
- }
- } else if (c > 0) {
- // code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow),
- // NULL); if (code) goto _err;
-
- pWriter->iRow++;
- if (pWriter->iRow >= pWriter->pBlockData->nRow) {
- pWriter->pBlockData = NULL;
- }
- }
- } else {
- TSDBKEY key = TSDBROW_KEY(pRow);
-
- while (true) {
- if (pWriter->iBlock >= pWriter->mBlock.nItem) break;
-
- SBlock block;
- int32_t c;
-
- tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock);
-
- // if (block.last) {
- // pWriter->pBlockData = &pWriter->bDataR;
-
- // code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL,
- // NULL); if (code) goto _err; pWriter->iRow = 0;
-
- // pWriter->iBlock++;
- // break;
- // }
-
- c = tsdbKeyCmprFn(&block.maxKey, &key);
-
- ASSERT(c);
-
- if (c < 0) {
- if (pWriter->bDataW.nRow) {
- // pWriter->blockW.last = 0;
- // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
- // &pWriter->blockW, pWriter->cmprAlg);
- // if (code) goto _err;
-
- code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock);
- if (code) goto _err;
-
- tBlockReset(&pWriter->blockW);
- tBlockDataClear(&pWriter->bDataW);
- }
-
- code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock);
- if (code) goto _err;
-
- pWriter->iBlock++;
- } else {
- c = tsdbKeyCmprFn(&tBlockDataLastKey(pBlockData), &block.minKey);
-
- ASSERT(c);
-
- if (c > 0) {
- pWriter->pBlockData = &pWriter->bDataR;
- // code =
- // tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL,
- // NULL);
- // if (code) goto _err;
- pWriter->iRow = 0;
-
- pWriter->iBlock++;
- }
- break;
- }
- }
-
- if (pWriter->pBlockData) continue;
-
- // code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL);
- // if (code) goto _err;
-
- iRow++;
- if (iRow < pBlockData->nRow) {
- *pRow = tsdbRowFromBlockData(pBlockData, iRow);
- } else {
- pRow = NULL;
- }
- }
-
- _check_write:
- if (pWriter->bDataW.nRow < pWriter->maxRow * 4 / 5) continue;
-
- _write_block:
- // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
- // &pWriter->blockW, pWriter->cmprAlg);
- // if (code) goto _err;
-
- code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock);
- if (code) goto _err;
-
- tBlockReset(&pWriter->blockW);
- tBlockDataClear(&pWriter->bDataW);
- }
-
- return code;
-
-_err:
- tsdbError("vgId:%d, vnode snapshot tsdb write table data impl for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode),
- pWriter->pTsdb->path, tstrerror(code));
- return code;
-}
-
-static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, TABLEID id) {
- int32_t code = 0;
- SBlockData* pBlockData = &pWriter->bData;
- TSDBKEY keyFirst = tBlockDataFirstKey(pBlockData);
- TSDBKEY keyLast = tBlockDataLastKey(pBlockData);
-
- // end last table write if should
- if (pWriter->pBlockIdxW) {
- int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdxW, &id);
- if (c < 0) {
- // end
- code = tsdbSnapWriteTableDataEnd(pWriter);
+ code = tMapDataPutItem(&pWriter->dWriter.mDataBlk, &dataBlk, tPutDataBlk);
if (code) goto _err;
+ }
- // reset
- pWriter->pBlockIdxW = NULL;
- } else if (c > 0) {
- ASSERT(0);
+ code = tsdbSnapNextTableData(pWriter);
+ if (code) goto _err;
+ }
+
+ if (pWriter->dWriter.mDataBlk.nItem) {
+ SBlockIdx blockIdx = {.suid = pWriter->id.suid, .uid = pWriter->id.uid};
+ code = tsdbWriteDataBlk(pWriter->dWriter.pWriter, &pWriter->dWriter.mDataBlk, &blockIdx);
+
+ if (taosArrayPush(pWriter->dWriter.aBlockIdx, &blockIdx) == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
}
}
- // start new table data write if need
- if (pWriter->pBlockIdxW == NULL) {
- // write table data ahead
- while (true) {
- if (pWriter->iBlockIdx >= taosArrayGetSize(pWriter->aBlockIdx)) break;
+ pWriter->id.suid = 0;
+ pWriter->id.uid = 0;
- SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx);
- int32_t c = tTABLEIDCmprFn(pBlockIdx, &id);
-
- if (c >= 0) break;
-
- code = tsdbSnapMoveWriteTableData(pWriter, pBlockIdx);
- if (code) goto _err;
-
- pWriter->iBlockIdx++;
- }
-
- // reader
- pWriter->pBlockIdx = NULL;
- if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) {
- ASSERT(pWriter->pDataFReader);
-
- SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx);
- int32_t c = tTABLEIDCmprFn(pBlockIdx, &id);
-
- ASSERT(c >= 0);
-
- if (c == 0) {
- pWriter->pBlockIdx = pBlockIdx;
- pWriter->iBlockIdx++;
- }
- }
-
- if (pWriter->pBlockIdx) {
- code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlock);
- if (code) goto _err;
- } else {
- tMapDataReset(&pWriter->mBlock);
- }
- pWriter->iBlock = 0;
- pWriter->pBlockData = NULL;
- pWriter->iRow = 0;
-
- // writer
- pWriter->pBlockIdxW = &pWriter->blockIdxW;
- pWriter->pBlockIdxW->suid = id.suid;
- pWriter->pBlockIdxW->uid = id.uid;
-
- tBlockReset(&pWriter->blockW);
- tBlockDataReset(&pWriter->bDataW);
- tMapDataReset(&pWriter->mBlockW);
- }
-
- ASSERT(pWriter->pBlockIdxW && pWriter->pBlockIdxW->suid == id.suid && pWriter->pBlockIdxW->uid == id.uid);
- ASSERT(pWriter->pBlockIdx == NULL || (pWriter->pBlockIdx->suid == id.suid && pWriter->pBlockIdx->uid == id.uid));
-
- code = tsdbSnapWriteTableDataImpl(pWriter);
- if (code) goto _err;
-
-_exit:
- tsdbDebug("vgId:%d, vnode snapshot tsdb write data impl for %s", TD_VID(pWriter->pTsdb->pVnode),
- pWriter->pTsdb->path);
return code;
_err:
- tsdbError("vgId:%d, vnode snapshot tsdb write data impl for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode),
- pWriter->pTsdb->path, tstrerror(code));
return code;
}
-static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) {
+static int32_t tsdbSnapWriteOpenFile(STsdbSnapWriter* pWriter, int32_t fid) {
int32_t code = 0;
STsdb* pTsdb = pWriter->pTsdb;
- if (pWriter->pDataFWriter == NULL) goto _exit;
+ ASSERT(pWriter->dWriter.pWriter == NULL);
+
+ pWriter->fid = fid;
+ pWriter->id = (TABLEID){0};
+ SDFileSet* pSet = taosArraySearch(pWriter->fs.aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, TD_EQ);
+
+ // Reader
+ if (pSet) {
+ code = tsdbDataFReaderOpen(&pWriter->dReader.pReader, pWriter->pTsdb, pSet);
+ if (code) goto _err;
+
+ code = tsdbReadBlockIdx(pWriter->dReader.pReader, pWriter->dReader.aBlockIdx);
+ if (code) goto _err;
+ } else {
+ ASSERT(pWriter->dReader.pReader == NULL);
+ taosArrayClear(pWriter->dReader.aBlockIdx);
+ }
+ pWriter->dReader.iBlockIdx = 0; // point to the next one
+ code = tsdbSnapNextTableData(pWriter);
+ if (code) goto _err;
+
+ // Writer
+ SHeadFile fHead = {.commitID = pWriter->commitID};
+ SDataFile fData = {.commitID = pWriter->commitID};
+ SSmaFile fSma = {.commitID = pWriter->commitID};
+ SSttFile fStt = {.commitID = pWriter->commitID};
+ SDFileSet wSet = {.fid = pWriter->fid, .pHeadF = &fHead, .pDataF = &fData, .pSmaF = &fSma};
+ if (pSet) {
+ wSet.diskId = pSet->diskId;
+ fData = *pSet->pDataF;
+ fSma = *pSet->pSmaF;
+ for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+ wSet.aSttF[iStt] = pSet->aSttF[iStt];
+ }
+ wSet.nSttF = pSet->nSttF + 1; // TODO: fix pSet->nSttF == pTsdb->maxFile
+ } else {
+ SDiskID did = {0};
+ tfsAllocDisk(pTsdb->pVnode->pTfs, 0, &did);
+ tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did);
+ wSet.diskId = did;
+ wSet.nSttF = 1;
+ }
+ wSet.aSttF[wSet.nSttF - 1] = &fStt;
+
+ code = tsdbDataFWriterOpen(&pWriter->dWriter.pWriter, pWriter->pTsdb, &wSet);
+ if (code) goto _err;
+ taosArrayClear(pWriter->dWriter.aBlockIdx);
+ tMapDataReset(&pWriter->dWriter.mDataBlk);
+ taosArrayClear(pWriter->dWriter.aSttBlk);
+ tBlockDataReset(&pWriter->dWriter.bData);
+ tBlockDataReset(&pWriter->dWriter.sData);
+
+ return code;
+
+_err:
+ return code;
+}
+
+static int32_t tsdbSnapWriteCloseFile(STsdbSnapWriter* pWriter) {
+ int32_t code = 0;
+
+ ASSERT(pWriter->dWriter.pWriter);
- // finish current table
code = tsdbSnapWriteTableDataEnd(pWriter);
if (code) goto _err;
- // move remain table
- while (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) {
- code = tsdbSnapMoveWriteTableData(pWriter, (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx));
- if (code) goto _err;
-
- pWriter->iBlockIdx++;
- }
-
- // write remain stuff
- if (taosArrayGetSize(pWriter->aBlockLW) > 0) {
- code = tsdbWriteBlockL(pWriter->pDataFWriter, pWriter->aBlockIdxW);
- if (code) goto _err;
- }
-
- if (taosArrayGetSize(pWriter->aBlockIdx) > 0) {
- code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW);
- if (code) goto _err;
- }
-
- code = tsdbFSUpsertFSet(&pWriter->fs, &pWriter->pDataFWriter->wSet);
+ // copy remain table data
+ TABLEID id = {.suid = INT64_MAX, .uid = INT64_MAX};
+ code = tsdbSnapWriteCopyData(pWriter, &id);
if (code) goto _err;
- code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 1);
+ code =
+ tsdbWriteSttBlock(pWriter->dWriter.pWriter, &pWriter->dWriter.sData, pWriter->dWriter.aSttBlk, pWriter->cmprAlg);
if (code) goto _err;
- if (pWriter->pDataFReader) {
- code = tsdbDataFReaderClose(&pWriter->pDataFReader);
+ // Indices
+ code = tsdbWriteBlockIdx(pWriter->dWriter.pWriter, pWriter->dWriter.aBlockIdx);
+ if (code) goto _err;
+
+ code = tsdbWriteSttBlk(pWriter->dWriter.pWriter, pWriter->dWriter.aSttBlk);
+ if (code) goto _err;
+
+ code = tsdbUpdateDFileSetHeader(pWriter->dWriter.pWriter);
+ if (code) goto _err;
+
+ code = tsdbFSUpsertFSet(&pWriter->fs, &pWriter->dWriter.pWriter->wSet);
+ if (code) goto _err;
+
+ code = tsdbDataFWriterClose(&pWriter->dWriter.pWriter, 1);
+ if (code) goto _err;
+
+ if (pWriter->dReader.pReader) {
+ code = tsdbDataFReaderClose(&pWriter->dReader.pReader);
if (code) goto _err;
}
_exit:
- tsdbInfo("vgId:%d, vnode snapshot tsdb writer data end for %s", TD_VID(pTsdb->pVnode), pTsdb->path);
return code;
_err:
- tsdbError("vgId:%d, vnode snapshot tsdb writer data end for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path,
- tstrerror(code));
+ return code;
+}
+
+static int32_t tsdbSnapWriteToDataFile(STsdbSnapWriter* pWriter, int32_t iRow, int8_t* done) {
+ int32_t code = 0;
+
+ SBlockData* pBData = &pWriter->bData;
+ TABLEID id = {.suid = pBData->suid, .uid = pBData->uid ? pBData->uid : pBData->aUid[iRow]};
+ TSDBROW row = tsdbRowFromBlockData(pBData, iRow);
+ TSDBKEY key = TSDBROW_KEY(&row);
+
+ *done = 0;
+ while (pWriter->dReader.iRow < pWriter->dReader.bData.nRow ||
+ pWriter->dReader.iDataBlk < pWriter->dReader.mDataBlk.nItem) {
+ // Merge row by row
+ for (; pWriter->dReader.iRow < pWriter->dReader.bData.nRow; pWriter->dReader.iRow++) {
+ TSDBROW trow = tsdbRowFromBlockData(&pWriter->dReader.bData, pWriter->dReader.iRow);
+ TSDBKEY tKey = TSDBROW_KEY(&trow);
+
+ ASSERT(pWriter->dReader.bData.suid == id.suid && pWriter->dReader.bData.uid == id.uid);
+
+ int32_t c = tsdbKeyCmprFn(&key, &tKey);
+ if (c < 0) {
+ code = tBlockDataAppendRow(&pWriter->dWriter.bData, &row, NULL, id.uid);
+ if (code) goto _err;
+ } else if (c > 0) {
+ code = tBlockDataAppendRow(&pWriter->dWriter.bData, &trow, NULL, id.uid);
+ if (code) goto _err;
+ } else {
+ ASSERT(0);
+ }
+
+ if (pWriter->dWriter.bData.nRow >= pWriter->maxRow) {
+ code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, &pWriter->dWriter.bData, &pWriter->dWriter.mDataBlk,
+ pWriter->cmprAlg);
+ if (code) goto _err;
+ }
+
+ if (c < 0) {
+ *done = 1;
+ goto _exit;
+ }
+ }
+
+ // Merge row by block
+ SDataBlk tDataBlk = {.minKey = key, .maxKey = key};
+ for (; pWriter->dReader.iDataBlk < pWriter->dReader.mDataBlk.nItem; pWriter->dReader.iDataBlk++) {
+ SDataBlk dataBlk;
+ tMapDataGetItemByIdx(&pWriter->dReader.mDataBlk, pWriter->dReader.iDataBlk, &dataBlk, tGetDataBlk);
+
+ int32_t c = tDataBlkCmprFn(&dataBlk, &tDataBlk);
+ if (c < 0) {
+ code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, &pWriter->dWriter.bData, &pWriter->dWriter.mDataBlk,
+ pWriter->cmprAlg);
+ if (code) goto _err;
+
+ code = tMapDataPutItem(&pWriter->dWriter.mDataBlk, &dataBlk, tPutDataBlk);
+ if (code) goto _err;
+ } else if (c > 0) {
+ code = tBlockDataAppendRow(&pWriter->dWriter.bData, &row, NULL, id.uid);
+ if (code) goto _err;
+
+ if (pWriter->dWriter.bData.nRow >= pWriter->maxRow) {
+ code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, &pWriter->dWriter.bData, &pWriter->dWriter.mDataBlk,
+ pWriter->cmprAlg);
+ if (code) goto _err;
+ }
+
+ *done = 1;
+ goto _exit;
+ } else {
+ code = tsdbReadDataBlockEx(pWriter->dReader.pReader, &dataBlk, &pWriter->dReader.bData);
+ if (code) goto _err;
+ pWriter->dReader.iRow = 0;
+
+ pWriter->dReader.iDataBlk++;
+ break;
+ }
+ }
+ }
+
+_exit:
+ return code;
+
+_err:
+ tsdbError("vgId:%d %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, tstrerror(code));
+ return code;
+}
+
+static int32_t tsdbSnapWriteToSttFile(STsdbSnapWriter* pWriter, int32_t iRow) {
+ int32_t code = 0;
+
+ TABLEID id = {.suid = pWriter->bData.suid,
+ .uid = pWriter->bData.uid ? pWriter->bData.uid : pWriter->bData.aUid[iRow]};
+ TSDBROW row = tsdbRowFromBlockData(&pWriter->bData, iRow);
+ SBlockData* pBData = &pWriter->dWriter.sData;
+
+ if (pBData->suid || pBData->uid) {
+ if (!TABLE_SAME_SCHEMA(pBData->suid, pBData->uid, id.suid, id.uid)) {
+ code = tsdbWriteSttBlock(pWriter->dWriter.pWriter, pBData, pWriter->dWriter.aSttBlk, pWriter->cmprAlg);
+ if (code) goto _err;
+
+ pBData->suid = 0;
+ pBData->uid = 0;
+ }
+ }
+
+ if (pBData->suid == 0 && pBData->uid == 0) {
+ code = tsdbUpdateTableSchema(pWriter->pTsdb->pVnode->pMeta, pWriter->id.suid, pWriter->id.uid, &pWriter->skmTable);
+ if (code) goto _err;
+
+ code = tBlockDataInit(pBData, pWriter->id.suid, pWriter->id.suid ? 0 : pWriter->id.uid, pWriter->skmTable.pTSchema);
+ if (code) goto _err;
+ }
+
+ code = tBlockDataAppendRow(pBData, &row, NULL, id.uid);
+ if (code) goto _err;
+
+ if (pBData->nRow >= pWriter->maxRow) {
+ code = tsdbWriteSttBlock(pWriter->dWriter.pWriter, pBData, pWriter->dWriter.aSttBlk, pWriter->cmprAlg);
+ if (code) goto _err;
+ }
+
+_exit:
+ return code;
+
+_err:
+ return code;
+}
+
+static int32_t tsdbSnapWriteRowData(STsdbSnapWriter* pWriter, int32_t iRow) {
+ int32_t code = 0;
+
+ SBlockData* pBlockData = &pWriter->bData;
+ TABLEID id = {.suid = pBlockData->suid, .uid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[iRow]};
+
+ // End last table data write if need
+ if (tTABLEIDCmprFn(&pWriter->id, &id) != 0) {
+ code = tsdbSnapWriteTableDataEnd(pWriter);
+ if (code) goto _err;
+ }
+
+ // Start new table data write if need
+ if (pWriter->id.suid == 0 && pWriter->id.uid == 0) {
+ code = tsdbSnapWriteTableDataStart(pWriter, &id);
+ if (code) goto _err;
+ }
+
+ // Merge with .data file data
+ int8_t done = 0;
+ if (pWriter->dReader.pBlockIdx && tTABLEIDCmprFn(pWriter->dReader.pBlockIdx, &id) == 0) {
+ code = tsdbSnapWriteToDataFile(pWriter, iRow, &done);
+ if (code) goto _err;
+ }
+
+ // Append to the .stt data block (todo: check if need to set/reload sst block)
+ if (!done) {
+ code = tsdbSnapWriteToSttFile(pWriter, iRow);
+ if (code) goto _err;
+ }
+
+_exit:
+ return code;
+
+_err:
+ tsdbError("vgId:%d %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, tstrerror(code));
return code;
}
static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
- int32_t code = 0;
- STsdb* pTsdb = pWriter->pTsdb;
- SSnapDataHdr* pHdr = (SSnapDataHdr*)pData;
- TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr));
- int64_t n;
-
- // decode
+ int32_t code = 0;
+ STsdb* pTsdb = pWriter->pTsdb;
SBlockData* pBlockData = &pWriter->bData;
- code = tDecmprBlockData(pData + sizeof(SSnapDataHdr) + sizeof(TABLEID), pHdr->size - sizeof(TABLEID), pBlockData,
- pWriter->aBuf);
+
+ // Decode data
+ SSnapDataHdr* pHdr = (SSnapDataHdr*)pData;
+ code = tDecmprBlockData(pHdr->data, pHdr->size, pBlockData, pWriter->aBuf);
if (code) goto _err;
- // open file
- TSDBKEY keyFirst = {.version = pBlockData->aVersion[0], .ts = pBlockData->aTSKEY[0]};
- TSDBKEY keyLast = {.version = pBlockData->aVersion[pBlockData->nRow - 1],
- .ts = pBlockData->aTSKEY[pBlockData->nRow - 1]};
+ ASSERT(pBlockData->nRow > 0);
- int32_t fid = tsdbKeyFid(keyFirst.ts, pWriter->minutes, pWriter->precision);
- ASSERT(fid == tsdbKeyFid(keyLast.ts, pWriter->minutes, pWriter->precision));
- if (pWriter->pDataFWriter == NULL || pWriter->fid != fid) {
- // end last file data write if need
- code = tsdbSnapWriteDataEnd(pWriter);
- if (code) goto _err;
+ // Loop to handle each row
+ for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) {
+ TSKEY ts = pBlockData->aTSKEY[iRow];
+ int32_t fid = tsdbKeyFid(ts, pWriter->minutes, pWriter->precision);
- pWriter->fid = fid;
+ if (pWriter->dWriter.pWriter == NULL || pWriter->fid != fid) {
+ if (pWriter->dWriter.pWriter) {
+ ASSERT(fid > pWriter->fid);
- // read
- SDFileSet* pSet = taosArraySearch(pWriter->fs.aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, TD_EQ);
- if (pSet) {
- code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet);
+ code = tsdbSnapWriteCloseFile(pWriter);
+ if (code) goto _err;
+ }
+
+ code = tsdbSnapWriteOpenFile(pWriter, fid);
if (code) goto _err;
-
- code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx);
- if (code) goto _err;
-
- code = tsdbReadBlockL(pWriter->pDataFReader, pWriter->aBlockL);
- if (code) goto _err;
- } else {
- ASSERT(pWriter->pDataFReader == NULL);
- taosArrayClear(pWriter->aBlockIdx);
- taosArrayClear(pWriter->aBlockL);
- }
- pWriter->iBlockIdx = 0;
- pWriter->pBlockIdx = NULL;
- tMapDataReset(&pWriter->mBlock);
- pWriter->iBlock = 0;
- pWriter->pBlockData = NULL;
- pWriter->iRow = 0;
- pWriter->iBlockL = 0;
- tBlockDataReset(&pWriter->bDataR);
- tBlockDataReset(&pWriter->lDataR);
-
- // write
- SHeadFile fHead;
- SDataFile fData;
- SLastFile fLast;
- SSmaFile fSma;
- SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .pLastF = &fLast, .pSmaF = &fSma};
-
- if (pSet) {
- wSet.diskId = pSet->diskId;
- wSet.fid = fid;
- fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0};
- fData = *pSet->pDataF;
- fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0};
- fSma = *pSet->pSmaF;
- } else {
- wSet.diskId = (SDiskID){.level = 0, .id = 0};
- wSet.fid = fid;
- fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0};
- fData = (SDataFile){.commitID = pWriter->commitID, .size = 0};
- fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0, .offset = 0};
- fSma = (SSmaFile){.commitID = pWriter->commitID, .size = 0};
}
- code = tsdbDataFWriterOpen(&pWriter->pDataFWriter, pTsdb, &wSet);
+ code = tsdbSnapWriteRowData(pWriter, iRow);
if (code) goto _err;
-
- taosArrayClear(pWriter->aBlockIdxW);
- taosArrayClear(pWriter->aBlockLW);
- tMapDataReset(&pWriter->mBlockW);
- pWriter->pBlockIdxW = NULL;
- tBlockDataReset(&pWriter->bDataW);
}
- code = tsdbSnapWriteTableData(pWriter, id);
- if (code) goto _err;
-
- tsdbInfo("vgId:%d, vnode snapshot tsdb write data for %s, fid:%d suid:%" PRId64 " uid:%" PRId64 " nRow:%d",
- TD_VID(pTsdb->pVnode), pTsdb->path, fid, id.suid, id.suid, pBlockData->nRow);
return code;
_err:
@@ -974,10 +1099,41 @@ _err:
return code;
}
+// SNAP_DATA_DEL
+static int32_t tsdbSnapMoveWriteDelData(STsdbSnapWriter* pWriter, TABLEID* pId) {
+ int32_t code = 0;
+
+ while (true) {
+ if (pWriter->iDelIdx >= taosArrayGetSize(pWriter->aDelIdxR)) break;
+
+ SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx);
+
+ if (tTABLEIDCmprFn(pDelIdx, pId) >= 0) break;
+
+ code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData);
+ if (code) goto _exit;
+
+ SDelIdx delIdx = *pDelIdx;
+ code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx);
+ if (code) goto _exit;
+
+ if (taosArrayPush(pWriter->aDelIdxW, &delIdx) == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _exit;
+ }
+
+ pWriter->iDelIdx++;
+ }
+
+_exit:
+ return code;
+}
+
static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
int32_t code = 0;
STsdb* pTsdb = pWriter->pTsdb;
+ // Open del file if not opened yet
if (pWriter->pDelFWriter == NULL) {
SDelFile* pDelFile = pWriter->fs.pDelFile;
@@ -988,38 +1144,28 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32
code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdxR);
if (code) goto _err;
+ } else {
+ taosArrayClear(pWriter->aDelIdxR);
}
+ pWriter->iDelIdx = 0;
// writer
- SDelFile delFile = {.commitID = pWriter->commitID, .offset = 0, .size = 0};
+ SDelFile delFile = {.commitID = pWriter->commitID};
code = tsdbDelFWriterOpen(&pWriter->pDelFWriter, &delFile, pTsdb);
if (code) goto _err;
+ taosArrayClear(pWriter->aDelIdxW);
}
- // process the del data
- TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr));
+ SSnapDataHdr* pHdr = (SSnapDataHdr*)pData;
+ TABLEID id = *(TABLEID*)pHdr->data;
- while (true) {
- if (pWriter->iDelIdx >= taosArrayGetSize(pWriter->aDelIdxR)) break;
- if (tTABLEIDCmprFn(taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx), &id) >= 0) break;
+ ASSERT(pHdr->size + sizeof(SSnapDataHdr) == nData);
- SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx);
-
- code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData);
- if (code) goto _err;
-
- SDelIdx delIdx = *pDelIdx;
- code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx);
- if (code) goto _err;
-
- if (taosArrayPush(pWriter->aDelIdxW, &delIdx) == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
-
- pWriter->iDelIdx++;
- }
+ // Move write data < id
+ code = tsdbSnapMoveWriteDelData(pWriter, &id);
+ if (code) goto _err;
+ // Merge incoming data with current
if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR) &&
tTABLEIDCmprFn(taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx), &id) == 0) {
SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx);
@@ -1053,7 +1199,6 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32
goto _err;
}
-_exit:
return code;
_err:
@@ -1066,23 +1211,14 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) {
int32_t code = 0;
STsdb* pTsdb = pWriter->pTsdb;
- if (pWriter->pDelFWriter == NULL) goto _exit;
+ if (pWriter->pDelFWriter == NULL) return code;
- for (; pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR); pWriter->iDelIdx++) {
- SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx);
+ TABLEID id = {.suid = INT64_MAX, .uid = INT64_MAX};
+ code = tsdbSnapMoveWriteDelData(pWriter, &id);
+ if (code) goto _err;
- code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData);
- if (code) goto _err;
-
- SDelIdx delIdx = *pDelIdx;
- code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx);
- if (code) goto _err;
-
- if (taosArrayPush(pWriter->aDelIdxR, &delIdx) == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
- }
+ code = tsdbWriteDelIdx(pWriter->pDelFWriter, pWriter->aDelIdxW);
+ if (code) goto _err;
code = tsdbUpdateDelFileHdr(pWriter->pDelFWriter);
if (code) goto _err;
@@ -1098,7 +1234,6 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) {
if (code) goto _err;
}
-_exit:
tsdbInfo("vgId:%d, vnode snapshot tsdb write del for %s end", TD_VID(pTsdb->pVnode), pTsdb->path);
return code;
@@ -1108,6 +1243,7 @@ _err:
return code;
}
+// APIs
int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) {
int32_t code = 0;
STsdbSnapWriter* pWriter = NULL;
@@ -1133,39 +1269,38 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
pWriter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression;
pWriter->commitID = pTsdb->pVnode->state.commitID;
- // for data file
+ // SNAP_DATA_TSDB
code = tBlockDataCreate(&pWriter->bData);
-
- if (code) goto _err;
- pWriter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
- if (pWriter->aBlockIdx == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
- code = tBlockDataCreate(&pWriter->bDataR);
if (code) goto _err;
- pWriter->aBlockL = taosArrayInit(0, sizeof(SBlockL));
- if (pWriter->aBlockL == NULL) {
+ pWriter->fid = INT32_MIN;
+ pWriter->id = (TABLEID){0};
+ // Reader
+ pWriter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
+ if (pWriter->dReader.aBlockIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
-
- pWriter->aBlockIdxW = taosArrayInit(0, sizeof(SBlockIdx));
- if (pWriter->aBlockIdxW == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
- code = tBlockDataCreate(&pWriter->bDataW);
+ code = tBlockDataCreate(&pWriter->dReader.bData);
if (code) goto _err;
- pWriter->aBlockLW = taosArrayInit(0, sizeof(SBlockL));
- if (pWriter->aBlockLW == NULL) {
+ // Writer
+ pWriter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
+ if (pWriter->dWriter.aBlockIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
+ pWriter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
+ if (pWriter->dWriter.aSttBlk == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
+ code = tBlockDataCreate(&pWriter->dWriter.bData);
+ if (code) goto _err;
+ code = tBlockDataCreate(&pWriter->dWriter.sData);
+ if (code) goto _err;
- // for del file
+ // SNAP_DATA_DEL
pWriter->aDelIdxR = taosArrayInit(0, sizeof(SDelIdx));
if (pWriter->aDelIdxR == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
@@ -1186,6 +1321,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
tsdbInfo("vgId:%d, tsdb snapshot writer open for %s succeed", TD_VID(pTsdb->pVnode), pTsdb->path);
return code;
+
_err:
tsdbError("vgId:%d, tsdb snapshot writer open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path,
tstrerror(code));
@@ -1196,14 +1332,17 @@ _err:
int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
int32_t code = 0;
STsdbSnapWriter* pWriter = *ppWriter;
+ STsdb* pTsdb = pWriter->pTsdb;
if (rollback) {
ASSERT(0);
// code = tsdbFSRollback(pWriter->pTsdb->pFS);
// if (code) goto _err;
} else {
- code = tsdbSnapWriteDataEnd(pWriter);
- if (code) goto _err;
+ if (pWriter->dWriter.pWriter) {
+ code = tsdbSnapWriteCloseFile(pWriter);
+ if (code) goto _err;
+ }
code = tsdbSnapWriteDelEnd(pWriter);
if (code) goto _err;
@@ -1211,14 +1350,44 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
code = tsdbFSCommit1(pWriter->pTsdb, &pWriter->fs);
if (code) goto _err;
+ // lock
+ taosThreadRwlockWrlock(&pTsdb->rwLock);
+
code = tsdbFSCommit2(pWriter->pTsdb, &pWriter->fs);
- if (code) goto _err;
+ if (code) {
+ taosThreadRwlockUnlock(&pTsdb->rwLock);
+ goto _err;
+ }
+
+ // unlock
+ taosThreadRwlockUnlock(&pTsdb->rwLock);
}
+ // SNAP_DATA_DEL
+ taosArrayDestroy(pWriter->aDelIdxW);
+ taosArrayDestroy(pWriter->aDelData);
+ taosArrayDestroy(pWriter->aDelIdxR);
+
+ // SNAP_DATA_TSDB
+
+ // Writer
+ tBlockDataDestroy(&pWriter->dWriter.sData, 1);
+ tBlockDataDestroy(&pWriter->dWriter.bData, 1);
+ taosArrayDestroy(pWriter->dWriter.aSttBlk);
+ tMapDataClear(&pWriter->dWriter.mDataBlk);
+ taosArrayDestroy(pWriter->dWriter.aBlockIdx);
+
+ // Reader
+ tBlockDataDestroy(&pWriter->dReader.bData, 1);
+ tMapDataClear(&pWriter->dReader.mDataBlk);
+ taosArrayDestroy(pWriter->dReader.aBlockIdx);
+
+ tBlockDataDestroy(&pWriter->bData, 1);
+ tTSchemaDestroy(pWriter->skmTable.pTSchema);
+
for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) {
tFree(pWriter->aBuf[iBuf]);
}
-
tsdbInfo("vgId:%d, vnode snapshot tsdb writer close for %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path);
taosMemoryFree(pWriter);
*ppWriter = NULL;
@@ -1243,8 +1412,8 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData)
goto _exit;
} else {
- if (pWriter->pDataFWriter) {
- code = tsdbSnapWriteDataEnd(pWriter);
+ if (pWriter->dWriter.pWriter) {
+ code = tsdbSnapWriteCloseFile(pWriter);
if (code) goto _err;
}
}
@@ -1257,7 +1426,6 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData)
_exit:
tsdbDebug("vgId:%d, tsdb snapshot write for %s succeed", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path);
-
return code;
_err:
diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index cfb04881e6..caeca45e01 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -51,6 +51,22 @@ _exit:
return code;
}
+int32_t tMapDataCopy(SMapData *pFrom, SMapData *pTo) {
+ int32_t code = 0;
+
+ pTo->nItem = pFrom->nItem;
+ pTo->nData = pFrom->nData;
+ code = tRealloc((uint8_t **)&pTo->aOffset, sizeof(int32_t) * pFrom->nItem);
+ if (code) goto _exit;
+ code = tRealloc(&pTo->pData, pFrom->nData);
+ if (code) goto _exit;
+ memcpy(pTo->aOffset, pFrom->aOffset, sizeof(int32_t) * pFrom->nItem);
+ memcpy(pTo->pData, pFrom->pData, pFrom->nData);
+
+_exit:
+ return code;
+}
+
int32_t tMapDataSearch(SMapData *pMapData, void *pSearchItem, int32_t (*tGetItemFn)(uint8_t *, void *),
int32_t (*tItemCmprFn)(const void *, const void *), void *pItem) {
int32_t code = 0;
@@ -198,7 +214,7 @@ int32_t tCmprBlockIdx(void const *lhs, void const *rhs) {
int32_t tCmprBlockL(void const *lhs, void const *rhs) {
SBlockIdx *lBlockIdx = (SBlockIdx *)lhs;
- SBlockL *rBlockL = (SBlockL *)rhs;
+ SSttBlk *rBlockL = (SSttBlk *)rhs;
if (lBlockIdx->suid < rBlockL->suid) {
return -1;
@@ -215,69 +231,69 @@ int32_t tCmprBlockL(void const *lhs, void const *rhs) {
return 0;
}
-// SBlock ======================================================
-void tBlockReset(SBlock *pBlock) {
- *pBlock = (SBlock){.minKey = TSDBKEY_MAX, .maxKey = TSDBKEY_MIN, .minVer = VERSION_MAX, .maxVer = VERSION_MIN};
+// SDataBlk ======================================================
+void tDataBlkReset(SDataBlk *pDataBlk) {
+ *pDataBlk = (SDataBlk){.minKey = TSDBKEY_MAX, .maxKey = TSDBKEY_MIN, .minVer = VERSION_MAX, .maxVer = VERSION_MIN};
}
-int32_t tPutBlock(uint8_t *p, void *ph) {
- int32_t n = 0;
- SBlock *pBlock = (SBlock *)ph;
+int32_t tPutDataBlk(uint8_t *p, void *ph) {
+ int32_t n = 0;
+ SDataBlk *pDataBlk = (SDataBlk *)ph;
- n += tPutI64v(p ? p + n : p, pBlock->minKey.version);
- n += tPutI64v(p ? p + n : p, pBlock->minKey.ts);
- n += tPutI64v(p ? p + n : p, pBlock->maxKey.version);
- n += tPutI64v(p ? p + n : p, pBlock->maxKey.ts);
- n += tPutI64v(p ? p + n : p, pBlock->minVer);
- n += tPutI64v(p ? p + n : p, pBlock->maxVer);
- n += tPutI32v(p ? p + n : p, pBlock->nRow);
- n += tPutI8(p ? p + n : p, pBlock->hasDup);
- n += tPutI8(p ? p + n : p, pBlock->nSubBlock);
- for (int8_t iSubBlock = 0; iSubBlock < pBlock->nSubBlock; iSubBlock++) {
- n += tPutI64v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].offset);
- n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szBlock);
- n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szKey);
+ n += tPutI64v(p ? p + n : p, pDataBlk->minKey.version);
+ n += tPutI64v(p ? p + n : p, pDataBlk->minKey.ts);
+ n += tPutI64v(p ? p + n : p, pDataBlk->maxKey.version);
+ n += tPutI64v(p ? p + n : p, pDataBlk->maxKey.ts);
+ n += tPutI64v(p ? p + n : p, pDataBlk->minVer);
+ n += tPutI64v(p ? p + n : p, pDataBlk->maxVer);
+ n += tPutI32v(p ? p + n : p, pDataBlk->nRow);
+ n += tPutI8(p ? p + n : p, pDataBlk->hasDup);
+ n += tPutI8(p ? p + n : p, pDataBlk->nSubBlock);
+ for (int8_t iSubBlock = 0; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) {
+ n += tPutI64v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].offset);
+ n += tPutI32v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].szBlock);
+ n += tPutI32v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].szKey);
}
- if (pBlock->nSubBlock == 1 && !pBlock->hasDup) {
- n += tPutI64v(p ? p + n : p, pBlock->smaInfo.offset);
- n += tPutI32v(p ? p + n : p, pBlock->smaInfo.size);
+ if (pDataBlk->nSubBlock == 1 && !pDataBlk->hasDup) {
+ n += tPutI64v(p ? p + n : p, pDataBlk->smaInfo.offset);
+ n += tPutI32v(p ? p + n : p, pDataBlk->smaInfo.size);
}
return n;
}
-int32_t tGetBlock(uint8_t *p, void *ph) {
- int32_t n = 0;
- SBlock *pBlock = (SBlock *)ph;
+int32_t tGetDataBlk(uint8_t *p, void *ph) {
+ int32_t n = 0;
+ SDataBlk *pDataBlk = (SDataBlk *)ph;
- n += tGetI64v(p + n, &pBlock->minKey.version);
- n += tGetI64v(p + n, &pBlock->minKey.ts);
- n += tGetI64v(p + n, &pBlock->maxKey.version);
- n += tGetI64v(p + n, &pBlock->maxKey.ts);
- n += tGetI64v(p + n, &pBlock->minVer);
- n += tGetI64v(p + n, &pBlock->maxVer);
- n += tGetI32v(p + n, &pBlock->nRow);
- n += tGetI8(p + n, &pBlock->hasDup);
- n += tGetI8(p + n, &pBlock->nSubBlock);
- for (int8_t iSubBlock = 0; iSubBlock < pBlock->nSubBlock; iSubBlock++) {
- n += tGetI64v(p + n, &pBlock->aSubBlock[iSubBlock].offset);
- n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szBlock);
- n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szKey);
+ n += tGetI64v(p + n, &pDataBlk->minKey.version);
+ n += tGetI64v(p + n, &pDataBlk->minKey.ts);
+ n += tGetI64v(p + n, &pDataBlk->maxKey.version);
+ n += tGetI64v(p + n, &pDataBlk->maxKey.ts);
+ n += tGetI64v(p + n, &pDataBlk->minVer);
+ n += tGetI64v(p + n, &pDataBlk->maxVer);
+ n += tGetI32v(p + n, &pDataBlk->nRow);
+ n += tGetI8(p + n, &pDataBlk->hasDup);
+ n += tGetI8(p + n, &pDataBlk->nSubBlock);
+ for (int8_t iSubBlock = 0; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) {
+ n += tGetI64v(p + n, &pDataBlk->aSubBlock[iSubBlock].offset);
+ n += tGetI32v(p + n, &pDataBlk->aSubBlock[iSubBlock].szBlock);
+ n += tGetI32v(p + n, &pDataBlk->aSubBlock[iSubBlock].szKey);
}
- if (pBlock->nSubBlock == 1 && !pBlock->hasDup) {
- n += tGetI64v(p + n, &pBlock->smaInfo.offset);
- n += tGetI32v(p + n, &pBlock->smaInfo.size);
+ if (pDataBlk->nSubBlock == 1 && !pDataBlk->hasDup) {
+ n += tGetI64v(p + n, &pDataBlk->smaInfo.offset);
+ n += tGetI32v(p + n, &pDataBlk->smaInfo.size);
} else {
- pBlock->smaInfo.offset = 0;
- pBlock->smaInfo.size = 0;
+ pDataBlk->smaInfo.offset = 0;
+ pDataBlk->smaInfo.size = 0;
}
return n;
}
-int32_t tBlockCmprFn(const void *p1, const void *p2) {
- SBlock *pBlock1 = (SBlock *)p1;
- SBlock *pBlock2 = (SBlock *)p2;
+int32_t tDataBlkCmprFn(const void *p1, const void *p2) {
+ SDataBlk *pBlock1 = (SDataBlk *)p1;
+ SDataBlk *pBlock2 = (SDataBlk *)p2;
if (tsdbKeyCmprFn(&pBlock1->maxKey, &pBlock2->minKey) < 0) {
return -1;
@@ -288,48 +304,48 @@ int32_t tBlockCmprFn(const void *p1, const void *p2) {
return 0;
}
-bool tBlockHasSma(SBlock *pBlock) {
- if (pBlock->nSubBlock > 1) return false;
- if (pBlock->hasDup) return false;
+bool tDataBlkHasSma(SDataBlk *pDataBlk) {
+ if (pDataBlk->nSubBlock > 1) return false;
+ if (pDataBlk->hasDup) return false;
- return pBlock->smaInfo.size > 0;
+ return pDataBlk->smaInfo.size > 0;
}
-// SBlockL ======================================================
-int32_t tPutBlockL(uint8_t *p, void *ph) {
+// SSttBlk ======================================================
+int32_t tPutSttBlk(uint8_t *p, void *ph) {
int32_t n = 0;
- SBlockL *pBlockL = (SBlockL *)ph;
+ SSttBlk *pSttBlk = (SSttBlk *)ph;
- n += tPutI64(p ? p + n : p, pBlockL->suid);
- n += tPutI64(p ? p + n : p, pBlockL->minUid);
- n += tPutI64(p ? p + n : p, pBlockL->maxUid);
- n += tPutI64v(p ? p + n : p, pBlockL->minKey);
- n += tPutI64v(p ? p + n : p, pBlockL->maxKey);
- n += tPutI64v(p ? p + n : p, pBlockL->minVer);
- n += tPutI64v(p ? p + n : p, pBlockL->maxVer);
- n += tPutI32v(p ? p + n : p, pBlockL->nRow);
- n += tPutI64v(p ? p + n : p, pBlockL->bInfo.offset);
- n += tPutI32v(p ? p + n : p, pBlockL->bInfo.szBlock);
- n += tPutI32v(p ? p + n : p, pBlockL->bInfo.szKey);
+ n += tPutI64(p ? p + n : p, pSttBlk->suid);
+ n += tPutI64(p ? p + n : p, pSttBlk->minUid);
+ n += tPutI64(p ? p + n : p, pSttBlk->maxUid);
+ n += tPutI64v(p ? p + n : p, pSttBlk->minKey);
+ n += tPutI64v(p ? p + n : p, pSttBlk->maxKey);
+ n += tPutI64v(p ? p + n : p, pSttBlk->minVer);
+ n += tPutI64v(p ? p + n : p, pSttBlk->maxVer);
+ n += tPutI32v(p ? p + n : p, pSttBlk->nRow);
+ n += tPutI64v(p ? p + n : p, pSttBlk->bInfo.offset);
+ n += tPutI32v(p ? p + n : p, pSttBlk->bInfo.szBlock);
+ n += tPutI32v(p ? p + n : p, pSttBlk->bInfo.szKey);
return n;
}
-int32_t tGetBlockL(uint8_t *p, void *ph) {
+int32_t tGetSttBlk(uint8_t *p, void *ph) {
int32_t n = 0;
- SBlockL *pBlockL = (SBlockL *)ph;
+ SSttBlk *pSttBlk = (SSttBlk *)ph;
- n += tGetI64(p + n, &pBlockL->suid);
- n += tGetI64(p + n, &pBlockL->minUid);
- n += tGetI64(p + n, &pBlockL->maxUid);
- n += tGetI64v(p + n, &pBlockL->minKey);
- n += tGetI64v(p + n, &pBlockL->maxKey);
- n += tGetI64v(p + n, &pBlockL->minVer);
- n += tGetI64v(p + n, &pBlockL->maxVer);
- n += tGetI32v(p + n, &pBlockL->nRow);
- n += tGetI64v(p + n, &pBlockL->bInfo.offset);
- n += tGetI32v(p + n, &pBlockL->bInfo.szBlock);
- n += tGetI32v(p + n, &pBlockL->bInfo.szKey);
+ n += tGetI64(p + n, &pSttBlk->suid);
+ n += tGetI64(p + n, &pSttBlk->minUid);
+ n += tGetI64(p + n, &pSttBlk->maxUid);
+ n += tGetI64v(p + n, &pSttBlk->minKey);
+ n += tGetI64v(p + n, &pSttBlk->maxKey);
+ n += tGetI64v(p + n, &pSttBlk->minVer);
+ n += tGetI64v(p + n, &pSttBlk->maxVer);
+ n += tGetI32v(p + n, &pSttBlk->nRow);
+ n += tGetI64v(p + n, &pSttBlk->bInfo.offset);
+ n += tGetI32v(p + n, &pSttBlk->bInfo.szBlock);
+ n += tGetI32v(p + n, &pSttBlk->bInfo.szKey);
return n;
}
@@ -1532,7 +1548,7 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut,
if (code) goto _exit;
blockCol.offset = aBufN[0];
- aBufN[0] = aBufN[0] + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM);
+ aBufN[0] = aBufN[0] + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue;
}
code = tRealloc(&aBuf[1], hdr.szBlkCol + tPutBlockCol(NULL, &blockCol));
@@ -1540,15 +1556,8 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut,
hdr.szBlkCol += tPutBlockCol(aBuf[1] + hdr.szBlkCol, &blockCol);
}
- aBufN[1] = 0;
- if (hdr.szBlkCol > 0) {
- aBufN[1] = hdr.szBlkCol + sizeof(TSCKSUM);
-
- code = tRealloc(&aBuf[1], aBufN[1]);
- if (code) goto _exit;
-
- taosCalcChecksumAppend(0, aBuf[1], aBufN[1]);
- }
+ // SBlockCol
+ aBufN[1] = hdr.szBlkCol;
// uid + version + tskey
aBufN[2] = 0;
@@ -1569,16 +1578,11 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut,
if (code) goto _exit;
aBufN[2] += hdr.szKey;
- aBufN[2] += sizeof(TSCKSUM);
- code = tRealloc(&aBuf[2], aBufN[2]);
- if (code) goto _exit;
-
// hdr
aBufN[3] = tPutDiskDataHdr(NULL, &hdr);
code = tRealloc(&aBuf[3], aBufN[3]);
if (code) goto _exit;
tPutDiskDataHdr(aBuf[3], &hdr);
- taosCalcChecksumAppend(taosCalcChecksum(0, aBuf[3], aBufN[3]), aBuf[2], aBufN[2]);
// aggragate
if (ppOut) {
@@ -1603,17 +1607,13 @@ _exit:
int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]) {
int32_t code = 0;
- tBlockDataClear(pBlockData);
+ tBlockDataReset(pBlockData);
int32_t n = 0;
SDiskDataHdr hdr = {0};
// SDiskDataHdr
n += tGetDiskDataHdr(pIn + n, &hdr);
- if (!taosCheckChecksumWhole(pIn, n + hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM))) {
- code = TSDB_CODE_FILE_CORRUPTED;
- goto _exit;
- }
ASSERT(hdr.delimiter == TSDB_FILE_DLMT);
pBlockData->suid = hdr.suid;
@@ -1641,7 +1641,7 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin
code = tsdbDecmprData(pIn + n, hdr.szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr.cmprAlg, (uint8_t **)&pBlockData->aTSKEY,
sizeof(TSKEY) * hdr.nRow, &aBuf[0]);
if (code) goto _exit;
- n = n + hdr.szKey + sizeof(TSCKSUM);
+ n += hdr.szKey;
// loop to decode each column data
if (hdr.szBlkCol == 0) goto _exit;
@@ -1663,8 +1663,8 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin
if (code) goto _exit;
}
} else {
- code = tsdbDecmprColData(pIn + n + hdr.szBlkCol + sizeof(TSCKSUM) + blockCol.offset, &blockCol, hdr.cmprAlg,
- hdr.nRow, pColData, &aBuf[0]);
+ code = tsdbDecmprColData(pIn + n + hdr.szBlkCol + blockCol.offset, &blockCol, hdr.cmprAlg, hdr.nRow, pColData,
+ &aBuf[0]);
if (code) goto _exit;
}
}
@@ -2046,12 +2046,6 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol
}
size += pBlockCol->szValue;
- // checksum
- size += sizeof(TSCKSUM);
- code = tRealloc(ppOut, nOut + size);
- if (code) goto _exit;
- taosCalcChecksumAppend(0, *ppOut + nOut, size);
-
_exit:
return code;
}
@@ -2060,12 +2054,6 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in
uint8_t **ppBuf) {
int32_t code = 0;
- int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM);
- if (!taosCheckChecksumWhole(pIn, size)) {
- code = TSDB_CODE_FILE_CORRUPTED;
- goto _exit;
- }
-
ASSERT(pColData->cid == pBlockCol->cid);
ASSERT(pColData->type == pBlockCol->type);
pColData->smaOn = pBlockCol->smaOn;
@@ -2137,37 +2125,3 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in
_exit:
return code;
}
-
-int32_t tsdbReadAndCheck(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck) {
- int32_t code = 0;
-
- // alloc
- code = tRealloc(ppOut, size);
- if (code) goto _exit;
-
- // seek
- int64_t n = taosLSeekFile(pFD, offset, SEEK_SET);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _exit;
- }
-
- // read
- n = taosReadFile(pFD, *ppOut, size);
- if (n < 0) {
- code = TAOS_SYSTEM_ERROR(errno);
- goto _exit;
- } else if (n < size) {
- code = TSDB_CODE_FILE_CORRUPTED;
- goto _exit;
- }
-
- // check
- if (toCheck && !taosCheckChecksumWhole(*ppOut, size)) {
- code = TSDB_CODE_FILE_CORRUPTED;
- goto _exit;
- }
-
-_exit:
- return code;
-}
diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c
index 383652531e..0a9fbf92a4 100644
--- a/source/dnode/vnode/src/tsdb/tsdbWrite.c
+++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c
@@ -39,7 +39,7 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *
SSubmitBlkRsp r = {0};
tGetSubmitMsgNext(&msgIter, &pBlock);
if (pBlock == NULL) break;
- if (tsdbInsertTableData(pTsdb, version, &msgIter, pBlock, &r) < 0) {
+ if ((terrno = tsdbInsertTableData(pTsdb, version, &msgIter, pBlock, &r)) < 0) {
return -1;
}
diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c
index 24dda29330..9ed72eeecd 100644
--- a/source/dnode/vnode/src/vnd/vnodeCfg.c
+++ b/source/dnode/vnode/src/vnd/vnodeCfg.c
@@ -13,6 +13,7 @@
* along with this program. If not, see , result block list, used to keep the multi-block that
// passed by downstream operator
SArray* pResultBlockList;
- int32_t rspBlockIndex; // indicate the return block index in pResultBlockList
- SSDataBlock* pDummyBlock; // dummy block, not keep data
- bool seqLoadData; // sequential load data or not, false by default
+ int32_t rspBlockIndex; // indicate the return block index in pResultBlockList
+ SSDataBlock* pDummyBlock; // dummy block, not keep data
+ bool seqLoadData; // sequential load data or not, false by default
int32_t current;
SLoadRemoteDataInfo loadInfo;
uint64_t self;
@@ -278,22 +278,22 @@ typedef struct SExchangeInfo {
} SExchangeInfo;
typedef struct SColMatchInfo {
- int32_t srcSlotId; // source slot id
+ int32_t srcSlotId; // source slot id
int32_t colId;
int32_t targetSlotId;
- bool output; // todo remove this?
+ bool output; // todo remove this?
bool reserved;
- int32_t matchType; // determinate the source according to col id or slot id
+ int32_t matchType; // determinate the source according to col id or slot id
} SColMatchInfo;
typedef struct SScanInfo {
- int32_t numOfAsc;
- int32_t numOfDesc;
+ int32_t numOfAsc;
+ int32_t numOfDesc;
} SScanInfo;
typedef struct SSampleExecInfo {
- double sampleRatio; // data block sample ratio, 1 by default
- uint32_t seed; // random seed value
+ double sampleRatio; // data block sample ratio, 1 by default
+ uint32_t seed; // random seed value
} SSampleExecInfo;
enum {
@@ -305,39 +305,40 @@ typedef struct SAggSupporter {
SSHashObj* pResultRowHashTable; // quick locate the window object for each result
char* keyBuf; // window key buffer
SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file
- int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row
- int32_t currentPageId; // current write page id
+ int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row
+ int32_t currentPageId; // current write page id
} SAggSupporter;
typedef struct {
- // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if current data block needs to be loaded.
- SInterval interval;
- SAggSupporter *pAggSup;
- SExprSupp *pExprSup; // expr supporter of aggregate operator
+ // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if
+ // current data block needs to be loaded.
+ SInterval interval;
+ SAggSupporter* pAggSup;
+ SExprSupp* pExprSup; // expr supporter of aggregate operator
} SAggOptrPushDownInfo;
typedef struct STableScanInfo {
- STsdbReader* dataReader;
- SReadHandle readHandle;
+ STsdbReader* dataReader;
+ SReadHandle readHandle;
SFileBlockLoadRecorder readRecorder;
- SScanInfo scanInfo;
- int32_t scanTimes;
- SNode* pFilterNode; // filter info, which is push down by optimizer
+ SScanInfo scanInfo;
+ int32_t scanTimes;
+ SNode* pFilterNode; // filter info, which is push down by optimizer
- SSDataBlock* pResBlock;
- SArray* pColMatchInfo;
- SExprSupp pseudoSup;
- SQueryTableDataCond cond;
- int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan
- int32_t dataBlockLoadFlag;
- SSampleExecInfo sample; // sample execution info
- int32_t currentGroupId;
- int32_t currentTable;
- int8_t scanMode;
- int8_t noTable;
+ SSDataBlock* pResBlock;
+ SArray* pColMatchInfo;
+ SExprSupp pseudoSup;
+ SQueryTableDataCond cond;
+ int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan
+ int32_t dataBlockLoadFlag;
+ SSampleExecInfo sample; // sample execution info
+ int32_t currentGroupId;
+ int32_t currentTable;
+ int8_t scanMode;
+ int8_t noTable;
SAggOptrPushDownInfo pdInfo;
- int8_t assignBlockUid;
+ int8_t assignBlockUid;
} STableScanInfo;
typedef struct STableMergeScanInfo {
@@ -370,7 +371,7 @@ typedef struct STableMergeScanInfo {
SArray* pColMatchInfo;
int32_t numOfOutput;
- SExprSupp pseudoSup;
+ SExprSupp pseudoSup;
SQueryTableDataCond cond;
int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan
@@ -384,33 +385,33 @@ typedef struct STableMergeScanInfo {
} STableMergeScanInfo;
typedef struct STagScanInfo {
- SColumnInfo *pCols;
- SSDataBlock *pRes;
- SArray *pColMatchInfo;
- int32_t curPos;
- SReadHandle readHandle;
- STableListInfo *pTableList;
+ SColumnInfo* pCols;
+ SSDataBlock* pRes;
+ SArray* pColMatchInfo;
+ int32_t curPos;
+ SReadHandle readHandle;
+ STableListInfo* pTableList;
} STagScanInfo;
typedef struct SLastrowScanInfo {
- SSDataBlock *pRes;
- SReadHandle readHandle;
- void *pLastrowReader;
- SArray *pColMatchInfo;
- int32_t *pSlotIds;
- SExprSupp pseudoExprSup;
- int32_t retrieveType;
- int32_t currentGroupIndex;
- SSDataBlock *pBufferredRes;
- SArray *pUidList;
- int32_t indexOfBufferedRes;
+ SSDataBlock* pRes;
+ SReadHandle readHandle;
+ void* pLastrowReader;
+ SArray* pColMatchInfo;
+ int32_t* pSlotIds;
+ SExprSupp pseudoExprSup;
+ int32_t retrieveType;
+ int32_t currentGroupIndex;
+ SSDataBlock* pBufferredRes;
+ SArray* pUidList;
+ int32_t indexOfBufferedRes;
} SLastrowScanInfo;
typedef enum EStreamScanMode {
STREAM_SCAN_FROM_READERHANDLE = 1,
STREAM_SCAN_FROM_RES,
STREAM_SCAN_FROM_UPDATERES,
- STREAM_SCAN_FROM_DELETERES,
+ STREAM_SCAN_FROM_DELETE_DATA,
STREAM_SCAN_FROM_DATAREADER_RETRIEVE,
STREAM_SCAN_FROM_DATAREADER_RANGE,
} EStreamScanMode;
@@ -432,10 +433,10 @@ typedef struct SStreamAggSupporter {
SArray* pCurWins;
int32_t valueSize;
int32_t keySize;
- char* pKeyBuf; // window key buffer
- SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file
- int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row
- int32_t currentPageId; // buffer page that is active
+ char* pKeyBuf; // window key buffer
+ SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file
+ int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row
+ int32_t currentPageId; // buffer page that is active
SSDataBlock* pScanBlock;
} SStreamAggSupporter;
@@ -447,22 +448,22 @@ typedef struct SWindowSupporter {
} SWindowSupporter;
typedef struct SPartitionBySupporter {
- SArray* pGroupCols; // group by columns, SArray
- SArray* pGroupColVals; // current group column values, SArray
- char* keyBuf; // group by keys for hash
- bool needCalc; // partition by column
+ SArray* pGroupCols; // group by columns, SArray
+ SArray* pGroupColVals; // current group column values, SArray
+ char* keyBuf; // group by keys for hash
+ bool needCalc; // partition by column
} SPartitionBySupporter;
typedef struct SPartitionDataInfo {
- uint64_t groupId;
- SArray* rowIds;
+ uint64_t groupId;
+ SArray* rowIds;
} SPartitionDataInfo;
typedef struct STimeWindowSupp {
- int8_t calTrigger;
- int64_t waterMark;
- TSKEY maxTs;
- SColumnInfoData timeWindowData; // query time window info for scalar function execution.
+ int8_t calTrigger;
+ int64_t waterMark;
+ TSKEY maxTs;
+ SColumnInfoData timeWindowData; // query time window info for scalar function execution.
} STimeWindowAggSupp;
typedef struct SStreamScanInfo {
@@ -487,22 +488,22 @@ typedef struct SStreamScanInfo {
uint64_t groupId;
SUpdateInfo* pUpdateInfo;
- EStreamScanMode scanMode;
- SOperatorInfo* pStreamScanOp;
- SOperatorInfo* pTableScanOp;
- SArray* childIds;
- SWindowSupporter windowSup;
- SPartitionBySupporter partitionSup;
- SExprSupp* pPartScalarSup;
- bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA.
- int32_t scanWinIndex; // for state operator
- int32_t pullDataResIndex;
- SSDataBlock* pPullDataRes; // pull data SSDataBlock
- SSDataBlock* pDeleteDataRes; // delete data SSDataBlock
- int32_t deleteDataIndex;
- STimeWindow updateWin;
- STimeWindowAggSupp twAggSup;
- SSDataBlock* pUpdateDataRes;
+ EStreamScanMode scanMode;
+ SOperatorInfo* pStreamScanOp;
+ SOperatorInfo* pTableScanOp;
+ SArray* childIds;
+ SWindowSupporter windowSup;
+ SPartitionBySupporter partitionSup;
+ SExprSupp* pPartScalarSup;
+ bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA.
+ int32_t scanWinIndex; // for state operator
+ int32_t pullDataResIndex;
+ SSDataBlock* pPullDataRes; // pull data SSDataBlock
+ SSDataBlock* pDeleteDataRes; // delete data SSDataBlock
+ int32_t deleteDataIndex;
+ STimeWindow updateWin;
+ STimeWindowAggSupp twAggSup;
+ SSDataBlock* pUpdateDataRes;
// status for tmq
SNodeList* pGroupTags;
SNode* pTagCond;
@@ -571,17 +572,17 @@ typedef struct SIntervalAggOperatorInfo {
EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model]
STimeWindowAggSupp twAggSup;
bool invertible;
- SArray* pPrevValues; // SArray used to keep the previous not null value for interpolation.
+ SArray* pPrevValues; // SArray used to keep the previous not null value for interpolation.
bool ignoreExpiredData;
SArray* pRecycledPages;
- SArray* pDelWins; // SWinRes
+ SArray* pDelWins; // SWinRes
int32_t delIndex;
SSDataBlock* pDelRes;
SNode* pCondition;
} SIntervalAggOperatorInfo;
typedef struct SMergeAlignedIntervalAggOperatorInfo {
- SIntervalAggOperatorInfo *intervalAggOperatorInfo;
+ SIntervalAggOperatorInfo* intervalAggOperatorInfo;
bool hasGroupId;
uint64_t groupId; // current groupId
@@ -590,7 +591,7 @@ typedef struct SMergeAlignedIntervalAggOperatorInfo {
SNode* pCondition;
} SMergeAlignedIntervalAggOperatorInfo;
-typedef struct SStreamFinalIntervalOperatorInfo {
+typedef struct SStreamIntervalOperatorInfo {
// SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
SOptrBasicInfo binfo; // basic info
SAggSupporter aggSup; // aggregate supporter
@@ -598,57 +599,75 @@ typedef struct SStreamFinalIntervalOperatorInfo {
SGroupResInfo groupResInfo; // multiple results build supporter
SInterval interval; // interval info
int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator.
- int32_t order; // current SSDataBlock scan order
+ STimeWindowAggSupp twAggSup;
+ bool invertible;
+ bool ignoreExpiredData;
+ SArray* pRecycledPages;
+ SArray* pDelWins; // SWinRes
+ int32_t delIndex;
+ SSDataBlock* pDelRes;
+ bool isFinal;
+} SStreamIntervalOperatorInfo;
+
+typedef struct SStreamFinalIntervalOperatorInfo {
+ // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
+ SOptrBasicInfo binfo; // basic info
+ SAggSupporter aggSup; // aggregate supporter
+ SExprSupp scalarSupp; // supporter for perform scalar function
+ SGroupResInfo groupResInfo; // multiple results build supporter
+ SInterval interval; // interval info
+ int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator.
+ int32_t order; // current SSDataBlock scan order
STimeWindowAggSupp twAggSup;
SArray* pChildren;
SSDataBlock* pUpdateRes;
bool returnUpdate;
- SPhysiNode* pPhyNode; // create new child
+ SPhysiNode* pPhyNode; // create new child
bool isFinal;
SHashObj* pPullDataMap;
- SArray* pPullWins; // SPullWindowInfo
+ SArray* pPullWins; // SPullWindowInfo
int32_t pullIndex;
SSDataBlock* pPullDataRes;
bool ignoreExpiredData;
SArray* pRecycledPages;
- SArray* pDelWins; // SWinRes
+ SArray* pDelWins; // SWinRes
int32_t delIndex;
SSDataBlock* pDelRes;
} SStreamFinalIntervalOperatorInfo;
typedef struct SAggOperatorInfo {
// SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
- SOptrBasicInfo binfo;
- SAggSupporter aggSup;
+ SOptrBasicInfo binfo;
+ SAggSupporter aggSup;
- STableQueryInfo *current;
- uint64_t groupId;
- SGroupResInfo groupResInfo;
- SExprSupp scalarExprSup;
- SNode *pCondition;
+ STableQueryInfo* current;
+ uint64_t groupId;
+ SGroupResInfo groupResInfo;
+ SExprSupp scalarExprSup;
+ SNode* pCondition;
} SAggOperatorInfo;
typedef struct SProjectOperatorInfo {
// SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
- SOptrBasicInfo binfo;
- SAggSupporter aggSup;
- SNode* pFilterNode; // filter info, which is push down by optimizer
- SArray* pPseudoColInfo;
- SLimitInfo limitInfo;
- bool mergeDataBlocks;
- SSDataBlock* pFinalRes;
- SNode* pCondition;
+ SOptrBasicInfo binfo;
+ SAggSupporter aggSup;
+ SNode* pFilterNode; // filter info, which is push down by optimizer
+ SArray* pPseudoColInfo;
+ SLimitInfo limitInfo;
+ bool mergeDataBlocks;
+ SSDataBlock* pFinalRes;
+ SNode* pCondition;
} SProjectOperatorInfo;
typedef struct SIndefOperatorInfo {
- SOptrBasicInfo binfo;
- SAggSupporter aggSup;
- SArray* pPseudoColInfo;
- SExprSupp scalarSup;
- SNode* pCondition;
- uint64_t groupId;
+ SOptrBasicInfo binfo;
+ SAggSupporter aggSup;
+ SArray* pPseudoColInfo;
+ SExprSupp scalarSup;
+ SNode* pCondition;
+ uint64_t groupId;
- SSDataBlock* pNextGroupRes;
+ SSDataBlock* pNextGroupRes;
} SIndefOperatorInfo;
typedef struct SFillOperatorInfo {
@@ -663,7 +682,7 @@ typedef struct SFillOperatorInfo {
SArray* pColMatchColInfo;
int32_t primaryTsCol;
int32_t primarySrcSlotId;
- uint64_t curGroupId; // current handled group id
+ uint64_t curGroupId; // current handled group id
SExprInfo* pExprInfo;
int32_t numOfExpr;
SExprInfo* pNotFillExprInfo;
@@ -672,23 +691,23 @@ typedef struct SFillOperatorInfo {
typedef struct SGroupbyOperatorInfo {
// SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
- SOptrBasicInfo binfo;
- SAggSupporter aggSup;
+ SOptrBasicInfo binfo;
+ SAggSupporter aggSup;
- SArray* pGroupCols; // group by columns, SArray
- SArray* pGroupColVals; // current group column values, SArray
- SNode* pCondition;
- bool isInit; // denote if current val is initialized or not
- char* keyBuf; // group by keys for hash
- int32_t groupKeyLen; // total group by column width
- SGroupResInfo groupResInfo;
- SExprSupp scalarSup;
+ SArray* pGroupCols; // group by columns, SArray
+ SArray* pGroupColVals; // current group column values, SArray
+ SNode* pCondition;
+ bool isInit; // denote if current val is initialized or not
+ char* keyBuf; // group by keys for hash
+ int32_t groupKeyLen; // total group by column width
+ SGroupResInfo groupResInfo;
+ SExprSupp scalarSup;
} SGroupbyOperatorInfo;
typedef struct SDataGroupInfo {
- uint64_t groupId;
- int64_t numOfRows;
- SArray* pPageList;
+ uint64_t groupId;
+ int64_t numOfRows;
+ SArray* pPageList;
} SDataGroupInfo;
// The sort in partition may be needed later.
@@ -700,12 +719,12 @@ typedef struct SPartitionOperatorInfo {
int32_t groupKeyLen; // total group by column width
SHashObj* pGroupSet; // quick locate the window object for each result
- SDiskbasedBuf* pBuf; // query result buffer based on blocked-wised disk file
- int32_t rowCapacity; // maximum number of rows for each buffer page
- int32_t* columnOffset; // start position for each column data
- SArray* sortedGroupArray; // SDataGroupInfo sorted by group id
- int32_t groupIndex; // group index
- int32_t pageIndex; // page index of current group
+ SDiskbasedBuf* pBuf; // query result buffer based on blocked-wised disk file
+ int32_t rowCapacity; // maximum number of rows for each buffer page
+ int32_t* columnOffset; // start position for each column data
+ SArray* sortedGroupArray; // SDataGroupInfo sorted by group id
+ int32_t groupIndex; // group index
+ int32_t pageIndex; // page index of current group
SExprSupp scalarSup;
} SPartitionOperatorInfo;
@@ -719,52 +738,52 @@ typedef struct SWindowRowsSup {
typedef struct SSessionAggOperatorInfo {
// SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
- SOptrBasicInfo binfo;
- SAggSupporter aggSup;
+ SOptrBasicInfo binfo;
+ SAggSupporter aggSup;
SGroupResInfo groupResInfo;
SWindowRowsSup winSup;
- bool reptScan; // next round scan
- int64_t gap; // session window gap
- int32_t tsSlotId; // primary timestamp slot id
+ bool reptScan; // next round scan
+ int64_t gap; // session window gap
+ int32_t tsSlotId; // primary timestamp slot id
STimeWindowAggSupp twAggSup;
- const SNode* pCondition;
+ const SNode* pCondition;
} SSessionAggOperatorInfo;
typedef struct SResultWindowInfo {
SResultRowPosition pos;
- STimeWindow win;
- uint64_t groupId;
- bool isOutput;
- bool isClosed;
+ STimeWindow win;
+ uint64_t groupId;
+ bool isOutput;
+ bool isClosed;
} SResultWindowInfo;
typedef struct SStateWindowInfo {
SResultWindowInfo winInfo;
- SStateKeys stateKey;
+ SStateKeys stateKey;
} SStateWindowInfo;
typedef struct SStreamSessionAggOperatorInfo {
- SOptrBasicInfo binfo;
- SStreamAggSupporter streamAggSup;
- SExprSupp scalarSupp; // supporter for perform scalar function
- SGroupResInfo groupResInfo;
- int64_t gap; // session window gap
- int32_t primaryTsIndex; // primary timestamp slot id
- int32_t endTsIndex; // window end timestamp slot id
- int32_t order; // current SSDataBlock scan order
- STimeWindowAggSupp twAggSup;
- SSDataBlock* pWinBlock; // window result
- SqlFunctionCtx* pDummyCtx; // for combine
- SSDataBlock* pDelRes; // delete result
- SSDataBlock* pUpdateRes; // update window
- bool returnUpdate;
- SHashObj* pStDeleted;
- void* pDelIterator;
- SArray* pChildren; // cache for children's result; final stream operator
- SPhysiNode* pPhyNode; // create new child
- bool isFinal;
- bool ignoreExpiredData;
+ SOptrBasicInfo binfo;
+ SStreamAggSupporter streamAggSup;
+ SExprSupp scalarSupp; // supporter for perform scalar function
+ SGroupResInfo groupResInfo;
+ int64_t gap; // session window gap
+ int32_t primaryTsIndex; // primary timestamp slot id
+ int32_t endTsIndex; // window end timestamp slot id
+ int32_t order; // current SSDataBlock scan order
+ STimeWindowAggSupp twAggSup;
+ SSDataBlock* pWinBlock; // window result
+ SqlFunctionCtx* pDummyCtx; // for combine
+ SSDataBlock* pDelRes; // delete result
+ SSDataBlock* pUpdateRes; // update window
+ bool returnUpdate;
+ SHashObj* pStDeleted;
+ void* pDelIterator;
+ SArray* pChildren; // cache for children's result; final stream operator
+ SPhysiNode* pPhyNode; // create new child
+ bool isFinal;
+ bool ignoreExpiredData;
} SStreamSessionAggOperatorInfo;
typedef struct SStreamPartitionOperatorInfo {
@@ -775,29 +794,30 @@ typedef struct SStreamPartitionOperatorInfo {
void* parIte;
SSDataBlock* pInputDataBlock;
int32_t tsColIndex;
+ SSDataBlock* pDelRes;
} SStreamPartitionOperatorInfo;
typedef struct STimeSliceOperatorInfo {
- SSDataBlock* pRes;
- STimeWindow win;
- SInterval interval;
- int64_t current;
- SArray* pPrevRow; // SArray
- SArray* pNextRow; // SArray
- SArray* pLinearInfo; // SArray
- bool fillLastPoint;
- bool isPrevRowSet;
- bool isNextRowSet;
- int32_t fillType; // fill type
- SColumn tsCol; // primary timestamp column
- SExprSupp scalarSup; // scalar calculation
- struct SFillColInfo* pFillColInfo; // fill column info
+ SSDataBlock* pRes;
+ STimeWindow win;
+ SInterval interval;
+ int64_t current;
+ SArray* pPrevRow; // SArray
+ SArray* pNextRow; // SArray
+ SArray* pLinearInfo; // SArray
+ bool fillLastPoint;
+ bool isPrevRowSet;
+ bool isNextRowSet;
+ int32_t fillType; // fill type
+ SColumn tsCol; // primary timestamp column
+ SExprSupp scalarSup; // scalar calculation
+ struct SFillColInfo* pFillColInfo; // fill column info
} STimeSliceOperatorInfo;
typedef struct SStateWindowOperatorInfo {
// SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
- SOptrBasicInfo binfo;
- SAggSupporter aggSup;
+ SOptrBasicInfo binfo;
+ SAggSupporter aggSup;
SGroupResInfo groupResInfo;
SWindowRowsSup winSup;
@@ -811,52 +831,52 @@ typedef struct SStateWindowOperatorInfo {
} SStateWindowOperatorInfo;
typedef struct SStreamStateAggOperatorInfo {
- SOptrBasicInfo binfo;
- SStreamAggSupporter streamAggSup;
- SExprSupp scalarSupp; // supporter for perform scalar function
- SGroupResInfo groupResInfo;
- int32_t primaryTsIndex; // primary timestamp slot id
- int32_t order; // current SSDataBlock scan order
- STimeWindowAggSupp twAggSup;
- SColumn stateCol;
- SqlFunctionCtx* pDummyCtx; // for combine
- SSDataBlock* pDelRes;
- SHashObj* pSeDeleted;
- void* pDelIterator;
- SArray* pChildren; // cache for children's result;
- bool ignoreExpiredData;
+ SOptrBasicInfo binfo;
+ SStreamAggSupporter streamAggSup;
+ SExprSupp scalarSupp; // supporter for perform scalar function
+ SGroupResInfo groupResInfo;
+ int32_t primaryTsIndex; // primary timestamp slot id
+ int32_t order; // current SSDataBlock scan order
+ STimeWindowAggSupp twAggSup;
+ SColumn stateCol;
+ SqlFunctionCtx* pDummyCtx; // for combine
+ SSDataBlock* pDelRes;
+ SHashObj* pSeDeleted;
+ void* pDelIterator;
+ SArray* pChildren; // cache for children's result;
+ bool ignoreExpiredData;
} SStreamStateAggOperatorInfo;
typedef struct SSortedMergeOperatorInfo {
// SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
- SOptrBasicInfo binfo;
- SAggSupporter aggSup;
+ SOptrBasicInfo binfo;
+ SAggSupporter aggSup;
- SArray* pSortInfo;
- int32_t numOfSources;
- SSortHandle *pSortHandle;
- int32_t bufPageSize;
- uint32_t sortBufSize; // max buffer size for in-memory sort
- int32_t resultRowFactor;
- bool hasGroupVal;
- SDiskbasedBuf *pTupleStore; // keep the final results
- int32_t numOfResPerPage;
- char** groupVal;
- SArray *groupInfo;
+ SArray* pSortInfo;
+ int32_t numOfSources;
+ SSortHandle* pSortHandle;
+ int32_t bufPageSize;
+ uint32_t sortBufSize; // max buffer size for in-memory sort
+ int32_t resultRowFactor;
+ bool hasGroupVal;
+ SDiskbasedBuf* pTupleStore; // keep the final results
+ int32_t numOfResPerPage;
+ char** groupVal;
+ SArray* groupInfo;
} SSortedMergeOperatorInfo;
typedef struct SSortOperatorInfo {
SOptrBasicInfo binfo;
- uint32_t sortBufSize; // max buffer size for in-memory sort
- SArray* pSortInfo;
- SSortHandle* pSortHandle;
- SArray* pColMatchInfo; // for index map from table scan output
- int32_t bufPageSize;
+ uint32_t sortBufSize; // max buffer size for in-memory sort
+ SArray* pSortInfo;
+ SSortHandle* pSortHandle;
+ SArray* pColMatchInfo; // for index map from table scan output
+ int32_t bufPageSize;
- int64_t startTs; // sort start time
- uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included.
- SLimitInfo limitInfo;
- SNode* pCondition;
+ int64_t startTs; // sort start time
+ uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included.
+ SLimitInfo limitInfo;
+ SNode* pCondition;
} SSortOperatorInfo;
typedef struct STagFilterOperatorInfo {
@@ -864,18 +884,18 @@ typedef struct STagFilterOperatorInfo {
} STagFilterOperatorInfo;
typedef struct SJoinOperatorInfo {
- SSDataBlock *pRes;
- int32_t joinType;
- int32_t inputOrder;
+ SSDataBlock* pRes;
+ int32_t joinType;
+ int32_t inputOrder;
- SSDataBlock *pLeft;
- int32_t leftPos;
- SColumnInfo leftCol;
+ SSDataBlock* pLeft;
+ int32_t leftPos;
+ SColumnInfo leftCol;
- SSDataBlock *pRight;
- int32_t rightPos;
- SColumnInfo rightCol;
- SNode *pCondAfterMerge;
+ SSDataBlock* pRight;
+ int32_t rightPos;
+ SColumnInfo rightCol;
+ SNode* pCondAfterMerge;
} SJoinOperatorInfo;
#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED)
@@ -884,8 +904,8 @@ typedef struct SJoinOperatorInfo {
void doDestroyExchangeOperatorInfo(void* param);
SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn,
- __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_encode_fn_t encode,
- __optr_decode_fn_t decode, __optr_explain_fn_t explain);
+ __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_encode_fn_t encode,
+ __optr_decode_fn_t decode, __optr_explain_fn_t explain);
int32_t operatorDummyOpenFn(SOperatorInfo* pOperator);
void operatorDummyCloseFn(void* param, int32_t numOfCols);
@@ -896,24 +916,25 @@ void cleanupBasicInfo(SOptrBasicInfo* pInfo);
int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr);
void cleanupExprSupp(SExprSupp* pSup);
void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs);
-int32_t initAggInfo(SExprSupp *pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
+int32_t initAggInfo(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
const char* pkey);
-void initResultSizeInfo(SResultInfo * pResultInfo, int32_t numOfRows);
-void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf);
-int32_t handleLimitOffset(SOperatorInfo *pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf);
+void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows);
+void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
+ SDiskbasedBuf* pBuf);
+int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf);
bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo);
void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo);
-void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData, int32_t offset,
- int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput);
+void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData, int32_t offset,
+ int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput);
-int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, int32_t numOfOutput, SArray* pColList, char** pNextStart);
-void updateLoadRemoteInfo(SLoadRemoteDataInfo *pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs,
- SOperatorInfo* pOperator);
+int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart);
+void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs,
+ SOperatorInfo* pOperator);
STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order);
-int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag);
+int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag);
int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz);
void doSetOperatorCompleted(SOperatorInfo* pOperator);
@@ -921,10 +942,10 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pC
int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr,
SSDataBlock* pBlock, const char* idStr);
-void cleanupAggSup(SAggSupporter* pAggSup);
-void destroyBasicOperatorInfo(void* param, int32_t numOfOutput);
-void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle);
-void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId);
+void cleanupAggSup(SAggSupporter* pAggSup);
+void destroyBasicOperatorInfo(void* param, int32_t numOfOutput);
+void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle);
+void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId);
int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts);
int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts);
@@ -933,29 +954,37 @@ SSDataBlock* loadNextDataBlock(void* param);
void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset);
-SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo,
- char* pData, int16_t bytes, bool masterscan, uint64_t groupId,
- SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup);
+SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData,
+ int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo,
+ bool isIntervalQuery, SAggSupporter* pSup);
SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
+ SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode,
STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode,
+ const char* pUser, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SNode* pCondition, SExprInfo* pScalarExprInfo,
+SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
+ SSDataBlock* pResultBlock, SNode* pCondition, SExprInfo* pScalarExprInfo,
int32_t numOfScalarExpr, bool mergeResult, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode *pNode, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode,
+ SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode,
+ SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortNode, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams,
+ SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
+ SExecTaskInfo* pTaskInfo);
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
- STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream);
+ STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode,
+ SExecTaskInfo* pTaskInfo, bool isStream);
SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMergeIntervalPhysiNode* pIntervalPhyNode,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SMergeAlignedIntervalPhysiNode* pNode,
@@ -975,33 +1004,41 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo);
-
-SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo);
-
-SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode,
+SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode,
+ SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode,
+ SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode,
SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream,
- SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream,
- SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild);
+SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode,
+ SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream,
+ SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo);
+
+SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
+ SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
+ SExecTaskInfo* pTaskInfo, int32_t numOfChild);
+SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream,
+ SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo);
+
+SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
+ SExecTaskInfo* pTaskInfo);
int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx,
- int32_t numOfOutput, SArray* pPseudoList);
+ int32_t numOfOutput, SArray* pPseudoList);
-void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol);
+void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order,
+ int32_t scanFlag, bool createDummyCol);
bool isTaskKilled(SExecTaskInfo* pTaskInfo);
int32_t checkForQueryBuf(size_t numOfTables);
-void setTaskKilled(SExecTaskInfo* pTaskInfo);
-void queryCostStatis(SExecTaskInfo* pTaskInfo);
+void setTaskKilled(SExecTaskInfo* pTaskInfo);
+void queryCostStatis(SExecTaskInfo* pTaskInfo);
void doDestroyTask(SExecTaskInfo* pTaskInfo);
int32_t getMaximumIdleDurationSec();
@@ -1013,7 +1050,7 @@ int32_t getMaximumIdleDurationSec();
* nOptrWithVal: *nOptrWithVal save the number of optr with value
* return: result code, 0 means success
*/
-int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t *length, int32_t *nOptrWithVal);
+int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t* length, int32_t* nOptrWithVal);
/*
* ops: root operator, created by caller
@@ -1026,7 +1063,7 @@ int32_t decodeOperator(SOperatorInfo* ops, const char* data, int32_t length);
void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status);
int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId,
char* sql, EOPTR_EXEC_MODEL model);
-int32_t createDataSinkParam(SDataSinkNode *pNode, void **pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle);
+int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle);
int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList);
int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result);
@@ -1035,42 +1072,50 @@ int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* len
STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval,
int32_t order);
int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey,
- __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order);
+ __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order);
int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order);
int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput,
- int32_t size);
-SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, int32_t interBufSize);
-SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs,
- TSKEY endTs, uint64_t groupId, int64_t gap, int32_t* pIndex);
-SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs,
- TSKEY endTs, uint64_t groupId, int64_t gap, int32_t* pIndex);
-bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap);
-bool functionNeedToExecute(SqlFunctionCtx* pCtx);
-bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup);
-bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup);
-bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup);
-void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp);
-void printDataBlock(SSDataBlock* pBlock, const char* flag);
+ int32_t size);
+SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, int32_t interBufSize);
+SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
+ int64_t gap, int32_t* pIndex);
+SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
+ int64_t gap, int32_t* pIndex);
+bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap);
+bool functionNeedToExecute(SqlFunctionCtx* pCtx);
+bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup);
+bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup);
+bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup);
+void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp);
+void printDataBlock(SSDataBlock* pBlock, const char* flag);
uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId);
int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition,
- SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset,
- SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo);
+ SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs,
+ const int32_t* rowCellOffset, SSDataBlock* pBlock,
+ SExecTaskInfo* pTaskInfo);
-int32_t createScanTableListInfo(SScanPhysiNode *pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle,
- STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* idstr);
+int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle,
+ STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond,
+ const char* idstr);
SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode,
SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo *pTableListInfo,
+SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo,
SReadHandle* readHandle, SExecTaskInfo* pTaskInfo);
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex);
-bool groupbyTbname(SNodeList* pGroupList);
+bool groupbyTbname(SNodeList* pGroupList);
int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey);
-SSDataBlock* createSpecialDataBlock(EStreamType type);
-void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput);
+void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput);
+int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup,
+ SGroupResInfo* pGroupResInfo);
+int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
+ int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup,
+ SExecTaskInfo* pTaskInfo);
+int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult);
+int32_t saveOutput(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize);
#ifdef __cplusplus
}
diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c
index 06b7c13fa2..40198615ea 100644
--- a/source/libs/executor/src/dataDeleter.c
+++ b/source/libs/executor/src/dataDeleter.c
@@ -168,7 +168,9 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE
taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf);
memcpy(&pDeleter->nextOutput, pBuf, sizeof(SDataDeleterBuf));
taosFreeQitem(pBuf);
- *pLen = ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->dataLen;
+
+ SDataCacheEntry* pEntry = (SDataCacheEntry*)pDeleter->nextOutput.pData;
+ *pLen = pEntry->dataLen;
*pQueryEnd = pDeleter->queryEnd;
qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->numOfRows);
}
diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c
index 20396046ba..1697ed63fb 100644
--- a/source/libs/executor/src/dataDispatcher.c
+++ b/source/libs/executor/src/dataDispatcher.c
@@ -93,6 +93,8 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
pBuf->useSize = sizeof(SDataCacheEntry);
blockEncode(pInput->pData, pEntry->data, &pEntry->dataLen, numOfCols, pEntry->compressed);
+ ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data+8));
+ ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data+8+4));
pBuf->useSize += pEntry->dataLen;
@@ -170,7 +172,13 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE
taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf);
memcpy(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf));
taosFreeQitem(pBuf);
- *pLen = ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->dataLen;
+
+ SDataCacheEntry* pEntry = (SDataCacheEntry*)pDispatcher->nextOutput.pData;
+ *pLen = pEntry->dataLen;
+
+ ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data+8));
+ ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data+8+4));
+
*pQueryEnd = pDispatcher->queryEnd;
qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->numOfRows);
}
@@ -191,6 +199,9 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
pOutput->numOfCols = pEntry->numOfCols;
pOutput->compressed = pEntry->compressed;
+ ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data+8));
+ ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data+8+4));
+
atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen);
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index 3965b7e5b2..0c1728386b 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -475,7 +475,6 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
qError("failed to create result, reason:%s", tstrerror(code));
- terrno = code;
goto end;
}
@@ -1012,6 +1011,100 @@ static SColumn* createColumn(int32_t blockId, int32_t slotId, int32_t colId, SDa
return pCol;
}
+void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode) {
+ pExp->pExpr = taosMemoryCalloc(1, sizeof(tExprNode));
+ pExp->pExpr->_function.num = 1;
+ pExp->pExpr->_function.functionId = -1;
+
+ int32_t type = nodeType(pTargetNode->pExpr);
+ // it is a project query, or group by column
+ if (type == QUERY_NODE_COLUMN) {
+ pExp->pExpr->nodeType = QUERY_NODE_COLUMN;
+ SColumnNode* pColNode = (SColumnNode*)pTargetNode->pExpr;
+
+ pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam));
+ pExp->base.numOfParams = 1;
+
+ SDataType* pType = &pColNode->node.resType;
+ pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale,
+ pType->precision, pColNode->colName);
+ pExp->base.pParam[0].pCol =
+ createColumn(pColNode->dataBlockId, pColNode->slotId, pColNode->colId, pType, pColNode->colType);
+ pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN;
+ } else if (type == QUERY_NODE_VALUE) {
+ pExp->pExpr->nodeType = QUERY_NODE_VALUE;
+ SValueNode* pValNode = (SValueNode*)pTargetNode->pExpr;
+
+ pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam));
+ pExp->base.numOfParams = 1;
+
+ SDataType* pType = &pValNode->node.resType;
+ pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale,
+ pType->precision, pValNode->node.aliasName);
+ pExp->base.pParam[0].type = FUNC_PARAM_TYPE_VALUE;
+ nodesValueNodeToVariant(pValNode, &pExp->base.pParam[0].param);
+ } else if (type == QUERY_NODE_FUNCTION) {
+ pExp->pExpr->nodeType = QUERY_NODE_FUNCTION;
+ SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr;
+
+ SDataType* pType = &pFuncNode->node.resType;
+ pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale,
+ pType->precision, pFuncNode->node.aliasName);
+
+ pExp->pExpr->_function.functionId = pFuncNode->funcId;
+ pExp->pExpr->_function.pFunctNode = pFuncNode;
+
+ strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName,
+ tListLen(pExp->pExpr->_function.functionName));
+#if 1
+ // todo refactor: add the parameter for tbname function
+ if (!pFuncNode->pParameterList && (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0)) {
+ pFuncNode->pParameterList = nodesMakeList();
+ ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0);
+ SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
+ if (NULL == res) { // todo handle error
+ } else {
+ res->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT};
+ nodesListAppend(pFuncNode->pParameterList, (SNode*)res);
+ }
+ }
+#endif
+
+ int32_t numOfParam = LIST_LENGTH(pFuncNode->pParameterList);
+
+ pExp->base.pParam = taosMemoryCalloc(numOfParam, sizeof(SFunctParam));
+ pExp->base.numOfParams = numOfParam;
+
+ for (int32_t j = 0; j < numOfParam; ++j) {
+ SNode* p1 = nodesListGetNode(pFuncNode->pParameterList, j);
+ if (p1->type == QUERY_NODE_COLUMN) {
+ SColumnNode* pcn = (SColumnNode*)p1;
+
+ pExp->base.pParam[j].type = FUNC_PARAM_TYPE_COLUMN;
+ pExp->base.pParam[j].pCol =
+ createColumn(pcn->dataBlockId, pcn->slotId, pcn->colId, &pcn->node.resType, pcn->colType);
+ } else if (p1->type == QUERY_NODE_VALUE) {
+ SValueNode* pvn = (SValueNode*)p1;
+ pExp->base.pParam[j].type = FUNC_PARAM_TYPE_VALUE;
+ nodesValueNodeToVariant(pvn, &pExp->base.pParam[j].param);
+ }
+ }
+ } else if (type == QUERY_NODE_OPERATOR) {
+ pExp->pExpr->nodeType = QUERY_NODE_OPERATOR;
+ SOperatorNode* pNode = (SOperatorNode*)pTargetNode->pExpr;
+
+ pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam));
+ pExp->base.numOfParams = 1;
+
+ SDataType* pType = &pNode->node.resType;
+ pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale,
+ pType->precision, pNode->node.aliasName);
+ pExp->pExpr->_optrRoot.pRootNode = pTargetNode->pExpr;
+ } else {
+ ASSERT(0);
+ }
+}
+
SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs) {
int32_t numOfFuncs = LIST_LENGTH(pNodeList);
int32_t numOfGroupKeys = 0;
@@ -1035,98 +1128,7 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t*
}
SExprInfo* pExp = &pExprs[i];
-
- pExp->pExpr = taosMemoryCalloc(1, sizeof(tExprNode));
- pExp->pExpr->_function.num = 1;
- pExp->pExpr->_function.functionId = -1;
-
- int32_t type = nodeType(pTargetNode->pExpr);
- // it is a project query, or group by column
- if (type == QUERY_NODE_COLUMN) {
- pExp->pExpr->nodeType = QUERY_NODE_COLUMN;
- SColumnNode* pColNode = (SColumnNode*)pTargetNode->pExpr;
-
- pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam));
- pExp->base.numOfParams = 1;
-
- SDataType* pType = &pColNode->node.resType;
- pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale,
- pType->precision, pColNode->colName);
- pExp->base.pParam[0].pCol =
- createColumn(pColNode->dataBlockId, pColNode->slotId, pColNode->colId, pType, pColNode->colType);
- pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN;
- } else if (type == QUERY_NODE_VALUE) {
- pExp->pExpr->nodeType = QUERY_NODE_VALUE;
- SValueNode* pValNode = (SValueNode*)pTargetNode->pExpr;
-
- pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam));
- pExp->base.numOfParams = 1;
-
- SDataType* pType = &pValNode->node.resType;
- pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale,
- pType->precision, pValNode->node.aliasName);
- pExp->base.pParam[0].type = FUNC_PARAM_TYPE_VALUE;
- nodesValueNodeToVariant(pValNode, &pExp->base.pParam[0].param);
- } else if (type == QUERY_NODE_FUNCTION) {
- pExp->pExpr->nodeType = QUERY_NODE_FUNCTION;
- SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr;
-
- SDataType* pType = &pFuncNode->node.resType;
- pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale,
- pType->precision, pFuncNode->node.aliasName);
-
- pExp->pExpr->_function.functionId = pFuncNode->funcId;
- pExp->pExpr->_function.pFunctNode = pFuncNode;
-
- strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName,
- tListLen(pExp->pExpr->_function.functionName));
-#if 1
- // todo refactor: add the parameter for tbname function
- if (!pFuncNode->pParameterList && (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0)) {
- pFuncNode->pParameterList = nodesMakeList();
- ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0);
- SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
- if (NULL == res) { // todo handle error
- } else {
- res->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT};
- nodesListAppend(pFuncNode->pParameterList, (SNode*)res);
- }
- }
-#endif
-
- int32_t numOfParam = LIST_LENGTH(pFuncNode->pParameterList);
-
- pExp->base.pParam = taosMemoryCalloc(numOfParam, sizeof(SFunctParam));
- pExp->base.numOfParams = numOfParam;
-
- for (int32_t j = 0; j < numOfParam; ++j) {
- SNode* p1 = nodesListGetNode(pFuncNode->pParameterList, j);
- if (p1->type == QUERY_NODE_COLUMN) {
- SColumnNode* pcn = (SColumnNode*)p1;
-
- pExp->base.pParam[j].type = FUNC_PARAM_TYPE_COLUMN;
- pExp->base.pParam[j].pCol =
- createColumn(pcn->dataBlockId, pcn->slotId, pcn->colId, &pcn->node.resType, pcn->colType);
- } else if (p1->type == QUERY_NODE_VALUE) {
- SValueNode* pvn = (SValueNode*)p1;
- pExp->base.pParam[j].type = FUNC_PARAM_TYPE_VALUE;
- nodesValueNodeToVariant(pvn, &pExp->base.pParam[j].param);
- }
- }
- } else if (type == QUERY_NODE_OPERATOR) {
- pExp->pExpr->nodeType = QUERY_NODE_OPERATOR;
- SOperatorNode* pNode = (SOperatorNode*)pTargetNode->pExpr;
-
- pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam));
- pExp->base.numOfParams = 1;
-
- SDataType* pType = &pNode->node.resType;
- pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale,
- pType->precision, pNode->node.aliasName);
- pExp->pExpr->_optrRoot.pRootNode = pTargetNode->pExpr;
- } else {
- ASSERT(0);
- }
+ createExprFromTargetNode(pExp, pTargetNode);
}
return pExprs;
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 12118fad72..95415e1113 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -52,7 +52,11 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
// TODO: if a block was set but not consumed,
// prevent setting a different type of block
pInfo->validBlockIndex = 0;
- taosArrayClear(pInfo->pBlockLists);
+ if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) {
+ taosArrayClearP(pInfo->pBlockLists, taosMemoryFree);
+ } else {
+ taosArrayClear(pInfo->pBlockLists);
+ }
if (type == STREAM_INPUT__MERGED_SUBMIT) {
// ASSERT(numOfBlocks > 1);
@@ -93,6 +97,8 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
}
}
+static FORCE_INLINE void streamInputBlockDataDestory(void* pBlock) { blockDataDestroy((SSDataBlock*)pBlock); }
+
void tdCleanupStreamInputDataBlock(qTaskInfo_t tinfo) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
if (!pTaskInfo || !pTaskInfo->pRoot || pTaskInfo->pRoot->numOfDownstream <= 0) {
@@ -103,11 +109,7 @@ void tdCleanupStreamInputDataBlock(qTaskInfo_t tinfo) {
if (pOptrInfo->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
SStreamScanInfo* pInfo = pOptrInfo->info;
if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) {
- for (int32_t i = 0; i < taosArrayGetSize(pInfo->pBlockLists); ++i) {
- SSDataBlock* p = *(SSDataBlock**)taosArrayGet(pInfo->pBlockLists, i);
- taosArrayDestroy(p->pDataBlock);
- taosMemoryFreeClear(p);
- }
+ taosArrayClearP(pInfo->pBlockLists, streamInputBlockDataDestory);
} else {
ASSERT(0);
}
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 3cb4ea8702..815d6a71ef 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -1866,13 +1866,11 @@ void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t
pOperator->resultInfo.totalRows += numOfRows;
}
-int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, int32_t numOfOutput, SArray* pColList,
- char** pNextStart) {
+int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart) {
if (pColList == NULL) { // data from other sources
blockDataCleanup(pRes);
*pNextStart = (char*)blockDecode(pRes, pData);
} else { // extract data according to pColList
- ASSERT(numOfOutput == taosArrayGetSize(pColList));
char* pStart = pData;
int32_t numOfCols = htonl(*(int32_t*)pStart);
@@ -1970,7 +1968,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
char* pStart = pRetrieveRsp->data;
while (index++ < pRetrieveRsp->numOfBlocks) {
SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false);
- code = extractDataBlockFromFetchRsp(pb, pStart, pRetrieveRsp->numOfCols, NULL, &pStart);
+ code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart);
if (code != 0) {
taosMemoryFreeClear(pDataInfo->pRsp);
goto _error;
@@ -2095,7 +2093,7 @@ static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) {
SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp;
char* pStart = pRetrieveRsp->data;
- int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, pRetrieveRsp->numOfCols, NULL, &pStart);
+ int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, NULL, &pStart);
if (pRsp->completed == 1) {
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64
@@ -3441,6 +3439,44 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t
}
}
+static bool isWstartColumnExist(SFillOperatorInfo* pInfo) {
+ if (pInfo->numOfNotFillExpr == 0) {
+ return false;
+ }
+ for (int32_t i = 0; i < pInfo->numOfNotFillExpr; ++i) {
+ SExprInfo* exprInfo = pInfo->pNotFillExprInfo + i;
+ if (exprInfo->pExpr->nodeType == QUERY_NODE_COLUMN &&
+ exprInfo->base.numOfParams == 1 &&
+ exprInfo->base.pParam[0].pCol->colType == COLUMN_TYPE_WINDOW_START) {
+ return true;
+ }
+ }
+ return false;
+}
+
+static int32_t createWStartTsAsNotFillExpr(SFillOperatorInfo* pInfo, SFillPhysiNode* pPhyFillNode) {
+ bool wstartExist = isWstartColumnExist(pInfo);
+ if (wstartExist == false) {
+ if (pPhyFillNode->pWStartTs->type != QUERY_NODE_TARGET) {
+ qError("pWStartTs of fill physical node is not a target node");
+ return TSDB_CODE_QRY_SYS_ERROR;
+ }
+
+ SExprInfo* notFillExprs = taosMemoryRealloc(pInfo->pNotFillExprInfo, (pInfo->numOfNotFillExpr + 1) * sizeof(SExprInfo));
+ if (notFillExprs == NULL) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
+ createExprFromTargetNode(notFillExprs + pInfo->numOfNotFillExpr, (STargetNode*)pPhyFillNode->pWStartTs);
+
+ ++pInfo->numOfNotFillExpr;
+ pInfo->pNotFillExprInfo = notFillExprs;
+ return TSDB_CODE_SUCCESS;
+ }
+
+ return TSDB_CODE_SUCCESS;
+}
+
SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode,
SExecTaskInfo* pTaskInfo) {
SFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SFillOperatorInfo));
@@ -3452,7 +3488,10 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
SSDataBlock* pResBlock = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc);
SExprInfo* pExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &pInfo->numOfExpr);
pInfo->pNotFillExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &pInfo->numOfNotFillExpr);
-
+ int32_t code = createWStartTsAsNotFillExpr(pInfo, pPhyFillNode);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _error;
+ }
SInterval* pInterval =
QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == downstream->operatorType
? &((SMergeAlignedIntervalAggOperatorInfo*)downstream->info)->intervalAggOperatorInfo->interval
@@ -3473,7 +3512,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
SArray* pColMatchColInfo = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc,
&numOfOutputCols, COL_MATCH_FROM_SLOT_ID);
- int32_t code = initFillInfo(pInfo, pExprInfo, pInfo->numOfExpr, pInfo->pNotFillExprInfo, pInfo->numOfNotFillExpr,
+ code = initFillInfo(pInfo, pExprInfo, pInfo->numOfExpr, pInfo->pNotFillExprInfo, pInfo->numOfNotFillExpr,
(SNodeListNode*)pPhyFillNode->pValues, pPhyFillNode->timeRange, pResultInfo->capacity,
pTaskInfo->id.str, pInterval, type, order);
if (code != TSDB_CODE_SUCCESS) {
@@ -3915,7 +3954,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
pOptr = createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pAggNode->node.pConditions,
pScalarExprInfo, numOfScalarExpr, pAggNode->mergeDataBlock, pTaskInfo);
}
- } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) {
+ } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type) {
SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num);
@@ -3940,6 +3979,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pIntervalPhyNode,
pTaskInfo, isStream);
+ } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) {
+ pOptr = createStreamIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) {
SMergeAlignedIntervalPhysiNode* pIntervalPhyNode = (SMergeAlignedIntervalPhysiNode*)pPhyNode;
pOptr = createMergeAlignedIntervalOperatorInfo(ops[0], pIntervalPhyNode, pTaskInfo);
@@ -4410,3 +4451,108 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlF
return code;
}
+
+int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
+ int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup,
+ SExecTaskInfo* pTaskInfo) {
+ SWinKey key = {
+ .ts = win->skey,
+ .groupId = tableGroupId,
+ };
+ char* value = NULL;
+ int32_t size = pAggSup->resultRowSize;
+ if (streamStateAddIfNotExist(pTaskInfo->streamInfo.pState, &key, (void**)&value, &size) < 0) {
+ return TSDB_CODE_QRY_OUT_OF_MEMORY;
+ }
+ *pResult = (SResultRow*)value;
+ ASSERT(*pResult);
+ // set time window for current result
+ (*pResult)->win = (*win);
+ setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset);
+ return TSDB_CODE_SUCCESS;
+}
+
+int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult) {
+ streamStateReleaseBuf(pTaskInfo->streamInfo.pState, pKey, pResult);
+ return TSDB_CODE_SUCCESS;
+}
+
+int32_t saveOutput(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize) {
+ streamStatePut(pTaskInfo->streamInfo.pState, pKey, pResult, resSize);
+ return TSDB_CODE_SUCCESS;
+}
+
+int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup,
+ SGroupResInfo* pGroupResInfo) {
+ SExprInfo* pExprInfo = pSup->pExprInfo;
+ int32_t numOfExprs = pSup->numOfExprs;
+ int32_t* rowEntryOffset = pSup->rowEntryInfoOffset;
+ SqlFunctionCtx* pCtx = pSup->pCtx;
+
+ int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);
+
+ for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) {
+ SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i);
+ int32_t size = 0;
+ void* pVal = NULL;
+ SWinKey key = {
+ .ts = *(TSKEY*)pPos->key,
+ .groupId = pPos->groupId,
+ };
+ int32_t code = streamStateGet(pTaskInfo->streamInfo.pState, &key, &pVal, &size);
+ ASSERT(code == 0);
+ SResultRow* pRow = (SResultRow*)pVal;
+ doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset);
+ // no results, continue to check the next one
+ if (pRow->numOfRows == 0) {
+ pGroupResInfo->index += 1;
+ releaseOutputBuf(pTaskInfo, &key, pRow);
+ continue;
+ }
+
+ if (pBlock->info.groupId == 0) {
+ pBlock->info.groupId = pPos->groupId;
+ } else {
+ // current value belongs to different group, it can't be packed into one datablock
+ if (pBlock->info.groupId != pPos->groupId) {
+ releaseOutputBuf(pTaskInfo, &key, pRow);
+ break;
+ }
+ }
+
+ if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {
+ ASSERT(pBlock->info.rows > 0);
+ releaseOutputBuf(pTaskInfo, &key, pRow);
+ break;
+ }
+
+ pGroupResInfo->index += 1;
+
+ for (int32_t j = 0; j < numOfExprs; ++j) {
+ int32_t slotId = pExprInfo[j].base.resSchema.slotId;
+
+ pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset);
+ if (pCtx[j].fpSet.finalize) {
+ int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock);
+ if (TAOS_FAILED(code)) {
+ qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code));
+ T_LONG_JMP(pTaskInfo->env, code);
+ }
+ } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) {
+ // do nothing, todo refactor
+ } else {
+ // expand the result into multiple rows. E.g., _wstart, top(k, 20)
+ // the _wstart needs to copy to 20 following rows, since the results of top-k expands to 20 different rows.
+ SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);
+ char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo);
+ for (int32_t k = 0; k < pRow->numOfRows; ++k) {
+ colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes);
+ }
+ }
+ }
+ releaseOutputBuf(pTaskInfo, &key, pRow);
+ pBlock->info.rows += pRow->numOfRows;
+ }
+ blockDataUpdateTsWindow(pBlock, 0);
+ return TSDB_CODE_SUCCESS;
+}
diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c
index 599edb0722..5eb6557dbd 100644
--- a/source/libs/executor/src/groupoperator.c
+++ b/source/libs/executor/src/groupoperator.c
@@ -13,26 +13,26 @@
* along with this program. If not, see
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see
+#include
+#include
+#include
+#include
+#include "index.h"
+#include "indexCache.h"
+#include "indexFst.h"
+#include "indexFstUtil.h"
+#include "indexInt.h"
+#include "indexTfile.h"
+#include "indexUtil.h"
+#include "tskiplist.h"
+#include "tutil.h"
+using namespace std;
+
+static std::string logDir = TD_TMP_DIR_PATH "log";
+
+static void initLog() {
+ const char *defaultLogFileNamePrefix = "taoslog";
+ const int32_t maxLogFileNum = 10;
+
+ tsAsyncLog = 0;
+ idxDebugFlag = 143;
+ strcpy(tsLogDir, logDir.c_str());
+ taosRemoveDir(tsLogDir);
+ taosMkDir(tsLogDir);
+
+ if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
+ printf("failed to open log file in directory:%s\n", tsLogDir);
+ }
+}
+
+struct WriteBatch {
+ SIndexMultiTerm *terms;
+};
+class Idx {
+ public:
+ Idx(int _cacheSize = 1024 * 1024 * 4, const char *_path = "tindex") {
+ opts.cacheSize = _cacheSize;
+ path += TD_TMP_DIR_PATH;
+ path += _path;
+ }
+ int SetUp(bool remove) {
+ initLog();
+
+ if (remove) taosRemoveDir(path.c_str());
+
+ int ret = indexJsonOpen(&opts, path.c_str(), &index);
+ return ret;
+ }
+ int Write(WriteBatch *batch, uint64_t uid) {
+ // write batch
+ indexJsonPut(index, batch->terms, uid);
+ return 0;
+ }
+ int Read(const char *json, void *key, int64_t *id) {
+ // read batch
+ return 0;
+ }
+
+ void TearDown() { indexJsonClose(index); }
+
+ std::string path;
+
+ SIndexOpts opts;
+ SIndex *index;
+};
+
+SIndexTerm *indexTermCreateT(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName,
+ int32_t nColName, const char *colVal, int32_t nColVal) {
+ char buf[256] = {0};
+ int16_t sz = nColVal;
+ memcpy(buf, (uint16_t *)&sz, 2);
+ memcpy(buf + 2, colVal, nColVal);
+ if (colType == TSDB_DATA_TYPE_BINARY) {
+ return indexTermCreate(suid, oper, colType, colName, nColName, buf, sizeof(buf));
+ } else {
+ return indexTermCreate(suid, oper, colType, colName, nColName, colVal, nColVal);
+ }
+ return NULL;
+}
+int initWriteBatch(WriteBatch *wb, int batchSize) {
+ SIndexMultiTerm *terms = indexMultiTermCreate();
+
+ std::string colName;
+ std::string colVal;
+
+ for (int i = 0; i < 64; i++) {
+ colName += '0' + i;
+ colVal += '0' + i;
+ }
+
+ for (int i = 0; i < batchSize; i++) {
+ colVal[i % colVal.size()] = '0' + i % 128;
+ colName[i % colName.size()] = '0' + i % 128;
+ SIndexTerm *term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ colVal.c_str(), colVal.size());
+ indexMultiTermAdd(terms, term);
+ }
+
+ wb->terms = terms;
+ return 0;
+}
+
+int BenchWrite(Idx *idx, int batchSize, int limit) {
+ for (int i = 0; i < limit; i += batchSize) {
+ WriteBatch wb;
+ idx->Write(&wb, i);
+ }
+ return 0;
+}
+
+int BenchRead(Idx *idx) { return 0; }
+
+int main() {
+ // Idx *idx = new Idx;
+ // if (idx->SetUp(true) != 0) {
+ // std::cout << "failed to setup index" << std::endl;
+ // return 0;
+ // } else {
+ // std::cout << "succ to setup index" << std::endl;
+ // }
+ // BenchWrite(idx, 100, 10000);
+ return 1;
+}
diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc
index 5b76de2ef8..08bf84ff60 100644
--- a/source/libs/index/test/indexTests.cc
+++ b/source/libs/index/test/indexTests.cc
@@ -271,20 +271,20 @@ void validateFst() {
}
delete m;
}
+
static std::string logDir = TD_TMP_DIR_PATH "log";
+static void initLog() {
+ const char* defaultLogFileNamePrefix = "taoslog";
+ const int32_t maxLogFileNum = 10;
-static void initLog() {
- const char* defaultLogFileNamePrefix = "taoslog";
- const int32_t maxLogFileNum = 10;
+ tsAsyncLog = 0;
+ idxDebugFlag = 143;
+ strcpy(tsLogDir, logDir.c_str());
+ taosRemoveDir(tsLogDir);
+ taosMkDir(tsLogDir);
- tsAsyncLog = 0;
- idxDebugFlag = 143;
- strcpy(tsLogDir, logDir.c_str());
- taosRemoveDir(tsLogDir);
- taosMkDir(tsLogDir);
-
- if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
- printf("failed to open log file in directory:%s\n", tsLogDir);
+ if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
+ printf("failed to open log file in directory:%s\n", tsLogDir);
}
}
class IndexEnv : public ::testing::Test {
diff --git a/source/libs/monitor/src/monMsg.c b/source/libs/monitor/src/monMsg.c
index 8fa7e88605..bbee8b1166 100644
--- a/source/libs/monitor/src/monMsg.c
+++ b/source/libs/monitor/src/monMsg.c
@@ -510,6 +510,7 @@ int32_t tSerializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo)
SVnodeLoad *pLoad = taosArrayGet(pInfo->pVloads, i);
if (tEncodeI32(&encoder, pLoad->vgId) < 0) return -1;
if (tEncodeI32(&encoder, pLoad->syncState) < 0) return -1;
+ if (tEncodeI64(&encoder, pLoad->cacheUsage) < 0) return -1;
if (tEncodeI64(&encoder, pLoad->numOfTables) < 0) return -1;
if (tEncodeI64(&encoder, pLoad->numOfTimeSeries) < 0) return -1;
if (tEncodeI64(&encoder, pLoad->totalStorage) < 0) return -1;
@@ -544,6 +545,7 @@ int32_t tDeserializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInf
SVnodeLoad load = {0};
if (tDecodeI32(&decoder, &load.vgId) < 0) return -1;
if (tDecodeI32(&decoder, &load.syncState) < 0) return -1;
+ if (tDecodeI64(&decoder, &load.cacheUsage) < 0) return -1;
if (tDecodeI64(&decoder, &load.numOfTables) < 0) return -1;
if (tDecodeI64(&decoder, &load.numOfTimeSeries) < 0) return -1;
if (tDecodeI64(&decoder, &load.totalStorage) < 0) return -1;
@@ -594,7 +596,6 @@ int32_t tDeserializeSMonMloadInfo(void *buf, int32_t bufLen, SMonMloadInfo *pInf
return 0;
}
-
int32_t tSerializeSQnodeLoad(void *buf, int32_t bufLen, SQnodeLoad *pInfo) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
@@ -639,5 +640,3 @@ int32_t tDeserializeSQnodeLoad(void *buf, int32_t bufLen, SQnodeLoad *pInfo) {
tDecoderClear(&decoder);
return 0;
}
-
-
diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c
index 8976daadbd..a41462e3fd 100644
--- a/source/libs/nodes/src/nodesCodeFuncs.c
+++ b/source/libs/nodes/src/nodesCodeFuncs.c
@@ -362,18 +362,14 @@ static int32_t jsonToTableComInfo(const SJson* pJson, void* pObj) {
int32_t code;
tjsonGetNumberValue(pJson, jkTableComInfoNumOfTags, pNode->numOfTags, code);
- ;
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableComInfoPrecision, pNode->precision, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableComInfoNumOfColumns, pNode->numOfColumns, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableComInfoRowSize, pNode->rowSize, code);
- ;
}
return code;
@@ -406,14 +402,11 @@ static int32_t jsonToSchema(const SJson* pJson, void* pObj) {
int32_t code;
tjsonGetNumberValue(pJson, jkSchemaType, pNode->type, code);
- ;
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkSchemaColId, pNode->colId, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkSchemaBytes, pNode->bytes, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkSchemaName, pNode->name);
@@ -466,26 +459,20 @@ static int32_t jsonToTableMeta(const SJson* pJson, void* pObj) {
int32_t code;
tjsonGetNumberValue(pJson, jkTableMetaVgId, pNode->vgId, code);
- ;
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableMetaTableType, pNode->tableType, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableMetaUid, pNode->uid, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableMetaSuid, pNode->suid, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableMetaSversion, pNode->sversion, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableMetaTversion, pNode->tversion, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonToObject(pJson, jkTableMetaComInfo, jsonToTableComInfo, &pNode->tableInfo);
@@ -926,7 +913,6 @@ static int32_t jsonToLogicFillNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToLogicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkFillLogicPlanMode, pNode->mode, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkFillLogicPlanWStartTs, &pNode->pWStartTs);
@@ -2815,7 +2801,6 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) {
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkColumnColType, pNode->colType, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkColumnDbName, pNode->dbName);
@@ -3118,7 +3103,6 @@ static int32_t jsonToOperatorNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToExprNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkOperatorType, pNode->opType, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkOperatorLeft, &pNode->pLeft);
@@ -3153,7 +3137,6 @@ static int32_t jsonToLogicConditionNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToExprNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkLogicCondType, pNode->condType, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkLogicCondParameters, &pNode->pParameterList);
@@ -3442,11 +3425,9 @@ static int32_t jsonToOrderByExprNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToNodeObject(pJson, jkOrderByExprExpr, &pNode->pExpr);
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkOrderByExprOrder, pNode->order, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkOrderByExprNullOrder, pNode->nullOrder, code);
- ;
}
return code;
@@ -3624,7 +3605,6 @@ static int32_t jsonToFillNode(const SJson* pJson, void* pObj) {
int32_t code;
tjsonGetNumberValue(pJson, jkFillMode, pNode->mode, code);
- ;
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkFillValues, &pNode->pValues);
}
diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c
new file mode 100644
index 0000000000..5fe31ed78e
--- /dev/null
+++ b/source/libs/nodes/src/nodesMsgFuncs.c
@@ -0,0 +1,3108 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see
#include
+#include
#include "cmdnodes.h"
#include "mockCatalogService.h"
@@ -251,6 +252,7 @@ class PlannerTestBaseImpl {
string splitLogicPlan_;
string scaledLogicPlan_;
string physiPlan_;
+ string physiPlanMsg_;
vector physiSubplans_;
};
@@ -274,6 +276,7 @@ class PlannerTestBaseImpl {
res_.splitLogicPlan_.clear();
res_.scaledLogicPlan_.clear();
res_.physiPlan_.clear();
+ res_.physiPlanMsg_.clear();
res_.physiSubplans_.clear();
}
@@ -408,6 +411,8 @@ class PlannerTestBaseImpl {
SNode* pSubplan;
FOREACH(pSubplan, ((SNodeListNode*)pNode)->pNodeList) { res_.physiSubplans_.push_back(toString(pSubplan)); }
}
+ res_.physiPlanMsg_ = toMsg((SNode*)(*pPlan));
+ cout << "json len: " << res_.physiPlan_.length() << ", msg len: " << res_.physiPlanMsg_.length() << endl;
}
void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) {
@@ -446,12 +451,45 @@ class PlannerTestBaseImpl {
string toString(const SNode* pRoot) {
char* pStr = NULL;
int32_t len = 0;
+
+ auto start = chrono::steady_clock::now();
DO_WITH_THROW(nodesNodeToString, pRoot, false, &pStr, &len)
+ if (QUERY_NODE_PHYSICAL_PLAN == nodeType(pRoot)) {
+ cout << "nodesNodeToString: "
+ << chrono::duration_cast(chrono::steady_clock::now() - start).count() << "us" << endl;
+ }
+
string str(pStr);
taosMemoryFreeClear(pStr);
return str;
}
+ string toMsg(const SNode* pRoot) {
+ char* pStr = NULL;
+ int32_t len = 0;
+
+ auto start = chrono::steady_clock::now();
+ DO_WITH_THROW(nodesNodeToMsg, pRoot, &pStr, &len)
+ cout << "nodesNodeToMsg: "
+ << chrono::duration_cast(chrono::steady_clock::now() - start).count() << "us" << endl;
+
+ SNode* pNode = NULL;
+ char* pNewStr = NULL;
+ int32_t newlen = 0;
+ DO_WITH_THROW(nodesMsgToNode, pStr, len, &pNode)
+ DO_WITH_THROW(nodesNodeToMsg, pNode, &pNewStr, &newlen)
+ if (newlen != len || 0 != memcmp(pStr, pNewStr, len)) {
+ cout << "nodesNodeToMsg error!!!!!!!!!!!!!! len = " << len << ", newlen = " << newlen << endl;
+ DO_WITH_THROW(nodesNodeToString, pNode, false, &pNewStr, &newlen)
+ cout << "nodesNodeToString " << pNewStr << endl;
+ }
+ taosMemoryFreeClear(pNewStr);
+
+ string str(pStr, len);
+ taosMemoryFreeClear(pStr);
+ return str;
+ }
+
caseEnv caseEnv_;
stmtEnv stmtEnv_;
stmtRes res_;
diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c
index e2d3ac1583..e54937114c 100644
--- a/source/libs/qcom/src/querymsg.c
+++ b/source/libs/qcom/src/querymsg.c
@@ -38,6 +38,8 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) {
pOut->dbVgroup->vgVersion = usedbRsp->vgVersion;
pOut->dbVgroup->hashMethod = usedbRsp->hashMethod;
+ pOut->dbVgroup->hashPrefix = usedbRsp->hashPrefix;
+ pOut->dbVgroup->hashSuffix = usedbRsp->hashSuffix;
qDebug("Got %d vgroup for db %s", usedbRsp->vgNum, usedbRsp->db);
diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c
index 9cab39c301..c5f161b66a 100644
--- a/source/libs/scheduler/src/schTask.c
+++ b/source/libs/scheduler/src/schTask.c
@@ -52,7 +52,7 @@ void schInitTaskRetryTimes(SSchJob *pJob, SSchTask *pTask, SSchLevel *pLevel) {
int32_t nodeNum = taosArrayGetSize(pJob->nodeList);
pTask->maxRetryTimes = TMAX(nodeNum, SCH_DEFAULT_MAX_RETRY_NUM);
}
-
+
pTask->maxExecTimes = pTask->maxRetryTimes * (pLevel->level + 1);
}
@@ -139,13 +139,15 @@ int32_t schUpdateTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle, int3
}
if ((execId != pTask->execId) || pTask->waitRetry) { // ignore it
- SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId, pTask->execId, pTask->waitRetry);
+ SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId,
+ pTask->execId, pTask->waitRetry);
return TSDB_CODE_SUCCESS;
}
SSchNodeInfo *nodeInfo = taosHashGet(pTask->execNodes, &execId, sizeof(execId));
if (NULL == nodeInfo) { // ignore it
- SCH_TASK_DLOG("handle not updated since execId %d already not exist, current execId %d, waitRetry %d", execId, pTask->execId, pTask->waitRetry);
+ SCH_TASK_DLOG("handle not updated since execId %d already not exist, current execId %d, waitRetry %d", execId,
+ pTask->execId, pTask->waitRetry);
return TSDB_CODE_SUCCESS;
}
@@ -314,7 +316,7 @@ int32_t schRescheduleTask(SSchJob *pJob, SSchTask *pTask) {
if (!schMgmt.cfg.enableReSchedule) {
return TSDB_CODE_SUCCESS;
}
-
+
if (SCH_IS_DATA_BIND_TASK(pTask)) {
return TSDB_CODE_SUCCESS;
}
@@ -341,7 +343,8 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32
}
if (((pTask->execId + 1) >= pTask->maxExecTimes) || ((pTask->retryTimes + 1) > pTask->maxRetryTimes)) {
- SCH_TASK_DLOG("task no more retry since reach max times %d:%d, execId %d", pTask->maxRetryTimes, pTask->maxExecTimes, pTask->execId);
+ SCH_TASK_DLOG("task no more retry since reach max times %d:%d, execId %d", pTask->maxRetryTimes,
+ pTask->maxExecTimes, pTask->execId);
schHandleJobFailure(pJob, rspCode);
return TSDB_CODE_SUCCESS;
}
@@ -548,7 +551,8 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo
if ((pTask->retryTimes + 1) > pTask->maxRetryTimes) {
*needRetry = false;
- SCH_TASK_DLOG("task no more retry since reach max retry times, retryTimes:%d/%d", pTask->retryTimes, pTask->maxRetryTimes);
+ SCH_TASK_DLOG("task no more retry since reach max retry times, retryTimes:%d/%d", pTask->retryTimes,
+ pTask->maxRetryTimes);
return TSDB_CODE_SUCCESS;
}
@@ -564,25 +568,25 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo
return TSDB_CODE_SUCCESS;
}
-/*
- if (SCH_IS_DATA_BIND_TASK(pTask)) {
- if ((pTask->execId + 1) >= SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)) {
- *needRetry = false;
- SCH_TASK_DLOG("task no more retry since all ep tried, execId:%d, epNum:%d", pTask->execId,
- SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode));
- return TSDB_CODE_SUCCESS;
- }
- } else {
- int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs);
+ /*
+ if (SCH_IS_DATA_BIND_TASK(pTask)) {
+ if ((pTask->execId + 1) >= SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)) {
+ *needRetry = false;
+ SCH_TASK_DLOG("task no more retry since all ep tried, execId:%d, epNum:%d", pTask->execId,
+ SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode));
+ return TSDB_CODE_SUCCESS;
+ }
+ } else {
+ int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs);
- if ((pTask->candidateIdx + 1) >= candidateNum && (TSDB_CODE_SCH_TIMEOUT_ERROR != errCode)) {
- *needRetry = false;
- SCH_TASK_DLOG("task no more retry since all candiates tried, candidateIdx:%d, candidateNum:%d",
- pTask->candidateIdx, candidateNum);
- return TSDB_CODE_SUCCESS;
+ if ((pTask->candidateIdx + 1) >= candidateNum && (TSDB_CODE_SCH_TIMEOUT_ERROR != errCode)) {
+ *needRetry = false;
+ SCH_TASK_DLOG("task no more retry since all candiates tried, candidateIdx:%d, candidateNum:%d",
+ pTask->candidateIdx, candidateNum);
+ return TSDB_CODE_SUCCESS;
+ }
}
- }
-*/
+ */
*needRetry = true;
SCH_TASK_DLOG("task need the %dth retry, errCode:%x - %s", pTask->execId + 1, errCode, tstrerror(errCode));
@@ -630,8 +634,9 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) {
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
- SCH_TASK_TLOG("set %dth candidate addr, id %d, inUse:%d/%d, fqdn:%s, port:%d", i, naddr->nodeId, naddr->epSet.inUse, naddr->epSet.numOfEps,
- SCH_GET_CUR_EP(naddr)->fqdn, SCH_GET_CUR_EP(naddr)->port);
+ SCH_TASK_TLOG("set %dth candidate addr, id %d, inUse:%d/%d, fqdn:%s, port:%d", i, naddr->nodeId,
+ naddr->epSet.inUse, naddr->epSet.numOfEps, SCH_GET_CUR_EP(naddr)->fqdn,
+ SCH_GET_CUR_EP(naddr)->port);
++addNum;
}
@@ -711,10 +716,10 @@ int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask) {
if (candidateNum <= 1) {
goto _return;
}
-
+
switch (schMgmt.cfg.schPolicy) {
case SCH_LOAD_SEQ:
- case SCH_ALL:
+ case SCH_ALL:
default:
if (++pTask->candidateIdx >= candidateNum) {
pTask->candidateIdx = 0;
@@ -732,7 +737,7 @@ int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask) {
_return:
SCH_TASK_DLOG("switch task candiateIdx to %d/%d", pTask->candidateIdx, candidateNum);
-
+
return TSDB_CODE_SUCCESS;
}
@@ -759,7 +764,7 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) {
return;
}
- int32_t i = 0;
+ int32_t i = 0;
SSchNodeInfo *nodeInfo = taosHashIterate(pTask->execNodes, NULL);
while (nodeInfo) {
if (nodeInfo->handle) {
@@ -821,16 +826,16 @@ int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList) {
int32_t schLaunchTaskImpl(void *param) {
SSchTaskCtx *pCtx = (SSchTaskCtx *)param;
- SSchJob *pJob = schAcquireJob(pCtx->jobRid);
+ SSchJob *pJob = schAcquireJob(pCtx->jobRid);
if (NULL == pJob) {
- taosMemoryFree(param);
qDebug("job refId 0x%" PRIx64 " already not exist", pCtx->jobRid);
+ taosMemoryFree(param);
SCH_RET(TSDB_CODE_SCH_JOB_IS_DROPPING);
}
-
+
SSchTask *pTask = pCtx->pTask;
- int8_t status = 0;
- int32_t code = 0;
+ int8_t status = 0;
+ int32_t code = 0;
atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1);
pTask->execId++;
@@ -891,13 +896,12 @@ _return:
SCH_RET(code);
}
-int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) {
-
+int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) {
SSchTaskCtx *param = taosMemoryCalloc(1, sizeof(SSchTaskCtx));
if (NULL == param) {
SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
-
+
param->jobRid = pJob->refId;
param->pTask = pTask;
@@ -906,7 +910,7 @@ int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) {
} else {
SCH_ERR_RET(schLaunchTaskImpl(param));
}
-
+
return TSDB_CODE_SUCCESS;
}
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index 9d4010f60e..a2a45938e4 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -243,6 +243,36 @@ FAIL:
return 0;
}
+int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, SSDataBlock* pDataBlock, int32_t vgSz,
+ int64_t groupId) {
+ char* ctbName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, groupId);
+ SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
+
+ // TODO: get hash function by hashMethod
+ uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));
+ taosMemoryFree(ctbName);
+ bool found = false;
+ // TODO: optimize search
+ int32_t j;
+ for (j = 0; j < vgSz; j++) {
+ SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j);
+ ASSERT(pVgInfo->vgId > 0);
+ if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) {
+ if (streamAddBlockToDispatchMsg(pDataBlock, &pReqs[j]) < 0) {
+ return -1;
+ }
+ if (pReqs[j].blockNum == 0) {
+ atomic_add_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1);
+ }
+ pReqs[j].blockNum++;
+ found = true;
+ break;
+ }
+ }
+ ASSERT(found);
+ return 0;
+}
+
int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pData) {
int32_t code = -1;
int32_t blockNum = taosArrayGetSize(pData->blocks);
@@ -317,20 +347,10 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat
for (int32_t i = 0; i < blockNum; i++) {
SSDataBlock* pDataBlock = taosArrayGet(pData->blocks, i);
- char* ctbName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, pDataBlock->info.groupId);
- // TODO: get hash function by hashMethod
- uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));
-
- taosMemoryFree(ctbName);
-
- bool found = false;
- // TODO: optimize search
- int32_t j;
- for (j = 0; j < vgSz; j++) {
- SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j);
- ASSERT(pVgInfo->vgId > 0);
- if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) {
+ // TODO: do not use broadcast
+ if (pDataBlock->info.type == STREAM_DELETE_RESULT) {
+ for (int32_t j = 0; j < vgSz; j++) {
if (streamAddBlockToDispatchMsg(pDataBlock, &pReqs[j]) < 0) {
goto FAIL_SHUFFLE_DISPATCH;
}
@@ -338,11 +358,13 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat
atomic_add_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1);
}
pReqs[j].blockNum++;
- found = true;
- break;
}
+ continue;
+ }
+
+ if (streamSearchAndAddBlock(pTask, pReqs, pDataBlock, vgSz, pDataBlock->info.groupId) < 0) {
+ goto FAIL_SHUFFLE_DISPATCH;
}
- ASSERT(found);
}
for (int32_t i = 0; i < vgSz; i++) {
diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c
index dfd6f012cc..5efdbb4679 100644
--- a/source/libs/stream/src/streamState.c
+++ b/source/libs/stream/src/streamState.c
@@ -112,6 +112,29 @@ int32_t streamStateDel(SStreamState* pState, const SWinKey* key) {
return tdbTbDelete(pState->pStateDb, key, sizeof(SWinKey), &pState->txn);
}
+int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) {
+ // todo refactor
+ int32_t size = *pVLen;
+ if (streamStateGet(pState, key, pVal, pVLen) == 0) {
+ return 0;
+ }
+ void* tmp = taosMemoryCalloc(1, size);
+ if (streamStatePut(pState, key, &tmp, size) == 0) {
+ taosMemoryFree(tmp);
+ int32_t code = streamStateGet(pState, key, pVal, pVLen);
+ ASSERT(code == 0);
+ return code;
+ }
+ taosMemoryFree(tmp);
+ return -1;
+}
+
+int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal) {
+ // todo refactor
+ streamFreeVal(pVal);
+ return 0;
+}
+
SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key) {
SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));
if (pCur == NULL) return NULL;
diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index 51098374b0..6f29b54f80 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -2766,8 +2766,6 @@ const char* syncStr(ESyncState state) {
}
int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) {
- SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg);
-
if (ths->state != TAOS_SYNC_STATE_FOLLOWER) {
syncNodeEventLog(ths, "I am not follower, can not do leader transfer");
return 0;
@@ -2799,6 +2797,8 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p
}
*/
+ SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg);
+
do {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "do leader transfer, index:%ld", pEntry->index);
diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c
index b42aba560f..faebe5bbec 100644
--- a/source/libs/sync/src/syncMessage.c
+++ b/source/libs/sync/src/syncMessage.c
@@ -1992,6 +1992,313 @@ void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg) {
}
}
+// ---- message process SyncHeartbeat----
+SyncHeartbeat* syncHeartbeatBuild(int32_t vgId) {
+ uint32_t bytes = sizeof(SyncHeartbeat);
+ SyncHeartbeat* pMsg = taosMemoryMalloc(bytes);
+ memset(pMsg, 0, bytes);
+ pMsg->bytes = bytes;
+ pMsg->vgId = vgId;
+ pMsg->msgType = TDMT_SYNC_HEARTBEAT;
+ return pMsg;
+}
+
+void syncHeartbeatDestroy(SyncHeartbeat* pMsg) {
+ if (pMsg != NULL) {
+ taosMemoryFree(pMsg);
+ }
+}
+
+void syncHeartbeatSerialize(const SyncHeartbeat* pMsg, char* buf, uint32_t bufLen) {
+ ASSERT(pMsg->bytes <= bufLen);
+ memcpy(buf, pMsg, pMsg->bytes);
+}
+
+void syncHeartbeatDeserialize(const char* buf, uint32_t len, SyncHeartbeat* pMsg) {
+ memcpy(pMsg, buf, len);
+ ASSERT(len == pMsg->bytes);
+}
+
+char* syncHeartbeatSerialize2(const SyncHeartbeat* pMsg, uint32_t* len) {
+ char* buf = taosMemoryMalloc(pMsg->bytes);
+ ASSERT(buf != NULL);
+ syncHeartbeatSerialize(pMsg, buf, pMsg->bytes);
+ if (len != NULL) {
+ *len = pMsg->bytes;
+ }
+ return buf;
+}
+
+SyncHeartbeat* syncHeartbeatDeserialize2(const char* buf, uint32_t len) {
+ uint32_t bytes = *((uint32_t*)buf);
+ SyncHeartbeat* pMsg = taosMemoryMalloc(bytes);
+ ASSERT(pMsg != NULL);
+ syncHeartbeatDeserialize(buf, len, pMsg);
+ ASSERT(len == pMsg->bytes);
+ return pMsg;
+}
+
+void syncHeartbeat2RpcMsg(const SyncHeartbeat* pMsg, SRpcMsg* pRpcMsg) {
+ memset(pRpcMsg, 0, sizeof(*pRpcMsg));
+ pRpcMsg->msgType = pMsg->msgType;
+ pRpcMsg->contLen = pMsg->bytes;
+ pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen);
+ syncHeartbeatSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen);
+}
+
+void syncHeartbeatFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeat* pMsg) {
+ syncHeartbeatDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg);
+}
+
+SyncHeartbeat* syncHeartbeatFromRpcMsg2(const SRpcMsg* pRpcMsg) {
+ SyncHeartbeat* pMsg = syncHeartbeatDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
+ ASSERT(pMsg != NULL);
+ return pMsg;
+}
+
+cJSON* syncHeartbeat2Json(const SyncHeartbeat* pMsg) {
+ char u64buf[128] = {0};
+ cJSON* pRoot = cJSON_CreateObject();
+
+ if (pMsg != NULL) {
+ cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
+ cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId);
+ cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
+
+ cJSON* pSrcId = cJSON_CreateObject();
+ snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr);
+ cJSON_AddStringToObject(pSrcId, "addr", u64buf);
+ {
+ uint64_t u64 = pMsg->srcId.addr;
+ cJSON* pTmp = pSrcId;
+ char host[128] = {0};
+ uint16_t port;
+ syncUtilU642Addr(u64, host, sizeof(host), &port);
+ cJSON_AddStringToObject(pTmp, "addr_host", host);
+ cJSON_AddNumberToObject(pTmp, "addr_port", port);
+ }
+ cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId);
+ cJSON_AddItemToObject(pRoot, "srcId", pSrcId);
+
+ cJSON* pDestId = cJSON_CreateObject();
+ snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr);
+ cJSON_AddStringToObject(pDestId, "addr", u64buf);
+ {
+ uint64_t u64 = pMsg->destId.addr;
+ cJSON* pTmp = pDestId;
+ char host[128] = {0};
+ uint16_t port;
+ syncUtilU642Addr(u64, host, sizeof(host), &port);
+ cJSON_AddStringToObject(pTmp, "addr_host", host);
+ cJSON_AddNumberToObject(pTmp, "addr_port", port);
+ }
+ cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId);
+ cJSON_AddItemToObject(pRoot, "destId", pDestId);
+
+ snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term);
+ cJSON_AddStringToObject(pRoot, "term", u64buf);
+
+ snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm);
+ cJSON_AddStringToObject(pRoot, "privateTerm", u64buf);
+
+ snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->commitIndex);
+ cJSON_AddStringToObject(pRoot, "commitIndex", u64buf);
+ }
+
+ cJSON* pJson = cJSON_CreateObject();
+ cJSON_AddItemToObject(pJson, "SyncHeartbeat", pRoot);
+ return pJson;
+}
+
+char* syncHeartbeat2Str(const SyncHeartbeat* pMsg) {
+ cJSON* pJson = syncHeartbeat2Json(pMsg);
+ char* serialized = cJSON_Print(pJson);
+ cJSON_Delete(pJson);
+ return serialized;
+}
+
+void syncHeartbeatPrint(const SyncHeartbeat* pMsg) {
+ char* serialized = syncHeartbeat2Str(pMsg);
+ printf("syncHeartbeatPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized);
+ fflush(NULL);
+ taosMemoryFree(serialized);
+}
+
+void syncHeartbeatPrint2(char* s, const SyncHeartbeat* pMsg) {
+ char* serialized = syncHeartbeat2Str(pMsg);
+ printf("syncHeartbeatPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized);
+ fflush(NULL);
+ taosMemoryFree(serialized);
+}
+
+void syncHeartbeatLog(const SyncHeartbeat* pMsg) {
+ char* serialized = syncHeartbeat2Str(pMsg);
+ sTrace("syncHeartbeatLog | len:%" PRIu64 " | %s", strlen(serialized), serialized);
+ taosMemoryFree(serialized);
+}
+
+void syncHeartbeatLog2(char* s, const SyncHeartbeat* pMsg) {
+ if (gRaftDetailLog) {
+ char* serialized = syncHeartbeat2Str(pMsg);
+ sTrace("syncHeartbeatLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized);
+ taosMemoryFree(serialized);
+ }
+}
+
+// ---- message process SyncHeartbeatReply----
+SyncHeartbeatReply* syncHeartbeatReplyBuild(int32_t vgId) {
+ uint32_t bytes = sizeof(SyncHeartbeatReply);
+ SyncHeartbeatReply* pMsg = taosMemoryMalloc(bytes);
+ memset(pMsg, 0, bytes);
+ pMsg->bytes = bytes;
+ pMsg->vgId = vgId;
+ pMsg->msgType = TDMT_SYNC_HEARTBEAT_REPLY;
+ return pMsg;
+}
+
+void syncHeartbeatReplyDestroy(SyncHeartbeatReply* pMsg) {
+ if (pMsg != NULL) {
+ taosMemoryFree(pMsg);
+ }
+}
+
+void syncHeartbeatReplySerialize(const SyncHeartbeatReply* pMsg, char* buf, uint32_t bufLen) {
+ ASSERT(pMsg->bytes <= bufLen);
+ memcpy(buf, pMsg, pMsg->bytes);
+}
+
+void syncHeartbeatReplyDeserialize(const char* buf, uint32_t len, SyncHeartbeatReply* pMsg) {
+ memcpy(pMsg, buf, len);
+ ASSERT(len == pMsg->bytes);
+}
+
+char* syncHeartbeatReplySerialize2(const SyncHeartbeatReply* pMsg, uint32_t* len) {
+ char* buf = taosMemoryMalloc(pMsg->bytes);
+ ASSERT(buf != NULL);
+ syncHeartbeatReplySerialize(pMsg, buf, pMsg->bytes);
+ if (len != NULL) {
+ *len = pMsg->bytes;
+ }
+ return buf;
+}
+
+SyncHeartbeatReply* syncHeartbeatReplyDeserialize2(const char* buf, uint32_t len) {
+ uint32_t bytes = *((uint32_t*)buf);
+ SyncHeartbeatReply* pMsg = taosMemoryMalloc(bytes);
+ ASSERT(pMsg != NULL);
+ syncHeartbeatReplyDeserialize(buf, len, pMsg);
+ ASSERT(len == pMsg->bytes);
+ return pMsg;
+}
+
+void syncHeartbeatReply2RpcMsg(const SyncHeartbeatReply* pMsg, SRpcMsg* pRpcMsg) {
+ memset(pRpcMsg, 0, sizeof(*pRpcMsg));
+ pRpcMsg->msgType = pMsg->msgType;
+ pRpcMsg->contLen = pMsg->bytes;
+ pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen);
+ syncHeartbeatReplySerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen);
+}
+
+void syncHeartbeatReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeatReply* pMsg) {
+ syncHeartbeatReplyDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg);
+}
+
+SyncHeartbeatReply* syncHeartbeatReplyFromRpcMsg2(const SRpcMsg* pRpcMsg) {
+ SyncHeartbeatReply* pMsg = syncHeartbeatReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
+ ASSERT(pMsg != NULL);
+ return pMsg;
+}
+
+cJSON* syncHeartbeatReply2Json(const SyncHeartbeatReply* pMsg) {
+ char u64buf[128] = {0};
+ cJSON* pRoot = cJSON_CreateObject();
+
+ if (pMsg != NULL) {
+ cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
+ cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId);
+ cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
+
+ cJSON* pSrcId = cJSON_CreateObject();
+ snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr);
+ cJSON_AddStringToObject(pSrcId, "addr", u64buf);
+ {
+ uint64_t u64 = pMsg->srcId.addr;
+ cJSON* pTmp = pSrcId;
+ char host[128] = {0};
+ uint16_t port;
+ syncUtilU642Addr(u64, host, sizeof(host), &port);
+ cJSON_AddStringToObject(pTmp, "addr_host", host);
+ cJSON_AddNumberToObject(pTmp, "addr_port", port);
+ }
+ cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId);
+ cJSON_AddItemToObject(pRoot, "srcId", pSrcId);
+
+ cJSON* pDestId = cJSON_CreateObject();
+ snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr);
+ cJSON_AddStringToObject(pDestId, "addr", u64buf);
+ {
+ uint64_t u64 = pMsg->destId.addr;
+ cJSON* pTmp = pDestId;
+ char host[128] = {0};
+ uint16_t port;
+ syncUtilU642Addr(u64, host, sizeof(host), &port);
+ cJSON_AddStringToObject(pTmp, "addr_host", host);
+ cJSON_AddNumberToObject(pTmp, "addr_port", port);
+ }
+ cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId);
+ cJSON_AddItemToObject(pRoot, "destId", pDestId);
+
+ snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm);
+ cJSON_AddStringToObject(pRoot, "privateTerm", u64buf);
+
+ snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term);
+ cJSON_AddStringToObject(pRoot, "term", u64buf);
+
+ cJSON_AddStringToObject(pRoot, "matchIndex", u64buf);
+ snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->startTime);
+ cJSON_AddStringToObject(pRoot, "startTime", u64buf);
+ }
+
+ cJSON* pJson = cJSON_CreateObject();
+ cJSON_AddItemToObject(pJson, "SyncHeartbeatReply", pRoot);
+ return pJson;
+}
+
+char* syncHeartbeatReply2Str(const SyncHeartbeatReply* pMsg) {
+ cJSON* pJson = syncHeartbeatReply2Json(pMsg);
+ char* serialized = cJSON_Print(pJson);
+ cJSON_Delete(pJson);
+ return serialized;
+}
+
+void syncHeartbeatReplyPrint(const SyncHeartbeatReply* pMsg) {
+ char* serialized = syncHeartbeatReply2Str(pMsg);
+ printf("syncHeartbeatReplyPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized);
+ fflush(NULL);
+ taosMemoryFree(serialized);
+}
+
+void syncHeartbeatReplyPrint2(char* s, const SyncHeartbeatReply* pMsg) {
+ char* serialized = syncHeartbeatReply2Str(pMsg);
+ printf("syncHeartbeatReplyPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized);
+ fflush(NULL);
+ taosMemoryFree(serialized);
+}
+
+void syncHeartbeatReplyLog(const SyncHeartbeatReply* pMsg) {
+ char* serialized = syncHeartbeatReply2Str(pMsg);
+ sTrace("syncHeartbeatReplyLog | len:%" PRIu64 " | %s", strlen(serialized), serialized);
+ taosMemoryFree(serialized);
+}
+
+void syncHeartbeatReplyLog2(char* s, const SyncHeartbeatReply* pMsg) {
+ if (gRaftDetailLog) {
+ char* serialized = syncHeartbeatReply2Str(pMsg);
+ sTrace("syncHeartbeatReplyLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized);
+ taosMemoryFree(serialized);
+ }
+}
+
// ---- message process SyncApplyMsg----
SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen) {
uint32_t bytes = sizeof(SyncApplyMsg) + dataLen;
diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt
index 72845d0c1d..b9cc7a391d 100644
--- a/source/libs/sync/test/CMakeLists.txt
+++ b/source/libs/sync/test/CMakeLists.txt
@@ -57,6 +57,8 @@ add_executable(syncLeaderTransferTest "")
add_executable(syncReconfigFinishTest "")
add_executable(syncRestoreFromSnapshot "")
add_executable(syncRaftCfgIndexTest "")
+add_executable(syncHeartbeatTest "")
+add_executable(syncHeartbeatReplyTest "")
target_sources(syncTest
@@ -295,6 +297,14 @@ target_sources(syncRaftCfgIndexTest
PRIVATE
"syncRaftCfgIndexTest.cpp"
)
+target_sources(syncHeartbeatTest
+ PRIVATE
+ "syncHeartbeatTest.cpp"
+)
+target_sources(syncHeartbeatReplyTest
+ PRIVATE
+ "syncHeartbeatReplyTest.cpp"
+)
target_include_directories(syncTest
@@ -592,6 +602,16 @@ target_include_directories(syncRaftCfgIndexTest
"${TD_SOURCE_DIR}/include/libs/sync"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
+target_include_directories(syncHeartbeatTest
+ PUBLIC
+ "${TD_SOURCE_DIR}/include/libs/sync"
+ "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
+target_include_directories(syncHeartbeatReplyTest
+ PUBLIC
+ "${TD_SOURCE_DIR}/include/libs/sync"
+ "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
target_link_libraries(syncTest
@@ -830,6 +850,14 @@ target_link_libraries(syncRaftCfgIndexTest
sync
gtest_main
)
+target_link_libraries(syncHeartbeatTest
+ sync
+ gtest_main
+)
+target_link_libraries(syncHeartbeatReplyTest
+ sync
+ gtest_main
+)
enable_testing()
diff --git a/source/libs/sync/test/syncHeartbeatReplyTest.cpp b/source/libs/sync/test/syncHeartbeatReplyTest.cpp
new file mode 100644
index 0000000000..0ccd7b70bb
--- /dev/null
+++ b/source/libs/sync/test/syncHeartbeatReplyTest.cpp
@@ -0,0 +1,105 @@
+#include
+#include
+#include "syncIO.h"
+#include "syncInt.h"
+#include "syncMessage.h"
+#include "syncUtil.h"
+
+void logTest() {
+ sTrace("--- sync log test: trace");
+ sDebug("--- sync log test: debug");
+ sInfo("--- sync log test: info");
+ sWarn("--- sync log test: warn");
+ sError("--- sync log test: error");
+ sFatal("--- sync log test: fatal");
+}
+
+SyncHeartbeatReply *createMsg() {
+ SyncHeartbeatReply *pMsg = syncHeartbeatReplyBuild(1000);
+ pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234);
+ pMsg->srcId.vgId = 100;
+ pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678);
+ pMsg->destId.vgId = 100;
+
+ pMsg->term = 33;
+ pMsg->privateTerm = 44;
+ pMsg->startTime = taosGetTimestampMs();
+ return pMsg;
+}
+
+void test1() {
+ SyncHeartbeatReply *pMsg = createMsg();
+ syncHeartbeatReplyLog2((char *)"test1:", pMsg);
+ syncHeartbeatReplyDestroy(pMsg);
+}
+
+void test2() {
+ SyncHeartbeatReply *pMsg = createMsg();
+ uint32_t len = pMsg->bytes;
+ char * serialized = (char *)taosMemoryMalloc(len);
+ syncHeartbeatReplySerialize(pMsg, serialized, len);
+ SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyBuild(1000);
+ syncHeartbeatReplyDeserialize(serialized, len, pMsg2);
+ syncHeartbeatReplyLog2((char *)"test2: syncHeartbeatReplySerialize -> syncHeartbeatReplyDeserialize ",
+ pMsg2);
+
+ taosMemoryFree(serialized);
+ syncHeartbeatReplyDestroy(pMsg);
+ syncHeartbeatReplyDestroy(pMsg2);
+}
+
+void test3() {
+ SyncHeartbeatReply *pMsg = createMsg();
+ uint32_t len;
+ char * serialized = syncHeartbeatReplySerialize2(pMsg, &len);
+ SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyDeserialize2(serialized, len);
+ syncHeartbeatReplyLog2((char *)"test3: syncHeartbeatReplySerialize3 -> syncHeartbeatReplyDeserialize2 ",
+ pMsg2);
+
+ taosMemoryFree(serialized);
+ syncHeartbeatReplyDestroy(pMsg);
+ syncHeartbeatReplyDestroy(pMsg2);
+}
+
+void test4() {
+ SyncHeartbeatReply *pMsg = createMsg();
+ SRpcMsg rpcMsg;
+ syncHeartbeatReply2RpcMsg(pMsg, &rpcMsg);
+ SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyBuild(1000);
+ syncHeartbeatReplyFromRpcMsg(&rpcMsg, pMsg2);
+ syncHeartbeatReplyLog2((char *)"test4: syncHeartbeatReply2RpcMsg -> syncHeartbeatReplyFromRpcMsg ",
+ pMsg2);
+
+ rpcFreeCont(rpcMsg.pCont);
+ syncHeartbeatReplyDestroy(pMsg);
+ syncHeartbeatReplyDestroy(pMsg2);
+}
+
+void test5() {
+ SyncHeartbeatReply *pMsg = createMsg();
+ SRpcMsg rpcMsg;
+ syncHeartbeatReply2RpcMsg(pMsg, &rpcMsg);
+ SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyFromRpcMsg2(&rpcMsg);
+ syncHeartbeatReplyLog2((char *)"test5: syncHeartbeatReply2RpcMsg -> syncHeartbeatReplyFromRpcMsg2 ",
+ pMsg2);
+
+ rpcFreeCont(rpcMsg.pCont);
+ syncHeartbeatReplyDestroy(pMsg);
+ syncHeartbeatReplyDestroy(pMsg2);
+}
+
+int main() {
+ gRaftDetailLog = true;
+
+ tsAsyncLog = 0;
+ sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
+ logTest();
+
+ test1();
+ test2();
+ test3();
+ test4();
+ test5();
+
+ return 0;
+}
diff --git a/source/libs/sync/test/syncHeartbeatTest.cpp b/source/libs/sync/test/syncHeartbeatTest.cpp
new file mode 100644
index 0000000000..d910c828f1
--- /dev/null
+++ b/source/libs/sync/test/syncHeartbeatTest.cpp
@@ -0,0 +1,99 @@
+#include
+#include
+#include "syncIO.h"
+#include "syncInt.h"
+#include "syncMessage.h"
+#include "syncUtil.h"
+
+void logTest() {
+ sTrace("--- sync log test: trace");
+ sDebug("--- sync log test: debug");
+ sInfo("--- sync log test: info");
+ sWarn("--- sync log test: warn");
+ sError("--- sync log test: error");
+ sFatal("--- sync log test: fatal");
+}
+
+SyncHeartbeat *createMsg() {
+ SyncHeartbeat *pMsg = syncHeartbeatBuild(789);
+ pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234);
+ pMsg->srcId.vgId = 100;
+ pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678);
+ pMsg->destId.vgId = 100;
+ pMsg->term = 8;
+ pMsg->commitIndex = 33;
+ pMsg->privateTerm = 44;
+ return pMsg;
+}
+
+void test1() {
+ SyncHeartbeat *pMsg = createMsg();
+ syncHeartbeatLog2((char *)"test1:", pMsg);
+ syncHeartbeatDestroy(pMsg);
+}
+
+void test2() {
+ SyncHeartbeat *pMsg = createMsg();
+ uint32_t len = pMsg->bytes;
+ char * serialized = (char *)taosMemoryMalloc(len);
+ syncHeartbeatSerialize(pMsg, serialized, len);
+ SyncHeartbeat *pMsg2 = syncHeartbeatBuild(789);
+ syncHeartbeatDeserialize(serialized, len, pMsg2);
+ syncHeartbeatLog2((char *)"test2: syncHeartbeatSerialize -> syncHeartbeatDeserialize ", pMsg2);
+
+ taosMemoryFree(serialized);
+ syncHeartbeatDestroy(pMsg);
+ syncHeartbeatDestroy(pMsg2);
+}
+
+void test3() {
+ SyncHeartbeat *pMsg = createMsg();
+ uint32_t len;
+ char * serialized = syncHeartbeatSerialize2(pMsg, &len);
+ SyncHeartbeat *pMsg2 = syncHeartbeatDeserialize2(serialized, len);
+ syncHeartbeatLog2((char *)"test3: syncHeartbeatSerialize2 -> syncHeartbeatDeserialize2 ", pMsg2);
+
+ taosMemoryFree(serialized);
+ syncHeartbeatDestroy(pMsg);
+ syncHeartbeatDestroy(pMsg2);
+}
+
+void test4() {
+ SyncHeartbeat *pMsg = createMsg();
+ SRpcMsg rpcMsg;
+ syncHeartbeat2RpcMsg(pMsg, &rpcMsg);
+ SyncHeartbeat *pMsg2 = (SyncHeartbeat *)taosMemoryMalloc(rpcMsg.contLen);
+ syncHeartbeatFromRpcMsg(&rpcMsg, pMsg2);
+ syncHeartbeatLog2((char *)"test4: syncHeartbeat2RpcMsg -> syncHeartbeatFromRpcMsg ", pMsg2);
+
+ rpcFreeCont(rpcMsg.pCont);
+ syncHeartbeatDestroy(pMsg);
+ syncHeartbeatDestroy(pMsg2);
+}
+
+void test5() {
+ SyncHeartbeat *pMsg = createMsg();
+ SRpcMsg rpcMsg;
+ syncHeartbeat2RpcMsg(pMsg, &rpcMsg);
+ SyncHeartbeat *pMsg2 =syncHeartbeatFromRpcMsg2(&rpcMsg);
+ syncHeartbeatLog2((char *)"test5: syncHeartbeat2RpcMsg -> syncHeartbeatFromRpcMsg2 ", pMsg2);
+
+ rpcFreeCont(rpcMsg.pCont);
+ syncHeartbeatDestroy(pMsg);
+ syncHeartbeatDestroy(pMsg2);
+}
+
+int main() {
+ tsAsyncLog = 0;
+ sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
+ gRaftDetailLog = true;
+ logTest();
+
+ test1();
+ test2();
+ test3();
+ test4();
+ test5();
+
+ return 0;
+}
diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c
index f90c392788..bec07b935e 100644
--- a/source/libs/tdb/src/db/tdbPager.c
+++ b/source/libs/tdb/src/db/tdbPager.c
@@ -502,7 +502,7 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) {
i64 offset;
int ret;
- offset = pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1);
+ offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1);
if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) {
ASSERT(0);
return -1;
diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c
index 275e7b42cc..1559c85e23 100644
--- a/source/libs/transport/src/thttp.c
+++ b/source/libs/transport/src/thttp.c
@@ -126,22 +126,22 @@ _OVER:
return code;
}
-static void destroyHttpClient(SHttpClient* cli) {
+static FORCE_INLINE void destroyHttpClient(SHttpClient* cli) {
taosMemoryFree(cli->wbuf);
taosMemoryFree(cli->rbuf);
taosMemoryFree(cli->addr);
taosMemoryFree(cli);
}
-static void clientCloseCb(uv_handle_t* handle) {
+static FORCE_INLINE void clientCloseCb(uv_handle_t* handle) {
SHttpClient* cli = handle->data;
destroyHttpClient(cli);
}
-static void clientAllocBuffCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
+static FORCE_INLINE void clientAllocBuffCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
SHttpClient* cli = handle->data;
buf->base = cli->rbuf;
buf->len = HTTP_RECV_BUF_SIZE;
}
-static void clientRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
+static FORCE_INLINE void clientRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
SHttpClient* cli = handle->data;
if (nread < 0) {
uError("http-report recv error:%s", uv_err_name(nread));
@@ -173,7 +173,7 @@ static void clientConnCb(uv_connect_t* req, int32_t status) {
uv_write(&cli->req, (uv_stream_t*)&cli->tcp, cli->wbuf, 2, clientSentCb);
}
-static int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) {
+static FORCE_INLINE int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) {
uint32_t ip = taosGetIpv4FromFqdn(server);
if (ip == 0xffffffff) {
terrno = TAOS_SYSTEM_ERROR(errno);
diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index 4b5441f738..4a0008b5ff 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -69,11 +69,9 @@ typedef struct SCliThrd {
SAsyncPool* asyncPool;
uv_prepare_t* prepare;
void* pool; // conn pool
-
+ // timer handles
SArray* timerList;
-
// msg queue
-
queue msg;
TdThreadMutex msgMtx;
SDelayQueue* delayQueue;
@@ -108,7 +106,7 @@ static void cliReadTimeoutCb(uv_timer_t* handle);
// register timer in each thread to clear expire conn
// static void cliTimeoutCb(uv_timer_t* handle);
// alloc buffer for recv
-static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
+static FORCE_INLINE void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
// callback after recv nbytes from socket
static void cliRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf);
// callback after send data to socket
@@ -132,10 +130,10 @@ static void cliSend(SCliConn* pConn);
static void cliDestroyConnMsgs(SCliConn* conn, bool destroy);
// cli util func
-static bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx);
-static void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr);
+static FORCE_INLINE bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx);
+static FORCE_INLINE void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr);
-static int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* resp);
+static FORCE_INLINE int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* resp);
// process data read from server, add decompress etc later
static void cliHandleResp(SCliConn* conn);
@@ -150,13 +148,11 @@ static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd);
static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease, NULL,
cliHandleUpdate};
-static void cliSendQuit(SCliThrd* thrd);
-static void destroyUserdata(STransMsg* userdata);
+static FORCE_INLINE void destroyUserdata(STransMsg* userdata);
+static FORCE_INLINE void destroyCmsg(void* cmsg);
+static FORCE_INLINE int cliRBChoseIdx(STrans* pTransInst);
+static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx);
-static int cliRBChoseIdx(STrans* pTransInst);
-
-static void destroyCmsg(void* cmsg);
-static void transDestroyConnCtx(STransConnCtx* ctx);
// thread obj
static SCliThrd* createThrdObj();
static void destroyThrdObj(SCliThrd* pThrd);
@@ -166,7 +162,7 @@ static void cliWalkCb(uv_handle_t* handle, void* arg);
static void cliReleaseUnfinishedMsg(SCliConn* conn) {
for (int i = 0; i < transQueueSize(&conn->cliMsgs); i++) {
SCliMsg* msg = transQueueGet(&conn->cliMsgs, i);
- if (msg != NULL && msg->ctx != NULL) {
+ if (msg != NULL && msg->ctx != NULL && msg->ctx->ahandle != (void*)0x9527) {
if (conn->ctx.freeFunc != NULL && msg->ctx->ahandle != NULL) {
conn->ctx.freeFunc(msg->ctx->ahandle);
}
@@ -200,22 +196,22 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) {
#define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL)
#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrd*)(conn)->hostThrd)->pTransInst))->label)
-#define CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle) \
- do { \
- int i = 0, sz = transQueueSize(&conn->cliMsgs); \
- for (; i < sz; i++) { \
- pMsg = transQueueGet(&conn->cliMsgs, i); \
- if (pMsg != NULL && pMsg->ctx != NULL && (uint64_t)pMsg->ctx->ahandle == ahandle) { \
- break; \
- } \
- } \
- if (i == sz) { \
- pMsg = NULL; \
- tDebug("msg not found, %" PRIu64 "", ahandle); \
- } else { \
- pMsg = transQueueRm(&conn->cliMsgs, i); \
- tDebug("msg found, %" PRIu64 "", ahandle); \
- } \
+#define CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle) \
+ do { \
+ int i = 0, sz = transQueueSize(&conn->cliMsgs); \
+ for (; i < sz; i++) { \
+ pMsg = transQueueGet(&conn->cliMsgs, i); \
+ if (pMsg->ctx != NULL && (uint64_t)pMsg->ctx->ahandle == ahandle) { \
+ break; \
+ } \
+ } \
+ if (i == sz) { \
+ pMsg = NULL; \
+ tDebug("msg not found, %" PRIu64 "", ahandle); \
+ } else { \
+ pMsg = transQueueRm(&conn->cliMsgs, i); \
+ tDebug("msg found, %" PRIu64 "", ahandle); \
+ } \
} while (0)
#define CONN_GET_NEXT_SENDMSG(conn) \
do { \
@@ -293,7 +289,12 @@ bool cliMaySendCachedMsg(SCliConn* conn) {
if (!transQueueEmpty(&conn->cliMsgs)) {
SCliMsg* pCliMsg = NULL;
CONN_GET_NEXT_SENDMSG(conn);
- cliSend(conn);
+ if (pCliMsg == NULL)
+ return false;
+ else {
+ cliSend(conn);
+ return true;
+ }
}
return false;
_RETURN:
@@ -380,8 +381,10 @@ void cliHandleResp(SCliConn* conn) {
return;
}
- if (cliAppCb(conn, &transMsg, pMsg) != 0) {
- return;
+ if (pMsg == NULL || (pMsg && pMsg->type != Release)) {
+ if (cliAppCb(conn, &transMsg, pMsg) != 0) {
+ return;
+ }
}
destroyCmsg(pMsg);
@@ -429,17 +432,20 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) {
transMsg.info.ahandle);
}
} else {
- transMsg.info.ahandle = pCtx ? pCtx->ahandle : NULL;
+ transMsg.info.ahandle = (pMsg->type != Release && pCtx) ? pCtx->ahandle : NULL;
}
if (pCtx == NULL || pCtx->pSem == NULL) {
if (transMsg.info.ahandle == NULL) {
+ if (REQUEST_NO_RESP(&pMsg->msg) || pMsg->type == Release) destroyCmsg(pMsg);
once = true;
continue;
}
}
- if (cliAppCb(pConn, &transMsg, pMsg) != 0) {
- return;
+ if (pMsg == NULL || (pMsg && pMsg->type != Release)) {
+ if (cliAppCb(pConn, &transMsg, pMsg) != 0) {
+ return;
+ }
}
destroyCmsg(pMsg);
tTrace("%s conn %p start to destroy, ref:%d", CONN_GET_INST_LABEL(pConn), pConn, T_REF_VAL_GET(pConn));
@@ -705,6 +711,9 @@ static bool cliHandleNoResp(SCliConn* conn) {
if (cliMaySendCachedMsg(conn) == false) {
SCliThrd* thrd = conn->hostThrd;
addConnToPool(thrd->pool, conn);
+ res = false;
+ } else {
+ res = true;
}
}
}
@@ -782,7 +791,13 @@ void cliSend(SCliConn* pConn) {
uv_buf_t wb = uv_buf_init((char*)pHead, msgLen);
uv_write_t* req = transReqQueuePush(&pConn->wreqQueue);
- uv_write(req, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb);
+
+ int status = uv_write(req, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb);
+ if (status != 0) {
+ tGError("%s conn %p failed to sent msg:%s, errmsg:%s", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pMsg->msgType),
+ uv_err_name(status));
+ cliHandleExcept(pConn);
+ }
return;
_RETURN:
return;
@@ -885,26 +900,23 @@ SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd, bool* ignore) {
}
return conn;
}
-void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) {
+FORCE_INLINE void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) {
if (pCvtAddr->cvt == false) {
return;
}
- for (int i = 0; i < pEpSet->numOfEps && pEpSet->numOfEps == 1; i++) {
- if (strncmp(pEpSet->eps[i].fqdn, pCvtAddr->fqdn, TSDB_FQDN_LEN) == 0) {
- memset(pEpSet->eps[i].fqdn, 0, TSDB_FQDN_LEN);
- memcpy(pEpSet->eps[i].fqdn, pCvtAddr->ip, TSDB_FQDN_LEN);
- }
+ if (pEpSet->numOfEps == 1 && strncmp(pEpSet->eps[0].fqdn, pCvtAddr->fqdn, TSDB_FQDN_LEN) == 0) {
+ memset(pEpSet->eps[0].fqdn, 0, TSDB_FQDN_LEN);
+ memcpy(pEpSet->eps[0].fqdn, pCvtAddr->ip, TSDB_FQDN_LEN);
}
}
-bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx) {
+FORCE_INLINE bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx) {
if (code != 0) return false;
if (pCtx->retryCnt == 0) return false;
if (transEpSetIsEqual(&pCtx->epSet, &pCtx->origEpSet)) return false;
return true;
}
-
-int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* pResp) {
+FORCE_INLINE int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* pResp) {
if (pMsg == NULL) return -1;
memset(pResp, 0, sizeof(STransMsg));
@@ -934,7 +946,9 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) {
// persist conn already release by server
STransMsg resp;
cliBuildExceptResp(pMsg, &resp);
- pTransInst->cfp(pTransInst->parent, &resp, NULL);
+ if (pMsg->type != Release) {
+ pTransInst->cfp(pTransInst->parent, &resp, NULL);
+ }
destroyCmsg(pMsg);
return;
}
@@ -980,6 +994,8 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) {
return;
}
}
+ STraceId* trace = &pMsg->msg.info.traceId;
+ tGTrace("%s conn %p ready", pTransInst->label, conn);
}
static void cliAsyncCb(uv_async_t* handle) {
SAsyncItem* item = handle->data;
@@ -1128,14 +1144,15 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads,
return cli;
}
-static void destroyUserdata(STransMsg* userdata) {
+static FORCE_INLINE void destroyUserdata(STransMsg* userdata) {
if (userdata->pCont == NULL) {
return;
}
transFreeMsg(userdata->pCont);
userdata->pCont = NULL;
}
-static void destroyCmsg(void* arg) {
+
+static FORCE_INLINE void destroyCmsg(void* arg) {
SCliMsg* pMsg = arg;
if (pMsg == NULL) {
return;
@@ -1201,7 +1218,7 @@ static void destroyThrdObj(SCliThrd* pThrd) {
taosMemoryFree(pThrd);
}
-static void transDestroyConnCtx(STransConnCtx* ctx) {
+static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx) {
//
taosMemoryFree(ctx);
}
@@ -1220,7 +1237,7 @@ void cliWalkCb(uv_handle_t* handle, void* arg) {
}
}
-int cliRBChoseIdx(STrans* pTransInst) {
+FORCE_INLINE int cliRBChoseIdx(STrans* pTransInst) {
int8_t index = pTransInst->index;
if (pTransInst->numOfThreads == 0) {
return -1;
@@ -1230,7 +1247,7 @@ int cliRBChoseIdx(STrans* pTransInst) {
}
return index % pTransInst->numOfThreads;
}
-static void doDelayTask(void* param) {
+static FORCE_INLINE void doDelayTask(void* param) {
STaskArg* arg = param;
SCliMsg* pMsg = arg->param1;
SCliThrd* pThrd = arg->param2;
@@ -1264,13 +1281,13 @@ static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) {
transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
}
-void cliCompareAndSwap(int8_t* val, int8_t exp, int8_t newVal) {
+FORCE_INLINE void cliCompareAndSwap(int8_t* val, int8_t exp, int8_t newVal) {
if (*val != exp) {
*val = newVal;
}
}
-bool cliTryExtractEpSet(STransMsg* pResp, SEpSet* dst) {
+FORCE_INLINE bool cliTryExtractEpSet(STransMsg* pResp, SEpSet* dst) {
if ((pResp == NULL || pResp->info.hasEpSet == 0)) {
return false;
}
@@ -1300,7 +1317,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
STrans* pTransInst = pThrd->pTransInst;
if (pMsg == NULL || pMsg->ctx == NULL) {
- tTrace("%s conn %p handle resp", pTransInst->label, pConn);
+ tDebug("%s conn %p handle resp", pTransInst->label, pConn);
pTransInst->cfp(pTransInst->parent, pResp, NULL);
return 0;
}
@@ -1402,53 +1419,57 @@ void transUnrefCliHandle(void* handle) {
cliDestroyConn((SCliConn*)handle, true);
}
}
-SCliThrd* transGetWorkThrdFromHandle(int64_t handle, bool* validHandle) {
+static FORCE_INLINE SCliThrd* transGetWorkThrdFromHandle(STrans* trans, int64_t handle) {
SCliThrd* pThrd = NULL;
SExHandle* exh = transAcquireExHandle(transGetRefMgt(), handle);
if (exh == NULL) {
return NULL;
}
- *validHandle = true;
+ if (exh->pThrd == NULL && trans != NULL) {
+ int idx = cliRBChoseIdx(trans);
+ if (idx < 0) return NULL;
+ exh->pThrd = ((SCliObj*)trans->tcphandle)->pThreadObj[idx];
+ }
+
pThrd = exh->pThrd;
transReleaseExHandle(transGetRefMgt(), handle);
return pThrd;
}
-SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle, bool* validHandle) {
+SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle) {
if (handle == 0) {
int idx = cliRBChoseIdx(trans);
if (idx < 0) return NULL;
return ((SCliObj*)trans->tcphandle)->pThreadObj[idx];
}
- SCliThrd* pThrd = transGetWorkThrdFromHandle(handle, validHandle);
- if (*validHandle == true && pThrd == NULL) {
- int idx = cliRBChoseIdx(trans);
- if (idx < 0) return NULL;
- pThrd = ((SCliObj*)trans->tcphandle)->pThreadObj[idx];
- }
+ SCliThrd* pThrd = transGetWorkThrdFromHandle(trans, handle);
return pThrd;
}
int transReleaseCliHandle(void* handle) {
int idx = -1;
bool valid = false;
- SCliThrd* pThrd = transGetWorkThrdFromHandle((int64_t)handle, &valid);
+ SCliThrd* pThrd = transGetWorkThrdFromHandle(NULL, (int64_t)handle);
if (pThrd == NULL) {
return -1;
}
- STransMsg tmsg = {.info.handle = handle};
+ STransMsg tmsg = {.info.handle = handle, .info.ahandle = (void*)0x9527};
TRACE_SET_MSGID(&tmsg.info.traceId, tGenIdPI64());
+ STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx));
+ pCtx->ahandle = tmsg.info.ahandle;
+
SCliMsg* cmsg = taosMemoryCalloc(1, sizeof(SCliMsg));
cmsg->msg = tmsg;
cmsg->type = Release;
+ cmsg->ctx = pCtx;
STraceId* trace = &tmsg.info.traceId;
tGDebug("send release request at thread:%08" PRId64 "", pThrd->pid);
if (0 != transAsyncSend(pThrd->asyncPool, &cmsg->q)) {
- taosMemoryFree(cmsg);
+ destroyCmsg(cmsg);
return -1;
}
return 0;
@@ -1461,9 +1482,8 @@ int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STran
return -1;
}
- bool valid = false;
- SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle, &valid);
- if (pThrd == NULL && valid == false) {
+ SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle);
+ if (pThrd == NULL) {
transFreeMsg(pReq->pCont);
transReleaseExHandle(transGetInstMgt(), (int64_t)shandle);
return TSDB_CODE_RPC_BROKEN_LINK;
@@ -1506,9 +1526,8 @@ int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMs
return -1;
}
- bool valid = false;
- SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle, &valid);
- if (pThrd == NULL && valid == false) {
+ SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle);
+ if (pThrd == NULL) {
transFreeMsg(pReq->pCont);
transReleaseExHandle(transGetInstMgt(), (int64_t)shandle);
return TSDB_CODE_RPC_BROKEN_LINK;
@@ -1592,6 +1611,7 @@ int64_t transAllocHandle() {
SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle));
exh->refId = transAddExHandle(transGetRefMgt(), exh);
tDebug("pre alloc refId %" PRId64 "", exh->refId);
+
return exh->refId;
}
#endif
diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c
index a4d679b281..5f3171ee0e 100644
--- a/source/libs/transport/src/transComm.c
+++ b/source/libs/transport/src/transComm.c
@@ -287,10 +287,10 @@ void transCtxMerge(STransCtx* dst, STransCtx* src) {
STransCtxVal* sVal = (STransCtxVal*)iter;
key = taosHashGetKey(sVal, &klen);
- STransCtxVal* dVal = taosHashGet(dst->args, key, klen);
- if (dVal) {
- dst->freeFunc(dVal->val);
- }
+ // STransCtxVal* dVal = taosHashGet(dst->args, key, klen);
+ // if (dVal) {
+ // dst->freeFunc(dVal->val);
+ // }
taosHashPut(dst->args, key, klen, sVal, sizeof(*sVal));
iter = taosHashIterate(src->args, iter);
}
@@ -424,7 +424,7 @@ void transQueueDestroy(STransQueue* queue) {
taosArrayDestroy(queue->q);
}
-static int32_t timeCompare(const HeapNode* a, const HeapNode* b) {
+static FORCE_INLINE int32_t timeCompare(const HeapNode* a, const HeapNode* b) {
SDelayTask* arg1 = container_of(a, SDelayTask, node);
SDelayTask* arg2 = container_of(b, SDelayTask, node);
if (arg1->execTime > arg2->execTime) {
diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c
index 7007079f87..d3277d1cc1 100644
--- a/source/libs/transport/src/transSvr.c
+++ b/source/libs/transport/src/transSvr.c
@@ -125,17 +125,17 @@ static void uvWorkAfterTask(uv_work_t* req, int status);
static void uvWalkCb(uv_handle_t* handle, void* arg);
static void uvFreeCb(uv_handle_t* handle);
-static void uvStartSendRespImpl(SSvrMsg* smsg);
+static FORCE_INLINE void uvStartSendRespImpl(SSvrMsg* smsg);
+
static void uvPrepareSendData(SSvrMsg* msg, uv_buf_t* wb);
static void uvStartSendResp(SSvrMsg* msg);
static void uvNotifyLinkBrokenToApp(SSvrConn* conn);
-static void destroySmsg(SSvrMsg* smsg);
-// check whether already read complete packet
-static SSvrConn* createConn(void* hThrd);
-static void destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/);
-static void destroyConnRegArg(SSvrConn* conn);
+static FORCE_INLINE void destroySmsg(SSvrMsg* smsg);
+static FORCE_INLINE SSvrConn* createConn(void* hThrd);
+static FORCE_INLINE void destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/);
+static FORCE_INLINE void destroyConnRegArg(SSvrConn* conn);
static int reallocConnRef(SSvrConn* conn);
@@ -413,7 +413,7 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
wb->len = len;
}
-static void uvStartSendRespImpl(SSvrMsg* smsg) {
+static FORCE_INLINE void uvStartSendRespImpl(SSvrMsg* smsg) {
SSvrConn* pConn = smsg->pConn;
if (pConn->broken) {
return;
@@ -447,7 +447,7 @@ static void uvStartSendResp(SSvrMsg* smsg) {
return;
}
-static void destroySmsg(SSvrMsg* smsg) {
+static FORCE_INLINE void destroySmsg(SSvrMsg* smsg) {
if (smsg == NULL) {
return;
}
@@ -812,7 +812,7 @@ void* transWorkerThread(void* arg) {
return NULL;
}
-static SSvrConn* createConn(void* hThrd) {
+static FORCE_INLINE SSvrConn* createConn(void* hThrd) {
SWorkThrd* pThrd = hThrd;
SSvrConn* pConn = (SSvrConn*)taosMemoryCalloc(1, sizeof(SSvrConn));
@@ -842,7 +842,7 @@ static SSvrConn* createConn(void* hThrd) {
return pConn;
}
-static void destroyConn(SSvrConn* conn, bool clear) {
+static FORCE_INLINE void destroyConn(SSvrConn* conn, bool clear) {
if (conn == NULL) {
return;
}
@@ -854,7 +854,7 @@ static void destroyConn(SSvrConn* conn, bool clear) {
}
}
}
-static void destroyConnRegArg(SSvrConn* conn) {
+static FORCE_INLINE void destroyConnRegArg(SSvrConn* conn) {
if (conn->regArg.init == 1) {
transFreeMsg(conn->regArg.msg.pCont);
conn->regArg.init = 0;
@@ -907,7 +907,7 @@ static void uvDestroyConn(uv_handle_t* handle) {
}
static void uvPipeListenCb(uv_stream_t* handle, int status) {
if (status != 0) {
- tError("server failed to init pipe");
+ tError("server failed to init pipe, errmsg: %s", uv_err_name(status));
return;
}
@@ -945,7 +945,10 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
uv_loop_init(srv->loop);
int ret = uv_pipe_init(srv->loop, &srv->pipeListen, 0);
- assert(ret == 0);
+ if (ret != 0) {
+ tError("failed to init pipe, errmsg: %s", uv_err_name(ret));
+ goto End;
+ }
#ifdef WINDOWS
char pipeName[64];
@@ -956,10 +959,16 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
taosGetSelfPthreadId());
#endif
ret = uv_pipe_bind(&srv->pipeListen, pipeName);
- assert(ret == 0);
+ if (ret != 0) {
+ tError("failed to bind pipe, errmsg: %s", uv_err_name(ret));
+ goto End;
+ }
ret = uv_listen((uv_stream_t*)&srv->pipeListen, SOMAXCONN, uvPipeListenCb);
- assert(ret == 0);
+ if (ret != 0) {
+ tError("failed to listen pipe, errmsg: %s", uv_err_name(ret));
+ goto End;
+ }
for (int i = 0; i < srv->numOfThreads; i++) {
SWorkThrd* thrd = (SWorkThrd*)taosMemoryCalloc(1, sizeof(SWorkThrd));
@@ -1082,12 +1091,12 @@ void transCloseServer(void* arg) {
if (srv->inited) {
uv_async_send(srv->pAcceptAsync);
taosThreadJoin(srv->thread, NULL);
- }
- SRV_RELEASE_UV(srv->loop);
+ SRV_RELEASE_UV(srv->loop);
- for (int i = 0; i < srv->numOfThreads; i++) {
- sendQuitToWorkThrd(srv->pThreadObj[i]);
- destroyWorkThrd(srv->pThreadObj[i]);
+ for (int i = 0; i < srv->numOfThreads; i++) {
+ sendQuitToWorkThrd(srv->pThreadObj[i]);
+ destroyWorkThrd(srv->pThreadObj[i]);
+ }
}
taosMemoryFree(srv->pThreadObj);
diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c
index 9db7d6c455..5c437e6f7a 100644
--- a/source/libs/wal/src/walRead.c
+++ b/source/libs/wal/src/walRead.c
@@ -168,6 +168,9 @@ static int32_t walReadChangeFile(SWalReader *pReader, int64_t fileFirstVer) {
}
pReader->pIdxFile = pIdxFile;
+
+ pReader->curFileFirstVer = fileFirstVer;
+
return 0;
}
@@ -372,7 +375,7 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) {
int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead) {
int64_t code;
-// ASSERT(pRead->curVersion == pHead->head.version);
+ // ASSERT(pRead->curVersion == pHead->head.version);
code = taosLSeekFile(pRead->pLogFile, pHead->head.bodyLen, SEEK_CUR);
if (code < 0) {
@@ -415,7 +418,8 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) {
}
if (walValidBodyCksum(*ppHead) != 0) {
- wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, ver);
+ wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId,
+ ver);
pRead->curInvalid = 1;
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index 3117152af6..044cdc86b4 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -618,6 +618,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_REMOVE_EXISTS, "Rsma remove exists"
TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP, "Rsma fetch msg is messed up")
TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_EMPTY_INFO, "Rsma info is empty")
TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_SCHEMA, "Rsma invalid schema")
+TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_REGEX_MATCH, "Rsma regex match")
//index
TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING, "Index is rebuilding")
diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c
index 0970485dad..65f1bac60a 100644
--- a/source/util/src/trbtree.c
+++ b/source/util/src/trbtree.c
@@ -13,179 +13,297 @@
* along with this program. If not, see (getBufPage(pBuf, writePageId));
ASSERT_EQ(*(int32_t*)pBufPagex->data, nx);
- SArray* pa = getDataBufPagesIdList(pBuf, groupId);
+ SArray* pa = getDataBufPagesIdList(pBuf);
ASSERT_EQ(taosArrayGetSize(pa), 5);
destroyDiskbasedBuf(pBuf);
@@ -152,7 +152,7 @@ void recyclePageTest() {
SFilePage* pBufPagex1 = static_cast(getBufPage(pBuf, 1));
- SArray* pa = getDataBufPagesIdList(pBuf, groupId);
+ SArray* pa = getDataBufPagesIdList(pBuf);
ASSERT_EQ(taosArrayGetSize(pa), 6);
destroyDiskbasedBuf(pBuf);
diff --git a/source/util/test/trbtreeTest.cpp b/source/util/test/trbtreeTest.cpp
new file mode 100644
index 0000000000..cabf315df0
--- /dev/null
+++ b/source/util/test/trbtreeTest.cpp
@@ -0,0 +1,40 @@
+#include
+
+#include
+#include
+
+#include "trbtree.h"
+
+static int32_t tCmprInteger(const void *p1, const void *p2) {
+ if (*(int *)p1 < *(int *)p2) {
+ return -1;
+ } else if (*(int *)p1 > *(int *)p2) {
+ return 1;
+ }
+ return 0;
+}
+
+TEST(trbtreeTest, rbtree_test1) {
+#if 0
+ SRBTree rt;
+ tRBTreeCreate(&rt, tCmprInteger);
+ int a[] = {1, 3, 4, 2, 7, 5, 8};
+
+ for (int i = 0; i < sizeof(a) / sizeof(a[0]); i++) {
+ SRBTreeNode *pNode = (SRBTreeNode *)taosMemoryMalloc(sizeof(*pNode) + sizeof(int));
+ *(int *)pNode->payload = a[i];
+
+ tRBTreePut(&rt, pNode);
+ }
+
+ SRBTreeIter rti = tRBTreeIterCreate(&rt, 1);
+ SRBTreeNode *pNode = tRBTreeIterNext(&rti);
+ int la = 0;
+ while (pNode) {
+ GTEST_ASSERT_GT(*(int *)pNode->payload, la);
+ la = *(int *)pNode->payload;
+ // printf("%d\n", la);
+ pNode = tRBTreeIterNext(&rti);
+ }
+#endif
+}
\ No newline at end of file
diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt
index 46bae734ea..161c878440 100644
--- a/tests/script/jenkins/basic.txt
+++ b/tests/script/jenkins/basic.txt
@@ -221,6 +221,7 @@
./test.sh -f tsim/table/describe.sim
./test.sh -f tsim/table/double.sim
./test.sh -f tsim/table/float.sim
+./test.sh -f tsim/table/hash.sim
./test.sh -f tsim/table/int.sim
./test.sh -f tsim/table/limit.sim
./test.sh -f tsim/table/smallint.sim
@@ -248,6 +249,12 @@
./test.sh -f tsim/stream/windowClose.sim
./test.sh -f tsim/stream/ignoreExpiredData.sim
./test.sh -f tsim/stream/sliding.sim
+#./test.sh -f tsim/stream/partitionbyColumnInterval.sim
+#./test.sh -f tsim/stream/partitionbyColumnSession.sim
+#./test.sh -f tsim/stream/partitionbyColumnState.sim
+#./test.sh -f tsim/stream/deleteInterval.sim
+#./test.sh -f tsim/stream/deleteSession.sim
+#./test.sh -f tsim/stream/deleteState.sim
# ---- transaction ----
./test.sh -f tsim/trans/lossdata1.sim
diff --git a/tests/script/sh/abs_max.c b/tests/script/sh/abs_max.c
deleted file mode 100644
index d623adacf9..0000000000
--- a/tests/script/sh/abs_max.c
+++ /dev/null
@@ -1,88 +0,0 @@
-#include
-#include
-#include
-
-typedef struct SUdfInit{
- int maybe_null; /* 1 if function can return NULL */
- int decimals; /* for real functions */
- long long length; /* For string functions */
- char *ptr; /* free pointer for function data */
- int const_item; /* 0 if result is independent of arguments */
-} SUdfInit;
-
-
-#define TSDB_DATA_INT_NULL 0x80000000LL
-#define TSDB_DATA_BIGINT_NULL 0x8000000000000000LL
-
-void abs_max(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput,
- int* numOfOutput, short otype, short obytes, SUdfInit* buf) {
- int i;
- int r = 0;
- printf("abs_max input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf);
- if (itype == 5) {
- r=*(long *)dataOutput;
- *numOfOutput=0;
-
- for(i=0;i r) {
- r = v;
- }
- }
-
- *(long *)dataOutput=r;
-
- printf("abs_max out, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput);
- }
-}
-
-
-
-void abs_max_finalize(char* dataOutput, char* interBuf, int* numOfOutput, SUdfInit* buf) {
- int i;
- int r = 0;
- printf("abs_max_finalize dataoutput:%p:%d, numOfOutput:%d, buf:%p\n", dataOutput, *dataOutput, *numOfOutput, buf);
- *numOfOutput=1;
- printf("abs_max finalize, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput);
-}
-
-void abs_max_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf) {
- int r = 0;
-
- if (numOfRows > 0) {
- r = *((long *)data);
- }
- printf("abs_max_merge numOfRows:%d, dataoutput:%p, buf:%p\n", numOfRows, dataOutput, buf);
- for (int i = 1; i < numOfRows; ++i) {
- printf("abs_max_merge %d - %ld\n", i, *((long *)data + i));
- if (*((long*)data + i) > r) {
- r= *((long*)data + i);
- }
- }
-
- *(long*)dataOutput=r;
- if (numOfRows > 0) {
- *numOfOutput=1;
- } else {
- *numOfOutput=0;
- }
-
- printf("abs_max_merge, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput);
-}
-
-
-int abs_max_init(SUdfInit* buf) {
- printf("abs_max init\n");
- return 0;
-}
-
-
-void abs_max_destroy(SUdfInit* buf) {
- printf("abs_max destroy\n");
-}
-
diff --git a/tests/script/sh/add_one.c b/tests/script/sh/add_one.c
deleted file mode 100644
index e12cf8f26f..0000000000
--- a/tests/script/sh/add_one.c
+++ /dev/null
@@ -1,33 +0,0 @@
-#include
-#include
-#include
-
-typedef struct SUdfInit{
- int maybe_null; /* 1 if function can return NULL */
- int decimals; /* for real functions */
- long long length; /* For string functions */
- char *ptr; /* free pointer for function data */
- int const_item; /* 0 if result is independent of arguments */
-} SUdfInit;
-
-void add_one(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBUf, char* tsOutput,
- int* numOfOutput, short otype, short obytes, SUdfInit* buf) {
- int i;
- int r = 0;
- printf("add_one input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf);
- if (itype == 4) {
- for(i=0;i
-#include
-#include
-
-typedef struct SUdfInit{
- int maybe_null; /* 1 if function can return NULL */
- int decimals; /* for real functions */
- long long length; /* For string functions */
- char *ptr; /* free pointer for function data */
- int const_item; /* 0 if result is independent of arguments */
-} SUdfInit;
-
-typedef struct SDemo{
- double sum;
- int num;
- short otype;
-}SDemo;
-
-#define FLOAT_NULL 0x7FF00000 // it is an NAN
-#define DOUBLE_NULL 0x7FFFFF0000000000LL // it is an NAN
-
-
-void demo(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput,
- int* numOfOutput, short otype, short obytes, SUdfInit* buf) {
- int i;
- double r = 0;
- SDemo *p = (SDemo *)interBuf;
- SDemo *q = (SDemo *)dataOutput;
- printf("demo input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, interBUf:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, interBuf, tsOutput, numOfOutput, buf);
-
- for(i=0;isum += r*r;
- }
-
- p->otype = otype;
- p->num += numOfRows;
-
- q->sum = p->sum;
- q->num = p->num;
- q->otype = p->otype;
-
- *numOfOutput=1;
-
- printf("demo out, sum:%f, num:%d, numOfOutput:%d\n", p->sum, p->num, *numOfOutput);
-}
-
-
-void demo_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf) {
- int i;
- SDemo *p = (SDemo *)data;
- SDemo res = {0};
- printf("demo_merge input data:%p, rows:%d, dataoutput:%p, numOfOutput:%p, buf:%p\n", data, numOfRows, dataOutput, numOfOutput, buf);
-
- for(i=0;isum * p->sum;
- res.num += p->num;
- p++;
- }
-
- p->sum = res.sum;
- p->num = res.num;
-
- *numOfOutput=1;
-
- printf("demo out, sum:%f, num:%d, numOfOutput:%d\n", p->sum, p->num, *numOfOutput);
-}
-
-
-
-void demo_finalize(char* dataOutput, char* interBuf, int* numOfOutput, SUdfInit* buf) {
- SDemo *p = (SDemo *)interBuf;
- printf("demo_finalize interbuf:%p, numOfOutput:%p, buf:%p, sum:%f, num:%d\n", interBuf, numOfOutput, buf, p->sum, p->num);
- if (p->otype == 6) {
- if (p->num != 30000) {
- *(unsigned int *)dataOutput = FLOAT_NULL;
- } else {
- *(float *)dataOutput = (float)(p->sum / p->num);
- }
- printf("finalize values:%f\n", *(float *)dataOutput);
- } else if (p->otype == 7) {
- if (p->num != 30000) {
- *(unsigned long long *)dataOutput = DOUBLE_NULL;
- } else {
- *(double *)dataOutput = (double)(p->sum / p->num);
- }
- printf("finalize values:%f\n", *(double *)dataOutput);
- }
-
- *numOfOutput=1;
-
- printf("demo finalize, numOfOutput:%d\n", *numOfOutput);
-}
-
-
-int demo_init(SUdfInit* buf) {
- printf("demo init\n");
- return 0;
-}
-
-
-void demo_destroy(SUdfInit* buf) {
- printf("demo destroy\n");
-}
-
diff --git a/tests/script/sh/demo.lua b/tests/script/sh/demo.lua
deleted file mode 100644
index c5e5582fc3..0000000000
--- a/tests/script/sh/demo.lua
+++ /dev/null
@@ -1,43 +0,0 @@
-funcName = "test"
-
-global = {}
-
-function test_init()
- return global
-end
-
-function test_add(rows, ans, key)
- t = {}
- t["sum"] = 0.0
- t["num"] = 0
- for i=1, #rows do
- t["sum"] = t["sum"] + rows[i] * rows[i]
- end
- t["num"] = #rows
-
-
- if (ans[key] ~= nil)
- then
- ans[key]["sum"] = ans[key]["sum"] + t["sum"]
- ans[key]["num"] = ans[key]["num"] + t["num"]
- else
- ans[key] = t
- end
-
- return ans;
-end
-
-function test_finalize(ans, key)
- local ret = 0.0
-
- if (ans[key] ~= nil and ans[key]["num"] == 30000)
- then
- ret = ans[key]["sum"]/ans[key]["num"]
- ans[key]["sum"] = 0.0
- ans[key]["num"] = 0
- else
- ret = inf
- end
-
- return ret, ans
-end
diff --git a/tests/script/sh/sum_double.c b/tests/script/sh/sum_double.c
deleted file mode 100644
index d6eea5d291..0000000000
--- a/tests/script/sh/sum_double.c
+++ /dev/null
@@ -1,84 +0,0 @@
-#include
-#include
-#include
-
-typedef struct SUdfInit{
- int maybe_null; /* 1 if function can return NULL */
- int decimals; /* for real functions */
- long long length; /* For string functions */
- char *ptr; /* free pointer for function data */
- int const_item; /* 0 if result is independent of arguments */
-} SUdfInit;
-
-#define TSDB_DATA_INT_NULL 0x80000000LL
-
-
-void sum_double(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput,
- int* numOfOutput, short otype, short obytes, SUdfInit* buf) {
- int i;
- int r = 0;
- printf("sum_double input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf);
- if (itype == 4) {
- r=*(int *)dataOutput;
- *numOfOutput=0;
-
- for(i=0;iptr)=*(int*)dataOutput*2;
- *(int*)dataOutput=*(int*)(buf->ptr);
- printf("sum_double finalize, dataoutput:%d, numOfOutput:%d\n", *(int *)dataOutput, *numOfOutput);
-}
-
-void sum_double_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf) {
- int r = 0;
- int sum = 0;
-
- printf("sum_double_merge numOfRows:%d, dataoutput:%p, buf:%p\n", numOfRows, dataOutput, buf);
- for (int i = 0; i < numOfRows; ++i) {
- printf("sum_double_merge %d - %d\n", i, *((int*)data + i));
- sum +=*((int*)data + i);
- }
-
- *(int*)dataOutput+=sum;
- if (numOfRows > 0) {
- *numOfOutput=1;
- } else {
- *numOfOutput=0;
- }
-
- printf("sum_double_merge, dataoutput:%d, numOfOutput:%d\n", *(int *)dataOutput, *numOfOutput);
-}
-
-
-int sum_double_init(SUdfInit* buf) {
- buf->maybe_null=1;
- buf->ptr = taosMemoryMalloc(sizeof(int));
- printf("sum_double init\n");
- return 0;
-}
-
-
-void sum_double_destroy(SUdfInit* buf) {
- taosMemoryFree(buf->ptr);
- printf("sum_double destroy\n");
-}
-
diff --git a/tests/script/tsim/column/table.sim b/tests/script/tsim/column/table.sim
index 4f1d32c373..03c4799681 100644
--- a/tests/script/tsim/column/table.sim
+++ b/tests/script/tsim/column/table.sim
@@ -159,6 +159,7 @@ if $data01 != 10 then
return -1
endi
if $data02 != 4.500000000 then
+ print expect 4.500000000, actual: $data02
return -1
endi
if $data03 != 4.500000000 then
diff --git a/tests/script/tsim/db/basic1.sim b/tests/script/tsim/db/basic1.sim
index 3f6bf05398..5d7d0bd9e9 100644
--- a/tests/script/tsim/db/basic1.sim
+++ b/tests/script/tsim/db/basic1.sim
@@ -133,10 +133,10 @@ if $data(4)[5] != localhost:7100 then
return -1
endi
-#sql show vnodes 'localhost:7100'
-#if $rows != 9 then
-# return -1
-#endi
+sql show vnodes 'localhost:7100'
+if $rows != 9 then
+ return -1
+endi
print =============== drop database
sql drop database d2
diff --git a/tests/script/tsim/parser/commit.sim b/tests/script/tsim/parser/commit.sim
index ae19a4803b..a9bf8b26eb 100644
--- a/tests/script/tsim/parser/commit.sim
+++ b/tests/script/tsim/parser/commit.sim
@@ -97,6 +97,7 @@ while $loop <= $loops
endw
sql select count(*) from $stb
if $data00 != $totalNum then
+ print expect $totalNum , actual: $data00
return -1
endi
$loop = $loop + 1
diff --git a/tests/script/tsim/parser/slimit_alter_tags.sim b/tests/script/tsim/parser/slimit_alter_tags.sim
index 3827b14b45..b5afbfa56e 100644
--- a/tests/script/tsim/parser/slimit_alter_tags.sim
+++ b/tests/script/tsim/parser/slimit_alter_tags.sim
@@ -128,6 +128,7 @@ if $rows != 5 then
return -1
endi
if $data00 != $rowNum then
+ print expect $rowNum , actual: $data00
return -1
endi
if $data10 != $rowNum then
diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim
index d9777d5133..8942f7f702 100644
--- a/tests/script/tsim/stream/basic1.sim
+++ b/tests/script/tsim/stream/basic1.sim
@@ -5,7 +5,7 @@ sleep 50
sql connect
print =============== create database
-sql create database test vgroups 1
+sql create database test vgroups 1;
sql select * from information_schema.ins_databases
if $rows != 3 then
return -1
@@ -13,7 +13,7 @@ endi
print $data00 $data01 $data02
-sql use test
+sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double);
@@ -588,4 +588,38 @@ if $data00 != 5 then
goto loop3
endi
+#max,min selectivity
+sql create database test3 vgroups 1;
+sql use test3;
+sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int);
+sql create table ts1 using st tags(1,1,1);
+sql create stream stream_t3 trigger at_once into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from ts1 interval(10s) ;
+
+sql insert into ts1 values(1648791211000,1,2,3);
+sleep 50
+sql insert into ts1 values(1648791222001,2,2,3);
+sleep 50
+
+$loop_count = 0
+loop3:
+sql select * from streamtST3;
+
+sleep 300
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+# row 0
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop3
+endi
+
+# row 1
+if $data12 != 2 then
+ print =====data12=$data12
+ goto loop3
+endi
+
system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/stream/deleteInterval.sim b/tests/script/tsim/stream/deleteInterval.sim
new file mode 100644
index 0000000000..dfd0ddc9d3
--- /dev/null
+++ b/tests/script/tsim/stream/deleteInterval.sim
@@ -0,0 +1,419 @@
+$loop_all = 0
+looptest:
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 200
+sql connect
+
+sql drop stream if exists streams0;
+sql drop stream if exists streams1;
+sql drop stream if exists streams2;
+sql drop stream if exists streams3;
+sql drop stream if exists streams4;
+sql drop database if exists test;
+sql create database test vgroups 1;
+sql use test;
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3 from t1 interval(10s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sleep 200
+sql delete from t1 where ts = 1648791213000;
+
+$loop_count = 0
+
+loop0:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 0 then
+ print =====rows=$rows
+ goto loop0
+endi
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop1:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop1
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop1
+endi
+
+sql insert into t1 values(1648791213000,1,1,1,1.0);
+sql insert into t1 values(1648791213001,2,2,2,2.0);
+sql insert into t1 values(1648791213002,3,3,3,3.0);
+sql insert into t1 values(1648791213003,4,4,4,4.0);
+
+sleep 200
+sql delete from t1 where ts >= 1648791213001 and ts <= 1648791213002;
+
+$loop_count = 0
+
+loop3:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop3
+endi
+
+if $data02 != 4 then
+ print =====data02=$data02
+ goto loop3
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+
+$loop_count = 0
+
+loop4:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop4
+endi
+
+sleep 200
+
+sql delete from t1 where ts >= 1648791223000 and ts <= 1648791223003;
+
+$loop_count = 0
+
+loop5:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop5
+endi
+
+if $data02 != 4 then
+ print =====data02=$data02
+ goto loop5
+endi
+
+sql insert into t1 values(1648791213000,1,1,1,1.0);
+sql insert into t1 values(1648791213005,2,2,2,2.0);
+sql insert into t1 values(1648791213006,3,3,3,3.0);
+sql insert into t1 values(1648791213007,4,4,4,4.0);
+
+sql insert into t1 values(1648791223000,1,1,1,1.0);
+sql insert into t1 values(1648791223001,2,2,2,2.0);
+sql insert into t1 values(1648791223002,3,3,3,3.0);
+sql insert into t1 values(1648791223003,4,4,4,4.0);
+
+sql insert into t1 values(1648791233000,1,1,1,1.0);
+sql insert into t1 values(1648791233001,2,2,2,2.0);
+sql insert into t1 values(1648791233008,3,3,3,3.0);
+sql insert into t1 values(1648791233009,4,4,4,4.0);
+
+sql delete from t1 where ts >= 1648791213001 and ts <= 1648791233005;
+
+$loop_count = 0
+
+loop6:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop6
+endi
+
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop6
+endi
+
+if $data11 != 2 then
+ print =====data11=$data11
+ goto loop6
+endi
+
+if $data12 != 4 then
+ print =====data12=$data12
+ goto loop6
+endi
+
+sql drop stream if exists streams2;
+sql drop database if exists test2;
+sql create database test2 vgroups 4;
+sql use test2;
+sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
+sql create table t1 using st tags(1,1,1);
+sql create table t2 using st tags(2,2,2);
+sql create stream streams2 trigger at_once into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st interval(10s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop7:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 1 then
+ print =====rows=$rows
+ goto loop7
+endi
+
+sleep 200
+
+sql delete from t1 where ts = 1648791213000;
+
+$loop_count = 0
+
+loop8:
+sleep 200
+
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop8
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop8
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+sql insert into t2 values(1648791223000,1,2,3,1.0);
+sql insert into t2 values(1648791223001,1,2,3,1.0);
+sql insert into t2 values(1648791223002,3,2,3,1.0);
+sql insert into t2 values(1648791223003,3,2,3,1.0);
+
+sleep 200
+
+sql delete from t2 where ts >= 1648791223000 and ts <= 1648791223001;
+
+$loop_count = 0
+
+loop11:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop11
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop11
+endi
+
+if $data11 != 6 then
+ print =====data11=$data11
+ goto loop11
+endi
+
+if $data12 != 3 then
+ print =====data12=$data12
+ goto loop11
+endi
+
+sleep 200
+
+sql delete from st where ts >= 1648791223000 and ts <= 1648791223003;
+
+$loop_count = 0
+
+loop12:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 1 then
+ print =====rows=$rows
+ goto loop12
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop12
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop12
+endi
+
+sql insert into t1 values(1648791213004,3,2,3,1.0);
+sql insert into t1 values(1648791213005,3,2,3,1.0);
+sql insert into t1 values(1648791213006,3,2,3,1.0);
+sql insert into t1 values(1648791223004,1,2,3,1.0);
+sql insert into t2 values(1648791213004,3,2,3,1.0);
+sql insert into t2 values(1648791213005,3,2,3,1.0);
+sql insert into t2 values(1648791213006,3,2,3,1.0);
+sql insert into t2 values(1648791223004,1,2,3,1.0);
+
+sleep 200
+
+sql delete from t2 where ts >= 1648791213004 and ts <= 1648791213006;
+
+$loop_count = 0
+
+loop13:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop13
+endi
+
+if $data01 != 4 then
+ print =====data01=$data01
+ goto loop13
+endi
+
+if $data02 != 3 then
+ print =====data02=$data02
+ goto loop13
+endi
+
+if $data11 != 2 then
+ print =====data11=$data11
+ goto loop13
+endi
+
+if $data12 != 1 then
+ print =====data12=$data12
+ goto loop13
+endi
+
+sql insert into t1 values(1648791223005,1,2,3,1.0);
+sql insert into t1 values(1648791223006,1,2,3,1.0);
+sql insert into t2 values(1648791223005,1,2,3,1.0);
+sql insert into t2 values(1648791223006,1,2,3,1.0);
+
+sql insert into t1 values(1648791233005,4,2,3,1.0);
+sql insert into t1 values(1648791233006,2,2,3,1.0);
+sql insert into t2 values(1648791233005,5,2,3,1.0);
+sql insert into t2 values(1648791233006,3,2,3,1.0);
+
+sleep 200
+
+sql delete from st where ts >= 1648791213001 and ts <= 1648791233005;
+
+$loop_count = 0
+
+loop14:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop14
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop14
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop14
+endi
+
+if $data11 != 2 then
+ print =====data11=$data11
+ goto loop14
+endi
+
+if $data12 != 3 then
+ print =====data12=$data12
+ goto loop14
+endi
+
+$loop_all = $loop_all + 1
+print ============loop_all=$loop_all
+
+system sh/stop_dnodes.sh
+
+#goto looptest
\ No newline at end of file
diff --git a/tests/script/tsim/stream/deleteSession.sim b/tests/script/tsim/stream/deleteSession.sim
new file mode 100644
index 0000000000..541609633b
--- /dev/null
+++ b/tests/script/tsim/stream/deleteSession.sim
@@ -0,0 +1,532 @@
+$loop_all = 0
+looptest:
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 200
+sql connect
+
+sql drop stream if exists streams0;
+sql drop stream if exists streams1;
+sql drop stream if exists streams2;
+sql drop stream if exists streams3;
+sql drop stream if exists streams4;
+sql drop database if exists test;
+sql create database test vgroups 1;
+sql use test;
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3 from t1 session(ts, 5s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sleep 200
+sql delete from t1 where ts = 1648791213000;
+
+$loop_count = 0
+
+loop0:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 0 then
+ print =====rows=$rows
+ goto loop0
+endi
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop1:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop1
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop1
+endi
+
+sql insert into t1 values(1648791213000,1,1,1,1.0);
+sql insert into t1 values(1648791213001,2,2,2,2.0);
+sql insert into t1 values(1648791213002,3,3,3,3.0);
+sql insert into t1 values(1648791213003,4,4,4,4.0);
+
+sleep 200
+sql delete from t1 where ts >= 1648791213001 and ts <= 1648791213002;
+
+$loop_count = 0
+
+loop3:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop3
+endi
+
+if $data02 != 4 then
+ print =====data02=$data02
+ goto loop3
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+
+$loop_count = 0
+
+loop4:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop4
+endi
+
+sleep 200
+
+sql delete from t1 where ts >= 1648791223000 and ts <= 1648791223003;
+
+$loop_count = 0
+
+loop5:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop5
+endi
+
+if $data02 != 4 then
+ print =====data02=$data02
+ goto loop5
+endi
+
+sql insert into t1 values(1648791213000,1,1,1,1.0);
+sql insert into t1 values(1648791213005,2,2,2,2.0);
+sql insert into t1 values(1648791213006,3,3,3,3.0);
+sql insert into t1 values(1648791213007,4,4,4,4.0);
+
+sql insert into t1 values(1648791223000,1,1,1,1.0);
+sql insert into t1 values(1648791223001,2,2,2,2.0);
+sql insert into t1 values(1648791223002,3,3,3,3.0);
+sql insert into t1 values(1648791223003,4,4,4,4.0);
+
+sql insert into t1 values(1648791233000,1,1,1,1.0);
+sql insert into t1 values(1648791233001,2,2,2,2.0);
+sql insert into t1 values(1648791233008,3,3,3,3.0);
+sql insert into t1 values(1648791233009,4,4,4,4.0);
+
+sql delete from t1 where ts >= 1648791213001 and ts <= 1648791233005;
+
+$loop_count = 0
+
+loop6:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop6
+endi
+
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop6
+endi
+
+if $data11 != 2 then
+ print =====data11=$data11
+ goto loop6
+endi
+
+if $data12 != 4 then
+ print =====data12=$data12
+ goto loop6
+endi
+
+sql drop stream if exists streams2;
+sql drop database if exists test2;
+sql create database test2 vgroups 4;
+sql use test2;
+sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
+sql create table t1 using st tags(1,1,1);
+sql create table t2 using st tags(2,2,2);
+sql create stream streams2 trigger at_once into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop7:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 1 then
+ print =====rows=$rows
+ goto loop7
+endi
+
+sleep 200
+
+sql delete from t1 where ts = 1648791213000;
+
+$loop_count = 0
+
+loop8:
+sleep 200
+
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop8
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop8
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+sql insert into t2 values(1648791223000,1,2,3,1.0);
+sql insert into t2 values(1648791223001,1,2,3,1.0);
+sql insert into t2 values(1648791223002,3,2,3,1.0);
+sql insert into t2 values(1648791223003,3,2,3,1.0);
+
+sleep 200
+
+sql delete from t2 where ts >= 1648791223000 and ts <= 1648791223001;
+
+$loop_count = 0
+
+loop11:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop11
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop11
+endi
+
+if $data11 != 6 then
+ print =====data11=$data11
+ goto loop11
+endi
+
+if $data12 != 3 then
+ print =====data12=$data12
+ goto loop11
+endi
+
+sleep 200
+
+sql delete from st where ts >= 1648791223000 and ts <= 1648791223003;
+
+$loop_count = 0
+
+loop12:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 1 then
+ print =====rows=$rows
+ goto loop12
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop12
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop12
+endi
+
+sql insert into t1 values(1648791213004,3,2,3,1.0);
+sql insert into t1 values(1648791213005,3,2,3,1.0);
+sql insert into t1 values(1648791213006,3,2,3,1.0);
+sql insert into t1 values(1648791223004,1,2,3,1.0);
+sql insert into t2 values(1648791213004,3,2,3,1.0);
+sql insert into t2 values(1648791213005,3,2,3,1.0);
+sql insert into t2 values(1648791213006,3,2,3,1.0);
+sql insert into t2 values(1648791223004,1,2,3,1.0);
+
+sleep 200
+
+sql delete from t2 where ts >= 1648791213004 and ts <= 1648791213006;
+
+$loop_count = 0
+
+loop13:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop13
+endi
+
+if $data01 != 4 then
+ print =====data01=$data01
+ goto loop13
+endi
+
+if $data02 != 3 then
+ print =====data02=$data02
+ goto loop13
+endi
+
+if $data11 != 2 then
+ print =====data11=$data11
+ goto loop13
+endi
+
+if $data12 != 1 then
+ print =====data12=$data12
+ goto loop13
+endi
+
+sql insert into t1 values(1648791223005,1,2,3,1.0);
+sql insert into t1 values(1648791223006,1,2,3,1.0);
+sql insert into t2 values(1648791223005,1,2,3,1.0);
+sql insert into t2 values(1648791223006,1,2,3,1.0);
+
+sql insert into t1 values(1648791233005,4,2,3,1.0);
+sql insert into t1 values(1648791233006,2,2,3,1.0);
+sql insert into t2 values(1648791233005,5,2,3,1.0);
+sql insert into t2 values(1648791233006,3,2,3,1.0);
+
+sleep 200
+
+sql delete from st where ts >= 1648791213001 and ts <= 1648791233005;
+
+$loop_count = 0
+
+loop14:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop14
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop14
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop14
+endi
+
+if $data11 != 2 then
+ print =====data11=$data11
+ goto loop14
+endi
+
+if $data12 != 3 then
+ print =====data12=$data12
+ goto loop14
+endi
+
+sql drop stream if exists streams1;
+sql drop stream if exists streams2;
+sql drop stream if exists streams3;
+sql drop database if exists test3;
+sql drop database if exists test;
+sql create database test3 vgroups 4;
+sql create database test vgroups 1;
+sql use test3;
+sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
+sql create table t1 using st tags(1,1,1);
+sql create table t2 using st tags(2,2,2);
+sql create stream streams3 trigger at_once into test.streamt3 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s);
+
+sql insert into t1 values(1648791210000,1,1,1,NULL);
+sql insert into t1 values(1648791210001,2,2,2,NULL);
+sql insert into t2 values(1648791213001,3,3,3,NULL);
+sql insert into t2 values(1648791213003,4,4,4,NULL);
+sql insert into t1 values(1648791216000,5,5,5,NULL);
+sql insert into t1 values(1648791216002,6,6,6,NULL);
+sql insert into t1 values(1648791216004,7,7,7,NULL);
+sql insert into t2 values(1648791218001,8,8,8,NULL);
+sql insert into t2 values(1648791218003,9,9,9,NULL);
+sql insert into t1 values(1648791222000,10,10,10,NULL);
+sql insert into t1 values(1648791222003,11,11,11,NULL);
+sql insert into t1 values(1648791222005,12,12,12,NULL);
+
+sql insert into t1 values(1648791232005,13,13,13,NULL);
+sql insert into t2 values(1648791242005,14,14,14,NULL);
+
+$loop_count = 0
+
+loop19:
+sleep 200
+sql select * from test.streamt3 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 3 then
+ print =====rows=$rows
+ goto loop19
+endi
+
+sql delete from t2 where ts >= 1648791213001 and ts <= 1648791218003;
+
+$loop_count = 0
+
+loop20:
+sleep 200
+sql select * from test.streamt3 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 5 then
+ print =====rows=$rows
+ goto loop20
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop20
+endi
+
+if $data02 != 2 then
+ print =====data02=$data02
+ goto loop20
+endi
+
+if $data11 != 3 then
+ print =====data11=$data11
+ goto loop20
+endi
+
+if $data12 != 7 then
+ print =====data12=$data12
+ goto loop20
+endi
+
+if $data21 != 3 then
+ print =====data21=$data21
+ goto loop20
+endi
+
+if $data22 != 12 then
+ print =====data22=$data22
+ goto loop20
+endi
+
+if $data31 != 1 then
+ print =====data31=$data31
+ goto loop20
+endi
+
+if $data32 != 13 then
+ print =====data32=$data32
+ goto loop20
+endi
+
+if $data41 != 1 then
+ print =====data41=$data41
+ goto loop20
+endi
+
+if $data42 != 14 then
+ print =====data42=$data42
+ goto loop20
+endi
+
+$loop_all = $loop_all + 1
+print ============loop_all=$loop_all
+
+system sh/stop_dnodes.sh
+
+#goto looptest
\ No newline at end of file
diff --git a/tests/script/tsim/stream/deleteState.sim b/tests/script/tsim/stream/deleteState.sim
new file mode 100644
index 0000000000..ecd9f55340
--- /dev/null
+++ b/tests/script/tsim/stream/deleteState.sim
@@ -0,0 +1,198 @@
+$loop_all = 0
+looptest:
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 200
+sql connect
+
+sql drop stream if exists streams0;
+sql drop stream if exists streams1;
+sql drop stream if exists streams2;
+sql drop stream if exists streams3;
+sql drop stream if exists streams4;
+sql drop database if exists test;
+sql create database test vgroups 1;
+sql use test;
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(b) c3 from t1 state_window(a);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sleep 200
+sql delete from t1 where ts = 1648791213000;
+
+$loop_count = 0
+
+loop0:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 0 then
+ print =====rows=$rows
+ goto loop0
+endi
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop1:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop1
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop1
+endi
+
+sql insert into t1 values(1648791213000,1,1,1,1.0);
+sql insert into t1 values(1648791213001,1,2,2,2.0);
+sql insert into t1 values(1648791213002,1,3,3,3.0);
+sql insert into t1 values(1648791213003,1,4,4,4.0);
+
+sleep 200
+sql delete from t1 where ts >= 1648791213001 and ts <= 1648791213002;
+
+$loop_count = 0
+
+loop3:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop3
+endi
+
+if $data02 != 4 then
+ print =====data02=$data02
+ goto loop3
+endi
+
+sql insert into t1 values(1648791223000,2,2,3,1.0);
+sql insert into t1 values(1648791223001,2,2,3,1.0);
+sql insert into t1 values(1648791223002,2,2,3,1.0);
+sql insert into t1 values(1648791223003,2,2,3,1.0);
+
+$loop_count = 0
+
+loop4:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop4
+endi
+
+sleep 200
+
+sql delete from t1 where ts >= 1648791223000 and ts <= 1648791223003;
+
+$loop_count = 0
+
+loop5:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop5
+endi
+
+if $data02 != 4 then
+ print =====data02=$data02
+ goto loop5
+endi
+
+sql insert into t1 values(1648791213000,1,1,1,1.0);
+sql insert into t1 values(1648791213005,1,2,2,2.0);
+sql insert into t1 values(1648791213006,1,3,3,3.0);
+sql insert into t1 values(1648791213007,1,4,4,4.0);
+
+sql insert into t1 values(1648791223000,2,1,1,1.0);
+sql insert into t1 values(1648791223001,2,2,2,2.0);
+sql insert into t1 values(1648791223002,2,3,3,3.0);
+sql insert into t1 values(1648791223003,2,4,4,4.0);
+
+sql insert into t1 values(1648791233000,3,1,1,1.0);
+sql insert into t1 values(1648791233001,3,2,2,2.0);
+sql insert into t1 values(1648791233008,3,3,3,3.0);
+sql insert into t1 values(1648791233009,3,4,4,4.0);
+
+sql delete from t1 where ts >= 1648791213001 and ts <= 1648791233005;
+
+$loop_count = 0
+
+loop6:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop6
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop6
+endi
+
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop6
+endi
+
+if $data11 != 2 then
+ print =====data11=$data11
+ goto loop6
+endi
+
+if $data12 != 4 then
+ print =====data12=$data12
+ goto loop6
+endi
+
+
+$loop_all = $loop_all + 1
+print ============loop_all=$loop_all
+
+system sh/stop_dnodes.sh
+
+#goto looptest
\ No newline at end of file
diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim
index b6b427343e..9b2e940556 100644
--- a/tests/script/tsim/stream/distributeInterval0.sim
+++ b/tests/script/tsim/stream/distributeInterval0.sim
@@ -198,7 +198,7 @@ endi
sql select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5, avg(d) from st interval(10s);
-sql create database test1 vgroups 1;
+sql create database test1 vgroups 4;
sql use test1;
sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int);
sql create table ts1 using st tags(1,1,1);
@@ -232,4 +232,43 @@ if $data11 != 2 then
goto loop2
endi
+#max,min selectivity
+sql create database test3 vgroups 4;
+sql use test3;
+sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int);
+sql create table ts1 using st tags(1,1,1);
+sql create table ts2 using st tags(2,2,2);
+sql create stream stream_t3 trigger at_once into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from st interval(10s) ;
+
+sql insert into ts1 values(1648791211000,1,2,3);
+sleep 50
+sql insert into ts1 values(1648791222001,2,2,3);
+sleep 50
+sql insert into ts2 values(1648791211000,1,2,3);
+sleep 50
+sql insert into ts2 values(1648791222001,2,2,3);
+sleep 50
+
+$loop_count = 0
+loop3:
+sql select * from streamtST3;
+
+sleep 300
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+# row 0
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop3
+endi
+
+# row 1
+if $data12 != 2 then
+ print =====data12=$data12
+ goto loop3
+endi
+
system sh/stop_dnodes.sh
diff --git a/tests/script/tsim/stream/partitionbyColumn0.sim b/tests/script/tsim/stream/partitionbyColumnInterval.sim
similarity index 98%
rename from tests/script/tsim/stream/partitionbyColumn0.sim
rename to tests/script/tsim/stream/partitionbyColumnInterval.sim
index d91d4b7bf0..24fdb9c994 100644
--- a/tests/script/tsim/stream/partitionbyColumn0.sim
+++ b/tests/script/tsim/stream/partitionbyColumnInterval.sim
@@ -24,7 +24,7 @@ sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
$loop_count = 0
loop0:
-sleep 100
+sleep 50
sql select * from streamt order by c1, c4, c2, c3;
$loop_count = $loop_count + 1
@@ -48,7 +48,7 @@ sql insert into t1 values(1648791213000,1,2,3,1.0);
$loop_count = 0
loop1:
-sleep 100
+sleep 50
sql select * from streamt order by c1, c4, c2, c3;
$loop_count = $loop_count + 1
@@ -71,7 +71,7 @@ sql insert into t1 values(1648791213000,2,2,3,1.0);
$loop_count = 0
loop2:
-sleep 100
+sleep 50
sql select * from streamt order by c1, c4, c2, c3;
$loop_count = $loop_count + 1
@@ -97,7 +97,7 @@ sql insert into t1 values(1648791213002,1,2,3,1.0);
$loop_count = 0
loop3:
-sleep 100
+sleep 50
sql select * from streamt order by c1, c4, c2, c3;
$loop_count = $loop_count + 1
@@ -134,7 +134,7 @@ sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (16
$loop_count = 0
loop4:
-sleep 100
+sleep 50
sql select * from streamt order by c1, c4, c2, c3;
$loop_count = $loop_count + 1
@@ -208,7 +208,7 @@ sql insert into t1 values(1648791213001,1,2,3,2.0);
$loop_count = 0
loop5:
-sleep 100
+sleep 50
sql select * from streamt1 order by c1, c4, c2, c3;
$loop_count = $loop_count + 1
@@ -229,7 +229,7 @@ sql insert into t1 values(1648791213001,1,1,6,2.0) (1648791223002,1,1,7,2.0);
$loop_count = 0
loop6:
-sleep 100
+sleep 50
sql select * from streamt1 order by c1, c4, c2, c3;
$loop_count = $loop_count + 1
@@ -294,7 +294,7 @@ sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
$loop_count = 0
loop7:
-sleep 100
+sleep 50
sql select * from test.streamt2 order by c1, c2, c3;
$loop_count = $loop_count + 1
@@ -318,7 +318,7 @@ sql insert into t2 values(1648791213000,1,2,3,1.0);
$loop_count = 0
loop8:
-sleep 100
+sleep 50
sql select * from test.streamt2 order by c1, c2, c3;
$loop_count = $loop_count + 1
@@ -342,7 +342,7 @@ sql insert into t2 values(1648791213000,2,2,3,1.0);
$loop_count = 0
loop9:
-sleep 100
+sleep 50
sql select * from test.streamt2 order by c1, c2, c3;
$loop_count = $loop_count + 1
@@ -372,7 +372,7 @@ sql insert into t2 values(1648791213002,1,2,3,1.0);
$loop_count = 0
loop10:
-sleep 100
+sleep 50
sql select * from test.streamt2 order by c1, c2, c3;
$loop_count = $loop_count + 1
@@ -414,7 +414,7 @@ sql insert into t2 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (16
$loop_count = 0
loop11:
-sleep 100
+sleep 50
sql select * from test.streamt2 order by c1, c2, c3;
$loop_count = $loop_count + 1
@@ -492,7 +492,7 @@ sql insert into t4 values(1648791213000,1,2,3,1.0);
$loop_count = 0
loop13:
-sleep 100
+sleep 50
sql select * from test.streamt4 order by c1, c2, c3;
$loop_count = $loop_count + 1
@@ -534,7 +534,7 @@ sql insert into t1 values(1648791213000,1,2,3,1.0);
$loop_count = 0
loop14:
-sleep 100
+sleep 50
sql select * from test.streamt4 order by c1, c2, c3;
$loop_count = $loop_count + 1
diff --git a/tests/script/tsim/stream/partitionbyColumn1.sim b/tests/script/tsim/stream/partitionbyColumnSession.sim
similarity index 93%
rename from tests/script/tsim/stream/partitionbyColumn1.sim
rename to tests/script/tsim/stream/partitionbyColumnSession.sim
index 7f5c53ebe3..1742d52cf0 100644
--- a/tests/script/tsim/stream/partitionbyColumn1.sim
+++ b/tests/script/tsim/stream/partitionbyColumnSession.sim
@@ -24,11 +24,11 @@ sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
$loop_count = 0
loop0:
-sleep 300
+sleep 50
sql select * from streamt order by c1, c4, c2, c3;
$loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
return -1
endi
@@ -45,12 +45,14 @@ endi
sql insert into t1 values(1648791213000,1,2,3,1.0);
+$loop_count = 0
+
loop1:
-sleep 300
+sleep 50
sql select * from streamt order by c1, c4, c2, c3;
$loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
return -1
endi
@@ -66,12 +68,14 @@ endi
sql insert into t1 values(1648791213000,2,2,3,1.0);
+$loop_count = 0
+
loop2:
-sleep 300
+sleep 50
sql select * from streamt order by c1, c4, c2, c3;
$loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
return -1
endi
@@ -90,12 +94,14 @@ sql insert into t1 values(1648791213001,2,2,3,1.0);
sql insert into t1 values(1648791213002,2,2,3,1.0);
sql insert into t1 values(1648791213002,1,2,3,1.0);
+$loop_count = 0
+
loop3:
-sleep 300
+sleep 50
sql select * from streamt order by c1, c4, c2, c3;
$loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
return -1
endi
@@ -125,12 +131,14 @@ sql insert into t1 values(1648791223002,3,2,3,1.0);
sql insert into t1 values(1648791223003,3,2,3,1.0);
sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
+$loop_count = 0
+
loop4:
-sleep 300
+sleep 50
sql select * from streamt order by c1, c4, c2, c3;
$loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
return -1
endi
@@ -199,11 +207,11 @@ sql insert into t1 values(1648791213001,1,2,3,2.0);
$loop_count = 0
loop5:
-sleep 300
+sleep 50
sql select * from streamt1 order by c1, c4, c2, c3;
$loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
return -1
endi
@@ -217,12 +225,14 @@ sql insert into t1 values(1648791223001,1,2,5,2.0);
sql insert into t1 values(1648791223002,1,2,5,2.0);
sql insert into t1 values(1648791213001,1,1,6,2.0) (1648791223002,1,1,7,2.0);
+$loop_count = 0
+
loop6:
-sleep 300
+sleep 50
sql select * from streamt1 order by c1, c4, c2, c3;
$loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
return -1
endi
@@ -282,11 +292,11 @@ sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
$loop_count = 0
loop7:
-sleep 300
+sleep 50
sql select * from test.streamt2 order by c1, c2, c3;
$loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
return -1
endi
@@ -303,12 +313,14 @@ endi
sql insert into t1 values(1648791213000,1,2,3,1.0);
sql insert into t2 values(1648791213000,1,2,3,1.0);
+$loop_count = 0
+
loop8:
-sleep 300
+sleep 50
sql select * from test.streamt2 order by c1, c2, c3;
$loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
return -1
endi
@@ -324,12 +336,15 @@ endi
sql insert into t1 values(1648791213000,2,2,3,1.0);
sql insert into t2 values(1648791213000,2,2,3,1.0);
+
+$loop_count = 0
+
loop9:
-sleep 300
+sleep 50
sql select * from test.streamt2 order by c1, c2, c3;
$loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
return -1
endi
@@ -352,12 +367,14 @@ sql insert into t2 values(1648791213001,2,2,3,1.0);
sql insert into t2 values(1648791213002,2,2,3,1.0);
sql insert into t2 values(1648791213002,1,2,3,1.0);
+$loop_count = 0
+
loop10:
-sleep 300
+sleep 50
sql select * from test.streamt2 order by c1, c2, c3;
$loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
return -1
endi
@@ -373,7 +390,7 @@ endi
if $data11 != 2 thenloop4
print =====data11=$data11
- goto loop3
+ goto loop10
endi
if $data12 != 1 then
@@ -392,12 +409,14 @@ sql insert into t2 values(1648791223002,3,2,3,1.0);
sql insert into t2 values(1648791223003,3,2,3,1.0);
sql insert into t2 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
+$loop_count = 0
+
loop11:
-sleep 300
+sleep 50
sql select * from test.streamt2 order by c1, c2, c3;
$loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
return -1
endi
@@ -470,17 +489,17 @@ sql insert into t4 values(1648791213000,1,2,3,1.0);
$loop_count = 0
loop13:
-sleep 300
+sleep 50
sql select * from test.streamt4 order by c1, c2, c3;
$loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
return -1
endi
if $rows != 2 then
print =====rows=$rows
- goto loop14
+ goto loop13
endi
if $data01 != 1 then
@@ -495,12 +514,12 @@ endi
if $data11 != 3 then
print =====data11=$data11
- goto loop11
+ goto loop13
endi
if $data12 != 2 then
print =====data12=$data12
- goto loop11
+ goto loop13
endi
sql insert into t4 values(1648791213000,2,2,3,1.0);
@@ -509,12 +528,14 @@ sql insert into t1 values(1648791233000,2,2,3,1.0);
sql insert into t1 values(1648791213000,1,2,3,1.0);
+$loop_count = 0
+
loop14:
-sleep 300
+sleep 50
sql select * from test.streamt4 order by c1, c2, c3;
$loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
return -1
endi
diff --git a/tests/script/tsim/stream/partitionbyColumn2.sim b/tests/script/tsim/stream/partitionbyColumnState.sim
similarity index 98%
rename from tests/script/tsim/stream/partitionbyColumn2.sim
rename to tests/script/tsim/stream/partitionbyColumnState.sim
index 3d9acbcac5..75d01b17ec 100644
--- a/tests/script/tsim/stream/partitionbyColumn2.sim
+++ b/tests/script/tsim/stream/partitionbyColumnState.sim
@@ -40,6 +40,8 @@ endi
sql insert into t1 values(1648791213000,1,1,3,1.0);
+$loop_count = 0
+
loop1:
sleep 300
sql select * from streamt order by c1, c4, c2, c3;
@@ -61,6 +63,8 @@ endi
sql insert into t1 values(1648791213000,2,1,3,1.0);
+$loop_count = 0
+
loop2:
sleep 300
sql select * from streamt order by c1, c4, c2, c3;
@@ -85,6 +89,8 @@ sql insert into t1 values(1648791213001,2,1,3,1.0);
sql insert into t1 values(1648791213002,2,1,3,1.0);
sql insert into t1 values(1648791213002,1,1,3,1.0);
+$loop_count = 0
+
loop3:
sleep 300
sql select * from streamt order by c1, c4, c2, c3;
@@ -120,6 +126,8 @@ sql insert into t1 values(1648791223002,3,2,3,1.0);
sql insert into t1 values(1648791223003,3,2,3,1.0);
sql insert into t1 values(1648791213001,1,1,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
+$loop_count = 0
+
loop4:
sleep 300
sql select * from streamt order by c1, c4, c2, c3;
@@ -212,6 +220,8 @@ sql insert into t1 values(1648791223001,1,2,2,5);
sql insert into t1 values(1648791223002,1,2,2,6);
sql insert into t1 values(1648791213001,1,1,1,7) (1648791223002,1,1,2,8);
+$loop_count = 0
+
loop6:
sleep 300
sql select * from streamt1 order by c1, c4, c2, c3;
diff --git a/tests/script/tsim/table/hash.sim b/tests/script/tsim/table/hash.sim
new file mode 100644
index 0000000000..664f867137
--- /dev/null
+++ b/tests/script/tsim/table/hash.sim
@@ -0,0 +1,84 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+#=========== prepare
+#sql create database d1 vgroups 2
+sql create database d1 vgroups 2 table_prefix 3 table_suffix 2
+sql select * from information_schema.ins_databases
+print $data(d1)[27] $data(d1)[28]
+if $data(d1)[27] != 3 then
+ return -1
+endi
+if $data(d1)[28] != 2 then
+ return -1
+endi
+
+sql use d1;
+sql create table st (ts timestamp, i int) tags (j int);
+sql create table st_ct_1 using st tags(3) st_ct_2 using st tags(4) st_ct_3 using st tags(5) st_ct_4 using st tags(6) st_ct_5 using st tags(7)
+sql insert into st_ct_1 values(now+1s, 1)
+sql insert into st_ct_1 values(now+2s, 2)
+sql insert into st_ct_1 values(now+3s, 3)
+sql insert into st_ct_2 values(now+1s, 1)
+sql insert into st_ct_2 values(now+2s, 2)
+sql insert into st_ct_2 values(now+3s, 3)
+sql insert into st_ct_3 values(now+1s, 1)
+sql insert into st_ct_3 values(now+2s, 2)
+sql insert into st_ct_3 values(now+3s, 2)
+sql insert into st_ct_4 values(now+1s, 1)
+sql insert into st_ct_4 values(now+2s, 2)
+sql insert into st_ct_4 values(now+3s, 2)
+sql insert into st_ct_5 values(now+1s, 1)
+sql insert into st_ct_5 values(now+2s, 2)
+sql insert into st_ct_5 values(now+3s, 2)
+
+# check query
+sql select * from st
+if $rows != 15 then
+ return -1
+endi
+
+# check table vgroup
+sql select * from information_schema.ins_tables where db_name = 'd1'
+if $data(st_ct_1)[6] != 2 then
+ return -1
+endi
+if $data(st_ct_2)[6] != 2 then
+ return -1
+endi
+if $data(st_ct_3)[6] != 2 then
+ return -1
+endi
+if $data(st_ct_4)[6] != 2 then
+ return -1
+endi
+if $data(st_ct_5)[6] != 2 then
+ return -1
+endi
+
+# check invalid table name
+sql create table c1 using st tags(3)
+sql create table c12 using st tags(3)
+sql create table c123 using st tags(3)
+sql create table c1234 using st tags(3)
+sql create table c12345 using st tags(3)
+sql select * from information_schema.ins_tables where db_name = 'd1'
+if $data(c1)[6] != 2 then
+ return -1
+endi
+if $data(c12)[6] != 3 then
+ return -1
+endi
+if $data(c123)[6] != 2 then
+ return -1
+endi
+if $data(c1234)[6] != 3 then
+ return -1
+endi
+if $data(c12345)[6] != 3 then
+ return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim
index d85a1bebc8..11a387ed4d 100644
--- a/tests/script/tsim/valgrind/checkError6.sim
+++ b/tests/script/tsim/valgrind/checkError6.sim
@@ -158,6 +158,8 @@ print =============== restart
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s start -v
+sleep 1000
+
sql select avg(tbcol) as c from stb
sql select avg(tbcol) as c from stb where ts <= 1601481840000
sql select avg(tbcol) as c from stb where tgcol < 5 and ts <= 1601481840000
diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh
index 4305ceff56..8987ba3bbd 100755
--- a/tests/system-test/fulltest.sh
+++ b/tests/system-test/fulltest.sh
@@ -156,8 +156,8 @@ python3 ./test.py -f 2-query/sin.py
python3 ./test.py -f 2-query/sin.py -R
python3 ./test.py -f 2-query/smaTest.py
python3 ./test.py -f 2-query/smaTest.py -R
-python3 ./test.py -f 2-query/sml.py
-python3 ./test.py -f 2-query/sml.py -R
+#python3 ./test.py -f 2-query/sml.py
+#python3 ./test.py -f 2-query/sml.py -R
python3 ./test.py -f 2-query/spread.py
python3 ./test.py -f 2-query/spread.py -R
python3 ./test.py -f 2-query/sqrt.py
@@ -333,7 +333,7 @@ python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py
python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py
python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py
python3 ./test.py -f 7-tmq/tmq_taosx.py
-# python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py
+python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py
#------------querPolicy 2-----------
@@ -512,6 +512,6 @@ python3 ./test.py -f 2-query/count_partition.py -Q 3
python3 ./test.py -f 2-query/max_partition.py -Q 3
python3 ./test.py -f 2-query/last_row.py -Q 3
python3 ./test.py -f 2-query/tsbsQuery.py -Q 3
-python3 ./test.py -f 2-query/sml.py -Q 3
+#python3 ./test.py -f 2-query/sml.py -Q 3
python3 ./test.py -f 2-query/interp.py -Q 3
diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt
new file mode 100644
index 0000000000..59cbbb3147
--- /dev/null
+++ b/utils/CMakeLists.txt
@@ -0,0 +1,4 @@
+#ADD_SUBDIRECTORY(examples/c)
+ADD_SUBDIRECTORY(tsim)
+ADD_SUBDIRECTORY(test/c)
+#ADD_SUBDIRECTORY(comparisonTest/tdengine)
diff --git a/tests/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt
similarity index 97%
rename from tests/test/c/CMakeLists.txt
rename to utils/test/c/CMakeLists.txt
index 31331b5265..839c65b633 100644
--- a/tests/test/c/CMakeLists.txt
+++ b/utils/test/c/CMakeLists.txt
@@ -1,4 +1,5 @@
add_executable(tmq_demo tmqDemo.c)
+add_dependencies(tmq_demo taos)
add_executable(tmq_sim tmqSim.c)
add_executable(create_table createTable.c)
add_executable(tmq_taosx_ci tmq_taosx_ci.c)
diff --git a/tests/test/c/createTable.c b/utils/test/c/createTable.c
similarity index 100%
rename from tests/test/c/createTable.c
rename to utils/test/c/createTable.c
diff --git a/tests/test/c/sdbDump.c b/utils/test/c/sdbDump.c
similarity index 100%
rename from tests/test/c/sdbDump.c
rename to utils/test/c/sdbDump.c
diff --git a/tests/test/c/sml_test.c b/utils/test/c/sml_test.c
similarity index 99%
rename from tests/test/c/sml_test.c
rename to utils/test/c/sml_test.c
index 1fd1def263..ca3d464da7 100644
--- a/tests/test/c/sml_test.c
+++ b/utils/test/c/sml_test.c
@@ -92,7 +92,7 @@ int smlProcess_telnet_Test() {
int smlProcess_json1_Test() {
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
- TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1");
+ TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db");
taos_free_result(pRes);
pRes = taos_query(taos, "use sml_db");
@@ -112,7 +112,7 @@ int smlProcess_json1_Test() {
" },"
" {"
" \"metric\": \"sys.cpu.nice\","
- " \"timestamp\": 1346846400,"
+ " \"timestamp\": 1662344042,"
" \"value\": 9,"
" \"tags\": {"
" \"host\": \"web02\","
@@ -141,7 +141,7 @@ int smlProcess_json2_Test() {
"{"
" \"metric\": \"meter_current0\","
" \"timestamp\": {"
- " \"value\" : 1346846400,"
+ " \"value\" : 1662344042,"
" \"type\" : \"s\""
" },"
" \"value\": {"
@@ -181,7 +181,7 @@ int smlProcess_json3_Test() {
"{"
" \"metric\": \"meter_current1\","
" \"timestamp\": {"
- " \"value\" : 1346846400,"
+ " \"value\" : 1662344042,"
" \"type\" : \"s\""
" },"
" \"value\": {"
@@ -249,7 +249,7 @@ int smlProcess_json4_Test() {
"{"
" \"metric\": \"meter_current2\","
" \"timestamp\": {"
- " \"value\" : 1346846500000,"
+ " \"value\" : 1662344042000,"
" \"type\" : \"ms\""
" },"
" \"value\": \"ni\","
diff --git a/tests/test/c/tmqDemo.c b/utils/test/c/tmqDemo.c
similarity index 100%
rename from tests/test/c/tmqDemo.c
rename to utils/test/c/tmqDemo.c
diff --git a/tests/test/c/tmqSim.c b/utils/test/c/tmqSim.c
similarity index 100%
rename from tests/test/c/tmqSim.c
rename to utils/test/c/tmqSim.c
diff --git a/tests/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c
similarity index 100%
rename from tests/test/c/tmq_taosx_ci.c
rename to utils/test/c/tmq_taosx_ci.c
diff --git a/tests/tsim/CMakeLists.txt b/utils/tsim/CMakeLists.txt
similarity index 100%
rename from tests/tsim/CMakeLists.txt
rename to utils/tsim/CMakeLists.txt
diff --git a/tests/tsim/inc/simInt.h b/utils/tsim/inc/simInt.h
similarity index 100%
rename from tests/tsim/inc/simInt.h
rename to utils/tsim/inc/simInt.h
diff --git a/tests/tsim/inc/simParse.h b/utils/tsim/inc/simParse.h
similarity index 100%
rename from tests/tsim/inc/simParse.h
rename to utils/tsim/inc/simParse.h
diff --git a/tests/tsim/src/simExe.c b/utils/tsim/src/simExe.c
similarity index 100%
rename from tests/tsim/src/simExe.c
rename to utils/tsim/src/simExe.c
diff --git a/tests/tsim/src/simMain.c b/utils/tsim/src/simMain.c
similarity index 100%
rename from tests/tsim/src/simMain.c
rename to utils/tsim/src/simMain.c
diff --git a/tests/tsim/src/simParse.c b/utils/tsim/src/simParse.c
similarity index 100%
rename from tests/tsim/src/simParse.c
rename to utils/tsim/src/simParse.c
diff --git a/tests/tsim/src/simSystem.c b/utils/tsim/src/simSystem.c
similarity index 100%
rename from tests/tsim/src/simSystem.c
rename to utils/tsim/src/simSystem.c