diff --git a/.gitignore b/.gitignore index 704b2e7415..08e3d57717 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ CMakeSettings.json cmake-build-debug/ cmake-build-release/ cscope.out +cscope.files .DS_Store debug/ release/ diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 8ddee6dbbd..d3fc05a1d2 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -355,7 +355,7 @@ pipeline { } parallel { stage('check docs') { - agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 || Mac_catalina "} + agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 || Mac_catalina "} steps { check_docs() } @@ -393,7 +393,7 @@ pipeline { agent{label " Mac_catalina "} steps { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { - timeout(time: 20, unit: 'MINUTES'){ + timeout(time: 30, unit: 'MINUTES'){ pre_test() pre_test_build_mac() } @@ -401,7 +401,7 @@ pipeline { } } stage('linux test') { - agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 "} + agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "} options { skipDefaultCheckout() } when { changeRequest() diff --git a/cmake/cmake.define b/cmake/cmake.define index 7710c071eb..73f9497809 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.0) -set(CMAKE_VERBOSE_MAKEFILE FALSE) +set(CMAKE_VERBOSE_MAKEFILE TRUE) set(TD_BUILD_TAOSA_INTERNAL FALSE) #set output directory @@ -159,6 +159,7 @@ ELSE () CHECK_C_COMPILER_FLAG("-mavx2" COMPILER_SUPPORT_AVX2) CHECK_C_COMPILER_FLAG("-mavx512f" COMPILER_SUPPORT_AVX512F) CHECK_C_COMPILER_FLAG("-mavx512vbmi" COMPILER_SUPPORT_AVX512BMI) + CHECK_C_COMPILER_FLAG("-mavx512vl" COMPILER_SUPPORT_AVX512VL) IF (COMPILER_SUPPORT_SSE42) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse4.2") @@ -166,11 +167,11 @@ ELSE () ENDIF() IF ("${SIMD_SUPPORT}" MATCHES "true") - IF (COMPILER_SUPPORT_FMA) + IF (COMPILER_SUPPORT_FMA) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfma") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfma") - ENDIF() - IF (COMPILER_SUPPORT_AVX) + ENDIF() + IF (COMPILER_SUPPORT_AVX) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx") ENDIF() @@ -180,11 +181,17 @@ ELSE () ENDIF() MESSAGE(STATUS "SIMD instructions (FMA/AVX/AVX2) is ACTIVATED") - IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi") - MESSAGE(STATUS "avx512 supported by gcc") - ENDIF() +# IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI) +# SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi") +# SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi") +# MESSAGE(STATUS "avx512f/avx512bmi supported by compiler") +# ENDIF() +# +# IF (COMPILER_SUPPORT_AVX512VL) +# SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512vl") +# SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512vl") +# MESSAGE(STATUS "avx512vl supported by compiler") +# ENDIF() ENDIF() # build mode diff --git a/cmake/cmake.options b/cmake/cmake.options index d34c34dd89..e12de3e4e6 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -151,6 +151,7 @@ IF(${BUILD_S3}) IF(${BUILD_WITH_S3}) +add_definitions(-DUSE_S3) option(BUILD_WITH_COS "If build with cos" OFF) ELSE () diff --git a/docs/en/05-get-started/03-package.md b/docs/en/05-get-started/03-package.md index 3e3c04682f..d34df2c970 100644 --- a/docs/en/05-get-started/03-package.md +++ b/docs/en/05-get-started/03-package.md @@ -243,7 +243,7 @@ You can use the TDengine CLI to monitor your TDengine deployment and execute ad taos ``` -The TDengine CLI displays a welcome message and version information to indicate that its connection to the TDengine service was successful. If an error message is displayed, see the [FAQ](/train-faq/faq) for troubleshooting information. At the following prompt, you can execute SQL commands. +The TDengine CLI displays a welcome message and version information to indicate that its connection to the TDengine service was successful. If an error message is displayed, see the [FAQ](../../train-faq/faq) for troubleshooting information. At the following prompt, you can execute SQL commands. ```cmd taos> diff --git a/docs/en/05-get-started/_pkg_install.mdx b/docs/en/05-get-started/_pkg_install.mdx index 32d7c1f376..2372d2ff26 100644 --- a/docs/en/05-get-started/_pkg_install.mdx +++ b/docs/en/05-get-started/_pkg_install.mdx @@ -10,7 +10,7 @@ Between official releases, beta versions may be released that contain new featur -For information about installing TDengine, see [Install and Uninstall](/operation/pkg-install). +For information about installing TDengine, see [Install and Uninstall](../../operation/pkg-install). For information about TDengine releases, see [All Downloads](https://tdengine.com/all-downloads) diff --git a/docs/en/05-get-started/index.md b/docs/en/05-get-started/index.md index cc3b4826dd..5c4ab59f75 100644 --- a/docs/en/05-get-started/index.md +++ b/docs/en/05-get-started/index.md @@ -12,7 +12,7 @@ import StackOverflowSVG from './stackoverflow.svg' You can install and run TDengine on Linux/Windows/macOS machines as well as Docker containers. You can also deploy TDengine as a managed service with TDengine Cloud. -The full package of TDengine includes the TDengine Server (`taosd`), TDengine Client (`taosc`), taosAdapter for connecting with third-party systems and providing a RESTful interface, a command-line interface, and some tools. In addition to connectors for multiple languages, TDengine also provides a [RESTful interface](/reference/rest-api) through [taosAdapter](/reference/taosadapter). +The full package of TDengine includes the TDengine Server (`taosd`), TDengine Client (`taosc`), taosAdapter for connecting with third-party systems and providing a RESTful interface, a command-line interface, and some tools. In addition to connectors for multiple languages, TDengine also provides a [RESTful interface](../reference/rest-api) through [taosAdapter](../reference/taosadapter). ```mdx-code-block import DocCardList from '@theme/DocCardList'; diff --git a/docs/en/07-develop/01-connect/_connect_java.mdx b/docs/en/07-develop/01-connect/_connect_java.mdx index 538e27fcc3..fda86f2221 100644 --- a/docs/en/07-develop/01-connect/_connect_java.mdx +++ b/docs/en/07-develop/01-connect/_connect_java.mdx @@ -12,4 +12,4 @@ When using REST connection, the feature of bulk pulling can be enabled if the si {{#include docs/examples/java/src/main/java/com/taos/example/WSConnectExample.java:main}} ``` -More configuration about connection, please refer to [Java Connector](/reference/connector/java) +More configuration about connection, please refer to [Java Connector](../../reference/connector/java) diff --git a/docs/en/07-develop/01-connect/index.md b/docs/en/07-develop/01-connect/index.md index 3ca44783c4..3f09f9fb6a 100644 --- a/docs/en/07-develop/01-connect/index.md +++ b/docs/en/07-develop/01-connect/index.md @@ -22,7 +22,7 @@ import VerifyLinux from "../../14-reference/03-connector/_verify_linux.mdx"; import VerifyWindows from "../../14-reference/03-connector/_verify_windows.mdx"; import VerifyMacOS from "../../14-reference/03-connector/_verify_macos.mdx"; -Any application running on any platform can access TDengine through the REST API provided by TDengine. For information, see [REST API](/reference/rest-api/). Applications can also use the connectors for various programming languages, including C/C++, Java, Python, Go, Node.js, C#, and Rust, to access TDengine. These connectors support connecting to TDengine clusters using both native interfaces (taosc). Some connectors also support connecting over a REST interface. Community developers have also contributed several unofficial connectors, such as the ADO.NET connector, the Lua connector, and the PHP connector. +Any application running on any platform can access TDengine through the REST API provided by TDengine. For information, see [REST API](../../reference/rest-api/). Applications can also use the connectors for various programming languages, including C/C++, Java, Python, Go, Node.js, C#, and Rust, to access TDengine. These connectors support connecting to TDengine clusters using both native interfaces (taosc). Some connectors also support connecting over a REST interface. Community developers have also contributed several unofficial connectors, such as the ADO.NET connector, the Lua connector, and the PHP connector. ## Establish Connection @@ -36,7 +36,7 @@ For REST and native connections, connectors provide similar APIs for performing Key differences: 3. The REST connection is more accessible with cross-platform support, however it results in a 30% performance downgrade. -1. The TDengine client driver (taosc) has the highest performance with all the features of TDengine like [Parameter Binding](/reference/connector/cpp#parameter-binding-api), [Subscription](/reference/connector/cpp#subscription-and-consumption-api), etc. +1. The TDengine client driver (taosc) has the highest performance with all the features of TDengine like [Parameter Binding](../../reference/connector/cpp#parameter-binding-api), [Subscription](../../reference/connector/cpp#subscription-and-consumption-api), etc. ## Install Client Driver taosc diff --git a/docs/en/07-develop/02-model/index.mdx b/docs/en/07-develop/02-model/index.mdx index 4524a66a41..a18e1932ab 100644 --- a/docs/en/07-develop/02-model/index.mdx +++ b/docs/en/07-develop/02-model/index.mdx @@ -3,9 +3,9 @@ title: Data Model description: This document describes the data model of TDengine. --- -The data model employed by TDengine is similar to that of a relational database. You have to create databases and tables. You must design the data model based on your own business and application requirements. You should design the [STable](/concept/#super-table-stable) (an abbreviation for super table) schema to fit your data. This chapter will explain the big picture without getting into syntactical details. +The data model employed by TDengine is similar to that of a relational database. You have to create databases and tables. You must design the data model based on your own business and application requirements. You should design the [STable](../../concept/#super-table-stable) (an abbreviation for super table) schema to fit your data. This chapter will explain the big picture without getting into syntactical details. -Note: before you read this chapter, please make sure you have already read through [Key Concepts](/concept/), since TDengine introduces new concepts like "one table for one [data collection point](/concept/#data-collection-point)" and "[super table](/concept/#super-table-stable)". +Note: before you read this chapter, please make sure you have already read through [Key Concepts](../../concept/), since TDengine introduces new concepts like "one table for one [data collection point](../../concept/#data-collection-point)" and "[super table](../../concept/#super-table-stable)". ## Create Database @@ -22,7 +22,7 @@ In the above SQL statement: - a new data file will be created every 10 days - the size of the write cache pool on each VNode is 16 MB - the number of vgroups is 100 -- WAL is enabled but fsync is disabled For more details please refer to [Database](/taos-sql/database). +- WAL is enabled but fsync is disabled For more details please refer to [Database](../../taos-sql/database). After creating a database, the current database in use can be switched using SQL command `USE`. For example the SQL statement below switches the current database to `power`. @@ -41,13 +41,13 @@ Without the current database specified, table name must be preceded with the cor ## Create STable -In a time-series application, there may be multiple kinds of data collection points. For example, in the electrical power system there are meters, transformers, bus bars, switches, etc. For easy and efficient aggregation of multiple tables, one STable needs to be created for each kind of data collection point. For example, for the meters in [table 1](/concept/#model_table1), the SQL statement below can be used to create the super table. +In a time-series application, there may be multiple kinds of data collection points. For example, in the electrical power system there are meters, transformers, bus bars, switches, etc. For easy and efficient aggregation of multiple tables, one STable needs to be created for each kind of data collection point. For example, for the meters in [table 1](../../concept/#model_table1), the SQL statement below can be used to create the super table. ```sql CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int); ``` -Similar to creating a regular table, when creating a STable, the name and schema need to be provided. In the STable schema, the first column must always be a timestamp (like ts in the example), and the other columns (like current, voltage and phase in the example) are the data collected. The remaining columns can [contain data of type](/taos-sql/data-type/) integer, float, double, string etc. In addition, the schema for tags, like location and groupId in the example, must be provided. The tag type can be integer, float, string, etc. Tags are essentially the static properties of a data collection point. For example, properties like the location, device type, device group ID, manager ID are tags. Tags in the schema can be added, removed or updated. Please refer to [STable](/taos-sql/stable) for more details. +Similar to creating a regular table, when creating a STable, the name and schema need to be provided. In the STable schema, the first column must always be a timestamp (like ts in the example), and the other columns (like current, voltage and phase in the example) are the data collected. The remaining columns can [contain data of type](../../taos-sql/data-type/) integer, float, double, string etc. In addition, the schema for tags, like location and groupId in the example, must be provided. The tag type can be integer, float, string, etc. Tags are essentially the static properties of a data collection point. For example, properties like the location, device type, device group ID, manager ID are tags. Tags in the schema can be added, removed or updated. Please refer to [STable](../../taos-sql/stable) for more details. For each kind of data collection point, a corresponding STable must be created. There may be many STables in an application. For electrical power system, we need to create a STable respectively for meters, transformers, busbars, switches. There may be multiple kinds of data collection points on a single device, for example there may be one data collection point for electrical data like current and voltage and another data collection point for environmental data like temperature, humidity and wind direction. Multiple STables are required for these kinds of devices. @@ -61,7 +61,7 @@ A specific table needs to be created for each data collection point. Similar to CREATE TABLE d1001 USING meters TAGS ("California.SanFrancisco", 2); ``` -In the above SQL statement, "d1001" is the table name, "meters" is the STable name, followed by the value of tag "Location" and the value of tag "groupId", which are "California.SanFrancisco" and "2" respectively in the example. The tag values can be updated after the table is created. Please refer to [Tables](/taos-sql/table) for details. +In the above SQL statement, "d1001" is the table name, "meters" is the STable name, followed by the value of tag "Location" and the value of tag "groupId", which are "California.SanFrancisco" and "2" respectively in the example. The tag values can be updated after the table is created. Please refer to [Tables](../../taos-sql/table) for details. It's suggested to use the globally unique ID of a data collection point as the table name. For example the device serial number could be used as a unique ID. If a unique ID doesn't exist, multiple IDs that are not globally unique can be combined to form a globally unique ID. It's not recommended to use a globally unique ID as tag value. @@ -75,7 +75,7 @@ INSERT INTO d1001 USING meters TAGS ("California.SanFrancisco", 2) VALUES (now, In the above SQL statement, a row with value `(now, 10.2, 219, 0.32)` will be inserted into table "d1001". If table "d1001" doesn't exist, it will be created automatically using STable "meters" as template with tag value `"California.SanFrancisco", 2`. -For more details please refer to [Create Table Automatically](/taos-sql/insert#automatically-create-table-when-inserting). +For more details please refer to [Create Table Automatically](../../taos-sql/insert#automatically-create-table-when-inserting). ## Single Column vs Multiple Column diff --git a/docs/en/07-develop/03-insert-data/01-sql-writing.mdx b/docs/en/07-develop/03-insert-data/01-sql-writing.mdx index 4d1b67e451..1e719283cc 100644 --- a/docs/en/07-develop/03-insert-data/01-sql-writing.mdx +++ b/docs/en/07-develop/03-insert-data/01-sql-writing.mdx @@ -33,7 +33,7 @@ The below SQL statement is used to insert one row into table "d1001". INSERT INTO d1001 VALUES (ts1, 10.3, 219, 0.31); ``` -`ts1` is Unix timestamp, the timestamps which is larger than the difference between current time and KEEP in config is only allowed. For further detail, refer to [TDengine SQL insert timestamp section](/taos-sql/insert). +`ts1` is Unix timestamp, the timestamps which is larger than the difference between current time and KEEP in config is only allowed. For further detail, refer to [TDengine SQL insert timestamp section](../../../taos-sql/insert). ### Insert Multiple Rows @@ -43,7 +43,7 @@ Multiple rows can be inserted in a single SQL statement. The example below inser INSERT INTO d1001 VALUES (ts2, 10.2, 220, 0.23) (ts2, 10.3, 218, 0.25); ``` -`ts1` and `ts2` is Unix timestamp, the timestamps which is larger than the difference between current time and KEEP in config is only allowed. For further detail, refer to [TDengine SQL insert timestamp section](/taos-sql/insert). +`ts1` and `ts2` is Unix timestamp, the timestamps which is larger than the difference between current time and KEEP in config is only allowed. For further detail, refer to [TDengine SQL insert timestamp section](../../../taos-sql/insert). ### Insert into Multiple Tables @@ -53,9 +53,9 @@ Data can be inserted into multiple tables in the same SQL statement. The example INSERT INTO d1001 VALUES (ts1, 10.3, 219, 0.31) (ts2, 12.6, 218, 0.33) d1002 VALUES (ts3, 12.3, 221, 0.31); ``` -`ts1`, `ts2` and `ts3` is Unix timestamp, the timestamps which is larger than the difference between current time and KEEP in config is only allowed. For further detail, refer to [TDengine SQL insert timestamp section](/taos-sql/insert). +`ts1`, `ts2` and `ts3` is Unix timestamp, the timestamps which is larger than the difference between current time and KEEP in config is only allowed. For further detail, refer to [TDengine SQL insert timestamp section](../../../taos-sql/insert). -For more details about `INSERT` please refer to [INSERT](/taos-sql/insert). +For more details about `INSERT` please refer to [INSERT](../../../taos-sql/insert). :::info diff --git a/docs/en/07-develop/03-insert-data/20-kafka-writting.mdx b/docs/en/07-develop/03-insert-data/20-kafka-writting.mdx index 89ca10b669..e8de010d5b 100644 --- a/docs/en/07-develop/03-insert-data/20-kafka-writting.mdx +++ b/docs/en/07-develop/03-insert-data/20-kafka-writting.mdx @@ -35,7 +35,7 @@ bin/kafka-topics.sh --bootstrap-server=localhost:9092 --describe ## Insert into TDengine -We can write data into TDengine via SQL or Schemaless. For more information, please refer to [Insert Using SQL](/develop/insert-data/sql-writing/) or [High Performance Writing](/develop/insert-data/high-volume/) or [Schemaless Writing](/reference/schemaless/). +We can write data into TDengine via SQL or Schemaless. For more information, please refer to [Insert Using SQL](../sql-writing/) or [High Performance Writing](../high-volume/) or [Schemaless Writing](../../../reference/schemaless/). ## Examples diff --git a/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx b/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx index bd430d5973..b90b92510a 100644 --- a/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx +++ b/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx @@ -46,7 +46,7 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0 ::: -For more details please refer to [InfluxDB Line Protocol](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) and [TDengine Schemaless](/reference/schemaless/#Schemaless-Line-Protocol) +For more details please refer to [InfluxDB Line Protocol](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) and [TDengine Schemaless](../../../reference/schemaless/#Schemaless-Line-Protocol) ## Examples diff --git a/docs/en/07-develop/04-query-data/index.mdx b/docs/en/07-develop/04-query-data/index.mdx index 5a05d22d85..413baf6b2c 100644 --- a/docs/en/07-develop/04-query-data/index.mdx +++ b/docs/en/07-develop/04-query-data/index.mdx @@ -128,7 +128,7 @@ For more information, see [Aggregate by Window](../../taos-sql/distinguished). ### Query -In the section describing [Insert](/develop/insert-data/sql-writing), a database named `power` is created and some data are inserted into STable `meters`. Below sample code demonstrates how to query the data in this STable. +In the section describing [Insert](../insert-data/sql-writing), a database named `power` is created and some data are inserted into STable `meters`. Below sample code demonstrates how to query the data in this STable. diff --git a/docs/en/07-develop/index.md b/docs/en/07-develop/index.md index 8f80b82b97..4ed5e8c19f 100644 --- a/docs/en/07-develop/index.md +++ b/docs/en/07-develop/index.md @@ -6,7 +6,7 @@ description: This document describes how to use the various components of TDengi Before creating an application to process time-series data with TDengine, consider the following: 1. Choose the method to connect to TDengine. TDengine offers a REST API that can be used with any programming language. It also has connectors for a variety of languages. -2. Design the data model based on your own use cases. Consider the main [concepts](/concept/) of TDengine, including "one table per data collection point" and the supertable. Learn about static labels, collected metrics, and subtables. Depending on the characteristics of your data and your requirements, you decide to create one or more databases and design a supertable schema that fit your data. +2. Design the data model based on your own use cases. Consider the main [concepts](../concept/) of TDengine, including "one table per data collection point" and the supertable. Learn about static labels, collected metrics, and subtables. Depending on the characteristics of your data and your requirements, you decide to create one or more databases and design a supertable schema that fit your data. 3. Decide how you will insert data. TDengine supports writing using standard SQL, but also supports schemaless writing, so that data can be written directly without creating tables manually. 4. Based on business requirements, find out what SQL query statements need to be written. You may be able to repurpose any existing SQL. 5. If you want to run real-time analysis based on time series data, including various dashboards, use the TDengine stream processing component instead of deploying complex systems such as Spark or Flink. @@ -14,7 +14,7 @@ Before creating an application to process time-series data with TDengine, consid 7. In many use cases (such as fleet management), the application needs to obtain the latest status of each data collection point. It is recommended that you use the cache function of TDengine instead of deploying Redis separately. 8. If you find that the SQL functions of TDengine cannot meet your requirements, then you can use user-defined functions to solve the problem. -This section is organized in the order described above. For ease of understanding, TDengine provides sample code for each supported programming language for each function. If you want to learn more about the use of SQL, please read the [SQL manual](/taos-sql/). For a more in-depth understanding of the use of each connector, please read the [Connector Reference Guide](/reference/connector/). If you also want to integrate TDengine with third-party systems, such as Grafana, please refer to the [third-party tools](/third-party/). +This section is organized in the order described above. For ease of understanding, TDengine provides sample code for each supported programming language for each function. If you want to learn more about the use of SQL, please read the [SQL manual](../taos-sql/). For a more in-depth understanding of the use of each connector, please read the [Connector Reference Guide](../reference/connector/). If you also want to integrate TDengine with third-party systems, such as Grafana, please refer to the [third-party tools](../third-party/). If you encounter any problems during the development process, please click ["Submit an issue"](https://github.com/taosdata/TDengine/issues/new/choose) at the bottom of each page and submit it on GitHub right away. diff --git a/docs/en/10-deployment/01-deploy.md b/docs/en/10-deployment/01-deploy.md index 4a6a3aae41..6e7e0d1b7c 100644 --- a/docs/en/10-deployment/01-deploy.md +++ b/docs/en/10-deployment/01-deploy.md @@ -72,7 +72,7 @@ For all the dnodes in a TDengine cluster, the below parameters must be configure ## Start Cluster -The first dnode can be started following the instructions in [Get Started](/get-started/). Then TDengine CLI `taos` can be launched to execute command `show dnodes`, the output is as following for example: +The first dnode can be started following the instructions in [Get Started](../../get-started/). Then TDengine CLI `taos` can be launched to execute command `show dnodes`, the output is as following for example: ``` taos> show dnodes; @@ -90,7 +90,7 @@ From the above output, it is shown that the end point of the started dnode is "h There are a few steps necessary to add other dnodes in the cluster. -Second, we can start `taosd` as instructed in [Get Started](/get-started/). +Second, we can start `taosd` as instructed in [Get Started](../../get-started/). Then, on the first dnode i.e. h1.tdengine.com in our example, use TDengine CLI `taos` to execute the following command: diff --git a/docs/en/12-taos-sql/02-database.md b/docs/en/12-taos-sql/02-database.md index 255263ecfd..a0e036664a 100644 --- a/docs/en/12-taos-sql/02-database.md +++ b/docs/en/12-taos-sql/02-database.md @@ -26,7 +26,6 @@ database_option: { | PAGESIZE value | PRECISION {'ms' | 'us' | 'ns'} | REPLICA value - | RETENTIONS ingestion_duration:keep_duration ... | WAL_LEVEL {1 | 2} | VGROUPS value | SINGLE_STABLE {0 | 1} @@ -61,7 +60,6 @@ database_option: { - PAGESIZE: specifies the size (in KB) of each page in the metadata storage engine cache on each vnode. The default value is 4. Enter a value between 1 and 16384. - PRECISION: specifies the precision at which a database records timestamps. Enter ms for milliseconds, us for microseconds, or ns for nanoseconds. The default value is ms. - REPLICA: specifies the number of replicas that are made of the database. Enter 1 or 3. The default value is 1. The value of the REPLICA parameter cannot exceed the number of dnodes in the cluster. -- RETENTIONS: specifies the retention period for data aggregated at various intervals. For example, RETENTIONS 15s:7d,1m:21d,15m:50d indicates that data aggregated every 15 seconds is retained for 7 days, data aggregated every 1 minute is retained for 21 days, and data aggregated every 15 minutes is retained for 50 days. You must enter three aggregation intervals and corresponding retention periods. - WAL_LEVEL: specifies whether fsync is enabled. The default value is 1. - 1: WAL is enabled but fsync is disabled. - 2: WAL and fsync are both enabled. diff --git a/docs/en/12-taos-sql/03-table.md b/docs/en/12-taos-sql/03-table.md index 10c44848c9..a10abd28a5 100644 --- a/docs/en/12-taos-sql/03-table.md +++ b/docs/en/12-taos-sql/03-table.md @@ -32,9 +32,6 @@ table_options: table_option: { COMMENT 'string_value' - | WATERMARK duration[,duration] - | MAX_DELAY duration[,duration] - | ROLLUP(func_name [, func_name] ...) | SMA(col_name [, col_name] ...) | TTL value } @@ -54,11 +51,8 @@ table_option: { **Parameter description** 1. COMMENT: specifies comments for the table. This parameter can be used with supertables, standard tables, and subtables. -2. WATERMARK: specifies the time after which the window is closed. The default value is 5 seconds. Enter a value between 0 and 15 minutes in milliseconds, seconds, or minutes. You can enter multiple values separated by commas (,). This parameter applies only to supertables and takes effect only when the RETENTIONS parameter has been specified for the database. -3. MAX_DELAY: specifies the maximum latency for pushing computation results. The default value is 15 minutes or the value of the INTERVAL parameter, whichever is smaller. Enter a value between 0 and 15 minutes in milliseconds, seconds, or minutes. You can enter multiple values separated by commas (,). Note: Retain the default value if possible. Configuring a small MAX_DELAY may cause results to be frequently pushed, affecting storage and query performance. This parameter applies only to supertables and takes effect only when the RETENTIONS parameter has been specified for the database. -4. ROLLUP: specifies aggregate functions to roll up. Rolling up a function provides downsampled results based on multiple axes. This parameter applies only to supertables and takes effect only when the RETENTIONS parameter has been specified for the database. You can specify only one function to roll up. The rollup takes effect on all columns except TS. Enter one of the following values: avg, sum, min, max, last, or first. -5. SMA: specifies functions on which to enable small materialized aggregates (SMA). SMA is user-defined precomputation of aggregates based on data blocks. Enter one of the following values: max, min, or sum This parameter can be used with supertables and standard tables. -6. TTL: specifies the time to live (TTL) for the table. If TTL is specified when creatinga table, after the time period for which the table has been existing is over TTL, TDengine will automatically delete the table. Please be noted that the system may not delete the table at the exact moment that the TTL expires but guarantee there is such a system and finally the table will be deleted. The unit of TTL is in days. The default value is 0, i.e. never expire. +2. SMA: specifies functions on which to enable small materialized aggregates (SMA). SMA is user-defined precomputation of aggregates based on data blocks. Enter one of the following values: max, min, or sum This parameter can be used with supertables and standard tables. +3. TTL: specifies the time to live (TTL) for the table. If TTL is specified when creatinga table, after the time period for which the table has been existing is over TTL, TDengine will automatically delete the table. Please be noted that the system may not delete the table at the exact moment that the TTL expires but guarantee there is such a system and finally the table will be deleted. The unit of TTL is in days. The default value is 0, i.e. never expire. ## Create Subtables diff --git a/docs/en/12-taos-sql/26-udf.md b/docs/en/12-taos-sql/26-udf.md index b533b98b3d..f86b535927 100644 --- a/docs/en/12-taos-sql/26-udf.md +++ b/docs/en/12-taos-sql/26-udf.md @@ -53,7 +53,7 @@ CREATE AGGREGATE FUNCTION function_name AS library_path OUTPUTTYPE output_type [ CREATE AGGREGATE FUNCTION l2norm AS "/home/taos/udf_example/libl2norm.so" OUTPUTTYPE DOUBLE bufsize 64; ``` -For more information about user-defined functions, see [User-Defined Functions](/develop/udf). +For more information about user-defined functions, see [User-Defined Functions](../../develop/udf). ## Manage UDF diff --git a/docs/en/13-operation/10-monitor.md b/docs/en/13-operation/10-monitor.md index f1be4c5fd3..009db425a4 100644 --- a/docs/en/13-operation/10-monitor.md +++ b/docs/en/13-operation/10-monitor.md @@ -41,7 +41,7 @@ Launch `TDinsight.sh` with the command above and restart Grafana, then open Dash ## log database -The data of tdinsight dashboard is stored in `log` database (default. You can change it in taoskeeper's config file. For more infrmation, please reference to [taoskeeper document](/reference/taosKeeper)). The taoskeeper will create log database on taoskeeper startup. +The data of tdinsight dashboard is stored in `log` database (default. You can change it in taoskeeper's config file. For more infrmation, please reference to [taoskeeper document](../../reference/taosKeeper)). The taoskeeper will create log database on taoskeeper startup. ### cluster\_info table diff --git a/docs/en/14-reference/02-rest-api/02-rest-api.mdx b/docs/en/14-reference/02-rest-api/02-rest-api.mdx index 4da987213c..8f8e966f8d 100644 --- a/docs/en/14-reference/02-rest-api/02-rest-api.mdx +++ b/docs/en/14-reference/02-rest-api/02-rest-api.mdx @@ -514,4 +514,4 @@ Response body: ## Reference -[taosAdapter](/reference/taosadapter/) +[taosAdapter](../taosadapter/) diff --git a/docs/en/14-reference/03-connector/03-cpp.mdx b/docs/en/14-reference/03-connector/03-cpp.mdx index f6ebf0fe47..27adb58c12 100644 --- a/docs/en/14-reference/03-connector/03-cpp.mdx +++ b/docs/en/14-reference/03-connector/03-cpp.mdx @@ -24,7 +24,7 @@ The dynamic libraries for the TDengine client driver are located in. ## Supported platforms -Please refer to [list of supported platforms](/reference/connector#supported-platforms) +Please refer to [list of supported platforms](../#supported-platforms) ## Supported versions @@ -32,7 +32,7 @@ The version number of the TDengine client driver and the version number of the T ## Installation Steps -Please refer to the [Installation Steps](/reference/connector#installation-steps) for TDengine client driver installation +Please refer to the [Installation Steps](../#installation-steps) for TDengine client driver installation ## Establishing a connection @@ -394,7 +394,7 @@ The specific functions related to the interface are as follows (see also the [pr ### Schemaless Writing API -In addition to writing data using the SQL method or the parameter binding API, writing can also be done using schemaless writing, which eliminates the need to create a super table/data sub-table structure in advance and writes the data directly. The TDengine system automatically creates and maintains the required table structure based on the written data content. The use of schemaless writing is described in the chapter [Schemaless Writing](/reference/schemaless/), and the C/C++ API used with it is described here. +In addition to writing data using the SQL method or the parameter binding API, writing can also be done using schemaless writing, which eliminates the need to create a super table/data sub-table structure in advance and writes the data directly. The TDengine system automatically creates and maintains the required table structure based on the written data content. The use of schemaless writing is described in the chapter [Schemaless Writing](../../schemaless/), and the C/C++ API used with it is described here. - `TAOS_RES* taos_schemaless_insert(TAOS* taos, const char* lines[], int numLines, int protocol, int precision)` diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx index f770ce0d5d..039395cc30 100644 --- a/docs/en/14-reference/03-connector/04-java.mdx +++ b/docs/en/14-reference/03-connector/04-java.mdx @@ -148,7 +148,7 @@ TDengine currently supports timestamp, number, character, Boolean type, and the **Note**: Only TAG supports JSON types Due to historical reasons, the BINARY type data in TDengine is not truly binary data and is no longer recommended for use. Please use VARBINARY type instead. -GEOMETRY type is binary data in little endian byte order, which complies with the WKB specification. For detailed information, please refer to [Data Type] (/tao-sql/data-type/#Data Types) +GEOMETRY type is binary data in little endian byte order, which complies with the WKB specification. For detailed information, please refer to [Data Type](../../../taos-sql/data-type/) For WKB specifications, please refer to [Well Known Binary (WKB)]（ https://libgeos.org/specifications/wkb/ ） For Java connector, the jts library can be used to easily create GEOMETRY type objects, serialize them, and write them to TDengine. Here is an example [Geometry example]（https://github.com/taosdata/TDengine/blob/3.0/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/GeometryDemo.java） @@ -160,7 +160,7 @@ For Java connector, the jts library can be used to easily create GEOMETRY type o Before using Java Connector to connect to the database, the following conditions are required. - Java 1.8 or above runtime environment and Maven 3.6 or above installed -- TDengine client driver installed (required for native connections, not required for REST connections), please refer to [Installing Client Driver](/reference/connector#Install-Client-Driver) +- TDengine client driver installed (required for native connections, not required for REST connections), please refer to [Installing Client Driver](../#Install-Client-Driver) ### Install the connectors @@ -368,7 +368,7 @@ The configuration parameters in properties are as follows. - TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: message transmission timeout in milliseconds, the default value is 60000 ms. It only takes effect when using JDBC REST connection and batchfetch is true. - TSDBDriver.PROPERTY_KEY_USE_SSL: connecting Securely Using SSL. true: using SSL connection, false: not using SSL connection. It only takes effect when using JDBC REST connection. - TSDBDriver.HTTP_POOL_SIZE: size of REST concurrent requests. The default value is 20. - For JDBC native connections, you can specify other parameters, such as log level, SQL length, etc., by specifying URL and Properties. For more detailed configuration, please refer to [Client Configuration](/reference/config/#Client-Only). + For JDBC native connections, you can specify other parameters, such as log level, SQL length, etc., by specifying URL and Properties. For more detailed configuration, please refer to [Client Configuration](../../config/#Client-Only). ### Priority of configuration parameters diff --git a/docs/en/14-reference/03-connector/05-go.mdx b/docs/en/14-reference/03-connector/05-go.mdx index a0be7a4a02..33f7a93439 100644 --- a/docs/en/14-reference/03-connector/05-go.mdx +++ b/docs/en/14-reference/03-connector/05-go.mdx @@ -74,7 +74,7 @@ If it is a TDengine error, you can get the error code and error information in t ### Pre-installation preparation * Install Go development environment (Go 1.14 and above, GCC 4.8.5 and above) -* If you use the native connector, please install the TDengine client driver. Please refer to [Install Client Driver](/reference/connector/#install-client-driver) for specific steps +* If you use the native connector, please install the TDengine client driver. Please refer to [Install Client Driver](../#install-client-driver) for specific steps Configure the environment variables and check the command. diff --git a/docs/en/14-reference/03-connector/06-rust.mdx b/docs/en/14-reference/03-connector/06-rust.mdx index 5a44b161cb..0981df6724 100644 --- a/docs/en/14-reference/03-connector/06-rust.mdx +++ b/docs/en/14-reference/03-connector/06-rust.mdx @@ -80,7 +80,7 @@ Note: Only TAG supports JSON types ### Pre-installation preparation * Install the Rust development toolchain -* If using the native connection, please install the TDengine client driver. Please refer to [install client driver](/reference/connector#install-client-driver) +* If using the native connection, please install the TDengine client driver. Please refer to [install client driver](../#install-client-driver) ### Install the connectors diff --git a/docs/en/14-reference/03-connector/07-python.mdx b/docs/en/14-reference/03-connector/07-python.mdx index b699019b44..ccc270d3be 100644 --- a/docs/en/14-reference/03-connector/07-python.mdx +++ b/docs/en/14-reference/03-connector/07-python.mdx @@ -7,7 +7,7 @@ description: This document describes taospy, the TDengine Python connector. import Tabs from "@theme/Tabs"; import TabItem from "@theme/TabItem"; -`taospy` is the official Python connector for TDengine. taospy provides a rich API that makes it easy for Python applications to use TDengine. `taospy` wraps both the [native interface](/reference/connector/cpp) and [REST interface](/reference/rest-api) of TDengine, which correspond to the `taos` and `taosrest` modules of the `taospy` package, respectively. +`taospy` is the official Python connector for TDengine. taospy provides a rich API that makes it easy for Python applications to use TDengine. `taospy` wraps both the [native interface](../cpp) and [REST interface](../../rest-api) of TDengine, which correspond to the `taos` and `taosrest` modules of the `taospy` package, respectively. In addition to wrapping the native and REST interfaces, `taospy` also provides a set of programming interfaces that conforms to the [Python Data Access Specification (PEP 249)](https://peps.python.org/pep-0249/). It is easy to integrate `taospy` with many third-party tools, such as [SQLAlchemy](https://www.sqlalchemy.org/) and [pandas](https://pandas.pydata.org/). `taos-ws-py` is an optional package to enable using WebSocket to connect TDengine. @@ -17,7 +17,7 @@ The direct connection to the server using the native interface provided by the c The source code for the Python connector is hosted on [GitHub](https://github.com/taosdata/taos-connector-python). ## Supported platforms -- The [supported platforms](/reference/connector/#supported-platforms) for the native connection are the same as the ones supported by the TDengine client. +- The [supported platforms](../#supported-platforms) for the native connection are the same as the ones supported by the TDengine client. - REST connections are supported on all platforms that can run Python. ### Supported features @@ -95,7 +95,7 @@ TDengine currently supports timestamp, number, character, Boolean type, and the 1. Install Python. The recent taospy package requires Python 3.6.2+. The earlier versions of taospy require Python 3.7+. The taos-ws-py package requires Python 3.7+. If Python is not available on your system, refer to the [Python BeginnersGuide](https://wiki.python.org/moin/BeginnersGuide/Download) to install it. 2. Install [pip](https://pypi.org/project/pip/). In most cases, the Python installer comes with the pip utility. If not, please refer to [pip documentation](https://pip.pypa.io/en/stable/installation/) to install it. -If you use a native connection, you will also need to [Install Client Driver](/reference/connector#Install-Client-Driver). The client install package includes the TDengine client dynamic link library (`libtaos.so` or `taos.dll`) and the TDengine CLI. +If you use a native connection, you will also need to [Install Client Driver](../#Install-Client-Driver). The client install package includes the TDengine client dynamic link library (`libtaos.so` or `taos.dll`) and the TDengine CLI. ### Install via pip @@ -444,7 +444,7 @@ The best practice for TaosCursor is to create a cursor at the beginning of a que ##### Use of the RestClient class -The `RestClient` class is a direct wrapper for the [REST API](/reference/rest-api). It contains only a `sql()` method for executing arbitrary SQL statements and returning the result. +The `RestClient` class is a direct wrapper for the [REST API](../../rest-api). It contains only a `sql()` method for executing arbitrary SQL statements and returning the result. ```python title="Use of RestClient" {{#include docs/examples/python/rest_client_example.py}} @@ -501,7 +501,7 @@ The queried results can only be fetched once. For example, only one of `fetch_al -The `RestClient` class is a direct wrapper for the [REST API](/reference/rest-api). It contains only a `sql()` method for executing arbitrary SQL statements and returning the result. +The `RestClient` class is a direct wrapper for the [REST API](../../rest-api). It contains only a `sql()` method for executing arbitrary SQL statements and returning the result. ```python {{#include docs/examples/python/rest_client_example.py}} @@ -561,7 +561,7 @@ The `TaosConnection` class and the `TaosResult` class already implement all the ##### Use of the RestClient class -The `RestClient` class is a direct wrapper for the [REST API](/reference/rest-api). It contains only a `sql()` method for executing arbitrary SQL statements and returning the result. +The `RestClient` class is a direct wrapper for the [REST API](../../rest-api). It contains only a `sql()` method for executing arbitrary SQL statements and returning the result. ```python title="Use of RestClient" {{#include docs/examples/python/rest_client_with_req_id_example.py}} diff --git a/docs/en/14-reference/03-connector/08-node.mdx b/docs/en/14-reference/03-connector/08-node.mdx index a02bcddfd6..bed06477f1 100644 --- a/docs/en/14-reference/03-connector/08-node.mdx +++ b/docs/en/14-reference/03-connector/08-node.mdx @@ -28,7 +28,7 @@ The REST connector supports all platforms that can run Node.js. ## Version support -Please refer to [version support list](/reference/connector#version-support) +Please refer to [version support list](../#version-support) ## Supported features @@ -58,7 +58,7 @@ Please refer to [version support list](/reference/connector#version-support) ### Pre-installation preparation - Install the Node.js development environment -- If you are using the REST connector, skip this step. However, if you use the native connector, please install the TDengine client driver. Please refer to [Install Client Driver](/reference/connector#Install-Client-Driver) for more details. We use [node-gyp](https://github.com/nodejs/node-gyp) to interact with TDengine instances and also need to install some dependencies mentioned below depending on the specific OS. +- If you are using the REST connector, skip this step. However, if you use the native connector, please install the TDengine client driver. Please refer to [Install Client Driver](../#Install-Client-Driver) for more details. We use [node-gyp](https://github.com/nodejs/node-gyp) to interact with TDengine instances and also need to install some dependencies mentioned below depending on the specific OS. diff --git a/docs/en/14-reference/03-connector/09-csharp.mdx b/docs/en/14-reference/03-connector/09-csharp.mdx index 203d44fe02..282be3af6b 100644 --- a/docs/en/14-reference/03-connector/09-csharp.mdx +++ b/docs/en/14-reference/03-connector/09-csharp.mdx @@ -36,7 +36,7 @@ Please note TDengine does not support 32bit Windows any more. ## Version support -Please refer to [version support list](/reference/connector#version-support) +Please refer to [version support list](../#version-support) ## Supported features @@ -69,7 +69,7 @@ Please refer to [version support list](/reference/connector#version-support) * Install the [.NET SDK](https://dotnet.microsoft.com/download) * [Nuget Client](https://docs.microsoft.com/en-us/nuget/install-nuget-client-tools) (optional installation) -* Install TDengine client driver, please refer to [Install client driver](/reference/connector/#install-client-driver) for details +* Install TDengine client driver, please refer to [Install client driver](../#install-client-driver) for details ### Install `TDengine.Connector` diff --git a/docs/en/14-reference/03-connector/80-php.mdx b/docs/en/14-reference/03-connector/80-php.mdx index b3c2065b6e..bff9e8e5d5 100644 --- a/docs/en/14-reference/03-connector/80-php.mdx +++ b/docs/en/14-reference/03-connector/80-php.mdx @@ -40,7 +40,7 @@ Because the version of TDengine client driver is tightly associated with that of ### Install TDengine Client Driver -Regarding how to install TDengine client driver please refer to [Install Client Driver](/reference/connector#installation-steps) +Regarding how to install TDengine client driver please refer to [Install Client Driver](../#installation-steps) ### Install php-tdengine diff --git a/docs/en/14-reference/03-connector/_preparation.mdx b/docs/en/14-reference/03-connector/_preparation.mdx index 25b78ec134..99887ac36b 100644 --- a/docs/en/14-reference/03-connector/_preparation.mdx +++ b/docs/en/14-reference/03-connector/_preparation.mdx @@ -2,7 +2,7 @@ :::info -Since the TDengine client driver is written in C, using the native connection requires loading the client driver shared library file, which is usually included in the TDengine installer. You can install either standard TDengine server installation package or [TDengine client installation package](/get-started/). For Windows development, you need to install the corresponding Windows client, please refer to [Install TDengine](../../get-started/package). +Since the TDengine client driver is written in C, using the native connection requires loading the client driver shared library file, which is usually included in the TDengine installer. You can install either standard TDengine server installation package or [TDengine client installation package](../../get-started/). For Windows development, you need to install the corresponding Windows client, please refer to [Install TDengine](../../get-started/package). - libtaos.so: After successful installation of TDengine on a Linux system, the dependent Linux version of the client driver `libtaos.so` file will be automatically linked to `/usr/lib/libtaos.so`, which is included in the Linux scannable path and does not need to be specified separately. - taos.dll: After installing the client on Windows, the dependent Windows version of the client driver taos.dll file will be automatically copied to the system default search path C:/Windows/System32, again without the need to specify it separately. diff --git a/docs/en/14-reference/04-taosadapter.md b/docs/en/14-reference/04-taosadapter.md index c75598b0df..a9330d21c7 100644 --- a/docs/en/14-reference/04-taosadapter.md +++ b/docs/en/14-reference/04-taosadapter.md @@ -186,7 +186,7 @@ See [example/config/taosadapter.toml](https://github.com/taosdata/taosadapter/bl ### TDengine RESTful interface -You can use any client that supports the http protocol to write data to or query data from TDengine by accessing the REST interface address `http://:6041/rest/sql`. See the [official documentation](/reference/rest-api/) for details. +You can use any client that supports the http protocol to write data to or query data from TDengine by accessing the REST interface address `http://:6041/rest/sql`. See the [official documentation](../rest-api/) for details. ### InfluxDB @@ -202,7 +202,7 @@ Support InfluxDB query parameters as follows. - `precision` The time precision used by TDengine - `u` TDengine user name - `p` TDengine password -- `ttl` The time to live of automatically created sub-table. This value cannot be updated. TDengine will use the ttl value of the first data of sub-table to create sub-table. For more information, please refer [Create Table](/taos-sql/table/#create-table) +- `ttl` The time to live of automatically created sub-table. This value cannot be updated. TDengine will use the ttl value of the first data of sub-table to create sub-table. For more information, please refer [Create Table](../../taos-sql/table/#create-table) Note: InfluxDB token authorization is not supported at present. Only Basic authorization and query parameter validation are supported. Example: curl --request POST http://127.0.0.1:6041/influxdb/v1/write?db=test --user "root:taosdata" --data-binary "measurement,host=host1 field1=2i,field2=2.0 1577836800000000000" diff --git a/docs/en/14-reference/06-taosdump.md b/docs/en/14-reference/06-taosdump.md index c07465a97c..5ec26df1fa 100644 --- a/docs/en/14-reference/06-taosdump.md +++ b/docs/en/14-reference/06-taosdump.md @@ -31,7 +31,7 @@ There are two ways to install taosdump: 2. backup multiple specified databases: use `-D db1,db2,... ` parameters; 3. back up some super or normal tables in the specified database: use `dbname stbname1 stbname2 tbname1 tbname2 ... ` parameters. Note that the first parameter of this input sequence is the database name, and only one database is supported. The second and subsequent parameters are the names of super or normal tables in that database, separated by spaces. 4. back up the system log database: TDengine clusters usually contain a system database named `log`. The data in this database is the data that TDengine runs itself, and the taosdump will not back up the log database by default. If users need to back up the log database, users can use the `-a` or `-allow-sys` command-line parameter. -5. Loose mode backup: taosdump version 1.4.1 onwards provides `-n` and `-L` parameters for backing up data without using escape characters and "loose" mode, which can reduce the number of backups if table names, column names, tag names do not use escape characters. This can also reduce the backup data time and backup data footprint. If you are unsure about using `-n` and `-L` conditions, please use the default parameters for "strict" mode backup. See the [official documentation](/taos-sql/escape) for a description of escaped characters. +5. Loose mode backup: taosdump version 1.4.1 onwards provides `-n` and `-L` parameters for backing up data without using escape characters and "loose" mode, which can reduce the number of backups if table names, column names, tag names do not use escape characters. This can also reduce the backup data time and backup data footprint. If you are unsure about using `-n` and `-L` conditions, please use the default parameters for "strict" mode backup. See the [official documentation](../../taos-sql/escape) for a description of escaped characters. :::tip - taosdump versions after 1.4.1 provide the `-I` argument for parsing Avro file schema and data. If users specify `-s` then only taosdump will parse schema. diff --git a/docs/en/14-reference/08-taos-shell.md b/docs/en/14-reference/08-taos-shell.md index 7e0433a8b2..8512f5b59d 100644 --- a/docs/en/14-reference/08-taos-shell.md +++ b/docs/en/14-reference/08-taos-shell.md @@ -8,7 +8,7 @@ The TDengine command-line interface (hereafter referred to as `TDengine CLI`) is ## Installation -If executed on the TDengine server-side, there is no need for additional installation steps to install TDengine CLI as it is already included and installed automatically. To run TDengine CLI in an environment where no TDengine server is running, the TDengine client installation package needs to be installed first. For details, please refer to [Connector](/reference/connector/). +If executed on the TDengine server-side, there is no need for additional installation steps to install TDengine CLI as it is already included and installed automatically. To run TDengine CLI in an environment where no TDengine server is running, the TDengine client installation package needs to be installed first. For details, please refer to [Connector](../connector/). ## Execution @@ -18,7 +18,7 @@ To access the TDengine CLI, you can execute `taos` command-line utility from a t taos ``` -TDengine CLI will display a welcome message and version information if it successfully connected to the TDengine service. If it fails, TDengine CLI will print an error message. See [FAQ](/train-faq/faq) to solve the problem of terminal connection failure to the server. The TDengine CLI prompts as follows: +TDengine CLI will display a welcome message and version information if it successfully connected to the TDengine service. If it fails, TDengine CLI will print an error message. See [FAQ](../../train-faq/faq) to solve the problem of terminal connection failure to the server. The TDengine CLI prompts as follows: ```cmd taos> diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index ac2364a65d..65c48f9190 100755 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -87,7 +87,7 @@ Ensure that your firewall rules do not block TCP port 6042 on any host in the c | Protocol | Default Port | Description | How to configure | | :------- | :----------- | :-------------------------------------------------------------------------------------------------------- | :--------------------------------------------------------------------------------------------- | | TCP | 6030 | Communication between client and server. In a multi-node cluster, communication between nodes. serverPort | -| TCP | 6041 | REST connection between client and server | Prior to 2.4.0.0: serverPort+11; After 2.4.0.0 refer to [taosAdapter](/reference/taosadapter/) | +| TCP | 6041 | REST connection between client and server | Prior to 2.4.0.0: serverPort+11; After 2.4.0.0 refer to [taosAdapter](../taosadapter/) | | TCP | 6043 | Service Port of taosKeeper | The parameter of taosKeeper | | TCP | 6044 | Data access port for StatsD | Configurable through taosAdapter parameters. | | UDP | 6045 | Data access for statsd | Configurable through taosAdapter parameters. | diff --git a/docs/en/14-reference/13-schemaless/13-schemaless.md b/docs/en/14-reference/13-schemaless/13-schemaless.md index eb336f4633..9b001ee79c 100644 --- a/docs/en/14-reference/13-schemaless/13-schemaless.md +++ b/docs/en/14-reference/13-schemaless/13-schemaless.md @@ -116,7 +116,7 @@ You can configure smlChildTableName in taos.cfg to specify table names, for exam 10. Taos.cfg adds the configuration of smlTsDefaultName (with a string value), which only works on the client side. After configuration, the time column name of the schemaless automatic table creation can be set through this configuration. If not configured, defaults to _ts. 11. Super table name or child table name are case sensitive. :::tip -All processing logic of schemaless will still follow TDengine's underlying restrictions on data structures, such as the total length of each row of data cannot exceed 48 KB(64 KB since version 3.0.5.0) and the total length of a tag value cannot exceed 16 KB. See [TDengine SQL Boundary Limits](/taos-sql/limit) for specific constraints in this area. +All processing logic of schemaless will still follow TDengine's underlying restrictions on data structures, such as the total length of each row of data cannot exceed 48 KB(64 KB since version 3.0.5.0) and the total length of a tag value cannot exceed 16 KB. See [TDengine SQL Boundary Limits](../../taos-sql/limit) for specific constraints in this area. ::: ## Time resolution recognition diff --git a/docs/en/20-third-party/02-prometheus.md b/docs/en/20-third-party/02-prometheus.md index bfdd3d015e..e6bd855939 100644 --- a/docs/en/20-third-party/02-prometheus.md +++ b/docs/en/20-third-party/02-prometheus.md @@ -16,7 +16,7 @@ Prometheus data can be stored in TDengine via the `remote_write` interface with To write Prometheus data to TDengine requires the following preparations. - The TDengine cluster is deployed and functioning properly -- taosAdapter is installed and running properly. Please refer to the [taosAdapter manual](/reference/taosadapter) for details. +- taosAdapter is installed and running properly. Please refer to the [taosAdapter manual](../../reference/taosadapter) for details. - Prometheus has been installed. Please refer to the [official documentation](https://prometheus.io/docs/prometheus/latest/installation/) for installing Prometheus ## Configuration steps diff --git a/docs/en/20-third-party/03-telegraf.md b/docs/en/20-third-party/03-telegraf.md index 7e99b84eab..900262f51f 100644 --- a/docs/en/20-third-party/03-telegraf.md +++ b/docs/en/20-third-party/03-telegraf.md @@ -14,7 +14,7 @@ Telegraf's data can be written to TDengine by simply adding the output configura To write Telegraf data to TDengine requires the following preparations. - The TDengine cluster is deployed and functioning properly -- taosAdapter is installed and running properly. Please refer to the [taosAdapter manual](/reference/taosadapter) for details. +- taosAdapter is installed and running properly. Please refer to the [taosAdapter manual](../../reference/taosadapter) for details. - Telegraf has been installed. Please refer to the [official documentation](https://docs.influxdata.com/telegraf/v1.22/install/) for Telegraf installation. - Telegraf collects the running status measurements of current system. You can enable [input plugins](https://docs.influxdata.com/telegraf/v1.22/plugins/) to insert [other formats](https://docs.influxdata.com/telegraf/v1.24/data_formats/input/) data to Telegraf then forward to TDengine. @@ -73,6 +73,6 @@ Query OK, 3 row(s) in set (0.013269s) - TDengine take influxdb format data and create unique ID for table names by the rule. The user can configure `smlChildTableName` parameter to generate specified table names if he/she needs. And he/she also need to insert data with specified data format. -For example, Add `smlChildTableName=tname` in the taos.cfg file. Insert data `st,tname=cpu1,t1=4 c1=3 1626006833639000000` then the table name will be cpu1. If there are multiple lines has same tname but different tag_set, the first line's tag_set will be used to automatically creating table and ignore other lines. Please refer to [TDengine Schemaless](/reference/schemaless/#Schemaless-Line-Protocol) +For example, Add `smlChildTableName=tname` in the taos.cfg file. Insert data `st,tname=cpu1,t1=4 c1=3 1626006833639000000` then the table name will be cpu1. If there are multiple lines has same tname but different tag_set, the first line's tag_set will be used to automatically creating table and ignore other lines. Please refer to [TDengine Schemaless](../../reference/schemaless/#Schemaless-Line-Protocol) ::: diff --git a/docs/en/20-third-party/05-collectd.md b/docs/en/20-third-party/05-collectd.md index d8c8e7f81d..5672852cd0 100644 --- a/docs/en/20-third-party/05-collectd.md +++ b/docs/en/20-third-party/05-collectd.md @@ -15,7 +15,7 @@ You can write the data collected by collectd to TDengine by simply modifying the Writing collectd data to the TDengine requires several preparations. - The TDengine cluster is deployed and running properly -- taosAdapter is installed and running, please refer to [taosAdapter's manual](/reference/taosadapter) for details +- taosAdapter is installed and running, please refer to [taosAdapter's manual](../../reference/taosadapter) for details - collectd has been installed. Please refer to the [official documentation](https://collectd.org/download.shtml) to install collectd ## Configuration steps diff --git a/docs/en/20-third-party/07-icinga2.md b/docs/en/20-third-party/07-icinga2.md index 540aae8689..43cd9b83ec 100644 --- a/docs/en/20-third-party/07-icinga2.md +++ b/docs/en/20-third-party/07-icinga2.md @@ -14,7 +14,7 @@ You can write the data collected by icinga2 to TDengine by simply modifying the To write icinga2 data to TDengine requires the following preparations. - The TDengine cluster is deployed and working properly -- taosAdapter is installed and running properly. Please refer to the [taosAdapter manual](/reference/taosadapter) for details. +- taosAdapter is installed and running properly. Please refer to the [taosAdapter manual](../../reference/taosadapter) for details. - icinga2 has been installed. Please refer to the [official documentation](https://icinga.com/docs/icinga-2/latest/doc/02-installation/) for icinga2 installation ## Configuration steps diff --git a/docs/en/20-third-party/08-tcollector.md b/docs/en/20-third-party/08-tcollector.md index f1c0ecd44d..83d01dda24 100644 --- a/docs/en/20-third-party/08-tcollector.md +++ b/docs/en/20-third-party/08-tcollector.md @@ -14,7 +14,7 @@ You can write the data collected by TCollector to TDengine by simply changing th To write data to the TDengine via TCollector requires the following preparations. - The TDengine cluster has been deployed and is working properly -- taosAdapter is installed and running properly. Please refer to the [taosAdapter manual](/reference/taosadapter) for details. +- taosAdapter is installed and running properly. Please refer to the [taosAdapter manual](../../reference/taosadapter) for details. - TCollector has been installed. Please refer to [official documentation](http://opentsdb.net/docs/build/html/user_guide/utilities/tcollector.html#installation-of-tcollector) for TCollector installation ## Configuration steps diff --git a/docs/en/20-third-party/09-emq-broker.md b/docs/en/20-third-party/09-emq-broker.md index 9dab6e52c9..7ca6cd4aa7 100644 --- a/docs/en/20-third-party/09-emq-broker.md +++ b/docs/en/20-third-party/09-emq-broker.md @@ -82,7 +82,7 @@ Edit the resource configuration to add the key/value pairing for Authorization. Basic cm9vdDp0YW9zZGF0YQ== ``` -Please refer to the [ TDengine REST API documentation ](/reference/rest-api/) for the authorization in details. +Please refer to the [ TDengine REST API documentation ](../../reference/rest-api/) for the authorization in details. Enter the rule engine replacement template in the message body: diff --git a/docs/en/20-third-party/11-kafka.md b/docs/en/20-third-party/11-kafka.md index b865c00bc3..cac4f5f604 100644 --- a/docs/en/20-third-party/11-kafka.md +++ b/docs/en/20-third-party/11-kafka.md @@ -94,7 +94,7 @@ The output as bellow: The role of the TDengine Sink Connector is to synchronize the data of the specified topic to TDengine. Users do not need to create databases and super tables in advance. The name of the target database can be specified manually (see the configuration parameter connection.database), or it can be generated according to specific rules (see the configuration parameter connection.database.prefix). -TDengine Sink Connector internally uses TDengine [modeless write interface](/reference/connector/cpp#modeless write-api) to write data to TDengine, currently supports data in three formats: [InfluxDB line protocol format](/develop /insert-data/influxdb-line), [OpenTSDB Telnet protocol format](/develop/insert-data/opentsdb-telnet), and [OpenTSDB JSON protocol format](/develop/insert-data/opentsdb-json). +TDengine Sink Connector internally uses TDengine [modeless write interface](../../reference/connector/cpp#modeless write-api) to write data to TDengine, currently supports data in three formats: [InfluxDB line protocol format](../../develop/insert-data/influxdb-line), [OpenTSDB Telnet protocol format](../../develop/insert-data/opentsdb-telnet), and [OpenTSDB JSON protocol format](../../develop/insert-data/opentsdb-json). The following example synchronizes the data of the topic meters to the target database power. The data format is the InfluxDB Line protocol format. @@ -213,7 +213,7 @@ If you see the above data, the synchronization is successful. If not, check the The role of the TDengine Source Connector is to push all the data of a specific TDengine database after a particular time to Kafka. The implementation principle of TDengine Source Connector is to first pull historical data in batches and then synchronize incremental data with the strategy of the regular query. At the same time, the changes in the table will be monitored, and the newly added table can be automatically synchronized. If Kafka Connect is restarted, synchronization will resume where it left off. -TDengine Source Connector will convert the data in TDengine data table into [InfluxDB Line protocol format](/develop/insert-data/influxdb-line/) or [OpenTSDB JSON protocol format](/develop/insert-data/opentsdb-json ) and then write to Kafka. +TDengine Source Connector will convert the data in TDengine data table into [InfluxDB Line protocol format](../../develop/insert-data/influxdb-line/) or [OpenTSDB JSON protocol format](../../develop/insert-data/opentsdb-json ) and then write to Kafka. The following sample program synchronizes the data in the database test to the topic tdengine-test-meters. diff --git a/docs/en/20-third-party/_deploytaosadapter.mdx b/docs/en/20-third-party/_deploytaosadapter.mdx index 840ca7640a..f8ee53af62 100644 --- a/docs/en/20-third-party/_deploytaosadapter.mdx +++ b/docs/en/20-third-party/_deploytaosadapter.mdx @@ -14,4 +14,4 @@ Check the running status of taosAdapter. systemctl status taosadapter ``` -taosAdapter Please refer to the `taosadapter --help` command output and [reference documentation](/reference/taosadapter) for detailed configuration parameters and usage of taosAdapter. +taosAdapter Please refer to the `taosadapter --help` command output and [reference documentation](../../reference/taosadapter) for detailed configuration parameters and usage of taosAdapter. diff --git a/docs/en/25-application/01-telegraf.md b/docs/en/25-application/01-telegraf.md index 1e3325b2b2..a6db826fa3 100644 --- a/docs/en/25-application/01-telegraf.md +++ b/docs/en/25-application/01-telegraf.md @@ -41,7 +41,7 @@ Download and install the [latest version of TDengine](https://docs.tdengine.com/ ### Install Grafana Plugin and Configure Data Source -Please refer to [Install Grafana Plugin and Configure Data Source](/third-party/grafana/#install-grafana-plugin-and-configure-data-source) +Please refer to [Install Grafana Plugin and Configure Data Source](../../third-party/grafana/#install-grafana-plugin-and-configure-data-source) ### Modify /etc/telegraf/telegraf.conf diff --git a/docs/en/25-application/02-collectd.md b/docs/en/25-application/02-collectd.md index ee1e944928..2c198dcdf5 100644 --- a/docs/en/25-application/02-collectd.md +++ b/docs/en/25-application/02-collectd.md @@ -44,7 +44,7 @@ Download and install the [latest version of TDengine](https://docs.tdengine.com/ ### Install Grafana Plugin and Configure Data Source -Please refer to [Install Grafana Plugin and Configure Data Source](/third-party/grafana/#install-grafana-plugin-and-configure-data-source) +Please refer to [Install Grafana Plugin and Configure Data Source](../../third-party/grafana/#install-grafana-plugin-and-configure-data-source) ### Configure collectd diff --git a/docs/en/25-application/_03-immigrate.md b/docs/en/25-application/_03-immigrate.md index 457a40614e..cdb3d5591c 100644 --- a/docs/en/25-application/_03-immigrate.md +++ b/docs/en/25-application/_03-immigrate.md @@ -70,7 +70,7 @@ You can use collectd and push the data to taosAdapter utilizing the write_tsdb p - **Tuning the Dashboard system** -After writing the data to TDengine, you can configure Grafana to visualize the data written to TDengine. To obtain and use the Grafana plugin provided by TDengine, please refer to [Links to other tools](/third-party/grafana). +After writing the data to TDengine, you can configure Grafana to visualize the data written to TDengine. To obtain and use the Grafana plugin provided by TDengine, please refer to [Links to other tools](../../third-party/grafana). TDengine provides two sets of Dashboard templates by default, and users only need to import the templates from the Grafana directory into Grafana to activate their use. @@ -396,7 +396,7 @@ Hard disk writing performance has little effect on TDengine. The TDengine writin ### Computational resource estimates -Due to the characteristics of IoT data, when the frequency of data generation is consistent, the writing process of TDengine maintains a relatively fixed amount of resource consumption (computing and storage). According to the [TDengine Operation and Maintenance Guide](/operation/) description, the system consumes less than 1 CPU core at 22,000 writes per second. +Due to the characteristics of IoT data, when the frequency of data generation is consistent, the writing process of TDengine maintains a relatively fixed amount of resource consumption (computing and storage). According to the [TDengine Operation and Maintenance Guide](../../operation/) description, the system consumes less than 1 CPU core at 22,000 writes per second. In estimating the CPU resources consumed by the query, assuming that the application requires the database to provide 10,000 QPS, the CPU time consumed by each query is about 1 ms. The query provided by each core per second is 1,000 QPS, which satisfies 10,000 QPS. The query request requires at least 10 cores. For the system as a whole system to have less than 50% CPU load, the entire cluster needs twice as many cores i.e. 20 cores. diff --git a/docs/examples/python/bind_param_example.py b/docs/examples/python/bind_param_example.py index 6a67434f87..e3df9f7d25 100644 --- a/docs/examples/python/bind_param_example.py +++ b/docs/examples/python/bind_param_example.py @@ -20,7 +20,7 @@ def get_ts(ts: str): def create_stable(): conn = taos.connect() try: - conn.execute("CREATE DATABASE power") + conn.execute("CREATE DATABASE power keep 36500") conn.execute("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " "TAGS (location BINARY(64), groupId INT)") finally: diff --git a/docs/examples/python/conn_websocket_pandas.py b/docs/examples/python/conn_websocket_pandas.py index 5cad5384b2..2986aace9f 100644 --- a/docs/examples/python/conn_websocket_pandas.py +++ b/docs/examples/python/conn_websocket_pandas.py @@ -4,7 +4,7 @@ import taos taos_conn = taos.connect() taos_conn.execute('drop database if exists power') -taos_conn.execute('create database if not exists power wal_retention_period 3600') +taos_conn.execute('create database if not exists power wal_retention_period 3600 keep 36500 ') taos_conn.execute("use power") taos_conn.execute( "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)") diff --git a/docs/examples/python/connect_rest_examples.py b/docs/examples/python/connect_rest_examples.py index 1c432dcc65..c8a9292547 100644 --- a/docs/examples/python/connect_rest_examples.py +++ b/docs/examples/python/connect_rest_examples.py @@ -11,7 +11,7 @@ conn = connect(url="http://localhost:6041", # create STable cursor = conn.cursor() cursor.execute("DROP DATABASE IF EXISTS power") -cursor.execute("CREATE DATABASE power") +cursor.execute("CREATE DATABASE power keep 36500 ") cursor.execute( "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)") diff --git a/docs/examples/python/connect_rest_with_req_id_examples.py b/docs/examples/python/connect_rest_with_req_id_examples.py index f1b5915ea3..568cbea168 100644 --- a/docs/examples/python/connect_rest_with_req_id_examples.py +++ b/docs/examples/python/connect_rest_with_req_id_examples.py @@ -11,7 +11,7 @@ conn = connect(url="http://localhost:6041", # create STable cursor = conn.cursor() cursor.execute("DROP DATABASE IF EXISTS power", req_id=1) -cursor.execute("CREATE DATABASE power", req_id=2) +cursor.execute("CREATE DATABASE power keep 36500", req_id=2) cursor.execute( "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)", req_id=3) diff --git a/docs/examples/python/connect_websocket_examples.py b/docs/examples/python/connect_websocket_examples.py index 29452bbf9d..75e7422a90 100644 --- a/docs/examples/python/connect_websocket_examples.py +++ b/docs/examples/python/connect_websocket_examples.py @@ -6,13 +6,13 @@ conn = taosws.connect("taosws://root:taosdata@localhost:6041") # ANCHOR: basic conn.execute("drop database if exists connwspy") -conn.execute("create database if not exists connwspy wal_retention_period 3600") +conn.execute("create database if not exists connwspy wal_retention_period 3600 keep 36500 ") conn.execute("use connwspy") conn.execute("create table if not exists stb (ts timestamp, c1 int) tags (t1 int)") conn.execute("create table if not exists tb1 using stb tags (1)") conn.execute("insert into tb1 values (now, 1)") -conn.execute("insert into tb1 values (now, 2)") -conn.execute("insert into tb1 values (now, 3)") +conn.execute("insert into tb1 values (now+1s, 2)") +conn.execute("insert into tb1 values (now+2s, 3)") r = conn.execute("select * from stb") result = conn.query("select * from stb") diff --git a/docs/examples/python/connect_websocket_with_req_id_examples.py b/docs/examples/python/connect_websocket_with_req_id_examples.py index f5f76c8446..3588b8e41f 100644 --- a/docs/examples/python/connect_websocket_with_req_id_examples.py +++ b/docs/examples/python/connect_websocket_with_req_id_examples.py @@ -6,7 +6,7 @@ conn = taosws.connect("taosws://root:taosdata@localhost:6041") # ANCHOR: basic conn.execute("drop database if exists connwspy", req_id=1) -conn.execute("create database if not exists connwspy", req_id=2) +conn.execute("create database if not exists connwspy keep 36500", req_id=2) conn.execute("use connwspy", req_id=3) conn.execute("create table if not exists stb (ts timestamp, c1 int) tags (t1 int)", req_id=4) conn.execute("create table if not exists tb1 using stb tags (1)", req_id=5) diff --git a/docs/examples/python/connection_usage_native_reference.py b/docs/examples/python/connection_usage_native_reference.py index 0a23c5f95b..3610087e7f 100644 --- a/docs/examples/python/connection_usage_native_reference.py +++ b/docs/examples/python/connection_usage_native_reference.py @@ -4,7 +4,7 @@ import taos conn = taos.connect() # Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement. conn.execute("DROP DATABASE IF EXISTS test") -conn.execute("CREATE DATABASE test") +conn.execute("CREATE DATABASE test keep 36500") # change database. same as execute "USE db" conn.select_db("test") conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") diff --git a/docs/examples/python/connection_usage_native_reference_with_req_id.py b/docs/examples/python/connection_usage_native_reference_with_req_id.py index 24d0914ad5..3d568a1e1e 100644 --- a/docs/examples/python/connection_usage_native_reference_with_req_id.py +++ b/docs/examples/python/connection_usage_native_reference_with_req_id.py @@ -4,7 +4,7 @@ import taos conn = taos.connect() # Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement. conn.execute("DROP DATABASE IF EXISTS test", req_id=1) -conn.execute("CREATE DATABASE test", req_id=2) +conn.execute("CREATE DATABASE test keep 36500", req_id=2) # change database. same as execute "USE db" conn.select_db("test") conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=3) diff --git a/docs/examples/python/cursor_usage_native_reference.py b/docs/examples/python/cursor_usage_native_reference.py index a5103810f0..32ee51354d 100644 --- a/docs/examples/python/cursor_usage_native_reference.py +++ b/docs/examples/python/cursor_usage_native_reference.py @@ -4,7 +4,7 @@ conn = taos.connect() cursor = conn.cursor() cursor.execute("DROP DATABASE IF EXISTS test") -cursor.execute("CREATE DATABASE test") +cursor.execute("CREATE DATABASE test keep 36500") cursor.execute("USE test") cursor.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") diff --git a/docs/examples/python/cursor_usage_native_reference_with_req_id.py b/docs/examples/python/cursor_usage_native_reference_with_req_id.py index 15207ee6bc..345a804923 100644 --- a/docs/examples/python/cursor_usage_native_reference_with_req_id.py +++ b/docs/examples/python/cursor_usage_native_reference_with_req_id.py @@ -4,7 +4,7 @@ conn = taos.connect() cursor = conn.cursor() cursor.execute("DROP DATABASE IF EXISTS test", req_id=1) -cursor.execute("CREATE DATABASE test", req_id=2) +cursor.execute("CREATE DATABASE test keep 36500", req_id=2) cursor.execute("USE test", req_id=3) cursor.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=4) diff --git a/docs/examples/python/fast_write_example.py b/docs/examples/python/fast_write_example.py index 626e3310b1..76e84e97ac 100644 --- a/docs/examples/python/fast_write_example.py +++ b/docs/examples/python/fast_write_example.py @@ -160,7 +160,7 @@ def main(infinity): conn = get_connection() conn.execute("DROP DATABASE IF EXISTS test") - conn.execute("CREATE DATABASE IF NOT EXISTS test") + conn.execute("CREATE DATABASE IF NOT EXISTS test keep 36500") conn.execute("CREATE STABLE IF NOT EXISTS test.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " "TAGS (location BINARY(64), groupId INT)") conn.close() diff --git a/docs/examples/python/json_protocol_example.py b/docs/examples/python/json_protocol_example.py index 58b38f3ff6..a38dcbf0ad 100644 --- a/docs/examples/python/json_protocol_example.py +++ b/docs/examples/python/json_protocol_example.py @@ -16,7 +16,7 @@ def get_connection(): def create_database(conn): - conn.execute("CREATE DATABASE test") + conn.execute("CREATE DATABASE test keep 36500") conn.execute("USE test") diff --git a/docs/examples/python/kafka_example_common.py b/docs/examples/python/kafka_example_common.py index 1c735abfc0..ed0540574f 100644 --- a/docs/examples/python/kafka_example_common.py +++ b/docs/examples/python/kafka_example_common.py @@ -5,7 +5,7 @@ LOCATIONS = ['California.SanFrancisco', 'California.LosAngles', 'California.SanD 'California.PaloAlto', 'California.Campbell', 'California.MountainView', 'California.Sunnyvale', 'California.SantaClara', 'California.Cupertino'] -CREATE_DATABASE_SQL = 'create database if not exists {} keep 365 duration 10 buffer 16 wal_level 1 wal_retention_period 3600' +CREATE_DATABASE_SQL = 'create database if not exists {} keep 36500 duration 10 buffer 16 wal_level 1 wal_retention_period 3600' USE_DATABASE_SQL = 'use {}' DROP_TABLE_SQL = 'drop table if exists meters' DROP_DATABASE_SQL = 'drop database if exists {}' diff --git a/docs/examples/python/line_protocol_example.py b/docs/examples/python/line_protocol_example.py index 735e8e7eb8..6482032e6e 100644 --- a/docs/examples/python/line_protocol_example.py +++ b/docs/examples/python/line_protocol_example.py @@ -15,7 +15,7 @@ def get_connection(): def create_database(conn): # the default precision is ms (microsecond), but we use us(microsecond) here. - conn.execute("CREATE DATABASE test precision 'us'") + conn.execute("CREATE DATABASE test precision 'us' keep 36500") conn.execute("USE test") diff --git a/docs/examples/python/multi_bind_example.py b/docs/examples/python/multi_bind_example.py index 205ba69fb2..b29e1cd17e 100644 --- a/docs/examples/python/multi_bind_example.py +++ b/docs/examples/python/multi_bind_example.py @@ -71,7 +71,7 @@ def insert_data(): def create_stable(): conn = taos.connect() try: - conn.execute("CREATE DATABASE power") + conn.execute("CREATE DATABASE power keep 36500") conn.execute("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " "TAGS (location BINARY(64), groupId INT)") finally: diff --git a/docs/examples/python/native_insert_example.py b/docs/examples/python/native_insert_example.py index cdde7d23d2..0fba375678 100644 --- a/docs/examples/python/native_insert_example.py +++ b/docs/examples/python/native_insert_example.py @@ -18,7 +18,7 @@ def get_connection() -> taos.TaosConnection: def create_stable(conn: taos.TaosConnection): - conn.execute("CREATE DATABASE power") + conn.execute("CREATE DATABASE power keep 36500") conn.execute("USE power") conn.execute("CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " "TAGS (location BINARY(64), groupId INT)") diff --git a/docs/examples/python/result_set_examples.py b/docs/examples/python/result_set_examples.py index 6cba0d3f73..234c624a4d 100644 --- a/docs/examples/python/result_set_examples.py +++ b/docs/examples/python/result_set_examples.py @@ -2,7 +2,7 @@ import taos conn = taos.connect() conn.execute("DROP DATABASE IF EXISTS test") -conn.execute("CREATE DATABASE test") +conn.execute("CREATE DATABASE test keep 36500") conn.select_db("test") conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") # prepare data diff --git a/docs/examples/python/result_set_with_req_id_examples.py b/docs/examples/python/result_set_with_req_id_examples.py index 90ae2f4f26..f46a3697b3 100644 --- a/docs/examples/python/result_set_with_req_id_examples.py +++ b/docs/examples/python/result_set_with_req_id_examples.py @@ -2,7 +2,7 @@ import taos conn = taos.connect() conn.execute("DROP DATABASE IF EXISTS test", req_id=1) -conn.execute("CREATE DATABASE test", req_id=2) +conn.execute("CREATE DATABASE test keep 36500", req_id=2) conn.select_db("test") conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=3) # prepare data diff --git a/docs/examples/python/schemaless_insert.py b/docs/examples/python/schemaless_insert.py index 334a4b728f..74ab4b15fe 100644 --- a/docs/examples/python/schemaless_insert.py +++ b/docs/examples/python/schemaless_insert.py @@ -3,7 +3,7 @@ import taos conn = taos.connect() dbname = "pytest_line" conn.execute("drop database if exists %s" % dbname) -conn.execute("create database if not exists %s precision 'us'" % dbname) +conn.execute("create database if not exists %s precision 'us' keep 36500" % dbname) conn.select_db(dbname) lines = [ diff --git a/docs/examples/python/schemaless_insert_raw.py b/docs/examples/python/schemaless_insert_raw.py index 0fda7dc505..b5ef5833a6 100644 --- a/docs/examples/python/schemaless_insert_raw.py +++ b/docs/examples/python/schemaless_insert_raw.py @@ -10,9 +10,9 @@ try: conn.execute("drop database if exists %s" % dbname) if taos.IS_V3: - conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname) + conn.execute("create database if not exists %s schemaless 1 precision 'ns' keep 36500" % dbname) else: - conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname) + conn.execute("create database if not exists %s update 2 precision 'ns' keep 36500" % dbname) conn.select_db(dbname) diff --git a/docs/examples/python/schemaless_insert_raw_req_id.py b/docs/examples/python/schemaless_insert_raw_req_id.py index 606e510986..5488e2ddc0 100644 --- a/docs/examples/python/schemaless_insert_raw_req_id.py +++ b/docs/examples/python/schemaless_insert_raw_req_id.py @@ -10,9 +10,9 @@ try: conn.execute("drop database if exists %s" % dbname) if taos.IS_V3: - conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname) + conn.execute("create database if not exists %s schemaless 1 precision 'ns' keep 36500" % dbname) else: - conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname) + conn.execute("create database if not exists %s update 2 precision 'ns' keep 36500" % dbname) conn.select_db(dbname) diff --git a/docs/examples/python/schemaless_insert_raw_ttl.py b/docs/examples/python/schemaless_insert_raw_ttl.py index cf57792534..73909e4290 100644 --- a/docs/examples/python/schemaless_insert_raw_ttl.py +++ b/docs/examples/python/schemaless_insert_raw_ttl.py @@ -10,9 +10,9 @@ try: conn.execute("drop database if exists %s" % dbname) if taos.IS_V3: - conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname) + conn.execute("create database if not exists %s schemaless 1 precision 'ns' keep 36500" % dbname) else: - conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname) + conn.execute("create database if not exists %s update 2 precision 'ns' keep 36500" % dbname) conn.select_db(dbname) diff --git a/docs/examples/python/schemaless_insert_req_id.py b/docs/examples/python/schemaless_insert_req_id.py index ee1472db69..a5091d80a8 100644 --- a/docs/examples/python/schemaless_insert_req_id.py +++ b/docs/examples/python/schemaless_insert_req_id.py @@ -4,7 +4,7 @@ from taos import SmlProtocol, SmlPrecision conn = taos.connect() dbname = "pytest_line" conn.execute("drop database if exists %s" % dbname) -conn.execute("create database if not exists %s precision 'us'" % dbname) +conn.execute("create database if not exists %s precision 'us' keep 36500" % dbname) conn.select_db(dbname) lines = [ diff --git a/docs/examples/python/schemaless_insert_ttl.py b/docs/examples/python/schemaless_insert_ttl.py index 85050439f2..6ad134fae1 100644 --- a/docs/examples/python/schemaless_insert_ttl.py +++ b/docs/examples/python/schemaless_insert_ttl.py @@ -4,7 +4,7 @@ from taos import SmlProtocol, SmlPrecision conn = taos.connect() dbname = "pytest_line" conn.execute("drop database if exists %s" % dbname) -conn.execute("create database if not exists %s precision 'us'" % dbname) +conn.execute("create database if not exists %s precision 'us' keep 36500" % dbname) conn.select_db(dbname) lines = [ diff --git a/docs/examples/python/sql_writer.py b/docs/examples/python/sql_writer.py index 3456981a7b..d62f4c8a8d 100644 --- a/docs/examples/python/sql_writer.py +++ b/docs/examples/python/sql_writer.py @@ -10,7 +10,7 @@ class SQLWriter: self._tb_tags = {} self._conn = get_connection_func() self._max_sql_length = self.get_max_sql_length() - self._conn.execute("create database if not exists test") + self._conn.execute("create database if not exists test keep 36500") self._conn.execute("USE test") def get_max_sql_length(self): diff --git a/docs/examples/python/stmt_example.py b/docs/examples/python/stmt_example.py index 83197a777a..cfdd81c90c 100644 --- a/docs/examples/python/stmt_example.py +++ b/docs/examples/python/stmt_example.py @@ -10,7 +10,7 @@ db_name = 'test_ws_stmt' def before(): taos_conn = taos.connect() taos_conn.execute("drop database if exists %s" % db_name) - taos_conn.execute("create database %s" % db_name) + taos_conn.execute("create database %s keep 36500" % db_name) taos_conn.select_db(db_name) taos_conn.execute("create table t1 (ts timestamp, a int, b float, c varchar(10))") taos_conn.execute( diff --git a/docs/examples/python/stmt_websocket_example.py b/docs/examples/python/stmt_websocket_example.py index d0824cfa9f..2acab188f3 100644 --- a/docs/examples/python/stmt_websocket_example.py +++ b/docs/examples/python/stmt_websocket_example.py @@ -9,7 +9,7 @@ import taos def before_test(db_name): taos_conn = taos.connect() taos_conn.execute("drop database if exists %s" % db_name) - taos_conn.execute("create database %s" % db_name) + taos_conn.execute("create database %s keep 36500" % db_name) taos_conn.select_db(db_name) taos_conn.execute("create table t1 (ts timestamp, a int, b float, c varchar(10))") taos_conn.execute( diff --git a/docs/examples/python/telnet_line_protocol_example.py b/docs/examples/python/telnet_line_protocol_example.py index d812e186af..0d524e8d4a 100644 --- a/docs/examples/python/telnet_line_protocol_example.py +++ b/docs/examples/python/telnet_line_protocol_example.py @@ -19,7 +19,7 @@ def get_connection(): def create_database(conn): - conn.execute("CREATE DATABASE test") + conn.execute("CREATE DATABASE test keep 36500") conn.execute("USE test") diff --git a/docs/examples/python/tmq_assignment_example.py b/docs/examples/python/tmq_assignment_example.py index c370db47a5..c063768af4 100644 --- a/docs/examples/python/tmq_assignment_example.py +++ b/docs/examples/python/tmq_assignment_example.py @@ -7,7 +7,7 @@ def prepare(): conn = taos.connect() conn.execute("drop topic if exists tmq_assignment_demo_topic") conn.execute("drop database if exists tmq_assignment_demo_db") - conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600") + conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600 keep 36500") conn.select_db("tmq_assignment_demo_db") conn.execute( "create table if not exists tmq_assignment_demo_table (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)") diff --git a/docs/examples/python/tmq_example.py b/docs/examples/python/tmq_example.py index 5b462fa153..e0bddd9911 100644 --- a/docs/examples/python/tmq_example.py +++ b/docs/examples/python/tmq_example.py @@ -6,7 +6,7 @@ def init_tmq_env(db, topic): conn = taos.connect() conn.execute("drop topic if exists {}".format(topic)) conn.execute("drop database if exists {}".format(db)) - conn.execute("create database if not exists {} wal_retention_period 3600".format(db)) + conn.execute("create database if not exists {} wal_retention_period 3600 keep 36500".format(db)) conn.select_db(db) conn.execute( "create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))") diff --git a/docs/examples/python/tmq_websocket_assgnment_example.py b/docs/examples/python/tmq_websocket_assgnment_example.py index a180ef840e..ca50015162 100644 --- a/docs/examples/python/tmq_websocket_assgnment_example.py +++ b/docs/examples/python/tmq_websocket_assgnment_example.py @@ -6,7 +6,7 @@ def prepare(): conn = taos.connect() conn.execute("drop topic if exists tmq_assignment_demo_topic") conn.execute("drop database if exists tmq_assignment_demo_db") - conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600") + conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600 keep 36500") conn.select_db("tmq_assignment_demo_db") conn.execute( "create table if not exists tmq_assignment_demo_table (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)") diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index bd33281bc0..c3e0e9a07a 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -26,7 +26,6 @@ database_option: { | PAGESIZE value | PRECISION {'ms' | 'us' | 'ns'} | REPLICA value - | RETENTIONS ingestion_duration:keep_duration ... | WAL_LEVEL {1 | 2} | VGROUPS value | SINGLE_STABLE {0 | 1} @@ -61,7 +60,6 @@ database_option: { - PAGESIZE：一个 VNODE 中元数据存储引擎的页大小，单位为 KB，默认为 4 KB。范围为 1 到 16384，即 1 KB 到 16 MB。 - PRECISION：数据库的时间戳精度。ms 表示毫秒，us 表示微秒，ns 表示纳秒，默认 ms 毫秒。 - REPLICA：表示数据库副本数，取值为 1 或 3，默认为 1。在集群中使用，副本数必须小于或等于 DNODE 的数目。 -- RETENTIONS：表示数据的聚合周期和保存时长，如 RETENTIONS 15s:7d,1m:21d,15m:50d 表示数据原始采集周期为 15 秒，原始数据保存 7 天；按 1 分钟聚合的数据保存 21 天；按 15 分钟聚合的数据保存 50 天。目前支持且只支持三级存储周期。 - WAL_LEVEL：WAL 级别，默认为 1。 - 1：写 WAL，但不执行 fsync。 - 2：写 WAL，而且执行 fsync。 diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md index 9258258263..7e20f20574 100644 --- a/docs/zh/12-taos-sql/03-table.md +++ b/docs/zh/12-taos-sql/03-table.md @@ -30,9 +30,6 @@ table_options: table_option: { COMMENT 'string_value' - | WATERMARK duration[,duration] - | MAX_DELAY duration[,duration] - | ROLLUP(func_name [, func_name] ...) | SMA(col_name [, col_name] ...) | TTL value } @@ -52,11 +49,8 @@ table_option: { **参数说明** 1. COMMENT：表注释。可用于超级表、子表和普通表。 -2. WATERMARK：指定窗口的关闭时间，默认值为 5 秒，最小单位毫秒，范围为 0 到 15 分钟，多个以逗号分隔。只可用于超级表，且只有当数据库使用了 RETENTIONS 参数时，才可以使用此表参数。 -3. MAX_DELAY：用于控制推送计算结果的最大延迟，默认值为 interval 的值(但不能超过最大值)，最小单位毫秒，范围为 1 毫秒到 15 分钟，多个以逗号分隔。注：不建议 MAX_DELAY 设置太小，否则会过于频繁的推送结果，影响存储和查询性能，如无特殊需求，取默认值即可。只可用于超级表，且只有当数据库使用了 RETENTIONS 参数时，才可以使用此表参数。 -4. ROLLUP：Rollup 指定的聚合函数，提供基于多层级的降采样聚合结果。只可用于超级表。只有当数据库使用了 RETENTIONS 参数时，才可以使用此表参数。作用于超级表除 TS 列外的其它所有列，但是只能定义一个聚合函数。 聚合函数支持 avg, sum, min, max, last, first。 -5. SMA：Small Materialized Aggregates，提供基于数据块的自定义预计算功能。预计算类型包括 MAX、MIN 和 SUM。可用于超级表/普通表。 -6. TTL：Time to Live，是用户用来指定表的生命周期的参数。如果创建表时指定了这个参数，当该表的存在时间超过 TTL 指定的时间后，TDengine 自动删除该表。这个 TTL 的时间只是一个大概时间，系统不保证到了时间一定会将其删除，而只保证存在这样一个机制且最终一定会删除。TTL 单位是天，默认为 0，表示不限制，到期时间为表创建时间加上 TTL 时间。TTL 与数据库 KEEP 参数没有关联，如果 KEEP 比 TTL 小，在表被删除之前数据也可能已经被删除。 +2. SMA：Small Materialized Aggregates，提供基于数据块的自定义预计算功能。预计算类型包括 MAX、MIN 和 SUM。可用于超级表/普通表。 +3. TTL：Time to Live，是用户用来指定表的生命周期的参数。如果创建表时指定了这个参数，当该表的存在时间超过 TTL 指定的时间后，TDengine 自动删除该表。这个 TTL 的时间只是一个大概时间，系统不保证到了时间一定会将其删除，而只保证存在这样一个机制且最终一定会删除。TTL 单位是天，默认为 0，表示不限制，到期时间为表创建时间加上 TTL 时间。TTL 与数据库 KEEP 参数没有关联，如果 KEEP 比 TTL 小，在表被删除之前数据也可能已经被删除。 ## 创建子表 diff --git a/include/common/cos.h b/include/common/cos.h index 21b645f604..c6b159c1da 100644 --- a/include/common/cos.h +++ b/include/common/cos.h @@ -43,6 +43,7 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, int32_t s3GetObjectsByPrefix(const char *prefix, const char *path); void s3EvictCache(const char *path, long object_size); long s3Size(const char *object_name); +int32_t s3GetObjectToFile(const char *object_name, char *fileName); #ifdef __cplusplus } diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 30720da8a7..87a6a90a7e 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -258,8 +258,6 @@ typedef struct SQueryTableDataCond { int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock); void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock); -int32_t tEncodeDataBlocks(void** buf, const SArray* blocks); -void* tDecodeDataBlocks(const void* buf, SArray** blocks); void colDataDestroy(SColumnInfoData* pColData); //====================================================================================================================== @@ -294,7 +292,7 @@ typedef struct STableBlockDistInfo { int32_t defMaxRows; int32_t firstSeekTimeUs; uint32_t numOfInmemRows; - uint32_t numOfSmallBlocks; + uint32_t numOfSttRows; uint32_t numOfVgroups; int32_t blockRowsHisto[20]; } STableBlockDistInfo; diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 58517a5db0..33cfada338 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -75,12 +75,15 @@ extern int32_t tsElectInterval; extern int32_t tsHeartbeatInterval; extern int32_t tsHeartbeatTimeout; +// vnode +extern int64_t tsVndCommitMaxIntervalMs; + // snode extern int32_t tsRsyncPort; -extern char tsCheckpointBackupDir[]; +extern char tsCheckpointBackupDir[]; // vnode checkpoint -extern char tsSnodeAddress[]; //127.0.0.1:873 +extern char tsSnodeAddress[]; // 127.0.0.1:873 // mnode extern int64_t tsMndSdbWriteDelta; @@ -104,8 +107,8 @@ extern int32_t tsMonitorMaxLogs; extern bool tsMonitorComp; // audit -extern bool tsEnableAudit; -extern bool tsEnableAuditCreateTable; +extern bool tsEnableAudit; +extern bool tsEnableAuditCreateTable; // telem extern bool tsEnableTelem; @@ -113,9 +116,9 @@ extern int32_t tsTelemInterval; extern char tsTelemServer[]; extern uint16_t tsTelemPort; extern bool tsEnableCrashReport; -extern char *tsTelemUri; -extern char *tsClientCrashReportUri; -extern char *tsSvrCrashReportUri; +extern char * tsTelemUri; +extern char * tsClientCrashReportUri; +extern char * tsSvrCrashReportUri; // query buffer management extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing diff --git a/include/common/tgrant.h b/include/common/tgrant.h index a5f3ab2e3f..f06fca8014 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -31,8 +31,6 @@ extern "C" { #endif #define GRANT_HEART_BEAT_MIN 2 -#define GRANT_ACTIVE_CODE "activeCode" -#define GRANT_C_ACTIVE_CODE "cActiveCode" typedef enum { TSDB_GRANT_ALL, @@ -52,11 +50,6 @@ typedef enum { TSDB_GRANT_TABLE, } EGrantType; -typedef struct { - int64_t grantedTime; - int64_t connGrantedTime; -} SGrantedInfo; - int32_t grantCheck(EGrantType grant); int32_t grantAlterActiveCode(int32_t did, const char* old, const char* newer, char* out, int8_t type); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index d9ac701369..e6ed0f250c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -35,12 +35,14 @@ extern "C" { #define TD_MSG_NUMBER_ #undef TD_MSG_DICT_ #undef TD_MSG_INFO_ +#undef TD_MSG_RANGE_CODE_ #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" #undef TD_MSG_NUMBER_ #undef TD_MSG_DICT_ #undef TD_MSG_INFO_ +#undef TD_MSG_RANGE_CODE_ #define TD_MSG_SEG_CODE_ #include "tmsgdef.h" @@ -48,33 +50,31 @@ extern "C" { #undef TD_MSG_DICT_ #undef TD_MSG_INFO_ #undef TD_MSG_SEG_CODE_ +#undef TD_MSG_RANGE_CODE_ #include "tmsgdef.h" extern char* tMsgInfo[]; extern int32_t tMsgDict[]; - -#define TMSG_SEG_CODE(TYPE) (((TYPE)&0xff00) >> 8) -#define TMSG_SEG_SEQ(TYPE) ((TYPE)&0xff) -#define TMSG_INFO(TYPE) \ - ((TYPE) < TDMT_DND_MAX_MSG || (TYPE) < TDMT_MND_MAX_MSG || (TYPE) < TDMT_VND_MAX_MSG || (TYPE) < TDMT_SCH_MAX_MSG || \ - (TYPE) < TDMT_STREAM_MAX_MSG || (TYPE) < TDMT_MON_MAX_MSG || (TYPE) < TDMT_SYNC_MAX_MSG) || \ - (TYPE) < TDMT_VND_STREAM_MSG || (TYPE) < TDMT_VND_TMQ_MSG || (TYPE) < TDMT_VND_TMQ_MAX_MSG \ - ? tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)] \ - : 0 - -#define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)) +extern int32_t tMsgRangeDict[]; typedef uint16_t tmsg_t; +#define TMSG_SEG_CODE(TYPE) (((TYPE)&0xff00) >> 8) +#define TMSG_SEG_SEQ(TYPE) ((TYPE)&0xff) +#define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)) + static inline bool tmsgIsValid(tmsg_t type) { - if (type < TDMT_DND_MAX_MSG || type < TDMT_MND_MAX_MSG || type < TDMT_VND_MAX_MSG || type < TDMT_SCH_MAX_MSG || - type < TDMT_STREAM_MAX_MSG || type < TDMT_MON_MAX_MSG || type < TDMT_SYNC_MAX_MSG || type < TDMT_VND_STREAM_MSG || - type < TDMT_VND_TMQ_MSG || type < TDMT_VND_TMQ_MAX_MSG) { - return true; - } else { - return false; + // static int8_t sz = sizeof(tMsgRangeDict) / sizeof(tMsgRangeDict[0]); + int8_t maxSegIdx = TMSG_SEG_CODE(TDMT_MAX_MSG); + int segIdx = TMSG_SEG_CODE(type); + if (segIdx >= 0 && segIdx < maxSegIdx) { + return type < tMsgRangeDict[segIdx]; } + return false; } + +#define TMSG_INFO(type) (tmsgIsValid(type) ? tMsgInfo[TMSG_INDEX(type)] : "unKnown") + static inline bool vnodeIsMsgBlock(tmsg_t type) { return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || (type == TDMT_VND_UPDATE_TAG_VAL) || (type == TDMT_VND_ALTER_CONFIRM) || (type == TDMT_VND_COMMIT) || @@ -169,14 +169,14 @@ typedef enum _mgmt_table { #define TSDB_FILL_PREV 6 #define TSDB_FILL_NEXT 7 -#define TSDB_ALTER_USER_PASSWD 0x1 -#define TSDB_ALTER_USER_SUPERUSER 0x2 -#define TSDB_ALTER_USER_ENABLE 0x3 -#define TSDB_ALTER_USER_SYSINFO 0x4 -#define TSDB_ALTER_USER_ADD_PRIVILEGES 0x5 -#define TSDB_ALTER_USER_DEL_PRIVILEGES 0x6 -#define TSDB_ALTER_USER_ADD_WHITE_LIST 0x7 -#define TSDB_ALTER_USER_DROP_WHITE_LIST 0x8 +#define TSDB_ALTER_USER_PASSWD 0x1 +#define TSDB_ALTER_USER_SUPERUSER 0x2 +#define TSDB_ALTER_USER_ENABLE 0x3 +#define TSDB_ALTER_USER_SYSINFO 0x4 +#define TSDB_ALTER_USER_ADD_PRIVILEGES 0x5 +#define TSDB_ALTER_USER_DEL_PRIVILEGES 0x6 +#define TSDB_ALTER_USER_ADD_WHITE_LIST 0x7 +#define TSDB_ALTER_USER_DROP_WHITE_LIST 0x8 #define TSDB_KILL_MSG_LEN 30 @@ -295,7 +295,36 @@ typedef enum ENodeType { QUERY_NODE_SYNCDB_STMT, QUERY_NODE_GRANT_STMT, QUERY_NODE_REVOKE_STMT, - QUERY_NODE_SHOW_DNODES_STMT, + // placeholder for [152, 180] + QUERY_NODE_SHOW_CREATE_VIEW_STMT = 181, + QUERY_NODE_SHOW_CREATE_DATABASE_STMT, + QUERY_NODE_SHOW_CREATE_TABLE_STMT, + QUERY_NODE_SHOW_CREATE_STABLE_STMT, + QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT, + QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT, + QUERY_NODE_SHOW_SCORES_STMT, + QUERY_NODE_SHOW_TABLE_TAGS_STMT, + QUERY_NODE_KILL_CONNECTION_STMT, + QUERY_NODE_KILL_QUERY_STMT, + QUERY_NODE_KILL_TRANSACTION_STMT, + QUERY_NODE_DELETE_STMT, + QUERY_NODE_INSERT_STMT, + QUERY_NODE_QUERY, + QUERY_NODE_SHOW_DB_ALIVE_STMT, + QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT, + QUERY_NODE_BALANCE_VGROUP_LEADER_STMT, + QUERY_NODE_RESTORE_DNODE_STMT, + QUERY_NODE_RESTORE_QNODE_STMT, + QUERY_NODE_RESTORE_MNODE_STMT, + QUERY_NODE_RESTORE_VNODE_STMT, + QUERY_NODE_PAUSE_STREAM_STMT, + QUERY_NODE_RESUME_STREAM_STMT, + QUERY_NODE_CREATE_VIEW_STMT, + QUERY_NODE_DROP_VIEW_STMT, + + // show statement nodes + // see 'sysTableShowAdapter', 'SYSTABLE_SHOW_TYPE_OFFSET' + QUERY_NODE_SHOW_DNODES_STMT = 400, QUERY_NODE_SHOW_MNODES_STMT, QUERY_NODE_SHOW_MODULES_STMT, QUERY_NODE_SHOW_QNODES_STMT, @@ -324,31 +353,6 @@ typedef enum ENodeType { QUERY_NODE_SHOW_VNODES_STMT, QUERY_NODE_SHOW_USER_PRIVILEGES_STMT, QUERY_NODE_SHOW_VIEWS_STMT, - QUERY_NODE_SHOW_CREATE_VIEW_STMT, - QUERY_NODE_SHOW_CREATE_DATABASE_STMT, - QUERY_NODE_SHOW_CREATE_TABLE_STMT, - QUERY_NODE_SHOW_CREATE_STABLE_STMT, - QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT, - QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT, - QUERY_NODE_SHOW_SCORES_STMT, - QUERY_NODE_SHOW_TABLE_TAGS_STMT, - QUERY_NODE_KILL_CONNECTION_STMT, - QUERY_NODE_KILL_QUERY_STMT, - QUERY_NODE_KILL_TRANSACTION_STMT, - QUERY_NODE_DELETE_STMT, - QUERY_NODE_INSERT_STMT, - QUERY_NODE_QUERY, - QUERY_NODE_SHOW_DB_ALIVE_STMT, - QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT, - QUERY_NODE_BALANCE_VGROUP_LEADER_STMT, - QUERY_NODE_RESTORE_DNODE_STMT, - QUERY_NODE_RESTORE_QNODE_STMT, - QUERY_NODE_RESTORE_MNODE_STMT, - QUERY_NODE_RESTORE_VNODE_STMT, - QUERY_NODE_PAUSE_STREAM_STMT, - QUERY_NODE_RESUME_STREAM_STMT, - QUERY_NODE_CREATE_VIEW_STMT, - QUERY_NODE_DROP_VIEW_STMT, // logic plan node QUERY_NODE_LOGIC_PLAN_SCAN = 1000, @@ -791,7 +795,7 @@ typedef struct { int32_t tSerializeSMDropStbReq(void* buf, int32_t bufLen, SMDropStbReq* pReq); int32_t tDeserializeSMDropStbReq(void* buf, int32_t bufLen, SMDropStbReq* pReq); -void tFreeSMDropStbReq(SMDropStbReq *pReq); +void tFreeSMDropStbReq(SMDropStbReq* pReq); typedef struct { char name[TSDB_TABLE_FNAME_LEN]; @@ -872,18 +876,18 @@ int32_t tSerializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq int32_t tDeserializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq); typedef struct { - char user[TSDB_USER_LEN]; + char user[TSDB_USER_LEN]; int32_t sqlLen; char* sql; } SDropUserReq, SDropAcctReq; int32_t tSerializeSDropUserReq(void* buf, int32_t bufLen, SDropUserReq* pReq); int32_t tDeserializeSDropUserReq(void* buf, int32_t bufLen, SDropUserReq* pReq); -void tFreeSDropUserReq(SDropUserReq *pReq); +void tFreeSDropUserReq(SDropUserReq* pReq); -typedef struct SIpV4Range{ - uint32_t ip; - uint32_t mask; +typedef struct SIpV4Range { + uint32_t ip; + uint32_t mask; } SIpV4Range; typedef struct { @@ -893,21 +897,21 @@ typedef struct { SIpWhiteList* cloneIpWhiteList(SIpWhiteList* pIpWhiteList); typedef struct { - int8_t createType; - int8_t superUser; // denote if it is a super user or not - int8_t sysInfo; - int8_t enable; - char user[TSDB_USER_LEN]; - char pass[TSDB_USET_PASSWORD_LEN]; + int8_t createType; + int8_t superUser; // denote if it is a super user or not + int8_t sysInfo; + int8_t enable; + char user[TSDB_USER_LEN]; + char pass[TSDB_USET_PASSWORD_LEN]; int32_t numIpRanges; SIpV4Range* pIpRanges; - int32_t sqlLen; - char* sql; + int32_t sqlLen; + char* sql; } SCreateUserReq; int32_t tSerializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq); int32_t tDeserializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq); -void tFreeSCreateUserReq(SCreateUserReq *pReq); +void tFreeSCreateUserReq(SCreateUserReq* pReq); typedef struct { int64_t ver; @@ -934,22 +938,22 @@ int32_t tSerializeRetrieveIpWhite(void* buf, int32_t bufLen, SRetrieveIpWhiteReq int32_t tDeserializeRetrieveIpWhite(void* buf, int32_t bufLen, SRetrieveIpWhiteReq* pReq); typedef struct { - int8_t alterType; - int8_t superUser; - int8_t sysInfo; - int8_t enable; - int8_t isView; - char user[TSDB_USER_LEN]; - char pass[TSDB_USET_PASSWORD_LEN]; - char objname[TSDB_DB_FNAME_LEN]; // db or topic - char tabName[TSDB_TABLE_NAME_LEN]; - char* tagCond; - int32_t tagCondLen; + int8_t alterType; + int8_t superUser; + int8_t sysInfo; + int8_t enable; + int8_t isView; + char user[TSDB_USER_LEN]; + char pass[TSDB_USET_PASSWORD_LEN]; + char objname[TSDB_DB_FNAME_LEN]; // db or topic + char tabName[TSDB_TABLE_NAME_LEN]; + char* tagCond; + int32_t tagCondLen; int32_t numIpRanges; SIpV4Range* pIpRanges; int64_t privileges; - int32_t sqlLen; - char* sql; + int32_t sqlLen; + char* sql; } SAlterUserReq; int32_t tSerializeSAlterUserReq(void* buf, int32_t bufLen, SAlterUserReq* pReq); @@ -979,9 +983,9 @@ typedef struct { SHashObj* alterTbs; SHashObj* readViews; SHashObj* writeViews; - SHashObj* alterViews; + SHashObj* alterViews; SHashObj* useDbs; - int64_t whiteListVer; + int64_t whiteListVer; } SGetUserAuthRsp; int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); @@ -996,8 +1000,8 @@ int32_t tSerializeSGetUserWhiteListReq(void* buf, int32_t bufLen, SGetUserWhiteL int32_t tDeserializeSGetUserWhiteListReq(void* buf, int32_t bufLen, SGetUserWhiteListReq* pReq); typedef struct { - char user[TSDB_USER_LEN]; - int32_t numWhiteLists; + char user[TSDB_USER_LEN]; + int32_t numWhiteLists; SIpV4Range* pWhiteLists; } SGetUserWhiteListRsp; @@ -1170,8 +1174,8 @@ int32_t tDeserializeSAlterDbReq(void* buf, int32_t bufLen, SAlterDbReq* pReq); void tFreeSAlterDbReq(SAlterDbReq* pReq); typedef struct { - char db[TSDB_DB_FNAME_LEN]; - int8_t ignoreNotExists; + char db[TSDB_DB_FNAME_LEN]; + int8_t ignoreNotExists; int32_t sqlLen; char* sql; } SDropDbReq; @@ -1379,7 +1383,7 @@ typedef struct { int32_t tSerializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq); int32_t tDeserializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq); -void tFreeSCompactDbReq(SCompactDbReq *pReq); +void tFreeSCompactDbReq(SCompactDbReq* pReq); typedef struct { char name[TSDB_FUNC_NAME_LEN]; @@ -1818,7 +1822,6 @@ int32_t tSerializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp); int32_t tDeserializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp); void tFreeSViewHbRsp(SViewHbRsp* pRsp); - typedef struct { int32_t numOfTables; int32_t numOfVgroup; @@ -2007,7 +2010,7 @@ typedef struct { int32_t tSerializeSRestoreDnodeReq(void* buf, int32_t bufLen, SRestoreDnodeReq* pReq); int32_t tDeserializeSRestoreDnodeReq(void* buf, int32_t bufLen, SRestoreDnodeReq* pReq); -void tFreeSRestoreDnodeReq(SRestoreDnodeReq *pReq); +void tFreeSRestoreDnodeReq(SRestoreDnodeReq* pReq); typedef struct { int32_t dnodeId; @@ -2019,7 +2022,7 @@ typedef struct { int32_t tSerializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq); int32_t tDeserializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq); -void tFreeSMCfgDnodeReq(SMCfgDnodeReq *pReq); +void tFreeSMCfgDnodeReq(SMCfgDnodeReq* pReq); typedef struct { char config[TSDB_DNODE_CONFIG_LEN]; @@ -2038,7 +2041,7 @@ typedef struct { int32_t tSerializeSCreateDropMQSNodeReq(void* buf, int32_t bufLen, SMCreateQnodeReq* pReq); int32_t tDeserializeSCreateDropMQSNodeReq(void* buf, int32_t bufLen, SMCreateQnodeReq* pReq); -void tFreeSMCreateQnodeReq(SMCreateQnodeReq *pReq); +void tFreeSMCreateQnodeReq(SMCreateQnodeReq* pReq); void tFreeSDDropQnodeReq(SDDropQnodeReq* pReq); typedef struct { int8_t replica; @@ -2080,7 +2083,7 @@ typedef struct { int32_t tSerializeSBalanceVgroupReq(void* buf, int32_t bufLen, SBalanceVgroupReq* pReq); int32_t tDeserializeSBalanceVgroupReq(void* buf, int32_t bufLen, SBalanceVgroupReq* pReq); -void tFreeSBalanceVgroupReq(SBalanceVgroupReq *pReq); +void tFreeSBalanceVgroupReq(SBalanceVgroupReq* pReq); typedef struct { int32_t vgId1; @@ -2101,7 +2104,7 @@ typedef struct { int32_t tSerializeSRedistributeVgroupReq(void* buf, int32_t bufLen, SRedistributeVgroupReq* pReq); int32_t tDeserializeSRedistributeVgroupReq(void* buf, int32_t bufLen, SRedistributeVgroupReq* pReq); -void tFreeSRedistributeVgroupReq(SRedistributeVgroupReq *pReq); +void tFreeSRedistributeVgroupReq(SRedistributeVgroupReq* pReq); typedef struct { int32_t useless; @@ -2112,7 +2115,7 @@ typedef struct { int32_t tSerializeSBalanceVgroupLeaderReq(void* buf, int32_t bufLen, SBalanceVgroupLeaderReq* pReq); int32_t tDeserializeSBalanceVgroupLeaderReq(void* buf, int32_t bufLen, SBalanceVgroupLeaderReq* pReq); -void tFreeSBalanceVgroupLeaderReq(SBalanceVgroupLeaderReq *pReq); +void tFreeSBalanceVgroupLeaderReq(SBalanceVgroupLeaderReq* pReq); typedef struct { int32_t vgId; @@ -2504,15 +2507,15 @@ typedef struct { } SMVSubscribeRsp; typedef struct { - char name[TSDB_TOPIC_FNAME_LEN]; - int8_t igNotExists; + char name[TSDB_TOPIC_FNAME_LEN]; + int8_t igNotExists; int32_t sqlLen; char* sql; } SMDropTopicReq; int32_t tSerializeSMDropTopicReq(void* buf, int32_t bufLen, SMDropTopicReq* pReq); int32_t tDeserializeSMDropTopicReq(void* buf, int32_t bufLen, SMDropTopicReq* pReq); -void tFreeSMDropTopicReq(SMDropTopicReq *pReq); +void tFreeSMDropTopicReq(SMDropTopicReq* pReq); typedef struct { char topic[TSDB_TOPIC_FNAME_LEN]; @@ -3083,8 +3086,8 @@ typedef struct { } SMqVDeleteRsp; typedef struct { - char name[TSDB_STREAM_FNAME_LEN]; - int8_t igNotExists; + char name[TSDB_STREAM_FNAME_LEN]; + int8_t igNotExists; int32_t sqlLen; char* sql; } SMDropStreamReq; @@ -3921,7 +3924,7 @@ int32_t tDeserializeSCMDropViewReq(void* buf, int32_t bufLen, SCMDropViewReq* pR void tFreeSCMDropViewReq(SCMDropViewReq* pReq); typedef struct { - char fullname[TSDB_VIEW_FNAME_LEN]; + char fullname[TSDB_VIEW_FNAME_LEN]; } SViewMetaReq; int32_t tSerializeSViewMetaReq(void* buf, int32_t bufLen, const SViewMetaReq* pReq); int32_t tDeserializeSViewMetaReq(void* buf, int32_t bufLen, SViewMetaReq* pReq); @@ -3943,7 +3946,6 @@ int32_t tSerializeSViewMetaRsp(void* buf, int32_t bufLen, const SViewMetaRsp* pR int32_t tDeserializeSViewMetaRsp(void* buf, int32_t bufLen, SViewMetaRsp* pRsp); void tFreeSViewMetaRsp(SViewMetaRsp* pRsp); - #pragma pack(pop) #ifdef __cplusplus diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 95526222dc..61b471912f 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -24,48 +24,70 @@ #if defined(TD_MSG_INFO_) -#undef TD_NEW_MSG_SEG -#undef TD_DEF_MSG_TYPE -#define TD_NEW_MSG_SEG(TYPE) "null", -#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) MSG, MSG "-rsp", + #undef TD_NEW_MSG_SEG + #undef TD_DEF_MSG_TYPE + #undef TD_CLOSE_MSG_TYPE + #define TD_NEW_MSG_SEG(TYPE) "null", + #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) MSG, MSG "-rsp", + #define TD_CLOSE_MSG_TYPE(TYPE) -char *tMsgInfo[] = { + char *tMsgInfo[] = { + +#elif defined(TD_MSG_RANGE_CODE_) + + #undef TD_NEW_MSG_SEG + #undef TD_DEF_MSG_TYPE + #undef TD_CLOSE_MSG_TYPE + #define TD_NEW_MSG_SEG(TYPE) + #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) + #define TD_CLOSE_MSG_TYPE(TYPE) TYPE, + int32_t tMsgRangeDict[] = { #elif defined(TD_MSG_NUMBER_) -#undef TD_NEW_MSG_SEG -#undef TD_DEF_MSG_TYPE -#define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM, -#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE##_NUM, TYPE##_RSP_NUM, + #undef TD_NEW_MSG_SEG + #undef TD_DEF_MSG_TYPE + #undef TD_CLOSE_MSG_TYPE + #define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM, + #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE##_NUM, TYPE##_RSP_NUM, + #define TD_CLOSE_MSG_TYPE(TYPE) -enum { + enum { #elif defined(TD_MSG_DICT_) -#undef TD_NEW_MSG_SEG -#undef TD_DEF_MSG_TYPE -#define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM, -#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) + #undef TD_NEW_MSG_SEG + #undef TD_DEF_MSG_TYPE + #undef TD_CLOSE_MSG_TYPE + #define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM, + #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) + #define TD_CLOSE_MSG_TYPE(type) + + int32_t tMsgDict[] = { -int32_t tMsgDict[] = { #elif defined(TD_MSG_SEG_CODE_) -#undef TD_NEW_MSG_SEG -#undef TD_DEF_MSG_TYPE -#define TD_NEW_MSG_SEG(TYPE) TYPE##_SEG_CODE, -#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) + #undef TD_NEW_MSG_SEG + #undef TD_DEF_MSG_TYPE + #undef TD_CLOSE_MSG_TYPE + #define TD_NEW_MSG_SEG(TYPE) TYPE##_SEG_CODE, + #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) + #define TD_CLOSE_MSG_TYPE(TYPE) -enum { + enum { -#else -#undef TD_NEW_MSG_SEG -#undef TD_DEF_MSG_TYPE -#define TD_NEW_MSG_SEG(TYPE) TYPE = ((TYPE##_SEG_CODE) << 8), -#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE, TYPE##_RSP, +#else -enum { // WARN: new msg should be appended to segment tail + #undef TD_NEW_MSG_SEG + #undef TD_DEF_MSG_TYPE + #undef TD_CLOSE_MSG_TYPE + #define TD_NEW_MSG_SEG(TYPE) TYPE = ((TYPE##_SEG_CODE) << 8), + #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE, TYPE##_RSP, + #define TD_CLOSE_MSG_TYPE(TYPE) TYPE, + + enum { // WARN: new msg should be appended to segment tail #endif TD_NEW_MSG_SEG(TDMT_DND_MSG) // 0<<8 TD_DEF_MSG_TYPE(TDMT_DND_CREATE_MNODE, "dnode-create-mnode", NULL, NULL) @@ -82,10 +104,12 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_DND_NET_TEST, "net-test", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "config-dnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_SYSTABLE_RETRIEVE, "dnode-retrieve", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_DND_MAX_MSG, "dnd-max", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_UNUSED_CODE, "dnd-unused", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_ALTER_MNODE_TYPE, "dnode-alter-mnode-type", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE_TYPE, "dnode-alter-vnode-type", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, "dnode-check-vnode-learner-catchup", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_MAX_MSG, "dnd-max", NULL, NULL) + TD_CLOSE_MSG_TYPE(TDMT_END_DND_MSG) TD_NEW_MSG_SEG(TDMT_MND_MSG) // 1<<8 TD_DEF_MSG_TYPE(TDMT_MND_CONNECT, "connect", NULL, NULL) @@ -186,6 +210,7 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_MND_RESUME_STREAM, "resume-stream", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_TIMER, "stream-checkpoint-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_BEGIN_CHECKPOINT, "stream-begin-checkpoint", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE, "stream-checkpoint-remain", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_NODECHANGE_CHECK, "stream-nodechange-check", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TRIM_DB_TIMER, "trim-db-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GRANT_NOTIFY, "grant-notify", NULL, NULL) @@ -193,6 +218,7 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_MND_DROP_VIEW, "drop-view", SCMDropViewReq, NULL) TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) + TD_CLOSE_MSG_TYPE(TDMT_END_MND_MSG) TD_NEW_MSG_SEG(TDMT_VND_MSG) // 2<<8 TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp) @@ -230,7 +256,7 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_VND_EXEC_RSMA, "vnode-exec-rsma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "delete-data", SVDeleteReq, SVDeleteRsp) TD_DEF_MSG_TYPE(TDMT_VND_BATCH_DEL, "batch-delete", SBatchDeleteReq, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) +TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "alter-replica", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIRM, "alter-confirm", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL) @@ -242,6 +268,7 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_VND_DROP_INDEX, "vnode-drop-index", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL) + TD_CLOSE_MSG_TYPE(TDMT_END_VND_MSG) TD_NEW_MSG_SEG(TDMT_SCH_MSG) // 3<<8 TD_DEF_MSG_TYPE(TDMT_SCH_QUERY, "query", NULL, NULL) @@ -256,6 +283,7 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SCH_TASK_NOTIFY, "task-notify", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SCH_MAX_MSG, "sch-max", NULL, NULL) + TD_CLOSE_MSG_TYPE(TDMT_END_SCH_MSG) TD_NEW_MSG_SEG(TDMT_STREAM_MSG) //4 << 8 @@ -273,9 +301,11 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_STOP, "stream-task-stop", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_HTASK_DROP, "stream-htask-drop", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL) + TD_CLOSE_MSG_TYPE(TDMT_END_STREAM_MSG) TD_NEW_MSG_SEG(TDMT_MON_MSG) //5 << 8 TD_DEF_MSG_TYPE(TDMT_MON_MAX_MSG, "monitor-max", NULL, NULL) + TD_CLOSE_MSG_TYPE(TDMT_END_MON_MSG) TD_NEW_MSG_SEG(TDMT_SYNC_MSG) //6 << 8 TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT, "sync-timer", NULL, NULL) @@ -307,6 +337,8 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_SYNC_PREP_SNAPSHOT_REPLY, "sync-prep-snapshot-reply", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_FORCE_FOLLOWER, "sync-force-become-follower", NULL, NULL) + TD_CLOSE_MSG_TYPE(TDMT_END_SYNC_MSG) + TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG) //7 << 8 TD_DEF_MSG_TYPE(TDMT_VND_STREAM_SCAN_HISTORY, "vnode-stream-scan-history", NULL, NULL) @@ -316,6 +348,7 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_RESET, "vnode-stream-reset", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL) + TD_CLOSE_MSG_TYPE(TDMT_END_VND_STREAM_MSG) TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG) //8 << 8 TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SUBSCRIBE, "vnode-tmq-subscribe", SMqRebVgReq, SMqRebVgRsp) @@ -329,9 +362,15 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_WALINFO, "vnode-tmq-vg-walinfo", SMqPollReq, SMqDataBlkRsp) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_COMMITTEDINFO, "vnode-tmq-committedinfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_MAX_MSG, "vnd-tmq-max", NULL, NULL) + TD_CLOSE_MSG_TYPE(TDMT_END_TMQ_MSG) + + TD_NEW_MSG_SEG(TDMT_MAX_MSG) // msg end mark + + + -#if defined(TD_MSG_NUMBER_) - TDMT_MAX -#endif + #if defined(TD_MSG_NUMBER_) + TDMT_MAX + #endif }; diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 279799b172..741e3663db 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -275,9 +275,11 @@ typedef struct { #define IS_VALID_TINYINT(_t) ((_t) >= INT8_MIN && (_t) <= INT8_MAX) #define IS_VALID_SMALLINT(_t) ((_t) >= INT16_MIN && (_t) <= INT16_MAX) #define IS_VALID_INT(_t) ((_t) >= INT32_MIN && (_t) <= INT32_MAX) +#define IS_VALID_INT64(_t) ((_t) >= INT64_MIN && (_t) <= INT64_MAX) #define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) <= UINT8_MAX) #define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) <= UINT16_MAX) #define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) <= UINT32_MAX) +#define IS_VALID_UINT64(_t) ((_t) >= 0 && (_t) <= UINT64_MAX) #define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX) #define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX) diff --git a/include/common/tvariant.h b/include/common/tvariant.h index 130945cce5..66b7302f4e 100644 --- a/include/common/tvariant.h +++ b/include/common/tvariant.h @@ -37,6 +37,10 @@ typedef struct SVariant { }; } SVariant; +int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value); +int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value); +int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double *value); + int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value); int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value); diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 045f2bad70..32b60b55ae 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -152,6 +152,18 @@ typedef struct { // clang-format off /*-------------------------------------------------new api format---------------------------------------------------*/ +typedef enum { + TSD_READER_NOTIFY_DURATION_START +} ETsdReaderNotifyType; + +typedef union { + struct { + int32_t filesetId; + } duration; +} STsdReaderNotifyInfo; + +typedef void (*TsdReaderNotifyCbFn)(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param); + typedef struct TsdReader { int32_t (*tsdReaderOpen)(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, SSDataBlock* pResBlock, void** ppReader, const char* idstr, bool countOnly, @@ -170,6 +182,9 @@ typedef struct TsdReader { int32_t (*tsdReaderGetDataBlockDistInfo)(); int64_t (*tsdReaderGetNumOfInMemRows)(); void (*tsdReaderNotifyClosing)(); + + void (*tsdSetFilesetDelimited)(void* pReader); + void (*tsdSetSetNotifyCb)(void* pReader, TsdReaderNotifyCbFn notifyFn, void* param); } TsdReader; typedef struct SStoreCacheReader { @@ -351,6 +366,8 @@ typedef struct SStateStore { TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol); bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts); bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid); + bool (*isIncrementalTimeStamp)(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts); + void (*updateInfoDestroy)(SUpdateInfo* pInfo); void (*windowSBfDelete)(SUpdateInfo *pInfo, uint64_t count); void (*windowSBfAdd)(SUpdateInfo *pInfo, uint64_t count); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index fc62800dda..06a6fd8752 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -118,6 +118,7 @@ typedef struct SScanLogicNode { bool igLastNull; bool groupOrderScan; bool onlyMetaCtbIdx; // for tag scan with no tbname + bool filesetDelimited; // returned blocks delimited by fileset } SScanLogicNode; typedef struct SJoinLogicNode { @@ -433,6 +434,7 @@ typedef struct STableScanPhysiNode { int8_t igExpired; bool assignBlockUid; int8_t igCheckUpdate; + bool filesetDelimited; } STableScanPhysiNode; typedef STableScanPhysiNode STableSeqScanPhysiNode; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 19dc8c9e4d..5c5172b9cd 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -449,6 +449,7 @@ typedef struct SVnodeModifyOpStmt { SHashObj* pSubTableHashObj; // SHashObj SHashObj* pTableNameHashObj; // set of table names for refreshing meta, sync mode SHashObj* pDbFNameHashObj; // set of db names for refreshing meta, sync mode + SHashObj* pTableCxtHashObj; // temp SHashObj for single request SArray* pVgDataBlocks; // SArray SVCreateTbReq* pCreateTblReq; TdFilePtr fp; diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index d0a2b311ee..ba90c0dc7a 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -35,6 +35,7 @@ int32_t streamStateBegin(SStreamState* pState); int32_t streamStateCommit(SStreamState* pState); void streamStateDestroy(SStreamState* pState, bool remove); int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark); +int32_t streamStateDelTaskDb(SStreamState* pState); int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen); @@ -133,4 +134,4 @@ char* streamStateIntervalDump(SStreamState* pState); } #endif -#endif /* ifndef _STREAM_STATE_H_ */ +#endif /* ifndef _STREAM_STATE_H_ */ \ No newline at end of file diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index e97f3e0192..8f3e100db6 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -58,7 +58,9 @@ typedef struct SStreamTask SStreamTask; typedef struct SStreamQueue SStreamQueue; typedef struct SStreamTaskSM SStreamTaskSM; -#define SSTREAM_TASK_VER 2 +#define SSTREAM_TASK_VER 2 +#define SSTREAM_TASK_INCOMPATIBLE_VER 1 +#define SSTREAM_TASK_NEED_CONVERT_VER 2 enum { STREAM_STATUS__NORMAL = 0, @@ -110,6 +112,7 @@ typedef enum { TASK_LEVEL__SOURCE = 1, TASK_LEVEL__AGG, TASK_LEVEL__SINK, + TASK_LEVEL_SMA, } ETASK_LEVEL; enum { @@ -213,9 +216,6 @@ int32_t streamQueuePush(SStreamQueue1* pQueue, SStreamQueueItem* pItem); SStreamQueueRes streamQueueGetRes(SStreamQueue1* pQueue); #endif -int32_t streamInit(); -void streamCleanUp(); - SStreamDataSubmit* streamDataSubmitNew(SPackedData* pData, int32_t type); void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit); @@ -304,11 +304,16 @@ typedef struct SStreamTaskId { typedef struct SCheckpointInfo { int64_t startTs; int64_t checkpointId; - int64_t checkpointVer; // latest checkpointId version - int64_t processedVer; // already processed ver, that has generated results version. + + int64_t checkpointVer; // latest checkpointId version + int64_t processedVer; int64_t nextProcessVer; // current offset in WAL, not serialize it int64_t failedId; // record the latest failed checkpoint id + int64_t checkpointingId; + int32_t downstreamAlignNum; + int32_t checkpointNotReadyTasks; bool dispatchCheckpointTrigger; + int64_t msgVer; } SCheckpointInfo; typedef struct SStreamStatus { @@ -394,7 +399,8 @@ typedef struct SHistoryTaskInfo { int32_t tickCount; int32_t retryTimes; int32_t waitInterval; - int64_t haltVer; // offset in wal when halt the stream task + int64_t haltVer; // offset in wal when halt the stream task + bool operatorOpen; // false by default } SHistoryTaskInfo; typedef struct STaskOutputInfo { @@ -447,12 +453,11 @@ struct SStreamTask { int64_t checkReqId; SArray* checkReqIds; // shuffle int32_t refCnt; - int64_t checkpointingId; - int32_t checkpointAlignCnt; - int32_t checkpointNotReadyTasks; int32_t transferStateAlignCnt; struct SStreamMeta* pMeta; SSHashObj* pNameMap; + void* pBackend; + int64_t backendRefId; char reserve[256]; }; @@ -490,20 +495,25 @@ typedef struct SStreamMeta { int32_t walScanCounter; void* streamBackend; int64_t streamBackendRid; - SHashObj* pTaskBackendUnique; + SHashObj* pTaskDbUnique; TdThreadMutex backendMutex; SMetaHbInfo* pHbInfo; STaskUpdateInfo updateInfo; + SHashObj* pUpdateTaskSet; int32_t numOfStreamTasks; // this value should be increased when a new task is added into the meta int32_t numOfPausedTasks; - int32_t chkptNotReadyTasks; int64_t rid; int64_t chkpId; + int32_t chkpCap; SArray* chkpSaved; SArray* chkpInUse; - int32_t chkpCap; SRWLatch chkpDirLock; + + void* qHandle; + int32_t pauseTaskNum; + + void* bkdChkptMgt; } SStreamMeta; int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); @@ -659,7 +669,7 @@ int32_t tDecodeStreamCheckpointReadyMsg(SDecoder* pDecoder, SStreamCheckpointRea typedef struct STaskStatusEntry { STaskId id; int32_t status; - int32_t statusLastDuration; // to record the last duration of current status + int32_t statusLastDuration; // to record the last duration of current status int64_t stage; int32_t nodeId; int64_t verStart; // start version in WAL, only valid for source task @@ -668,10 +678,12 @@ typedef struct STaskStatusEntry { int32_t relatedHTask; // has related fill-history task int64_t activeCheckpointId; // current active checkpoint id bool checkpointFailed; // denote if the checkpoint is failed or not + bool inputQChanging; // inputQ is changing or not + int64_t inputQUnchangeCounter; double inputQUsed; // in MiB double inputRate; - double sinkQuota; // existed quota size for sink task - double sinkDataSize; // sink to dst data size + double sinkQuota; // existed quota size for sink task + double sinkDataSize; // sink to dst data size } STaskStatusEntry; typedef struct SStreamHbMsg { @@ -832,11 +844,14 @@ int32_t streamMetaReopen(SStreamMeta* pMeta); void streamMetaInitBackend(SStreamMeta* pMeta); int32_t streamMetaCommit(SStreamMeta* pMeta); int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta); +int32_t streamMetaReloadAllTasks(SStreamMeta* pMeta); int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta); void streamMetaNotifyClose(SStreamMeta* pMeta); +int32_t streamTaskSetDb(SStreamMeta* pMeta, void* pTask, char* key); void streamMetaStartHb(SStreamMeta* pMeta); bool streamMetaTaskInTimer(SStreamMeta* pMeta); -int32_t streamMetaUpdateTaskDownstreamStatus(SStreamTask* pTask, int64_t startTs, int64_t endTs, bool succ); +int32_t streamMetaUpdateTaskDownstreamStatus(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, int64_t startTs, + int64_t endTs, bool ready); void streamMetaRLock(SStreamMeta* pMeta); void streamMetaRUnLock(SStreamMeta* pMeta); void streamMetaWLock(SStreamMeta* pMeta); @@ -855,8 +870,10 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa int32_t buildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SRpcMsg* pMsg, int8_t isSucceed); +SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask); +void* streamDestroyStateMachine(SStreamTaskSM* pSM); #ifdef __cplusplus } #endif -#endif /* ifndef _STREAM_H_ */ +#endif /* ifndef _STREAM_H_ */ \ No newline at end of file diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h index 41ada56904..af93c6ac01 100644 --- a/include/libs/stream/tstreamUpdate.h +++ b/include/libs/stream/tstreamUpdate.h @@ -55,6 +55,7 @@ int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *p int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo); void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count); void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count); +bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts); #ifdef __cplusplus } diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index ece1e40585..a428a9ae6a 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -289,6 +289,9 @@ const char* syncStr(ESyncState state); int32_t syncNodeGetConfig(int64_t rid, SSyncCfg *cfg); +// util +int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size); + #ifdef __cplusplus } #endif diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 0256a496df..6ab06d06a3 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -558,7 +558,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_GRANT_GEN_IVLD_KEY TAOS_DEF_ERROR_CODE(0, 0x0812) #define TSDB_CODE_GRANT_GEN_APP_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0813) #define TSDB_CODE_GRANT_GEN_ENC_IVLD_KLEN TAOS_DEF_ERROR_CODE(0, 0x0814) -#define TSDB_CODE_GRANT_PAR_IVLD_DIST TAOS_DEF_ERROR_CODE(0, 0x0815) // sync // #define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x diff --git a/include/util/tcompression.h b/include/util/tcompression.h index ab0c22fc9b..75ddbb12e7 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -139,6 +139,8 @@ int32_t getWordLength(char type); int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, char *const output, const char type); int32_t tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output); int32_t tsDecompressFloatImplAvx2(const char *const input, const int32_t nelements, char *const output); +int32_t tsDecompressTimestampAvx512(const char* const input, const int32_t nelements, char *const output, bool bigEndian); +int32_t tsDecompressTimestampAvx2(const char* const input, const int32_t nelements, char *const output, bool bigEndian); /************************************************************************* * STREAM COMPRESSION diff --git a/include/util/tdef.h b/include/util/tdef.h index 69d0c1126d..51b0b63da2 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -288,6 +288,7 @@ typedef enum ELogicConditionType { #define TSDB_CONN_ACTIVE_KEY_LEN 255 #define TSDB_DEFAULT_PKT_SIZE 65480 // same as RPC_MAX_UDP_SIZE +#define TSDB_SNAP_DATA_PAYLOAD_SIZE (1 * 1024 * 1024) #define TSDB_PAYLOAD_SIZE TSDB_DEFAULT_PKT_SIZE #define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value @@ -305,7 +306,7 @@ typedef enum ELogicConditionType { #define TSDB_SYNC_APPLYQ_SIZE_LIMIT 512 #define TSDB_SYNC_NEGOTIATION_WIN 512 -#define TSDB_SYNC_SNAP_BUFFER_SIZE 2048 +#define TSDB_SYNC_SNAP_BUFFER_SIZE 1024 #define TSDB_TBNAME_COLUMN_INDEX (-1) #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta diff --git a/packaging/tools/set_core.sh b/packaging/tools/set_core.sh index db95aeb343..084c4465f9 100755 --- a/packaging/tools/set_core.sh +++ b/packaging/tools/set_core.sh @@ -38,3 +38,4 @@ source /etc/profile ${csudo}mkdir -p ${corePath} ||: ${csudo}sysctl -w kernel.core_pattern=${corePath}/core-%e-%p ||: ${csudo}echo "${corePath}/core-%e-%p" | ${csudo}tee /proc/sys/kernel/core_pattern ||: +${csudo}echo "kernel.core_pattern = ${corePath}/core_%e-%p" >> /etc/sysctl.conf ||: diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 30d4792279..9ae28dd55e 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -65,14 +65,32 @@ extern "C" { #define IS_INVALID_COL_LEN(len) ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN) #define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN) -//#define TS "_ts" -//#define TS_LEN 3 #define VALUE "_value" -#define VALUE_LEN 6 +#define VALUE_LEN (sizeof(VALUE)-1) #define OTD_JSON_FIELDS_NUM 4 #define MAX_RETRY_TIMES 10 -typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; + +#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \ +&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0) + +#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \ +&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0) + +#define IS_SAME_KEY (maxKV->type == kv->type && maxKV->keyLen == kv->keyLen && memcmp(maxKV->key, kv->key, kv->keyLen) == 0) + +#define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \ + (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE)) + +#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) + +#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen) \ + for (int i = 1; i < keyLen; ++i) { \ + if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) { \ + MOVE_FORWARD_ONE(key + i, keyLen - i); \ + keyLen--; \ + } \ + } typedef enum { SCHEMA_ACTION_NULL, @@ -83,18 +101,6 @@ typedef enum { SCHEMA_ACTION_CHANGE_TAG_SIZE, } ESchemaAction; -typedef struct { - const void *key; - int32_t keyLen; - void *value; - bool used; -}Node; - -typedef struct NodeList{ - Node data; - struct NodeList* next; -}NodeList; - typedef struct { char *measure; char *tags; @@ -117,7 +123,6 @@ typedef struct { int32_t sTableNameLen; char childTableName[TSDB_TABLE_NAME_LEN]; uint64_t uid; -// void *key; // for openTsdb SArray *tags; @@ -161,7 +166,8 @@ typedef struct { typedef struct { int64_t id; - SMLProtocolType protocol; + TSDB_SML_PROTOCOL_TYPE protocol; + int8_t precision; bool reRun; bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) @@ -201,29 +207,8 @@ typedef struct { bool needModifySchema; } SSmlHandle; -#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \ -&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0) - -#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \ -&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0) - -#define IS_SAME_KEY (maxKV->keyLen == kv.keyLen && memcmp(maxKV->key, kv.key, kv.keyLen) == 0) - -#define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \ - (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE)) - -#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) - -#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen) \ - for (int i = 1; i < keyLen; ++i) { \ - if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) { \ - MOVE_FORWARD_ONE(key + i, keyLen - i); \ - keyLen--; \ - } \ - } - -extern int64_t smlFactorNS[3]; -extern int64_t smlFactorS[3]; +extern int64_t smlFactorNS[]; +extern int64_t smlFactorS[]; typedef int32_t (*_equal_fn_sml)(const void *, const void *); @@ -231,16 +216,10 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos); void smlDestroyInfo(SSmlHandle *info); int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset); int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset); -//SArray *smlJsonParseTags(char *start, char *end); bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); -//void* nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn); -//int nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn); -//int nodeListSize(NodeList* list); -bool smlDoubleToInt64OverFlow(double num); int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision); -int8_t smlGetTsTypeByLen(int32_t len); SSmlTableInfo* smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen); SSmlSTableMeta* smlBuildSTableMeta(bool isDataFormat); int32_t smlSetCTableName(SSmlTableInfo *oneTable); @@ -253,12 +232,45 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg); uint8_t smlGetTimestampLen(int64_t num); void smlDestroyTableInfo(void *para); -void freeSSmlKv(void* data); +void freeSSmlKv(void* data); int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseJSON(SSmlHandle *info, char *payload); -void smlStrReplace(char* src, int32_t len); +SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement); +bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv); +bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv); +int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements); +int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements); +int32_t smlJoinMeasureTag(SSmlLineInfo *elements); +void smlBuildTsKv(SSmlKv *kv, int64_t ts); +int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv); +int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs); + +static inline bool smlDoubleToInt64OverFlow(double num) { + if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; + return false; +} + +static inline void smlStrReplace(char* src, int32_t len){ + if (!tsSmlDot2Underline) return; + for(int i = 0; i < len; i++){ + if(src[i] == '.'){ + src[i] = '_'; + } + } +} + +static inline int8_t smlGetTsTypeByLen(int32_t len) { + if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { + return TSDB_TIME_PRECISION_SECONDS; + } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { + return TSDB_TIME_PRECISION_MILLI; + } else { + return -1; + } +} + #ifdef __cplusplus } #endif diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 212c42125e..85624c31c5 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -732,7 +732,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { return -1; } newstr[0] = '"'; - strncpy(newstr+1, str, len); + memcpy(newstr+1, str, len); newstr[len + 1] = '"'; newstr[len + 2] = '\0'; str = newstr; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 6ab0bfc563..67b23792ad 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -20,75 +20,93 @@ #include "clientSml.h" -int64_t smlToMilli[3] = {3600000LL, 60000LL, 1000LL}; -int64_t smlFactorNS[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; -int64_t smlFactorS[3] = {1000LL, 1000000LL, 1000000000LL}; +#define RETURN_FALSE \ + smlBuildInvalidDataMsg(msg, "invalid data", pVal); \ + return false; -//void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) { -// NodeList *tmp = list; -// while (tmp) { -// if (fn == NULL) { -// if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { -// return tmp->data.value; -// } -// } else { -// if (tmp->data.used && fn(tmp->data.key, key) == 0) { -// return tmp->data.value; -// } -// } -// -// tmp = tmp->next; -// } -// return NULL; -//} -// -//int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) { -// NodeList *tmp = *list; -// while (tmp) { -// if (!tmp->data.used) break; -// if (fn == NULL) { -// if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { -// return -1; -// } -// } else { -// if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) { -// return -1; -// } -// } -// -// tmp = tmp->next; -// } -// if (tmp) { -// tmp->data.key = key; -// tmp->data.keyLen = len; -// tmp->data.value = value; -// tmp->data.used = true; -// } else { -// NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList)); -// if (newNode == NULL) { -// return -1; -// } -// newNode->data.key = key; -// newNode->data.keyLen = len; -// newNode->data.value = value; -// newNode->data.used = true; -// newNode->next = *list; -// *list = newNode; -// } -// return 0; -//} -// -//int nodeListSize(NodeList *list) { -// int cnt = 0; -// while (list) { -// if (list->data.used) -// cnt++; -// else -// break; -// list = list->next; -// } -// return cnt; -//} +#define SET_DOUBLE \ + kvVal->type = TSDB_DATA_TYPE_DOUBLE; \ + kvVal->d = result; + +#define SET_FLOAT \ + if (!IS_VALID_FLOAT(result)) { \ + smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_FLOAT; \ + kvVal->f = (float)result; + +#define SET_BIGINT \ + errno = 0; \ + int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \ + if (errno == ERANGE) { \ + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_BIGINT; \ + kvVal->i = tmp; + +#define SET_INT \ + if (!IS_VALID_INT(result)) { \ + smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_INT; \ + kvVal->i = result; + +#define SET_SMALL_INT \ + if (!IS_VALID_SMALLINT(result)) { \ + smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_SMALLINT; \ + kvVal->i = result; + +#define SET_UBIGINT \ + errno = 0; \ + uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \ + if (errno == ERANGE || result < 0) { \ + smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ + kvVal->u = tmp; + +#define SET_UINT \ + if (!IS_VALID_UINT(result)) { \ + smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UINT; \ + kvVal->u = result; + +#define SET_USMALL_INT \ + if (!IS_VALID_USMALLINT(result)) { \ + smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_USMALLINT; \ + kvVal->u = result; + +#define SET_TINYINT \ + if (!IS_VALID_TINYINT(result)) { \ + smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_TINYINT; \ + kvVal->i = result; + +#define SET_UTINYINT \ + if (!IS_VALID_UTINYINT(result)) { \ + smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UTINYINT; \ + kvVal->u = result; + +int64_t smlToMilli[] = {3600000LL, 60000LL, 1000LL}; +int64_t smlFactorNS[] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; +int64_t smlFactorS[] = {1000LL, 1000000LL, 1000000000LL}; static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo* conn, const char* pTabName, AUTH_TYPE type){ SUserAuthInfo pAuth = {0}; @@ -109,19 +127,6 @@ static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo* conn, const cha return (code == TSDB_CODE_SUCCESS) ? (authRes.pass[AUTH_RES_BASIC] ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code; } -inline bool smlDoubleToInt64OverFlow(double num) { - if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; - return false; -} - -void smlStrReplace(char* src, int32_t len){ - if (!tsSmlDot2Underline) return; - for(int i = 0; i < len; i++){ - if(src[i] == '.'){ - src[i] = '_'; - } - } -} int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { if (pBuf->buf) { @@ -155,16 +160,6 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u return convertTimePrecision(tsInt64, fromPrecision, toPrecision); } -int8_t smlGetTsTypeByLen(int32_t len) { - if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { - return TSDB_TIME_PRECISION_SECONDS; - } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { - return TSDB_TIME_PRECISION_MILLI; - } else { - return -1; - } -} - SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen) { SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); if (!tag) { @@ -180,11 +175,6 @@ SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measu goto cleanup; } - // tag->tags = taosArrayInit(16, sizeof(SSmlKv)); - // if (tag->tags == NULL) { - // uError("SML:smlBuildTableInfo failed to allocate memory"); - // goto cleanup; - // } return tag; cleanup: @@ -192,6 +182,242 @@ cleanup: return NULL; } +void smlBuildTsKv(SSmlKv *kv, int64_t ts){ + kv->key = tsSmlTsDefaultName; + kv->keyLen = strlen(tsSmlTsDefaultName); + kv->type = TSDB_DATA_TYPE_TIMESTAMP; + kv->i = ts; + kv->length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; +} + +SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement){ + SSmlSTableMeta* sMeta = NULL; + char *measure = currElement->measure; + int measureLen = currElement->measureLen; + if (currElement->measureEscaped) { + measure = (char *)taosMemoryMalloc(measureLen); + memcpy(measure, currElement->measure, measureLen); + PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); + smlStrReplace(measure, measureLen); + } + STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen); + if (currElement->measureEscaped) { + taosMemoryFree(measure); + } + if (pTableMeta == NULL) { + info->dataFormat = false; + info->reRun = true; + terrno = TSDB_CODE_SUCCESS; + return sMeta; + } + sMeta = smlBuildSTableMeta(info->dataFormat); + if(sMeta == NULL){ + taosMemoryFreeClear(pTableMeta); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return sMeta; + } + sMeta->tableMeta = pTableMeta; + taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES); + for (int i = 1; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) { + SSchema *col = pTableMeta->schema + i; + SSmlKv kv = {.key = col->name, .keyLen = strlen(col->name), .type = col->type}; + if (col->type == TSDB_DATA_TYPE_NCHAR) { + kv.length = (col->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + } else if (col->type == TSDB_DATA_TYPE_BINARY || col->type == TSDB_DATA_TYPE_GEOMETRY || col->type == TSDB_DATA_TYPE_VARBINARY) { + kv.length = col->bytes - VARSTR_HEADER_SIZE; + } else{ + kv.length = col->bytes; + } + + if(i < pTableMeta->tableInfo.numOfColumns){ + taosArrayPush(sMeta->cols, &kv); + }else{ + taosArrayPush(sMeta->tags, &kv); + } + } + return sMeta; +} + +bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv){ + // cnt begin 0, add ts so + 2 + if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) { + info->dataFormat = false; + info->reRun = true; + return false; + } + // bind data + int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, cnt + 1); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uDebug("smlBuildCol error, retry"); + info->dataFormat = false; + info->reRun = true; + return false; + } + if (cnt >= taosArrayGetSize(info->maxColKVs)) { + info->dataFormat = false; + info->reRun = true; + return false; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt); + + if (unlikely(!IS_SAME_KEY)) { + info->dataFormat = false; + info->reRun = true; + return false; + } + + if (unlikely(IS_VAR_DATA_TYPE(kv->type) && kv->length > maxKV->length)) { + maxKV->length = kv->length; + info->needModifySchema = true; + } + return true; +} + +bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv){ + if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) { + goto END; + } + + if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) { + goto END; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); + + if (unlikely(!IS_SAME_KEY)) { + goto END; + } + + if (unlikely(kv->length > maxKV->length)) { + maxKV->length = kv->length; + info->needModifySchema = true; + } + return true; + +END: + info->dataFormat = false; + info->reRun = true; + return false; +} + +int32_t smlJoinMeasureTag(SSmlLineInfo *elements){ + elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen); + if(elements->measureTag == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(elements->measureTag, elements->measure, elements->measureLen); + memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen); + elements->measureTagsLen = elements->measureLen + elements->tagsLen; + return TSDB_CODE_SUCCESS; +} + +int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) { + bool isSameMeasure = IS_SAME_SUPER_TABLE; + if(isSameMeasure) { + return 0; + } + SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); + + SSmlSTableMeta *sMeta = NULL; + if (unlikely(tmp == NULL)) { + sMeta = smlBuildSuperTableInfo(info, elements); + if(sMeta == NULL) return -1; + }else{ + sMeta = *tmp; + } + ASSERT(sMeta != NULL); + info->currSTableMeta = sMeta->tableMeta; + info->maxTagKVs = sMeta->tags; + info->maxColKVs = sMeta->cols; + return 0; +} + +int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements){ + SSmlTableInfo **oneTable = + (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen); + SSmlTableInfo *tinfo = NULL; + if (unlikely(oneTable == NULL)) { + tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); + if (unlikely(!tinfo)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES); + + tinfo->tags = taosArrayDup(info->preLineTagKV, NULL); + for (size_t i = 0; i < taosArrayGetSize(info->preLineTagKV); i++) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(info->preLineTagKV, i); + if (kv->keyEscaped) kv->key = NULL; + if (kv->valueEscaped) kv->value = NULL; + } + + smlSetCTableName(tinfo); + getTableUid(info, elements, tinfo); + if (info->dataFormat) { + info->currSTableMeta->uid = tinfo->uid; + tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); + if (tinfo->tableDataCtx == NULL) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); + smlDestroyTableInfo(&tinfo); + return TSDB_CODE_SML_INVALID_DATA; + } + } + }else{ + tinfo = *oneTable; + } + ASSERT(tinfo != NULL); + if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; + return TSDB_CODE_SUCCESS; +} + +int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv){ + if (info->dataFormat) { + uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format true, ts:%" PRId64, info->id, kvTs->i); + int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0); + if (ret == TSDB_CODE_SUCCESS) { + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1); + } + if (ret == TSDB_CODE_SUCCESS) { + ret = smlBuildRow(info->currTableDataCtx); + } + clearColValArraySml(info->currTableDataCtx->pValues); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + return ret; + } + } else { + uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format false, ts:%" PRId64, info->id, kvTs->i); + if (elements->colArray == NULL) { + elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + } + taosArrayPush(elements->colArray, kvTs); + taosArrayPush(elements->colArray, kv); + } + info->preLine = *elements; + + return TSDB_CODE_SUCCESS; +} + +int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs){ + if (info->dataFormat) { + uDebug("SML:0x%" PRIx64 " smlParseEndLine format true, ts:%" PRId64, info->id, kvTs->i); + int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0); + if (ret == TSDB_CODE_SUCCESS) { + ret = smlBuildRow(info->currTableDataCtx); + } + + clearColValArraySml(info->currTableDataCtx->pValues); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + return ret; + } + } else { + uDebug("SML:0x%" PRIx64 " smlParseEndLine format false, ts:%" PRId64, info->id, kvTs->i); + taosArraySet(elements->colArray, 0, kvTs); + } + info->preLine = *elements; + + return TSDB_CODE_SUCCESS; +} + static int32_t smlParseTableName(SArray *tags, char *childTableName) { bool autoChildName = false; size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter); @@ -328,98 +554,6 @@ cleanup: return NULL; } -// uint16_t smlCalTypeSum(char* endptr, int32_t left){ -// uint16_t sum = 0; -// for(int i = 0; i < left; i++){ -// sum += endptr[i]; -// } -// return sum; -// } - -#define RETURN_FALSE \ - smlBuildInvalidDataMsg(msg, "invalid data", pVal); \ - return false; - -#define SET_DOUBLE \ - kvVal->type = TSDB_DATA_TYPE_DOUBLE; \ - kvVal->d = result; - -#define SET_FLOAT \ - if (!IS_VALID_FLOAT(result)) { \ - smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_FLOAT; \ - kvVal->f = (float)result; - -#define SET_BIGINT \ - errno = 0; \ - int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \ - if (errno == ERANGE) { \ - smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_BIGINT; \ - kvVal->i = tmp; - -#define SET_INT \ - if (!IS_VALID_INT(result)) { \ - smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_INT; \ - kvVal->i = result; - -#define SET_SMALL_INT \ - if (!IS_VALID_SMALLINT(result)) { \ - smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_SMALLINT; \ - kvVal->i = result; - -#define SET_UBIGINT \ - errno = 0; \ - uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \ - if (errno == ERANGE || result < 0) { \ - smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ - kvVal->u = tmp; - -#define SET_UINT \ - if (!IS_VALID_UINT(result)) { \ - smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_UINT; \ - kvVal->u = result; - -#define SET_USMALL_INT \ - if (!IS_VALID_USMALLINT(result)) { \ - smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_USMALLINT; \ - kvVal->u = result; - -#define SET_TINYINT \ - if (!IS_VALID_TINYINT(result)) { \ - smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_TINYINT; \ - kvVal->i = result; - -#define SET_UTINYINT \ - if (!IS_VALID_UTINYINT(result)) { \ - smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_UTINYINT; \ - kvVal->u = result; - bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->length; @@ -765,8 +899,6 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO return TSDB_CODE_SUCCESS; } -// static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData, -// int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta, ESchemaAction action) { SRequestObj *pRequest = NULL; @@ -1121,35 +1253,6 @@ end: return code; } -/* -static int32_t smlCheckDupUnit(SHashObj *dumplicateKey, SArray *tags, SSmlMsgBuf *msg){ - for(int i = 0; i < taosArrayGetSize(tags); i++) { - SSmlKv *tag = taosArrayGet(tags, i); - if (smlCheckDuplicateKey(tag->key, tag->keyLen, dumplicateKey)) { - smlBuildInvalidDataMsg(msg, "dumplicate key", tag->key); - return TSDB_CODE_TSC_DUP_NAMES; - } - } - return TSDB_CODE_SUCCESS; -} - -static int32_t smlJudgeDupColName(SArray *cols, SArray *tags, SSmlMsgBuf *msg) { - SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - int ret = smlCheckDupUnit(dumplicateKey, cols, msg); - if(ret != TSDB_CODE_SUCCESS){ - goto end; - } - ret = smlCheckDupUnit(dumplicateKey, tags, msg); - if(ret != TSDB_CODE_SUCCESS){ - goto end; - } - - end: - taosHashCleanup(dumplicateKey); - return ret; -} -*/ - static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) { for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); @@ -1206,11 +1309,6 @@ void smlDestroyTableInfo(void *para) { taosHashCleanup(kvHash); } - // if (info->parseJsonByLib) { - // SSmlLineInfo *key = (SSmlLineInfo *)(tag->key); - // if (key != NULL) taosMemoryFree(key->tags); - // } - // taosMemoryFree(tag->key); taosArrayDestroy(tag->cols); taosArrayDestroyEx(tag->tags, freeSSmlKv); taosMemoryFree(tag); @@ -1228,21 +1326,6 @@ void smlDestroyInfo(SSmlHandle *info) { if (!info) return; qDestroyQuery(info->pQuery); - // destroy info->childTables -// SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); -// while (oneTable) { -// smlDestroyTableInfo(oneTable); -// oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); -// } - - // destroy info->superTables -// SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); -// while (oneSTable) { -// smlDestroySTableMeta(*oneSTable); -// oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); -// } - - // destroy info->pVgHash taosHashCleanup(info->pVgHash); taosHashCleanup(info->childTables); taosHashCleanup(info->superTables); @@ -1268,7 +1351,9 @@ void smlDestroyInfo(SSmlHandle *info) { if (info->parseJsonByLib) { taosMemoryFree(info->lines[i].tags); } - if (info->lines[i].measureTagsLen != 0) taosMemoryFree(info->lines[i].measureTag); + if (info->lines[i].measureTagsLen != 0 && info->protocol != TSDB_SML_LINE_PROTOCOL) { + taosMemoryFree(info->lines[i].measureTag); + } } taosMemoryFree(info->lines); } @@ -1399,11 +1484,6 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { return ret; } } else { - // ret = smlJudgeDupColName(elements->colArray, tinfo->tags, &info->msgBuf); - // if (ret != TSDB_CODE_SUCCESS) { - // uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); - // return ret; - // } uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); @@ -1526,30 +1606,17 @@ static void smlPrintStatisticInfo(SSmlHandle *info) { uDebug( "SML:0x%" PRIx64 " smlInsertLines result, code:%d, msg:%s, lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \ - parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64 - "", + parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64, info->id, info->cost.code, tstrerror(info->cost.code), info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables, info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, info->cost.numOfAlterColSTables, info->cost.schemaTime - info->cost.parseTime, info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime, info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime); + } int32_t smlClearForRerun(SSmlHandle *info) { info->reRun = false; - // clear info->childTables -// SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); -// while (oneTable) { -// smlDestroyTableInfo(info, *oneTable); -// oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); -// } - - // clear info->superTables -// SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); -// while (oneSTable) { -// smlDestroySTableMeta(*oneSTable); -// oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); -// } taosHashClear(info->childTables); taosHashClear(info->superTables); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 5e656c71a7..845884d5ac 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -29,199 +29,6 @@ (start)++; \ } -// SArray *smlJsonParseTags(char *start, char *end){ -// SArray *tags = taosArrayInit(4, sizeof(SSmlKv)); -// while(start < end){ -// SSmlKv kv = {0}; -// kv.type = TSDB_DATA_TYPE_NCHAR; -// bool isInQuote = false; -// while(start < end){ -// if(unlikely(!isInQuote && *start == '"')){ -// start++; -// kv.key = start; -// isInQuote = true; -// continue; -// } -// if(unlikely(isInQuote && *start == '"')){ -// kv.keyLen = start - kv.key; -// start++; -// break; -// } -// start++; -// } -// bool hasColon = false; -// while(start < end){ -// if(unlikely(!hasColon && *start == ':')){ -// start++; -// hasColon = true; -// continue; -// } -// if(unlikely(hasColon && kv.value == NULL && (*start > 32 && *start != '"'))){ -// kv.value = start; -// start++; -// continue; -// } -// -// if(unlikely(hasColon && kv.value != NULL && (*start == '"' || *start == ',' || *start == '}'))){ -// kv.length = start - kv.value; -// taosArrayPush(tags, &kv); -// start++; -// break; -// } -// start++; -// } -// } -// return tags; -// } - -// static int32_t smlParseTagsFromJSON(SSmlHandle *info, SSmlLineInfo *elements) { -// int32_t ret = TSDB_CODE_SUCCESS; -// -// if(is_same_child_table_telnet(elements, &info->preLine) == 0){ -// return TSDB_CODE_SUCCESS; -// } -// -// bool isSameMeasure = IS_SAME_SUPER_TABLE; -// -// int cnt = 0; -// SArray *preLineKV = info->preLineTagKV; -// bool isSuperKVInit = true; -// SArray *superKV = NULL; -// if(info->dataFormat){ -// if(unlikely(!isSameMeasure)){ -// SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, -// elements->measureLen, NULL); -// -// if(unlikely(sMeta == NULL)){ -// sMeta = smlBuildSTableMeta(info->dataFormat); -// STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); -// sMeta->tableMeta = pTableMeta; -// if(pTableMeta == NULL){ -// info->dataFormat = false; -// info->reRun = true; -// return TSDB_CODE_SUCCESS; -// } -// nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL); -// } -// info->currSTableMeta = sMeta->tableMeta; -// superKV = sMeta->tags; -// -// if(unlikely(taosArrayGetSize(superKV) == 0)){ -// isSuperKVInit = false; -// } -// taosArraySetSize(preLineKV, 0); -// } -// }else{ -// taosArraySetSize(preLineKV, 0); -// } -// -// SArray *tags = smlJsonParseTags(elements->tags, elements->tags + elements->tagsLen); -// int32_t tagNum = taosArrayGetSize(tags); -// if (tagNum == 0) { -// uError("SML:tag is empty:%s", elements->tags) -// taosArrayDestroy(tags); -// return TSDB_CODE_SML_INVALID_DATA; -// } -// for (int32_t i = 0; i < tagNum; ++i) { -// SSmlKv kv = *(SSmlKv*)taosArrayGet(tags, i); -// -// if(info->dataFormat){ -// if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ -// info->dataFormat = false; -// info->reRun = true; -// taosArrayDestroy(tags); -// return TSDB_CODE_SUCCESS; -// } -// -// if(isSameMeasure){ -// if(unlikely(cnt >= taosArrayGetSize(preLineKV))) { -// info->dataFormat = false; -// info->reRun = true; -// taosArrayDestroy(tags); -// return TSDB_CODE_SUCCESS; -// } -// SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); -// if(unlikely(kv.length > preKV->length)){ -// preKV->length = kv.length; -// SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, -// elements->measureLen, NULL); -// if(unlikely(NULL == tableMeta)){ -// uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); -// return TSDB_CODE_SML_INTERNAL_ERROR; -// } -// -// SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt); -// oldKV->length = kv.length; -// info->needModifySchema = true; -// } -// if(unlikely(!IS_SAME_KEY)){ -// info->dataFormat = false; -// info->reRun = true; -// taosArrayDestroy(tags); -// return TSDB_CODE_SUCCESS; -// } -// }else{ -// if(isSuperKVInit){ -// if(unlikely(cnt >= taosArrayGetSize(superKV))) { -// info->dataFormat = false; -// info->reRun = true; -// taosArrayDestroy(tags); -// return TSDB_CODE_SUCCESS; -// } -// SSmlKv *preKV = (SSmlKv *)taosArrayGet(superKV, cnt); -// if(unlikely(kv.length > preKV->length)) { -// preKV->length = kv.length; -// }else{ -// kv.length = preKV->length; -// } -// info->needModifySchema = true; -// -// if(unlikely(!IS_SAME_KEY)){ -// info->dataFormat = false; -// info->reRun = true; -// taosArrayDestroy(tags); -// return TSDB_CODE_SUCCESS; -// } -// }else{ -// taosArrayPush(superKV, &kv); -// } -// taosArrayPush(preLineKV, &kv); -// } -// }else{ -// taosArrayPush(preLineKV, &kv); -// } -// cnt++; -// } -// taosArrayDestroy(tags); -// -// SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, -// is_same_child_table_telnet); if (unlikely(tinfo == NULL)) { -// tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); -// if (unlikely(!tinfo)) { -// return TSDB_CODE_OUT_OF_MEMORY; -// } -// tinfo->tags = taosArrayDup(preLineKV, NULL); -// -// smlSetCTableName(tinfo); -// if (info->dataFormat) { -// info->currSTableMeta->uid = tinfo->uid; -// tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); -// if (tinfo->tableDataCtx == NULL) { -// smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); -// return TSDB_CODE_SML_INVALID_DATA; -// } -// } -// -// SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); -// *key = *elements; -// tinfo->key = key; -// nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet); -// } -// if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; -// -// return ret; -// } - static char *smlJsonGetObj(char *payload) { int leftBracketCnt = 0; bool isInQuote = false; @@ -659,12 +466,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { break; } case cJSON_String: { - /* set default JSON type to binary/nchar according to - * user configured parameter tsDefaultJSONStrType - */ - - char *tsDefaultJSONStrType = "binary"; // todo - smlConvertJSONString(kv, tsDefaultJSONStrType, root); + smlConvertJSONString(kv, "binary", root); break; } case cJSON_Object: { @@ -682,138 +484,70 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { return TSDB_CODE_SUCCESS; } -static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { - int32_t ret = TSDB_CODE_SUCCESS; - - bool isSameMeasure = IS_SAME_SUPER_TABLE; - - int cnt = 0; +static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ SArray *preLineKV = info->preLineTagKV; - if (info->dataFormat) { - if (unlikely(!isSameMeasure)) { - SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); - SSmlSTableMeta *sMeta = NULL; - if (unlikely(tmp == NULL)) { - STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); - if (pTableMeta == NULL) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - sMeta = smlBuildSTableMeta(info->dataFormat); - if(sMeta == NULL){ - taosMemoryFreeClear(pTableMeta); - return TSDB_CODE_OUT_OF_MEMORY; - } - sMeta->tableMeta = pTableMeta; - taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES); - for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){ - SSchema *tag = pTableMeta->schema + i; - SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE }; - taosArrayPush(sMeta->tags, &kv); - } - tmp = &sMeta; - } - info->currSTableMeta = (*tmp)->tableMeta; - info->maxTagKVs = (*tmp)->tags; - } - } - taosArrayClear(preLineKV); + taosArrayClearEx(preLineKV, freeSSmlKv); + int cnt = 0; int32_t tagNum = cJSON_GetArraySize(tags); if (unlikely(tagNum == 0)) { uError("SML:Tag should not be empty"); - return TSDB_CODE_TSC_INVALID_JSON; + terrno = TSDB_CODE_TSC_INVALID_JSON; + return -1; } for (int32_t i = 0; i < tagNum; ++i) { cJSON *tag = cJSON_GetArrayItem(tags, i); if (unlikely(tag == NULL)) { - return TSDB_CODE_TSC_INVALID_JSON; + terrno = TSDB_CODE_TSC_INVALID_JSON; + return -1; } - // if(unlikely(tag == cMeasure)) continue; size_t keyLen = strlen(tag->string); if (unlikely(IS_INVALID_COL_LEN(keyLen))) { uError("OTD:Tag key length is 0 or too large than 64"); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + return -1; } // add kv to SSmlKv - SSmlKv kv = {.key = tag->string, .keyLen = keyLen}; + SSmlKv kv = {0}; + kv.key = tag->string; + kv.keyLen = keyLen; + // value - ret = smlParseValueFromJSON(tag, &kv); + int32_t ret = smlParseValueFromJSON(tag, &kv); if (unlikely(ret != TSDB_CODE_SUCCESS)) { - return ret; - } - - if (info->dataFormat) { - if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - - if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); - if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - if (unlikely(kv.length > maxKV->length)) { - maxKV->length = kv.length; - info->needModifySchema = true; - } + terrno = ret; + return -1; } taosArrayPush(preLineKV, &kv); + + if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { + terrno = TSDB_CODE_SUCCESS; + return -1; + } + cnt++; } + return 0; +} - elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen); - memcpy(elements->measureTag, elements->measure, elements->measureLen); - memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen); - elements->measureTagsLen = elements->measureLen + elements->tagsLen; - - SSmlTableInfo **tmp = - (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen); - SSmlTableInfo *tinfo = NULL; - if (unlikely(tmp == NULL)) { - tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); - if (unlikely(!tinfo)) { - return TSDB_CODE_OUT_OF_MEMORY; +static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { + int32_t ret = 0; + if(info->dataFormat){ + ret = smlProcessSuperTable(info, elements); + if(ret != 0){ + return terrno; } - tinfo->tags = taosArrayDup(preLineKV, NULL); - - smlSetCTableName(tinfo); - getTableUid(info, elements, tinfo); - if (info->dataFormat) { - info->currSTableMeta->uid = tinfo->uid; - tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); - if (tinfo->tableDataCtx == NULL) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); - smlDestroyTableInfo(&tinfo); - return TSDB_CODE_SML_INVALID_DATA; - } - } - - // SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); - // *key = *elements; - // if(info->parseJsonByLib){ - // key->tags = taosMemoryMalloc(elements->tagsLen + 1); - // memcpy(key->tags, elements->tags, elements->tagsLen); - // key->tags[elements->tagsLen] = 0; - // } - // tinfo->key = key; - taosHashPut(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen, &tinfo, - POINTER_BYTES); - tmp = &tinfo; } - if (info->dataFormat) info->currTableDataCtx = (*tmp)->tableDataCtx; - - return ret; + ret = smlProcessTagJson(info, tags); + if(ret != 0){ + return terrno; + } + ret = smlJoinMeasureTag(elements); + if(ret != 0){ + return ret; + } + return smlProcessChildTable(info, elements); } static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPrecision) { @@ -998,35 +732,10 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); return TSDB_CODE_INVALID_TIMESTAMP; } - SSmlKv kvTs = {.key = tsSmlTsDefaultName, - .keyLen = strlen(tsSmlTsDefaultName), - .type = TSDB_DATA_TYPE_TIMESTAMP, - .i = ts, - .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + SSmlKv kvTs = {0}; + smlBuildTsKv(&kvTs, ts); - if (info->dataFormat) { - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); - } - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildRow(info->currTableDataCtx); - } - clearColValArraySml(info->currTableDataCtx->pValues); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); - return ret; - } - } else { - if (elements->colArray == NULL) { - elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); - } - taosArrayPush(elements->colArray, &kvTs); - taosArrayPush(elements->colArray, &kv); - } - info->preLine = *elements; - - return TSDB_CODE_SUCCESS; + return smlParseEndTelnetJson(info, elements, &kvTs, &kv); } static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { @@ -1054,7 +763,6 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { return TSDB_CODE_TSC_INVALID_JSON; } - if (unlikely(info->lines != NULL)) { for (int i = 0; i < info->lineNum; i++) { taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv); @@ -1202,35 +910,10 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * return TSDB_CODE_INVALID_TIMESTAMP; } } - SSmlKv kvTs = {.key = tsSmlTsDefaultName, - .keyLen = strlen(tsSmlTsDefaultName), - .type = TSDB_DATA_TYPE_TIMESTAMP, - .i = ts, - .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + SSmlKv kvTs = {0}; + smlBuildTsKv(&kvTs, ts); - if (info->dataFormat) { - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); - } - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildRow(info->currTableDataCtx); - } - clearColValArraySml(info->currTableDataCtx->pValues); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); - return ret; - } - } else { - if (elements->colArray == NULL) { - elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); - } - taosArrayPush(elements->colArray, &kvTs); - taosArrayPush(elements->colArray, &kv); - } - info->preLine = *elements; - - return TSDB_CODE_SUCCESS; + return smlParseEndTelnetJson(info, elements, &kvTs, &kv); } int32_t smlParseJSON(SSmlHandle *info, char *payload) { diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 006475654a..0c610a4611 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -20,15 +20,9 @@ #include "clientSml.h" -// comma , #define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH) -// space #define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH) -// equal = #define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH) -// quote " -// #define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH) -// SLASH #define IS_SLASH_LETTER_IN_FIELD_VALUE(sql) (*((sql)-1) == SLASH && (*(sql) == QUOTE || *(sql) == SLASH)) @@ -51,10 +45,10 @@ } \ } -#define BINARY_ADD_LEN 2 // "binary" 2 means " " -#define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" " +#define BINARY_ADD_LEN (sizeof("\"\"")-1) // "binary" 2 means length of ("") +#define NCHAR_ADD_LEN (sizeof("L\"\"")-1) // L"nchar" 3 means length of (L"") -uint8_t smlPrecisionConvert[7] = {TSDB_TIME_PRECISION_NANO, TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES, +uint8_t smlPrecisionConvert[] = {TSDB_TIME_PRECISION_NANO, TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES, TSDB_TIME_PRECISION_SECONDS, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO}; @@ -198,61 +192,10 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { return TSDB_CODE_TSC_INVALID_VALUE; } -static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement, bool isSameMeasure, - bool isSameCTable) { - if (isSameCTable) { - return TSDB_CODE_SUCCESS; - } - - int cnt = 0; +static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ SArray *preLineKV = info->preLineTagKV; - if (info->dataFormat) { - if (unlikely(!isSameMeasure)) { - SSmlSTableMeta **tmp = - (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen); - - SSmlSTableMeta *sMeta = NULL; - if (unlikely(tmp == NULL)) { - char *measure = currElement->measure; - int measureLen = currElement->measureLen; - if (currElement->measureEscaped) { - measure = (char *)taosMemoryMalloc(currElement->measureLen); - memcpy(measure, currElement->measure, currElement->measureLen); - PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); - smlStrReplace(measure, measureLen); - } - STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen); - if (currElement->measureEscaped) { - taosMemoryFree(measure); - } - if (pTableMeta == NULL) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - sMeta = smlBuildSTableMeta(info->dataFormat); - if(sMeta == NULL){ - taosMemoryFreeClear(pTableMeta); - return TSDB_CODE_OUT_OF_MEMORY; - } - sMeta->tableMeta = pTableMeta; - taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES); - for (int i = pTableMeta->tableInfo.numOfColumns; - i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) { - SSchema *tag = pTableMeta->schema + i; - SSmlKv kv = {.key = tag->name, - .keyLen = strlen(tag->name), - .type = tag->type, - .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE}; - taosArrayPush(sMeta->tags, &kv); - } - tmp = &sMeta; - } - info->currSTableMeta = (*tmp)->tableMeta; - info->maxTagKVs = (*tmp)->tags; - } - } taosArrayClearEx(preLineKV, freeSSmlKv); + int cnt = 0; while (*sql < sqlEnd) { if (unlikely(IS_SPACE(*sql))) { @@ -267,7 +210,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin while (*sql < sqlEnd) { if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); - return TSDB_CODE_SML_INVALID_DATA; + terrno = TSDB_CODE_SML_INVALID_DATA; + return -1; } if (unlikely(IS_EQUAL(*sql))) { keyLen = *sql - key; @@ -283,7 +227,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + return -1; } // parse value @@ -297,7 +242,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin break; } else if (unlikely(IS_EQUAL(*sql))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); - return TSDB_CODE_SML_INVALID_DATA; + terrno = TSDB_CODE_SML_INVALID_DATA; + return -1; } if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { @@ -311,11 +257,13 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin if (unlikely(valueLen == 0)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); - return TSDB_CODE_SML_INVALID_DATA; + terrno = TSDB_CODE_SML_INVALID_DATA; + return -1; } if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + return -1; } if (keyEscaped) { @@ -331,37 +279,17 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin value = tmp; } SSmlKv kv = {.key = key, - .keyLen = keyLen, - .type = TSDB_DATA_TYPE_NCHAR, - .value = value, - .length = valueLen, - .keyEscaped = keyEscaped, - .valueEscaped = valueEscaped}; + .keyLen = keyLen, + .type = TSDB_DATA_TYPE_NCHAR, + .value = value, + .length = valueLen, + .keyEscaped = keyEscaped, + .valueEscaped = valueEscaped}; taosArrayPush(preLineKV, &kv); - if (info->dataFormat) { - if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); - - if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - - if (unlikely(kv.length > maxKV->length)) { - maxKV->length = kv.length; - info->needModifySchema = true; - } + if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { + terrno = TSDB_CODE_SUCCESS; + return -1; } cnt++; @@ -370,99 +298,33 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin } (*sql)++; } + return 0; +} - void *oneTable = taosHashGet(info->childTables, currElement->measure, currElement->measureTagsLen); - if ((oneTable != NULL)) { +static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) { + bool isSameCTable = IS_SAME_CHILD_TABLE; + if(isSameCTable){ return TSDB_CODE_SUCCESS; } - SSmlTableInfo *tinfo = smlBuildTableInfo(1, currElement->measure, currElement->measureLen); - if (unlikely(!tinfo)) { - return TSDB_CODE_OUT_OF_MEMORY; - } - tinfo->tags = taosArrayDup(preLineKV, NULL); - for (size_t i = 0; i < taosArrayGetSize(preLineKV); i++) { - SSmlKv *kv = (SSmlKv *)taosArrayGet(preLineKV, i); - if (kv->keyEscaped) kv->key = NULL; - if (kv->valueEscaped) kv->value = NULL; - } - - smlSetCTableName(tinfo); - getTableUid(info, currElement, tinfo); - if (info->dataFormat) { - info->currSTableMeta->uid = tinfo->uid; - tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); - if (tinfo->tableDataCtx == NULL) { - smlDestroyTableInfo(&tinfo); - smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); - return TSDB_CODE_SML_INVALID_DATA; + int32_t ret = 0; + if(info->dataFormat){ + ret = smlProcessSuperTable(info, elements); + if(ret != 0){ + return terrno; } } - taosHashPut(info->childTables, currElement->measure, currElement->measureTagsLen, &tinfo, POINTER_BYTES); + ret = smlProcessTagLine(info, sql, sqlEnd); + if(ret != 0){ + return terrno; + } - return TSDB_CODE_SUCCESS; + return smlProcessChildTable(info, elements); } -static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement, bool isSameMeasure, - bool isSameCTable) { +static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement) { int cnt = 0; - if (info->dataFormat) { - if (unlikely(!isSameCTable)) { - SSmlTableInfo **oneTable = - (SSmlTableInfo **)taosHashGet(info->childTables, currElement->measure, currElement->measureTagsLen); - if (unlikely(oneTable == NULL)) { - smlBuildInvalidDataMsg(&info->msgBuf, "child table should inside", currElement->measure); - return TSDB_CODE_SML_INVALID_DATA; - } - info->currTableDataCtx = (*oneTable)->tableDataCtx; - } - - if (unlikely(!isSameMeasure)) { - SSmlSTableMeta **tmp = - (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen); - if (unlikely(tmp == NULL)) { - char *measure = currElement->measure; - int measureLen = currElement->measureLen; - if (currElement->measureEscaped) { - measure = (char *)taosMemoryMalloc(currElement->measureLen); - memcpy(measure, currElement->measure, currElement->measureLen); - PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); - smlStrReplace(measure, measureLen); - } - STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen); - if (currElement->measureEscaped) { - taosMemoryFree(measure); - } - if (pTableMeta == NULL) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - *tmp = smlBuildSTableMeta(info->dataFormat); - if(*tmp == NULL){ - taosMemoryFreeClear(pTableMeta); - return TSDB_CODE_OUT_OF_MEMORY; - } - (*tmp)->tableMeta = pTableMeta; - taosHashPut(info->superTables, currElement->measure, currElement->measureLen, tmp, POINTER_BYTES); - - for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - SSchema *tag = pTableMeta->schema + i; - SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type}; - if (tag->type == TSDB_DATA_TYPE_NCHAR) { - kv.length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; - } else if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_GEOMETRY || tag->type == TSDB_DATA_TYPE_VARBINARY) { - kv.length = tag->bytes - VARSTR_HEADER_SIZE; - } - taosArrayPush((*tmp)->cols, &kv); - } - } - info->currSTableMeta = (*tmp)->tableMeta; - info->maxColKVs = (*tmp)->cols; - } - } - while (*sql < sqlEnd) { if (unlikely(IS_SPACE(*sql))) { break; @@ -562,47 +424,11 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin } if (info->dataFormat) { - // cnt begin 0, add ts so + 2 - if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) { - info->dataFormat = false; - info->reRun = true; - freeSSmlKv(&kv); - return TSDB_CODE_SUCCESS; - } - // bind data - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, cnt + 1); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - uDebug("smlBuildCol error, retry"); - info->dataFormat = false; - info->reRun = true; - freeSSmlKv(&kv); - return TSDB_CODE_SUCCESS; - } - if (cnt >= taosArrayGetSize(info->maxColKVs)) { - info->dataFormat = false; - info->reRun = true; - freeSSmlKv(&kv); - return TSDB_CODE_SUCCESS; - } - SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt); - if (kv.type != maxKV->type) { - info->dataFormat = false; - info->reRun = true; - freeSSmlKv(&kv); - return TSDB_CODE_SUCCESS; - } - if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - freeSSmlKv(&kv); - return TSDB_CODE_SUCCESS; - } - - if (unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > maxKV->length)) { - maxKV->length = kv.length; - info->needModifySchema = true; - } + bool isAligned = isSmlColAligned(info, cnt, &kv); freeSSmlKv(&kv); + if(!isAligned){ + return TSDB_CODE_SUCCESS; + } } else { if (currElement->colArray == NULL) { currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1); @@ -625,7 +451,6 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine JUMP_SPACE(sql, sqlEnd) if (unlikely(*sql == COMMA)) return TSDB_CODE_SML_INVALID_DATA; elements->measure = sql; - // parse measure size_t measureLenEscaped = 0; while (sql < sqlEnd) { @@ -659,20 +484,12 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine tmp++; } elements->measureTagsLen = tmp - elements->measure; - - bool isSameCTable = false; - bool isSameMeasure = false; - if (IS_SAME_CHILD_TABLE) { - isSameCTable = true; - isSameMeasure = true; - } else if (info->dataFormat) { - isSameMeasure = IS_SAME_SUPER_TABLE; - } + elements->measureTag = elements->measure; // parse tag if (*sql == COMMA) sql++; elements->tags = sql; - int ret = smlParseTagKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable); + int ret = smlParseTagLine(info, &sql, sqlEnd, elements); if (unlikely(ret != TSDB_CODE_SUCCESS)) { return ret; } @@ -687,7 +504,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine JUMP_SPACE(sql, sqlEnd) elements->cols = sql; - ret = smlParseColKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable); + ret = smlParseColLine(info, &sql, sqlEnd, elements); if (unlikely(ret != TSDB_CODE_SUCCESS)) { return ret; } @@ -718,31 +535,9 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); return TSDB_CODE_INVALID_TIMESTAMP; } - // add ts to - SSmlKv kv = {.key = tsSmlTsDefaultName, - .keyLen = strlen(tsSmlTsDefaultName), - .type = TSDB_DATA_TYPE_TIMESTAMP, - .i = ts, - .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, - .keyEscaped = false, - .valueEscaped = false}; - if (info->dataFormat) { - uDebug("SML:0x%" PRIx64 " smlParseInfluxString format true, ts:%" PRId64, info->id, ts); - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0); - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildRow(info->currTableDataCtx); - } - clearColValArraySml(info->currTableDataCtx->pValues); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); - return ret; - } - } else { - uDebug("SML:0x%" PRIx64 " smlParseInfluxString format false, ts:%" PRId64, info->id, ts); - taosArraySet(elements->colArray, 0, &kv); - } - info->preLine = *elements; + SSmlKv kvTs = {0}; + smlBuildTsKv(&kvTs, ts); - return ret; + return smlParseEndLine(info, elements, &kvTs); } diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index f9f8602c5a..f715f32556 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -23,8 +23,6 @@ int32_t is_same_child_table_telnet(const void *a, const void *b) { SSmlLineInfo *t1 = (SSmlLineInfo *)a; SSmlLineInfo *t2 = (SSmlLineInfo *)b; - // uError("is_same_child_table_telnet len:%d,%d %s,%s @@@ len:%d,%d %s,%s", t1->measureLen, t2->measureLen, - // t1->measure, t2->measure, t1->tagsLen, t2->tagsLen, t1->tags, t2->tags); if (t1 == NULL || t2 == NULL || t1->measure == NULL || t2->measure == NULL || t1->tags == NULL || t2->tags == NULL) return 1; return (((t1->measureLen == t2->measureLen) && memcmp(t1->measure, t2->measure, t1->measureLen) == 0) && @@ -69,47 +67,11 @@ static void smlParseTelnetElement(char **sql, char *sqlEnd, char **data, int32_t } } -static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) { - if (is_same_child_table_telnet(elements, &info->preLine) == 0) { - elements->measureTag = info->preLine.measureTag; - return TSDB_CODE_SUCCESS; - } - - bool isSameMeasure = IS_SAME_SUPER_TABLE; - - int cnt = 0; +static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ SArray *preLineKV = info->preLineTagKV; - if (info->dataFormat) { - if (!isSameMeasure) { - SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); - SSmlSTableMeta *sMeta = NULL; - if (unlikely(tmp == NULL)) { - STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); - if (pTableMeta == NULL) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - sMeta = smlBuildSTableMeta(info->dataFormat); - if(sMeta == NULL){ - taosMemoryFreeClear(pTableMeta); - return TSDB_CODE_OUT_OF_MEMORY; - } - sMeta->tableMeta = pTableMeta; - taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES); - for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){ - SSchema *tag = pTableMeta->schema + i; - SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE }; - taosArrayPush(sMeta->tags, &kv); - } - tmp = &sMeta; - } - info->currSTableMeta = (*tmp)->tableMeta; - info->maxTagKVs = (*tmp)->tags; - } - } + taosArrayClearEx(preLineKV, freeSSmlKv); + int cnt = 0; - taosArrayClear(preLineKV); const char *sql = data; while (sql < sqlEnd) { JUMP_SPACE(sql, sqlEnd) @@ -121,8 +83,9 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS // parse key while (sql < sqlEnd) { if (unlikely(*sql == SPACE)) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); + terrno = TSDB_CODE_SML_INVALID_DATA; + return -1; } if (unlikely(*sql == EQUAL)) { keyLen = sql - key; @@ -133,13 +96,10 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS } if (unlikely(IS_INVALID_COL_LEN(keyLen))) { - smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); + terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + return -1; } - // if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { - // smlBuildInvalidDataMsg(msg, "dumplicate key", key); - // return TSDB_CODE_TSC_DUP_NAMES; - // } // parse value const char *value = sql; @@ -150,87 +110,67 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS break; } if (unlikely(*sql == EQUAL)) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); + terrno = TSDB_CODE_SML_INVALID_DATA; + return -1; } sql++; } valueLen = sql - value; if (unlikely(valueLen == 0)) { - smlBuildInvalidDataMsg(msg, "invalid value", value); - return TSDB_CODE_TSC_INVALID_VALUE; + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); + terrno = TSDB_CODE_TSC_INVALID_VALUE; + return -1; } if (unlikely(valueLen > (TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + return -1; } - SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen}; - - if (info->dataFormat) { - if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); - if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - if (unlikely(kv.length > maxKV->length)) { - maxKV->length = kv.length; - info->needModifySchema = true; - } - } + SSmlKv kv = {.key = key, + .keyLen = keyLen, + .type = TSDB_DATA_TYPE_NCHAR, + .value = value, + .length = valueLen, + .keyEscaped = false, + .valueEscaped = false}; taosArrayPush(preLineKV, &kv); + if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { + terrno = TSDB_CODE_SUCCESS; + return -1; + } cnt++; } + return 0; +} - elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen); - memcpy(elements->measureTag, elements->measure, elements->measureLen); - memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen); - elements->measureTagsLen = elements->measureLen + elements->tagsLen; - - SSmlTableInfo **tmp = - (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen); - SSmlTableInfo *tinfo = NULL; - if (unlikely(tmp == NULL)) { - tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); - if (!tinfo) { - return TSDB_CODE_OUT_OF_MEMORY; - } - tinfo->tags = taosArrayDup(preLineKV, NULL); - - smlSetCTableName(tinfo); - getTableUid(info, elements, tinfo); - if (info->dataFormat) { - info->currSTableMeta->uid = tinfo->uid; - tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); - if (tinfo->tableDataCtx == NULL) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); - smlDestroyTableInfo(&tinfo); - return TSDB_CODE_SML_INVALID_DATA; - } - } - - // SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); - // *key = *elements; - // tinfo->key = key; - taosHashPut(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen, &tinfo, - POINTER_BYTES); - tmp = &tinfo; +static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements) { + if (is_same_child_table_telnet(elements, &info->preLine) == 0) { + elements->measureTag = info->preLine.measureTag; + return TSDB_CODE_SUCCESS; } - if (info->dataFormat) info->currTableDataCtx = (*tmp)->tableDataCtx; - return TSDB_CODE_SUCCESS; + int32_t ret = 0; + if(info->dataFormat){ + ret = smlProcessSuperTable(info, elements); + if(ret != 0){ + return terrno; + } + } + + ret = smlProcessTagTelnet(info, data, sqlEnd); + if(ret != 0){ + return terrno; + } + + ret = smlJoinMeasureTag(elements); + if(ret != 0){ + return ret; + } + + return smlProcessChildTable(info, elements); } // format: =[ =] @@ -251,7 +191,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine return TSDB_CODE_SML_INVALID_DATA; } - bool needConverTime = false; // get TS before parse tag(get meta), so need conver time + bool needConverTime = false; // get TS before parse tag(get meta), so need convert time if (info->dataFormat && info->currSTableMeta == NULL) { needConverTime = true; } @@ -260,11 +200,6 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); return TSDB_CODE_INVALID_TIMESTAMP; } - SSmlKv kvTs = {.key = tsSmlTsDefaultName, - .keyLen = strlen(tsSmlTsDefaultName), - .type = TSDB_DATA_TYPE_TIMESTAMP, - .i = ts, - .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; // parse value smlParseTelnetElement(&sql, sqlEnd, &elements->cols, &elements->colsLen); @@ -287,7 +222,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine return TSDB_CODE_TSC_INVALID_VALUE; } - int ret = smlParseTelnetTags(info, sql, sqlEnd, elements, &info->msgBuf); + int ret = smlParseTelnetTags(info, sql, sqlEnd, elements); if (unlikely(ret != TSDB_CODE_SUCCESS)) { return ret; } @@ -296,30 +231,11 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine return TSDB_CODE_SUCCESS; } - if (info->dataFormat && info->currSTableMeta != NULL) { - if (needConverTime) { - kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision); - } - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); - } - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildRow(info->currTableDataCtx); - } - clearColValArraySml(info->currTableDataCtx->pValues); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); - return ret; - } - } else { - if (elements->colArray == NULL) { - elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); - } - taosArrayPush(elements->colArray, &kvTs); - taosArrayPush(elements->colArray, &kv); + SSmlKv kvTs = {0}; + smlBuildTsKv(&kvTs, ts); + if (needConverTime) { + kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision); } - info->preLine = *elements; - return TSDB_CODE_SUCCESS; + return smlParseEndTelnetJson(info, elements, &kvTs, &kv); } \ No newline at end of file diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index addf0aa629..e78783cf3c 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -126,9 +126,9 @@ void queryCallback(void* param, void* res, int32_t code) { taos_fetch_raw_block_a(res, fetchCallback, param); } -void createNewTable(TAOS* pConn, int32_t index) { +void createNewTable(TAOS* pConn, int32_t index, int32_t numOfRows, int64_t startTs, const char* pVarchar) { char str[1024] = {0}; - sprintf(str, "create table tu%d using st2 tags(%d)", index, index); + sprintf(str, "create table if not exists tu%d using st2 tags(%d)", index, index); TAOS_RES* pRes = taos_query(pConn, str); if (taos_errno(pRes) != 0) { @@ -136,22 +136,43 @@ void createNewTable(TAOS* pConn, int32_t index) { } taos_free_result(pRes); - for (int32_t i = 0; i < 10000; i += 20) { - char sql[1024] = {0}; - sprintf(sql, - "insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" - "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" - "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" - "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)", - index, i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, - i + 7, i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, - i + 14, i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19); - TAOS_RES* p = taos_query(pConn, sql); - if (taos_errno(p) != 0) { - printf("failed to insert data, reason:%s\n", taos_errstr(p)); - } + if (startTs == 0) { + for (int32_t i = 0; i < numOfRows; i += 20) { + char sql[1024] = {0}; + sprintf(sql, + "insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)", + index, i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, + i + 7, i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, + i + 14, i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19); + TAOS_RES* p = taos_query(pConn, sql); + if (taos_errno(p) != 0) { + printf("failed to insert data, reason:%s\n", taos_errstr(p)); + } - taos_free_result(p); + taos_free_result(p); + } + } else { + for (int32_t i = 0; i < numOfRows; i += 20) { + char sql[1024*50] = {0}; + sprintf(sql, + "insert into tu%d values(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, " + "%d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, " + "'%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')", + index, startTs, i, pVarchar, startTs + 1, i + 1, pVarchar, startTs + 2, i + 2, pVarchar, startTs + 3, i + 3, pVarchar, startTs + 4, i + 4, + pVarchar, startTs + 5, i + 5, pVarchar, startTs + 6, i + 6, pVarchar, startTs + 7, i + 7, pVarchar, startTs + 8, i + 8, pVarchar, startTs + 9, i + 9, + pVarchar, startTs + 10, i + 10, pVarchar, startTs + 11, i + 11, pVarchar, startTs + 12, i + 12, pVarchar, startTs + 13, i + 13, pVarchar, startTs + 14, + i + 14, pVarchar, startTs + 15, i + 15, pVarchar, startTs + 16, i + 16, pVarchar, startTs + 17, i + 17, pVarchar, startTs + 18, i + 18, + pVarchar, startTs + 19, i + 19, pVarchar); + TAOS_RES* p = taos_query(pConn, sql); + if (taos_errno(p) != 0) { + printf("failed to insert data, reason:%s\n", taos_errstr(p)); + } + + taos_free_result(p); + } } } @@ -808,14 +829,7 @@ TEST(clientCase, projection_query_tables) { TAOS_RES* pRes = taos_query(pConn, "use abc1"); taos_free_result(pRes); - pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); - if (taos_errno(pRes) != 0) { - printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - - pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int) tags(a int)"); + pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int, f varchar(4096)) tags(a int)"); if (taos_errno(pRes) != 0) { printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); } @@ -828,28 +842,32 @@ TEST(clientCase, projection_query_tables) { taos_free_result(pRes); int64_t start = 1685959190000; + const char* pstr = + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefgh" + "ijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnop" + "qrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwx" + "yzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdef" + "ghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz!@#$%^&&*&^^%$#@!qQWERTYUIOPASDFGHJKL:" + "QWERTYUIOP{}"; - int32_t code = -1; - for(int32_t i = 0; i < 1000000; ++i) { - char t[512] = {0}; + for(int32_t i = 0; i < 10000; ++i) { + char str[1024] = {0}; + sprintf(str, "create table if not exists tu%d using st2 tags(%d)", i, i); - sprintf(t, "insert into t1 values(now, %d)", i); - while(1) { - void* p = taos_query(pConn, t); - code = taos_errno(p); - taos_free_result(p); - if (code != 0) { - printf("insert data error, retry\n"); - } else { - break; - } + TAOS_RES* px = taos_query(pConn, str); + if (taos_errno(px) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(px); + } + + for(int32_t j = 0; j < 5000; ++j) { + start += 20; + for (int32_t i = 0; i < 10000; ++i) { + createNewTable(pConn, i, 20, start, pstr); } } - for (int32_t i = 0; i < 1; ++i) { - printf("create table :%d\n", i); - createNewTable(pConn, i); - } // // pRes = taos_query(pConn, "select * from tu"); // if (taos_errno(pRes) != 0) { diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 2a334a34b8..7c8676e9f5 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -86,7 +86,7 @@ typedef struct { char err_msg[128]; S3Status status; uint64_t content_length; - char *buf; + char * buf; int64_t buf_pos; } TS3SizeCBD; @@ -270,7 +270,7 @@ typedef struct list_parts_callback_data { typedef struct MultipartPartData { put_object_callback_data put_object_data; int seq; - UploadManager *manager; + UploadManager * manager; } MultipartPartData; static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) { @@ -317,7 +317,7 @@ S3Status MultipartResponseProperiesCallback(const S3ResponseProperties *properti MultipartPartData *data = (MultipartPartData *)callbackData; int seq = data->seq; - const char *etag = properties->eTag; + const char * etag = properties->eTag; data->manager->etags[seq - 1] = strdup(etag); data->manager->next_etags_pos = seq; return S3StatusOK; @@ -450,10 +450,10 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { int32_t code = 0; const char *key = object; // const char *uploadId = 0; - const char *filename = 0; + const char * filename = 0; uint64_t contentLength = 0; - const char *cacheControl = 0, *contentType = 0, *md5 = 0; - const char *contentDispositionFilename = 0, *contentEncoding = 0; + const char * cacheControl = 0, *contentType = 0, *md5 = 0; + const char * contentDispositionFilename = 0, *contentEncoding = 0; int64_t expires = -1; S3CannedAcl cannedAcl = S3CannedAclPrivate; int metaPropertiesCount = 0; @@ -467,6 +467,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { // data.infileFD = NULL; // data.noStatus = noStatus; + // uError("ERROR: %s stat file %s: ", __func__, file); if (taosStatFile(file, &contentLength, NULL, NULL) < 0) { uError("ERROR: %s Failed to stat file %s: ", __func__, file); code = TAOS_SYSTEM_ERROR(errno); @@ -647,7 +648,7 @@ typedef struct list_bucket_callback_data { char nextMarker[1024]; int keyCount; int allDetails; - SArray *objectArray; + SArray * objectArray; } list_bucket_callback_data; static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int contentsCount, @@ -692,11 +693,11 @@ static void s3FreeObjectKey(void *pItem) { static SArray *getListByPrefix(const char *prefix) { S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, - 0, awsRegionG}; + 0, awsRegionG}; S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback}, &listBucketCallback}; - const char *marker = 0, *delimiter = 0; + const char * marker = 0, *delimiter = 0; int maxkeys = 0, allDetails = 0; list_bucket_callback_data data; data.objectArray = taosArrayInit(32, sizeof(void *)); @@ -737,7 +738,7 @@ static SArray *getListByPrefix(const char *prefix) { void s3DeleteObjects(const char *object_name[], int nobject) { S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, - 0, awsRegionG}; + 0, awsRegionG}; S3ResponseHandler responseHandler = {0, &responseCompleteCallback}; for (int i = 0; i < nobject; ++i) { @@ -788,7 +789,7 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, const char *ifMatch = 0, *ifNotMatch = 0; S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, - 0, awsRegionG}; + 0, awsRegionG}; S3GetConditions getConditions = {ifModifiedSince, ifNotModifiedSince, ifMatch, ifNotMatch}; S3GetObjectHandler getObjectHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, &getObjectDataCallback}; @@ -826,7 +827,7 @@ int32_t s3GetObjectToFile(const char *object_name, char *fileName) { const char *ifMatch = 0, *ifNotMatch = 0; S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, - 0, awsRegionG}; + 0, awsRegionG}; S3GetConditions getConditions = {ifModifiedSince, ifNotModifiedSince, ifMatch, ifNotMatch}; S3GetObjectHandler getObjectHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback}, &getObjectCallback}; @@ -857,7 +858,7 @@ int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) { if (objectArray == NULL) return -1; for (size_t i = 0; i < taosArrayGetSize(objectArray); i++) { - char *object = taosArrayGetP(objectArray, i); + char * object = taosArrayGetP(objectArray, i); const char *tmp = strchr(object, '/'); tmp = (tmp == NULL) ? object : tmp + 1; char fileName[PATH_MAX] = {0}; @@ -948,12 +949,12 @@ static void s3InitRequestOptions(cos_request_options_t *options, int is_cname) { int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) { int32_t code = 0; - cos_pool_t *p = NULL; + cos_pool_t * p = NULL; int is_cname = 0; - cos_status_t *s = NULL; + cos_status_t * s = NULL; cos_request_options_t *options = NULL; cos_string_t bucket, object, file; - cos_table_t *resp_headers; + cos_table_t * resp_headers; // int traffic_limit = 0; cos_pool_create(&p, NULL); @@ -984,14 +985,14 @@ int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) { int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str) { int32_t code = 0; - cos_pool_t *p = NULL; + cos_pool_t * p = NULL; int is_cname = 0; - cos_status_t *s = NULL; - cos_request_options_t *options = NULL; + cos_status_t * s = NULL; + cos_request_options_t * options = NULL; cos_string_t bucket, object, file; - cos_table_t *resp_headers; + cos_table_t * resp_headers; int traffic_limit = 0; - cos_table_t *headers = NULL; + cos_table_t * headers = NULL; cos_resumable_clt_params_t *clt_params = NULL; cos_pool_create(&p, NULL); @@ -1024,11 +1025,11 @@ int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str) { } void s3DeleteObjectsByPrefix(const char *prefix_str) { - cos_pool_t *p = NULL; + cos_pool_t * p = NULL; cos_request_options_t *options = NULL; int is_cname = 0; cos_string_t bucket; - cos_status_t *s = NULL; + cos_status_t * s = NULL; cos_string_t prefix; cos_pool_create(&p, NULL); @@ -1043,10 +1044,10 @@ void s3DeleteObjectsByPrefix(const char *prefix_str) { } void s3DeleteObjects(const char *object_name[], int nobject) { - cos_pool_t *p = NULL; + cos_pool_t * p = NULL; int is_cname = 0; cos_string_t bucket; - cos_table_t *resp_headers = NULL; + cos_table_t * resp_headers = NULL; cos_request_options_t *options = NULL; cos_list_t object_list; cos_list_t deleted_object_list; @@ -1080,14 +1081,14 @@ void s3DeleteObjects(const char *object_name[], int nobject) { bool s3Exists(const char *object_name) { bool ret = false; - cos_pool_t *p = NULL; + cos_pool_t * p = NULL; int is_cname = 0; - cos_status_t *s = NULL; - cos_request_options_t *options = NULL; + cos_status_t * s = NULL; + cos_request_options_t * options = NULL; cos_string_t bucket; cos_string_t object; - cos_table_t *resp_headers; - cos_table_t *headers = NULL; + cos_table_t * resp_headers; + cos_table_t * headers = NULL; cos_object_exist_status_e object_exist; cos_pool_create(&p, NULL); @@ -1114,15 +1115,15 @@ bool s3Exists(const char *object_name) { bool s3Get(const char *object_name, const char *path) { bool ret = false; - cos_pool_t *p = NULL; + cos_pool_t * p = NULL; int is_cname = 0; - cos_status_t *s = NULL; + cos_status_t * s = NULL; cos_request_options_t *options = NULL; cos_string_t bucket; cos_string_t object; cos_string_t file; - cos_table_t *resp_headers = NULL; - cos_table_t *headers = NULL; + cos_table_t * resp_headers = NULL; + cos_table_t * headers = NULL; int traffic_limit = 0; //创建内存池 @@ -1158,15 +1159,15 @@ bool s3Get(const char *object_name, const char *path) { int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_size, bool check, uint8_t **ppBlock) { (void)check; int32_t code = 0; - cos_pool_t *p = NULL; + cos_pool_t * p = NULL; int is_cname = 0; - cos_status_t *s = NULL; + cos_status_t * s = NULL; cos_request_options_t *options = NULL; cos_string_t bucket; cos_string_t object; - cos_table_t *resp_headers; - cos_table_t *headers = NULL; - cos_buf_t *content = NULL; + cos_table_t * resp_headers; + cos_table_t * headers = NULL; + cos_buf_t * content = NULL; // cos_string_t file; // int traffic_limit = 0; char range_buf[64]; @@ -1260,7 +1261,7 @@ void s3EvictCache(const char *path, long object_size) { terrno = TAOS_SYSTEM_ERROR(errno); vError("failed to open %s since %s", dir_name, terrstr()); } - SArray *evict_files = taosArrayInit(16, sizeof(SEvictFile)); + SArray * evict_files = taosArrayInit(16, sizeof(SEvictFile)); tdbDirEntryPtr pDirEntry; while ((pDirEntry = taosReadDir(pDir)) != NULL) { char *name = taosGetDirEntryName(pDirEntry); @@ -1302,13 +1303,13 @@ void s3EvictCache(const char *path, long object_size) { long s3Size(const char *object_name) { long size = 0; - cos_pool_t *p = NULL; + cos_pool_t * p = NULL; int is_cname = 0; - cos_status_t *s = NULL; + cos_status_t * s = NULL; cos_request_options_t *options = NULL; cos_string_t bucket; cos_string_t object; - cos_table_t *resp_headers = NULL; + cos_table_t * resp_headers = NULL; //创建内存池 cos_pool_create(&p, NULL); @@ -1354,5 +1355,6 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, void s3EvictCache(const char *path, long object_size) {} long s3Size(const char *object_name) { return 0; } int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) { return 0; } +int32_t s3GetObjectToFile(const char *object_name, char *fileName) { return 0; } #endif diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index f97f9c0c11..02dfbfebfe 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1248,6 +1248,7 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { n += tPutCStr(p ? p + n : p, pTagVal->pKey); } else { n += tPutI16v(p ? p + n : p, pTagVal->cid); + ASSERTS(pTagVal->cid > 0, "Invalid tag cid:%" PRIi16, pTagVal->cid); } // type diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c6b964f348..98d79c5295 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -107,7 +107,7 @@ bool tsEnableTelem = true; int32_t tsTelemInterval = 43200; char tsTelemServer[TSDB_FQDN_LEN] = "telemetry.tdengine.com"; uint16_t tsTelemPort = 80; -char *tsTelemUri = "/report"; +char * tsTelemUri = "/report"; #ifdef TD_ENTERPRISE bool tsEnableCrashReport = false; @@ -354,16 +354,24 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input char cfgFile[PATH_MAX + 100] = {0}; taosExpandDir(inputCfgDir, cfgDir, PATH_MAX); - char lastC = cfgDir[strlen(cfgDir) - 1]; + char lastC = cfgDir[strlen(cfgDir) - 1]; char *tdDirsep = TD_DIRSEP; if (lastC == '\\' || lastC == '/') { tdDirsep = ""; } if (taosIsDir(cfgDir)) { #ifdef CUS_PROMPT - snprintf(cfgFile, sizeof(cfgFile), "%s" "%s" "%s.cfg", cfgDir, tdDirsep, CUS_PROMPT); + snprintf(cfgFile, sizeof(cfgFile), + "%s" + "%s" + "%s.cfg", + cfgDir, tdDirsep, CUS_PROMPT); #else - snprintf(cfgFile, sizeof(cfgFile), "%s" "%s" "taos.cfg", cfgDir, tdDirsep); + snprintf(cfgFile, sizeof(cfgFile), + "%s" + "%s" + "taos.cfg", + cfgDir, tdDirsep); #endif } else { tstrncpy(cfgFile, cfgDir, sizeof(cfgDir)); @@ -729,6 +737,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; @@ -1273,7 +1282,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi static int32_t taosCheckGlobalCfg() { uint32_t ipv4 = taosGetIpv4FromFqdn(tsLocalFqdn); if (ipv4 == 0xffffffff) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TSDB_CODE_RPC_FQDN_ERROR; uError("failed to get ip from fqdn:%s since %s, dnode can not be initialized", tsLocalFqdn, terrstr()); return -1; } @@ -1347,7 +1356,7 @@ void taosCleanupCfg() { typedef struct { const char *optionName; - void *optionVar; + void * optionVar; } OptionNameAndVar; static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, SConfigItem *pItem, bool isDebugflag) { @@ -1360,7 +1369,7 @@ static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, switch (pItem->dtype) { case CFG_DTYPE_BOOL: { int32_t flag = pItem->i32; - bool *pVar = pOptions[d].optionVar; + bool * pVar = pOptions[d].optionVar; uInfo("%s set from %d to %d", optName, *pVar, flag); *pVar = flag; terrno = TSDB_CODE_SUCCESS; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 01b1df9d5f..d69542c98b 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -18,42 +18,51 @@ #undef TD_MSG_NUMBER_ #undef TD_MSG_DICT_ +#undef TD_MSG_RANGE_CODE_ #define TD_MSG_INFO_ #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" #undef TD_MSG_NUMBER_ #undef TD_MSG_INFO_ +#undef TD_MSG_RANGE_CODE_ #define TD_MSG_DICT_ #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" +#undef TD_MSG_NUMBER_ +#undef TD_MSG_INFO_ +#undef TD_MSG_DICT_ +#undef TD_MSG_SEG_CODE_ +#define TD_MSG_RANGE_CODE_ +#include "tmsgdef.h" + #include "tlog.h" -#define DECODESQL() \ - do { \ - if(!tDecodeIsEnd(&decoder)){ \ - if(tDecodeI32(&decoder, &pReq->sqlLen) < 0) return -1; \ - if(pReq->sqlLen > 0){ \ - if (tDecodeBinaryAlloc(&decoder, (void **)&pReq->sql, NULL) < 0) return -1; \ - } \ - } \ +#define DECODESQL() \ + do { \ + if (!tDecodeIsEnd(&decoder)) { \ + if (tDecodeI32(&decoder, &pReq->sqlLen) < 0) return -1; \ + if (pReq->sqlLen > 0) { \ + if (tDecodeBinaryAlloc(&decoder, (void **)&pReq->sql, NULL) < 0) return -1; \ + } \ + } \ } while (0) -#define ENCODESQL() \ - do { \ - if (pReq->sqlLen > 0 && pReq->sql != NULL){ \ - if (tEncodeI32(&encoder, pReq->sqlLen) < 0) return -1; \ - if (tEncodeBinary(&encoder, pReq->sql, pReq->sqlLen) < 0) return -1; \ - } \ +#define ENCODESQL() \ + do { \ + if (pReq->sqlLen > 0 && pReq->sql != NULL) { \ + if (tEncodeI32(&encoder, pReq->sqlLen) < 0) return -1; \ + if (tEncodeBinary(&encoder, pReq->sql, pReq->sqlLen) < 0) return -1; \ + } \ } while (0) -#define FREESQL() \ - do { \ - if(pReq->sql != NULL){ \ - taosMemoryFree(pReq->sql); \ - } \ - pReq->sql = NULL; \ +#define FREESQL() \ + do { \ + if (pReq->sql != NULL) { \ + taosMemoryFree(pReq->sql); \ + } \ + pReq->sql = NULL; \ } while (0) static int32_t tDecodeSVAlterTbReqCommon(SDecoder *pDecoder, SVAlterTbReq *pReq); @@ -742,9 +751,7 @@ int32_t tDeserializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) return 0; } -void tFreeSMDropStbReq(SMDropStbReq *pReq) { - FREESQL(); -} +void tFreeSMDropStbReq(SMDropStbReq *pReq) { FREESQL(); } int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq) { SEncoder encoder = {0}; @@ -1489,9 +1496,7 @@ int32_t tDeserializeSDropUserReq(void *buf, int32_t bufLen, SDropUserReq *pReq) return 0; } -void tFreeSDropUserReq(SDropUserReq *pReq) { - FREESQL(); -} +void tFreeSDropUserReq(SDropUserReq *pReq) { FREESQL(); } SIpWhiteList *cloneIpWhiteList(SIpWhiteList *pIpWhiteList) { if (pIpWhiteList == NULL) return NULL; @@ -1822,7 +1827,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) char *tb = taosHashIterate(pRsp->readTbs, NULL); while (tb != NULL) { size_t keyLen = 0; - void *key = taosHashGetKey(tb, &keyLen); + void * key = taosHashGetKey(tb, &keyLen); if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1; @@ -1837,7 +1842,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) tb = taosHashIterate(pRsp->writeTbs, NULL); while (tb != NULL) { size_t keyLen = 0; - void *key = taosHashGetKey(tb, &keyLen); + void * key = taosHashGetKey(tb, &keyLen); if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1; @@ -1852,7 +1857,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) tb = taosHashIterate(pRsp->alterTbs, NULL); while (tb != NULL) { size_t keyLen = 0; - void *key = taosHashGetKey(tb, &keyLen); + void * key = taosHashGetKey(tb, &keyLen); if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1; @@ -1867,7 +1872,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) tb = taosHashIterate(pRsp->readViews, NULL); while (tb != NULL) { size_t keyLen = 0; - void *key = taosHashGetKey(tb, &keyLen); + void * key = taosHashGetKey(tb, &keyLen); if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1; @@ -1882,7 +1887,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) tb = taosHashIterate(pRsp->writeViews, NULL); while (tb != NULL) { size_t keyLen = 0; - void *key = taosHashGetKey(tb, &keyLen); + void * key = taosHashGetKey(tb, &keyLen); if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1; @@ -1897,7 +1902,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) tb = taosHashIterate(pRsp->alterViews, NULL); while (tb != NULL) { size_t keyLen = 0; - void *key = taosHashGetKey(tb, &keyLen); + void * key = taosHashGetKey(tb, &keyLen); if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1; @@ -1912,7 +1917,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) int32_t *useDb = taosHashIterate(pRsp->useDbs, NULL); while (useDb != NULL) { size_t keyLen = 0; - void *key = taosHashGetKey(useDb, &keyLen); + void * key = taosHashGetKey(useDb, &keyLen); if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1; @@ -1954,8 +1959,8 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs pRsp->alterViews = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); pRsp->useDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (pRsp->createdDbs == NULL || pRsp->readDbs == NULL || pRsp->writeDbs == NULL || pRsp->readTbs == NULL || - pRsp->writeTbs == NULL || pRsp->alterTbs == NULL || pRsp->readViews == NULL || - pRsp->writeViews == NULL || pRsp->alterViews == NULL ||pRsp->useDbs == NULL) { + pRsp->writeTbs == NULL || pRsp->alterTbs == NULL || pRsp->readViews == NULL || pRsp->writeViews == NULL || + pRsp->alterViews == NULL || pRsp->useDbs == NULL) { goto _err; } @@ -2219,7 +2224,7 @@ int32_t tDeserializeSGetUserWhiteListReq(void *buf, int32_t bufLen, SGetUserWhit return 0; } -int32_t tSerializeSGetUserWhiteListRsp(void* buf, int32_t bufLen, SGetUserWhiteListRsp* pRsp) { +int32_t tSerializeSGetUserWhiteListRsp(void *buf, int32_t bufLen, SGetUserWhiteListRsp *pRsp) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -2237,7 +2242,7 @@ int32_t tSerializeSGetUserWhiteListRsp(void* buf, int32_t bufLen, SGetUserWhiteL return tlen; } -int32_t tDeserializeSGetUserWhiteListRsp(void* buf, int32_t bufLen, SGetUserWhiteListRsp* pRsp) { +int32_t tDeserializeSGetUserWhiteListRsp(void *buf, int32_t bufLen, SGetUserWhiteListRsp *pRsp) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -2257,9 +2262,7 @@ int32_t tDeserializeSGetUserWhiteListRsp(void* buf, int32_t bufLen, SGetUserWhit return 0; } -void tFreeSGetUserWhiteListRsp(SGetUserWhiteListRsp* pRsp) { - taosMemoryFree(pRsp->pWhiteLists); -} +void tFreeSGetUserWhiteListRsp(SGetUserWhiteListRsp *pRsp) { taosMemoryFree(pRsp->pWhiteLists); } int32_t tSerializeSCreateDropMQSNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) { SEncoder encoder = {0}; @@ -2288,13 +2291,9 @@ int32_t tDeserializeSCreateDropMQSNodeReq(void *buf, int32_t bufLen, SMCreateQno return 0; } -void tFreeSMCreateQnodeReq(SMCreateQnodeReq *pReq){ - FREESQL(); -} +void tFreeSMCreateQnodeReq(SMCreateQnodeReq *pReq) { FREESQL(); } -void tFreeSDDropQnodeReq(SDDropQnodeReq* pReq) { - FREESQL(); -} +void tFreeSDDropQnodeReq(SDDropQnodeReq *pReq) { FREESQL(); } int32_t tSerializeSDropDnodeReq(void *buf, int32_t bufLen, SDropDnodeReq *pReq) { SEncoder encoder = {0}; @@ -2336,9 +2335,7 @@ int32_t tDeserializeSDropDnodeReq(void *buf, int32_t bufLen, SDropDnodeReq *pReq return 0; } -void tFreeSDropDnodeReq(SDropDnodeReq *pReq) { - FREESQL(); -} +void tFreeSDropDnodeReq(SDropDnodeReq *pReq) { FREESQL(); } int32_t tSerializeSRestoreDnodeReq(void *buf, int32_t bufLen, SRestoreDnodeReq *pReq) { SEncoder encoder = {0}; @@ -2369,9 +2366,7 @@ int32_t tDeserializeSRestoreDnodeReq(void *buf, int32_t bufLen, SRestoreDnodeReq return 0; } -void tFreeSRestoreDnodeReq(SRestoreDnodeReq *pReq) { - FREESQL(); -} +void tFreeSRestoreDnodeReq(SRestoreDnodeReq *pReq) { FREESQL(); } int32_t tSerializeSMCfgDnodeReq(void *buf, int32_t bufLen, SMCfgDnodeReq *pReq) { SEncoder encoder = {0}; @@ -2404,9 +2399,7 @@ int32_t tDeserializeSMCfgDnodeReq(void *buf, int32_t bufLen, SMCfgDnodeReq *pReq return 0; } -void tFreeSMCfgDnodeReq(SMCfgDnodeReq *pReq) { - FREESQL(); -} +void tFreeSMCfgDnodeReq(SMCfgDnodeReq *pReq) { FREESQL(); } int32_t tSerializeSDCfgDnodeReq(void *buf, int32_t bufLen, SDCfgDnodeReq *pReq) { SEncoder encoder = {0}; @@ -2464,9 +2457,7 @@ int32_t tDeserializeSCreateDnodeReq(void *buf, int32_t bufLen, SCreateDnodeReq * return 0; } -void tFreeSCreateDnodeReq(SCreateDnodeReq *pReq) { - FREESQL(); -} +void tFreeSCreateDnodeReq(SCreateDnodeReq *pReq) { FREESQL(); } int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq) { SEncoder encoder = {0}; @@ -3121,9 +3112,7 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { return 0; } -void tFreeSAlterDbReq(SAlterDbReq *pReq) { - FREESQL(); -} +void tFreeSAlterDbReq(SAlterDbReq *pReq) { FREESQL(); } int32_t tSerializeSDropDbReq(void *buf, int32_t bufLen, SDropDbReq *pReq) { SEncoder encoder = {0}; @@ -3154,9 +3143,7 @@ int32_t tDeserializeSDropDbReq(void *buf, int32_t bufLen, SDropDbReq *pReq) { return 0; } -void tFreeSDropDbReq(SDropDbReq *pReq) { - FREESQL(); -} +void tFreeSDropDbReq(SDropDbReq *pReq) { FREESQL(); } int32_t tSerializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) { SEncoder encoder = {0}; @@ -3435,9 +3422,7 @@ int32_t tDeserializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq return 0; } -void tFreeSCompactDbReq(SCompactDbReq *pReq) { - FREESQL(); -} +void tFreeSCompactDbReq(SCompactDbReq *pReq) { FREESQL(); } int32_t tSerializeSUseDbRspImp(SEncoder *pEncoder, const SUseDbRsp *pRsp) { if (tEncodeCStr(pEncoder, pRsp->db) < 0) return -1; @@ -4611,9 +4596,7 @@ int32_t tDeserializeSMDropTopicReq(void *buf, int32_t bufLen, SMDropTopicReq *pR return 0; } -void tFreeSMDropTopicReq(SMDropTopicReq *pReq) { - FREESQL(); -} +void tFreeSMDropTopicReq(SMDropTopicReq *pReq) { FREESQL(); } int32_t tSerializeSMDropCgroupReq(void *buf, int32_t bufLen, SMDropCgroupReq *pReq) { SEncoder encoder = {0}; @@ -5501,9 +5484,7 @@ int32_t tDeserializeSBalanceVgroupReq(void *buf, int32_t bufLen, SBalanceVgroupR return 0; } -void tFreeSBalanceVgroupReq(SBalanceVgroupReq *pReq) { - FREESQL(); -} +void tFreeSBalanceVgroupReq(SBalanceVgroupReq *pReq) { FREESQL(); } int32_t tSerializeSBalanceVgroupLeaderReq(void *buf, int32_t bufLen, SBalanceVgroupLeaderReq *pReq) { SEncoder encoder = {0}; @@ -5526,7 +5507,7 @@ int32_t tDeserializeSBalanceVgroupLeaderReq(void *buf, int32_t bufLen, SBalanceV if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1; - if(!tDecodeIsEnd(&decoder)){ + if (!tDecodeIsEnd(&decoder)) { if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; } @@ -5537,9 +5518,7 @@ int32_t tDeserializeSBalanceVgroupLeaderReq(void *buf, int32_t bufLen, SBalanceV return 0; } -void tFreeSBalanceVgroupLeaderReq(SBalanceVgroupLeaderReq *pReq) { - FREESQL(); -} +void tFreeSBalanceVgroupLeaderReq(SBalanceVgroupLeaderReq *pReq) { FREESQL(); } int32_t tSerializeSMergeVgroupReq(void *buf, int32_t bufLen, SMergeVgroupReq *pReq) { SEncoder encoder = {0}; @@ -5601,9 +5580,7 @@ int32_t tDeserializeSRedistributeVgroupReq(void *buf, int32_t bufLen, SRedistrib return 0; } -void tFreeSRedistributeVgroupReq(SRedistributeVgroupReq *pReq) { - FREESQL(); -} +void tFreeSRedistributeVgroupReq(SRedistributeVgroupReq *pReq) { FREESQL(); } int32_t tSerializeSSplitVgroupReq(void *buf, int32_t bufLen, SSplitVgroupReq *pReq) { SEncoder encoder = {0}; @@ -7152,9 +7129,7 @@ int32_t tDeserializeSMDropStreamReq(void *buf, int32_t bufLen, SMDropStreamReq * return 0; } -void tFreeMDropStreamReq(SMDropStreamReq *pReq) { - FREESQL(); -} +void tFreeMDropStreamReq(SMDropStreamReq *pReq) { FREESQL(); } int32_t tSerializeSMRecoverStreamReq(void *buf, int32_t bufLen, const SMRecoverStreamReq *pReq) { SEncoder encoder = {0}; @@ -7297,8 +7272,8 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) { } else { ASSERT(0); } - //ENCODESQL - if(pReq->sqlLen > 0 && pReq->sql != NULL) { + // ENCODESQL + if (pReq->sqlLen > 0 && pReq->sql != NULL) { if (tEncodeI32(pCoder, pReq->sqlLen) < 0) return -1; if (tEncodeBinary(pCoder, pReq->sql, pReq->sqlLen) < 0) return -1; } @@ -7345,11 +7320,11 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) { ASSERT(0); } - //DECODESQL - if(!tDecodeIsEnd(pCoder)){ - if(tDecodeI32(pCoder, &pReq->sqlLen) < 0) return -1; - if(pReq->sqlLen > 0){ - if (tDecodeBinaryAlloc(pCoder, (void**)&pReq->sql, NULL) < 0) return -1; + // DECODESQL + if (!tDecodeIsEnd(pCoder)) { + if (tDecodeI32(pCoder, &pReq->sqlLen) < 0) return -1; + if (pReq->sqlLen > 0) { + if (tDecodeBinaryAlloc(pCoder, (void **)&pReq->sql, NULL) < 0) return -1; } } @@ -7375,7 +7350,7 @@ void tDestroySVCreateTbReq(SVCreateTbReq *pReq, int32_t flags) { } } - if(pReq->sql != NULL){ + if (pReq->sql != NULL) { taosMemoryFree(pReq->sql); } pReq->sql = NULL; @@ -8194,7 +8169,7 @@ int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { for (int32_t i = 0; i < pRsp->blockNum; i++) { int32_t bLen = *(int32_t *)taosArrayGet(pRsp->blockDataLen, i); - void *data = taosArrayGetP(pRsp->blockData, i); + void * data = taosArrayGetP(pRsp->blockData, i); if (tEncodeBinary(pEncoder, (const uint8_t *)data, bLen) < 0) return -1; if (pRsp->withSchema) { SSchemaWrapper *pSW = (SSchemaWrapper *)taosArrayGetP(pRsp->blockSchema, i); @@ -8227,7 +8202,7 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { } for (int32_t i = 0; i < pRsp->blockNum; i++) { - void *data; + void * data; uint64_t bLen; if (tDecodeBinaryAlloc(pDecoder, &data, &bLen) < 0) return -1; taosArrayPush(pRsp->blockData, &data); @@ -8273,7 +8248,7 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) { if (tEncodeI32(pEncoder, pRsp->createTableNum) < 0) return -1; if (pRsp->createTableNum) { for (int32_t i = 0; i < pRsp->createTableNum; i++) { - void *createTableReq = taosArrayGetP(pRsp->createTableReq, i); + void * createTableReq = taosArrayGetP(pRsp->createTableReq, i); int32_t createTableLen = *(int32_t *)taosArrayGet(pRsp->createTableLen, i); if (tEncodeBinary(pEncoder, createTableReq, createTableLen) < 0) return -1; } @@ -8289,7 +8264,7 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) { pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t)); pRsp->createTableReq = taosArrayInit(pRsp->createTableNum, sizeof(void *)); for (int32_t i = 0; i < pRsp->createTableNum; i++) { - void *pCreate = NULL; + void * pCreate = NULL; uint64_t len; if (tDecodeBinaryAlloc(pDecoder, &pCreate, &len) < 0) return -1; int32_t l = (int32_t)len; @@ -8591,7 +8566,7 @@ void tDestroySubmitTbData(SSubmitTbData *pTbData, int32_t flag) { taosArrayDestroy(pTbData->aCol); } else { int32_t nRow = TARRAY_SIZE(pTbData->aRowP); - SRow **rows = (SRow **)TARRAY_DATA(pTbData->aRowP); + SRow ** rows = (SRow **)TARRAY_DATA(pTbData->aRowP); for (int32_t i = 0; i < nRow; ++i) { tRowDestroy(rows[i]); @@ -8855,7 +8830,7 @@ int32_t tDeserializeSCMCreateViewReq(void *buf, int32_t bufLen, SCMCreateViewReq } for (int32_t i = 0; i < pReq->numOfCols; ++i) { - SSchema* pSchema = pReq->pSchema + i; + SSchema *pSchema = pReq->pSchema + i; if (tDecodeSSchema(&decoder, pSchema) < 0) return -1; } } @@ -8866,17 +8841,17 @@ int32_t tDeserializeSCMCreateViewReq(void *buf, int32_t bufLen, SCMCreateViewReq return 0; } -void tFreeSCMCreateViewReq(SCMCreateViewReq* pReq) { +void tFreeSCMCreateViewReq(SCMCreateViewReq *pReq) { if (NULL == pReq) { return; } taosMemoryFreeClear(pReq->querySql); - taosMemoryFreeClear(pReq->sql); + taosMemoryFreeClear(pReq->sql); taosMemoryFreeClear(pReq->pSchema); } -int32_t tSerializeSCMDropViewReq(void* buf, int32_t bufLen, const SCMDropViewReq* pReq) { +int32_t tSerializeSCMDropViewReq(void *buf, int32_t bufLen, const SCMDropViewReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -8894,7 +8869,7 @@ int32_t tSerializeSCMDropViewReq(void* buf, int32_t bufLen, const SCMDropViewReq return tlen; } -int32_t tDeserializeSCMDropViewReq(void* buf, int32_t bufLen, SCMDropViewReq* pReq) { +int32_t tDeserializeSCMDropViewReq(void *buf, int32_t bufLen, SCMDropViewReq *pReq) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -8910,15 +8885,15 @@ int32_t tDeserializeSCMDropViewReq(void* buf, int32_t bufLen, SCMDropViewReq* pR tDecoderClear(&decoder); return 0; } -void tFreeSCMDropViewReq(SCMDropViewReq* pReq) { +void tFreeSCMDropViewReq(SCMDropViewReq *pReq) { if (NULL == pReq) { return; } - + taosMemoryFree(pReq->sql); } -int32_t tSerializeSViewMetaReq(void* buf, int32_t bufLen, const SViewMetaReq* pReq) { +int32_t tSerializeSViewMetaReq(void *buf, int32_t bufLen, const SViewMetaReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -8932,7 +8907,7 @@ int32_t tSerializeSViewMetaReq(void* buf, int32_t bufLen, const SViewMetaReq* pR return tlen; } -int32_t tDeserializeSViewMetaReq(void* buf, int32_t bufLen, SViewMetaReq* pReq) { +int32_t tDeserializeSViewMetaReq(void *buf, int32_t bufLen, SViewMetaReq *pReq) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -8964,8 +8939,7 @@ static int32_t tEncodeSViewMetaRsp(SEncoder *pEncoder, const SViewMetaRsp *pRsp) return 0; } - -int32_t tSerializeSViewMetaRsp(void* buf, int32_t bufLen, const SViewMetaRsp* pRsp) { +int32_t tSerializeSViewMetaRsp(void *buf, int32_t bufLen, const SViewMetaRsp *pRsp) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -8998,7 +8972,7 @@ static int32_t tDecodeSViewMetaRsp(SDecoder *pDecoder, SViewMetaRsp *pRsp) { } for (int32_t i = 0; i < pRsp->numOfCols; ++i) { - SSchema* pSchema = pRsp->pSchema + i; + SSchema *pSchema = pRsp->pSchema + i; if (tDecodeSSchema(pDecoder, pSchema) < 0) return -1; } } @@ -9006,7 +8980,7 @@ static int32_t tDecodeSViewMetaRsp(SDecoder *pDecoder, SViewMetaRsp *pRsp) { return 0; } -int32_t tDeserializeSViewMetaRsp(void* buf, int32_t bufLen, SViewMetaRsp* pRsp) { +int32_t tDeserializeSViewMetaRsp(void *buf, int32_t bufLen, SViewMetaRsp *pRsp) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -9019,7 +8993,7 @@ int32_t tDeserializeSViewMetaRsp(void* buf, int32_t bufLen, SViewMetaRsp* pRsp) return 0; } -void tFreeSViewMetaRsp(SViewMetaRsp* pRsp) { +void tFreeSViewMetaRsp(SViewMetaRsp *pRsp) { if (NULL == pRsp) { return; } @@ -9064,7 +9038,7 @@ int32_t tDeserializeSViewHbRsp(void *buf, int32_t bufLen, SViewHbRsp *pRsp) { } for (int32_t i = 0; i < numOfMeta; ++i) { - SViewMetaRsp* metaRsp = taosMemoryCalloc(1, sizeof(SViewMetaRsp)); + SViewMetaRsp *metaRsp = taosMemoryCalloc(1, sizeof(SViewMetaRsp)); if (NULL == metaRsp) return -1; if (tDecodeSViewMetaRsp(&decoder, metaRsp) < 0) return -1; taosArrayPush(pRsp->pViewRsp, &metaRsp); @@ -9086,7 +9060,3 @@ void tFreeSViewHbRsp(SViewHbRsp *pRsp) { taosArrayDestroy(pRsp->pViewRsp); } - - - - diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 227de7f5fc..ec00fce4dd 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -174,6 +174,10 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) { i += 2; } + if (hour > 12 || hour < 0) { + return -1; + } + // return error if there're illegal charaters after min(2 Digits) char* minStr = &str[i]; if (minStr[1] != '\0' && minStr[2] != '\0') { @@ -181,7 +185,7 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) { } int64_t minute = strnatoi(&str[i], 2); - if (minute > 59) { + if (minute > 59 || (hour == 12 && minute > 0)) { return -1; } @@ -646,7 +650,7 @@ int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* dura /* get the basic numeric value */ int64_t timestamp = taosStr2Int64(token, &endPtr, 10); - if (timestamp < 0 || errno != 0) { + if ((timestamp == 0 && token[0] != '0') || errno != 0) { return -1; } diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 5f2796260c..dd5e8240b9 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -19,6 +19,278 @@ #include "ttokendef.h" #include "tvariant.h" +int32_t parseBinaryUInteger(const char *z, int32_t n, uint64_t *value) { + // skip head 0b + const char *p = z + 2; + int32_t l = n - 2; + + while (*p == '0' && l > 0) { + p++; + l--; + } + if (l > 64) { // too big + return TSDB_CODE_FAILED; + } + + uint64_t val = 0; + for (int32_t i = 0; i < l; i++) { + val = val << 1; + if (p[i] == '1') { + val |= 1; + } else if (p[i] != '0') { // abnormal char + return TSDB_CODE_FAILED; + } + } + *value = val; + return TSDB_CODE_SUCCESS; +} + +int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *value) { + // parse sign + bool has_sign = false; + if (z[0] == '-') { + *is_neg = true; + has_sign = true; + } else if (z[0] == '+') { + has_sign = true; + } else if (z[0] != '.' && (z[0] < '0' || z[0] > '9')) { + return TSDB_CODE_FAILED; + } + if (has_sign) { + if (n < 2) { + return TSDB_CODE_FAILED; + } + z++; + n--; + } + + errno = 0; + char *endPtr = NULL; + bool parsed = false; + if (z[0] == '0' && n > 2) { + if (z[1] == 'b' || z[1] == 'B') { + // paring as binary + return parseBinaryUInteger(z, n, value); + } + + if (z[1] == 'x' || z[1] == 'X') { + // parsing as hex + *value = taosStr2UInt64(z, &endPtr, 16); + parsed = true; + } + } + + if (!parsed) { + // parsing as double + double val = taosStr2Double(z, &endPtr); + if (val > UINT64_MAX) { + return TSDB_CODE_FAILED; + } + *value = round(val); + } + + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + + +int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) { + if (n == 0) { + *value = 0; + return TSDB_CODE_SUCCESS; + } + + errno = 0; + char* endPtr = NULL; + *value = taosStr2Double(z, &endPtr); + + if (errno == ERANGE || errno == EINVAL) { + return TSDB_CODE_FAILED; + } + + if (endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { + errno = 0; + char *endPtr = NULL; + switch (type) + { + case TK_NK_INTEGER: { + *value = taosStr2Int64(z, &endPtr, 10); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } break; + case TK_NK_FLOAT: { + double val = round(taosStr2Double(z, &endPtr)); + if (!IS_VALID_INT64(val)) { + return TSDB_CODE_FAILED; + } + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + *value = val; + return TSDB_CODE_SUCCESS; + } break; + default: + break; + } + + if (n == 0) { + *value = 0; + return TSDB_CODE_SUCCESS; + } + + // 1. try to parse as integer + *value = taosStr2Int64(z, &endPtr, 10); + if (endPtr - z == n) { + if (errno == ERANGE || errno == EINVAL) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } else if (errno == 0 && *endPtr == '.') { + // pure decimal part + const char *s = endPtr + 1; + const char *end = z + n; + bool pure = true; + while (s < end) { + if (*s < '0' || *s > '9') { + pure = false; + break; + } + s++; + } + if (pure) { + if (endPtr+1 < end && endPtr[1] > '4') { + const char *p = z; + while (*p == ' ') { + p++; + } + if (*p == '-') { + if ( *value > INT64_MIN) { + (*value)--; + } + } else { + if ( *value < INT64_MAX) { + (*value)++; + } + } + } + return TSDB_CODE_SUCCESS; + } + } + + // 2. parse as other + bool is_neg = false; + uint64_t uv = 0; + int32_t code = parseSignAndUInteger(z, n, &is_neg, &uv); + if (code == TSDB_CODE_SUCCESS) { + // truncate into int64 + if (is_neg) { + if (uv > 1ull + INT64_MAX) { + *value = INT64_MIN; + return TSDB_CODE_FAILED; + } else { + *value = -uv; + } + } else { + if (uv > INT64_MAX) { + *value = INT64_MAX; + return TSDB_CODE_FAILED; + } + *value = uv; + } + } + + return code; +} + +int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { + errno = 0; + char *endPtr = NULL; + const char *p = z; + while (*p == ' ') { + p++; + } + switch (type) { + case TK_NK_INTEGER: { + *value = taosStr2UInt64(p, &endPtr, 10); + if (*p == '-' && *value) { + return TSDB_CODE_FAILED; + } + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } break; + case TK_NK_FLOAT: { + double val = round(taosStr2Double(p, &endPtr)); + if (!IS_VALID_UINT64(val)) { + return TSDB_CODE_FAILED; + } + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + *value = val; + return TSDB_CODE_SUCCESS; + } break; + default: + break; + } + + if (n == 0) { + *value = 0; + return TSDB_CODE_SUCCESS; + } + + // 1. parse as integer + *value = taosStr2UInt64(p, &endPtr, 10); + if (*p == '-' && *value) { + return TSDB_CODE_FAILED; + } + if (endPtr - z == n) { + if (errno == ERANGE || errno == EINVAL) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } else if (errno == 0 && *endPtr == '.') { + const char *s = endPtr + 1; + const char *end = z + n; + bool pure = true; + while (s < end) { + if (*s < '0' || *s > '9') { + pure = false; + break; + } + s++; + } + if (pure) { + if (endPtr + 1 < end && endPtr[1] > '4' && *value < UINT64_MAX) { + (*value)++; + } + return TSDB_CODE_SUCCESS; + } + } + + // 2. parse as other + bool is_neg = false; + int32_t code = parseSignAndUInteger(z, n, &is_neg, value); + if (is_neg) { + if (TSDB_CODE_SUCCESS == code && 0 == *value) { + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_FAILED; + } + return code; +} + int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value) { errno = 0; char *endPtr = NULL; @@ -26,10 +298,10 @@ int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value) { *value = taosStr2Int64(z, &endPtr, base); if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { errno = 0; - return -1; + return TSDB_CODE_FAILED; } - return 0; + return TSDB_CODE_SUCCESS; } int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value) { @@ -39,16 +311,15 @@ int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value) { const char *p = z; while (*p == ' ') p++; if (*p == '-') { - return -1; + return TSDB_CODE_FAILED; } *value = taosStr2UInt64(z, &endPtr, base); if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { errno = 0; - return -1; + return TSDB_CODE_FAILED; } - - return 0; + return TSDB_CODE_SUCCESS; } /** @@ -147,14 +418,13 @@ void taosVariantDestroy(SVariant *pVar) { taosMemoryFreeClear(pVar->pz); pVar->nLen = 0; } - } void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { if (pSrc == NULL || pDst == NULL) return; pDst->nType = pSrc->nType; - if (pSrc->nType == TSDB_DATA_TYPE_BINARY ||pSrc->nType == TSDB_DATA_TYPE_VARBINARY || + if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_VARBINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR || pSrc->nType == TSDB_DATA_TYPE_JSON || pSrc->nType == TSDB_DATA_TYPE_GEOMETRY) { int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE; @@ -172,7 +442,6 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { if (IS_NUMERIC_TYPE(pSrc->nType) || (pSrc->nType == TSDB_DATA_TYPE_BOOL)) { pDst->i = pSrc->i; } - } int32_t taosVariantCompare(const SVariant *p1, const SVariant *p2) { diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 107276d7f9..9f7ee165ac 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -14,6 +14,7 @@ #include "tdef.h" #include "tvariant.h" #include "ttime.h" +#include "ttokendef.h" namespace { // @@ -24,40 +25,177 @@ int main(int argc, char** argv) { return RUN_ALL_TESTS(); } -TEST(testCase, toInteger_test) { + +TEST(testCase, toUIntegerEx_test) { + uint64_t val = 0; + char* s = "123"; - uint32_t type = 0; - - int64_t val = 0; - - int32_t ret = toInteger(s, strlen(s), 10, &val); + int32_t ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 123); + s = "1000u"; + ret = toUIntegerEx(s, strlen(s), 0, &val); + ASSERT_EQ(ret, -1); + + s = "0x1f"; + ret = toUIntegerEx(s, strlen(s), TK_NK_HEX, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 31); + + s = "0b110"; + ret = toUIntegerEx(s, strlen(s), 0, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 6); + + s = "2567.4787"; + ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 2567); + + s = "1.869895343e4"; + ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 18699); + + s = "-1"; + ret = toUIntegerEx(s, strlen(s),TK_NK_INTEGER, &val); + ASSERT_EQ(ret, -1); + + s = "-0b10010"; + ret = toUIntegerEx(s, strlen(s), 0, &val); + ASSERT_EQ(ret, -1); + + s = "-0x40"; + ret = toUIntegerEx(s, strlen(s), TK_NK_HEX, &val); + ASSERT_EQ(ret, -1); + + s = "-80.9999"; + ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, -1); + + s = "-5.2343544534e10"; + ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, -1); + + // INT64_MAX s = "9223372036854775807"; - ret = toInteger(s, strlen(s), 10, &val); + ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 9223372036854775807); - s = "9323372036854775807"; - ret = toInteger(s, strlen(s), 10, &val); + // UINT64_MAX + s = "18446744073709551615"; + ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 9323372036854775807u); + ASSERT_EQ(val, 18446744073709551615u); + + // out of range + s = "18446744073709551616"; + ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, -1); + + s = "5.23e25"; + ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, -1); +} + +TEST(testCase, toIntegerEx_test) { + int64_t val = 0; + + char* s = "123"; + int32_t ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 123); + + s = "-1"; + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -1); + + s = "1000u"; + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, -1); + + s = "0x1f"; + ret = toIntegerEx(s, strlen(s), TK_NK_HEX, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 31); + + s = "-0x40"; + ret = toIntegerEx(s, strlen(s), TK_NK_HEX, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -64); + + s = "0b110"; + ret = toIntegerEx(s, strlen(s), TK_NK_BIN, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 6); + + s = "-0b10010"; + ret = toIntegerEx(s, strlen(s), 0, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -18); + + s = "-80.9999"; + ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -81); + + s = "2567.8787"; + ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 2568); + + s = "-5.2343544534e10"; + ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -52343544534); + + s = "1.869895343e4"; + ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 18699); + + // INT64_MAX + s = "9223372036854775807"; + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 9223372036854775807LL); + + s = "-9223372036854775808"; + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -9223372036854775808); + + // out of range + s = "9323372036854775807"; + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, -1); s = "-9323372036854775807"; - ret = toInteger(s, strlen(s), 10, &val); + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); ASSERT_EQ(ret, -1); + // UINT64_MAX + s = "18446744073709551615"; + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, -1); +} + +TEST(testCase, toInteger_test) { + int64_t val = 0; + + char* s = "123"; + int32_t ret = toInteger(s, strlen(s), 10, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 123); + s = "-1"; ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, -1); - s = "-9223372036854775807"; - ret = toInteger(s, strlen(s), 10, &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, -9223372036854775807); - s = "1000u"; ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, -1); @@ -77,13 +215,22 @@ TEST(testCase, toInteger_test) { ASSERT_EQ(ret, 0); ASSERT_EQ(val, 72); - // 18446744073709551615 UINT64_MAX - s = "18446744073709551615"; + s = "9223372036854775807"; ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 18446744073709551615u); + ASSERT_EQ(val, 9223372036854775807); - s = "18446744073709551616"; + s = "-9223372036854775808"; + ret = toInteger(s, strlen(s), 10, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -9223372036854775808); + + // out of range + s = "9323372036854775807"; + ret = toInteger(s, strlen(s), 10, &val); + ASSERT_EQ(ret, -1); + + s = "-9323372036854775807"; ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, -1); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 6f13abcebc..e0503c83c6 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -28,6 +28,9 @@ } \ } while (0) +extern int32_t streamTimerInit(); +extern void streamTimerCleanUp(); + static SDnode globalDnode = {0}; SDnode *dmInstance() { return &globalDnode; } @@ -166,6 +169,7 @@ int32_t dmInit() { #if defined(USE_S3) if (s3Begin() != 0) return -1; #endif + if (streamTimerInit() != 0) return -1; dInfo("dnode env is initialized"); return 0; @@ -194,6 +198,8 @@ void dmCleanup() { #if defined(USE_S3) s3End(); #endif + streamTimerCleanUp(); + dInfo("dnode env is cleaned up"); taosCleanupCfg(); diff --git a/source/dnode/mnode/impl/inc/mndCluster.h b/source/dnode/mnode/impl/inc/mndCluster.h index 2b59d9dbf5..e33ffdb372 100644 --- a/source/dnode/mnode/impl/inc/mndCluster.h +++ b/source/dnode/mnode/impl/inc/mndCluster.h @@ -27,8 +27,6 @@ void mndCleanupCluster(SMnode *pMnode); int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len); int64_t mndGetClusterId(SMnode *pMnode); int64_t mndGetClusterCreateTime(SMnode *pMnode); -int32_t mndGetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); -int32_t mndSetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); int64_t mndGetClusterUpTime(SMnode *pMnode); #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 27fef1e81e..08c0aec46a 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -192,8 +192,6 @@ typedef struct { int64_t createdTime; int64_t updateTime; int32_t upTime; - int64_t grantedTime; - int64_t connGrantedTime; } SClusterObj; typedef struct { @@ -700,6 +698,11 @@ typedef struct { } SStreamObj; +typedef struct SStreamSeq { + char name[24]; + uint64_t seq; + SRWLatch lock; +} SStreamSeq; int32_t tEncodeSStreamObj(SEncoder* pEncoder, const SStreamObj* pObj); int32_t tDecodeSStreamObj(SDecoder* pDecoder, SStreamObj* pObj, int32_t sver); void tFreeStreamObj(SStreamObj* pObj); @@ -731,14 +734,13 @@ typedef struct { int8_t type; int32_t numOfCols; SSchema* pSchema; - SRWLatch lock; + SRWLatch lock; } SViewObj; int32_t tEncodeSViewObj(SEncoder* pEncoder, const SViewObj* pObj); int32_t tDecodeSViewObj(SDecoder* pDecoder, SViewObj* pObj, int32_t sver); void tFreeSViewObj(SViewObj* pObj); - #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 244a6d08dd..e3ac4fd6fc 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -28,18 +28,28 @@ typedef struct SStreamTransInfo { const char *name; } SStreamTransInfo; +// time to generated the checkpoint, if now() - checkpointTs >= tsCheckpointInterval, this checkpoint will be discard +// to avoid too many checkpoints for a taskk in the waiting list +typedef struct SCheckpointCandEntry { + char * pName; + int64_t streamId; + int64_t checkpointTs; + int64_t checkpointId; +} SCheckpointCandEntry; + typedef struct SStreamTransMgmt { SHashObj *pDBTrans; + SHashObj *pWaitingList; // stream id list, of which timed checkpoint failed to be issued due to the trans conflict. } SStreamTransMgmt; typedef struct SStreamExecInfo { - SArray *pNodeList; - int64_t ts; // snapshot ts - SStreamTransMgmt transMgmt; - int64_t activeCheckpoint; // active check point id - SHashObj * pTaskMap; - SArray * pTaskList; - TdThreadMutex lock; + SArray * pNodeList; + int64_t ts; // snapshot ts + SStreamTransMgmt transMgmt; + int64_t activeCheckpoint; // active check point id + SHashObj * pTaskMap; + SArray * pTaskList; + TdThreadMutex lock; } SStreamExecInfo; extern SStreamExecInfo execInfo; @@ -51,8 +61,9 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream); int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); -int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pName, const char* pSrcDb, const char* pDstDb); -bool streamTransConflictOtherTrans(SMnode *pMnode, const char *pSrcDb, const char *pDstDb); +int32_t mndStreamRegisterTrans(STrans *pTrans, const char *pName, const char *pSrcDb, const char *pDstDb); +int32_t mndAddtoCheckpointWaitingList(SStreamObj *pStream, int64_t checkpointId); +bool streamTransConflictOtherTrans(SMnode *pMnode, const char *pSrcDb, const char *pDstDb, bool lock); // for sma // TODO refactor diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 26c678b513..4c799e1e1e 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -19,7 +19,7 @@ #include "mndTrans.h" #define CLUSTER_VER_NUMBE 1 -#define CLUSTER_RESERVE_SIZE 44 +#define CLUSTER_RESERVE_SIZE 60 int64_t tsExpireTime = 0; static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster); @@ -112,19 +112,6 @@ int64_t mndGetClusterCreateTime(SMnode *pMnode) { return createTime; } -int32_t mndGetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo) { - void *pIter = NULL; - SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); - if (pCluster != NULL) { - pInfo->grantedTime = pCluster->grantedTime; - pInfo->connGrantedTime = pCluster->connGrantedTime; - mndReleaseCluster(pMnode, pCluster, pIter); - return 0; - } - - return -1; -} - static int32_t mndGetClusterUpTimeImp(SClusterObj *pCluster) { #if 0 int32_t upTime = taosGetTimestampSec() - pCluster->updateTime / 1000; @@ -159,8 +146,6 @@ static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { SDB_SET_INT64(pRaw, dataPos, pCluster->updateTime, _OVER) SDB_SET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN, _OVER) SDB_SET_INT32(pRaw, dataPos, pCluster->upTime, _OVER) - SDB_SET_INT64(pRaw, dataPos, pCluster->grantedTime, _OVER) - SDB_SET_INT64(pRaw, dataPos, pCluster->connGrantedTime, _OVER) SDB_SET_RESERVE(pRaw, dataPos, CLUSTER_RESERVE_SIZE, _OVER) terrno = 0; @@ -201,8 +186,6 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pCluster->updateTime, _OVER) SDB_GET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN, _OVER) SDB_GET_INT32(pRaw, dataPos, &pCluster->upTime, _OVER) - SDB_GET_INT64(pRaw, dataPos, &pCluster->grantedTime, _OVER); - SDB_GET_INT64(pRaw, dataPos, &pCluster->connGrantedTime, _OVER); SDB_GET_RESERVE(pRaw, dataPos, CLUSTER_RESERVE_SIZE, _OVER) terrno = 0; @@ -235,8 +218,6 @@ static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pOld, SClusterObj mTrace("cluster:%" PRId64 ", perform update action, old row:%p new row:%p, uptime from %d to %d", pOld->id, pOld, pNew, pOld->upTime, pNew->upTime); pOld->upTime = pNew->upTime; - pOld->grantedTime = pNew->grantedTime; - pOld->connGrantedTime = pNew->connGrantedTime; pOld->updateTime = taosGetTimestampMs(); return 0; } @@ -378,44 +359,3 @@ static int32_t mndProcessUptimeTimer(SRpcMsg *pReq) { mndTransDrop(pTrans); return 0; } - -int32_t mndSetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo) { - SClusterObj clusterObj = {0}; - void *pIter = NULL; - SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); - if (pCluster != NULL) { - if (pCluster->grantedTime >= pInfo->grantedTime && pCluster->connGrantedTime >= pInfo->connGrantedTime) { - mndReleaseCluster(pMnode, pCluster, pIter); - return 0; - } - memcpy(&clusterObj, pCluster, sizeof(SClusterObj)); - if (pCluster->grantedTime < pInfo->grantedTime) clusterObj.grantedTime = pInfo->grantedTime; - if (pCluster->connGrantedTime < pInfo->connGrantedTime) clusterObj.connGrantedTime = pInfo->connGrantedTime; - mndReleaseCluster(pMnode, pCluster, pIter); - } - - if (clusterObj.id <= 0) { - mError("can't get cluster info while update granted info"); - return -1; - } - - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "granted-info"); - if (pTrans == NULL) return -1; - - SSdbRaw *pCommitRaw = mndClusterActionEncode(&clusterObj); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - - mndTransDrop(pTrans); - return 0; -} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index b0bffcc83e..e224aceec2 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -790,9 +790,7 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg if (cfgAll) { // alter all dnodes: if (!failRecord) failRecord = taosArrayInit(1, sizeof(int32_t)); if (failRecord) taosArrayPush(failRecord, &pDnode->id); - if (0 == cfgAllErr || cfgAllErr == TSDB_CODE_GRANT_PAR_IVLD_ACTIVE) { - cfgAllErr = terrno; // output 1st or more specific error - } + if (0 == cfgAllErr) cfgAllErr = terrno; // output 1st terrno. } } else { terrno = 0; // no action for dup active code @@ -808,9 +806,7 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg if (cfgAll) { if (!failRecord) failRecord = taosArrayInit(1, sizeof(int32_t)); if (failRecord) taosArrayPush(failRecord, &pDnode->id); - if (0 == cfgAllErr || cfgAllErr == TSDB_CODE_GRANT_PAR_IVLD_ACTIVE) { - cfgAllErr = terrno; // output 1st or more specific error - } + if (0 == cfgAllErr) cfgAllErr = terrno; } } else { terrno = 0; @@ -1287,12 +1283,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { strcpy(dcfgReq.config, "supportvnodes"); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); - } else if (strncasecmp(cfgReq.config, GRANT_ACTIVE_CODE, 10) == 0 || - strncasecmp(cfgReq.config, GRANT_C_ACTIVE_CODE, 11) == 0) { - if (cfgReq.dnodeId != -1) { - terrno = TSDB_CODE_INVALID_CFG; - goto _err_out; - } + } else if (strncasecmp(cfgReq.config, "activeCode", 10) == 0 || strncasecmp(cfgReq.config, "cActiveCode", 11) == 0) { int8_t opt = strncasecmp(cfgReq.config, "a", 1) == 0 ? DND_ACTIVE_CODE : DND_CONN_ACTIVE_CODE; int8_t index = opt == DND_ACTIVE_CODE ? 10 : 11; if (' ' != cfgReq.config[index] && 0 != cfgReq.config[index]) { @@ -1310,11 +1301,12 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { goto _err_out; } - strcpy(dcfgReq.config, opt == DND_ACTIVE_CODE ? GRANT_ACTIVE_CODE : GRANT_C_ACTIVE_CODE); + strcpy(dcfgReq.config, opt == DND_ACTIVE_CODE ? "activeCode" : "cActiveCode"); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%s", cfgReq.value); - if ((terrno = mndConfigDnode(pMnode, pReq, &cfgReq, opt)) != 0) { + if (mndConfigDnode(pMnode, pReq, &cfgReq, opt) != 0) { mError("dnode:%d, failed to config activeCode since %s", cfgReq.dnodeId, terrstr()); + terrno = TSDB_CODE_INVALID_CFG; goto _err_out; } tFreeSMCfgDnodeReq(&cfgReq); diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 8d00dfefb6..fdfec610d9 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -146,6 +146,15 @@ static void mndStreamCheckpointTick(SMnode *pMnode, int64_t sec) { } } +static void mndStreamCheckpointRemain(SMnode* pMnode) { + int32_t contLen = 0; + void *pReq = mndBuildCheckpointTickMsg(&contLen, 0); + if (pReq != NULL) { + SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE, .pCont = pReq, .contLen = contLen}; + tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); + } +} + static void mndStreamCheckNode(SMnode* pMnode) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); @@ -286,6 +295,10 @@ static void *mndThreadFp(void *param) { mndStreamCheckpointTick(pMnode, sec); } + if (sec % 5 == 0) { + mndStreamCheckpointRemain(pMnode); + } + if (sec % tsStreamNodeCheckInterval == 0) { mndStreamCheckNode(pMnode); } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index f4f9cbb535..1275ba7962 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -291,12 +291,17 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { goto _OVER; } - pObj = mndAcquireSnode(pMnode, createReq.dnodeId); - if (pObj != NULL) { +// pObj = mndAcquireSnode(pMnode, createReq.dnodeId); +// if (pObj != NULL) { +// terrno = TSDB_CODE_MND_SNODE_ALREADY_EXIST; +// goto _OVER; +// } else if (terrno != TSDB_CODE_MND_SNODE_NOT_EXIST) { +// goto _OVER; +// } + + if (sdbGetSize(pMnode->pSdb, SDB_SNODE) >= 1){ terrno = TSDB_CODE_MND_SNODE_ALREADY_EXIST; goto _OVER; - } else if (terrno != TSDB_CODE_MND_SNODE_NOT_EXIST) { - goto _OVER; } pDnode = mndAcquireDnode(pMnode, createReq.dnodeId); @@ -314,7 +319,7 @@ _OVER: return -1; } - mndReleaseSnode(pMnode, pObj); +// mndReleaseSnode(pMnode, pObj); mndReleaseDnode(pMnode, pDnode); tFreeSMCreateQnodeReq(&createReq); return code; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index b424777707..e8d5dfd1f5 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -18,23 +18,21 @@ #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" -#include "mndSnode.h" #include "mndPrivilege.h" #include "mndScheduler.h" #include "mndShow.h" +#include "mndSnode.h" #include "mndStb.h" -#include "mndTopic.h" #include "mndTrans.h" -#include "mndUser.h" #include "mndVgroup.h" #include "osMemory.h" #include "parser.h" #include "tmisce.h" #include "tname.h" -#define MND_STREAM_VER_NUMBER 4 -#define MND_STREAM_RESERVE_SIZE 64 -#define MND_STREAM_MAX_NUM 60 +#define MND_STREAM_VER_NUMBER 4 +#define MND_STREAM_RESERVE_SIZE 64 +#define MND_STREAM_MAX_NUM 60 #define MND_STREAM_CHECKPOINT_NAME "stream-checkpoint" #define MND_STREAM_PAUSE_NAME "stream-pause" @@ -65,6 +63,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq); static int32_t mndProcessDropStreamReq(SRpcMsg *pReq); static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq); static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq); +static int32_t mndProcessStreamCheckpointInCandid(SRpcMsg *pReq); static int32_t mndProcessStreamHb(SRpcMsg *pReq); static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextStream(SMnode *pMnode, void *pIter); @@ -83,7 +82,7 @@ static SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId); static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList); -static STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const char *name, const char* pMsg); +static STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const char *name, const char *pMsg); static int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans); static void initTransAction(STransAction *pAction, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset, int32_t retryCode); @@ -91,12 +90,19 @@ static int32_t createStreamUpdateTrans(SStreamObj *pStream, SVgroupChangeInfo *p static void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); static void saveStreamTasksInfo(SStreamObj *pStream, SStreamExecInfo *pExecNode); static int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot); -static int32_t killActiveCheckpointTrans(SMnode *pMnode, const char* pDbName, size_t len); +static int32_t killActiveCheckpointTrans(SMnode *pMnode, const char *pDbName, size_t len); static int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList); +static void freeCheckpointCandEntry(void *); static SSdbRaw *mndStreamActionEncode(SStreamObj *pStream); static SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw); +SSdbRaw * mndStreamSeqActionEncode(SStreamObj *pStream); +SSdbRow * mndStreamSeqActionDecode(SSdbRaw *pRaw); +static int32_t mndStreamSeqActionInsert(SSdb *pSdb, SStreamSeq *pStream); +static int32_t mndStreamSeqActionDelete(SSdb *pSdb, SStreamSeq *pStream); +static int32_t mndStreamSeqActionUpdate(SSdb *pSdb, SStreamSeq *pOldStream, SStreamSeq *pNewStream); + int32_t mndInitStream(SMnode *pMnode) { SSdbTable table = { .sdbType = SDB_STREAM, @@ -107,6 +113,15 @@ int32_t mndInitStream(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndStreamActionUpdate, .deleteFp = (SdbDeleteFp)mndStreamActionDelete, }; + SSdbTable tableSeq = { + .sdbType = SDB_STREAM_SEQ, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndStreamSeqActionEncode, + .decodeFp = (SdbDecodeFp)mndStreamSeqActionDecode, + .insertFp = (SdbInsertFp)mndStreamSeqActionInsert, + .updateFp = (SdbUpdateFp)mndStreamSeqActionUpdate, + .deleteFp = (SdbDeleteFp)mndStreamSeqActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STREAM, mndProcessCreateStreamReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq); @@ -123,6 +138,7 @@ int32_t mndInitStream(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_VND_STREAM_CHECK_POINT_SOURCE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHECKPOINT_TIMER, mndProcessStreamCheckpointTmr); mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamDoCheckpoint); + mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE, mndProcessStreamCheckpointInCandid); mndSetMsgHandle(pMnode, TDMT_MND_STREAM_HEARTBEAT, mndProcessStreamHb); mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_REPORT_CHECKPOINT, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_STREAM_NODECHANGE_CHECK, mndProcessNodeCheckReq); @@ -141,14 +157,23 @@ int32_t mndInitStream(SMnode *pMnode) { execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId)); execInfo.pTaskMap = taosHashInit(64, fn, true, HASH_NO_LOCK); execInfo.transMgmt.pDBTrans = taosHashInit(32, fn, true, HASH_NO_LOCK); + execInfo.transMgmt.pWaitingList = taosHashInit(32, fn, true, HASH_NO_LOCK); + taosHashSetFreeFp(execInfo.transMgmt.pWaitingList, freeCheckpointCandEntry); - return sdbSetTable(pMnode->pSdb, table); + if (sdbSetTable(pMnode->pSdb, table) != 0) { + return -1; + } + if (sdbSetTable(pMnode->pSdb, tableSeq) != 0) { + return -1; + } + return 0; } void mndCleanupStream(SMnode *pMnode) { taosArrayDestroy(execInfo.pTaskList); taosHashCleanup(execInfo.pTaskMap); taosHashCleanup(execInfo.transMgmt.pDBTrans); + taosHashCleanup(execInfo.transMgmt.pWaitingList); taosThreadMutexDestroy(&execInfo.lock); mDebug("mnd stream exec info cleanup"); } @@ -195,7 +220,8 @@ STREAM_ENCODE_OVER: return NULL; } - mTrace("stream:%s, encode to raw:%p, row:%p", pStream->name, pRaw, pStream); + mTrace("stream:%s, encode to raw:%p, row:%p, checkpoint:%" PRId64 "", pStream->name, pRaw, pStream, + pStream->checkpointId); return pRaw; } @@ -248,7 +274,8 @@ STREAM_DECODE_OVER: return NULL; } - mTrace("stream:%s, decode from raw:%p, row:%p", pStream->name, pRaw, pStream); + mTrace("stream:%s, decode from raw:%p, row:%p, checkpoint:%" PRId64 "", pStream->name, pRaw, pStream, + pStream->checkpointId); return pRow; } @@ -274,6 +301,8 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStream pOldStream->status = pNewStream->status; pOldStream->updateTime = pNewStream->updateTime; + pOldStream->checkpointId = pNewStream->checkpointId; + pOldStream->checkpointFreq = pNewStream->checkpointFreq; taosWUnLockLatch(&pOldStream->lock); return 0; @@ -308,6 +337,12 @@ static void mndShowStreamStatus(char *dst, SStreamObj *pStream) { } } +SSdbRaw *mndStreamSeqActionEncode(SStreamObj *pStream) { return NULL; } +SSdbRow *mndStreamSeqActionDecode(SSdbRaw *pRaw) { return NULL; } +int32_t mndStreamSeqActionInsert(SSdb *pSdb, SStreamSeq *pStream) { return 0; } +int32_t mndStreamSeqActionDelete(SSdb *pSdb, SStreamSeq *pStream) { return 0; } +int32_t mndStreamSeqActionUpdate(SSdb *pSdb, SStreamSeq *pOldStream, SStreamSeq *pNewStream) { return 0; } + static void mndShowStreamTrigger(char *dst, SStreamObj *pStream) { int8_t trigger = pStream->conf.trigger; if (trigger == STREAM_TRIGGER_AT_ONCE) { @@ -657,7 +692,7 @@ _OVER: return -1; } -static int32_t mndPersistTaskDropReq(SMnode* pMnode, STrans *pTrans, SStreamTask *pTask) { +static int32_t mndPersistTaskDropReq(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) { SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq)); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -671,8 +706,17 @@ static int32_t mndPersistTaskDropReq(SMnode* pMnode, STrans *pTrans, SStreamTask STransAction action = {0}; SEpSet epset = {0}; if(pTask->info.nodeId == SNODE_HANDLE){ - SSnodeObj* pObj = mndAcquireSnode(pMnode, pTask->info.nodeId); - addEpIntoEpSet(&epset, pObj->pDnode->fqdn, pObj->pDnode->port); + SSnodeObj *pObj = NULL; + void *pIter = NULL; + while (1) { + pIter = sdbFetch(pMnode->pSdb, SDB_SNODE, pIter, (void **)&pObj); + if (pIter == NULL) { + break; + } + + addEpIntoEpSet(&epset, pObj->pDnode->fqdn, pObj->pDnode->port); + sdbRelease(pMnode->pSdb, pObj); + } }else{ SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); epset = mndGetVgroupEpset(pMnode, pVgObj); @@ -707,7 +751,7 @@ int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) static int32_t checkForNumOfStreams(SMnode *pMnode, SStreamObj *pStreamObj) { // check for number of existed tasks int32_t numOfStream = 0; SStreamObj *pStream = NULL; - void *pIter = NULL; + void * pIter = NULL; while ((pIter = sdbFetch(pMnode->pSdb, SDB_STREAM, pIter, (void **)&pStream)) != NULL) { if (pStream->sourceDbUid == pStreamObj->sourceDbUid) { @@ -719,13 +763,15 @@ static int32_t checkForNumOfStreams(SMnode *pMnode, SStreamObj *pStreamObj) { / if (numOfStream > MND_STREAM_MAX_NUM) { mError("too many streams, no more than %d for each database", MND_STREAM_MAX_NUM); sdbCancelFetch(pMnode->pSdb, pIter); - return TSDB_CODE_MND_TOO_MANY_STREAMS; + terrno = TSDB_CODE_MND_TOO_MANY_STREAMS; + return terrno; } if (pStream->targetStbUid == pStreamObj->targetStbUid) { mError("Cannot write the same stable as other stream:%s", pStream->name); sdbCancelFetch(pMnode->pSdb, pIter); - return TSDB_CODE_MND_INVALID_TARGET_TABLE; + terrno = TSDB_CODE_MND_TOO_MANY_STREAMS; + return terrno; } } @@ -733,14 +779,14 @@ static int32_t checkForNumOfStreams(SMnode *pMnode, SStreamObj *pStreamObj) { / } static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { - SMnode * pMnode = pReq->info.node; - int32_t code = -1; - SStreamObj * pStream = NULL; - SDbObj * pDb = NULL; - SCMCreateStreamReq createStreamReq = {0}; - SStreamObj streamObj = {0}; - char* sql = NULL; + SMnode *pMnode = pReq->info.node; + SStreamObj *pStream = NULL; + SStreamObj streamObj = {0}; + char * sql = NULL; + int32_t sqlLen = 0; + terrno = TSDB_CODE_SUCCESS; + SCMCreateStreamReq createStreamReq = {0}; if (tDeserializeSCMCreateStreamReq(pReq->pCont, pReq->contLen, &createStreamReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; @@ -761,7 +807,6 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { if (pStream != NULL) { if (createStreamReq.igExists) { mInfo("stream:%s, already exist, ignore exist is set", createStreamReq.name); - code = 0; goto _OVER; } else { terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST; @@ -771,8 +816,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } - int32_t sqlLen = 0; - if(createStreamReq.sql != NULL){ + if (createStreamReq.sql != NULL) { sqlLen = strlen(createStreamReq.sql); sql = taosMemoryMalloc(sqlLen + 1); memset(sql, 0, sqlLen + 1); @@ -785,8 +829,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } - code = checkForNumOfStreams(pMnode, &streamObj); - if (code != TSDB_CODE_SUCCESS) { + if (checkForNumOfStreams(pMnode, &streamObj) < 0) { goto _OVER; } @@ -849,8 +892,6 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { saveStreamTasksInfo(&streamObj, &execInfo); taosThreadMutexUnlock(&execInfo.lock); - code = TSDB_CODE_ACTION_IN_PROGRESS; - SName dbname = {0}; tNameFromString(&dbname, createStreamReq.sourceDB, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); @@ -859,26 +900,26 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { // reuse this function for stream if (sql != NULL && sqlLen > 0) { - auditRecord(pReq, pMnode->clusterId, "createStream", dbname.dbname, name.dbname, sql, - sqlLen); - } - else{ + auditRecord(pReq, pMnode->clusterId, "createStream", dbname.dbname, name.dbname, sql, sqlLen); + } else { char detail[1000] = {0}; sprintf(detail, "dbname:%s, stream name:%s", dbname.dbname, name.dbname); auditRecord(pReq, pMnode->clusterId, "createStream", dbname.dbname, name.dbname, detail, strlen(detail)); } + _OVER: - if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { + if (terrno != TSDB_CODE_SUCCESS && terrno != TSDB_CODE_ACTION_IN_PROGRESS) { mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr()); } mndReleaseStream(pMnode, pStream); tFreeSCMCreateStreamReq(&createStreamReq); tFreeStreamObj(&streamObj); - if(sql != NULL){ + if (sql != NULL) { taosMemoryFreeClear(sql); } - return code; + + return terrno; } int64_t mndStreamGenChkpId(SMnode *pMnode) { @@ -891,8 +932,11 @@ int64_t mndStreamGenChkpId(SMnode *pMnode) { if (pIter == NULL) break; maxChkpId = TMAX(maxChkpId, pStream->checkpointId); + mDebug("stream %p checkpoint %" PRId64 "", pStream, pStream->checkpointId); sdbRelease(pSdb, pStream); } + + mDebug("generated checkpoint %" PRId64 "", maxChkpId + 1); return maxChkpId + 1; } @@ -912,6 +956,22 @@ static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) { return 0; } +static int32_t mndProcessStreamRemainChkptTmr(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SSdb * pSdb = pMnode->pSdb; + if (sdbGetSize(pSdb, SDB_STREAM) <= 0) { + return 0; + } + + SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg)); + pMsg->checkpointId = 0; + + int32_t size = sizeof(SMStreamDoCheckpointMsg); + SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE, .pCont = pMsg, .contLen = size}; + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); + return 0; +} + static int32_t mndBuildStreamCheckpointSourceReq2(void **pBuf, int32_t *pLen, int32_t nodeId, int64_t checkpointId, int64_t streamId, int32_t taskId) { SStreamCheckpointSourceReq req = {0}; @@ -954,107 +1014,104 @@ static int32_t mndBuildStreamCheckpointSourceReq2(void **pBuf, int32_t *pLen, in return 0; } -// static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStream, int64_t checkpointId) { -// int64_t timestampMs = taosGetTimestampMs(); -// if (timestampMs - pStream->checkpointFreq < tsStreamCheckpointInterval * 1000) { -// return -1; -// } -// STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, NULL, "stream-checkpoint"); -// if (pTrans == NULL) return -1; -// mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb); -// if (mndTrancCheckConflict(pMnode, pTrans) != 0) { -// mError("failed to checkpoint of stream name%s, checkpointId: %" PRId64 ", reason:%s", pStream->name, -// checkpointId, -// tstrerror(TSDB_CODE_MND_TRANS_CONFLICT)); -// mndTransDrop(pTrans); -// return -1; -// } -// mDebug("start to trigger checkpoint for stream:%s, checkpoint: %" PRId64 "", pStream->name, checkpointId); -// atomic_store_64(&pStream->currentTick, 1); -// taosWLockLatch(&pStream->lock); -// // 1. redo action: broadcast checkpoint source msg for all source vg -// int32_t totLevel = taosArrayGetSize(pStream->tasks); -// for (int32_t i = 0; i < totLevel; i++) { -// SArray *pLevel = taosArrayGetP(pStream->tasks, i); -// SStreamTask *pTask = taosArrayGetP(pLevel, 0); -// if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { -// int32_t sz = taosArrayGetSize(pLevel); -// for (int32_t j = 0; j < sz; j++) { -// SStreamTask *pTask = taosArrayGetP(pLevel, j); -// /*A(pTask->info.nodeId > 0);*/ -// SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); -// if (pVgObj == NULL) { -// taosWUnLockLatch(&pStream->lock); -// mndTransDrop(pTrans); -// return -1; -// } +static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStream, int64_t checkpointId) { + int32_t code = -1; + int64_t timestampMs = taosGetTimestampMs(); + if (timestampMs - pStream->checkpointFreq < tsStreamCheckpointInterval * 1000) { + return -1; + } -// void *buf; -// int32_t tlen; -// if (mndBuildStreamCheckpointSourceReq2(&buf, &tlen, pTask->info.nodeId, checkpointId, pTask->id.streamId, -// pTask->id.taskId) < 0) { -// mndReleaseVgroup(pMnode, pVgObj); -// taosWUnLockLatch(&pStream->lock); -// mndTransDrop(pTrans); -// return -1; -// } + bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb, true); + if (conflict) { + mndAddtoCheckpointWaitingList(pStream, checkpointId); + mWarn("checkpoint conflict with other trans in %s, ignore the checkpoint for stream:%s %" PRIx64, pStream->sourceDb, + pStream->name, pStream->uid); + return -1; + } -// STransAction action = {0}; -// action.epSet = mndGetVgroupEpset(pMnode, pVgObj); -// action.pCont = buf; -// action.contLen = tlen; -// action.msgType = TDMT_VND_STREAM_CHECK_POINT_SOURCE; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, NULL, MND_STREAM_CHECKPOINT_NAME); + if (pTrans == NULL) { + return -1; + } -// mndReleaseVgroup(pMnode, pVgObj); + mndStreamRegisterTrans(pTrans, MND_STREAM_CHECKPOINT_NAME, pStream->sourceDb, pStream->targetDb); -// if (mndTransAppendRedoAction(pTrans, &action) != 0) { -// taosMemoryFree(buf); -// taosWUnLockLatch(&pStream->lock); -// mndReleaseStream(pMnode, pStream); -// mndTransDrop(pTrans); -// return -1; -// } -// } -// } -// } -// // 2. reset tick -// pStream->checkpointFreq = checkpointId; -// pStream->checkpointId = checkpointId; -// pStream->checkpointFreq = taosGetTimestampMs(); -// atomic_store_64(&pStream->currentTick, 0); -// // 3. commit log: stream checkpoint info -// pStream->version = pStream->version + 1; -// taosWUnLockLatch(&pStream->lock); + mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb); + if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + mError("failed to checkpoint of stream name%s, checkpointId: %" PRId64 ", reason:%s", pStream->name, checkpointId, + tstrerror(TSDB_CODE_MND_TRANS_CONFLICT)); + goto _ERR; + } -// // // code condtion + mDebug("start to trigger checkpoint for stream:%s, checkpoint: %" PRId64 "", pStream->name, checkpointId); -// SSdbRaw *pCommitRaw = mndStreamActionEncode(pStream); -// if (pCommitRaw == NULL) { -// mError("failed to prepare trans rebalance since %s", terrstr()); -// goto _ERR; -// } -// if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { -// sdbFreeRaw(pCommitRaw); -// mError("failed to prepare trans rebalance since %s", terrstr()); -// goto _ERR; -// } -// if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) { -// sdbFreeRaw(pCommitRaw); -// mError("failed to prepare trans rebalance since %s", terrstr()); -// goto _ERR; -// } + taosWLockLatch(&pStream->lock); + pStream->currentTick = 1; -// if (mndTransPrepare(pMnode, pTrans) != 0) { -// mError("failed to prepare trans rebalance since %s", terrstr()); -// goto _ERR; -// } -// mndTransDrop(pTrans); -// return 0; -// _ERR: -// mndTransDrop(pTrans); -// return -1; -// } + // 1. redo action: broadcast checkpoint source msg for all source vg + int32_t totLevel = taosArrayGetSize(pStream->tasks); + for (int32_t i = 0; i < totLevel; i++) { + SArray * pLevel = taosArrayGetP(pStream->tasks, i); + SStreamTask *p = taosArrayGetP(pLevel, 0); + + if (p->info.taskLevel == TASK_LEVEL__SOURCE) { + int32_t sz = taosArrayGetSize(pLevel); + for (int32_t j = 0; j < sz; j++) { + SStreamTask *pTask = taosArrayGetP(pLevel, j); + + SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); + if (pVgObj == NULL) { + taosWUnLockLatch(&pStream->lock); + goto _ERR; + } + + void * buf; + int32_t tlen; + if (mndBuildStreamCheckpointSourceReq2(&buf, &tlen, pTask->info.nodeId, checkpointId, pTask->id.streamId, + pTask->id.taskId) < 0) { + mndReleaseVgroup(pMnode, pVgObj); + taosWUnLockLatch(&pStream->lock); + goto _ERR; + } + + STransAction act = {0}; + SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); + mndReleaseVgroup(pMnode, pVgObj); + + initTransAction(&act, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY); + if (mndTransAppendRedoAction(pTrans, &act) != 0) { + taosMemoryFree(buf); + taosWUnLockLatch(&pStream->lock); + goto _ERR; + } + } + } + } + + // 2. reset tick + pStream->checkpointId = checkpointId; + pStream->checkpointFreq = taosGetTimestampMs(); + pStream->currentTick = 0; + + // 3. commit log: stream checkpoint info + pStream->version = pStream->version + 1; + taosWUnLockLatch(&pStream->lock); + + if ((code = mndPersistTransLog(pStream, pTrans)) != TSDB_CODE_SUCCESS) { + return code; + } + + if ((code = mndTransPrepare(pMnode, pTrans)) != TSDB_CODE_SUCCESS) { + mError("failed to prepare trans rebalance since %s", terrstr()); + goto _ERR; + } + + code = 0; +_ERR: + mndTransDrop(pTrans); + return code; +} static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream, SMnode *pMnode, int64_t chkptId) { taosWLockLatch(&pStream->lock); @@ -1130,23 +1187,7 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream return 0; } -static const char *mndGetStreamDB(SMnode *pMnode) { - SSdb * pSdb = pMnode->pSdb; - SStreamObj *pStream = NULL; - void * pIter = NULL; - - pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); - if (pIter == NULL) { - return NULL; - } - - const char *p = taosStrdup(pStream->sourceDb); - mndReleaseStream(pMnode, pStream); - sdbCancelFetch(pSdb, pIter); - return p; -} - -static int32_t initStreamNodeList(SMnode* pMnode) { +static int32_t initStreamNodeList(SMnode *pMnode) { if (execInfo.pNodeList == NULL || (taosArrayGetSize(execInfo.pNodeList) == 0)) { execInfo.pNodeList = taosArrayDestroy(execInfo.pNodeList); execInfo.pNodeList = extractNodeListFromStream(pMnode); @@ -1155,11 +1196,11 @@ static int32_t initStreamNodeList(SMnode* pMnode) { return taosArrayGetSize(execInfo.pNodeList); } -static bool taskNodeIsUpdated(SMnode* pMnode) { +static bool taskNodeIsUpdated(SMnode *pMnode) { // check if the node update happens or not taosThreadMutexLock(&execInfo.lock); - int32_t numOfNodes = initStreamNodeList(pMnode); + int32_t numOfNodes = initStreamNodeList(pMnode); if (numOfNodes == 0) { mDebug("stream task node change checking done, no vgroups exist, do nothing"); execInfo.ts = taosGetTimestampSec(); @@ -1199,6 +1240,38 @@ static bool taskNodeIsUpdated(SMnode* pMnode) { return nodeUpdated; } +static int32_t mndCheckNodeStatus(SMnode *pMnode) { + bool ready = true; + int64_t ts = taosGetTimestampSec(); + if (taskNodeIsUpdated(pMnode)) { + return -1; + } + + taosThreadMutexLock(&execInfo.lock); + if (taosArrayGetSize(execInfo.pNodeList) == 0) { + mDebug("stream task node change checking done, no vgroups exist, do nothing"); + execInfo.ts = ts; + } + + for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { + STaskId * p = taosArrayGet(execInfo.pTaskList, i); + STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p)); + if (pEntry == NULL) { + continue; + } + + if (pEntry->status != TASK_STATUS__READY) { + mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s not ready, checkpoint msg not issued", + pEntry->id.streamId, (int32_t)pEntry->id.taskId, 0, streamTaskGetStatusStr(pEntry->status)); + ready = false; + break; + } + } + + taosThreadMutexUnlock(&execInfo.lock); + return ready ? 0 : -1; +} + static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { SMnode * pMnode = pReq->info.node; SSdb * pSdb = pMnode->pSdb; @@ -1206,90 +1279,65 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { SStreamObj *pStream = NULL; int32_t code = 0; - // check if the node update happens or not - bool updated = taskNodeIsUpdated(pMnode); - if (updated) { - mWarn("checkpoint ignore, stream task nodes update detected"); - return -1; - } - - { // check if all tasks are in TASK_STATUS__READY status - bool ready = true; - taosThreadMutexLock(&execInfo.lock); - - // no streams exists, abort - int32_t numOfTasks = taosArrayGetSize(execInfo.pTaskList); - if (numOfTasks <= 0) { - taosThreadMutexUnlock(&execInfo.lock); - return 0; - } - - for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { - STaskId * p = taosArrayGet(execInfo.pTaskList, i); - STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p)); - if (pEntry == NULL) { - continue; - } - - if (pEntry->status != TASK_STATUS__READY) { - mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s not ready, checkpoint msg not issued", - pEntry->id.streamId, (int32_t)pEntry->id.taskId, 0, streamTaskGetStatusStr(pEntry->status)); - ready = false; - break; - } - } - taosThreadMutexUnlock(&execInfo.lock); - if (!ready) { - return 0; - } + if ((code = mndCheckNodeStatus(pMnode)) != 0) { + return code; } SMStreamDoCheckpointMsg *pMsg = (SMStreamDoCheckpointMsg *)pReq->pCont; - int64_t checkpointId = pMsg->checkpointId; - - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, NULL, MND_STREAM_CHECKPOINT_NAME); - if (pTrans == NULL) { - mError("failed to trigger checkpoint, reason: %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - return -1; - } - - mDebug("start to trigger checkpoint, checkpointId: %" PRId64, checkpointId); - - const char *pDb = mndGetStreamDB(pMnode); - mndTransSetDbName(pTrans, pDb, pDb); - mndStreamRegisterTrans(pTrans, MND_STREAM_CHECKPOINT_NAME, pDb, pDb); - taosMemoryFree((void *)pDb); - - if (mndTransCheckConflict(pMnode, pTrans) != 0) { - mError("failed to trigger checkpoint, checkpointId: %" PRId64 ", reason:%s", checkpointId, - tstrerror(TSDB_CODE_MND_TRANS_CONFLICT)); - mndTransDrop(pTrans); - return -1; - } - - while (1) { - pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); - if (pIter == NULL) break; - - code = mndAddStreamCheckpointToTrans(pTrans, pStream, pMnode, checkpointId); + while ((pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream)) != NULL) { + code = mndProcessStreamCheckpointTrans(pMnode, pStream, pMsg->checkpointId); sdbRelease(pSdb, pStream); if (code == -1) { break; } } - if (code == 0) { - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("failed to prepare trans rebalance since %s", terrstr()); + return code; +} + +static int32_t mndProcessStreamCheckpointInCandid(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + void * pIter = NULL; + int32_t code = 0; + + taosThreadMutexLock(&execInfo.lock); + int32_t num = taosHashGetSize(execInfo.transMgmt.pWaitingList); + taosThreadMutexUnlock(&execInfo.lock); + if (num == 0) { + return code; + } + + if ((code = mndCheckNodeStatus(pMnode)) != 0) { + return code; + } + + SArray *pList = taosArrayInit(4, sizeof(int64_t)); + while ((pIter = taosHashIterate(execInfo.transMgmt.pWaitingList, pIter)) != NULL) { + SCheckpointCandEntry *pEntry = pIter; + + SStreamObj *ps = mndAcquireStream(pMnode, pEntry->pName); + if (ps == NULL) { + continue; + } + mDebug("start to launch checkpoint for stream:%s %" PRIx64 " in candidate list", pEntry->pName, pEntry->streamId); + + code = mndProcessStreamCheckpointTrans(pMnode, ps, pEntry->checkpointId); + mndReleaseStream(pMnode, ps); + + if (code == TSDB_CODE_SUCCESS) { + taosArrayPush(pList, &pEntry->streamId); } } - mndTransDrop(pTrans); + for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) { + int64_t *pId = taosArrayGet(pList, i); - // only one trans here - taosThreadMutexLock(&execInfo.lock); - execInfo.activeCheckpoint = checkpointId; - taosThreadMutexUnlock(&execInfo.lock); + taosHashRemove(execInfo.transMgmt.pWaitingList, pId, sizeof(*pId)); + } + + int32_t remain = taosHashGetSize(execInfo.transMgmt.pWaitingList); + mDebug("%d in candidate list generated checkpoint, remaining:%d", (int32_t)taosArrayGetSize(pList), remain); + taosArrayDestroy(pList); return code; } @@ -1326,7 +1374,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { } // check if it is conflict with other trans in both sourceDb and targetDb. - bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb); + bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb, true); if (conflict) { sdbRelease(pMnode->pSdb, pStream); tFreeMDropStreamReq(&dropReq); @@ -1527,7 +1575,7 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB char dstStr[20] = {0}; STR_TO_VARSTR(dstStr, sinkQuota) pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char*) dstStr, false); + colDataSetVal(pColInfo, numOfRows, (const char *)dstStr, false); char scanHistoryIdle[20 + VARSTR_HEADER_SIZE] = {0}; strcpy(scanHistoryIdle, "100a"); @@ -1535,7 +1583,7 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB memset(dstStr, 0, tListLen(dstStr)); STR_TO_VARSTR(dstStr, scanHistoryIdle) pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char*) dstStr, false); + colDataSetVal(pColInfo, numOfRows, (const char *)dstStr, false); numOfRows++; sdbRelease(pSdb, pStream); @@ -1550,7 +1598,7 @@ static void mndCancelGetNextStream(SMnode *pMnode, void *pIter) { sdbCancelFetch(pSdb, pIter); } -static void setTaskAttrInResBlock(SStreamObj* pStream, SStreamTask* pTask, SSDataBlock* pBlock, int32_t numOfRows) { +static void setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SSDataBlock *pBlock, int32_t numOfRows) { SColumnInfoData *pColInfo; int32_t cols = 0; @@ -1604,7 +1652,7 @@ static void setTaskAttrInResBlock(SStreamObj* pStream, SStreamTask* pTask, SSDat colDataSetVal(pColInfo, numOfRows, (const char *)level, false); // status - char status[20 + VARSTR_HEADER_SIZE] = {0}; + char status[20 + VARSTR_HEADER_SIZE] = {0}; STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; STaskStatusEntry *pe = taosHashGet(execInfo.pTaskMap, &id, sizeof(id)); @@ -1655,7 +1703,7 @@ static void setTaskAttrInResBlock(SStreamObj* pStream, SStreamTask* pTask, SSDat colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); } -static int32_t getNumOfTasks(SArray* pTaskList) { +static int32_t getNumOfTasks(SArray *pTaskList) { int32_t numOfLevels = taosArrayGetSize(pTaskList); int32_t count = 0; @@ -1713,7 +1761,7 @@ static void mndCancelGetNextStreamTask(SMnode *pMnode, void *pIter) { sdbCancelFetch(pSdb, pIter); } -static int32_t mndPauseStreamTask(SMnode* pMnode, STrans *pTrans, SStreamTask *pTask) { +static int32_t mndPauseStreamTask(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) { SVPauseStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVPauseStreamTaskReq)); if (pReq == NULL) { mError("failed to malloc in pause stream, size:%" PRIzu ", code:%s", sizeof(SVPauseStreamTaskReq), @@ -1739,7 +1787,7 @@ static int32_t mndPauseStreamTask(SMnode* pMnode, STrans *pTrans, SStreamTask *p return 0; } -int32_t mndPauseAllStreamTasks(SMnode* pMnode, STrans *pTrans, SStreamObj *pStream) { +int32_t mndPauseAllStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { SArray *tasks = pStream->tasks; int32_t size = taosArrayGetSize(tasks); @@ -1809,7 +1857,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { } // check if it is conflict with other trans in both sourceDb and targetDb. - bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb); + bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb, true); if (conflict) { sdbRelease(pMnode->pSdb, pStream); return -1; @@ -1867,7 +1915,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { return TSDB_CODE_ACTION_IN_PROGRESS; } -static int32_t mndResumeStreamTask(STrans *pTrans, SMnode* pMnode, SStreamTask *pTask, int8_t igUntreated) { +static int32_t mndResumeStreamTask(STrans *pTrans, SMnode *pMnode, SStreamTask *pTask, int8_t igUntreated) { SVResumeStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResumeStreamTaskReq)); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1891,7 +1939,7 @@ static int32_t mndResumeStreamTask(STrans *pTrans, SMnode* pMnode, SStreamTask * return 0; } -int32_t mndResumeAllStreamTasks(STrans *pTrans, SMnode* pMnode, SStreamObj *pStream, int8_t igUntreated) { +int32_t mndResumeAllStreamTasks(STrans *pTrans, SMnode *pMnode, SStreamObj *pStream, int8_t igUntreated) { int32_t size = taosArrayGetSize(pStream->tasks); for (int32_t i = 0; i < size; i++) { SArray *pTasks = taosArrayGetP(pStream->tasks, i); @@ -1944,7 +1992,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { } // check if it is conflict with other trans in both sourceDb and targetDb. - bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb); + bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb, true); if (conflict) { sdbRelease(pMnode->pSdb, pStream); return -1; @@ -2128,8 +2176,8 @@ static bool isNodeEpsetChanged(const SEpSet *pPrevEpset, const SEpSet *pCurrent) // 1. increase the replica does not affect the stream process. // 2. decreasing the replica may affect the stream task execution in the way that there is one or more running stream // tasks on the will be removed replica. -// 3. vgroup redistribution is an combination operation of first increase replica and then decrease replica. So we will -// handle it as mentioned in 1 & 2 items. +// 3. vgroup redistribution is an combination operation of first increase replica and then decrease replica. So we +// will handle it as mentioned in 1 & 2 items. static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList) { SVgroupChangeInfo info = { .pUpdateNodeList = taosArrayInit(4, sizeof(SNodeUpdateInfo)), @@ -2157,10 +2205,8 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP epsetAssign(&updateInfo.prevEp, &pPrevEntry->epset); epsetAssign(&updateInfo.newEp, &pCurrent->epset); taosArrayPush(info.pUpdateNodeList, &updateInfo); - - } - if(pCurrent->nodeId != SNODE_HANDLE){ + if (pCurrent->nodeId != SNODE_HANDLE) { SVgObj *pVgroup = mndAcquireVgroup(pMnode, pCurrent->nodeId); taosHashPut(info.pDBMap, pVgroup->dbName, strlen(pVgroup->dbName), NULL, 0); mndReleaseVgroup(pMnode, pVgroup); @@ -2373,7 +2419,6 @@ static int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId) { if (p == NULL) { return TSDB_CODE_SUCCESS; } - taosHashRemove(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId)); for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) { @@ -2411,7 +2456,7 @@ int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) { STaskId * pId = taosArrayGet(execInfo.pTaskList, i); STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId)); - if(pEntry->nodeId == SNODE_HANDLE) continue; + if (pEntry->nodeId == SNODE_HANDLE) continue; bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId); if (!existed) { @@ -2428,9 +2473,9 @@ int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) { (int32_t)taosArrayGetSize(execInfo.pTaskList)); int32_t size = taosArrayGetSize(pNodeSnapshot); - SArray* pValidNodeEntryList = taosArrayInit(4, sizeof(SNodeEntry)); - for(int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeList); ++i) { - SNodeEntry* p = taosArrayGet(execInfo.pNodeList, i); + SArray *pValidNodeEntryList = taosArrayInit(4, sizeof(SNodeEntry)); + for (int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeList); ++i) { + SNodeEntry *p = taosArrayGet(execInfo.pNodeList, i); for (int32_t j = 0; j < size; ++j) { SNodeEntry *pEntry = taosArrayGet(pNodeSnapshot, j); @@ -2449,13 +2494,14 @@ int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) { return 0; } -static void killAllCheckpointTrans(SMnode* pMnode, SVgroupChangeInfo* pChangeInfo) { - void* pIter = NULL; - while((pIter = taosHashIterate(pChangeInfo->pDBMap, pIter)) != NULL) { - char* pDb = (char*) pIter; +// kill all trans in the dst DB +static void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo) { + void *pIter = NULL; + while ((pIter = taosHashIterate(pChangeInfo->pDBMap, pIter)) != NULL) { + char *pDb = (char *)pIter; size_t len = 0; - void* pKey = taosHashGetKey(pDb, &len); + void * pKey = taosHashGetKey(pDb, &len); killActiveCheckpointTrans(pMnode, pKey, len); } } @@ -2604,7 +2650,7 @@ void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList)); } -STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const char *name, const char* pMsg) { +STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const char *name, const char *pMsg) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, name); if (pTrans == NULL) { mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); @@ -2612,7 +2658,7 @@ STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const return NULL; } - mDebug("s-task:0x%"PRIx64" start to build trans %s", pStream->uid, pMsg); + mDebug("s-task:0x%" PRIx64 " start to build trans %s", pStream->uid, pMsg); mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb); if (mndTransCheckConflict(pMnode, pTrans) != 0) { @@ -2648,6 +2694,7 @@ int32_t createStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to malloc in reset stream, size:%" PRIzu ", code:%s", sizeof(SVResetStreamTaskReq), tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + taosWUnLockLatch(&pStream->lock); return terrno; } @@ -2691,9 +2738,9 @@ int32_t createStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { return TSDB_CODE_ACTION_IN_PROGRESS; } -int32_t killActiveCheckpointTrans(SMnode *pMnode, const char* pDBName, size_t len) { +int32_t killActiveCheckpointTrans(SMnode *pMnode, const char *pDBName, size_t len) { // data in the hash table will be removed automatically, no need to remove it here. - SStreamTransInfo* pTransInfo = taosHashGet(execInfo.transMgmt.pDBTrans, pDBName, len); + SStreamTransInfo *pTransInfo = taosHashGet(execInfo.transMgmt.pDBTrans, pDBName, len); if (pTransInfo == NULL) { return TSDB_CODE_SUCCESS; } @@ -2704,7 +2751,7 @@ int32_t killActiveCheckpointTrans(SMnode *pMnode, const char* pDBName, size_t le return TSDB_CODE_SUCCESS; } - STrans* pTrans = mndAcquireTrans(pMnode, pTransInfo->transId); + STrans *pTrans = mndAcquireTrans(pMnode, pTransInfo->transId); if (pTrans != NULL) { mInfo("kill checkpoint transId:%d in Db:%s", pTransInfo->transId, pDBName); mndKillTrans(pMnode, pTrans); @@ -2715,7 +2762,7 @@ int32_t killActiveCheckpointTrans(SMnode *pMnode, const char* pDBName, size_t le } static int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int32_t transId) { - STrans* pTrans = mndAcquireTrans(pMnode, transId); + STrans *pTrans = mndAcquireTrans(pMnode, transId); if (pTrans != NULL) { mInfo("kill checkpoint transId:%d to reset task status", transId); mndKillTrans(pMnode, pTrans); @@ -2732,10 +2779,10 @@ static int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int32_t transId) { break; } - bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb); + bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb, false); if (conflict) { - mError("stream:%s other trans exists in DB:%s & %s failed to start reset-status trans", - pStream->name, pStream->sourceDb, pStream->targetDb); + mError("stream:%s other trans exists in DB:%s & %s failed to start reset-status trans", pStream->name, + pStream->sourceDb, pStream->targetDb); continue; } @@ -2746,11 +2793,10 @@ static int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int32_t transId) { return code; } } - return 0; } -static SStreamTask* mndGetStreamTask(STaskId* pId, SStreamObj* pStream) { +static SStreamTask *mndGetStreamTask(STaskId *pId, SStreamObj *pStream) { for (int32_t i = 0; i < taosArrayGetSize(pStream->tasks); i++) { SArray *pLevel = taosArrayGetP(pStream->tasks, i); @@ -2768,7 +2814,7 @@ static SStreamTask* mndGetStreamTask(STaskId* pId, SStreamObj* pStream) { static bool needDropRelatedFillhistoryTask(STaskStatusEntry *pTaskEntry, SStreamExecInfo *pExecNode) { if (pTaskEntry->status == TASK_STATUS__STREAM_SCAN_HISTORY && pTaskEntry->statusLastDuration >= 10) { - if (fabs(pTaskEntry->inputQUsed) <= DBL_EPSILON) { + if (!pTaskEntry->inputQChanging && pTaskEntry->inputQUnchangeCounter > 10) { int32_t numOfReady = 0; int32_t numOfTotal = 0; for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) { @@ -2805,7 +2851,7 @@ static bool needDropRelatedFillhistoryTask(STaskStatusEntry *pTaskEntry, SStream static int32_t mndDropRelatedFillhistoryTask(SMnode *pMnode, STaskStatusEntry *pTaskEntry, SStreamObj *pStream) { SStreamTask *pTask = mndGetStreamTask(&pTaskEntry->id, pStream); if (pTask == NULL) { - mError("failed to get the stream task:0x%x, may have been dropped", (int32_t) pTaskEntry->id.taskId); + mError("failed to get the stream task:0x%x, may have been dropped", (int32_t)pTaskEntry->id.taskId); return -1; } @@ -2840,12 +2886,12 @@ int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList) { mInfo("set node expired for %d nodes", num); for (int k = 0; k < num; ++k) { - int32_t* pVgId = taosArrayGet(pNodeList, k); + int32_t *pVgId = taosArrayGet(pNodeList, k); mInfo("set node expired for nodeId:%d, total:%d", *pVgId, num); int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeList); for (int i = 0; i < numOfNodes; ++i) { - SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeList, i); + SNodeEntry *pNodeEntry = taosArrayGet(execInfo.pNodeList, i); if (pNodeEntry->nodeId == *pVgId) { mInfo("vgId:%d expired for some stream tasks, needs update nodeEp", *pVgId); @@ -2858,13 +2904,13 @@ int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList) { return TSDB_CODE_SUCCESS; } -static void updateStageInfo(STaskStatusEntry* pTaskEntry, int64_t stage) { +static void updateStageInfo(STaskStatusEntry *pTaskEntry, int64_t stage) { int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeList); - for(int32_t j = 0; j < numOfNodes; ++j) { - SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeList, j); + for (int32_t j = 0; j < numOfNodes; ++j) { + SNodeEntry *pNodeEntry = taosArrayGet(execInfo.pNodeList, j); if (pNodeEntry->nodeId == pTaskEntry->nodeId) { - mInfo("vgId:%d stage updated from %"PRId64 " to %"PRId64 ", nodeUpdate trigger by s-task:0x%" PRIx64, pTaskEntry->nodeId, - pTaskEntry->stage, stage, pTaskEntry->id.taskId); + mInfo("vgId:%d stage updated from %" PRId64 " to %" PRId64 ", nodeUpdate trigger by s-task:0x%" PRIx64, + pTaskEntry->nodeId, pTaskEntry->stage, stage, pTaskEntry->id.taskId); pNodeEntry->stageUpdated = true; pTaskEntry->stage = stage; @@ -2911,6 +2957,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { bool snodeChanged = false; for (int32_t i = 0; i < req.numOfTasks; ++i) { STaskStatusEntry *p = taosArrayGet(req.pTaskStatus, i); + STaskStatusEntry *pTaskEntry = taosHashGet(execInfo.pTaskMap, &p->id, sizeof(p->id)); if (pTaskEntry == NULL) { mError("s-task:0x%" PRIx64 " not found in mnode task list", p->id.taskId); @@ -2919,8 +2966,20 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (pTaskEntry->stage != p->stage && pTaskEntry->stage != -1) { updateStageInfo(pTaskEntry, p->stage); - if(pTaskEntry->nodeId == SNODE_HANDLE) snodeChanged = true; + if (pTaskEntry->nodeId == SNODE_HANDLE) snodeChanged = true; } else { + // task is idle for more than 50 sec. + if (fabs(pTaskEntry->inputQUsed - p->inputQUsed) <= DBL_EPSILON) { + if (!pTaskEntry->inputQChanging) { + pTaskEntry->inputQUnchangeCounter++; + } else { + pTaskEntry->inputQChanging = false; + } + } else { + pTaskEntry->inputQChanging = true; + pTaskEntry->inputQUnchangeCounter = 0; + } + streamTaskStatusCopy(pTaskEntry, p); if (p->activeCheckpointId != 0) { if (activeCheckpointId != 0) { @@ -2947,7 +3006,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (p->status == TASK_STATUS__STREAM_SCAN_HISTORY) { bool drop = needDropRelatedFillhistoryTask(pTaskEntry, &execInfo); - if(drop) { + if (drop) { SStreamObj *pStreamObj = mndGetStreamObj(pMnode, pTaskEntry->id.streamId); if (pStreamObj == NULL) { mError("failed to acquire the streamObj:0x%" PRIx64 " it may have been dropped", pStreamObj->uid); @@ -2984,9 +3043,13 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { return TSDB_CODE_SUCCESS; } +void freeCheckpointCandEntry(void *param) { + SCheckpointCandEntry *pEntry = param; + taosMemoryFreeClear(pEntry->pName); +} SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId) { - void *pIter = NULL; - SSdb *pSdb = pMnode->pSdb; + void * pIter = NULL; + SSdb * pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; while ((pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream)) != NULL) { @@ -2997,4 +3060,4 @@ SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId) { } return NULL; -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index 2345de290a..fa36d69d6e 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "mndTrans.h" #include "mndStream.h" +#include "mndTrans.h" typedef struct SKeyInfo { void* pKey; @@ -35,17 +35,15 @@ int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pName, const char* pS } int32_t clearFinishedTrans(SMnode* pMnode) { - SArray* pList = taosArrayInit(4, sizeof(SKeyInfo)); size_t keyLen = 0; + SArray* pList = taosArrayInit(4, sizeof(SKeyInfo)); + void* pIter = NULL; - taosThreadMutexLock(&execInfo.lock); - - void* pIter = NULL; while ((pIter = taosHashIterate(execInfo.transMgmt.pDBTrans, pIter)) != NULL) { - SStreamTransInfo *pEntry = (SStreamTransInfo *)pIter; - STrans* pTrans = mndAcquireTrans(pMnode, pEntry->transId); + SStreamTransInfo* pEntry = (SStreamTransInfo*)pIter; // let's clear the finished trans + STrans* pTrans = mndAcquireTrans(pMnode, pEntry->transId); if (pTrans == NULL) { void* pKey = taosHashGetKey(pEntry, &keyLen); // key is the name of src/dst db name @@ -60,46 +58,76 @@ int32_t clearFinishedTrans(SMnode* pMnode) { } size_t num = taosArrayGetSize(pList); - for(int32_t i = 0; i < num; ++i) { + for (int32_t i = 0; i < num; ++i) { SKeyInfo* pKey = taosArrayGet(pList, i); taosHashRemove(execInfo.transMgmt.pDBTrans, pKey->pKey, pKey->keyLen); } - mDebug("clear %d finished stream-trans, remained:%d", (int32_t) num, taosHashGetSize(execInfo.transMgmt.pDBTrans)); - taosThreadMutexUnlock(&execInfo.lock); + mDebug("clear %d finished stream-trans, remained:%d", (int32_t)num, taosHashGetSize(execInfo.transMgmt.pDBTrans)); terrno = TSDB_CODE_SUCCESS; taosArrayDestroy(pList); return 0; } -bool streamTransConflictOtherTrans(SMnode* pMnode, const char* pSrcDb, const char* pDstDb) { - clearFinishedTrans(pMnode); +bool streamTransConflictOtherTrans(SMnode* pMnode, const char* pSrcDb, const char* pDstDb, bool lock) { + if (lock) { + taosThreadMutexLock(&execInfo.lock); + } - taosThreadMutexLock(&execInfo.lock); int32_t num = taosHashGetSize(execInfo.transMgmt.pDBTrans); if (num <= 0) { - taosThreadMutexUnlock(&execInfo.lock); + if (lock) { + taosThreadMutexUnlock(&execInfo.lock); + } return false; } + clearFinishedTrans(pMnode); + SStreamTransInfo *pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, pSrcDb, strlen(pSrcDb)); if (pEntry != NULL) { - taosThreadMutexUnlock(&execInfo.lock); + if (lock) { + taosThreadMutexUnlock(&execInfo.lock); + } mWarn("conflict with other transId:%d in Db:%s, trans:%s", pEntry->transId, pSrcDb, pEntry->name); return true; } pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, pDstDb, strlen(pDstDb)); if (pEntry != NULL) { - taosThreadMutexUnlock(&execInfo.lock); + if (lock) { + taosThreadMutexUnlock(&execInfo.lock); + } mWarn("conflict with other transId:%d in Db:%s, trans:%s", pEntry->transId, pSrcDb, pEntry->name); return true; } - taosThreadMutexUnlock(&execInfo.lock); + if (lock) { + taosThreadMutexUnlock(&execInfo.lock); + } + return false; } +int32_t mndAddtoCheckpointWaitingList(SStreamObj* pStream, int64_t checkpointId) { + SCheckpointCandEntry* pEntry = taosHashGet(execInfo.transMgmt.pWaitingList, &pStream->uid, sizeof(pStream->uid)); + if (pEntry == NULL) { + SCheckpointCandEntry entry = {.streamId = pStream->uid, + .checkpointTs = taosGetTimestampMs(), + .checkpointId = checkpointId, + .pName = taosStrdup(pStream->name)}; + taosHashPut(execInfo.transMgmt.pWaitingList, &pStream->uid, sizeof(pStream->uid), &entry, sizeof(entry)); + int32_t size = taosHashGetSize(execInfo.transMgmt.pWaitingList); + mDebug("stream:%" PRIx64 " add into waiting list due to conflict, ts:%" PRId64 " , checkpointId: %" PRId64 + ", total in waitingList:%d", + pStream->uid, entry.checkpointTs, checkpointId, size); + } else { + mDebug("stream:%" PRIx64 " ts:%" PRId64 ", checkpointId:%" PRId64 " already in waiting list, no need to add into", + pStream->uid, pEntry->checkpointTs, checkpointId); + } + + return TSDB_CODE_SUCCESS; +} diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 320587bb57..44ee804d22 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -161,7 +161,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg, SSubplan* pPlan) { if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { - if(pRebVg->oldConsumerId == -1) return 0; + if(pRebVg->oldConsumerId == -1) return 0; //drop stream, no consumer, while split vnode,all consumerId is -1 terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; return -1; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index c9756ef814..7749decf91 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -834,7 +834,7 @@ int32_t mndTransCheckConflict(SMnode *pMnode, STrans *pTrans) { if (mndCheckTransConflict(pMnode, pTrans)) { terrno = TSDB_CODE_MND_TRANS_CONFLICT; mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - return -1; + return terrno; } return 0; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 66abfd6bc1..0e3b544508 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -708,9 +708,8 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { int32_t numOfAlterViews = taosHashGetSize(pUser->alterViews); int32_t numOfTopics = taosHashGetSize(pUser->topics); int32_t numOfUseDbs = taosHashGetSize(pUser->useDbs); - int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + - (numOfReadDbs + numOfWriteDbs + numOfUseDbs) * TSDB_DB_FNAME_LEN + numOfTopics * TSDB_TOPIC_FNAME_LEN + - ipWhiteReserve; + int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN + + numOfTopics * TSDB_TOPIC_FNAME_LEN + ipWhiteReserve; char *stb = taosHashIterate(pUser->readTbs, NULL); while (stb != NULL) { @@ -720,7 +719,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { size += keyLen; size_t valueLen = 0; - valueLen = strlen(stb); + valueLen = strlen(stb) + 1; size += sizeof(int32_t); size += valueLen; stb = taosHashIterate(pUser->readTbs, stb); @@ -734,7 +733,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { size += keyLen; size_t valueLen = 0; - valueLen = strlen(stb); + valueLen = strlen(stb) + 1; size += sizeof(int32_t); size += valueLen; stb = taosHashIterate(pUser->writeTbs, stb); @@ -748,7 +747,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { size += keyLen; size_t valueLen = 0; - valueLen = strlen(stb); + valueLen = strlen(stb) + 1; size += sizeof(int32_t); size += valueLen; stb = taosHashIterate(pUser->alterTbs, stb); @@ -762,7 +761,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { size += keyLen; size_t valueLen = 0; - valueLen = strlen(stb); + valueLen = strlen(stb) + 1; size += sizeof(int32_t); size += valueLen; stb = taosHashIterate(pUser->readViews, stb); @@ -776,7 +775,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { size += keyLen; size_t valueLen = 0; - valueLen = strlen(stb); + valueLen = strlen(stb) + 1; size += sizeof(int32_t); size += valueLen; stb = taosHashIterate(pUser->writeViews, stb); @@ -790,11 +789,21 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { size += keyLen; size_t valueLen = 0; - valueLen = strlen(stb); + valueLen = strlen(stb) + 1; size += sizeof(int32_t); size += valueLen; stb = taosHashIterate(pUser->alterViews, stb); - } + } + + int32_t *useDb = taosHashIterate(pUser->useDbs, NULL); + while (useDb != NULL) { + size_t keyLen = 0; + void *key = taosHashGetKey(useDb, &keyLen); + size += sizeof(int32_t); + size += keyLen; + size += sizeof(int32_t); + useDb = taosHashIterate(pUser->useDbs, useDb); + } SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size); if (pRaw == NULL) goto _OVER; @@ -925,7 +934,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { stb = taosHashIterate(pUser->alterViews, stb); } - int32_t *useDb = taosHashIterate(pUser->useDbs, NULL); + useDb = taosHashIterate(pUser->useDbs, NULL); while (useDb != NULL) { size_t keyLen = 0; void *key = taosHashGetKey(useDb, &keyLen); diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index ddde645fae..0a20dcfd09 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -149,7 +149,8 @@ typedef enum { SDB_FUNC = 20, SDB_IDX = 21, SDB_VIEW = 22, - SDB_MAX = 23 + SDB_STREAM_SEQ = 23, + SDB_MAX = 24 } ESdbType; typedef struct SSdbRaw { @@ -169,11 +170,11 @@ typedef struct SSdbRow { } SSdbRow; typedef struct SSdb { - SMnode *pMnode; - SWal *pWal; + SMnode * pMnode; + SWal * pWal; int64_t sync; - char *currDir; - char *tmpDir; + char * currDir; + char * tmpDir; int64_t commitIndex; int64_t commitTerm; int64_t commitConfig; @@ -183,7 +184,7 @@ typedef struct SSdb { int64_t tableVer[SDB_MAX]; int64_t maxId[SDB_MAX]; EKeyType keyTypes[SDB_MAX]; - SHashObj *hashObjs[SDB_MAX]; + SHashObj * hashObjs[SDB_MAX]; TdThreadRwlock locks[SDB_MAX]; SdbInsertFp insertFps[SDB_MAX]; SdbUpdateFp updateFps[SDB_MAX]; @@ -198,25 +199,25 @@ typedef struct SSdb { typedef struct SSdbIter { TdFilePtr file; int64_t total; - char *name; + char * name; } SSdbIter; typedef struct { - ESdbType sdbType; - EKeyType keyType; - SdbDeployFp deployFp; - SdbEncodeFp encodeFp; - SdbDecodeFp decodeFp; - SdbInsertFp insertFp; - SdbUpdateFp updateFp; - SdbDeleteFp deleteFp; + ESdbType sdbType; + EKeyType keyType; + SdbDeployFp deployFp; + SdbEncodeFp encodeFp; + SdbDecodeFp decodeFp; + SdbInsertFp insertFp; + SdbUpdateFp updateFp; + SdbDeleteFp deleteFp; SdbValidateFp validateFp; } SSdbTable; typedef struct SSdbOpt { const char *path; - SMnode *pMnode; - SWal *pWal; + SMnode * pMnode; + SWal * pWal; int64_t sync; } SSdbOpt; @@ -393,7 +394,7 @@ int32_t sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver); int32_t sdbGetRawTotalSize(SSdbRaw *pRaw); SSdbRow *sdbAllocRow(int32_t objSize); -void *sdbGetRowObj(SSdbRow *pRow); +void * sdbGetRowObj(SSdbRow *pRow); void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc); int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *term, int64_t *config); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 72ef6d14bd..380be1dd38 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -13,31 +13,31 @@ * along with this program. If not, see . */ -#include "rsync.h" #include "executor.h" +#include "rsync.h" #include "sndInt.h" #include "tqCommon.h" #include "tuuid.h" #define sndError(...) \ - do { \ + do { \ if (sndDebugFlag & DEBUG_ERROR) { \ taosPrintLog("SND ERROR ", DEBUG_ERROR, sndDebugFlag, __VA_ARGS__); \ - } \ + } \ } while (0) -#define sndInfo(...) \ +#define sndInfo(...) \ do { \ - if (sndDebugFlag & DEBUG_INFO) { \ + if (sndDebugFlag & DEBUG_INFO) { \ taosPrintLog("SND INFO ", DEBUG_INFO, sndDebugFlag, __VA_ARGS__); \ } \ } while (0) #define sndDebug(...) \ - do { \ + do { \ if (sndDebugFlag & DEBUG_DEBUG) { \ taosPrintLog("SND ", DEBUG_DEBUG, sndDebugFlag, __VA_ARGS__); \ - } \ + } \ } while (0) int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer) { @@ -46,10 +46,11 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer if (code != TSDB_CODE_SUCCESS) { return code; } + pTask->pBackend = NULL; streamTaskOpenAllUpstreamInput(pTask); - SStreamTask* pSateTask = pTask; + SStreamTask *pSateTask = pTask; SStreamTask task = {0}; if (pTask->info.fillHistory) { task.id.streamId = pTask->streamTaskId.streamId; @@ -84,7 +85,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer streamTaskResetUpstreamStageInfo(pTask); streamSetupScheduleTrigger(pTask); - SCheckpointInfo* pChkInfo = &pTask->chkInfo; + SCheckpointInfo *pChkInfo = &pTask->chkInfo; // checkpoint ver is the kept version, handled data should be the next version. if (pTask->chkInfo.checkpointId != 0) { pTask->chkInfo.nextProcessVer = pTask->chkInfo.checkpointVer + 1; @@ -92,7 +93,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); } - char* p = NULL; + char *p = NULL; streamTaskGetStatus(pTask, &p); if (pTask->info.fillHistory) { @@ -194,7 +195,7 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { switch (pMsg->msgType) { case TDMT_STREAM_TASK_DEPLOY: { - void *pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + void * pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t len = pMsg->contLen - sizeof(SMsgHead); return tqStreamTaskProcessDeployReq(pSnode->pMeta, -1, pReq, len, true, true); } diff --git a/source/dnode/snode/src/snodeInitApi.c b/source/dnode/snode/src/snodeInitApi.c index 570feffc14..d1e0aefca0 100644 --- a/source/dnode/snode/src/snodeInitApi.c +++ b/source/dnode/snode/src/snodeInitApi.c @@ -80,6 +80,7 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->updateInfoDestroy = updateInfoDestroy; pStore->windowSBfDelete = windowSBfDelete; pStore->windowSBfAdd = windowSBfAdd; + pStore->isIncrementalTimeStamp = isIncrementalTimeStamp; pStore->updateInfoInitP = updateInfoInitP; pStore->updateInfoAddCloseWindowSBF = updateInfoAddCloseWindowSBF; diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 9aeb14cd60..114051f02b 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -14,6 +14,8 @@ set( "src/vnd/vnodeSnapshot.c" "src/vnd/vnodeRetention.c" "src/vnd/vnodeInitApi.c" + "src/vnd/vnodeAsync.c" + "src/vnd/vnodeHash.c" # meta "src/meta/metaOpen.c" diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 9307a620ad..74d62da5c5 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -153,26 +153,6 @@ typedef struct STsdbReader STsdbReader; #define CACHESCAN_RETRIEVE_LAST_ROW 0x4 #define CACHESCAN_RETRIEVE_LAST 0x8 -int32_t tsdbReaderOpen(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, - SSDataBlock *pResBlock, void **ppReader, const char *idstr, bool countOnly, - SHashObj **pIgnoreTables); -int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num); -void tsdbReaderSetId(STsdbReader *pReader, const char *idstr); -void tsdbReaderClose(STsdbReader *pReader); -int32_t tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext); -int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA); -void tsdbReleaseDataBlock(STsdbReader *pReader); -SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); -int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); -int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); -int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle); -void *tsdbGetIdx(SMeta *pMeta); -void *tsdbGetIvtIdx(SMeta *pMeta); -uint64_t tsdbGetReaderMaxVersion(STsdbReader *pReader); -void tsdbReaderSetCloseFlag(STsdbReader *pReader); -int64_t tsdbGetLastTimestamp(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr); - -//====================================================================================================================== int32_t tsdbReaderOpen2(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, SSDataBlock *pResBlock, void **ppReader, const char *idstr, bool countOnly, SHashObj **pIgnoreTables); @@ -191,7 +171,8 @@ void *tsdbGetIvtIdx2(SMeta *pMeta); uint64_t tsdbGetReaderMaxVersion2(STsdbReader *pReader); void tsdbReaderSetCloseFlag(STsdbReader *pReader); int64_t tsdbGetLastTimestamp2(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr); -//====================================================================================================================== +void tsdbSetFilesetDelimited(STsdbReader* pReader); +void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param); int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index ca9d22a987..95982abfbe 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -309,7 +309,12 @@ int32_t tsdbTakeReadSnap2(STsdbReader *pReader, _query_reseek_func_t reseek, STs void tsdbUntakeReadSnap2(STsdbReader *pReader, STsdbReadSnap *pSnap, bool proactive); // tsdbMerge.c ============================================================================================== -int32_t tsdbSchedMerge(STsdb *tsdb, int32_t fid); +typedef struct { + STsdb *tsdb; + int32_t fid; +} SMergeArg; + +int32_t tsdbMerge(void *arg); // tsdbDiskData ============================================================================================== int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder); @@ -372,6 +377,7 @@ struct STsdb { SVnode *pVnode; STsdbKeepCfg keepCfg; TdThreadMutex mutex; + bool bgTaskDisabled; SMemTable *mem; SMemTable *imem; STsdbFS fs; // old @@ -675,41 +681,54 @@ struct SDelFWriter { typedef struct STFileSet STFileSet; typedef TARRAY2(STFileSet *) TFileSetArray; -typedef struct STSnapRange STSnapRange; -typedef TARRAY2(STSnapRange *) TSnapRangeArray; // disjoint snap ranges +// fset range +typedef struct STFileSetRange STFileSetRange; +typedef TARRAY2(STFileSetRange *) TFileSetRangeArray; // disjoint ranges -// util -int32_t tSerializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR); -int32_t tDeserializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR); -void tsdbSnapRangeArrayDestroy(TSnapRangeArray **ppSnap); -SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges); - -// snap partition list -typedef TARRAY2(SVersionRange) SVerRangeList; -typedef struct STsdbSnapPartition STsdbSnapPartition; -typedef TARRAY2(STsdbSnapPartition *) STsdbSnapPartList; -// util -STsdbSnapPartList *tsdbSnapPartListCreate(); -void tsdbSnapPartListDestroy(STsdbSnapPartList **ppList); -int32_t tSerializeTsdbSnapPartList(void *buf, int32_t bufLen, STsdbSnapPartList *pList); -int32_t tDeserializeTsdbSnapPartList(void *buf, int32_t bufLen, STsdbSnapPartList *pList); -int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList *pList, TSnapRangeArray **ppRanges); +int32_t tsdbTFileSetRangeClear(STFileSetRange **fsr); +int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray **ppArr); +// fset partition enum { - TSDB_SNAP_RANGE_TYP_HEAD = 0, - TSDB_SNAP_RANGE_TYP_DATA, - TSDB_SNAP_RANGE_TYP_SMA, - TSDB_SNAP_RANGE_TYP_TOMB, - TSDB_SNAP_RANGE_TYP_STT, - TSDB_SNAP_RANGE_TYP_MAX, + TSDB_FSET_RANGE_TYP_HEAD = 0, + TSDB_FSET_RANGE_TYP_DATA, + TSDB_FSET_RANGE_TYP_SMA, + TSDB_FSET_RANGE_TYP_TOMB, + TSDB_FSET_RANGE_TYP_STT, + TSDB_FSET_RANGE_TYP_MAX, }; -struct STsdbSnapPartition { +typedef TARRAY2(SVersionRange) SVerRangeList; + +struct STsdbFSetPartition { int64_t fid; int8_t stat; - SVerRangeList verRanges[TSDB_SNAP_RANGE_TYP_MAX]; + SVerRangeList verRanges[TSDB_FSET_RANGE_TYP_MAX]; }; +typedef struct STsdbFSetPartition STsdbFSetPartition; +typedef TARRAY2(STsdbFSetPartition *) STsdbFSetPartList; + +STsdbFSetPartList *tsdbFSetPartListCreate(); +void tsdbFSetPartListDestroy(STsdbFSetPartList **ppList); +int32_t tSerializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList); +int32_t tDeserializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList); +int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList *pList, TFileSetRangeArray **ppRanges); + +// snap rep format +typedef enum ETsdbRepFmt { + TSDB_SNAP_REP_FMT_DEFAULT = 0, + TSDB_SNAP_REP_FMT_RAW, + TSDB_SNAP_REP_FMT_HYBRID, +} ETsdbRepFmt; + +typedef struct STsdbRepOpts { + ETsdbRepFmt format; +} STsdbRepOpts; + +int32_t tSerializeTsdbRepOpts(void *buf, int32_t bufLen, STsdbRepOpts *pInfo); +int32_t tDeserializeTsdbRepOpts(void *buf, int32_t bufLen, STsdbRepOpts *pInfo); + // snap read struct STsdbReadSnap { SMemTable *pMem; @@ -880,10 +899,9 @@ void tMergeTreeUnpinSttBlock(SMergeTree *pMTree); bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree); void tMergeTreeClose(SMergeTree *pMTree); -SSttBlockLoadInfo *tCreateOneLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols); -void resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); +SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols); void getSttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, SSttBlockLoadCostInfo *pLoadCost); -void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); +void *destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); void *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost); // tsdbCache ============================================================================================== @@ -1037,7 +1055,7 @@ typedef enum { // utils ETsdbFsState tsdbSnapGetFsState(SVnode *pVnode); -int32_t tsdbSnapGetDetails(SVnode *pVnode, SSnapshot *pSnap); +int32_t tsdbSnapPrepDescription(SVnode *pVnode, SSnapshot *pSnap); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 55b62dfe48..4036200d73 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -48,9 +48,32 @@ int32_t vnodeCheckCfg(const SVnodeCfg*); int32_t vnodeEncodeConfig(const void* pObj, SJson* pJson); int32_t vnodeDecodeConfig(const SJson* pJson, void* pObj); +// vnodeAsync.c +typedef struct SVAsync SVAsync; + +typedef enum { + EVA_PRIORITY_HIGH = 0, + EVA_PRIORITY_NORMAL, + EVA_PRIORITY_LOW, +} EVAPriority; + +#define VNODE_ASYNC_VALID_CHANNEL_ID(channelId) ((channelId) > 0) +#define VNODE_ASYNC_VALID_TASK_ID(taskId) ((taskId) > 0) + +int32_t vnodeAsyncInit(SVAsync** async, char* label); +int32_t vnodeAsyncDestroy(SVAsync** async); +int32_t vnodeAChannelInit(SVAsync* async, int64_t* channelId); +int32_t vnodeAChannelDestroy(SVAsync* async, int64_t channelId, bool waitRunning); +int32_t vnodeAsync(SVAsync* async, EVAPriority priority, int32_t (*execute)(void*), void (*complete)(void*), void* arg, + int64_t* taskId); +int32_t vnodeAsyncC(SVAsync* async, int64_t channelId, EVAPriority priority, int32_t (*execute)(void*), + void (*complete)(void*), void* arg, int64_t* taskId); +int32_t vnodeAWait(SVAsync* async, int64_t taskId); +int32_t vnodeACancel(SVAsync* async, int64_t taskId); +int32_t vnodeAsyncSetWorkers(SVAsync* async, int32_t numWorkers); + // vnodeModule.c -int vnodeScheduleTask(int (*execute)(void*), void* arg); -int vnodeScheduleTaskEx(int tpid, int (*execute)(void*), void* arg); +extern SVAsync* vnodeAsyncHandle[2]; // vnodeBufPool.c typedef struct SVBufPoolNode SVBufPoolNode; @@ -110,7 +133,7 @@ int32_t vnodeAsyncCommit(SVnode* pVnode); bool vnodeShouldRollback(SVnode* pVnode); // vnodeSync.c -int32_t vnodeSyncOpen(SVnode *pVnode, char *path, int32_t vnodeVersion); +int32_t vnodeSyncOpen(SVnode* pVnode, char* path, int32_t vnodeVersion); int32_t vnodeSyncStart(SVnode* pVnode); void vnodeSyncPreClose(SVnode* pVnode); void vnodeSyncPostClose(SVnode* pVnode); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 473329bfee..50a28357e5 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -65,6 +65,8 @@ typedef struct SMetaSnapReader SMetaSnapReader; typedef struct SMetaSnapWriter SMetaSnapWriter; typedef struct STsdbSnapReader STsdbSnapReader; typedef struct STsdbSnapWriter STsdbSnapWriter; +typedef struct STsdbSnapRAWReader STsdbSnapRAWReader; +typedef struct STsdbSnapRAWWriter STsdbSnapRAWWriter; typedef struct STqSnapReader STqSnapReader; typedef struct STqSnapWriter STqSnapWriter; typedef struct STqOffsetReader STqOffsetReader; @@ -93,7 +95,11 @@ typedef struct SQueryNode SQueryNode; #define VNODE_RSMA2_DIR "rsma2" #define VNODE_TQ_STREAM "stream" +#if SUSPEND_RESUME_TEST // only for test purpose +#define VNODE_BUFPOOL_SEGMENTS 1 +#else #define VNODE_BUFPOOL_SEGMENTS 3 +#endif #define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" @@ -209,7 +215,7 @@ int32_t tsdbBegin(STsdb* pTsdb); // int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo); int32_t tsdbCacheCommit(STsdb* pTsdb); int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo); -int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync); +int32_t tsdbRetention(STsdb* tsdb, int64_t now, int32_t sync); // int32_t tsdbFinishCommit(STsdb* pTsdb); // int32_t tsdbRollbackCommit(STsdb* pTsdb); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg); @@ -219,8 +225,6 @@ int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg); // tq -int tqInit(); -void tqCleanUp(); STQ* tqOpen(const char* path, SVnode* pVnode); void tqNotifyClose(STQ*); void tqClose(STQ*); @@ -311,6 +315,15 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, void* pRang int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, SSnapDataHdr* pHdr); int32_t tsdbSnapWriterPrepareClose(STsdbSnapWriter* pWriter); int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback); +// STsdbSnapRAWReader ======================================== +int32_t tsdbSnapRAWReaderOpen(STsdb* pTsdb, int64_t ever, int8_t type, STsdbSnapRAWReader** ppReader); +int32_t tsdbSnapRAWReaderClose(STsdbSnapRAWReader** ppReader); +int32_t tsdbSnapRAWRead(STsdbSnapRAWReader* pReader, uint8_t** ppData); +// STsdbSnapRAWWriter ======================================== +int32_t tsdbSnapRAWWriterOpen(STsdb* pTsdb, int64_t ever, STsdbSnapRAWWriter** ppWriter); +int32_t tsdbSnapRAWWrite(STsdbSnapRAWWriter* pWriter, SSnapDataHdr* pHdr); +int32_t tsdbSnapRAWWriterPrepareClose(STsdbSnapRAWWriter* pWriter); +int32_t tsdbSnapRAWWriterClose(STsdbSnapRAWWriter** ppWriter, int8_t rollback); // STqSnapshotReader == int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** ppReader); int32_t tqSnapReaderClose(STqSnapReader** ppReader); @@ -448,13 +461,16 @@ struct SVnode { SVBufPool* recycleTail; SVBufPool* onRecycle; + // commit variables + int64_t commitChannel; + int64_t commitTask; + SMeta* pMeta; SSma* pSma; STsdb* pTsdb; SWal* pWal; STQ* pTq; SSink* pSink; - tsem_t canCommit; int64_t sync; TdThreadMutex lock; bool blocked; @@ -494,18 +510,18 @@ struct SSma { void* pRSmaEnv; }; -#define SMA_CFG(s) (&(s)->pVnode->config) -#define SMA_TSDB_CFG(s) (&(s)->pVnode->config.tsdbCfg) -#define SMA_RETENTION(s) ((SRetention*)&(s)->pVnode->config.tsdbCfg.retentions) -#define SMA_LOCKED(s) ((s)->locked) -#define SMA_META(s) ((s)->pVnode->pMeta) -#define SMA_VID(s) TD_VID((s)->pVnode) -#define SMA_TFS(s) ((s)->pVnode->pTfs) -#define SMA_TSMA_ENV(s) ((s)->pTSmaEnv) -#define SMA_RSMA_ENV(s) ((s)->pRSmaEnv) -#define SMA_RSMA_TSDB0(s) ((s)->pVnode->pTsdb) -#define SMA_RSMA_TSDB1(s) ((s)->pRSmaTsdb[TSDB_RETENTION_L0]) -#define SMA_RSMA_TSDB2(s) ((s)->pRSmaTsdb[TSDB_RETENTION_L1]) +#define SMA_CFG(s) (&(s)->pVnode->config) +#define SMA_TSDB_CFG(s) (&(s)->pVnode->config.tsdbCfg) +#define SMA_RETENTION(s) ((SRetention*)&(s)->pVnode->config.tsdbCfg.retentions) +#define SMA_LOCKED(s) ((s)->locked) +#define SMA_META(s) ((s)->pVnode->pMeta) +#define SMA_VID(s) TD_VID((s)->pVnode) +#define SMA_TFS(s) ((s)->pVnode->pTfs) +#define SMA_TSMA_ENV(s) ((s)->pTSmaEnv) +#define SMA_RSMA_ENV(s) ((s)->pRSmaEnv) +#define SMA_RSMA_TSDB0(s) ((s)->pVnode->pTsdb) +#define SMA_RSMA_TSDB1(s) ((s)->pRSmaTsdb[TSDB_RETENTION_L0]) +#define SMA_RSMA_TSDB2(s) ((s)->pRSmaTsdb[TSDB_RETENTION_L1]) #define SMA_RSMA_GET_TSDB(pVnode, level) ((level == 0) ? pVnode->pTsdb : pVnode->pSma->pRSmaTsdb[level - 1]) // sma @@ -526,6 +542,7 @@ enum { SNAP_DATA_STREAM_STATE = 11, SNAP_DATA_STREAM_STATE_BACKEND = 12, SNAP_DATA_TQ_CHECKINFO = 13, + SNAP_DATA_RAW = 14, }; struct SSnapDataHdr { diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 5dc29509a0..abe4c3f2fc 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -61,7 +61,7 @@ struct SRSmaQTaskInfoItem { int32_t len; int8_t type; int64_t suid; - void *qTaskInfo; + void * qTaskInfo; }; static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) { @@ -185,7 +185,7 @@ int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore, bool isAdd) { void *pIter = NULL; while ((pIter = taosHashIterate(pStore->uidHash, pIter))) { tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - SArray *pTbUids = *(SArray **)pIter; + SArray * pTbUids = *(SArray **)pIter; if (tdUpdateTbUidListImpl(pSma, pTbSuid, pTbUids, isAdd) != TSDB_CODE_SUCCESS) { taosHashCancelIterate(pStore->uidHash, pIter); @@ -213,7 +213,7 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui } SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - SHashObj *infoHash = NULL; + SHashObj * infoHash = NULL; if (!pStat || !(infoHash = RSMA_INFO_HASH(pStat))) { terrno = TSDB_CODE_RSMA_INVALID_STAT; return TSDB_CODE_FAILED; @@ -264,11 +264,11 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat int8_t idx) { if ((param->qmsgLen > 0) && param->qmsg[idx]) { SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]); - SRetention *pRetention = SMA_RETENTION(pSma); - STsdbCfg *pTsdbCfg = SMA_TSDB_CFG(pSma); - SVnode *pVnode = pSma->pVnode; + SRetention * pRetention = SMA_RETENTION(pSma); + STsdbCfg * pTsdbCfg = SMA_TSDB_CFG(pSma); + SVnode * pVnode = pSma->pVnode; char taskInfDir[TSDB_FILENAME_LEN] = {0}; - void *pStreamState = NULL; + void * pStreamState = NULL; // set the backend of stream state tdRSmaQTaskInfoGetFullPath(pVnode, pRSmaInfo->suid, idx + 1, pVnode->pTfs, taskInfDir); @@ -297,6 +297,8 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat sprintf(pStreamTask->exec.qmsg, "%s", RSMA_EXEC_TASK_FLAG); pStreamTask->chkInfo.checkpointId = streamMetaGetLatestCheckpointId(pStreamTask->pMeta); tdRSmaTaskInit(pStreamTask->pMeta, pItem, &pStreamTask->id); + pStreamTask->status.pSM = streamCreateStateMachine(pStreamTask); + pStreamState = streamStateOpen(taskInfDir, pStreamTask, true, -1, -1); if (!pStreamState) { terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN; @@ -372,7 +374,7 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con } #endif - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SSmaEnv * pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; @@ -651,9 +653,7 @@ static int32_t tdRSmaProcessDelReq(SSma *pSma, int64_t suid, int8_t level, SBatc ((SMsgHead *)pBuf)->vgId = TD_VID(pSma->pVnode); - SRpcMsg delMsg = {.msgType = TDMT_VND_BATCH_DEL, - .pCont = pBuf, - .contLen = len + sizeof(SMsgHead)}; + SRpcMsg delMsg = {.msgType = TDMT_VND_BATCH_DEL, .pCont = pBuf, .contLen = len + sizeof(SMsgHead)}; code = tmsgPutToQueue(&pSma->pVnode->msgCb, WRITE_QUEUE, &delMsg); TSDB_CHECK_CODE(code, lino, _exit); } @@ -673,8 +673,8 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma int32_t code = 0; int32_t lino = 0; SSDataBlock *output = NULL; - SArray *pResList = pItem->pResList; - STSchema *pTSchema = pInfo->pTSchema; + SArray * pResList = pItem->pResList; + STSchema * pTSchema = pInfo->pTSchema; int64_t suid = pInfo->suid; while (1) { @@ -733,7 +733,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma } } - STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); + STsdb * sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); SSubmitReq2 *pReq = NULL; if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, output->info.id.groupId, SMA_VID(pSma), suid) < 0) { @@ -795,7 +795,7 @@ _exit: static int32_t tdExecuteRSmaImplAsync(SSma *pSma, int64_t version, const void *pMsg, int32_t len, int32_t inputType, SRSmaInfo *pInfo, tb_uid_t suid) { int32_t size = RSMA_EXEC_MSG_HLEN + len; // header + payload - void *qItem = taosAllocateQitem(size, DEF_QITEM, 0); + void * qItem = taosAllocateQitem(size, DEF_QITEM, 0); if (!qItem) { return TSDB_CODE_FAILED; @@ -870,10 +870,10 @@ static int32_t tdRsmaPrintSubmitReq(SSma *pSma, SSubmitReq *pReq) { * @param level * @return int32_t */ -static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int64_t version, int32_t inputType, SRSmaInfo *pInfo, - ERsmaExecType type, int8_t level) { +static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int64_t version, int32_t inputType, + SRSmaInfo *pInfo, ERsmaExecType type, int8_t level) { int32_t idx = level - 1; - void *qTaskInfo = RSMA_INFO_QTASK(pInfo, idx); + void * qTaskInfo = RSMA_INFO_QTASK(pInfo, idx); SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, idx); if (!qTaskInfo) { @@ -887,8 +887,9 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, return TSDB_CODE_FAILED; } - smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p, suid:%" PRIu64 ", nMsg:%d, submitReqVer:%" PRIi64 ", inputType:%d", SMA_VID(pSma), level, - RSMA_INFO_QTASK(pInfo, idx), pInfo->suid, msgSize, version, inputType); + smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p, suid:%" PRIu64 ", nMsg:%d, submitReqVer:%" PRIi64 + ", inputType:%d", + SMA_VID(pSma), level, RSMA_INFO_QTASK(pInfo, idx), pInfo->suid, msgSize, version, inputType); if ((terrno = qSetSMAInput(qTaskInfo, pMsg, msgSize, inputType)) < 0) { smaError("vgId:%d, rsma %" PRIi8 " qSetStreamInput failed since %s", SMA_VID(pSma), level, tstrerror(terrno)); @@ -912,7 +913,7 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) { int32_t code = 0; int32_t lino = 0; - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SSmaEnv * pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pStat = NULL; SRSmaInfo *pRSmaInfo = NULL; @@ -1067,8 +1068,8 @@ _err: static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { int32_t code = 0; int32_t lino = 0; - SVnode *pVnode = pSma->pVnode; - SArray *suidList = NULL; + SVnode * pVnode = pSma->pVnode; + SArray * suidList = NULL; STbUidStore uidStore = {0}; SMetaReader mr = {0}; tb_uid_t suid = 0; @@ -1196,7 +1197,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { int32_t code = 0; int32_t lino = 0; int32_t nTaskInfo = 0; - SSma *pSma = pRSmaStat->pSma; + SSma * pSma = pRSmaStat->pSma; SVnode *pVnode = pSma->pVnode; if (taosHashGetSize(pInfoHash) <= 0) { @@ -1229,7 +1230,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { do { int32_t nStreamFlushed = 0; int32_t nSleep = 0; - void *infoHash = NULL; + void * infoHash = NULL; while (true) { while ((infoHash = taosHashIterate(pInfoHash, infoHash))) { SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; @@ -1271,7 +1272,7 @@ _checkpoint: SStreamMeta *pMeta = NULL; int64_t checkpointId = taosGetTimestampNs(); bool checkpointBuilt = false; - void *infoHash = NULL; + void * infoHash = NULL; while ((infoHash = taosHashIterate(pInfoHash, infoHash))) { SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; if (RSMA_INFO_IS_DEL(pRSmaInfo)) { @@ -1282,11 +1283,12 @@ _checkpoint: SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pRSmaInfo, i); if (pItem && pItem->pStreamTask) { SStreamTask *pTask = pItem->pStreamTask; - atomic_store_32(&pTask->pMeta->chkptNotReadyTasks, 1); - pTask->checkpointingId = checkpointId; - pTask->chkInfo.checkpointId = pTask->checkpointingId; + // atomic_store_32(&pTask->pMeta->chkptNotReadyTasks, 1); + pTask->chkInfo.checkpointingId = checkpointId; + pTask->chkInfo.checkpointId = checkpointId; // 1pTask->checkpointingId; pTask->chkInfo.checkpointVer = pItem->submitReqVer; pTask->info.triggerParam = pItem->fetchResultVer; + pTask->info.taskLevel = TASK_LEVEL_SMA; if (!checkpointBuilt) { // the stream states share one checkpoint @@ -1342,10 +1344,10 @@ _exit: * @param tmrId */ static void tdRSmaFetchTrigger(void *param, void *tmrId) { - SRSmaRef *pRSmaRef = NULL; - SSma *pSma = NULL; - SRSmaStat *pStat = NULL; - SRSmaInfo *pRSmaInfo = NULL; + SRSmaRef * pRSmaRef = NULL; + SSma * pSma = NULL; + SRSmaStat * pStat = NULL; + SRSmaInfo * pRSmaInfo = NULL; SRSmaInfoItem *pItem = NULL; if (!(pRSmaRef = taosHashGet(smaMgmt.refHash, ¶m, POINTER_BYTES))) { @@ -1513,7 +1515,7 @@ _err: } static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SArray *pSubmitArr, ERsmaExecType type) { - void *msg = NULL; + void * msg = NULL; int8_t resume = 0; int32_t nSubmit = 0; int32_t nDelete = 0; @@ -1628,11 +1630,11 @@ _err: int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { int32_t code = 0; int32_t lino = 0; - SVnode *pVnode = pSma->pVnode; - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SVnode * pVnode = pSma->pVnode; + SSmaEnv * pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - SHashObj *infoHash = NULL; - SArray *pSubmitArr = NULL; + SHashObj * infoHash = NULL; + SArray * pSubmitArr = NULL; bool isFetchAll = false; if (!pRSmaStat || !(infoHash = RSMA_INFO_HASH(pRSmaStat))) { @@ -1731,4 +1733,4 @@ _exit: smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); } return code; -} +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index e66c0357a1..ee76a27414 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -17,12 +17,6 @@ #include "vnd.h" #include "tqCommon.h" -typedef struct { - int8_t inited; -} STqMgmt; - -static STqMgmt tqMgmt = {0}; - // 0: not init // 1: already inited // 2: wait to be inited or cleaup @@ -32,36 +26,6 @@ static FORCE_INLINE bool tqIsHandleExec(STqHandle* pHandle) { return TMQ_HANDLE_ static FORCE_INLINE void tqSetHandleExec(STqHandle* pHandle) { pHandle->status = TMQ_HANDLE_STATUS_EXEC; } static FORCE_INLINE void tqSetHandleIdle(STqHandle* pHandle) { pHandle->status = TMQ_HANDLE_STATUS_IDLE; } -int32_t tqInit() { - int8_t old; - while (1) { - old = atomic_val_compare_exchange_8(&tqMgmt.inited, 0, 2); - if (old != 2) break; - } - - if (old == 0) { - if (streamInit() < 0) { - return -1; - } - atomic_store_8(&tqMgmt.inited, 1); - } - - return 0; -} - -void tqCleanUp() { - int8_t old; - while (1) { - old = atomic_val_compare_exchange_8(&tqMgmt.inited, 1, 2); - if (old != 2) break; - } - - if (old == 1) { - streamCleanUp(); - atomic_store_8(&tqMgmt.inited, 0); - } -} - void tqDestroyTqHandle(void* data) { STqHandle* pData = (STqHandle*)data; qDestroyTask(pData->execHandle.task); @@ -337,7 +301,7 @@ int32_t tqProcessPollPush(STQ* pTq, SRpcMsg* pMsg) { while (pIter) { STqHandle* pHandle = *(STqHandle**)pIter; - tqInfo("vgId:%d start set submit for pHandle:%p, consumer:0x%" PRIx64, vgId, pHandle, pHandle->consumerId); + tqDebug("vgId:%d start set submit for pHandle:%p, consumer:0x%" PRIx64, vgId, pHandle, pHandle->consumerId); if (ASSERT(pHandle->msg != NULL)) { tqError("pHandle->msg should not be null"); @@ -750,21 +714,27 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { SStreamTask* pStateTask = pTask; - SStreamTask task = {0}; + + STaskId taskId = {.streamId = 0, .taskId = 0}; if (pTask->info.fillHistory) { - task.id.streamId = pTask->streamTaskId.streamId; - task.id.taskId = pTask->streamTaskId.taskId; - task.pMeta = pTask->pMeta; - pStateTask = &task; + taskId.streamId = pTask->id.streamId; + taskId.taskId = pTask->id.taskId; + + pTask->id.streamId = pTask->streamTaskId.streamId; + pTask->id.taskId = pTask->streamTaskId.taskId; } - pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pStateTask, false, -1, -1); + pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1); if (pTask->pState == NULL) { tqError("s-task:%s (vgId:%d) failed to open state for task", pTask->id.idStr, vgId); return -1; } else { tqDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState); } + if (pTask->info.fillHistory) { + pTask->id.streamId = taskId.streamId; + pTask->id.taskId = taskId.taskId; + } SReadHandle handle = { .checkpointId = pTask->chkInfo.checkpointId, @@ -785,15 +755,17 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId); } else if (pTask->info.taskLevel == TASK_LEVEL__AGG) { SStreamTask* pSateTask = pTask; - SStreamTask task = {0}; + // SStreamTask task = {0}; + + STaskId taskId = {.streamId = 0, .taskId = 0}; if (pTask->info.fillHistory) { - task.id.streamId = pTask->streamTaskId.streamId; - task.id.taskId = pTask->streamTaskId.taskId; - task.pMeta = pTask->pMeta; - pSateTask = &task; + taskId.streamId = pTask->id.streamId; + taskId.taskId = pTask->id.taskId; + pTask->id.streamId = pTask->streamTaskId.streamId; + pTask->id.taskId = pTask->streamTaskId.taskId; } - pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pSateTask, false, -1, -1); + pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1); if (pTask->pState == NULL) { tqError("s-task:%s (vgId:%d) failed to open state for task", pTask->id.idStr, vgId); return -1; @@ -801,6 +773,11 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { tqDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState); } + if (pTask->info.fillHistory) { + pTask->id.streamId = taskId.streamId; + pTask->id.taskId = taskId.taskId; + } + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList); SReadHandle handle = { .checkpointId = pTask->chkInfo.checkpointId, @@ -1280,14 +1257,13 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) return TSDB_CODE_SUCCESS; } - // downstream not ready, current the stream tasks are not all ready. Ignore this checkpoint req. if (pTask->status.downstreamReady != 1) { pTask->chkInfo.failedId = req.checkpointId; // record the latest failed checkpoint id - pTask->checkpointingId = req.checkpointId; + pTask->chkInfo.checkpointingId = req.checkpointId; - qError("s-task:%s not ready for checkpoint, since downstream not ready, ignore this checkpoint:%" PRId64 - ", set it failure", - pTask->id.idStr, req.checkpointId); + tqError("s-task:%s not ready for checkpoint, since downstream not ready, ignore this checkpoint:%" PRId64 + ", set it failure", + pTask->id.idStr, req.checkpointId); streamMetaReleaseTask(pMeta, pTask); SRpcMsg rsp = {0}; @@ -1316,10 +1292,10 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) // check if the checkpoint msg already sent or not. if (status == TASK_STATUS__CK) { - ASSERT(pTask->checkpointingId == req.checkpointId); + ASSERT(pTask->chkInfo.checkpointingId == req.checkpointId); tqWarn("s-task:%s recv checkpoint-source msg again checkpointId:%" PRId64 " already received, ignore this msg and continue process checkpoint", - pTask->id.idStr, pTask->checkpointingId); + pTask->id.idStr, pTask->chkInfo.checkpointingId); taosThreadMutexUnlock(&pTask->lock); streamMetaReleaseTask(pMeta, pTask); @@ -1335,10 +1311,6 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) // set the initial value for generating check point // set the mgmt epset info according to the checkout source msg from mnode, todo update mgmt epset if needed - if (pMeta->chkptNotReadyTasks == 0) { - pMeta->chkptNotReadyTasks = pMeta->numOfStreamTasks; - } - total = pMeta->numOfStreamTasks; streamMetaWUnLock(pMeta); @@ -1390,7 +1362,7 @@ int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) { } int32_t tqProcessTaskDropHTask(STQ* pTq, SRpcMsg* pMsg) { - SVDropHTaskReq* pReq = (SVDropHTaskReq*) pMsg->pCont; + SVDropHTaskReq* pReq = (SVDropHTaskReq*)pMsg->pCont; SStreamMeta* pMeta = pTq->pStreamMeta; SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); @@ -1415,6 +1387,9 @@ int32_t tqProcessTaskDropHTask(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskId id = {.streamId = pTask->hTaskInfo.id.streamId, .taskId = pTask->hTaskInfo.id.taskId}; streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &id); + // clear the scheduler status + streamTaskSetSchedStatusInactive(pTask); + tqDebug("s-task:%s set scheduler status:%d after drop fill-history task", pTask->id.idStr, pTask->status.schedStatus); streamMetaReleaseTask(pMeta, pTask); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index f367bc96f8..8fee1d5904 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -72,7 +72,7 @@ int32_t tqRegisterPushHandle(STQ* pTq, void* handle, SRpcMsg* pMsg) { memcpy(pHandle->msg->pCont, pMsg->pCont, pMsg->contLen); pHandle->msg->contLen = pMsg->contLen; int32_t ret = taosHashPut(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey), &pHandle, POINTER_BYTES); - tqInfo("vgId:%d data is over, ret:%d, consumerId:0x%" PRIx64 ", register to pHandle:%p, pCont:%p, len:%d", vgId, ret, + tqDebug("vgId:%d data is over, ret:%d, consumerId:0x%" PRIx64 ", register to pHandle:%p, pCont:%p, len:%d", vgId, ret, pHandle->consumerId, pHandle, pHandle->msg->pCont, pHandle->msg->contLen); return 0; } diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c index 7a8147f83b..2ab710176d 100644 --- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c @@ -104,8 +104,8 @@ int32_t streamStateSnapRead(SStreamStateReader* pReader, uint8_t** ppData) { pHdr->type = SNAP_DATA_STREAM_STATE_BACKEND; pHdr->size = len; memcpy(pHdr->data, rowData, len); - tqDebug("vgId:%d, vnode stream-state snapshot read data success", TD_VID(pReader->pTq->pVnode)); taosMemoryFree(rowData); + tqDebug("vgId:%d, vnode stream-state snapshot read data success", TD_VID(pReader->pTq->pVnode)); return code; _err: @@ -139,7 +139,7 @@ int32_t streamStateSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamS pWriter->sver = sver; pWriter->ever = ever; - sprintf(tdir, "%s%s%s%s%s", pTq->path, TD_DIRSEP, VNODE_TQ_STREAM, TD_DIRSEP, "received"); + sprintf(tdir, "%s%s%s", pTq->path, TD_DIRSEP, VNODE_TQ_STREAM); taosMkDir(tdir); SStreamSnapWriter* pSnapWriter = NULL; @@ -167,25 +167,19 @@ int32_t streamStateSnapWriterClose(SStreamStateWriter* pWriter, int8_t rollback) return code; } -int32_t streamStateRebuildFromSnap(SStreamStateWriter* pWriter, int64_t chkpId) { - tqDebug("vgId:%d, vnode %s start to rebuild stream-state", TD_VID(pWriter->pTq->pVnode), STREAM_STATE_TRANSFER); - - streamMetaWLock(pWriter->pTq->pStreamMeta); - int32_t code = streamMetaReopen(pWriter->pTq->pStreamMeta); - if (code == 0) { - streamMetaInitBackend(pWriter->pTq->pStreamMeta); - code = streamStateLoadTasks(pWriter); - } - - streamMetaWUnLock(pWriter->pTq->pStreamMeta); - tqDebug("vgId:%d, vnode %s succ to rebuild stream-state", TD_VID(pWriter->pTq->pVnode), STREAM_STATE_TRANSFER); - taosMemoryFree(pWriter); - return code; -} - -int32_t streamStateLoadTasks(SStreamStateWriter* pWriter) { return streamMetaLoadAllTasks(pWriter->pTq->pStreamMeta); } int32_t streamStateSnapWrite(SStreamStateWriter* pWriter, uint8_t* pData, uint32_t nData) { tqDebug("vgId:%d, vnode %s snapshot write data", TD_VID(pWriter->pTq->pVnode), STREAM_STATE_TRANSFER); return streamSnapWrite(pWriter->pWriterImpl, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); } +int32_t streamStateRebuildFromSnap(SStreamStateWriter* pWriter, int64_t chkpId) { + tqDebug("vgId:%d, vnode %s start to rebuild stream-state", TD_VID(pWriter->pTq->pVnode), STREAM_STATE_TRANSFER); + int32_t code = streamStateLoadTasks(pWriter); + tqDebug("vgId:%d, vnode %s succ to rebuild stream-state", TD_VID(pWriter->pTq->pVnode), STREAM_STATE_TRANSFER); + taosMemoryFree(pWriter); + return code; +} + +int32_t streamStateLoadTasks(SStreamStateWriter* pWriter) { + return streamMetaReloadAllTasks(pWriter->pTq->pStreamMeta); +} diff --git a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c index f22ecc3daf..f966e90b9a 100644 --- a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c @@ -238,7 +238,6 @@ int32_t streamTaskSnapWrite(SStreamTaskWriter* pWriter, uint8_t* pData, uint32_t goto _err; } tDecoderClear(&decoder); - // tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) int64_t key[2] = {taskId.streamId, taskId.taskId}; taosWLockLatch(&pTq->pStreamMeta->lock); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 8f62928d22..110cf79b4e 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -168,6 +168,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, taosWUnLockLatch(&pTq->lock); } + dataRsp.reqOffset = *pOffset; // reqOffset represents the current date offset, may be changed if wal not exists code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); end : { diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index aee2aaa244..b1d49bf31b 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -475,6 +475,7 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); if (!isLeader) { + streamMetaUpdateTaskDownstreamStatus(pMeta, rsp.streamId, rsp.upstreamTaskId, 0, taosGetTimestampMs(), false); tqError("vgId:%d not leader, task:0x%x not handle the check rsp, downstream:0x%x (vgId:%d)", vgId, rsp.upstreamTaskId, rsp.downstreamTaskId, rsp.downstreamNodeId); return code; @@ -482,6 +483,7 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe SStreamTask* pTask = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.upstreamTaskId); if (pTask == NULL) { + streamMetaUpdateTaskDownstreamStatus(pMeta, rsp.streamId, rsp.upstreamTaskId, 0, taosGetTimestampMs(), false); tqError("tq failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped", rsp.streamId, rsp.upstreamTaskId, vgId); terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST; @@ -670,7 +672,8 @@ int32_t startStreamTasks(SStreamMeta* pMeta) { streamLaunchFillHistoryTask(pTask); } - streamMetaUpdateTaskDownstreamStatus(pTask, pTask->execInfo.init, pTask->execInfo.start, true); + streamMetaUpdateTaskDownstreamStatus(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, pTask->execInfo.init, + pTask->execInfo.start, true); streamMetaReleaseTask(pMeta, pTask); continue; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 918d0bd7d0..b1424c1944 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1131,9 +1131,13 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE char **values_list = taosMemoryCalloc(num_keys * 2, sizeof(char *)); size_t *values_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t)); char **errs = taosMemoryCalloc(num_keys * 2, sizeof(char *)); + + (void)tsdbCacheCommit(pTsdb); + taosThreadMutexLock(&pTsdb->lruMutex); + taosThreadMutexLock(&pTsdb->rCache.rMutex); - rocksMayWrite(pTsdb, true, false, false); + // rocksMayWrite(pTsdb, true, false, false); rocksdb_multi_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, num_keys * 2, (const char *const *)keys_list, keys_list_sizes, values_list, values_list_sizes, errs); taosThreadMutexUnlock(&pTsdb->rCache.rMutex); @@ -1163,26 +1167,33 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE // taosThreadMutexLock(&pTsdb->lruMutex); + bool erase = false; LRUHandle *h = taosLRUCacheLookup(pTsdb->lruCache, keys_list[i], klen); if (h) { SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pTsdb->lruCache, h); - if (pLastCol->dirty) { + if (pLastCol->dirty && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) { pLastCol->dirty = 0; + erase = true; } - taosLRUCacheRelease(pTsdb->lruCache, h, true); + taosLRUCacheRelease(pTsdb->lruCache, h, erase); + } + if (erase) { + taosLRUCacheErase(pTsdb->lruCache, keys_list[i], klen); } - taosLRUCacheErase(pTsdb->lruCache, keys_list[i], klen); + erase = false; h = taosLRUCacheLookup(pTsdb->lruCache, keys_list[num_keys + i], klen); if (h) { SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pTsdb->lruCache, h); - if (pLastCol->dirty) { + if (pLastCol->dirty && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) { pLastCol->dirty = 0; + erase = true; } - taosLRUCacheRelease(pTsdb->lruCache, h, true); + taosLRUCacheRelease(pTsdb->lruCache, h, erase); + } + if (erase) { + taosLRUCacheErase(pTsdb->lruCache, keys_list[num_keys + i], klen); } - taosLRUCacheErase(pTsdb->lruCache, keys_list[num_keys + i], klen); - // taosThreadMutexUnlock(&pTsdb->lruMutex); } for (int i = 0; i < num_keys; ++i) { @@ -1793,8 +1804,8 @@ static int32_t loadTombFromBlk(const TTombBlkArray *pTombBlkArray, SCacheRowsRea } if (record.version <= pReader->info.verRange.maxVer) { - tsdbError("tomb xx load/cache: vgId:%d fid:%d commit %" PRId64 "~%" PRId64 "~%" PRId64 " tomb records", - TD_VID(pReader->pTsdb->pVnode), pReader->pCurFileSet->fid, record.skey, record.ekey, uid); + /*tsdbError("tomb xx load/cache: vgId:%d fid:%d record %" PRId64 "~%" PRId64 "~%" PRId64 " tomb records", + TD_VID(pReader->pTsdb->pVnode), pReader->pCurFileSet->fid, record.skey, record.ekey, uid);*/ SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey}; taosArrayPush(pInfo->pTombData, &delData); diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index b6aa791cf0..f668ea5f72 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -22,19 +22,34 @@ #define HASTYPE(_type, _t) (((_type) & (_t)) == (_t)) +static void setFirstLastResColToNull(SColumnInfoData* pCol, int32_t row) { + char *buf = taosMemoryCalloc(1, pCol->info.bytes); + SFirstLastRes* pRes = (SFirstLastRes*)((char*)buf + VARSTR_HEADER_SIZE); + pRes->bytes = 0; + pRes->hasResult = true; + pRes->isNull = true; + varDataSetLen(buf, pCol->info.bytes - VARSTR_HEADER_SIZE); + colDataSetVal(pCol, row, buf, false); + taosMemoryFree(buf); +} + static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* pReader, const int32_t* slotIds, const int32_t* dstSlotIds, void** pRes, const char* idStr) { int32_t numOfRows = pBlock->info.rows; // bool allNullRow = true; if (HASTYPE(pReader->type, CACHESCAN_RETRIEVE_LAST)) { - uint64_t ts = 0; - SFirstLastRes* p; - col_id_t colId; + uint64_t ts = TSKEY_MIN; + SFirstLastRes* p = NULL; + col_id_t colId = -1; for (int32_t i = 0; i < pReader->numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]); - int32_t slotId = slotIds[i]; - SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i); + if (slotIds[i] == -1) { + setFirstLastResColToNull(pColInfoData, numOfRows); + continue; + } + int32_t slotId = slotIds[i]; + SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i); colId = pColVal->colVal.cid; p = (SFirstLastRes*)varDataVal(pRes[i]); @@ -63,10 +78,14 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p for (int32_t idx = 0; idx < taosArrayGetSize(pBlock->pDataBlock); ++idx) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, idx); if (pCol->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID && pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP) { - colDataSetVal(pCol, numOfRows, (const char*)&ts, false); + if (ts == TSKEY_MIN) { + colDataSetNULL(pCol, numOfRows); + } else { + colDataSetVal(pCol, numOfRows, (const char*)&ts, false); + } continue; - } else if (pReader->numOfCols == 1 && idx != dstSlotIds[0] && pCol->info.colId == colId) { - if (!p->isNull) { + } else if (pReader->numOfCols == 1 && idx != dstSlotIds[0] && (pCol->info.colId == colId || colId == -1)) { + if (p && !p->isNull) { colDataSetVal(pCol, numOfRows, p->buf, false); } else { colDataSetNULL(pCol, numOfRows); @@ -81,6 +100,10 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]); int32_t slotId = slotIds[i]; + if (slotId == -1) { + colDataSetNULL(pColInfoData, numOfRows); + continue; + } SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i); SColVal* pVal = &pColVal->colVal; @@ -300,7 +323,13 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } for (int32_t j = 0; j < pr->numOfCols; ++j) { - pRes[j] = taosMemoryCalloc(1, sizeof(SFirstLastRes) + pr->pSchema->columns[slotIds[j]].bytes + VARSTR_HEADER_SIZE); + int32_t bytes; + if (slotIds[j] == -1) + bytes = 1; + else + bytes = pr->pSchema->columns[slotIds[j]].bytes; + + pRes[j] = taosMemoryCalloc(1, sizeof(SFirstLastRes) + bytes + VARSTR_HEADER_SIZE); SFirstLastRes* p = (SFirstLastRes*)varDataVal(pRes[j]); p->ts = INT64_MIN; } @@ -324,6 +353,11 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 for (int32_t i = 0; i < pr->numOfCols; ++i) { int32_t slotId = slotIds[i]; + if (slotId == -1) { + SLastCol p = {.ts = INT64_MIN, .colVal.type = TSDB_DATA_TYPE_BOOL, .colVal.flag = CV_FLAG_NULL}; + taosArrayPush(pLastCols, &p); + continue; + } struct STColumn* pCol = &pr->pSchema->columns[slotId]; SLastCol p = {.ts = INT64_MIN, .colVal.type = pCol->type, .colVal.flag = CV_FLAG_NULL}; @@ -348,6 +382,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 bool hasNotNullRow = true; int64_t singleTableLastTs = INT64_MAX; for (int32_t k = 0; k < pr->numOfCols; ++k) { + if (slotIds[k] == -1) continue; SLastCol* p = taosArrayGet(pLastCols, k); SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, k); diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c new file mode 100644 index 0000000000..3f448379c9 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbDataFileRAW.h" + +// SDataFileRAWReader ============================================= +int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderConfig *config, + SDataFileRAWReader **reader) { + int32_t code = 0; + int32_t lino = 0; + + reader[0] = taosMemoryCalloc(1, sizeof(SDataFileRAWReader)); + if (reader[0] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + reader[0]->config[0] = config[0]; + + if (fname) { + if (fname) { + code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else { + char fname1[TSDB_FILENAME_LEN]; + tsdbTFileName(config->tsdb, &config->file, fname1); + code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileRAWReaderClose(SDataFileRAWReader **reader) { + if (reader[0] == NULL) return 0; + + if (reader[0]->fd) { + tsdbCloseFile(&reader[0]->fd); + } + + taosMemoryFree(reader[0]); + reader[0] = NULL; + return 0; +} + +int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlockHeader *pBlock) { + int32_t code = 0; + int32_t lino = 0; + + pBlock->file.type = reader->config->file.type; + pBlock->file.fid = reader->config->file.fid; + pBlock->file.cid = reader->config->file.cid; + pBlock->file.size = reader->config->file.size; + pBlock->file.minVer = reader->config->file.minVer; + pBlock->file.maxVer = reader->config->file.maxVer; + pBlock->file.stt->level = reader->config->file.stt->level; + + code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +// SDataFileRAWWriter ============================================= +int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataFileRAWWriter **ppWriter) { + int32_t code = 0; + int32_t lino = 0; + + SDataFileRAWWriter *writer = taosMemoryCalloc(1, sizeof(SDataFileRAWWriter)); + if (!writer) return TSDB_CODE_OUT_OF_MEMORY; + + writer->config[0] = config[0]; + + code = tsdbDataFileRAWWriterDoOpen(writer); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + taosMemoryFree(writer); + writer = NULL; + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + ppWriter[0] = writer; + return code; +} + +static int32_t tsdbDataFileRAWWriterCloseAbort(SDataFileRAWWriter *writer) { + ASSERT(0); + return 0; +} + +static int32_t tsdbDataFileRAWWriterDoClose(SDataFileRAWWriter *writer) { return 0; } + +static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFileOpArray *opArr) { + int32_t code = 0; + int32_t lino = 0; + ASSERT(writer->ctx->offset == writer->file.size); + ASSERT(writer->config->fid == writer->file.fid); + + STFileOp op = (STFileOp){ + .optype = TSDB_FOP_CREATE, + .fid = writer->config->fid, + .nf = writer->file, + }; + code = TARRAY2_APPEND(opArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + + if (writer->fd) { + code = tsdbFsyncFile(writer->fd); + TSDB_CHECK_CODE(code, lino, _exit); + tsdbCloseFile(&writer->fd); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileRAWWriterOpenDataFD(SDataFileRAWWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + char fname[TSDB_FILENAME_LEN]; + int32_t flag = TD_FILE_READ | TD_FILE_WRITE; + + if (writer->ctx->offset == 0) { + flag |= (TD_FILE_CREATE | TD_FILE_TRUNC); + } + + tsdbTFileName(writer->config->tsdb, &writer->file, fname); + code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + writer->file = writer->config->file; + writer->ctx->offset = 0; + + code = tsdbDataFileRAWWriterOpenDataFD(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->ctx->opened = true; +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr) { + if (writer[0] == NULL) return 0; + + int32_t code = 0; + int32_t lino = 0; + + if (writer[0]->ctx->opened) { + if (abort) { + code = tsdbDataFileRAWWriterCloseAbort(writer[0]); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbDataFileRAWWriterCloseCommit(writer[0], opArr); + TSDB_CHECK_CODE(code, lino, _exit); + } + tsdbDataFileRAWWriterDoClose(writer[0]); + } + taosMemoryFree(writer[0]); + writer[0] = NULL; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer[0]->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *pDataBlock) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->ctx->offset += pDataBlock->dataLength; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h new file mode 100644 index 0000000000..d765671698 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tarray2.h" +#include "tsdbDef.h" +#include "tsdbFSet2.h" +#include "tsdbFile2.h" +#include "tsdbUtil2.h" + +#ifndef _TSDB_DATA_FILE_RAW_H +#define _TSDB_DATA_FILE_RAW_H + +#ifdef __cplusplus +extern "C" { +#endif + +// STsdbDataRAWBlockHeader ======================================= +typedef struct STsdbDataRAWBlockHeader { + struct { + int32_t type; + int64_t fid; + int64_t cid; + int64_t size; + int64_t minVer; + int64_t maxVer; + union { + struct { + int32_t level; + } stt[1]; + }; + } file; + + int64_t offset; + int64_t dataLength; + uint8_t data[0]; +} STsdbDataRAWBlockHeader; + +// SDataFileRAWReader ============================================= +typedef struct SDataFileRAWReaderConfig { + STsdb *tsdb; + int32_t szPage; + + STFile file; +} SDataFileRAWReaderConfig; + +typedef struct SDataFileRAWReader { + SDataFileRAWReaderConfig config[1]; + + struct { + bool opened; + int64_t offset; + } ctx[1]; + + STsdbFD *fd; +} SDataFileRAWReader; + +typedef TARRAY2(SDataFileRAWReader *) SDataFileRAWReaderArray; + +int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderConfig *config, + SDataFileRAWReader **reader); +int32_t tsdbDataFileRAWReaderClose(SDataFileRAWReader **reader); + +int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlockHeader *bHdr); + +// SDataFileRAWWriter ============================================= +typedef struct SDataFileRAWWriterConfig { + STsdb *tsdb; + int32_t szPage; + + SDiskID did; + int64_t fid; + int64_t cid; + int32_t level; + + STFile file; +} SDataFileRAWWriterConfig; + +typedef struct SDataFileRAWWriter { + SDataFileRAWWriterConfig config[1]; + + struct { + bool opened; + int64_t offset; + } ctx[1]; + + STFile file; + STsdbFD *fd; +} SDataFileRAWWriter; + +typedef struct SDataFileRAWWriter SDataFileRAWWriter; + +int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataFileRAWWriter **writer); +int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr); + +int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer); +int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *bHdr); +int32_t tsdbDataFileRAWFlush(SDataFileRAWWriter *writer); + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_DATA_FILE_RAW_H*/ diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 89bed6b42f..e933b3a7dc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -20,8 +20,6 @@ #define BLOCK_COMMIT_FACTOR 3 -extern int vnodeScheduleTask(int (*execute)(void *), void *arg); -extern int vnodeScheduleTaskEx(int tpid, int (*execute)(void *), void *arg); extern void remove_file(const char *fname, bool last_level); #define TSDB_FS_EDIT_MIN TSDB_FEDIT_COMMIT @@ -651,7 +649,6 @@ _exit: static int32_t close_file_system(STFileSystem *fs) { TARRAY2_CLEAR(fs->fSetArr, tsdbTFileSetClear); TARRAY2_CLEAR(fs->fSetArrTmp, tsdbTFileSetClear); - // TODO return 0; } @@ -748,36 +745,48 @@ _exit: return code; } -static void tsdbDoWaitBgTask(STFileSystem *fs, STFSBgTask *task) { - task->numWait++; - taosThreadCondWait(task->done, &fs->tsdb->mutex); - task->numWait--; +static int32_t tsdbFSSetBlockCommit(STFileSet *fset, bool block); - if (task->numWait == 0) { - taosThreadCondDestroy(task->done); - if (task->destroy) { - task->destroy(task->arg); +int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) { + STFileSystem *fs = pTsdb->pFS; + TARRAY2(int64_t) channelArr = {0}; + + taosThreadMutexLock(&fs->tsdb->mutex); + + // disable + pTsdb->bgTaskDisabled = true; + + // collect channel + STFileSet *fset; + TARRAY2_FOREACH(fs->fSetArr, fset) { + if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) { + TARRAY2_APPEND(&channelArr, fset->bgTaskChannel); + fset->bgTaskChannel = 0; } - taosMemoryFree(task); + fset->mergeScheduled = false; + tsdbFSSetBlockCommit(fset, false); } + + taosThreadMutexUnlock(&fs->tsdb->mutex); + + // destroy all channels + int64_t channel; + TARRAY2_FOREACH(&channelArr, channel) { vnodeAChannelDestroy(vnodeAsyncHandle[1], channel, true); } + TARRAY2_DESTROY(&channelArr, NULL); + return 0; } -static void tsdbDoDoneBgTask(STFileSystem *fs, STFSBgTask *task) { - if (task->numWait > 0) { - taosThreadCondBroadcast(task->done); - } else { - taosThreadCondDestroy(task->done); - if (task->destroy) { - task->destroy(task->arg); - } - taosMemoryFree(task); - } +int32_t tsdbEnableBgTask(STsdb *pTsdb) { + taosThreadMutexLock(&pTsdb->mutex); + pTsdb->bgTaskDisabled = false; + taosThreadMutexUnlock(&pTsdb->mutex); + return 0; } int32_t tsdbCloseFS(STFileSystem **fs) { if (fs[0] == NULL) return 0; - tsdbFSDisableBgTask(fs[0]); + tsdbDisableAndCancelAllBgTask((*fs)->tsdb); close_file_system(fs[0]); destroy_fs(fs); return 0; @@ -865,7 +874,7 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) { // schedule merge int32_t sttTrigger = fs->tsdb->pVnode->config.sttTrigger; - if (sttTrigger > 1) { + if (sttTrigger > 1 && !fs->tsdb->bgTaskDisabled) { STFileSet *fset; TARRAY2_FOREACH_REVERSE(fs->fSetArr, fset) { if (TARRAY2_SIZE(fset->lvlArr) == 0) { @@ -881,7 +890,7 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) { bool skipMerge = false; int32_t numFile = TARRAY2_SIZE(lvl->fobjArr); - if (numFile >= sttTrigger) { + if (numFile >= sttTrigger && (!fset->mergeScheduled)) { // launch merge { extern int8_t tsS3Enabled; @@ -910,8 +919,22 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) { } if (!skipMerge) { - code = tsdbSchedMerge(fs->tsdb, fset->fid); + code = tsdbTFileSetOpenChannel(fset); TSDB_CHECK_CODE(code, lino, _exit); + + SMergeArg *arg = taosMemoryMalloc(sizeof(*arg)); + if (arg == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + arg->tsdb = fs->tsdb; + arg->fid = fset->fid; + + code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree, + arg, NULL); + TSDB_CHECK_CODE(code, lino, _exit); + fset->mergeScheduled = true; } } @@ -939,7 +962,11 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) { } } - if (tsdbTFileSetIsEmpty(fset) && fset->bgTaskRunning == NULL) { + if (tsdbTFileSetIsEmpty(fset)) { + if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) { + vnodeAChannelDestroy(vnodeAsyncHandle[1], fset->bgTaskChannel, false); + fset->bgTaskChannel = 0; + } TARRAY2_REMOVE(fs->fSetArr, i, tsdbTFileSetClear); } else { i++; @@ -1045,7 +1072,25 @@ int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr) { return 0; } -int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRanges, TFileSetArray **fsetArr, +static SHashObj *tsdbFSetRangeArrayToHash(TFileSetRangeArray *pRanges) { + int32_t capacity = TARRAY2_SIZE(pRanges) * 2; + SHashObj *pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); + if (pHash == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + for (int32_t i = 0; i < TARRAY2_SIZE(pRanges); i++) { + STFileSetRange *u = TARRAY2_GET(pRanges, i); + int32_t fid = u->fid; + int32_t code = taosHashPut(pHash, &fid, sizeof(fid), u, sizeof(*u)); + ASSERT(code == 0); + tsdbDebug("range diff hash fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever); + } + return pHash; +} + +int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TFileSetRangeArray *pRanges, TFileSetArray **fsetArr, TFileOpArray *fopArr) { int32_t code = 0; STFileSet *fset; @@ -1057,7 +1102,7 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRange TARRAY2_INIT(fsetArr[0]); if (pRanges) { - pHash = tsdbGetSnapRangeHash(pRanges); + pHash = tsdbFSetRangeArrayToHash(pRanges); if (pHash == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _out; @@ -1069,7 +1114,7 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRange int64_t ever = VERSION_MAX; if (pHash) { int32_t fid = fset->fid; - STSnapRange *u = taosHashGet(pHash, &fid, sizeof(fid)); + STFileSetRange *u = taosHashGet(pHash, &fid, sizeof(fid)); if (u) { ever = u->sver - 1; } @@ -1096,29 +1141,13 @@ _out: return code; } -SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges) { - int32_t capacity = TARRAY2_SIZE(pRanges) * 2; - SHashObj *pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); - if (pHash == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } +int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr) { return tsdbFSDestroyCopySnapshot(fsetArr); } - for (int32_t i = 0; i < TARRAY2_SIZE(pRanges); i++) { - STSnapRange *u = TARRAY2_GET(pRanges, i); - int32_t fid = u->fid; - int32_t code = taosHashPut(pHash, &fid, sizeof(fid), u, sizeof(*u)); - ASSERT(code == 0); - tsdbDebug("range diff hash fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever); - } - return pHash; -} - -int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pRanges, - TSnapRangeArray **fsrArr) { +int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TFileSetRangeArray *pRanges, + TFileSetRangeArray **fsrArr) { int32_t code = 0; STFileSet *fset; - STSnapRange *fsr1 = NULL; + STFileSetRange *fsr1 = NULL; SHashObj *pHash = NULL; fsrArr[0] = taosMemoryCalloc(1, sizeof(*fsrArr[0])); @@ -1129,7 +1158,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev tsdbInfo("pRanges size:%d", (pRanges == NULL ? 0 : TARRAY2_SIZE(pRanges))); if (pRanges) { - pHash = tsdbGetSnapRangeHash(pRanges); + pHash = tsdbFSetRangeArrayToHash(pRanges); if (pHash == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _out; @@ -1143,7 +1172,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev if (pHash) { int32_t fid = fset->fid; - STSnapRange *u = taosHashGet(pHash, &fid, sizeof(fid)); + STFileSetRange *u = taosHashGet(pHash, &fid, sizeof(fid)); if (u) { sver1 = u->sver; tsdbDebug("range hash get fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever); @@ -1157,7 +1186,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev tsdbDebug("fsrArr:%p, fid:%d, sver:%" PRId64 ", ever:%" PRId64, fsrArr, fset->fid, sver1, ever1); - code = tsdbTSnapRangeInitRef(fs->tsdb, fset, sver1, ever1, &fsr1); + code = tsdbTFileSetRangeInitRef(fs->tsdb, fset, sver1, ever1, &fsr1); if (code) break; code = TARRAY2_APPEND(fsrArr[0], fsr1); @@ -1168,8 +1197,8 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev taosThreadMutexUnlock(&fs->tsdb->mutex); if (code) { - tsdbTSnapRangeClear(&fsr1); - TARRAY2_DESTROY(fsrArr[0], tsdbTSnapRangeClear); + tsdbTFileSetRangeClear(&fsr1); + TARRAY2_DESTROY(fsrArr[0], tsdbTFileSetRangeClear); fsrArr[0] = NULL; } @@ -1181,134 +1210,4 @@ _out: return code; } -const char *gFSBgTaskName[] = {NULL, "MERGE", "RETENTION", "COMPACT"}; - -static int32_t tsdbFSRunBgTask(void *arg) { - STFSBgTask *task = (STFSBgTask *)arg; - STFileSystem *fs = task->fs; - - task->launchTime = taosGetTimestampMs(); - task->run(task->arg); - task->finishTime = taosGetTimestampMs(); - - tsdbDebug("vgId:%d bg task:%s task id:%" PRId64 " finished, schedule time:%" PRId64 " launch time:%" PRId64 - " finish time:%" PRId64, - TD_VID(fs->tsdb->pVnode), gFSBgTaskName[task->type], task->taskid, task->scheduleTime, task->launchTime, - task->finishTime); - - taosThreadMutexLock(&fs->tsdb->mutex); - - STFileSet *fset = NULL; - tsdbFSGetFSet(fs, task->fid, &fset); - ASSERT(fset != NULL && fset->bgTaskRunning == task); - - // free last - tsdbDoDoneBgTask(fs, task); - fset->bgTaskRunning = NULL; - - // schedule next - if (fset->bgTaskNum > 0) { - if (fs->stop) { - while (fset->bgTaskNum > 0) { - STFSBgTask *nextTask = fset->bgTaskQueue->next; - nextTask->prev->next = nextTask->next; - nextTask->next->prev = nextTask->prev; - fset->bgTaskNum--; - tsdbDoDoneBgTask(fs, nextTask); - } - } else { - // pop task from head - fset->bgTaskRunning = fset->bgTaskQueue->next; - fset->bgTaskRunning->prev->next = fset->bgTaskRunning->next; - fset->bgTaskRunning->next->prev = fset->bgTaskRunning->prev; - fset->bgTaskNum--; - vnodeScheduleTaskEx(1, tsdbFSRunBgTask, fset->bgTaskRunning); - } - } - - taosThreadMutexUnlock(&fs->tsdb->mutex); - return 0; -} - -// IMPORTANT: the caller must hold the fs->tsdb->mutex -int32_t tsdbFSScheduleBgTask(STFileSystem *fs, int32_t fid, EFSBgTaskT type, int32_t (*run)(void *), - void (*destroy)(void *), void *arg, int64_t *taskid) { - if (fs->stop) { - if (destroy) { - destroy(arg); - } - return 0; - } - - STFileSet *fset; - tsdbFSGetFSet(fs, fid, &fset); - - ASSERT(fset != NULL); - - for (STFSBgTask *task = fset->bgTaskQueue->next; task != fset->bgTaskQueue; task = task->next) { - if (task->type == type) { - if (destroy) { - destroy(arg); - } - return 0; - } - } - - // do schedule task - STFSBgTask *task = taosMemoryCalloc(1, sizeof(STFSBgTask)); - if (task == NULL) return TSDB_CODE_OUT_OF_MEMORY; - taosThreadCondInit(task->done, NULL); - - task->fs = fs; - task->fid = fid; - task->type = type; - task->run = run; - task->destroy = destroy; - task->arg = arg; - task->scheduleTime = taosGetTimestampMs(); - task->taskid = ++fs->taskid; - - if (fset->bgTaskRunning == NULL && fset->bgTaskNum == 0) { - // launch task directly - fset->bgTaskRunning = task; - vnodeScheduleTaskEx(1, tsdbFSRunBgTask, task); - } else { - // add to the queue tail - fset->bgTaskNum++; - task->next = fset->bgTaskQueue; - task->prev = fset->bgTaskQueue->prev; - task->prev->next = task; - task->next->prev = task; - } - - if (taskid) *taskid = task->taskid; - return 0; -} - -int32_t tsdbFSDisableBgTask(STFileSystem *fs) { - taosThreadMutexLock(&fs->tsdb->mutex); - for (;;) { - fs->stop = true; - bool done = true; - - STFileSet *fset; - TARRAY2_FOREACH(fs->fSetArr, fset) { - if (fset->bgTaskRunning) { - tsdbDoWaitBgTask(fs, fset->bgTaskRunning); - done = false; - break; - } - } - - if (done) break; - } - taosThreadMutexUnlock(&fs->tsdb->mutex); - return 0; -} - -int32_t tsdbFSEnableBgTask(STFileSystem *fs) { - taosThreadMutexLock(&fs->tsdb->mutex); - fs->stop = false; - taosThreadMutexUnlock(&fs->tsdb->mutex); - return 0; -} +int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr) { return tsdbTFileSetRangeArrayDestroy(fsrArr); } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h index a3a8e2f575..714bf5bf16 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -44,22 +44,17 @@ int32_t tsdbFSCreateRefSnapshot(STFileSystem *fs, TFileSetArray **fsetArr); int32_t tsdbFSCreateRefSnapshotWithoutLock(STFileSystem *fs, TFileSetArray **fsetArr); int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr); -int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pExclude, TFileSetArray **fsetArr, +int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TFileSetRangeArray *pExclude, TFileSetArray **fsetArr, TFileOpArray *fopArr); -int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr, TFileOpArray *fopArr); -int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pRanges, - TSnapRangeArray **fsrArr); -int32_t tsdbFSDestroyRefRangedSnapshot(TSnapRangeArray **fsrArr); -// txn +int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr); +int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TFileSetRangeArray *pRanges, + TFileSetRangeArray **fsrArr); +int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr); +// txn int64_t tsdbFSAllocEid(STFileSystem *fs); int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype); int32_t tsdbFSEditCommit(STFileSystem *fs); int32_t tsdbFSEditAbort(STFileSystem *fs); -// background task -int32_t tsdbFSScheduleBgTask(STFileSystem *fs, int32_t fid, EFSBgTaskT type, int32_t (*run)(void *), - void (*destroy)(void *), void *arg, int64_t *taskid); -int32_t tsdbFSDisableBgTask(STFileSystem *fs); -int32_t tsdbFSEnableBgTask(STFileSystem *fs); // other int32_t tsdbFSGetFSet(STFileSystem *fs, int32_t fid, STFileSet **fset); int32_t tsdbFSCheckCommit(STsdb *tsdb, int32_t fid); diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index 61bedcb996..e088f54930 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -14,6 +14,7 @@ */ #include "tsdbFSet2.h" +#include "vnd.h" int32_t tsdbSttLvlInit(int32_t level, SSttLvl **lvl) { if (!(lvl[0] = taosMemoryMalloc(sizeof(SSttLvl)))) return TSDB_CODE_OUT_OF_MEMORY; @@ -451,10 +452,8 @@ int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset) { TARRAY2_INIT(fset[0]->lvlArr); // background task queue - fset[0]->bgTaskNum = 0; - fset[0]->bgTaskQueue->next = fset[0]->bgTaskQueue; - fset[0]->bgTaskQueue->prev = fset[0]->bgTaskQueue; - fset[0]->bgTaskRunning = NULL; + fset[0]->bgTaskChannel = 0; + fset[0]->mergeScheduled = false; // block commit variables taosThreadCondInit(&fset[0]->canCommit, NULL); @@ -534,7 +533,8 @@ int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_ return 0; } -int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STSnapRange **fsr) { +int32_t tsdbTFileSetRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, + STFileSetRange **fsr) { fsr[0] = taosMemoryCalloc(1, sizeof(*fsr[0])); if (fsr[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; fsr[0]->fid = fset1->fid; @@ -576,7 +576,7 @@ int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs return 0; } -int32_t tsdbTSnapRangeClear(STSnapRange **fsr) { +int32_t tsdbTFileSetRangeClear(STFileSetRange **fsr) { if (!fsr[0]) return 0; tsdbTFileSetClear(&fsr[0]->fset); @@ -585,6 +585,15 @@ int32_t tsdbTSnapRangeClear(STSnapRange **fsr) { return 0; } +int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray** ppArr) { + if (ppArr && ppArr[0]) { + TARRAY2_DESTROY(ppArr[0], tsdbTFileSetRangeClear); + taosMemoryFree(ppArr[0]); + ppArr[0] = NULL; + } + return 0; +} + int32_t tsdbTFileSetClear(STFileSet **fset) { if (!fset[0]) return 0; @@ -650,3 +659,8 @@ bool tsdbTFileSetIsEmpty(const STFileSet *fset) { } return TARRAY2_SIZE(fset->lvlArr) == 0; } + +int32_t tsdbTFileSetOpenChannel(STFileSet *fset) { + if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) return 0; + return vnodeAChannelInit(vnodeAsyncHandle[1], &fset->bgTaskChannel); +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h index 34f174ade7..0951a28f4e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h @@ -22,14 +22,12 @@ extern "C" { #endif -typedef struct STFileSet STFileSet; -typedef struct STFileOp STFileOp; -typedef struct SSttLvl SSttLvl; +typedef struct STFileOp STFileOp; +typedef struct SSttLvl SSttLvl; typedef TARRAY2(STFileObj *) TFileObjArray; typedef TARRAY2(SSttLvl *) TSttLvlArray; typedef TARRAY2(STFileOp) TFileOpArray; typedef struct STFileSystem STFileSystem; -typedef struct STFSBgTask STFSBgTask; typedef enum { TSDB_FOP_NONE = 0, @@ -51,8 +49,8 @@ int32_t tsdbTFileSetRemove(STFileSet *fset); int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_t ever, STFileSet **fset, TFileOpArray *fopArr); -int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STSnapRange **fsr); -int32_t tsdbTSnapRangeClear(STSnapRange **fsr); +int32_t tsdbTFileSetRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, + STFileSetRange **fsr); // to/from json int32_t tsdbTFileSetToJson(const STFileSet *fset, cJSON *json); @@ -72,33 +70,8 @@ bool tsdbTFileSetIsEmpty(const STFileSet *fset); // stt int32_t tsdbSttLvlInit(int32_t level, SSttLvl **lvl); int32_t tsdbSttLvlClear(SSttLvl **lvl); - -typedef enum { - TSDB_BG_TASK_MERGER = 1, - TSDB_BG_TASK_RETENTION, - TSDB_BG_TASK_COMPACT, -} EFSBgTaskT; - -struct STFSBgTask { - STFileSystem *fs; - int32_t fid; - - EFSBgTaskT type; - int32_t (*run)(void *arg); - void (*destroy)(void *arg); - void *arg; - - TdThreadCond done[1]; - int32_t numWait; - - int64_t taskid; - int64_t scheduleTime; - int64_t launchTime; - int64_t finishTime; - - struct STFSBgTask *prev; - struct STFSBgTask *next; -}; +// open channel +int32_t tsdbTFileSetOpenChannel(STFileSet *fset); struct STFileOp { tsdb_fop_t optype; @@ -118,10 +91,9 @@ struct STFileSet { STFileObj *farr[TSDB_FTYPE_MAX]; // file array TSttLvlArray lvlArr[1]; // level array - // background task queue - int32_t bgTaskNum; - STFSBgTask bgTaskQueue[1]; - STFSBgTask *bgTaskRunning; + // background task channel + int64_t bgTaskChannel; + bool mergeScheduled; // block commit variables TdThreadCond canCommit; @@ -129,7 +101,7 @@ struct STFileSet { bool blockCommit; }; -struct STSnapRange { +struct STFileSetRange { int32_t fid; int64_t sver; int64_t ever; diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c new file mode 100644 index 0000000000..03c12502d5 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbFSetRAW.h" + +// SFSetRAWWriter ================================================== +typedef struct SFSetRAWWriter { + SFSetRAWWriterConfig config[1]; + + struct { + TFileOpArray fopArr[1]; + STFile file; + int64_t offset; + } ctx[1]; + + // writer + SDataFileRAWWriter *dataWriter; +} SFSetRAWWriter; + +int32_t tsdbFSetRAWWriterOpen(SFSetRAWWriterConfig *config, SFSetRAWWriter **writer) { + int32_t code = 0; + int32_t lino = 0; + + writer[0] = taosMemoryCalloc(1, sizeof(SFSetRAWWriter)); + if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + writer[0]->config[0] = config[0]; + + TARRAY2_INIT(writer[0]->ctx->fopArr); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbFSetRAWWriterFinish(SFSetRAWWriter *writer, TFileOpArray *fopArr) { + int32_t code = 0; + int32_t lino = 0; + + STsdb *tsdb = writer->config->tsdb; + + STFileOp op; + TARRAY2_FOREACH(writer->ctx->fopArr, op) { + code = TARRAY2_APPEND(fopArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + } + + TARRAY2_CLEAR(writer->ctx->fopArr, NULL); +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr) { + int32_t code = 0; + int32_t lino = 0; + + SDataFileRAWWriterConfig config = { + .tsdb = writer->config->tsdb, + .szPage = writer->config->szPage, + .fid = bHdr->file.fid, + .did = writer->config->did, + .cid = writer->config->cid, + .level = writer->config->level, + + .file = + { + .type = bHdr->file.type, + .fid = bHdr->file.fid, + .did = writer->config->did, + .cid = writer->config->cid, + .size = bHdr->file.size, + .minVer = bHdr->file.minVer, + .maxVer = bHdr->file.maxVer, + .stt = {{ + .level = bHdr->file.stt->level, + }}, + }, + }; + + writer->ctx->offset = 0; + writer->ctx->file = config.file; + + code = tsdbDataFileRAWWriterOpen(&config, &writer->dataWriter); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbFSetRAWWriteFileDataEnd(SFSetRAWWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbDataFileRAWWriterClose(&writer->dataWriter, false, writer->ctx->fopArr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFSetRAWWriterClose(SFSetRAWWriter **writer, bool abort, TFileOpArray *fopArr) { + if (writer[0] == NULL) return 0; + + int32_t code = 0; + int32_t lino = 0; + + STsdb *tsdb = writer[0]->config->tsdb; + + // end + code = tsdbFSetRAWWriteFileDataEnd(writer[0]); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileRAWWriterClose(&writer[0]->dataWriter, abort, writer[0]->ctx->fopArr); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFSetRAWWriterFinish(writer[0], fopArr); + TSDB_CHECK_CODE(code, lino, _exit); + // free + TARRAY2_DESTROY(writer[0]->ctx->fopArr, NULL); + taosMemoryFree(writer[0]); + writer[0] = NULL; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(writer->ctx->offset >= 0 && writer->ctx->offset <= writer->ctx->file.size); + + if (writer->ctx->offset == writer->ctx->file.size) { + code = tsdbFSetRAWWriteFileDataEnd(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFSetRAWWriteFileDataBegin(writer, bHdr); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbDataFileRAWWriteBlockData(writer->dataWriter, bHdr); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->ctx->offset += bHdr->dataLength; + ASSERT(writer->ctx->offset == writer->dataWriter->ctx->offset); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.h b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.h new file mode 100644 index 0000000000..205c785e99 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbDataFileRAW.h" + +#ifndef _TSDB_FSET_RAW_H +#define _TSDB_FSET_RAW_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SFSetRAWWriterConfig { + STsdb *tsdb; + int32_t szPage; + + SDiskID did; + int64_t fid; + int64_t cid; + int32_t level; +} SFSetRAWWriterConfig; + +typedef struct SFSetRAWWriter SFSetRAWWriter; + +int32_t tsdbFSetRAWWriterOpen(SFSetRAWWriterConfig *config, SFSetRAWWriter **writer); +int32_t tsdbFSetRAWWriterClose(SFSetRAWWriter **writer, bool abort, TFileOpArray *fopArr); +int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr); + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_FSET_RAW_H*/ diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index 6d968d0828..7f33f08794 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -17,11 +17,6 @@ #define TSDB_MAX_LEVEL 2 // means max level is 3 -typedef struct { - STsdb *tsdb; - int32_t fid; -} SMergeArg; - typedef struct { STsdb *tsdb; int32_t fid; @@ -519,6 +514,8 @@ static int32_t tsdbMergeGetFSet(SMerger *merger) { return 0; } + fset->mergeScheduled = false; + int32_t code = tsdbTFileSetInitCopy(merger->tsdb, fset, &merger->fset); if (code) { taosThreadMutexUnlock(&merger->tsdb->mutex); @@ -528,7 +525,7 @@ static int32_t tsdbMergeGetFSet(SMerger *merger) { return 0; } -static int32_t tsdbMerge(void *arg) { +int32_t tsdbMerge(void *arg) { int32_t code = 0; int32_t lino = 0; SMergeArg *mergeArg = (SMergeArg *)arg; @@ -597,18 +594,3 @@ _exit: tsdbTFileSetClear(&merger->fset); return code; } - -int32_t tsdbSchedMerge(STsdb *tsdb, int32_t fid) { - SMergeArg *arg = taosMemoryMalloc(sizeof(*arg)); - if (arg == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - arg->tsdb = tsdb; - arg->fid = fid; - - int32_t code = tsdbFSScheduleBgTask(tsdb->pFS, fid, TSDB_BG_TASK_MERGER, tsdbMerge, taosMemoryFree, arg, NULL); - if (code) taosMemoryFree(arg); - - return code; -} diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 36675a771c..8017f1f4d0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -22,7 +22,7 @@ static void tLDataIterClose2(SLDataIter *pIter); // SLDataIter ================================================= -SSttBlockLoadInfo *tCreateOneLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols) { +SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols) { SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(1, sizeof(SSttBlockLoadInfo)); if (pLoadInfo == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -61,7 +61,7 @@ void getSttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, SSttBlockLoadCostInfo* pL } } -void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { +void *destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { if (pLoadInfo == NULL) { return NULL; } @@ -78,14 +78,19 @@ void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { pInfo->sttBlockIndex = -1; pInfo->pin = false; + if (pLoadInfo->statisBlock != NULL) { + tStatisBlockDestroy(pLoadInfo->statisBlock); + taosMemoryFreeClear(pLoadInfo->statisBlock); + } + taosArrayDestroy(pLoadInfo->aSttBlk); taosMemoryFree(pLoadInfo); return NULL; } -static void destroyLDataIter(SLDataIter *pIter) { +void destroyLDataIter(SLDataIter *pIter) { tLDataIterClose2(pIter); - destroyLastBlockLoadInfo(pIter->pBlockLoadInfo); + destroySttBlockLoadInfo(pIter->pBlockLoadInfo); taosMemoryFree(pIter); } @@ -732,25 +737,6 @@ static FORCE_INLINE int32_t tLDataIterDescCmprFn(const SRBTreeNode *p1, const SR return -1 * tLDataIterCmprFn(p1, p2); } -static void adjustValidLDataIters(SArray *pLDIterList, int32_t numOfFileObj) { - int32_t size = taosArrayGetSize(pLDIterList); - - if (size < numOfFileObj) { - int32_t inc = numOfFileObj - size; - for (int32_t k = 0; k < inc; ++k) { - SLDataIter *pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); - taosArrayPush(pLDIterList, &pIter); - } - } else if (size > numOfFileObj) { // remove unused LDataIter - int32_t inc = size - numOfFileObj; - - for (int i = 0; i < inc; ++i) { - SLDataIter *pIter = taosArrayPop(pLDIterList); - destroyLDataIter(pIter); - } - } -} - int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf) { int32_t code = TSDB_CODE_SUCCESS; @@ -773,19 +759,13 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf) { } // add the list/iter placeholder - while (taosArrayGetSize(pConf->pSttFileBlockIterArray) < numOfLevels) { - SArray *pList = taosArrayInit(4, POINTER_BYTES); - taosArrayPush(pConf->pSttFileBlockIterArray, &pList); - } + adjustLDataIters(pConf->pSttFileBlockIterArray, pConf->pCurrentFileset); for (int32_t j = 0; j < numOfLevels; ++j) { SSttLvl *pSttLevel = ((STFileSet *)pConf->pCurrentFileset)->lvlArr->data[j]; SArray *pList = taosArrayGetP(pConf->pSttFileBlockIterArray, j); - int32_t numOfFileObj = TARRAY2_SIZE(pSttLevel->fobjArr); - adjustValidLDataIters(pList, numOfFileObj); - - for (int32_t i = 0; i < numOfFileObj; ++i) { // open all last file + for (int32_t i = 0; i < TARRAY2_SIZE(pSttLevel->fobjArr); ++i) { // open all last file SLDataIter *pIter = taosArrayGetP(pList, i); SSttFileReader *pSttFileReader = pIter->pReader; @@ -805,7 +785,7 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf) { } if (pLoadInfo == NULL) { - pLoadInfo = tCreateOneLastBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols); + pLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols); } memset(pIter, 0, sizeof(SLDataIter)); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 751df706ab..6bb5b51a30 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -23,14 +23,14 @@ #include "tsimplehash.h" #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) -#define getCurrentKeyInLastBlock(_r) ((_r)->currentKey) +#define getCurrentKeyInSttBlock(_r) ((_r)->currentKey) static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter); static int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, STsdbReader* pReader); static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader); static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader); -static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, +static int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, SRowMerger* pMerger, SVersionRange* pVerRange, const char* id); static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, STsdbReader* pReader); static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, @@ -47,17 +47,22 @@ static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScan STsdbReader* pReader, SRow** pTSRow); static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader); +static int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pBlockScanInfo, + STsdbReader* pReader); -static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SCostSummary* pCost); +static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SReadCostSummary* pCost); static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idstr, int8_t* pLevel); static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level); -static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); static int32_t doBuildDataBlock(STsdbReader* pReader); static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader); static bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo); +static bool hasDataInSttBlock(SSttBlockReader* pSttBlockReader); static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter); static int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order); +static void resetTableListIndex(SReaderStatus* pStatus); +static void getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey, int64_t* pMinKey); +static void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInfo* pBlockScanInfo); static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } @@ -137,16 +142,16 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, TFileSetArray* pFileSetA pIter->pFilesetList = pFileSetArray; pIter->numOfFiles = numOfFileset; - if (pIter->pLastBlockReader == NULL) { - pIter->pLastBlockReader = taosMemoryCalloc(1, sizeof(struct SLastBlockReader)); - if (pIter->pLastBlockReader == NULL) { + if (pIter->pSttBlockReader == NULL) { + pIter->pSttBlockReader = taosMemoryCalloc(1, sizeof(struct SSttBlockReader)); + if (pIter->pSttBlockReader == NULL) { int32_t code = TSDB_CODE_OUT_OF_MEMORY; tsdbError("failed to prepare the last block iterator, since:%s %s", tstrerror(code), pReader->idStr); return code; } } - SLastBlockReader* pLReader = pIter->pLastBlockReader; + SSttBlockReader* pLReader = pIter->pSttBlockReader; pLReader->order = pReader->info.order; pLReader->window = pReader->info.window; pLReader->verRange = pReader->info.verRange; @@ -168,10 +173,10 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo return TSDB_CODE_SUCCESS; } - SCostSummary* pCost = &pReader->cost; + SReadCostSummary* pCost = &pReader->cost; - pIter->pLastBlockReader->uid = 0; - tMergeTreeClose(&pIter->pLastBlockReader->mergeTree); + pIter->pSttBlockReader->uid = 0; + tMergeTreeClose(&pIter->pSttBlockReader->mergeTree); pReader->status.pLDataIterArray = destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost); pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES); @@ -241,6 +246,7 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo tsdbDebug("%p file found fid:%d for qrange:%" PRId64 "-%" PRId64 ", %s", pReader, fid, pReader->info.window.skey, pReader->info.window.ekey, pReader->idStr); + *hasNext = true; return TSDB_CODE_SUCCESS; } @@ -291,11 +297,7 @@ static SSDataBlock* createResBlock(SQueryTableDataCond* pCond, int32_t capacity) } static int32_t tsdbInitReaderLock(STsdbReader* pReader) { - int32_t code = -1; - qTrace("tsdb/read: %p, pre-init read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); - - code = taosThreadMutexInit(&pReader->readerMutex, NULL); - + int32_t code = taosThreadMutexInit(&pReader->readerMutex, NULL); qTrace("tsdb/read: %p, post-init read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); return code; @@ -324,22 +326,14 @@ static int32_t tsdbAcquireReader(STsdbReader* pReader) { } static int32_t tsdbTryAcquireReader(STsdbReader* pReader) { - int32_t code = -1; - qTrace("tsdb/read: %p, pre-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); - - code = taosThreadMutexTryLock(&pReader->readerMutex); - + int32_t code = taosThreadMutexTryLock(&pReader->readerMutex); qTrace("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); return code; } static int32_t tsdbReleaseReader(STsdbReader* pReader) { - int32_t code = -1; - qTrace("tsdb/read: %p, pre-untake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); - - code = taosThreadMutexUnlock(&pReader->readerMutex); - + int32_t code = taosThreadMutexUnlock(&pReader->readerMutex); qTrace("tsdb/read: %p, post-untake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); return code; @@ -431,7 +425,10 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void goto _end; } + pReader->bFilesetDelimited = false; + tsdbInitReaderLock(pReader); + tsem_init(&pReader->resumeAfterSuspend, 0, 0); *ppReader = pReader; return code; @@ -1015,8 +1012,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { // check if current block are all handled if (pDumpInfo->rowIndex >= 0 && pDumpInfo->rowIndex < pRecord->numRow) { int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex]; - if (outOfTimeWindow(ts, - &pReader->info.window)) { // the remain data has out of query time window, ignore current block + if (outOfTimeWindow(ts, &pReader->info.window)) { + // the remain data has out of query time window, ignore current block setBlockAllDumped(pDumpInfo, ts, pReader->info.order); } } else { @@ -1123,16 +1120,12 @@ static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlo } int32_t step = asc ? 1 : -1; - // *nextIndex = pBlockInfo->tbBlockIdx + step; - // *pBlockIndex = *(SBlockIndex*)taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex); STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pTableBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx + step); SFileDataBlockInfo* p = taosArrayGet(pBlockIter->blockList, pTableDataBlockIdx->globalIndex); memcpy(pRecord, &p->record, sizeof(SBrinRecord)); *nextIndex = pBlockInfo->tbBlockIdx + step; - - // tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, pIndex->ordinalIndex, pBlock, tGetDataBlk); return true; } @@ -1376,23 +1369,19 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* return TSDB_CODE_SUCCESS; } - SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; - int64_t st = taosGetTimestampUs(); + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->resBlockInfo.capacity, pReader); - blockDataUpdateTsWindow(pBlock, pReader->suppInfo.slotId[0]); - pBlock->info.id.uid = pBlockScanInfo->uid; + double el = (taosGetTimestampUs() - st) / 1000.0; + updateComposedBlockInfo(pReader, el, pBlockScanInfo); - setComposedBlockFlag(pReader, true); - - double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%" PRId64 ", brange:%" PRId64 " - %" PRId64 ", uid:%" PRIu64 ", %s", - pReader, elapsedTime, pBlock->info.rows, pBlock->info.window.skey, pBlock->info.window.ekey, + pReader, el, pBlock->info.rows, pBlock->info.window.skey, pBlock->info.window.ekey, pBlockScanInfo->uid, pReader->idStr); - pReader->cost.buildmemBlock += elapsedTime; + pReader->cost.buildmemBlock += el; return code; } @@ -1422,26 +1411,26 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB return code; } -static bool nextRowFromSttBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, +static bool nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, SVersionRange* pVerRange) { - int32_t step = ASCENDING_TRAVERSE(pLastBlockReader->order) ? 1 : -1; + int32_t step = ASCENDING_TRAVERSE(pSttBlockReader->order) ? 1 : -1; while (1) { - bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree); + bool hasVal = tMergeTreeNext(&pSttBlockReader->mergeTree); if (!hasVal) { // the next value will be the accessed key in stt pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA; pScanInfo->sttKeyInfo.nextProcKey += step; return false; } - TSDBROW* pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBROW* pRow = tMergeTreeGetRow(&pSttBlockReader->mergeTree); int64_t key = pRow->pBlockData->aTSKEY[pRow->iRow]; int64_t ver = pRow->pBlockData->aVersion[pRow->iRow]; - pLastBlockReader->currentKey = key; + pSttBlockReader->currentKey = key; pScanInfo->sttKeyInfo.nextProcKey = key; - if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, pLastBlockReader->order, + if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, pSttBlockReader->order, pVerRange)) { pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA; return true; @@ -1449,26 +1438,26 @@ static bool nextRowFromSttBlocks(SLastBlockReader* pLastBlockReader, STableBlock } } -static void doPinSttBlock(SLastBlockReader* pLastBlockReader) { - tMergeTreePinSttBlock(&pLastBlockReader->mergeTree); +static void doPinSttBlock(SSttBlockReader* pSttBlockReader) { + tMergeTreePinSttBlock(&pSttBlockReader->mergeTree); } -static void doUnpinSttBlock(SLastBlockReader* pLastBlockReader) { - tMergeTreeUnpinSttBlock(&pLastBlockReader->mergeTree); +static void doUnpinSttBlock(SSttBlockReader* pSttBlockReader) { + tMergeTreeUnpinSttBlock(&pSttBlockReader->mergeTree); } -static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLastBlockReader, +static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader, bool* copied) { int32_t code = TSDB_CODE_SUCCESS; *copied = false; // avoid the fetch next row replace the referenced stt block in buffer - doPinSttBlock(pLastBlockReader); - bool hasVal = nextRowFromSttBlocks(pLastBlockReader, pScanInfo, &pReader->info.verRange); - doUnpinSttBlock(pLastBlockReader); + doPinSttBlock(pSttBlockReader); + bool hasVal = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, &pReader->info.verRange); + doUnpinSttBlock(pSttBlockReader); if (hasVal) { - int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); + int64_t next1 = getCurrentKeyInSttBlock(pSttBlockReader); if (next1 != ts) { code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, fRow->pBlockData, fRow->iRow); if (code) { @@ -1525,15 +1514,15 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* } static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, - SIterInfo* pIter, int64_t key, SLastBlockReader* pLastBlockReader) { + SIterInfo* pIter, int64_t key, SSttBlockReader* pSttBlockReader) { SRowMerger* pMerger = &pReader->status.merger; SRow* pTSRow = NULL; SBlockData* pBlockData = &pReader->status.fileBlockData; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; int64_t tsLast = INT64_MIN; - if (hasDataInLastBlock(pLastBlockReader)) { - tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + if (hasDataInSttBlock(pSttBlockReader)) { + tsLast = getCurrentKeyInSttBlock(pSttBlockReader); } TSDBKEY k = TSDBROW_KEY(pRow); @@ -1551,7 +1540,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* int64_t minKey = 0; if (pReader->info.order == TSDB_ORDER_ASC) { minKey = INT64_MAX; // chosen the minimum value - if (minKey > tsLast && hasDataInLastBlock(pLastBlockReader)) { + if (minKey > tsLast && hasDataInSttBlock(pSttBlockReader)) { minKey = tsLast; } @@ -1564,7 +1553,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } } else { minKey = INT64_MIN; - if (minKey < tsLast && hasDataInLastBlock(pLastBlockReader)) { + if (minKey < tsLast && hasDataInSttBlock(pSttBlockReader)) { minKey = tsLast; } @@ -1589,12 +1578,12 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == tsLast) { - TSDBROW* fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBROW* fRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); int32_t code = tsdbRowMergerAdd(pMerger, fRow1, NULL); if (code != TSDB_CODE_SUCCESS) { return code; } - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, + doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); } @@ -1639,12 +1628,12 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == tsLast) { - TSDBROW* fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBROW* fRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); int32_t code = tsdbRowMergerAdd(pMerger, fRow1, NULL); if (code != TSDB_CODE_SUCCESS) { return code; } - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, + doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); } @@ -1670,27 +1659,27 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return code; } -static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, STsdbReader* pReader, +static int32_t doMergeFileBlockAndLastBlock(SSttBlockReader* pSttBlockReader, STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, bool mergeBlockData) { SRowMerger* pMerger = &pReader->status.merger; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); + int64_t tsLastBlock = getCurrentKeyInSttBlock(pSttBlockReader); bool copied = false; int32_t code = TSDB_CODE_SUCCESS; SRow* pTSRow = NULL; - TSDBROW* pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBROW* pRow = tMergeTreeGetRow(&pSttBlockReader->mergeTree); // create local variable to hold the row value TSDBROW fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData}; - tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", ts:%" PRId64 " %s", pRow->pBlockData, pRow->iRow, pLastBlockReader->uid, + tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", ts:%" PRId64 " %s", pRow->pBlockData, pRow->iRow, pSttBlockReader->uid, fRow.pBlockData->aTSKEY[fRow.iRow], pReader->idStr); // only stt block exists if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { - code = tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader, &copied); + code = tryCopyDistinctRowFromSttBlock(&fRow, pSttBlockReader, pBlockScanInfo, tsLastBlock, pReader, &copied); if (code) { return code; } @@ -1704,9 +1693,9 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, return code; } - TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); tsdbRowMergerAdd(pMerger, pRow1, NULL); - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->info.verRange, + doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->info.verRange, pReader->idStr); code = tsdbRowMergerGetRow(pMerger, &pTSRow); @@ -1729,7 +1718,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, return code; } - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->info.verRange, + doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->info.verRange, pReader->idStr); // merge with block data if ts == key @@ -1755,10 +1744,11 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, return TSDB_CODE_SUCCESS; } -static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader* pLastBlockReader, int64_t key, - STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { +static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* pSttBlockReader, int64_t key, + STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SRowMerger* pMerger = &pReader->status.merger; + int32_t code = TSDB_CODE_SUCCESS; // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized if (pMerger->pArray == NULL) { @@ -1769,45 +1759,68 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader } } - if (hasDataInFileBlock(pBlockData, pDumpInfo)) { - // no last block available, only data block exists - if (!hasDataInLastBlock(pLastBlockReader)) { - return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); - } - - // row in last file block + bool dataInDataFile = hasDataInFileBlock(pBlockData, pDumpInfo); + bool dataInSttFile = hasDataInSttBlock(pSttBlockReader); + if (dataInDataFile && (!dataInSttFile)) { + // no stt file block available, only data block exists + return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); + } else if ((!dataInDataFile) && dataInSttFile) { + // no data ile block exists + return mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader); + } else { + // row in both stt file blocks and data file blocks TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + int64_t tsLast = getCurrentKeyInSttBlock(pSttBlockReader); if (ASCENDING_TRAVERSE(pReader->info.order)) { - if (key < tsLast) { + if (key < tsLast) { // asc return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); } else if (key > tsLast) { - return doMergeFileBlockAndLastBlock(pLastBlockReader, pReader, pBlockScanInfo, NULL, false); + return mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader); } - } else { + } else { // desc if (key > tsLast) { return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); } else if (key < tsLast) { - return doMergeFileBlockAndLastBlock(pLastBlockReader, pReader, pBlockScanInfo, NULL, false); + return mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader); } } + // the following for key == tsLast + // ASC: file block ------> stt block + // DESC: stt block ------> file block SRow* pTSRow = NULL; - int32_t code = tsdbRowMergerAdd(pMerger, &fRow, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; + if (ASCENDING_TRAVERSE(pReader->info.order)) { + code = tsdbRowMergerAdd(pMerger, &fRow, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); + + TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); + code = tsdbRowMergerAdd(pMerger, pRow1, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); + } else { + TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); + code = tsdbRowMergerAdd(pMerger, pRow1, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); + + code = tsdbRowMergerAdd(pMerger, &fRow, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); - - TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - code = tsdbRowMergerAdd(pMerger, pRow1, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); - code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1818,14 +1831,11 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader taosMemoryFree(pTSRow); tsdbRowMergerClear(pMerger); return code; - - } else { // only last block exists - return doMergeFileBlockAndLastBlock(pLastBlockReader, pReader, pBlockScanInfo, NULL, false); } } static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, - SLastBlockReader* pLastBlockReader) { + SSttBlockReader* pSttBlockReader) { SRowMerger* pMerger = &pReader->status.merger; SRow* pTSRow = NULL; int32_t code = TSDB_CODE_SUCCESS; @@ -1836,8 +1846,8 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader); int64_t tsLast = INT64_MIN; - if (hasDataInLastBlock(pLastBlockReader)) { - tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + if (hasDataInSttBlock(pSttBlockReader)) { + tsLast = getCurrentKeyInSttBlock(pSttBlockReader); } int64_t key = hasDataInFileBlock(pBlockData, pDumpInfo) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : INT64_MIN; @@ -1885,7 +1895,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* minKey = key; } - if (minKey > tsLast && hasDataInLastBlock(pLastBlockReader)) { + if (minKey > tsLast && hasDataInSttBlock(pSttBlockReader)) { minKey = tsLast; } } else { @@ -1902,13 +1912,13 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* minKey = key; } - if (minKey < tsLast && hasDataInLastBlock(pLastBlockReader)) { + if (minKey < tsLast && hasDataInSttBlock(pSttBlockReader)) { minKey = tsLast; } } - // ASC: file block -----> last block -----> imem -----> mem - // DESC: mem -----> imem -----> last block -----> file block + // ASC: file block -----> stt block -----> imem -----> mem + // DESC: mem -----> imem -----> stt block -----> file block if (ASCENDING_TRAVERSE(pReader->info.order)) { if (minKey == key) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); @@ -1921,13 +1931,13 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == tsLast) { - TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); code = tsdbRowMergerAdd(pMerger, pRow1, NULL); if (code != TSDB_CODE_SUCCESS) { return code; } - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, + doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); } @@ -1980,13 +1990,13 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == tsLast) { - TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); code = tsdbRowMergerAdd(pMerger, pRow1, NULL); if (code != TSDB_CODE_SUCCESS) { return code; } - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, + doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); } @@ -2105,10 +2115,10 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum return true; } -static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { +static bool initSttBlockReader(SSttBlockReader* pLBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { // the last block reader has been initialized for this table. if (pLBlockReader->uid == pScanInfo->uid) { - return hasDataInLastBlock(pLBlockReader); + return hasDataInSttBlock(pLBlockReader); } if (pLBlockReader->uid != 0) { @@ -2157,13 +2167,13 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan code = nextRowFromSttBlocks(pLBlockReader, pScanInfo, &pReader->info.verRange); int64_t el = taosGetTimestampUs() - st; - pReader->cost.initLastBlockReader += (el / 1000.0); + pReader->cost.initSttBlockReader += (el / 1000.0); tsdbDebug("init last block reader completed, elapsed time:%" PRId64 "us %s", el, pReader->idStr); return code; } -static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->mergeTree.pIter != NULL; } +static bool hasDataInSttBlock(SSttBlockReader* pSttBlockReader) { return pSttBlockReader->mergeTree.pIter != NULL; } bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo) { if ((pBlockData->nRow > 0) && (pBlockData->nRow != pDumpInfo->totalRows)) { @@ -2218,8 +2228,52 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc } } +int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { + bool copied = false; + SRow* pTSRow = NULL; + int64_t tsLastBlock = getCurrentKeyInSttBlock(pSttBlockReader); + + SRowMerger* pMerger = &pReader->status.merger; + TSDBROW* pRow = tMergeTreeGetRow(&pSttBlockReader->mergeTree); + TSDBROW fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData}; + + tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", ts:%" PRId64 " %s", pRow->pBlockData, pRow->iRow, pSttBlockReader->uid, + fRow.pBlockData->aTSKEY[fRow.iRow], pReader->idStr); + + int32_t code = tryCopyDistinctRowFromSttBlock(&fRow, pSttBlockReader, pBlockScanInfo, tsLastBlock, pReader, &copied); + if (code) { + return code; + } + + if (copied) { + pBlockScanInfo->lastProcKey = tsLastBlock; + return TSDB_CODE_SUCCESS; + } else { + code = tsdbRowMergerAdd(pMerger, &fRow, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); + tsdbRowMergerAdd(pMerger, pRow1, NULL); + doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->info.verRange, + pReader->idStr); + + code = tsdbRowMergerGetRow(pMerger, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); + + taosMemoryFree(pTSRow); + tsdbRowMergerClear(pMerger); + return code; + } +} + static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, - SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { + SBlockData* pBlockData, SSttBlockReader* pSttBlockReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; TSDBROW *pRow = NULL, *piRow = NULL; @@ -2236,21 +2290,21 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI // two levels of mem-table does contain the valid rows if (pRow != NULL && piRow != NULL) { - return doMergeMultiLevelRows(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); + return doMergeMultiLevelRows(pReader, pBlockScanInfo, pBlockData, pSttBlockReader); } - // imem + file + last block + // imem + file + stt block if (pBlockScanInfo->iiter.hasVal) { - return doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, key, pLastBlockReader); + return doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, key, pSttBlockReader); } - // mem + file + last block + // mem + file + stt block if (pBlockScanInfo->iter.hasVal) { - return doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, key, pLastBlockReader); + return doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, key, pSttBlockReader); } - // files data blocks + last block - return mergeFileBlockAndLastBlock(pReader, pLastBlockReader, key, pBlockScanInfo, pBlockData); + // files data blocks + stt block + return mergeFileBlockAndSttBlock(pReader, pSttBlockReader, key, pBlockScanInfo, pBlockData); } static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pBlockScanInfo, @@ -2293,13 +2347,12 @@ static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlock return code; } -static void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInfo* pBlockScanInfo) { +void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInfo* pBlockScanInfo) { SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0; pResBlock->info.dataLoad = 1; blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]); - setComposedBlockFlag(pReader, true); pReader->cost.composedBlocks += 1; @@ -2312,7 +2365,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); - SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + SSttBlockReader* pSttBlockReader = pReader->status.fileIter.pSttBlockReader; bool asc = ASCENDING_TRAVERSE(pReader->info.order); int64_t st = taosGetTimestampUs(); @@ -2356,13 +2409,12 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { pBlockScanInfo = *pReader->status.pTableIter; if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockScanInfo->uid, sizeof(pBlockScanInfo->uid))) { - // setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->info.order); return code; } } SBlockData* pBlockData = &pReader->status.fileBlockData; - initLastBlockReader(pLastBlockReader, pBlockScanInfo, pReader); + initSttBlockReader(pSttBlockReader, pBlockScanInfo, pReader); while (1) { bool hasBlockData = false; @@ -2396,7 +2448,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { break; } - code = buildComposedDataBlockImpl(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); + code = buildComposedDataBlockImpl(pReader, pBlockScanInfo, pBlockData, pSttBlockReader); if (code) { goto _end; } @@ -2436,7 +2488,7 @@ int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order) { return ASCENDING_TRAVERSE(order) ? 0 : taosArrayGetSize(pDelSkyline) - 1; } -int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SCostSummary* pCost) { +int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SReadCostSummary* pCost) { int32_t code = 0; int32_t newDelDataInFile = taosArrayGetSize(pBlockScanInfo->pFileDelData); if (newDelDataInFile == 0 && @@ -2509,6 +2561,41 @@ TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) } } +static void prepareDurationForNextFileSet(STsdbReader* pReader) { + if (pReader->status.bProcMemFirstFileset) { + pReader->status.prevFilesetStartKey = INT64_MIN; + pReader->status.prevFilesetEndKey = INT64_MAX; + pReader->status.bProcMemFirstFileset = false; + } + + int32_t fid = pReader->status.pCurrentFileset->fid; + STimeWindow winFid = {0}; + tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &winFid.skey, &winFid.ekey); + + if (ASCENDING_TRAVERSE(pReader->info.order)) { + pReader->status.bProcMemPreFileset = !(pReader->status.memTableMaxKey < pReader->status.prevFilesetStartKey || + (winFid.skey-1) < pReader->status.memTableMinKey); + } else { + pReader->status.bProcMemPreFileset = !( pReader->status.memTableMaxKey < (winFid.ekey+1) || + pReader->status.prevFilesetEndKey < pReader->status.memTableMinKey); + } + + if (pReader->status.bProcMemPreFileset) { + resetTableListIndex(&pReader->status); + } + + if (!pReader->status.bProcMemPreFileset) { + if (pReader->notifyFn) { + STsdReaderNotifyInfo info = {0}; + info.duration.filesetId = fid; + pReader->notifyFn(TSD_READER_NOTIFY_DURATION_START, &info, pReader->notifyParam); + } + } + + pReader->status.prevFilesetStartKey = winFid.skey; + pReader->status.prevFilesetEndKey = winFid.ekey; +} + static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SArray* pTableList) { SReaderStatus* pStatus = &pReader->status; pBlockNum->numOfBlocks = 0; @@ -2550,6 +2637,9 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr } if (pBlockNum->numOfBlocks + pBlockNum->numOfSttFiles > 0) { + if (pReader->bFilesetDelimited) { + prepareDurationForNextFileSet(pReader); + } break; } } @@ -2581,9 +2671,9 @@ static bool moveToNextTable(STableUidList* pOrderedCheckInfo, SReaderStatus* pSt return (pStatus->pTableIter != NULL); } -static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { +static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; - SLastBlockReader* pLastBlockReader = pStatus->fileIter.pLastBlockReader; + SSttBlockReader* pSttBlockReader = pStatus->fileIter.pSttBlockReader; STableUidList* pUidList = &pStatus->uidList; int32_t code = TSDB_CODE_SUCCESS; @@ -2621,7 +2711,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { continue; } - bool hasDataInLastFile = initLastBlockReader(pLastBlockReader, pScanInfo, pReader); + bool hasDataInLastFile = initSttBlockReader(pSttBlockReader, pScanInfo, pReader); if (!hasDataInLastFile) { bool hasNexTable = moveToNextTable(pUidList, pStatus); if (!hasNexTable) { @@ -2633,14 +2723,14 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { int64_t st = taosGetTimestampUs(); while (1) { - bool hasBlockLData = hasDataInLastBlock(pLastBlockReader); + bool hasBlockLData = hasDataInSttBlock(pSttBlockReader); // no data in last block and block, no need to proceed. if (hasBlockLData == false) { break; } - code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader); + code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pSttBlockReader); if (code) { return code; } @@ -2687,7 +2777,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { SDataBlockIter* pBlockIter = &pStatus->blockIter; STableBlockScanInfo* pScanInfo = NULL; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); - SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + SSttBlockReader* pSttBlockReader = pReader->status.fileIter.pSttBlockReader; bool asc = ASCENDING_TRAVERSE(pReader->info.order); if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) { @@ -2705,7 +2795,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { } if (pScanInfo->sttKeyInfo.status == STT_FILE_READER_UNINIT) { - initLastBlockReader(pLastBlockReader, pScanInfo, pReader); + initSttBlockReader(pSttBlockReader, pScanInfo, pReader); } TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); @@ -2750,13 +2840,13 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { int64_t st = taosGetTimestampUs(); // let's load data from stt files - initLastBlockReader(pLastBlockReader, pScanInfo, pReader); + initSttBlockReader(pSttBlockReader, pScanInfo, pReader); // no data in last block, no need to proceed. - while (hasDataInLastBlock(pLastBlockReader)) { + while (hasDataInSttBlock(pSttBlockReader)) { ASSERT(pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA); - code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader); + code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pSttBlockReader); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2766,7 +2856,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { } // data in stt now overlaps with current active file data block, need to composed with file data block. - int64_t lastKeyInStt = getCurrentKeyInLastBlock(pLastBlockReader); + int64_t lastKeyInStt = getCurrentKeyInSttBlock(pSttBlockReader); if ((lastKeyInStt >= pBlockInfo->record.firstKey && asc) || (lastKeyInStt <= pBlockInfo->record.lastKey && (!asc))) { tsdbDebug("%p lastKeyInStt:%" PRId64 ", overlap with file block, brange:%" PRId64 "-%" PRId64 " %s", pReader, @@ -2836,11 +2926,11 @@ _end: static int32_t doSumSttBlockRows(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; - SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + SSttBlockReader* pSttBlockReader = pReader->status.fileIter.pSttBlockReader; SSttBlockLoadInfo* pBlockLoadInfo = NULL; #if 0 for (int32_t i = 0; i < pReader->pFileReader->pSet->nSttF; ++i) { // open all last file - pBlockLoadInfo = &pLastBlockReader->pInfo[i]; + pBlockLoadInfo = &pSttBlockReader->pInfo[i]; code = tsdbReadSttBlk(pReader->pFileReader, i, pBlockLoadInfo->aSttBlk); if (code) { @@ -2931,10 +3021,12 @@ static int32_t readRowsCountFromMem(STsdbReader* pReader) { return code; } -static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { +static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t endKey) { SReaderStatus* pStatus = &pReader->status; STableUidList* pUidList = &pStatus->uidList; + tsdbDebug("seq load data blocks from cache, %s", pReader->idStr); + while (1) { if (pReader->code != TSDB_CODE_SUCCESS) { tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); @@ -2948,13 +3040,12 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { if (!hasNexTable) { return TSDB_CODE_SUCCESS; } - pBlockScanInfo = pStatus->pTableIter; + continue; } initMemDataIterator(*pBlockScanInfo, pReader); initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost); - int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN; int32_t code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey); if (code != TSDB_CODE_SUCCESS) { return code; @@ -3038,15 +3129,17 @@ typedef enum { TSDB_READ_CONTINUE = 0x2, } ERetrieveType; -static ERetrieveType doReadDataFromLastFiles(STsdbReader* pReader) { +static ERetrieveType doReadDataFromSttFiles(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; SDataBlockIter* pBlockIter = &pReader->status.blockIter; + tsdbDebug("seq load data blocks from stt files %s", pReader->idStr); + while (1) { terrno = 0; - code = doLoadLastBlockSequentially(pReader); + code = doLoadSttBlockSequentially(pReader); if (code != TSDB_CODE_SUCCESS) { terrno = code; return TSDB_READ_RETURN; @@ -3083,7 +3176,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { if (pBlockIter->numOfBlocks == 0) { // let's try to extract data from stt files. - ERetrieveType type = doReadDataFromLastFiles(pReader); + ERetrieveType type = doReadDataFromSttFiles(pReader); if (type == TSDB_READ_RETURN) { return terrno; } @@ -3118,7 +3211,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { resetDataBlockIterator(pBlockIter, pReader->info.order); resetTableListIndex(&pReader->status); - ERetrieveType type = doReadDataFromLastFiles(pReader); + ERetrieveType type = doReadDataFromSttFiles(pReader); if (type == TSDB_READ_RETURN) { return terrno; } @@ -3450,12 +3543,12 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc return TSDB_CODE_SUCCESS; } -int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, - SRowMerger* pMerger, SVersionRange* pVerRange, const char* idStr) { - while (nextRowFromSttBlocks(pLastBlockReader, pScanInfo, pVerRange)) { - int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); +int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, + SRowMerger* pMerger, SVersionRange* pVerRange, const char* idStr) { + while (nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pVerRange)) { + int64_t next1 = getCurrentKeyInSttBlock(pSttBlockReader); if (next1 == ts) { - TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); tsdbRowMergerAdd(pMerger, pRow1, NULL); } else { tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid, @@ -3774,7 +3867,6 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e int32_t code = TSDB_CODE_SUCCESS; do { - // SRow* pTSRow = NULL; TSDBROW row = {.type = -1}; bool freeTSRow = false; tsdbGetNextRowInMem(pBlockScanInfo, pReader, &row, endKey, &freeTSRow); @@ -3783,6 +3875,7 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e } if (row.type == TSDBROW_ROW_FMT) { + int64_t ts = row.pTSRow->ts;; code = doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo); if (freeTSRow) { @@ -3792,13 +3885,17 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e if (code) { return code; } + + pBlockScanInfo->lastProcKey = ts; } else { code = doAppendRowFromFileBlock(pBlock, pReader, row.pBlockData, row.iRow); if (code) { break; } + pBlockScanInfo->lastProcKey = row.pBlockData->aTSKEY[row.iRow]; } + // no data in buffer, return immediately if (!(pBlockScanInfo->iter.hasVal || pBlockScanInfo->iiter.hasVal)) { break; @@ -3885,6 +3982,11 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; + if (pReader->bFilesetDelimited) { + getMemTableTimeRange(pReader, &pReader->status.memTableMaxKey, &pReader->status.memTableMinKey); + pReader->status.bProcMemFirstFileset = true; + } + initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); resetDataBlockIterator(&pStatus->blockIter, pReader->info.order); @@ -4107,10 +4209,10 @@ void tsdbReaderClose2(STsdbReader* pReader) { tsdbDataFileReaderClose(&pReader->pFileReader); } - SCostSummary* pCost = &pReader->cost; + SReadCostSummary* pCost = &pReader->cost; SFilesetIter* pFilesetIter = &pReader->status.fileIter; - if (pFilesetIter->pLastBlockReader != NULL) { - SLastBlockReader* pLReader = pFilesetIter->pLastBlockReader; + if (pFilesetIter->pSttBlockReader != NULL) { + SSttBlockReader* pLReader = pFilesetIter->pSttBlockReader; tMergeTreeClose(&pLReader->mergeTree); taosMemoryFree(pLReader); } @@ -4122,6 +4224,7 @@ void tsdbReaderClose2(STsdbReader* pReader) { tsdbUntakeReadSnap2(pReader, pReader->pReadSnap, true); pReader->pReadSnap = NULL; + tsem_destroy(&pReader->resumeAfterSuspend); tsdbReleaseReader(pReader); tsdbUninitReaderLock(pReader); @@ -4132,12 +4235,12 @@ void tsdbReaderClose2(STsdbReader* pReader) { "build in-memory-block-time:%.2f ms, sttBlocks:%" PRId64 ", sttBlocks-time:%.2f ms, sttStatisBlock:%" PRId64 ", stt-statis-Block-time:%.2f ms, composed-blocks:%" PRId64 ", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb, createTime:%.2f ms,createSkylineIterTime:%.2f " - "ms, initLastBlockReader:%.2fms, %s", + "ms, initSttBlockReader:%.2fms, %s", pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, pCost->numOfBlocks, pCost->blockLoadTime, pCost->buildmemBlock, pCost->sttCost.loadBlocks, pCost->sttCost.blockElapsedTime, pCost->sttCost.loadStatisBlocks, pCost->sttCost.statisElapsedTime, pCost->composedBlocks, pCost->buildComposedBlockTime, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pCost->createScanInfoList, - pCost->createSkylineIterTime, pCost->initLastBlockReader, pReader->idStr); + pCost->createSkylineIterTime, pCost->initSttBlockReader, pReader->idStr); taosMemoryFree(pReader->idStr); @@ -4154,6 +4257,8 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; STableBlockScanInfo* pBlockScanInfo = NULL; + pReader->status.suspendInvoked = true; // record the suspend status + if (pStatus->loadFromFile) { SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); if (pBlockInfo != NULL) { @@ -4167,84 +4272,38 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { tsdbDataFileReaderClose(&pReader->pFileReader); - SCostSummary* pCost = &pReader->cost; + SReadCostSummary* pCost = &pReader->cost; pReader->status.pLDataIterArray = destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost); pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES); - // resetDataBlockScanInfo excluding lastKey - STableBlockScanInfo** p = NULL; - int32_t iter = 0; - - while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { - STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; - - pInfo->iterInit = false; - pInfo->iter.hasVal = false; - pInfo->iiter.hasVal = false; - - if (pInfo->iter.iter != NULL) { - pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter); - } - - if (pInfo->iiter.iter != NULL) { - pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter); - } - - pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); - pInfo->pFileDelData = taosArrayDestroy(pInfo->pFileDelData); - } - } else { - // resetDataBlockScanInfo excluding lastKey - STableBlockScanInfo** p = NULL; - int32_t iter = 0; - - while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { - STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; - - pInfo->iterInit = false; - pInfo->iter.hasVal = false; - pInfo->iiter.hasVal = false; - - if (pInfo->iter.iter != NULL) { - pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter); - } - - if (pInfo->iiter.iter != NULL) { - pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter); - } - - pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); - } - - pBlockScanInfo = pStatus->pTableIter == NULL ? NULL : *pStatus->pTableIter; - if (pBlockScanInfo) { - // save lastKey to restore memory iterator - STimeWindow w = pReader->resBlockInfo.pResBlock->info.window; - pBlockScanInfo->lastProcKey = ASCENDING_TRAVERSE(pReader->info.order) ? w.ekey : w.skey; - - // reset current current table's data block scan info, - pBlockScanInfo->iterInit = false; - - pBlockScanInfo->iter.hasVal = false; - pBlockScanInfo->iiter.hasVal = false; - if (pBlockScanInfo->iter.iter != NULL) { - pBlockScanInfo->iter.iter = tsdbTbDataIterDestroy(pBlockScanInfo->iter.iter); - } - - if (pBlockScanInfo->iiter.iter != NULL) { - pBlockScanInfo->iiter.iter = tsdbTbDataIterDestroy(pBlockScanInfo->iiter.iter); - } - - pBlockScanInfo->pBlockList = taosArrayDestroy(pBlockScanInfo->pBlockList); - pBlockScanInfo->pBlockIdxList = taosArrayDestroy(pBlockScanInfo->pBlockIdxList); - // TODO: keep skyline for reuse - pBlockScanInfo->delSkyline = taosArrayDestroy(pBlockScanInfo->delSkyline); - } } + // resetDataBlockScanInfo excluding lastKey + STableBlockScanInfo** p = NULL; + + int32_t step = ASCENDING_TRAVERSE(pReader->info.order)? 1:-1; + + int32_t iter = 0; + while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { + STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; + clearBlockScanInfo(pInfo); + pInfo->sttKeyInfo.nextProcKey = pInfo->lastProcKey + step; + } + + pStatus->uidList.currentIndex = 0; + initReaderStatus(pStatus); + tsdbUntakeReadSnap2(pReader, pReader->pReadSnap, false); pReader->pReadSnap = NULL; + if (pReader->bFilesetDelimited) { + pReader->status.memTableMinKey = INT64_MAX; + pReader->status.memTableMaxKey = INT64_MIN; + } pReader->flag = READER_STATUS_SUSPEND; +#if SUSPEND_RESUME_TEST + tsem_post(&pReader->resumeAfterSuspend); +#endif + tsdbDebug("reader: %p suspended uid %" PRIu64 " in this query %s", pReader, pBlockScanInfo ? pBlockScanInfo->uid : 0, pReader->idStr); return code; @@ -4350,6 +4409,72 @@ static bool tsdbReadRowsCountOnly(STsdbReader* pReader) { return pBlock->info.rows > 0; } +static int32_t doTsdbNextDataBlockFilesetDelimited(STsdbReader* pReader) { + SReaderStatus* pStatus = &pReader->status; + int32_t code = TSDB_CODE_SUCCESS; + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; + + if (pStatus->loadFromFile) { + if (pStatus->bProcMemPreFileset) { + int32_t fid = pReader->status.pCurrentFileset->fid; + STimeWindow win = {0}; + tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &win.skey, &win.ekey); + + int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? win.skey : win.ekey; + code = buildBlockFromBufferSequentially(pReader, endKey); + if (code != TSDB_CODE_SUCCESS || pBlock->info.rows > 0) { + return code; + } else { + pStatus->bProcMemPreFileset = false; + if (pReader->notifyFn) { + STsdReaderNotifyInfo info = {0}; + info.duration.filesetId = fid; + pReader->notifyFn(TSD_READER_NOTIFY_DURATION_START, &info, pReader->notifyParam); + } + resetTableListIndex(pStatus); + } + } + + code = buildBlockFromFiles(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (pBlock->info.rows <= 0) { + resetTableListIndex(&pReader->status); + int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN; + code = buildBlockFromBufferSequentially(pReader, endKey); + } + } else { // no data in files, let's try the buffer + int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN; + code = buildBlockFromBufferSequentially(pReader, endKey); + } + return code; +} + +static int32_t doTsdbNextDataBlockFilesFirst(STsdbReader* pReader) { + SReaderStatus* pStatus = &pReader->status; + int32_t code = TSDB_CODE_SUCCESS; + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; + + if (pStatus->loadFromFile) { + code = buildBlockFromFiles(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (pBlock->info.rows <= 0) { + resetTableListIndex(&pReader->status); + int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN; + code = buildBlockFromBufferSequentially(pReader, endKey); + } + } else { // no data in files, let's try the buffer + int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN; + code = buildBlockFromBufferSequentially(pReader, endKey); + } + return code; +} + static int32_t doTsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { int32_t code = TSDB_CODE_SUCCESS; @@ -4367,19 +4492,10 @@ static int32_t doTsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { if (READ_MODE_COUNT_ONLY == pReader->info.readMode) { return tsdbReadRowsCountOnly(pReader); } - - if (pStatus->loadFromFile) { - code = buildBlockFromFiles(pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (pBlock->info.rows <= 0) { - resetTableListIndex(&pReader->status); - code = buildBlockFromBufferSequentially(pReader); - } - } else { // no data in files, let's try the buffer - code = buildBlockFromBufferSequentially(pReader); + if (!pReader->bFilesetDelimited) { + code = doTsdbNextDataBlockFilesFirst(pReader); + } else { + code = doTsdbNextDataBlockFilesetDelimited(pReader); } *hasNext = pBlock->info.rows > 0; @@ -4399,6 +4515,16 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { SReaderStatus* pStatus = &pReader->status; + // NOTE: the following codes is used to perform test for suspend/resume for tsdbReader when it blocks the commit + // the data should be ingested in round-robin and all the child tables should be createted before ingesting data + // the version range of query will be used to identify the correctness of suspend/resume functions. + // this function will blocked before loading the SECOND block from vnode-buffer, and restart itself from sst-files +#if SUSPEND_RESUME_TEST + if (!pReader->status.suspendInvoked && !pReader->status.loadFromFile) { + tsem_wait(&pReader->resumeAfterSuspend); + } +#endif + code = tsdbAcquireReader(pReader); qTrace("tsdb/read: %p, take read mutex, code: %d", pReader, code); @@ -4779,7 +4905,20 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT return code; } } + + SMergeTreeConf conf = { + .pReader = pReader, + .pSchema = pReader->info.pSchema, + .pCols = pReader->suppInfo.colId, + .numOfCols = pReader->suppInfo.numOfCols, + .suid = pReader->info.suid, + }; + SReaderStatus* pStatus = &pReader->status; + if (pStatus->pCurrentFileset != NULL) { + pTableBlockInfo->numOfSttRows += tsdbGetRowsInSttFiles(pStatus->pCurrentFileset, pStatus->pLDataIterArray, + pReader->pTsdb, &conf, pReader->idStr); + } STsdbCfg* pc = &pReader->pTsdb->pVnode->config.tsdbCfg; pTableBlockInfo->defMinRows = pc->minRows; @@ -4816,10 +4955,6 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT pTableBlockInfo->minRows = numOfRows; } - if (numOfRows < defaultRows) { - pTableBlockInfo->numOfSmallBlocks += 1; - } - pTableBlockInfo->totalSize += pBlockInfo->record.blockSize; int32_t bucketIndex = getBucketIndex(pTableBlockInfo->defMinRows, bucketRange, numOfRows, numOfBuckets); @@ -4832,17 +4967,70 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT break; } + // add the data in stt files of new fileset + if (pStatus->pCurrentFileset != NULL) { + pTableBlockInfo->numOfSttRows += tsdbGetRowsInSttFiles(pStatus->pCurrentFileset, pStatus->pLDataIterArray, + pReader->pTsdb, &conf, pReader->idStr); + } + pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks; hasNext = (pBlockIter->numOfBlocks > 0); } - - // tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %s", pReader, numOfBlocks, numOfTables, - // pReader->pFileGroup->fid, pReader->idStr); } + + // record the data in stt files tsdbReleaseReader(pReader); return code; } +static void getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey, int64_t* pMinKey) { + int32_t code = TSDB_CODE_SUCCESS; + int64_t rows = 0; + + SReaderStatus* pStatus = &pReader->status; + + int32_t iter = 0; + int64_t maxKey = INT64_MIN; + int64_t minKey = INT64_MAX; + + void* pHashIter = tSimpleHashIterate(pStatus->pTableMap, NULL, &iter); + while (pHashIter!= NULL) { + STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)pHashIter; + + STbData* d = NULL; + if (pReader->pReadSnap->pMem != NULL) { + d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->info.suid, pBlockScanInfo->uid); + if (d != NULL) { + if (d->maxKey > maxKey) { + maxKey = d->maxKey; + } + if (d->minKey < minKey) { + minKey = d->minKey; + } + } + } + + STbData* di = NULL; + if (pReader->pReadSnap->pIMem != NULL) { + di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->info.suid, pBlockScanInfo->uid); + if (di != NULL) { + if (di->maxKey > maxKey) { + maxKey = di->maxKey; + } + if (di->minKey < minKey) { + minKey = di->minKey; + } + } + } + + // current table is exhausted, let's try the next table + pHashIter = tSimpleHashIterate(pStatus->pTableMap, pHashIter, &iter); + } + + *pMaxKey = maxKey; + *pMinKey = minKey; +} + int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; int64_t rows = 0; @@ -5025,8 +5213,17 @@ void tsdbUntakeReadSnap2(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proact void tsdbReaderSetId2(STsdbReader* pReader, const char* idstr) { taosMemoryFreeClear(pReader->idStr); pReader->idStr = taosStrdup(idstr); - pReader->status.fileIter.pLastBlockReader->mergeTree.idStr = pReader->idStr; + pReader->status.fileIter.pSttBlockReader->mergeTree.idStr = pReader->idStr; } void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/ } + +void tsdbSetFilesetDelimited(STsdbReader* pReader) { + pReader->bFilesetDelimited = true; +} + +void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param) { + pReader->notifyFn = notifyFn; + pReader->notifyParam = param; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 305399e0af..a058c0173d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -210,6 +210,7 @@ void clearBlockScanInfo(STableBlockScanInfo* p) { p->iterInit = false; p->iter.hasVal = false; p->iiter.hasVal = false; + p->sttKeyInfo.status = STT_FILE_READER_UNINIT; if (p->iter.iter != NULL) { p->iter.iter = tsdbTbDataIterDestroy(p->iter.iter); @@ -487,6 +488,8 @@ typedef enum { BLK_CHECK_QUIT = 0x2, } ETombBlkCheckEnum; +static void loadNextStatisticsBlock(SSttFileReader* pSttFileReader, const SSttBlockLoadInfo* pBlockLoadInfo, + const TStatisBlkArray* pStatisBlkArray, int32_t numOfRows, int32_t* i, int32_t* j); static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_t numOfTables, int32_t* j, ETombBlkCheckEnum* pRet) { int32_t code = 0; @@ -658,3 +661,177 @@ void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemT } } } + +int32_t getNumOfRowsInSttBlock(SSttFileReader *pSttFileReader, SSttBlockLoadInfo *pBlockLoadInfo, uint64_t suid, + const uint64_t* pUidList, int32_t numOfTables) { + int32_t num = 0; + + const TStatisBlkArray *pStatisBlkArray = pBlockLoadInfo->pSttStatisBlkArray; + if (TARRAY2_SIZE(pStatisBlkArray) <= 0) { + return 0; + } + + int32_t i = 0; + while((i < TARRAY2_SIZE(pStatisBlkArray)) && (pStatisBlkArray->data[i].minTbid.suid < suid)) { + ++i; + } + + if (i >= TARRAY2_SIZE(pStatisBlkArray)) { + return 0; + } + + SStatisBlk *p = &pStatisBlkArray->data[i]; + if (pBlockLoadInfo->statisBlock == NULL) { + pBlockLoadInfo->statisBlock = taosMemoryCalloc(1, sizeof(STbStatisBlock)); + tStatisBlockInit(pBlockLoadInfo->statisBlock); + } + + int64_t st = taosGetTimestampMs(); + tsdbSttFileReadStatisBlock(pSttFileReader, p, pBlockLoadInfo->statisBlock); + pBlockLoadInfo->statisBlockIndex = i; + + double el = (taosGetTimestampMs() - st) / 1000.0; + pBlockLoadInfo->cost.loadStatisBlocks += 1; + pBlockLoadInfo->cost.statisElapsedTime += el; + + STbStatisBlock *pBlock = pBlockLoadInfo->statisBlock; + + int32_t index = 0; + while (index < TARRAY2_SIZE(pBlock->suid) && pBlock->suid->data[index] < suid) { + ++index; + } + + if (index >= TARRAY2_SIZE(pBlock->suid)) { + return num; + } + + int32_t j = index; + int32_t uidIndex = 0; + while (i < TARRAY2_SIZE(pStatisBlkArray) && uidIndex <= numOfTables) { + p = &pStatisBlkArray->data[i]; + if (p->minTbid.suid > suid) { + return num; + } + + uint64_t uid = pUidList[uidIndex]; + + if (pBlock->uid->data[j] == uid) { + num += pBlock->count->data[j]; + uidIndex += 1; + j += 1; + loadNextStatisticsBlock(pSttFileReader, pBlockLoadInfo, pStatisBlkArray, pBlock->suid->size, &i, &j); + } else if (pBlock->uid->data[j] < uid) { + j += 1; + loadNextStatisticsBlock(pSttFileReader, pBlockLoadInfo, pStatisBlkArray, pBlock->suid->size, &i, &j); + } else { + uidIndex += 1; + } + } + + return num; +} + +// load next stt statistics block +static void loadNextStatisticsBlock(SSttFileReader* pSttFileReader, const SSttBlockLoadInfo* pBlockLoadInfo, + const TStatisBlkArray* pStatisBlkArray, int32_t numOfRows, int32_t* i, int32_t* j) { + if ((*j) >= numOfRows) { + (*i) += 1; + (*j) = 0; + if ((*i) < TARRAY2_SIZE(pStatisBlkArray)) { + tsdbSttFileReadStatisBlock(pSttFileReader, &pStatisBlkArray->data[(*i)], pBlockLoadInfo->statisBlock); + } + } +} + +void doAdjustValidDataIters(SArray* pLDIterList, int32_t numOfFileObj) { + int32_t size = taosArrayGetSize(pLDIterList); + + if (size < numOfFileObj) { + int32_t inc = numOfFileObj - size; + for (int32_t k = 0; k < inc; ++k) { + SLDataIter *pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); + taosArrayPush(pLDIterList, &pIter); + } + } else if (size > numOfFileObj) { // remove unused LDataIter + int32_t inc = size - numOfFileObj; + + for (int i = 0; i < inc; ++i) { + SLDataIter *pIter = taosArrayPop(pLDIterList); + destroyLDataIter(pIter); + } + } +} + +int32_t adjustLDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet) { + int32_t numOfLevels = pFileSet->lvlArr->size; + + // add the list/iter placeholder + while (taosArrayGetSize(pSttFileBlockIterArray) < numOfLevels) { + SArray* pList = taosArrayInit(4, POINTER_BYTES); + taosArrayPush(pSttFileBlockIterArray, &pList); + } + + for(int32_t j = 0; j < numOfLevels; ++j) { + SSttLvl* pSttLevel = pFileSet->lvlArr->data[j]; + SArray* pList = taosArrayGetP(pSttFileBlockIterArray, j); + doAdjustValidDataIters(pList, TARRAY2_SIZE(pSttLevel->fobjArr)); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArray, STsdb* pTsdb, SMergeTreeConf* pConf, + const char* pstr) { + int32_t numOfRows = 0; + + // no data exists, go to end + int32_t numOfLevels = pFileSet->lvlArr->size; + if (numOfLevels == 0) { + return numOfRows; + } + + // add the list/iter placeholder + adjustLDataIters(pSttFileBlockIterArray, pFileSet); + + for (int32_t j = 0; j < numOfLevels; ++j) { + SSttLvl* pSttLevel = pFileSet->lvlArr->data[j]; + SArray* pList = taosArrayGetP(pSttFileBlockIterArray, j); + + for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) { // open all last file + SLDataIter* pIter = taosArrayGetP(pList, i); + + // open stt file reader if not opened yet + // if failed to open this stt file, ignore the error and try next one + if (pIter->pReader == NULL) { + SSttFileReaderConfig conf = {.tsdb = pTsdb, .szPage = pTsdb->pVnode->config.tsdbPageSize}; + conf.file[0] = *pSttLevel->fobjArr->data[i]->f; + + const char* pName = pSttLevel->fobjArr->data[i]->fname; + int32_t code = tsdbSttFileReaderOpen(pName, &conf, &pIter->pReader); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("open stt file reader error. file:%s, code %s, %s", pName, tstrerror(code), pstr); + continue; + } + } + + if (pIter->pBlockLoadInfo == NULL) { + pIter->pBlockLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols); + } + + // load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file + int32_t code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray **)&pIter->pBlockLoadInfo->pSttStatisBlkArray); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), pstr); + continue; + } + + // extract rows from each stt file one-by-one + STsdbReader* pReader = pConf->pReader; + int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); + uint64_t* pUidList = pReader->status.uidList.tableUidList; + numOfRows += getNumOfRowsInSttBlock(pIter->pReader, pIter->pBlockLoadInfo, pConf->suid, pUidList, numOfTables); + } + } + + return numOfRows; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index 60e6e6960a..f4fa7bc2c8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -22,6 +22,7 @@ extern "C" { #include "tsdbDataFileRW.h" #include "tsdbUtil2.h" +#include "storageapi.h" #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) @@ -96,7 +97,7 @@ typedef struct SResultBlockInfo { int64_t capacity; } SResultBlockInfo; -typedef struct SCostSummary { +typedef struct SReadCostSummary { int64_t numOfBlocks; double blockLoadTime; double buildmemBlock; @@ -109,8 +110,8 @@ typedef struct SCostSummary { double buildComposedBlockTime; double createScanInfoList; double createSkylineIterTime; - double initLastBlockReader; -} SCostSummary; + double initSttBlockReader; +} SReadCostSummary; typedef struct STableUidList { uint64_t* tableUidList; // access table uid list in uid ascending order list @@ -122,12 +123,6 @@ typedef struct { int32_t numOfSttFiles; } SBlockNumber; -typedef struct SBlockIndex { - int32_t ordinalIndex; - int64_t inFileOffset; - STimeWindow window; // todo replace it with overlap flag. -} SBlockIndex; - typedef struct SBlockOrderWrapper { int64_t uid; int64_t offset; @@ -151,21 +146,21 @@ typedef struct SBlockLoadSuppInfo { bool smaValid; // the sma on all queried columns are activated } SBlockLoadSuppInfo; -typedef struct SLastBlockReader { +typedef struct SSttBlockReader { STimeWindow window; SVersionRange verRange; int32_t order; uint64_t uid; SMergeTree mergeTree; int64_t currentKey; -} SLastBlockReader; +} SSttBlockReader; typedef struct SFilesetIter { int32_t numOfFiles; // number of total files int32_t index; // current accessed index in the list TFileSetArray* pFilesetList; // data file set list int32_t order; - SLastBlockReader* pLastBlockReader; // last file block reader + SSttBlockReader* pSttBlockReader; // last file block reader } SFilesetIter; typedef struct SFileDataBlockInfo { @@ -192,6 +187,7 @@ typedef struct SFileBlockDumpInfo { } SFileBlockDumpInfo; typedef struct SReaderStatus { + bool suspendInvoked; bool loadFromFile; // check file stage bool composedDataBlock; // the returned data block is a composed block or not SSHashObj* pTableMap; // SHash @@ -205,6 +201,12 @@ typedef struct SReaderStatus { SArray* pLDataIterArray; SRowMerger merger; SColumnInfoData* pPrimaryTsCol; // primary time stamp output col info data + bool bProcMemPreFileset; + int64_t memTableMaxKey; + int64_t memTableMinKey; + int64_t prevFilesetStartKey; + int64_t prevFilesetEndKey; + bool bProcMemFirstFileset; } SReaderStatus; struct STsdbReader { @@ -220,13 +222,17 @@ struct STsdbReader { int32_t type; // query type: 1. retrieve all data blocks, 2. retrieve direct prev|next rows SBlockLoadSuppInfo suppInfo; STsdbReadSnap* pReadSnap; - SCostSummary cost; + tsem_t resumeAfterSuspend; + SReadCostSummary cost; SHashObj** pIgnoreTables; SSHashObj* pSchemaMap; // keep the retrieved schema info, to avoid the overhead by repeatly load schema SDataFileReader* pFileReader; // the file reader SBlockInfoBuf blockInfoBuf; EContentData step; STsdbReader* innerReader[2]; + bool bFilesetDelimited; // duration by duration output + TsdReaderNotifyCbFn notifyFn; + void* notifyParam; }; typedef struct SBrinRecordIter { @@ -266,7 +272,12 @@ bool blockIteratorNext(SDataBlockIter* pBlockIter, const char* idStr); void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver); int32_t loadDataFileTombDataForAll(STsdbReader* pReader); int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo); - +int32_t getNumOfRowsInSttBlock(SSttFileReader *pSttFileReader, SSttBlockLoadInfo *pBlockLoadInfo, uint64_t suid, + const uint64_t* pUidList, int32_t numOfTables); +void destroyLDataIter(SLDataIter* pIter); +int32_t adjustLDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet); +int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArray, STsdb* pTsdb, SMergeTreeConf* pConf, + const char* pstr); typedef struct { SArray* pTombData; } STableLoadInfo; diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index f6888ba9cb..dc98f46ac5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -249,7 +249,7 @@ _exit: if (code) { TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); } else { - tsdbInfo("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__); + tsdbDebug("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__); } return code; } @@ -279,7 +279,7 @@ _exit: if (code) { TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); } else { - tsdbInfo("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__); + tsdbDebug("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__); } tsdbFSDestroyCopySnapshot(&rtner->fsetArr); return code; @@ -391,32 +391,6 @@ _exit: static void tsdbFreeRtnArg(void *arg) { taosMemoryFree(arg); } -static int32_t tsdbDoRetentionSync(void *arg) { - int32_t code = 0; - int32_t lino = 0; - SRTNer rtner[1] = {0}; - - code = tsdbDoRetentionBegin(arg, rtner); - TSDB_CHECK_CODE(code, lino, _exit); - - STFileSet *fset; - TARRAY2_FOREACH(rtner->fsetArr, fset) { - code = tsdbDoRetentionOnFileSet(rtner, fset); - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = tsdbDoRetentionEnd(rtner); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); - } - tsem_post(&((SRtnArg *)arg)->tsdb->pVnode->canCommit); - tsdbFreeRtnArg(arg); - return code; -} - static int32_t tsdbDoRetentionAsync(void *arg) { int32_t code = 0; int32_t lino = 0; @@ -454,49 +428,46 @@ _exit: int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) { int32_t code = 0; - if (sync) { // sync retention + taosThreadMutexLock(&tsdb->mutex); + + if (tsdb->bgTaskDisabled) { + taosThreadMutexUnlock(&tsdb->mutex); + return 0; + } + + STFileSet *fset; + TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) { + code = tsdbTFileSetOpenChannel(fset); + if (code) { + taosThreadMutexUnlock(&tsdb->mutex); + return code; + } + SRtnArg *arg = taosMemoryMalloc(sizeof(*arg)); if (arg == NULL) { + taosThreadMutexUnlock(&tsdb->mutex); return TSDB_CODE_OUT_OF_MEMORY; } arg->tsdb = tsdb; arg->now = now; - arg->fid = INT32_MAX; + arg->fid = fset->fid; - tsem_wait(&tsdb->pVnode->canCommit); - code = vnodeScheduleTask(tsdbDoRetentionSync, arg); + if (sync) { + code = vnodeAsyncC(vnodeAsyncHandle[0], tsdb->pVnode->commitChannel, EVA_PRIORITY_LOW, tsdbDoRetentionAsync, + tsdbFreeRtnArg, arg, NULL); + } else { + code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_LOW, tsdbDoRetentionAsync, + tsdbFreeRtnArg, arg, NULL); + } if (code) { - tsem_post(&tsdb->pVnode->canCommit); - taosMemoryFree(arg); + tsdbFreeRtnArg(arg); + taosThreadMutexUnlock(&tsdb->mutex); return code; } - } else { // async retention - taosThreadMutexLock(&tsdb->mutex); - - STFileSet *fset; - TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) { - SRtnArg *arg = taosMemoryMalloc(sizeof(*arg)); - if (arg == NULL) { - taosThreadMutexUnlock(&tsdb->mutex); - return TSDB_CODE_OUT_OF_MEMORY; - } - - arg->tsdb = tsdb; - arg->now = now; - arg->fid = fset->fid; - - code = tsdbFSScheduleBgTask(tsdb->pFS, fset->fid, TSDB_BG_TASK_RETENTION, tsdbDoRetentionAsync, tsdbFreeRtnArg, - arg, NULL); - if (code) { - tsdbFreeRtnArg(arg); - taosThreadMutexUnlock(&tsdb->mutex); - return code; - } - } - - taosThreadMutexUnlock(&tsdb->mutex); } + taosThreadMutexUnlock(&tsdb->mutex); + return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c new file mode 100644 index 0000000000..9dae9bdd36 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -0,0 +1,627 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" +#include "tsdbFS2.h" + +#define TSDB_SNAP_MSG_VER 1 + +// fset partition +static int32_t tsdbFSetPartCmprFn(STsdbFSetPartition* x, STsdbFSetPartition* y) { + if (x->fid < y->fid) return -1; + if (x->fid > y->fid) return 1; + return 0; +} + +static int32_t tVersionRangeCmprFn(SVersionRange* x, SVersionRange* y) { + if (x->minVer < y->minVer) return -1; + if (x->minVer > y->minVer) return 1; + if (x->maxVer < y->maxVer) return -1; + if (x->maxVer > y->maxVer) return 1; + return 0; +} + +static int32_t tsdbTFileSetRangeCmprFn(STFileSetRange* x, STFileSetRange* y) { + if (x->fid < y->fid) return -1; + if (x->fid > y->fid) return 1; + return 0; +} + +STsdbFSetPartition* tsdbFSetPartitionCreate() { + STsdbFSetPartition* pSP = taosMemoryCalloc(1, sizeof(STsdbFSetPartition)); + if (pSP == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + for (int32_t i = 0; i < TSDB_FSET_RANGE_TYP_MAX; i++) { + TARRAY2_INIT(&pSP->verRanges[i]); + } + return pSP; +} + +void tsdbFSetPartitionClear(STsdbFSetPartition** ppSP) { + if (ppSP == NULL || ppSP[0] == NULL) { + return; + } + for (int32_t i = 0; i < TSDB_FSET_RANGE_TYP_MAX; i++) { + TARRAY2_DESTROY(&ppSP[0]->verRanges[i], NULL); + } + taosMemoryFree(ppSP[0]); + ppSP[0] = NULL; +} + +static int32_t tsdbFTypeToFRangeType(tsdb_ftype_t ftype) { + switch (ftype) { + case TSDB_FTYPE_HEAD: + return TSDB_FSET_RANGE_TYP_HEAD; + case TSDB_FTYPE_DATA: + return TSDB_FSET_RANGE_TYP_DATA; + case TSDB_FTYPE_SMA: + return TSDB_FSET_RANGE_TYP_SMA; + case TSDB_FTYPE_TOMB: + return TSDB_FSET_RANGE_TYP_TOMB; + case TSDB_FTYPE_STT: + return TSDB_FSET_RANGE_TYP_STT; + } + return TSDB_FSET_RANGE_TYP_MAX; +} + +static int32_t tsdbTFileSetToFSetPartition(STFileSet* fset, STsdbFSetPartition** ppSP) { + STsdbFSetPartition* p = tsdbFSetPartitionCreate(); + if (p == NULL) { + goto _err; + } + + p->fid = fset->fid; + + int32_t code = 0; + int32_t typ = 0; + int32_t corrupt = false; + int32_t count = 0; + for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (fset->farr[ftype] == NULL) continue; + typ = tsdbFTypeToFRangeType(ftype); + ASSERT(typ < TSDB_FSET_RANGE_TYP_MAX); + STFile* f = fset->farr[ftype]->f; + if (f->maxVer > fset->maxVerValid) { + corrupt = true; + tsdbError("skip incomplete data file: fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 + ", ftype: %d", + fset->fid, fset->maxVerValid, f->minVer, f->maxVer, ftype); + continue; + } + count++; + SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer}; + code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); + ASSERT(code == 0); + } + + typ = TSDB_FSET_RANGE_TYP_STT; + const SSttLvl* lvl; + TARRAY2_FOREACH(fset->lvlArr, lvl) { + STFileObj* fobj; + TARRAY2_FOREACH(lvl->fobjArr, fobj) { + STFile* f = fobj->f; + if (f->maxVer > fset->maxVerValid) { + corrupt = true; + tsdbError("skip incomplete stt file.fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 + ", ftype: %d", + fset->fid, fset->maxVerValid, f->minVer, f->maxVer, typ); + continue; + } + count++; + SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer}; + code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); + ASSERT(code == 0); + } + } + if (corrupt && count == 0) { + SVersionRange vr = {.minVer = VERSION_MIN, .maxVer = fset->maxVerValid}; + code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); + ASSERT(code == 0); + } + ppSP[0] = p; + return 0; + +_err: + tsdbFSetPartitionClear(&p); + return -1; +} + +// fset partition list +STsdbFSetPartList* tsdbFSetPartListCreate() { + STsdbFSetPartList* pList = taosMemoryCalloc(1, sizeof(STsdbFSetPartList)); + if (pList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + TARRAY2_INIT(pList); + return pList; +} + +void tsdbFSetPartListDestroy(STsdbFSetPartList** ppList) { + if (ppList == NULL || ppList[0] == NULL) return; + + TARRAY2_DESTROY(ppList[0], tsdbFSetPartitionClear); + taosMemoryFree(ppList[0]); + ppList[0] = NULL; +} + +int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList* pList, TFileSetRangeArray** ppRanges) { + TFileSetRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TFileSetRangeArray)); + if (pDiff == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + TARRAY2_INIT(pDiff); + + STsdbFSetPartition* part; + TARRAY2_FOREACH(pList, part) { + STFileSetRange* r = taosMemoryCalloc(1, sizeof(STFileSetRange)); + if (r == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + int64_t maxVerValid = -1; + int32_t typMax = TSDB_FSET_RANGE_TYP_MAX; + for (int32_t i = 0; i < typMax; i++) { + SVerRangeList* iList = &part->verRanges[i]; + SVersionRange vr = {0}; + TARRAY2_FOREACH(iList, vr) { + if (vr.maxVer < vr.minVer) { + continue; + } + maxVerValid = TMAX(maxVerValid, vr.maxVer); + } + } + r->fid = part->fid; + r->sver = maxVerValid + 1; + r->ever = VERSION_MAX; + tsdbDebug("range diff fid:%" PRId64 ", sver:%" PRId64 ", ever:%" PRId64, part->fid, r->sver, r->ever); + int32_t code = TARRAY2_SORT_INSERT(pDiff, r, tsdbTFileSetRangeCmprFn); + ASSERT(code == 0); + } + ppRanges[0] = pDiff; + + tsdbInfo("pDiff size:%d", TARRAY2_SIZE(pDiff)); + return 0; + +_err: + if (pDiff) { + tsdbTFileSetRangeArrayDestroy(&pDiff); + } + return -1; +} + +// serialization +int32_t tTsdbFSetPartListDataLenCalc(STsdbFSetPartList* pList) { + int32_t hdrLen = sizeof(int32_t); + int32_t datLen = 0; + + int8_t msgVer = 1; + int32_t len = TARRAY2_SIZE(pList); + hdrLen += sizeof(msgVer); + hdrLen += sizeof(len); + datLen += hdrLen; + + for (int32_t u = 0; u < len; u++) { + STsdbFSetPartition* p = TARRAY2_GET(pList, u); + int32_t typMax = TSDB_FSET_RANGE_TYP_MAX; + int32_t uItem = 0; + uItem += sizeof(STsdbFSetPartition); + uItem += sizeof(typMax); + + for (int32_t i = 0; i < typMax; i++) { + int32_t iLen = TARRAY2_SIZE(&p->verRanges[i]); + int32_t jItem = 0; + jItem += sizeof(SVersionRange); + jItem += sizeof(int64_t); + uItem += sizeof(iLen) + jItem * iLen; + } + datLen += uItem; + } + return datLen; +} + +int32_t tSerializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* pList) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + int8_t reserved8 = 0; + int16_t reserved16 = 0; + int64_t reserved64 = 0; + + int8_t msgVer = TSDB_SNAP_MSG_VER; + int32_t len = TARRAY2_SIZE(pList); + + if (tStartEncode(&encoder) < 0) goto _err; + if (tEncodeI8(&encoder, msgVer) < 0) goto _err; + if (tEncodeI32(&encoder, len) < 0) goto _err; + + for (int32_t u = 0; u < len; u++) { + STsdbFSetPartition* p = TARRAY2_GET(pList, u); + if (tEncodeI64(&encoder, p->fid) < 0) goto _err; + if (tEncodeI8(&encoder, p->stat) < 0) goto _err; + if (tEncodeI8(&encoder, reserved8) < 0) goto _err; + if (tEncodeI16(&encoder, reserved16) < 0) goto _err; + + int32_t typMax = TSDB_FSET_RANGE_TYP_MAX; + if (tEncodeI32(&encoder, typMax) < 0) goto _err; + + for (int32_t i = 0; i < typMax; i++) { + SVerRangeList* iList = &p->verRanges[i]; + int32_t iLen = TARRAY2_SIZE(iList); + + if (tEncodeI32(&encoder, iLen) < 0) goto _err; + for (int32_t j = 0; j < iLen; j++) { + SVersionRange r = TARRAY2_GET(iList, j); + if (tEncodeI64(&encoder, r.minVer) < 0) goto _err; + if (tEncodeI64(&encoder, r.maxVer) < 0) goto _err; + if (tEncodeI64(&encoder, reserved64) < 0) goto _err; + } + } + } + + tEndEncode(&encoder); + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; + +_err: + tEncoderClear(&encoder); + return -1; +} + +int32_t tDeserializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* pList) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + int8_t reserved8 = 0; + int16_t reserved16 = 0; + int64_t reserved64 = 0; + + STsdbFSetPartition* p = NULL; + + int8_t msgVer = 0; + int32_t len = 0; + if (tStartDecode(&decoder) < 0) goto _err; + if (tDecodeI8(&decoder, &msgVer) < 0) goto _err; + if (msgVer != TSDB_SNAP_MSG_VER) goto _err; + if (tDecodeI32(&decoder, &len) < 0) goto _err; + + for (int32_t u = 0; u < len; u++) { + p = tsdbFSetPartitionCreate(); + if (p == NULL) goto _err; + if (tDecodeI64(&decoder, &p->fid) < 0) goto _err; + if (tDecodeI8(&decoder, &p->stat) < 0) goto _err; + if (tDecodeI8(&decoder, &reserved8) < 0) goto _err; + if (tDecodeI16(&decoder, &reserved16) < 0) goto _err; + + int32_t typMax = 0; + if (tDecodeI32(&decoder, &typMax) < 0) goto _err; + + for (int32_t i = 0; i < typMax; i++) { + SVerRangeList* iList = &p->verRanges[i]; + int32_t iLen = 0; + if (tDecodeI32(&decoder, &iLen) < 0) goto _err; + for (int32_t j = 0; j < iLen; j++) { + SVersionRange r = {0}; + if (tDecodeI64(&decoder, &r.minVer) < 0) goto _err; + if (tDecodeI64(&decoder, &r.maxVer) < 0) goto _err; + if (tDecodeI64(&decoder, &reserved64) < 0) goto _err; + TARRAY2_APPEND(iList, r); + } + } + TARRAY2_APPEND(pList, p); + p = NULL; + } + + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; + +_err: + if (p) { + tsdbFSetPartitionClear(&p); + } + tDecoderClear(&decoder); + return -1; +} + +// fs state +static STsdbFSetPartList* tsdbSnapGetFSetPartList(STFileSystem* fs) { + STsdbFSetPartList* pList = tsdbFSetPartListCreate(); + if (pList == NULL) { + return NULL; + } + + int32_t code = 0; + taosThreadMutexLock(&fs->tsdb->mutex); + STFileSet* fset; + TARRAY2_FOREACH(fs->fSetArr, fset) { + STsdbFSetPartition* pItem = NULL; + if (tsdbTFileSetToFSetPartition(fset, &pItem) < 0) { + code = -1; + break; + } + ASSERT(pItem != NULL); + code = TARRAY2_SORT_INSERT(pList, pItem, tsdbFSetPartCmprFn); + ASSERT(code == 0); + } + taosThreadMutexUnlock(&fs->tsdb->mutex); + + if (code) { + TARRAY2_DESTROY(pList, tsdbFSetPartitionClear); + taosMemoryFree(pList); + pList = NULL; + } + return pList; +} + +ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) { + if (!VND_IS_RSMA(pVnode)) { + return pVnode->pTsdb->pFS->fsstate; + } + for (int32_t lvl = 0; lvl < TSDB_RETENTION_MAX; ++lvl) { + STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, lvl); + if (pTsdb && pTsdb->pFS->fsstate != TSDB_FS_STATE_NORMAL) { + return TSDB_FS_STATE_INCOMPLETE; + } + } + return TSDB_FS_STATE_NORMAL; +} + +// description +typedef struct STsdbPartitionInfo { + int32_t vgId; + int32_t tsdbMaxCnt; + int32_t subTyps[TSDB_RETENTION_MAX]; + STsdbFSetPartList* pLists[TSDB_RETENTION_MAX]; +} STsdbPartitionInfo; + +static int32_t tsdbPartitionInfoInit(SVnode* pVnode, STsdbPartitionInfo* pInfo) { + int32_t subTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2}; + pInfo->vgId = TD_VID(pVnode); + pInfo->tsdbMaxCnt = (!VND_IS_RSMA(pVnode) ? 1 : TSDB_RETENTION_MAX); + + ASSERT(sizeof(pInfo->subTyps) == sizeof(subTyps)); + memcpy(pInfo->subTyps, (char*)subTyps, sizeof(subTyps)); + + // fset partition list + memset(pInfo->pLists, 0, sizeof(pInfo->pLists[0]) * TSDB_RETENTION_MAX); + for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) { + STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, j); + pInfo->pLists[j] = tsdbSnapGetFSetPartList(pTsdb->pFS); + if (pInfo->pLists[j] == NULL) return -1; + } + return 0; +} + +static void tsdbPartitionInfoClear(STsdbPartitionInfo* pInfo) { + for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) { + if (pInfo->pLists[j] == NULL) continue; + tsdbFSetPartListDestroy(&pInfo->pLists[j]); + } +} + +static int32_t tsdbPartitionInfoEstSize(STsdbPartitionInfo* pInfo) { + int32_t dataLen = 0; + for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) { + dataLen += sizeof(SSyncTLV); // subTyps[j] + dataLen += tTsdbFSetPartListDataLenCalc(pInfo->pLists[j]); + } + return dataLen; +} + +static int32_t tsdbPartitionInfoSerialize(STsdbPartitionInfo* pInfo, uint8_t* buf, int32_t bufLen) { + int32_t tlen = 0; + int32_t offset = 0; + for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) { + SSyncTLV* pSubHead = (void*)((char*)buf + offset); + int32_t valOffset = offset + sizeof(*pSubHead); + ASSERT(pSubHead->val == (char*)buf + valOffset); + if ((tlen = tSerializeTsdbFSetPartList(pSubHead->val, bufLen - valOffset, pInfo->pLists[j])) < 0) { + tsdbError("vgId:%d, failed to serialize fset partition list of tsdb %d since %s", pInfo->vgId, j, terrstr()); + return -1; + } + pSubHead->typ = pInfo->subTyps[j]; + pSubHead->len = tlen; + offset += sizeof(*pSubHead) + tlen; + } + return offset; +} + +// tsdb replication opts +static int32_t tTsdbRepOptsDataLenCalc(STsdbRepOpts* pInfo) { + int32_t hdrLen = sizeof(int32_t); + int32_t datLen = 0; + + int8_t msgVer = 0; + int64_t reserved64 = 0; + int16_t format = 0; + hdrLen += sizeof(msgVer); + datLen += hdrLen; + datLen += sizeof(format); + datLen += sizeof(reserved64); + datLen += sizeof(*pInfo); + return datLen; +} + +int32_t tSerializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + int64_t reserved64 = 0; + int8_t msgVer = TSDB_SNAP_MSG_VER; + + if (tStartEncode(&encoder) < 0) goto _err; + if (tEncodeI8(&encoder, msgVer) < 0) goto _err; + int16_t format = pOpts->format; + if (tEncodeI16(&encoder, format) < 0) goto _err; + if (tEncodeI64(&encoder, reserved64) < 0) goto _err; + + tEndEncode(&encoder); + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; + +_err: + tEncoderClear(&encoder); + return -1; +} + +int32_t tDeserializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + int64_t reserved64 = 0; + int8_t msgVer = 0; + + if (tStartDecode(&decoder) < 0) goto _err; + if (tDecodeI8(&decoder, &msgVer) < 0) goto _err; + if (msgVer != TSDB_SNAP_MSG_VER) goto _err; + int16_t format = 0; + if (tDecodeI16(&decoder, &format) < 0) goto _err; + pOpts->format = format; + if (tDecodeI64(&decoder, &reserved64) < 0) goto _err; + + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; + +_err: + tDecoderClear(&decoder); + return -1; +} + +static int32_t tsdbRepOptsEstSize(STsdbRepOpts* pOpts) { + int32_t dataLen = 0; + dataLen += sizeof(SSyncTLV); + dataLen += tTsdbRepOptsDataLenCalc(pOpts); + return dataLen; +} + +static int32_t tsdbRepOptsSerialize(STsdbRepOpts* pOpts, void* buf, int32_t bufLen) { + SSyncTLV* pSubHead = buf; + int32_t offset = 0; + int32_t tlen = 0; + if ((tlen = tSerializeTsdbRepOpts(pSubHead->val, bufLen, pOpts)) < 0) { + return -1; + } + pSubHead->typ = SNAP_DATA_RAW; + pSubHead->len = tlen; + offset += sizeof(*pSubHead) + tlen; + return offset; +} + +// snap info +static int32_t tsdbSnapPrepDealWithSnapInfo(SVnode* pVnode, SSnapshot* pSnap, STsdbRepOpts* pInfo) { + if (!pSnap->data) return 0; + int32_t code = -1; + + SSyncTLV* pHead = (void*)pSnap->data; + int32_t offset = 0; + + while (offset + sizeof(*pHead) < pHead->len) { + SSyncTLV* pField = (void*)(pHead->val + offset); + offset += sizeof(*pField) + pField->len; + void* buf = pField->val; + int32_t bufLen = pField->len; + + switch (pField->typ) { + case SNAP_DATA_TSDB: + case SNAP_DATA_RSMA1: + case SNAP_DATA_RSMA2: { + } break; + case SNAP_DATA_RAW: { + if (tDeserializeTsdbRepOpts(buf, bufLen, pInfo) < 0) { + terrno = TSDB_CODE_INVALID_DATA_FMT; + tsdbError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + } break; + default: + tsdbError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), pField->typ); + goto _out; + } + } + + code = 0; +_out: + return code; +} + +int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { + ASSERT(pSnap->type == TDMT_SYNC_PREP_SNAPSHOT || pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY); + STsdbPartitionInfo partitionInfo = {0}; + int code = -1; + STsdbPartitionInfo* pInfo = &partitionInfo; + + if (tsdbPartitionInfoInit(pVnode, pInfo) != 0) { + goto _out; + } + + // deal with snap info for reply + STsdbRepOpts opts = {.format = TSDB_SNAP_REP_FMT_RAW}; + if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { + STsdbRepOpts leaderOpts = {0}; + if (tsdbSnapPrepDealWithSnapInfo(pVnode, pSnap, &leaderOpts) < 0) { + tsdbError("vgId:%d, failed to deal with snap info for reply since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + opts.format = TMIN(opts.format, leaderOpts.format); + } + + // info data realloc + const int32_t headLen = sizeof(SSyncTLV); + int32_t bufLen = headLen; + bufLen += tsdbPartitionInfoEstSize(pInfo); + bufLen += tsdbRepOptsEstSize(&opts); + if (syncSnapInfoDataRealloc(pSnap, bufLen) != 0) { + tsdbError("vgId:%d, failed to realloc memory for data of snap info. bytes:%d", TD_VID(pVnode), bufLen); + goto _out; + } + + // serialization + char* buf = (void*)pSnap->data; + int32_t offset = headLen; + int32_t tlen = 0; + + if ((tlen = tsdbPartitionInfoSerialize(pInfo, buf + offset, bufLen - offset)) < 0) { + tsdbError("vgId:%d, failed to serialize tsdb partition info since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + offset += tlen; + ASSERT(offset <= bufLen); + + if ((tlen = tsdbRepOptsSerialize(&opts, buf + offset, bufLen - offset)) < 0) { + tsdbError("vgId:%d, failed to serialize tsdb rep opts since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + offset += tlen; + ASSERT(offset <= bufLen); + + // set header of info data + SSyncTLV* pHead = pSnap->data; + pHead->typ = pSnap->type; + pHead->len = offset - headLen; + + tsdbInfo("vgId:%d, tsdb snap info prepared. type:%s, val length:%d", TD_VID(pVnode), TMSG_INFO(pHead->typ), + pHead->len); + code = 0; +_out: + tsdbPartitionInfoClear(pInfo); + return code; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index a9da0fbcec..6aff1c2930 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -32,12 +32,12 @@ struct STsdbSnapReader { uint8_t* aBuf[5]; SSkmInfo skmTb[1]; - TSnapRangeArray* fsrArr; + TFileSetRangeArray* fsrArr; // context struct { int32_t fsrArrIdx; - STSnapRange* fsr; + STFileSetRange* fsr; bool isDataDone; bool isTombDone; } ctx[1]; @@ -278,6 +278,15 @@ _exit: return code; } +static int64_t tBlockDataSize(SBlockData* pBlockData) { + int64_t nData = 0; + for (int32_t iCol = 0; iCol < pBlockData->nColData; iCol++) { + SColData* pColData = tBlockDataGetColDataByIdx(pBlockData, iCol); + nData += pColData->nData; + } + return nData; +} + static int32_t tsdbSnapReadTimeSeriesData(STsdbSnapReader* reader, uint8_t** data) { int32_t code = 0; int32_t lino = 0; @@ -320,8 +329,11 @@ static int32_t tsdbSnapReadTimeSeriesData(STsdbSnapReader* reader, uint8_t** dat code = tsdbIterMergerNext(reader->dataIterMerger); TSDB_CHECK_CODE(code, lino, _exit); - if (reader->blockData->nRow >= 81920) { - break; + if (!(reader->blockData->nRow % 16)) { + int64_t nData = tBlockDataSize(reader->blockData); + if (nData >= TSDB_SNAP_DATA_PAYLOAD_SIZE) { + break; + } } } @@ -425,14 +437,14 @@ int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type, reader[0]->ever = ever; reader[0]->type = type; - code = tsdbFSCreateRefRangedSnapshot(tsdb->pFS, sver, ever, (TSnapRangeArray*)pRanges, &reader[0]->fsrArr); + code = tsdbFSCreateRefRangedSnapshot(tsdb->pFS, sver, ever, (TFileSetRangeArray*)pRanges, &reader[0]->fsrArr); TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { tsdbError("vgId:%d %s failed at line %d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code), sver, ever, type); - tsdbSnapRangeArrayDestroy(&reader[0]->fsrArr); + tsdbTFileSetRangeArrayDestroy(&reader[0]->fsrArr); taosMemoryFree(reader[0]); reader[0] = NULL; } else { @@ -460,7 +472,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** reader) { TARRAY2_DESTROY(reader[0]->sttReaderArr, tsdbSttFileReaderClose); tsdbDataFileReaderClose(&reader[0]->dataReader); - tsdbSnapRangeArrayDestroy(&reader[0]->fsrArr); + tsdbFSDestroyRefRangedSnapshot(&reader[0]->fsrArr); tDestroyTSchema(reader[0]->skmTb->pTSchema); for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->aBuf); ++i) { @@ -1032,9 +1044,6 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, void* pRang int32_t code = 0; int32_t lino = 0; - // disable background tasks - tsdbFSDisableBgTask(pTsdb->pFS); - // start to write writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; @@ -1052,7 +1061,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, void* pRang writer[0]->compactVersion = INT64_MAX; writer[0]->now = taosGetTimestampMs(); - code = tsdbFSCreateCopyRangedSnapshot(pTsdb->pFS, (TSnapRangeArray*)pRanges, &writer[0]->fsetArr, writer[0]->fopArr); + code = tsdbFSCreateCopyRangedSnapshot(pTsdb->pFS, (TFileSetRangeArray*)pRanges, &writer[0]->fsetArr, writer[0]->fopArr); TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -1107,7 +1116,6 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** writer, int8_t rollback) { taosThreadMutexUnlock(&writer[0]->tsdb->mutex); } - tsdbFSEnableBgTask(tsdb->pFS); tsdbIterMergerClose(&writer[0]->ctx->tombIterMerger); tsdbIterMergerClose(&writer[0]->ctx->dataIterMerger); @@ -1117,7 +1125,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** writer, int8_t rollback) { tsdbDataFileReaderClose(&writer[0]->ctx->dataReader); TARRAY2_DESTROY(writer[0]->fopArr, NULL); - tsdbFSDestroyCopySnapshot(&writer[0]->fsetArr); + tsdbFSDestroyCopyRangedSnapshot(&writer[0]->fsetArr); for (int32_t i = 0; i < ARRAY_SIZE(writer[0]->aBuf); ++i) { tFree(writer[0]->aBuf[i]); @@ -1159,439 +1167,3 @@ _exit: } return code; } - -// snap part -static int32_t tsdbSnapPartCmprFn(STsdbSnapPartition* x, STsdbSnapPartition* y) { - if (x->fid < y->fid) return -1; - if (x->fid > y->fid) return 1; - return 0; -} - -static int32_t tVersionRangeCmprFn(SVersionRange* x, SVersionRange* y) { - if (x->minVer < y->minVer) return -1; - if (x->minVer > y->minVer) return 1; - if (x->maxVer < y->maxVer) return -1; - if (x->maxVer > y->maxVer) return 1; - return 0; -} - -static int32_t tsdbSnapRangeCmprFn(STSnapRange* x, STSnapRange* y) { - if (x->fid < y->fid) return -1; - if (x->fid > y->fid) return 1; - return 0; -} - -STsdbSnapPartition* tsdbSnapPartitionCreate() { - STsdbSnapPartition* pSP = taosMemoryCalloc(1, sizeof(STsdbSnapPartition)); - if (pSP == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) { - TARRAY2_INIT(&pSP->verRanges[i]); - } - return pSP; -} - -void tsdbSnapPartitionClear(STsdbSnapPartition** ppSP) { - if (ppSP == NULL || ppSP[0] == NULL) { - return; - } - for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) { - TARRAY2_DESTROY(&ppSP[0]->verRanges[i], NULL); - } - taosMemoryFree(ppSP[0]); - ppSP[0] = NULL; -} - -static int32_t tsdbFTypeToSRangeTyp(tsdb_ftype_t ftype) { - switch (ftype) { - case TSDB_FTYPE_HEAD: - return TSDB_SNAP_RANGE_TYP_HEAD; - case TSDB_FTYPE_DATA: - return TSDB_SNAP_RANGE_TYP_DATA; - case TSDB_FTYPE_SMA: - return TSDB_SNAP_RANGE_TYP_SMA; - case TSDB_FTYPE_TOMB: - return TSDB_SNAP_RANGE_TYP_TOMB; - case TSDB_FTYPE_STT: - return TSDB_SNAP_RANGE_TYP_STT; - } - return TSDB_SNAP_RANGE_TYP_MAX; -} - -static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbSnapPartition** ppSP) { - STsdbSnapPartition* p = tsdbSnapPartitionCreate(); - if (p == NULL) { - goto _err; - } - - p->fid = fset->fid; - - int32_t code = 0; - int32_t typ = 0; - int32_t corrupt = false; - int32_t count = 0; - for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { - if (fset->farr[ftype] == NULL) continue; - typ = tsdbFTypeToSRangeTyp(ftype); - ASSERT(typ < TSDB_SNAP_RANGE_TYP_MAX); - STFile* f = fset->farr[ftype]->f; - if (f->maxVer > fset->maxVerValid) { - corrupt = true; - tsdbError("skip incomplete data file: fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 - ", ftype: %d", - fset->fid, fset->maxVerValid, f->minVer, f->maxVer, ftype); - continue; - } - count++; - SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer}; - code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); - ASSERT(code == 0); - } - - typ = TSDB_SNAP_RANGE_TYP_STT; - const SSttLvl* lvl; - TARRAY2_FOREACH(fset->lvlArr, lvl) { - STFileObj* fobj; - TARRAY2_FOREACH(lvl->fobjArr, fobj) { - STFile* f = fobj->f; - if (f->maxVer > fset->maxVerValid) { - corrupt = true; - tsdbError("skip incomplete stt file.fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 - ", ftype: %d", - fset->fid, fset->maxVerValid, f->minVer, f->maxVer, typ); - continue; - } - count++; - SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer}; - code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); - ASSERT(code == 0); - } - } - if (corrupt && count == 0) { - SVersionRange vr = {.minVer = VERSION_MIN, .maxVer = fset->maxVerValid}; - code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); - ASSERT(code == 0); - } - ppSP[0] = p; - return 0; - -_err: - tsdbSnapPartitionClear(&p); - return -1; -} - -STsdbSnapPartList* tsdbSnapPartListCreate() { - STsdbSnapPartList* pList = taosMemoryCalloc(1, sizeof(STsdbSnapPartList)); - if (pList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - TARRAY2_INIT(pList); - return pList; -} - -static STsdbSnapPartList* tsdbGetSnapPartList(STFileSystem* fs) { - STsdbSnapPartList* pList = tsdbSnapPartListCreate(); - if (pList == NULL) { - return NULL; - } - - int32_t code = 0; - taosThreadMutexLock(&fs->tsdb->mutex); - STFileSet* fset; - TARRAY2_FOREACH(fs->fSetArr, fset) { - STsdbSnapPartition* pItem = NULL; - if (tsdbTFileSetToSnapPart(fset, &pItem) < 0) { - code = -1; - break; - } - ASSERT(pItem != NULL); - code = TARRAY2_SORT_INSERT(pList, pItem, tsdbSnapPartCmprFn); - ASSERT(code == 0); - } - taosThreadMutexUnlock(&fs->tsdb->mutex); - - if (code) { - TARRAY2_DESTROY(pList, tsdbSnapPartitionClear); - taosMemoryFree(pList); - pList = NULL; - } - return pList; -} - -int32_t tTsdbSnapPartListDataLenCalc(STsdbSnapPartList* pList) { - int32_t hdrLen = sizeof(int32_t); - int32_t datLen = 0; - - int8_t msgVer = 1; - int32_t len = TARRAY2_SIZE(pList); - hdrLen += sizeof(msgVer); - hdrLen += sizeof(len); - datLen += hdrLen; - - for (int32_t u = 0; u < len; u++) { - STsdbSnapPartition* p = TARRAY2_GET(pList, u); - int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; - int32_t uItem = 0; - uItem += sizeof(STsdbSnapPartition); - uItem += sizeof(typMax); - - for (int32_t i = 0; i < typMax; i++) { - int32_t iLen = TARRAY2_SIZE(&p->verRanges[i]); - int32_t jItem = 0; - jItem += sizeof(SVersionRange); - jItem += sizeof(int64_t); - uItem += sizeof(iLen) + jItem * iLen; - } - datLen += uItem; - } - return datLen; -} - -int32_t tSerializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); - - int8_t reserved8 = 0; - int16_t reserved16 = 0; - int64_t reserved64 = 0; - - int8_t msgVer = 1; - int32_t len = TARRAY2_SIZE(pList); - - if (tStartEncode(&encoder) < 0) goto _err; - if (tEncodeI8(&encoder, msgVer) < 0) goto _err; - if (tEncodeI32(&encoder, len) < 0) goto _err; - - for (int32_t u = 0; u < len; u++) { - STsdbSnapPartition* p = TARRAY2_GET(pList, u); - if (tEncodeI64(&encoder, p->fid) < 0) goto _err; - if (tEncodeI8(&encoder, p->stat) < 0) goto _err; - if (tEncodeI8(&encoder, reserved8) < 0) goto _err; - if (tEncodeI16(&encoder, reserved16) < 0) goto _err; - - int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; - if (tEncodeI32(&encoder, typMax) < 0) goto _err; - - for (int32_t i = 0; i < typMax; i++) { - SVerRangeList* iList = &p->verRanges[i]; - int32_t iLen = TARRAY2_SIZE(iList); - - if (tEncodeI32(&encoder, iLen) < 0) goto _err; - for (int32_t j = 0; j < iLen; j++) { - SVersionRange r = TARRAY2_GET(iList, j); - if (tEncodeI64(&encoder, r.minVer) < 0) goto _err; - if (tEncodeI64(&encoder, r.maxVer) < 0) goto _err; - if (tEncodeI64(&encoder, reserved64) < 0) goto _err; - } - } - } - - tEndEncode(&encoder); - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; - -_err: - tEncoderClear(&encoder); - return -1; -} - -int32_t tDeserializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - int8_t reserved8 = 0; - int16_t reserved16 = 0; - int64_t reserved64 = 0; - - STsdbSnapPartition* p = NULL; - - int8_t msgVer = 0; - int32_t len = 0; - if (tStartDecode(&decoder) < 0) goto _err; - if (tDecodeI8(&decoder, &msgVer) < 0) goto _err; - if (tDecodeI32(&decoder, &len) < 0) goto _err; - - for (int32_t u = 0; u < len; u++) { - p = tsdbSnapPartitionCreate(); - if (p == NULL) goto _err; - if (tDecodeI64(&decoder, &p->fid) < 0) goto _err; - if (tDecodeI8(&decoder, &p->stat) < 0) goto _err; - if (tDecodeI8(&decoder, &reserved8) < 0) goto _err; - if (tDecodeI16(&decoder, &reserved16) < 0) goto _err; - - int32_t typMax = 0; - if (tDecodeI32(&decoder, &typMax) < 0) goto _err; - - for (int32_t i = 0; i < typMax; i++) { - SVerRangeList* iList = &p->verRanges[i]; - int32_t iLen = 0; - if (tDecodeI32(&decoder, &iLen) < 0) goto _err; - for (int32_t j = 0; j < iLen; j++) { - SVersionRange r = {0}; - if (tDecodeI64(&decoder, &r.minVer) < 0) goto _err; - if (tDecodeI64(&decoder, &r.maxVer) < 0) goto _err; - if (tDecodeI64(&decoder, &reserved64) < 0) goto _err; - TARRAY2_APPEND(iList, r); - } - } - TARRAY2_APPEND(pList, p); - p = NULL; - } - - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; - -_err: - if (p) { - tsdbSnapPartitionClear(&p); - } - tDecoderClear(&decoder); - return -1; -} - -int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TSnapRangeArray** ppRanges) { - TSnapRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TSnapRangeArray)); - if (pDiff == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - TARRAY2_INIT(pDiff); - - STsdbSnapPartition* part; - TARRAY2_FOREACH(pList, part) { - STSnapRange* r = taosMemoryCalloc(1, sizeof(STSnapRange)); - if (r == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - int64_t maxVerValid = -1; - int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; - for (int32_t i = 0; i < typMax; i++) { - SVerRangeList* iList = &part->verRanges[i]; - SVersionRange vr = {0}; - TARRAY2_FOREACH(iList, vr) { - if (vr.maxVer < vr.minVer) { - continue; - } - maxVerValid = TMAX(maxVerValid, vr.maxVer); - } - } - r->fid = part->fid; - r->sver = maxVerValid + 1; - r->ever = VERSION_MAX; - tsdbDebug("range diff fid:%" PRId64 ", sver:%" PRId64 ", ever:%" PRId64, part->fid, r->sver, r->ever); - int32_t code = TARRAY2_SORT_INSERT(pDiff, r, tsdbSnapRangeCmprFn); - ASSERT(code == 0); - } - ppRanges[0] = pDiff; - - tsdbInfo("pDiff size:%d", TARRAY2_SIZE(pDiff)); - return 0; - -_err: - if (pDiff) { - tsdbSnapRangeArrayDestroy(&pDiff); - } - return -1; -} - -void tsdbSnapRangeArrayDestroy(TSnapRangeArray** ppSnap) { - if (ppSnap && ppSnap[0]) { - TARRAY2_DESTROY(ppSnap[0], tsdbTSnapRangeClear); - taosMemoryFree(ppSnap[0]); - ppSnap[0] = NULL; - } -} - -void tsdbSnapPartListDestroy(STsdbSnapPartList** ppList) { - if (ppList == NULL || ppList[0] == NULL) return; - - TARRAY2_DESTROY(ppList[0], tsdbSnapPartitionClear); - taosMemoryFree(ppList[0]); - ppList[0] = NULL; -} - -ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) { - if (!VND_IS_RSMA(pVnode)) { - return pVnode->pTsdb->pFS->fsstate; - } - for (int32_t lvl = 0; lvl < TSDB_RETENTION_MAX; ++lvl) { - STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, lvl); - if (pTsdb && pTsdb->pFS->fsstate != TSDB_FS_STATE_NORMAL) { - return TSDB_FS_STATE_INCOMPLETE; - } - } - return TSDB_FS_STATE_NORMAL; -} - -int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { - int code = -1; - int32_t tsdbMaxCnt = (!VND_IS_RSMA(pVnode) ? 1 : TSDB_RETENTION_MAX); - int32_t subTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2}; - STsdbSnapPartList* pLists[TSDB_RETENTION_MAX] = {0}; - - for (int32_t j = 0; j < tsdbMaxCnt; ++j) { - STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, j); - pLists[j] = tsdbGetSnapPartList(pTsdb->pFS); - if (pLists[j] == NULL) goto _out; - } - - // estimate bufLen and prepare - int32_t bufLen = sizeof(SSyncTLV); // typ: TDMT_SYNC_PREP_SNAPSHOT or TDMT_SYNC_PREP_SNAPSOT_REPLY - for (int32_t j = 0; j < tsdbMaxCnt; ++j) { - bufLen += sizeof(SSyncTLV); // subTyps[j] - bufLen += tTsdbSnapPartListDataLenCalc(pLists[j]); - } - - tsdbInfo("vgId:%d, allocate %d bytes for data of snapshot info.", TD_VID(pVnode), bufLen); - - void* data = taosMemoryRealloc(pSnap->data, bufLen); - if (data == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tsdbError("vgId:%d, failed to realloc memory for data of snapshot info. bytes:%d", TD_VID(pVnode), bufLen); - goto _out; - } - pSnap->data = data; - - // header - SSyncTLV* head = data; - head->len = 0; - head->typ = pSnap->type; - int32_t offset = sizeof(SSyncTLV); - int32_t tlen = 0; - - // fill snapshot info - for (int32_t j = 0; j < tsdbMaxCnt; ++j) { - if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { - } - - // subHead - SSyncTLV* subHead = (void*)((char*)data + offset); - subHead->typ = subTyps[j]; - ASSERT(subHead->val == (char*)data + offset + sizeof(SSyncTLV)); - - if ((tlen = tSerializeTsdbSnapPartList(subHead->val, bufLen - offset - sizeof(SSyncTLV), pLists[j])) < 0) { - tsdbError("vgId:%d, failed to serialize snap partition list of tsdb %d since %s", TD_VID(pVnode), j, terrstr()); - goto _out; - } - subHead->len = tlen; - offset += sizeof(SSyncTLV) + tlen; - } - - head->len = offset - sizeof(SSyncTLV); - ASSERT(offset <= bufLen); - code = 0; - -_out: - for (int32_t j = 0; j < tsdbMaxCnt; ++j) { - if (pLists[j] == NULL) continue; - tsdbSnapPartListDestroy(&pLists[j]); - } - - return code; -} diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c new file mode 100644 index 0000000000..b7c22aa0e9 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -0,0 +1,599 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" +#include "tsdbDataFileRAW.h" +#include "tsdbFS2.h" +#include "tsdbFSetRAW.h" + +static int32_t tsdbSnapRAWReadFileSetCloseReader(STsdbSnapRAWReader* reader); + +// reader +typedef struct SDataFileRAWReaderIter { + int32_t count; + int32_t idx; +} SDataFileRAWReaderIter; + +typedef struct STsdbSnapRAWReader { + STsdb* tsdb; + int64_t ever; + int8_t type; + + TFileSetArray* fsetArr; + + // context + struct { + int32_t fsetArrIdx; + STFileSet* fset; + bool isDataDone; + } ctx[1]; + + // reader + SDataFileRAWReaderArray dataReaderArr[1]; + + // iter + SDataFileRAWReaderIter dataIter[1]; +} STsdbSnapRAWReader; + +int32_t tsdbSnapRAWReaderOpen(STsdb* tsdb, int64_t ever, int8_t type, STsdbSnapRAWReader** reader) { + int32_t code = 0; + int32_t lino = 0; + + reader[0] = taosMemoryCalloc(1, sizeof(STsdbSnapRAWReader)); + if (reader[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + reader[0]->tsdb = tsdb; + reader[0]->ever = ever; + reader[0]->type = type; + + code = tsdbFSCreateRefSnapshot(tsdb->pFS, &reader[0]->fsetArr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s, sver:0, ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), __func__, + lino, tstrerror(code), ever, type); + tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr); + taosMemoryFree(reader[0]); + reader[0] = NULL; + } else { + tsdbInfo("vgId:%d tsdb snapshot reader opened. sver:0, ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), ever, type); + } + return code; +} + +int32_t tsdbSnapRAWReaderClose(STsdbSnapRAWReader** reader) { + if (reader[0] == NULL) return 0; + + int32_t code = 0; + int32_t lino = 0; + + STsdb* tsdb = reader[0]->tsdb; + + TARRAY2_DESTROY(reader[0]->dataReaderArr, tsdbDataFileRAWReaderClose); + tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr); + taosMemoryFree(reader[0]); + reader[0] = NULL; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); + } + return code; +} + +static int32_t tsdbSnapRAWReadFileSetOpenReader(STsdbSnapRAWReader* reader) { + int32_t code = 0; + int32_t lino = 0; + + // data + for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ftype++) { + if (reader->ctx->fset->farr[ftype] == NULL) { + continue; + } + STFileObj* fobj = reader->ctx->fset->farr[ftype]; + SDataFileRAWReader* dataReader; + SDataFileRAWReaderConfig config = { + .tsdb = reader->tsdb, + .szPage = reader->tsdb->pVnode->config.tsdbPageSize, + .file = fobj->f[0], + }; + code = tsdbDataFileRAWReaderOpen(NULL, &config, &dataReader); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(reader->dataReaderArr, dataReader); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // stt + SSttLvl* lvl; + TARRAY2_FOREACH(reader->ctx->fset->lvlArr, lvl) { + STFileObj* fobj; + TARRAY2_FOREACH(lvl->fobjArr, fobj) { + SDataFileRAWReader* dataReader; + SDataFileRAWReaderConfig config = { + .tsdb = reader->tsdb, + .szPage = reader->tsdb->pVnode->config.tsdbPageSize, + .file = fobj->f[0], + }; + code = tsdbDataFileRAWReaderOpen(NULL, &config, &dataReader); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(reader->dataReaderArr, dataReader); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + return code; +} + +static int32_t tsdbSnapRAWReadFileSetCloseReader(STsdbSnapRAWReader* reader) { + int32_t code = 0; + int32_t lino = 0; + + TARRAY2_CLEAR(reader->dataReaderArr, tsdbDataFileRAWReaderClose); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + return code; +} + +static int32_t tsdbSnapRAWReadFileSetOpenIter(STsdbSnapRAWReader* reader) { + int32_t code = 0; + int32_t lino = 0; + + reader->dataIter->count = TARRAY2_SIZE(reader->dataReaderArr); + reader->dataIter->idx = 0; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + return code; +} + +static int32_t tsdbSnapRAWReadFileSetCloseIter(STsdbSnapRAWReader* reader) { + reader->dataIter->count = 0; + reader->dataIter->idx = 0; + return 0; +} + +static int64_t tsdbSnapRAWReadPeek(SDataFileRAWReader* reader) { + int64_t size = TMIN(reader->config->file.size - reader->ctx->offset, TSDB_SNAP_DATA_PAYLOAD_SIZE); + return size; +} + +static SDataFileRAWReader* tsdbSnapRAWReaderIterNext(STsdbSnapRAWReader* reader) { + ASSERT(reader->dataIter->idx <= reader->dataIter->count); + + while (reader->dataIter->idx < reader->dataIter->count) { + SDataFileRAWReader* dataReader = TARRAY2_GET(reader->dataReaderArr, reader->dataIter->idx); + ASSERT(dataReader); + if (dataReader->ctx->offset < dataReader->config->file.size) { + return dataReader; + } + reader->dataIter->idx++; + } + return NULL; +} + +static int32_t tsdbSnapRAWReadNext(STsdbSnapRAWReader* reader, SSnapDataHdr** ppData) { + int32_t code = 0; + int32_t lino = 0; + int8_t type = reader->type; + ppData[0] = NULL; + + SDataFileRAWReader* dataReader = tsdbSnapRAWReaderIterNext(reader); + if (dataReader == NULL) { + return 0; + } + + // prepare + int64_t dataLength = tsdbSnapRAWReadPeek(dataReader); + ASSERT(dataLength > 0); + + void* pBuf = taosMemoryCalloc(1, sizeof(SSnapDataHdr) + sizeof(STsdbDataRAWBlockHeader) + dataLength); + if (pBuf == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + SSnapDataHdr* pHdr = pBuf; + pHdr->type = type; + pHdr->size = sizeof(STsdbDataRAWBlockHeader) + dataLength; + + // read + STsdbDataRAWBlockHeader* pBlock = (void*)pHdr->data; + pBlock->offset = dataReader->ctx->offset; + pBlock->dataLength = dataLength; + + code = tsdbDataFileRAWReadBlockData(dataReader, pBlock); + TSDB_CHECK_CODE(code, lino, _exit); + + // finish + dataReader->ctx->offset += pBlock->dataLength; + ASSERT(dataReader->ctx->offset <= dataReader->config->file.size); + ppData[0] = pBuf; + +_exit: + if (code) { + taosMemoryFree(pBuf); + pBuf = NULL; + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + return code; +} + +static int32_t tsdbSnapRAWReadData(STsdbSnapRAWReader* reader, uint8_t** ppData) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbSnapRAWReadNext(reader, (SSnapDataHdr**)ppData); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + return code; +} + +static int32_t tsdbSnapRAWReadBegin(STsdbSnapRAWReader* reader) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(reader->ctx->fset == NULL); + + if (reader->ctx->fsetArrIdx < TARRAY2_SIZE(reader->fsetArr)) { + reader->ctx->fset = TARRAY2_GET(reader->fsetArr, reader->ctx->fsetArrIdx++); + reader->ctx->isDataDone = false; + + code = tsdbSnapRAWReadFileSetOpenReader(reader); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbSnapRAWReadFileSetOpenIter(reader); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + return code; +} + +static int32_t tsdbSnapRAWReadEnd(STsdbSnapRAWReader* reader) { + tsdbSnapRAWReadFileSetCloseIter(reader); + tsdbSnapRAWReadFileSetCloseReader(reader); + reader->ctx->fset = NULL; + return 0; +} + +int32_t tsdbSnapRAWRead(STsdbSnapRAWReader* reader, uint8_t** data) { + int32_t code = 0; + int32_t lino = 0; + + data[0] = NULL; + + for (;;) { + if (reader->ctx->fset == NULL) { + code = tsdbSnapRAWReadBegin(reader); + TSDB_CHECK_CODE(code, lino, _exit); + + if (reader->ctx->fset == NULL) { + break; + } + } + + if (!reader->ctx->isDataDone) { + code = tsdbSnapRAWReadData(reader, data); + TSDB_CHECK_CODE(code, lino, _exit); + if (data[0]) { + goto _exit; + } else { + reader->ctx->isDataDone = true; + } + } + + code = tsdbSnapRAWReadEnd(reader); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } else { + tsdbDebug("vgId:%d %s done", TD_VID(reader->tsdb->pVnode), __func__); + } + return code; +} + +// writer +struct STsdbSnapRAWWriter { + STsdb* tsdb; + int64_t sver; + int64_t ever; + int32_t minutes; + int8_t precision; + int32_t minRow; + int32_t maxRow; + int8_t cmprAlg; + int64_t commitID; + int32_t szPage; + int64_t compactVersion; + int64_t now; + + TFileSetArray* fsetArr; + TFileOpArray fopArr[1]; + + struct { + bool fsetWriteBegin; + int32_t fid; + STFileSet* fset; + SDiskID did; + int64_t cid; + int64_t level; + + // writer + SFSetRAWWriter* fsetWriter; + } ctx[1]; +}; + +int32_t tsdbSnapRAWWriterOpen(STsdb* pTsdb, int64_t ever, STsdbSnapRAWWriter** writer) { + int32_t code = 0; + int32_t lino = 0; + + // start to write + writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); + if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + writer[0]->tsdb = pTsdb; + writer[0]->ever = ever; + writer[0]->minutes = pTsdb->keepCfg.days; + writer[0]->precision = pTsdb->keepCfg.precision; + writer[0]->minRow = pTsdb->pVnode->config.tsdbCfg.minRows; + writer[0]->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; + writer[0]->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; + writer[0]->commitID = tsdbFSAllocEid(pTsdb->pFS); + writer[0]->szPage = pTsdb->pVnode->config.tsdbPageSize; + writer[0]->compactVersion = INT64_MAX; + writer[0]->now = taosGetTimestampMs(); + + code = tsdbFSCreateCopySnapshot(pTsdb->pFS, &writer[0]->fsetArr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbInfo("vgId:%d %s done, sver:0, ever:%" PRId64, TD_VID(pTsdb->pVnode), __func__, ever); + } + return code; +} + +static int32_t tsdbSnapRAWWriteFileSetOpenIter(STsdbSnapRAWWriter* writer) { + int32_t code = 0; + int32_t lino = 0; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbSnapRAWWriteFileSetCloseIter(STsdbSnapRAWWriter* writer) { return 0; } + +static int32_t tsdbSnapRAWWriteFileSetOpenWriter(STsdbSnapRAWWriter* writer) { + int32_t code = 0; + int32_t lino = 0; + + SFSetRAWWriterConfig config = { + .tsdb = writer->tsdb, + .szPage = writer->szPage, + .fid = writer->ctx->fid, + .cid = writer->commitID, + .did = writer->ctx->did, + .level = writer->ctx->level, + }; + + code = tsdbFSetRAWWriterOpen(&config, &writer->ctx->fsetWriter); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbSnapRAWWriteFileSetCloseWriter(STsdbSnapRAWWriter* writer) { + return tsdbFSetRAWWriterClose(&writer->ctx->fsetWriter, 0, writer->fopArr); +} + +static int32_t tsdbSnapRAWWriteFileSetBegin(STsdbSnapRAWWriter* writer, int32_t fid) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(writer->ctx->fsetWriteBegin == false); + + STFileSet* fset = &(STFileSet){.fid = fid}; + + writer->ctx->fid = fid; + STFileSet** fsetPtr = TARRAY2_SEARCH(writer->fsetArr, &fset, tsdbTFileSetCmprFn, TD_EQ); + writer->ctx->fset = (fsetPtr == NULL) ? NULL : *fsetPtr; + + int32_t level = tsdbFidLevel(fid, &writer->tsdb->keepCfg, taosGetTimestampSec()); + if (tfsAllocDisk(writer->tsdb->pVnode->pTfs, level, &writer->ctx->did)) { + code = TSDB_CODE_NO_AVAIL_DISK; + TSDB_CHECK_CODE(code, lino, _exit); + } + tfsMkdirRecurAt(writer->tsdb->pVnode->pTfs, writer->tsdb->path, writer->ctx->did); + + code = tsdbSnapRAWWriteFileSetOpenWriter(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->ctx->level = level; + writer->ctx->fsetWriteBegin = true; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbSnapRAWWriteFileSetEnd(STsdbSnapRAWWriter* writer) { + if (!writer->ctx->fsetWriteBegin) return 0; + + int32_t code = 0; + int32_t lino = 0; + + // close write + code = tsdbSnapRAWWriteFileSetCloseWriter(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->ctx->fsetWriteBegin = false; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbSnapRAWWriterPrepareClose(STsdbSnapRAWWriter* writer) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbSnapRAWWriteFileSetEnd(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFSEditBegin(writer->tsdb->pFS, writer->fopArr, TSDB_FEDIT_COMMIT); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d %s done", TD_VID(writer->tsdb->pVnode), __func__); + } + return code; +} + +int32_t tsdbSnapRAWWriterClose(STsdbSnapRAWWriter** writer, int8_t rollback) { + if (writer[0] == NULL) return 0; + + int32_t code = 0; + int32_t lino = 0; + + STsdb* tsdb = writer[0]->tsdb; + + if (rollback) { + code = tsdbFSEditAbort(writer[0]->tsdb->pFS); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + taosThreadMutexLock(&writer[0]->tsdb->mutex); + + code = tsdbFSEditCommit(writer[0]->tsdb->pFS); + if (code) { + taosThreadMutexUnlock(&writer[0]->tsdb->mutex); + TSDB_CHECK_CODE(code, lino, _exit); + } + + writer[0]->tsdb->pFS->fsstate = TSDB_FS_STATE_NORMAL; + + taosThreadMutexUnlock(&writer[0]->tsdb->mutex); + } + + TARRAY2_DESTROY(writer[0]->fopArr, NULL); + tsdbFSDestroyCopySnapshot(&writer[0]->fsetArr); + + taosMemoryFree(writer[0]); + writer[0] = NULL; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } else { + tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); + } + return code; +} + +static int32_t tsdbSnapRAWWriteTimeSeriesData(STsdbSnapRAWWriter* writer, STsdbDataRAWBlockHeader* bHdr) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFSetRAWWriteBlockData(writer->ctx->fsetWriter, bHdr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbSnapRAWWriteData(STsdbSnapRAWWriter* writer, SSnapDataHdr* hdr) { + int32_t code = 0; + int32_t lino = 0; + + STsdbDataRAWBlockHeader* bHdr = (void*)hdr->data; + int32_t fid = bHdr->file.fid; + if (!writer->ctx->fsetWriteBegin || fid != writer->ctx->fid) { + code = tsdbSnapRAWWriteFileSetEnd(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbSnapRAWWriteFileSetBegin(writer, fid); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbSnapRAWWriteTimeSeriesData(writer, bHdr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbSnapRAWWrite(STsdbSnapRAWWriter* writer, SSnapDataHdr* hdr) { + ASSERT(hdr->type == SNAP_DATA_RAW); + + int32_t code = 0; + int32_t lino = 0; + + code = tsdbSnapRAWWriteData(writer, hdr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s, type:%d index:%" PRId64 " size:%" PRId64, + TD_VID(writer->tsdb->pVnode), __func__, lino, tstrerror(code), hdr->type, hdr->index, hdr->size); + } else { + tsdbDebug("vgId:%d %s done, type:%d index:%" PRId64 " size:%" PRId64, TD_VID(writer->tsdb->pVnode), __func__, + hdr->type, hdr->index, hdr->size); + } + return code; +} diff --git a/source/dnode/vnode/src/vnd/vnodeAsync.c b/source/dnode/vnode/src/vnd/vnodeAsync.c new file mode 100644 index 0000000000..c95d2324aa --- /dev/null +++ b/source/dnode/vnode/src/vnd/vnodeAsync.c @@ -0,0 +1,719 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "vnd.h" +#include "vnodeHash.h" + +typedef struct SVATask SVATask; +typedef struct SVAChannel SVAChannel; + +#define VNODE_ASYNC_DEFAULT_WORKERS 4 +#define VNODE_ASYNC_MAX_WORKERS 256 + +// priority + +#define EVA_PRIORITY_MAX (EVA_PRIORITY_LOW + 1) + +// worker +typedef enum { + EVA_WORKER_STATE_UINIT = 0, + EVA_WORKER_STATE_ACTIVE, + EVA_WORKER_STATE_IDLE, + EVA_WORKER_STATE_STOP, +} EVWorkerState; + +typedef struct { + SVAsync *async; + int32_t workerId; + EVWorkerState state; + TdThread thread; + SVATask *runningTask; +} SVWorker; + +// task +typedef enum { + EVA_TASK_STATE_WAITTING = 0, + EVA_TASK_STATE_RUNNING, +} EVATaskState; + +struct SVATask { + int64_t taskId; + EVAPriority priority; + int32_t priorScore; + SVAChannel *channel; + int32_t (*execute)(void *); + void (*complete)(void *); + void *arg; + EVATaskState state; + + // wait + int32_t numWait; + TdThreadCond waitCond; + + // queue + struct SVATask *prev; + struct SVATask *next; +}; + +#define VATASK_PIORITY(task_) ((task_)->priority - ((task_)->priorScore / 4)) + +// async channel +typedef enum { + EVA_CHANNEL_STATE_OPEN = 0, + EVA_CHANNEL_STATE_CLOSE, +} EVAChannelState; + +struct SVAChannel { + int64_t channelId; + EVAChannelState state; + SVATask queue[EVA_PRIORITY_MAX]; + SVATask *scheduled; + + SVAChannel *prev; + SVAChannel *next; +}; + +// async handle +struct SVAsync { + const char *label; + + TdThreadMutex mutex; + TdThreadCond hasTask; + bool stop; + + // worker + int32_t numWorkers; + int32_t numLaunchWorkers; + int32_t numIdleWorkers; + SVWorker workers[VNODE_ASYNC_MAX_WORKERS]; + + // channel + int64_t nextChannelId; + int32_t numChannels; + SVAChannel chList; + SVHashTable *channelTable; + + // task + int64_t nextTaskId; + int32_t numTasks; + SVATask queue[EVA_PRIORITY_MAX]; + SVHashTable *taskTable; +}; + +static int32_t vnodeAsyncTaskDone(SVAsync *async, SVATask *task) { + int32_t ret; + + if (task->channel != NULL && task->channel->scheduled == task) { + task->channel->scheduled = NULL; + if (task->channel->state == EVA_CHANNEL_STATE_CLOSE) { + taosMemoryFree(task->channel); + } else { + for (int32_t i = 0; i < EVA_PRIORITY_MAX; i++) { + SVATask *nextTask = task->channel->queue[i].next; + if (nextTask != &task->channel->queue[i]) { + if (task->channel->scheduled == NULL) { + task->channel->scheduled = nextTask; + nextTask->next->prev = nextTask->prev; + nextTask->prev->next = nextTask->next; + } else { + nextTask->priorScore++; + int32_t newPriority = VATASK_PIORITY(nextTask); + if (newPriority != i) { + // remove from current priority queue + nextTask->prev->next = nextTask->next; + nextTask->next->prev = nextTask->prev; + // add to new priority queue + nextTask->next = &task->channel->queue[newPriority]; + nextTask->prev = task->channel->queue[newPriority].prev; + nextTask->next->prev = nextTask; + nextTask->prev->next = nextTask; + } + } + } + } + + if (task->channel->scheduled != NULL) { + int32_t priority = VATASK_PIORITY(task->channel->scheduled); + task->channel->scheduled->next = &async->queue[priority]; + task->channel->scheduled->prev = async->queue[priority].prev; + task->channel->scheduled->next->prev = task->channel->scheduled; + task->channel->scheduled->prev->next = task->channel->scheduled; + } + } + } + + ret = vHashDrop(async->taskTable, task); + if (ret != 0) { + ASSERT(0); + } + async->numTasks--; + + // call complete callback + if (task->complete) { + task->complete(task->arg); + } + + if (task->numWait == 0) { + taosThreadCondDestroy(&task->waitCond); + taosMemoryFree(task); + } else if (task->numWait == 1) { + taosThreadCondSignal(&task->waitCond); + } else { + taosThreadCondBroadcast(&task->waitCond); + } + return 0; +} + +static int32_t vnodeAsyncCancelAllTasks(SVAsync *async) { + for (int32_t i = 0; i < EVA_PRIORITY_MAX; i++) { + while (async->queue[i].next != &async->queue[i]) { + SVATask *task = async->queue[i].next; + task->prev->next = task->next; + task->next->prev = task->prev; + vnodeAsyncTaskDone(async, task); + } + } + return 0; +} + +static void *vnodeAsyncLoop(void *arg) { + SVWorker *worker = (SVWorker *)arg; + SVAsync *async = worker->async; + + setThreadName(async->label); + + for (;;) { + taosThreadMutexLock(&async->mutex); + + // finish last running task + if (worker->runningTask != NULL) { + vnodeAsyncTaskDone(async, worker->runningTask); + worker->runningTask = NULL; + } + + for (;;) { + if (async->stop || worker->workerId >= async->numWorkers) { + if (async->stop) { // cancel all tasks + vnodeAsyncCancelAllTasks(async); + } + worker->state = EVA_WORKER_STATE_STOP; + async->numLaunchWorkers--; + taosThreadMutexUnlock(&async->mutex); + return NULL; + } + + for (int32_t i = 0; i < EVA_PRIORITY_MAX; i++) { + SVATask *task = async->queue[i].next; + if (task != &async->queue[i]) { + if (worker->runningTask == NULL) { + worker->runningTask = task; + task->prev->next = task->next; + task->next->prev = task->prev; + } else { // promote priority + task->priorScore++; + int32_t priority = VATASK_PIORITY(task); + if (priority != i) { + // remove from current priority queue + task->prev->next = task->next; + task->next->prev = task->prev; + // add to new priority queue + task->next = &async->queue[priority]; + task->prev = async->queue[priority].prev; + task->next->prev = task; + task->prev->next = task; + } + } + } + } + + if (worker->runningTask == NULL) { + worker->state = EVA_WORKER_STATE_IDLE; + async->numIdleWorkers++; + taosThreadCondWait(&async->hasTask, &async->mutex); + async->numIdleWorkers--; + worker->state = EVA_WORKER_STATE_ACTIVE; + } else { + worker->runningTask->state = EVA_TASK_STATE_RUNNING; + break; + } + } + + taosThreadMutexUnlock(&async->mutex); + + // do run the task + worker->runningTask->execute(worker->runningTask->arg); + } + + return NULL; +} + +static uint32_t vnodeAsyncTaskHash(const void *obj) { + SVATask *task = (SVATask *)obj; + return MurmurHash3_32((const char *)(&task->taskId), sizeof(task->taskId)); +} + +static int32_t vnodeAsyncTaskCompare(const void *obj1, const void *obj2) { + SVATask *task1 = (SVATask *)obj1; + SVATask *task2 = (SVATask *)obj2; + if (task1->taskId < task2->taskId) { + return -1; + } else if (task1->taskId > task2->taskId) { + return 1; + } + return 0; +} + +static uint32_t vnodeAsyncChannelHash(const void *obj) { + SVAChannel *channel = (SVAChannel *)obj; + return MurmurHash3_32((const char *)(&channel->channelId), sizeof(channel->channelId)); +} + +static int32_t vnodeAsyncChannelCompare(const void *obj1, const void *obj2) { + SVAChannel *channel1 = (SVAChannel *)obj1; + SVAChannel *channel2 = (SVAChannel *)obj2; + if (channel1->channelId < channel2->channelId) { + return -1; + } else if (channel1->channelId > channel2->channelId) { + return 1; + } + return 0; +} + +int32_t vnodeAsyncInit(SVAsync **async, char *label) { + int32_t ret; + + if (async == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + if (label == NULL) { + label = "anonymous"; + } + + (*async) = (SVAsync *)taosMemoryCalloc(1, sizeof(SVAsync) + strlen(label) + 1); + if ((*async) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + strcpy((char *)((*async) + 1), label); + (*async)->label = (const char *)((*async) + 1); + + taosThreadMutexInit(&(*async)->mutex, NULL); + taosThreadCondInit(&(*async)->hasTask, NULL); + (*async)->stop = false; + + // worker + (*async)->numWorkers = VNODE_ASYNC_DEFAULT_WORKERS; + (*async)->numLaunchWorkers = 0; + (*async)->numIdleWorkers = 0; + for (int32_t i = 0; i < VNODE_ASYNC_MAX_WORKERS; i++) { + (*async)->workers[i].async = (*async); + (*async)->workers[i].workerId = i; + (*async)->workers[i].state = EVA_WORKER_STATE_UINIT; + (*async)->workers[i].runningTask = NULL; + } + + // channel + (*async)->nextChannelId = 0; + (*async)->numChannels = 0; + (*async)->chList.prev = &(*async)->chList; + (*async)->chList.next = &(*async)->chList; + ret = vHashInit(&(*async)->channelTable, vnodeAsyncChannelHash, vnodeAsyncChannelCompare); + if (ret != 0) { + taosThreadMutexDestroy(&(*async)->mutex); + taosThreadCondDestroy(&(*async)->hasTask); + taosMemoryFree(*async); + return ret; + } + + // task + (*async)->nextTaskId = 0; + (*async)->numTasks = 0; + for (int32_t i = 0; i < EVA_PRIORITY_MAX; i++) { + (*async)->queue[i].next = &(*async)->queue[i]; + (*async)->queue[i].prev = &(*async)->queue[i]; + } + ret = vHashInit(&(*async)->taskTable, vnodeAsyncTaskHash, vnodeAsyncTaskCompare); + if (ret != 0) { + vHashDestroy(&(*async)->channelTable); + taosThreadMutexDestroy(&(*async)->mutex); + taosThreadCondDestroy(&(*async)->hasTask); + taosMemoryFree(*async); + return ret; + } + + return 0; +} + +int32_t vnodeAsyncDestroy(SVAsync **async) { + if ((*async) == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + // set stop and broadcast + taosThreadMutexLock(&(*async)->mutex); + (*async)->stop = true; + taosThreadCondBroadcast(&(*async)->hasTask); + taosThreadMutexUnlock(&(*async)->mutex); + + // join all workers + for (int32_t i = 0; i < VNODE_ASYNC_MAX_WORKERS; i++) { + taosThreadMutexLock(&(*async)->mutex); + EVWorkerState state = (*async)->workers[i].state; + taosThreadMutexUnlock(&(*async)->mutex); + + if (state == EVA_WORKER_STATE_UINIT) { + continue; + } + + taosThreadJoin((*async)->workers[i].thread, NULL); + ASSERT((*async)->workers[i].state == EVA_WORKER_STATE_STOP); + (*async)->workers[i].state = EVA_WORKER_STATE_UINIT; + } + + // close all channels + for (SVAChannel *channel = (*async)->chList.next; channel != &(*async)->chList; channel = (*async)->chList.next) { + channel->next->prev = channel->prev; + channel->prev->next = channel->next; + + int32_t ret = vHashDrop((*async)->channelTable, channel); + if (ret) { + ASSERT(0); + } + (*async)->numChannels--; + taosMemoryFree(channel); + } + + ASSERT((*async)->numLaunchWorkers == 0); + ASSERT((*async)->numIdleWorkers == 0); + ASSERT((*async)->numChannels == 0); + ASSERT((*async)->numTasks == 0); + + taosThreadMutexDestroy(&(*async)->mutex); + taosThreadCondDestroy(&(*async)->hasTask); + + vHashDestroy(&(*async)->channelTable); + vHashDestroy(&(*async)->taskTable); + taosMemoryFree(*async); + *async = NULL; + + return 0; +} + +static int32_t vnodeAsyncLaunchWorker(SVAsync *async) { + for (int32_t i = 0; i < async->numWorkers; i++) { + ASSERT(async->workers[i].state != EVA_WORKER_STATE_IDLE); + if (async->workers[i].state == EVA_WORKER_STATE_ACTIVE) { + continue; + } else if (async->workers[i].state == EVA_WORKER_STATE_STOP) { + taosThreadJoin(async->workers[i].thread, NULL); + async->workers[i].state = EVA_WORKER_STATE_UINIT; + } + + taosThreadCreate(&async->workers[i].thread, NULL, vnodeAsyncLoop, &async->workers[i]); + async->workers[i].state = EVA_WORKER_STATE_ACTIVE; + async->numLaunchWorkers++; + break; + } + return 0; +} + +int32_t vnodeAsync(SVAsync *async, EVAPriority priority, int32_t (*execute)(void *), void (*complete)(void *), + void *arg, int64_t *taskId) { + return vnodeAsyncC(async, 0, priority, execute, complete, arg, taskId); +} + +int32_t vnodeAsyncC(SVAsync *async, int64_t channelId, EVAPriority priority, int32_t (*execute)(void *), + void (*complete)(void *), void *arg, int64_t *taskId) { + if (async == NULL || execute == NULL || channelId < 0) { + return TSDB_CODE_INVALID_PARA; + } + + int64_t id; + + // create task object + SVATask *task = (SVATask *)taosMemoryCalloc(1, sizeof(SVATask)); + if (task == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + task->priority = priority; + task->priorScore = 0; + task->execute = execute; + task->complete = complete; + task->arg = arg; + task->state = EVA_TASK_STATE_WAITTING; + task->numWait = 0; + taosThreadCondInit(&task->waitCond, NULL); + + // schedule task + taosThreadMutexLock(&async->mutex); + + if (channelId == 0) { + task->channel = NULL; + } else { + SVAChannel channel = {.channelId = channelId}; + vHashGet(async->channelTable, &channel, (void **)&task->channel); + if (task->channel == NULL) { + taosThreadMutexUnlock(&async->mutex); + taosThreadCondDestroy(&task->waitCond); + taosMemoryFree(task); + return TSDB_CODE_INVALID_PARA; + } + } + + task->taskId = id = ++async->nextTaskId; + + // add task to hash table + int32_t ret = vHashPut(async->taskTable, task); + if (ret != 0) { + taosThreadMutexUnlock(&async->mutex); + taosThreadCondDestroy(&task->waitCond); + taosMemoryFree(task); + return ret; + } + + async->numTasks++; + + // add task to queue + if (task->channel == NULL || task->channel->scheduled == NULL) { + // add task to async->queue + if (task->channel) { + task->channel->scheduled = task; + } + + task->next = &async->queue[priority]; + task->prev = async->queue[priority].prev; + task->next->prev = task; + task->prev->next = task; + + // signal worker or launch new worker + if (async->numIdleWorkers > 0) { + taosThreadCondSignal(&(async->hasTask)); + } else if (async->numLaunchWorkers < async->numWorkers) { + vnodeAsyncLaunchWorker(async); + } + } else if (task->channel->scheduled->state == EVA_TASK_STATE_RUNNING || + priority >= VATASK_PIORITY(task->channel->scheduled)) { + // add task to task->channel->queue + task->next = &task->channel->queue[priority]; + task->prev = task->channel->queue[priority].prev; + task->next->prev = task; + task->prev->next = task; + } else { + // remove task->channel->scheduled from queue + task->channel->scheduled->prev->next = task->channel->scheduled->next; + task->channel->scheduled->next->prev = task->channel->scheduled->prev; + + // promote priority and add task->channel->scheduled to task->channel->queue + task->channel->scheduled->priorScore++; + int32_t newPriority = VATASK_PIORITY(task->channel->scheduled); + task->channel->scheduled->next = &task->channel->queue[newPriority]; + task->channel->scheduled->prev = task->channel->queue[newPriority].prev; + task->channel->scheduled->next->prev = task->channel->scheduled; + task->channel->scheduled->prev->next = task->channel->scheduled; + + // add task to queue + task->channel->scheduled = task; + task->next = &async->queue[priority]; + task->prev = async->queue[priority].prev; + task->next->prev = task; + task->prev->next = task; + } + + taosThreadMutexUnlock(&async->mutex); + + if (taskId != NULL) { + *taskId = id; + } + + return 0; +} + +int32_t vnodeAWait(SVAsync *async, int64_t taskId) { + if (async == NULL || taskId <= 0) { + return TSDB_CODE_INVALID_PARA; + } + + SVATask *task = NULL; + SVATask task2 = {.taskId = taskId}; + + taosThreadMutexLock(&async->mutex); + + vHashGet(async->taskTable, &task2, (void **)&task); + if (task) { + task->numWait++; + taosThreadCondWait(&task->waitCond, &async->mutex); + task->numWait--; + + if (task->numWait == 0) { + taosThreadCondDestroy(&task->waitCond); + taosMemoryFree(task); + } + } + + taosThreadMutexUnlock(&async->mutex); + + return 0; +} + +int32_t vnodeACancel(SVAsync *async, int64_t taskId) { + if (async == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + int32_t ret = 0; + SVATask *task = NULL; + SVATask task2 = {.taskId = taskId}; + + taosThreadMutexLock(&async->mutex); + + vHashGet(async->taskTable, &task2, (void **)&task); + if (task) { + if (task->state == EVA_TASK_STATE_WAITTING) { + // remove from queue + task->next->prev = task->prev; + task->prev->next = task->next; + vnodeAsyncTaskDone(async, task); + } else { + ret = 0; // task is running, should return code TSDB_CODE_BUSY ?? + } + } + + taosThreadMutexUnlock(&async->mutex); + + return ret; +} + +int32_t vnodeAsyncSetWorkers(SVAsync *async, int32_t numWorkers) { + if (async == NULL || numWorkers <= 0 || numWorkers > VNODE_ASYNC_MAX_WORKERS) { + return TSDB_CODE_INVALID_PARA; + } + + taosThreadMutexLock(&async->mutex); + async->numWorkers = numWorkers; + if (async->numIdleWorkers > 0) { + taosThreadCondBroadcast(&async->hasTask); + } + taosThreadMutexUnlock(&async->mutex); + + return 0; +} + +int32_t vnodeAChannelInit(SVAsync *async, int64_t *channelId) { + if (async == NULL || channelId == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + // create channel object + SVAChannel *channel = (SVAChannel *)taosMemoryMalloc(sizeof(SVAChannel)); + if (channel == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + channel->state = EVA_CHANNEL_STATE_OPEN; + for (int32_t i = 0; i < EVA_PRIORITY_MAX; i++) { + channel->queue[i].next = &channel->queue[i]; + channel->queue[i].prev = &channel->queue[i]; + } + channel->scheduled = NULL; + + // register channel + taosThreadMutexLock(&async->mutex); + + channel->channelId = *channelId = ++async->nextChannelId; + + // add to hash table + int32_t ret = vHashPut(async->channelTable, channel); + if (ret != 0) { + taosThreadMutexUnlock(&async->mutex); + taosMemoryFree(channel); + return ret; + } + + // add to list + channel->next = &async->chList; + channel->prev = async->chList.prev; + channel->next->prev = channel; + channel->prev->next = channel; + + async->numChannels++; + + taosThreadMutexUnlock(&async->mutex); + + return 0; +} + +int32_t vnodeAChannelDestroy(SVAsync *async, int64_t channelId, bool waitRunning) { + if (async == NULL || channelId <= 0) { + return TSDB_CODE_INVALID_PARA; + } + + SVAChannel *channel = NULL; + SVAChannel channel2 = {.channelId = channelId}; + + taosThreadMutexLock(&async->mutex); + + vHashGet(async->channelTable, &channel2, (void **)&channel); + if (channel) { + // unregister channel + channel->next->prev = channel->prev; + channel->prev->next = channel->next; + vHashDrop(async->channelTable, channel); + async->numChannels--; + + // cancel all waiting tasks + for (int32_t i = 0; i < EVA_PRIORITY_MAX; i++) { + while (channel->queue[i].next != &channel->queue[i]) { + SVATask *task = channel->queue[i].next; + task->prev->next = task->next; + task->next->prev = task->prev; + vnodeAsyncTaskDone(async, task); + } + } + + // cancel or wait the scheduled task + if (channel->scheduled == NULL || channel->scheduled->state == EVA_TASK_STATE_WAITTING) { + if (channel->scheduled) { + channel->scheduled->prev->next = channel->scheduled->next; + channel->scheduled->next->prev = channel->scheduled->prev; + vnodeAsyncTaskDone(async, channel->scheduled); + } + taosMemoryFree(channel); + } else { + if (waitRunning) { + // wait task + SVATask *task = channel->scheduled; + task->numWait++; + taosThreadCondWait(&task->waitCond, &async->mutex); + task->numWait--; + if (task->numWait == 0) { + taosThreadCondDestroy(&task->waitCond); + taosMemoryFree(task); + } + + taosMemoryFree(channel); + } else { + channel->state = EVA_CHANNEL_STATE_CLOSE; + } + } + } else { + taosThreadMutexUnlock(&async->mutex); + return TSDB_CODE_INVALID_PARA; + } + + taosThreadMutexUnlock(&async->mutex); + + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index a178e1f772..c8cd167393 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -203,10 +203,8 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { // free info binary taosMemoryFree(data); - vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d selfIndex:%d changeVersion:%d", - pInfo->config.vgId, fname, - pInfo->config.syncCfg.replicaNum, pInfo->config.syncCfg.myIndex, - pInfo->config.syncCfg.changeVersion); + vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d selfIndex:%d changeVersion:%d", pInfo->config.vgId, fname, + pInfo->config.syncCfg.replicaNum, pInfo->config.syncCfg.myIndex, pInfo->config.syncCfg.changeVersion); return 0; @@ -289,9 +287,10 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) { char dir[TSDB_FILENAME_LEN] = {0}; int64_t lastCommitted = pInfo->info.state.committed; - tsem_wait(&pVnode->canCommit); + // wait last commit task + vnodeAWait(vnodeAsyncHandle[0], pVnode->commitTask); - if(syncNodeGetConfig(pVnode->sync, &pVnode->config.syncCfg) != 0) goto _exit; + if (syncNodeGetConfig(pVnode->sync, &pVnode->config.syncCfg) != 0) goto _exit; pVnode->state.commitTerm = pVnode->state.applyTerm; @@ -379,12 +378,11 @@ static int32_t vnodeCommitTask(void *arg) { vnodeReturnBufPool(pVnode); _exit: - // end commit - tsem_post(&pVnode->canCommit); - taosMemoryFree(pInfo); return code; } +static void vnodeCompleteCommit(void *arg) { taosMemoryFree(arg); } + int vnodeAsyncCommit(SVnode *pVnode) { int32_t code = 0; @@ -401,14 +399,14 @@ int vnodeAsyncCommit(SVnode *pVnode) { } // schedule the task - code = vnodeScheduleTask(vnodeCommitTask, pInfo); + code = vnodeAsyncC(vnodeAsyncHandle[0], pVnode->commitChannel, EVA_PRIORITY_HIGH, vnodeCommitTask, + vnodeCompleteCommit, pInfo, &pVnode->commitTask); _exit: if (code) { if (NULL != pInfo) { taosMemoryFree(pInfo); } - tsem_post(&pVnode->canCommit); vError("vgId:%d, %s failed since %s, commit id:%" PRId64, TD_VID(pVnode), __func__, tstrerror(code), pVnode->state.commitID); } else { @@ -420,8 +418,7 @@ _exit: int vnodeSyncCommit(SVnode *pVnode) { vnodeAsyncCommit(pVnode); - tsem_wait(&pVnode->canCommit); - tsem_post(&pVnode->canCommit); + vnodeAWait(vnodeAsyncHandle[0], pVnode->commitTask); return 0; } @@ -501,7 +498,7 @@ _exit: } bool vnodeShouldRollback(SVnode *pVnode) { - char tFName[TSDB_FILENAME_LEN] = {0}; + char tFName[TSDB_FILENAME_LEN] = {0}; int32_t offset = 0; vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, tFName, TSDB_FILENAME_LEN); @@ -512,7 +509,7 @@ bool vnodeShouldRollback(SVnode *pVnode) { } void vnodeRollback(SVnode *pVnode) { - char tFName[TSDB_FILENAME_LEN] = {0}; + char tFName[TSDB_FILENAME_LEN] = {0}; int32_t offset = 0; vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, tFName, TSDB_FILENAME_LEN); diff --git a/source/dnode/vnode/src/vnd/vnodeHash.c b/source/dnode/vnode/src/vnd/vnodeHash.c new file mode 100644 index 0000000000..33602f6581 --- /dev/null +++ b/source/dnode/vnode/src/vnd/vnodeHash.c @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "vnodeHash.h" + +#define VNODE_HASH_DEFAULT_NUM_BUCKETS 1024 + +typedef struct SVHashEntry SVHashEntry; + +struct SVHashEntry { + SVHashEntry* next; + void* obj; +}; + +struct SVHashTable { + uint32_t (*hash)(const void*); + int32_t (*compare)(const void*, const void*); + int32_t numEntries; + uint32_t numBuckets; + SVHashEntry** buckets; +}; + +static int32_t vHashRehash(SVHashTable* ht, uint32_t newNumBuckets) { + SVHashEntry** newBuckets = (SVHashEntry**)taosMemoryCalloc(newNumBuckets, sizeof(SVHashEntry*)); + if (newBuckets == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < ht->numBuckets; i++) { + SVHashEntry* entry = ht->buckets[i]; + while (entry != NULL) { + SVHashEntry* next = entry->next; + uint32_t bucketIndex = ht->hash(entry->obj) % newNumBuckets; + entry->next = newBuckets[bucketIndex]; + newBuckets[bucketIndex] = entry; + entry = next; + } + } + + taosMemoryFree(ht->buckets); + ht->buckets = newBuckets; + ht->numBuckets = newNumBuckets; + + return 0; +} + +int32_t vHashInit(SVHashTable** ht, uint32_t (*hash)(const void*), int32_t (*compare)(const void*, const void*)) { + if (ht == NULL || hash == NULL || compare == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + (*ht) = (SVHashTable*)taosMemoryMalloc(sizeof(SVHashTable)); + if (*ht == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + (*ht)->hash = hash; + (*ht)->compare = compare; + (*ht)->numEntries = 0; + (*ht)->numBuckets = VNODE_HASH_DEFAULT_NUM_BUCKETS; + (*ht)->buckets = (SVHashEntry**)taosMemoryCalloc((*ht)->numBuckets, sizeof(SVHashEntry*)); + if ((*ht)->buckets == NULL) { + taosMemoryFree(*ht); + return TSDB_CODE_OUT_OF_MEMORY; + } + + return 0; +} + +int32_t vHashDestroy(SVHashTable** ht) { + if (ht == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + if (*ht) { + ASSERT((*ht)->numEntries == 0); + taosMemoryFree((*ht)->buckets); + taosMemoryFree(*ht); + (*ht) = NULL; + } + return 0; +} + +int32_t vHashPut(SVHashTable* ht, void* obj) { + if (ht == NULL || obj == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + uint32_t bucketIndex = ht->hash(obj) % ht->numBuckets; + for (SVHashEntry* entry = ht->buckets[bucketIndex]; entry != NULL; entry = entry->next) { + if (ht->compare(entry->obj, obj) == 0) { + return TSDB_CODE_DUP_KEY; + } + } + + if (ht->numEntries >= ht->numBuckets) { + vHashRehash(ht, ht->numBuckets * 2); + bucketIndex = ht->hash(obj) % ht->numBuckets; + } + + SVHashEntry* entry = (SVHashEntry*)taosMemoryMalloc(sizeof(SVHashEntry)); + if (entry == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + entry->obj = obj; + entry->next = ht->buckets[bucketIndex]; + ht->buckets[bucketIndex] = entry; + ht->numEntries++; + + return 0; +} + +int32_t vHashGet(SVHashTable* ht, const void* obj, void** retObj) { + if (ht == NULL || obj == NULL || retObj == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + uint32_t bucketIndex = ht->hash(obj) % ht->numBuckets; + for (SVHashEntry* entry = ht->buckets[bucketIndex]; entry != NULL; entry = entry->next) { + if (ht->compare(entry->obj, obj) == 0) { + *retObj = entry->obj; + return 0; + } + } + + *retObj = NULL; + return TSDB_CODE_NOT_FOUND; +} + +int32_t vHashDrop(SVHashTable* ht, const void* obj) { + if (ht == NULL || obj == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + uint32_t bucketIndex = ht->hash(obj) % ht->numBuckets; + for (SVHashEntry** entry = &ht->buckets[bucketIndex]; *entry != NULL; entry = &(*entry)->next) { + if (ht->compare((*entry)->obj, obj) == 0) { + SVHashEntry* tmp = *entry; + *entry = (*entry)->next; + taosMemoryFree(tmp); + ht->numEntries--; + if (ht->numBuckets > VNODE_HASH_DEFAULT_NUM_BUCKETS && ht->numEntries < ht->numBuckets / 4) { + vHashRehash(ht, ht->numBuckets / 2); + } + return 0; + } + } + + return TSDB_CODE_NOT_FOUND; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeHash.h b/source/dnode/vnode/src/vnd/vnodeHash.h new file mode 100644 index 0000000000..86f6f9ac87 --- /dev/null +++ b/source/dnode/vnode/src/vnd/vnodeHash.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _VNODE_HAS_H_ +#define _VNODE_HAS_H_ + +#include "vnd.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SVHashTable SVHashTable; + +int32_t vHashInit(SVHashTable** ht, uint32_t (*hash)(const void*), int32_t (*compare)(const void*, const void*)); +int32_t vHashDestroy(SVHashTable** ht); +int32_t vHashPut(SVHashTable* ht, void* obj); +int32_t vHashGet(SVHashTable* ht, const void* obj, void** retObj); +int32_t vHashDrop(SVHashTable* ht, const void* obj); + +#ifdef __cplusplus +} +#endif + +#endif /*_VNODE_HAS_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index a6673917bf..6b7b778cd5 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -60,6 +60,9 @@ void initTsdbReaderAPI(TsdReader* pReader) { pReader->tsdSetQueryTableList = tsdbSetTableList2; pReader->tsdSetReaderTaskId = (void (*)(void*, const char*))tsdbReaderSetId2; + + pReader->tsdSetFilesetDelimited = (void (*)(void*))tsdbSetFilesetDelimited; + pReader->tsdSetSetNotifyCb = (void (*)(void*, TsdReaderNotifyCbFn, void*))tsdbReaderSetNotifyCb; } void initMetadataAPI(SStoreMeta* pMeta) { @@ -189,6 +192,7 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->updateInfoDestroy = updateInfoDestroy; pStore->windowSBfDelete = windowSBfDelete; pStore->windowSBfAdd = windowSBfAdd; + pStore->isIncrementalTimeStamp = isIncrementalTimeStamp; pStore->updateInfoInitP = updateInfoInitP; pStore->updateInfoAddCloseWindowSBF = updateInfoAddCloseWindowSBF; diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index df08fb8a2b..44fcbefba7 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -16,173 +16,41 @@ #include "cos.h" #include "vnd.h" -typedef struct SVnodeTask SVnodeTask; -struct SVnodeTask { - SVnodeTask* next; - SVnodeTask* prev; - int (*execute)(void*); - void* arg; -}; +static volatile int32_t VINIT = 0; -typedef struct { - int nthreads; - TdThread* threads; - TdThreadMutex mutex; - TdThreadCond hasTask; - SVnodeTask queue; -} SVnodeThreadPool; - -struct SVnodeGlobal { - int8_t init; - int8_t stop; - SVnodeThreadPool tp[2]; -}; - -struct SVnodeGlobal vnodeGlobal; - -static void* loop(void* arg); +SVAsync* vnodeAsyncHandle[2]; int vnodeInit(int nthreads) { - int8_t init; - int ret; + int32_t init; - init = atomic_val_compare_exchange_8(&(vnodeGlobal.init), 0, 1); + init = atomic_val_compare_exchange_32(&VINIT, 0, 1); if (init) { return 0; } - vnodeGlobal.stop = 0; - for (int32_t i = 0; i < ARRAY_SIZE(vnodeGlobal.tp); i++) { - taosThreadMutexInit(&vnodeGlobal.tp[i].mutex, NULL); - taosThreadCondInit(&vnodeGlobal.tp[i].hasTask, NULL); + // vnode-commit + vnodeAsyncInit(&vnodeAsyncHandle[0], "vnode-commit"); + vnodeAsyncSetWorkers(vnodeAsyncHandle[0], nthreads); - taosThreadMutexLock(&vnodeGlobal.tp[i].mutex); - - vnodeGlobal.tp[i].queue.next = &vnodeGlobal.tp[i].queue; - vnodeGlobal.tp[i].queue.prev = &vnodeGlobal.tp[i].queue; - - taosThreadMutexUnlock(&(vnodeGlobal.tp[i].mutex)); - - vnodeGlobal.tp[i].nthreads = nthreads; - vnodeGlobal.tp[i].threads = taosMemoryCalloc(nthreads, sizeof(TdThread)); - if (vnodeGlobal.tp[i].threads == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - vError("failed to init vnode module since:%s", tstrerror(terrno)); - return -1; - } - - for (int j = 0; j < nthreads; j++) { - taosThreadCreate(&(vnodeGlobal.tp[i].threads[j]), NULL, loop, &vnodeGlobal.tp[i]); - } - } + // vnode-merge + vnodeAsyncInit(&vnodeAsyncHandle[1], "vnode-merge"); + vnodeAsyncSetWorkers(vnodeAsyncHandle[1], nthreads); if (walInit() < 0) { return -1; } - if (tqInit() < 0) { - return -1; - } - if (s3Init() < 0) { - return -1; - } return 0; } void vnodeCleanup() { - int8_t init; - - init = atomic_val_compare_exchange_8(&(vnodeGlobal.init), 1, 0); + int32_t init = atomic_val_compare_exchange_32(&VINIT, 1, 0); if (init == 0) return; // set stop - vnodeGlobal.stop = 1; - for (int32_t i = 0; i < ARRAY_SIZE(vnodeGlobal.tp); i++) { - taosThreadMutexLock(&(vnodeGlobal.tp[i].mutex)); - taosThreadCondBroadcast(&(vnodeGlobal.tp[i].hasTask)); - taosThreadMutexUnlock(&(vnodeGlobal.tp[i].mutex)); - - // wait for threads - for (int j = 0; j < vnodeGlobal.tp[i].nthreads; j++) { - taosThreadJoin(vnodeGlobal.tp[i].threads[j], NULL); - } - - // clear source - taosMemoryFreeClear(vnodeGlobal.tp[i].threads); - taosThreadCondDestroy(&(vnodeGlobal.tp[i].hasTask)); - taosThreadMutexDestroy(&(vnodeGlobal.tp[i].mutex)); - } + vnodeAsyncDestroy(&vnodeAsyncHandle[0]); + vnodeAsyncDestroy(&vnodeAsyncHandle[1]); walCleanUp(); - tqCleanUp(); smaCleanUp(); - s3CleanUp(); -} - -int vnodeScheduleTaskEx(int tpid, int (*execute)(void*), void* arg) { - SVnodeTask* pTask; - - ASSERT(!vnodeGlobal.stop); - - pTask = taosMemoryMalloc(sizeof(*pTask)); - if (pTask == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - pTask->execute = execute; - pTask->arg = arg; - - taosThreadMutexLock(&(vnodeGlobal.tp[tpid].mutex)); - pTask->next = &vnodeGlobal.tp[tpid].queue; - pTask->prev = vnodeGlobal.tp[tpid].queue.prev; - vnodeGlobal.tp[tpid].queue.prev->next = pTask; - vnodeGlobal.tp[tpid].queue.prev = pTask; - taosThreadCondSignal(&(vnodeGlobal.tp[tpid].hasTask)); - taosThreadMutexUnlock(&(vnodeGlobal.tp[tpid].mutex)); - - return 0; -} - -int vnodeScheduleTask(int (*execute)(void*), void* arg) { return vnodeScheduleTaskEx(0, execute, arg); } - -/* ------------------------ STATIC METHODS ------------------------ */ -static void* loop(void* arg) { - SVnodeThreadPool* tp = (SVnodeThreadPool*)arg; - SVnodeTask* pTask; - int ret; - - if (tp == &vnodeGlobal.tp[0]) { - setThreadName("vnode-commit"); - } else if (tp == &vnodeGlobal.tp[1]) { - setThreadName("vnode-merge"); - } - - for (;;) { - taosThreadMutexLock(&(tp->mutex)); - for (;;) { - pTask = tp->queue.next; - if (pTask == &tp->queue) { - // no task - if (vnodeGlobal.stop) { - taosThreadMutexUnlock(&(tp->mutex)); - return NULL; - } else { - taosThreadCondWait(&(tp->hasTask), &(tp->mutex)); - } - } else { - // has task - pTask->prev->next = pTask->next; - pTask->next->prev = pTask->prev; - break; - } - } - - taosThreadMutexUnlock(&(tp->mutex)); - - pTask->execute(pTask->arg); - taosMemoryFree(pTask); - } - - return NULL; } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index ff79e83d72..946ce9d278 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -129,8 +129,8 @@ int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, int32_t } pCfg->changeVersion = pReq->changeVersion; - vInfo("vgId:%d, save config while alter, replicas:%d totalReplicas:%d selfIndex:%d changeVersion:%d", - pReq->vgId, pCfg->replicaNum, pCfg->totalReplicaNum, pCfg->myIndex, pCfg->changeVersion); + vInfo("vgId:%d, save config while alter, replicas:%d totalReplicas:%d selfIndex:%d changeVersion:%d", pReq->vgId, + pCfg->replicaNum, pCfg->totalReplicaNum, pCfg->myIndex, pCfg->changeVersion); info.config.syncCfg = *pCfg; ret = vnodeSaveInfo(dir, &info); @@ -396,10 +396,14 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC pVnode->blocked = false; tsem_init(&pVnode->syncSem, 0, 0); - tsem_init(&(pVnode->canCommit), 0, 1); taosThreadMutexInit(&pVnode->mutex, NULL); taosThreadCondInit(&pVnode->poolNotEmpty, NULL); + if (vnodeAChannelInit(vnodeAsyncHandle[0], &pVnode->commitChannel) != 0) { + vError("vgId:%d, failed to init commit channel", TD_VID(pVnode)); + goto _err; + } + int8_t rollback = vnodeShouldRollback(pVnode); // open buffer pool @@ -487,7 +491,6 @@ _err: if (pVnode->pMeta) metaClose(&pVnode->pMeta); if (pVnode->freeList) vnodeCloseBufPool(pVnode); - tsem_destroy(&(pVnode->canCommit)); taosMemoryFree(pVnode); return NULL; } @@ -501,7 +504,8 @@ void vnodePostClose(SVnode *pVnode) { vnodeSyncPostClose(pVnode); } void vnodeClose(SVnode *pVnode) { if (pVnode) { - tsem_wait(&pVnode->canCommit); + vnodeAWait(vnodeAsyncHandle[0], pVnode->commitTask); + vnodeAChannelDestroy(vnodeAsyncHandle[0], pVnode->commitChannel, true); vnodeSyncClose(pVnode); vnodeQueryClose(pVnode); tqClose(pVnode->pTq); @@ -510,10 +514,8 @@ void vnodeClose(SVnode *pVnode) { smaClose(pVnode->pSma); if (pVnode->pMeta) metaClose(&pVnode->pMeta); vnodeCloseBufPool(pVnode); - tsem_post(&pVnode->canCommit); // destroy handle - tsem_destroy(&(pVnode->canCommit)); tsem_destroy(&pVnode->syncSem); taosThreadCondDestroy(&pVnode->poolNotEmpty); taosThreadMutexDestroy(&pVnode->mutex); diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 91244e321f..ed1dcc64c9 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -13,8 +13,28 @@ * along with this program. If not, see . */ -#include "vnd.h" #include "tsdb.h" +#include "vnd.h" + +static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TFileSetRangeArray **ppRanges) { + int32_t code = -1; + STsdbFSetPartList *pList = tsdbFSetPartListCreate(); + if (pList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _out; + } + if (tDeserializeTsdbFSetPartList(buf, bufLen, pList) < 0) { + terrno = TSDB_CODE_INVALID_DATA_FMT; + goto _out; + } + if (tsdbFSetPartListToRangeDiff(pList, ppRanges) < 0) { + goto _out; + } + code = 0; +_out: + tsdbFSetPartListDestroy(&pList); + return code; +} // SVSnapReader ======================================================== struct SVSnapReader { @@ -29,14 +49,18 @@ struct SVSnapReader { SMetaSnapReader *pMetaReader; // tsdb int8_t tsdbDone; - TSnapRangeArray *pRanges; + TFileSetRangeArray *pRanges; STsdbSnapReader *pTsdbReader; + // tsdb raw + int8_t tsdbRAWDone; + STsdbSnapRAWReader *pTsdbRAWReader; + // tq - int8_t tqHandleDone; - STqSnapReader *pTqSnapReader; - int8_t tqOffsetDone; - STqOffsetReader *pTqOffsetReader; - int8_t tqCheckInfoDone; + int8_t tqHandleDone; + STqSnapReader *pTqSnapReader; + int8_t tqOffsetDone; + STqOffsetReader *pTqOffsetReader; + int8_t tqCheckInfoDone; STqCheckInfoReader *pTqCheckInfoReader; // stream int8_t streamTaskDone; @@ -45,31 +69,11 @@ struct SVSnapReader { SStreamStateReader *pStreamStateReader; // rsma int8_t rsmaDone; - TSnapRangeArray *pRsmaRanges[TSDB_RETENTION_L2]; + TFileSetRangeArray *pRsmaRanges[TSDB_RETENTION_L2]; SRSmaSnapReader *pRsmaReader; }; -static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TSnapRangeArray **ppRanges) { - int32_t code = -1; - STsdbSnapPartList *pList = tsdbSnapPartListCreate(); - if (pList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _out; - } - if (tDeserializeTsdbSnapPartList(buf, bufLen, pList) < 0) { - terrno = TSDB_CODE_INVALID_DATA_FMT; - goto _out; - } - if (tsdbSnapPartListToRangeDiff(pList, ppRanges) < 0) { - goto _out; - } - code = 0; -_out: - tsdbSnapPartListDestroy(&pList); - return code; -} - -static TSnapRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int32_t tsdbTyp) { +static TFileSetRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int32_t tsdbTyp) { ASSERTS(sizeof(pReader->pRsmaRanges) / sizeof(pReader->pRsmaRanges[0]) == 2, "Unexpected array size"); switch (tsdbTyp) { case SNAP_DATA_TSDB: @@ -83,37 +87,66 @@ static TSnapRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int } } -static int32_t vnodeSnapReaderDoSnapInfo(SVSnapReader *pReader, SSnapshotParam *pParam) { +static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotParam *pParam) { SVnode *pVnode = pReader->pVnode; int32_t code = -1; if (pParam->data) { + // decode SSyncTLV *datHead = (void *)pParam->data; if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { terrno = TSDB_CODE_INVALID_DATA_FMT; goto _out; } - TSnapRangeArray **ppRanges = NULL; - int32_t offset = 0; + STsdbRepOpts tsdbOpts = {0}; + TFileSetRangeArray **ppRanges = NULL; + int32_t offset = 0; while (offset + sizeof(SSyncTLV) < datHead->len) { SSyncTLV *subField = (void *)(datHead->val + offset); offset += sizeof(SSyncTLV) + subField->len; void *buf = subField->val; int32_t bufLen = subField->len; - ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, subField->typ); - if (ppRanges == NULL) { - vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); - goto _out; - } - if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { - vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); - goto _out; + + switch (subField->typ) { + case SNAP_DATA_TSDB: + case SNAP_DATA_RSMA1: + case SNAP_DATA_RSMA2: { + ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, subField->typ); + if (ppRanges == NULL) { + vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); + goto _out; + } + if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { + vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + } break; + case SNAP_DATA_RAW: { + if (tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts) < 0) { + vError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + } break; + default: + vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ); + goto _out; } } - } + // toggle snap replication mode + vInfo("vgId:%d, vnode snap reader supported tsdb rep of format:%d", TD_VID(pVnode), tsdbOpts.format); + if (pReader->sver == 0 && tsdbOpts.format == TSDB_SNAP_REP_FMT_RAW) { + pReader->tsdbDone = true; + } else { + pReader->tsdbRAWDone = true; + } + + ASSERT(pReader->tsdbDone != pReader->tsdbRAWDone); + vInfo("vgId:%d, vnode snap writer enabled replication mode: %s", TD_VID(pVnode), + (pReader->tsdbDone ? "raw" : "normal")); + } code = 0; _out: return code; @@ -135,7 +168,7 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader pReader->ever = ever; // snapshot info - if (vnodeSnapReaderDoSnapInfo(pReader, pParam) < 0) { + if (vnodeSnapReaderDealWithSnapInfo(pReader, pParam) < 0) { goto _err; } @@ -152,9 +185,9 @@ _err: static void vnodeSnapReaderDestroyTsdbRanges(SVSnapReader *pReader) { int32_t tsdbTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2}; for (int32_t j = 0; j < TSDB_RETENTION_MAX; ++j) { - TSnapRangeArray **ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, tsdbTyps[j]); + TFileSetRangeArray **ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, tsdbTyps[j]); if (ppRanges == NULL) continue; - tsdbSnapRangeArrayDestroy(ppRanges); + tsdbTFileSetRangeArrayDestroy(ppRanges); } } @@ -170,6 +203,10 @@ void vnodeSnapReaderClose(SVSnapReader *pReader) { tsdbSnapReaderClose(&pReader->pTsdbReader); } + if (pReader->pTsdbRAWReader) { + tsdbSnapRAWReaderClose(&pReader->pTsdbRAWReader); + } + if (pReader->pMetaReader) { metaSnapReaderClose(&pReader->pMetaReader); } @@ -285,6 +322,28 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) } } + if (!pReader->tsdbRAWDone) { + // open if not + if (pReader->pTsdbRAWReader == NULL) { + ASSERT(pReader->sver == 0); + code = tsdbSnapRAWReaderOpen(pReader->pVnode->pTsdb, pReader->ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader); + if (code) goto _err; + } + + code = tsdbSnapRAWRead(pReader->pTsdbRAWReader, ppData); + if (code) { + goto _err; + } else { + if (*ppData) { + goto _exit; + } else { + pReader->tsdbRAWDone = 1; + code = tsdbSnapRAWReaderClose(&pReader->pTsdbRAWReader); + if (code) goto _err; + } + } + } + // TQ ================ vInfo("vgId:%d tq transform start", vgId); if (!pReader->tqHandleDone) { @@ -455,21 +514,23 @@ struct SVSnapWriter { // meta SMetaSnapWriter *pMetaSnapWriter; // tsdb - TSnapRangeArray *pRanges; + TFileSetRangeArray *pRanges; STsdbSnapWriter *pTsdbSnapWriter; + // tsdb raw + STsdbSnapRAWWriter *pTsdbSnapRAWWriter; // tq - STqSnapWriter *pTqSnapWriter; - STqOffsetWriter *pTqOffsetWriter; + STqSnapWriter *pTqSnapWriter; + STqOffsetWriter *pTqOffsetWriter; STqCheckInfoWriter *pTqCheckInfoWriter; // stream SStreamTaskWriter *pStreamTaskWriter; SStreamStateWriter *pStreamStateWriter; // rsma - TSnapRangeArray *pRsmaRanges[TSDB_RETENTION_L2]; + TFileSetRangeArray *pRsmaRanges[TSDB_RETENTION_L2]; SRSmaSnapWriter *pRsmaSnapWriter; }; -TSnapRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t tsdbTyp) { +TFileSetRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t tsdbTyp) { ASSERTS(sizeof(pWriter->pRsmaRanges) / sizeof(pWriter->pRsmaRanges[0]) == 2, "Unexpected array size"); switch (tsdbTyp) { case SNAP_DATA_TSDB: @@ -483,7 +544,7 @@ TSnapRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t ts } } -static int32_t vnodeSnapWriterDoSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *pParam) { +static int32_t vnodeSnapWriterDealWithSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *pParam) { SVnode *pVnode = pWriter->pVnode; int32_t code = -1; @@ -494,7 +555,8 @@ static int32_t vnodeSnapWriterDoSnapInfo(SVSnapWriter *pWriter, SSnapshotParam * goto _out; } - TSnapRangeArray **ppRanges = NULL; + STsdbRepOpts tsdbOpts = {0}; + TFileSetRangeArray **ppRanges = NULL; int32_t offset = 0; while (offset + sizeof(SSyncTLV) < datHead->len) { @@ -502,16 +564,34 @@ static int32_t vnodeSnapWriterDoSnapInfo(SVSnapWriter *pWriter, SSnapshotParam * offset += sizeof(SSyncTLV) + subField->len; void *buf = subField->val; int32_t bufLen = subField->len; - ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, subField->typ); - if (ppRanges == NULL) { - vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); - goto _out; - } - if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { - vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); - goto _out; + + switch (subField->typ) { + case SNAP_DATA_TSDB: + case SNAP_DATA_RSMA1: + case SNAP_DATA_RSMA2: { + ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, subField->typ); + if (ppRanges == NULL) { + vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); + goto _out; + } + if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { + vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + } break; + case SNAP_DATA_RAW: { + if (tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts) < 0) { + vError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + } break; + default: + vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ); + goto _out; } } + + vInfo("vgId:%d, vnode snap writer supported tsdb rep of format:%d", TD_VID(pVnode), tsdbOpts.format); } code = 0; @@ -519,15 +599,30 @@ _out: return code; } +extern int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb); +extern int32_t tsdbEnableBgTask(STsdb *pTsdb); + +static int32_t vnodeCancelAndDisableAllBgTask(SVnode *pVnode) { + tsdbDisableAndCancelAllBgTask(pVnode->pTsdb); + vnodeSyncCommit(pVnode); + vnodeAChannelDestroy(vnodeAsyncHandle[0], pVnode->commitChannel, true); + return 0; +} + +static int32_t vnodeEnableBgTask(SVnode *pVnode) { + tsdbEnableBgTask(pVnode->pTsdb); + vnodeAChannelInit(vnodeAsyncHandle[0], &pVnode->commitChannel); + return 0; +} + int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter **ppWriter) { int32_t code = 0; SVSnapWriter *pWriter = NULL; int64_t sver = pParam->start; int64_t ever = pParam->end; - // commit memory data - vnodeAsyncCommit(pVnode); - tsem_wait(&pVnode->canCommit); + // cancel and disable all bg task + vnodeCancelAndDisableAllBgTask(pVnode); // alloc pWriter = (SVSnapWriter *)taosMemoryCalloc(1, sizeof(*pWriter)); @@ -543,7 +638,7 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter pWriter->commitID = ++pVnode->state.commitID; // snapshot info - if (vnodeSnapWriterDoSnapInfo(pWriter, pParam) < 0) { + if (vnodeSnapWriterDealWithSnapInfo(pWriter, pParam) < 0) { goto _err; } @@ -561,9 +656,9 @@ _err: static void vnodeSnapWriterDestroyTsdbRanges(SVSnapWriter *pWriter) { int32_t tsdbTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2}; for (int32_t j = 0; j < TSDB_RETENTION_MAX; ++j) { - TSnapRangeArray **ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, tsdbTyps[j]); + TFileSetRangeArray **ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, tsdbTyps[j]); if (ppRanges == NULL) continue; - tsdbSnapRangeArrayDestroy(ppRanges); + tsdbTFileSetRangeArrayDestroy(ppRanges); } } @@ -578,6 +673,10 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * tsdbSnapWriterPrepareClose(pWriter->pTsdbSnapWriter); } + if (pWriter->pTsdbSnapRAWWriter) { + tsdbSnapRAWWriterPrepareClose(pWriter->pTsdbSnapRAWWriter); + } + if (pWriter->pRsmaSnapWriter) { rsmaSnapWriterPrepareClose(pWriter->pRsmaSnapWriter); } @@ -614,6 +713,11 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * if (code) goto _exit; } + if (pWriter->pTsdbSnapRAWWriter) { + code = tsdbSnapRAWWriterClose(&pWriter->pTsdbSnapRAWWriter, rollback); + if (code) goto _exit; + } + if (pWriter->pTqSnapWriter) { code = tqSnapWriterClose(&pWriter->pTqSnapWriter, rollback); if (code) goto _exit; @@ -657,7 +761,7 @@ _exit: vInfo("vgId:%d, vnode snapshot writer closed, rollback:%d", TD_VID(pVnode), rollback); taosMemoryFree(pWriter); } - tsem_post(&pVnode->canCommit); + vnodeEnableBgTask(pVnode); return code; } @@ -737,6 +841,17 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pHdr); if (code) goto _err; } break; + case SNAP_DATA_RAW: { + // tsdb + if (pWriter->pTsdbSnapRAWWriter == NULL) { + ASSERT(pWriter->sver == 0); + code = tsdbSnapRAWWriterOpen(pVnode->pTsdb, pWriter->ever, &pWriter->pTsdbSnapRAWWriter); + if (code) goto _err; + } + + code = tsdbSnapRAWWrite(pWriter->pTsdbSnapRAWWriter, pHdr); + if (code) goto _err; + } break; case SNAP_DATA_TQ_HANDLE: { // tq handle if (pWriter->pTqSnapWriter == NULL) { diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 817d5124a2..5871a60c9e 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -804,7 +804,7 @@ int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnap) { } if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT || pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { - code = tsdbSnapGetDetails(pVnode, pSnap); + code = tsdbSnapPrepDescription(pVnode, pSnap); } return code; } diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index e29583d8fc..cba26c46b5 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -270,6 +270,7 @@ typedef struct STableScanInfo { bool hasGroupByTag; bool countOnly; // TsdReader readerAPI; + bool filesetDelimited; } STableScanInfo; typedef struct STableMergeScanInfo { @@ -297,6 +298,9 @@ typedef struct STableMergeScanInfo { SHashObj* mSkipTables; int64_t mergeLimit; SSortExecInfo sortExecInfo; + bool bNewFileset; + bool bOnlyRetrieveBlock; + bool filesetDelimited; } STableMergeScanInfo; typedef struct STagScanFilterContext { @@ -338,18 +342,19 @@ enum { }; typedef struct SStreamAggSupporter { - int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row - SSDataBlock* pScanBlock; - SStreamState* pState; - int64_t gap; // stream session window gap - SqlFunctionCtx* pDummyCtx; // for combine - SSHashObj* pResultRows; - int32_t stateKeySize; - int16_t stateKeyType; - SDiskbasedBuf* pResultBuf; - SStateStore stateStore; - STimeWindow winRange; - SStorageAPI* pSessionAPI; + int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row + SSDataBlock* pScanBlock; + SStreamState* pState; + int64_t gap; // stream session window gap + SqlFunctionCtx* pDummyCtx; // for combine + SSHashObj* pResultRows; + int32_t stateKeySize; + int16_t stateKeyType; + SDiskbasedBuf* pResultBuf; + SStateStore stateStore; + STimeWindow winRange; + SStorageAPI* pSessionAPI; + struct SUpdateInfo* pUpdateInfo; } SStreamAggSupporter; typedef struct SWindowSupporter { @@ -502,38 +507,39 @@ typedef struct SOpCheckPointInfo { } SOpCheckPointInfo; typedef struct SStreamIntervalOperatorInfo { - SOptrBasicInfo binfo; // basic info - SAggSupporter aggSup; // aggregate supporter - SExprSupp scalarSupp; // supporter for perform scalar function - SGroupResInfo groupResInfo; // multiple results build supporter - SInterval interval; // interval info - int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. - STimeWindowAggSupp twAggSup; - bool invertible; - bool ignoreExpiredData; - bool ignoreExpiredDataSaved; - SArray* pDelWins; // SWinRes - int32_t delIndex; - SSDataBlock* pDelRes; - SPhysiNode* pPhyNode; // create new child - SHashObj* pPullDataMap; - SArray* pPullWins; // SPullWindowInfo - int32_t pullIndex; - SSDataBlock* pPullDataRes; - SArray* pChildren; - int32_t numOfChild; - SStreamState* pState; // void - SWinKey delKey; - uint64_t numOfDatapack; - SArray* pUpdated; - SSHashObj* pUpdatedMap; - int64_t dataVersion; - SStateStore stateStore; - bool recvGetAll; - SHashObj* pFinalPullDataMap; - SOpCheckPointInfo checkPointInfo; - bool reCkBlock; - SSDataBlock* pCheckpointRes; + SOptrBasicInfo binfo; // basic info + SAggSupporter aggSup; // aggregate supporter + SExprSupp scalarSupp; // supporter for perform scalar function + SGroupResInfo groupResInfo; // multiple results build supporter + SInterval interval; // interval info + int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. + STimeWindowAggSupp twAggSup; + bool invertible; + bool ignoreExpiredData; + bool ignoreExpiredDataSaved; + SArray* pDelWins; // SWinRes + int32_t delIndex; + SSDataBlock* pDelRes; + SPhysiNode* pPhyNode; // create new child + SHashObj* pPullDataMap; + SArray* pPullWins; // SPullWindowInfo + int32_t pullIndex; + SSDataBlock* pPullDataRes; + SArray* pChildren; + int32_t numOfChild; + SStreamState* pState; // void + SWinKey delKey; + uint64_t numOfDatapack; + SArray* pUpdated; + SSHashObj* pUpdatedMap; + int64_t dataVersion; + SStateStore stateStore; + bool recvGetAll; + SHashObj* pFinalPullDataMap; + SOpCheckPointInfo checkPointInfo; + bool reCkBlock; + SSDataBlock* pCheckpointRes; + struct SUpdateInfo* pUpdateInfo; } SStreamIntervalOperatorInfo; typedef struct SDataGroupInfo { @@ -830,6 +836,7 @@ void compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeW SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool addGap); int32_t releaseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI); void resetWinRange(STimeWindow* winRange); +bool checkExpiredData(SStateStore* pAPI, SUpdateInfo* pUpdateInfo, STimeWindowAggSupp* pTwSup, uint64_t tableId, TSKEY ts); int32_t encodeSSessionKey(void** buf, SSessionKey* key); void* decodeSSessionKey(void* buf, SSessionKey* key); diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 6d59698855..75fe3c51dd 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -352,6 +352,7 @@ int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTask for (int32_t i = 0; i < numOfCols; ++i) { SColMatchItem* pColMatch = taosArrayGet(pColMatchInfo, i); + bool found = false; for (int32_t j = 0; j < pWrapper->nCols; ++j) { /* if (pColMatch->colId == pWrapper->pSchema[j].colId && pColMatch->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { (*pSlotIds)[pColMatch->dstSlotId] = -1; @@ -361,9 +362,14 @@ int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTask if (pColMatch->colId == pWrapper->pSchema[j].colId) { (*pSlotIds)[i] = j; (*pDstSlotIds)[i] = pColMatch->dstSlotId; + found = true; break; } } + if (!found) { + (*pSlotIds)[i] = -1; + (*pDstSlotIds)[i] = pColMatch->dstSlotId; + } } return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index f301ddf4be..00b58263e2 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -189,7 +189,7 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp } int64_t lastTs = TSKEY_MIN; - bool ignoreRow = false; + bool updateLastRow = false; bool disorderTs = false; for (int32_t j = 0; j < rows; ++j) { // iterate by row @@ -249,7 +249,7 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp } else { if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId) { if (*(int64_t*)var == lastTs) { - ignoreRow = true; + updateLastRow = true; } else if (*(int64_t*)var < lastTs) { disorderTs = true; } else { @@ -269,15 +269,6 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp } break; } - - if (ignoreRow) { - break; - } - } - - if (ignoreRow) { - ignoreRow = false; - continue; } SRow* pRow = NULL; @@ -285,7 +276,14 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); goto _end; } - taosArrayPush(tbData.aRowP, &pRow); + if (updateLastRow) { + updateLastRow = false; + SRow** lastRow = taosArrayPop(tbData.aRowP); + tRowDestroy(*lastRow); + taosArrayPush(tbData.aRowP, &pRow); + } else { + taosArrayPush(tbData.aRowP, &pRow); + } } if (disorderTs) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e3068ce319..4fb6e26cc5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -893,7 +893,9 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } - + if (pInfo->filesetDelimited) { + pAPI->tsdReader.tsdSetFilesetDelimited(pInfo->base.dataReader); + } if (pInfo->pResBlock->info.capacity > pOperator->resultInfo.capacity) { pOperator->resultInfo.capacity = pInfo->pResBlock->info.capacity; } @@ -1085,6 +1087,8 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->countOnly = true; } + pInfo->filesetDelimited = pTableScanNode->filesetDelimited; + taosLRUCacheSetStrictCapacity(pInfo->base.metaCache.pTableMetaEntryCache, false); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo, optrDefaultBufFn, getTableScannerExecInfo, optrDefaultGetNextExtFn, NULL); @@ -1620,6 +1624,15 @@ void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKE pBlock->info.rows++; } +bool checkExpiredData(SStateStore* pAPI, SUpdateInfo* pUpdateInfo, STimeWindowAggSupp* pTwSup, uint64_t tableId, TSKEY ts) { + bool isExpired = false; + bool isInc = pAPI->isIncrementalTimeStamp(pUpdateInfo, tableId, ts); + if (!isInc) { + isExpired = isOverdue(ts, pTwSup); + } + return isExpired; +} + static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock* pBlock, bool out) { if (out) { blockDataCleanup(pInfo->pUpdateDataRes); @@ -3215,6 +3228,7 @@ static int32_t tableMergeScanDoSkipTable(STableMergeScanInfo* pInfo, SSDataBlock tSimpleHashPut(pInfo->mTableNumRows, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid), &nRows, sizeof(nRows)); } else { *(int64_t*)pNum = *(int64_t*)pNum + pBlock->info.rows; + nRows = *(int64_t*)pNum; } if (nRows >= pInfo->mergeLimit) { @@ -3243,23 +3257,28 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { STsdbReader* reader = pInfo->base.dataReader; while (true) { - code = pAPI->tsdReader.tsdNextDataBlock(reader, &hasNext); - if (code != 0) { - pAPI->tsdReader.tsdReaderReleaseDataBlock(reader); - qError("table merge scan fetch next data block error code: %d, %s", code, GET_TASKID(pTaskInfo)); - T_LONG_JMP(pTaskInfo->env, code); - } + if (!pInfo->bOnlyRetrieveBlock) { + code = pAPI->tsdReader.tsdNextDataBlock(reader, &hasNext); + if (code != 0) { + pAPI->tsdReader.tsdReaderReleaseDataBlock(reader); + qError("table merge scan fetch next data block error code: %d, %s", code, GET_TASKID(pTaskInfo)); + T_LONG_JMP(pTaskInfo->env, code); + } - if (!hasNext) { - break; - } + if (!hasNext || isTaskKilled(pTaskInfo)) { + pInfo->bNewFileset = false; + if (isTaskKilled(pTaskInfo)) { + qInfo("table merge scan fetch next data block found task killed. %s", GET_TASKID(pTaskInfo)); + pAPI->tsdReader.tsdReaderReleaseDataBlock(reader); + } + break; + } - if (isTaskKilled(pTaskInfo)) { - qInfo("table merge scan fetch next data block found task killed. %s", GET_TASKID(pTaskInfo)); - pAPI->tsdReader.tsdReaderReleaseDataBlock(reader); - break; + if (pInfo->bNewFileset) { + pInfo->bOnlyRetrieveBlock = true; + return NULL; + } } - // process this data block based on the probabilities bool processThisBlock = processBlockWithProbability(&pInfo->sample); if (!processThisBlock) { @@ -3268,7 +3287,9 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { uint32_t status = 0; code = loadDataBlock(pOperator, &pInfo->base, pBlock, &status); - // code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, pBlock, &status); + if (pInfo->bOnlyRetrieveBlock) { + pInfo->bOnlyRetrieveBlock = false; + } if (code != TSDB_CODE_SUCCESS) { qInfo("table merge scan load datablock code %d, %s", code, GET_TASKID(pTaskInfo)); T_LONG_JMP(pTaskInfo->env, code); @@ -3291,7 +3312,7 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { pOperator->resultInfo.totalRows += pBlock->info.rows; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; - + return pBlock; } @@ -3327,6 +3348,60 @@ int32_t dumpQueryTableCond(const SQueryTableDataCond* src, SQueryTableDataCond* return 0; } +void tableMergeScanTsdbNotifyCb(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param) { + STableMergeScanInfo* pTmsInfo = param; + pTmsInfo->bNewFileset = true; + return; +} + +int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) { + STableMergeScanInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + int32_t code = TSDB_CODE_SUCCESS; + int32_t numOfTable = pInfo->tableEndIndex - pInfo->tableStartIndex + 1; + + pInfo->bNewFileset = false; + + pInfo->sortBufSize = 2048 * pInfo->bufPageSize; + int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage, + pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0); + + tsortSetMergeLimit(pInfo->pSortHandle, pInfo->mergeLimit); + tsortSetAbortCheckFn(pInfo->pSortHandle, isTaskKilled, pOperator->pTaskInfo); + + tsortSetFetchRawDataFp(pInfo->pSortHandle, getBlockForTableMergeScan, NULL, NULL); + + STableMergeScanSortSourceParam *param = taosMemoryCalloc(1, sizeof(STableMergeScanSortSourceParam)); + param->pOperator = pOperator; + + SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); + ps->param = param; + ps->onlyRef = false; + tsortAddSource(pInfo->pSortHandle, ps); + + if (numOfTable == 1) { + tsortSetSingleTableMerge(pInfo->pSortHandle); + } else { + code = tsortOpen(pInfo->pSortHandle); + } + return code; +} + +void stopDurationForGroupTableMergeScan(SOperatorInfo* pOperator) { + STableMergeScanInfo* pInfo = pOperator->info; + + SSortExecInfo sortExecInfo = tsortGetSortExecInfo(pInfo->pSortHandle); + pInfo->sortExecInfo.sortMethod = sortExecInfo.sortMethod; + pInfo->sortExecInfo.sortBuffer = sortExecInfo.sortBuffer; + pInfo->sortExecInfo.loops += sortExecInfo.loops; + pInfo->sortExecInfo.readBytes += sortExecInfo.readBytes; + pInfo->sortExecInfo.writeBytes += sortExecInfo.writeBytes; + + tsortDestroySortHandle(pInfo->pSortHandle); + pInfo->pSortHandle = NULL; +} + int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -3350,43 +3425,16 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { tSimpleHashClear(pInfo->mTableNumRows); - size_t szRow = blockDataGetRowSize(pInfo->pResBlock); -// if (pInfo->mergeLimit != -1) { -// pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, -// NULL, pTaskInfo->id.str, pInfo->mergeLimit, szRow+8, tsPQSortMemThreshold * 1024* 1024); -// } else - { - pInfo->sortBufSize = 2048 * pInfo->bufPageSize; - int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage, - pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0); - - tsortSetMergeLimit(pInfo->pSortHandle, pInfo->mergeLimit); - tsortSetAbortCheckFn(pInfo->pSortHandle, isTaskKilled, pOperator->pTaskInfo); - } - - tsortSetFetchRawDataFp(pInfo->pSortHandle, getBlockForTableMergeScan, NULL, NULL); - - // one table has one data block int32_t numOfTable = tableEndIdx - tableStartIdx + 1; - - STableMergeScanSortSourceParam *param = taosMemoryCalloc(1, sizeof(STableMergeScanSortSourceParam)); - param->pOperator = pOperator; STableKeyInfo* startKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, tableStartIdx); pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, &pInfo->base.cond, startKeyInfo, numOfTable, pInfo->pReaderBlock, (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), false, &pInfo->mSkipTables); - - SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); - ps->param = param; - ps->onlyRef = false; - tsortAddSource(pInfo->pSortHandle, ps); - - int32_t code = TSDB_CODE_SUCCESS; - if (numOfTable == 1) { - tsortSetSingleTableMerge(pInfo->pSortHandle); - } else { - code = tsortOpen(pInfo->pSortHandle); + if (pInfo->filesetDelimited) { + pAPI->tsdReader.tsdSetFilesetDelimited(pInfo->base.dataReader); } + pAPI->tsdReader.tsdSetSetNotifyCb(pInfo->base.dataReader, tableMergeScanTsdbNotifyCb, pInfo); + + int32_t code = startDurationForGroupTableMergeScan(pOperator); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, terrno); @@ -3400,21 +3448,13 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; - SSortExecInfo sortExecInfo = tsortGetSortExecInfo(pInfo->pSortHandle); - pInfo->sortExecInfo.sortMethod = sortExecInfo.sortMethod; - pInfo->sortExecInfo.sortBuffer = sortExecInfo.sortBuffer; - pInfo->sortExecInfo.loops += sortExecInfo.loops; - pInfo->sortExecInfo.readBytes += sortExecInfo.readBytes; - pInfo->sortExecInfo.writeBytes += sortExecInfo.writeBytes; + stopDurationForGroupTableMergeScan(pOperator); if (pInfo->base.dataReader != NULL) { pAPI->tsdReader.tsdReaderClose(pInfo->base.dataReader); pInfo->base.dataReader = NULL; } - tsortDestroySortHandle(pInfo->pSortHandle); - pInfo->pSortHandle = NULL; - resetLimitInfoForNextGroup(&pInfo->limitInfo); taosHashCleanup(pInfo->mSkipTables); pInfo->mSkipTables = NULL; @@ -3497,17 +3537,22 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { pOperator->resultInfo.totalRows += pBlock->info.rows; return pBlock; } else { - // Data of this group are all dumped, let's try the next group - stopGroupTableMergeScan(pOperator); - if (pInfo->tableEndIndex >= tableListSize - 1) { - setOperatorCompleted(pOperator); - break; - } + if (pInfo->bNewFileset) { + stopDurationForGroupTableMergeScan(pOperator); + startDurationForGroupTableMergeScan(pOperator); + } else { + // Data of this group are all dumped, let's try the next group + stopGroupTableMergeScan(pOperator); + if (pInfo->tableEndIndex >= tableListSize - 1) { + setOperatorCompleted(pOperator); + break; + } - pInfo->tableStartIndex = pInfo->tableEndIndex + 1; - pInfo->groupId = tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex)->groupId; - startGroupTableMergeScan(pOperator); - resetLimitInfoForNextGroup(&pInfo->limitInfo); + pInfo->tableStartIndex = pInfo->tableEndIndex + 1; + pInfo->groupId = tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex)->groupId; + startGroupTableMergeScan(pOperator); + resetLimitInfoForNextGroup(&pInfo->limitInfo); + } } } @@ -3633,6 +3678,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN uint32_t nCols = taosArrayGetSize(pInfo->pResBlock->pDataBlock); pInfo->bufPageSize = getProperSortPageSize(rowSize, nCols); + pInfo->filesetDelimited = pTableScanNode->filesetDelimited; setOperatorInfo(pOperator, "TableMergeScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 9b987ff1a4..914b95ba83 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -195,6 +195,10 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW pWinInfo->pWinFlag->endFlag = ends[i]; } else if (pWin->ekey == pTsData[i]) { pWinInfo->pWinFlag->endFlag |= ends[i]; + } else { + *pRebuild = true; + pWinInfo->pWinFlag->endFlag |= ends[i]; + return i + 1 - start; } memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey)); @@ -297,7 +301,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl int32_t rows = pSDataBlock->info.rows; blockDataEnsureCapacity(pAggSup->pScanBlock, rows); for (int32_t i = 0; i < rows; i += winRows) { - if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) { + if (pInfo->ignoreExpiredData && checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo, + &pInfo->twAggSup, pSDataBlock->info.id.uid, tsCols[i])) { i++; continue; } @@ -318,6 +323,9 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); doDeleteEventWindow(pAggSup, pSeUpdated, &curWin.winInfo.sessionWin); releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore); + SSessionKey tmpSeInfo = {0}; + getSessionHashKey(&curWin.winInfo.sessionWin, &tmpSeInfo); + tSimpleHashPut(pStDeleted, &tmpSeInfo, sizeof(SSessionKey), NULL, 0); continue; } code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput, @@ -610,6 +618,13 @@ void streamEventReloadState(SOperatorInfo* pOperator) { compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, false); qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey, curInfo.winInfo.sessionWin.groupId); + if (IS_VALID_SESSION_WIN(curInfo.winInfo)) { + saveSessionOutputBuf(pAggSup, &curInfo.winInfo); + } + + if (!curInfo.pWinFlag->endFlag) { + continue; + } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { saveResult(curInfo.winInfo, pInfo->pSeUpdated); @@ -621,10 +636,6 @@ void streamEventReloadState(SOperatorInfo* pOperator) { getSessionHashKey(&curInfo.winInfo.sessionWin, &key); tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curInfo.winInfo, sizeof(SResultWindowInfo)); } - - if (IS_VALID_SESSION_WIN(curInfo.winInfo)) { - saveSessionOutputBuf(pAggSup, &curInfo.winInfo); - } } taosMemoryFree(pBuf); diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 4a88c160e6..fe19700cc9 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -276,15 +276,15 @@ static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SSHashObj* resWins) { void* pIte = NULL; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { - SWinKey* pKey = tSimpleHashGetKey(pIte, NULL); - uint64_t groupId = pKey->groupId; - TSKEY ts = pKey->ts; + SWinKey* pKey = tSimpleHashGetKey(pIte, NULL); + uint64_t groupId = pKey->groupId; + TSKEY ts = pKey->ts; SRowBuffPos* pPos = *(SRowBuffPos**)pIte; if (!pPos->beUpdated) { continue; } pPos->beUpdated = false; - int32_t code = saveWinResultInfo(ts, groupId, pPos, resWins); + int32_t code = saveWinResultInfo(ts, groupId, pPos, resWins); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -450,6 +450,7 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamInt pScanInfo->interval = pInfo->interval; pScanInfo->twAggSup = pInfo->twAggSup; pScanInfo->pState = pInfo->pState; + pInfo->pUpdateInfo = pScanInfo->pUpdateInfo; } void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput, @@ -800,7 +801,9 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat } while (1) { bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup); - if ((pInfo->ignoreExpiredData && isClosed && !IS_FINAL_INTERVAL_OP(pOperator)) || + if ((!IS_FINAL_INTERVAL_OP(pOperator) && pInfo->ignoreExpiredData && + checkExpiredData(&pInfo->stateStore, pInfo->pUpdateInfo, &pInfo->twAggSup, pSDataBlock->info.id.uid, + nextWin.ekey)) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) { startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin); if (startPos < 0) { @@ -1091,10 +1094,10 @@ void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera int32_t mapSize = 0; buf = taosDecodeFixedI32(buf, &mapSize); for (int32_t i = 0; i < mapSize; i++) { - SWinKey key = {0}; + SWinKey key = {0}; buf = decodeSWinKey(buf, &key); SRowBuffPos* pPos = NULL; - int32_t resSize = pInfo->aggSup.resultRowSize; + int32_t resSize = pInfo->aggSup.resultRowSize; pInfo->stateStore.streamStateAddIfNotExist(pInfo->pState, &key, (void**)&pPos, &resSize); tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pPos, POINTER_BYTES); } @@ -1165,7 +1168,7 @@ static SSDataBlock* buildIntervalResult(SOperatorInfo* pOperator) { return NULL; } -int32_t copyUpdateResult(SSHashObj** ppWinUpdated, SArray* pUpdated, __compar_fn_t compar) { +int32_t copyUpdateResult(SSHashObj** ppWinUpdated, SArray* pUpdated, __compar_fn_t compar) { void* pIte = NULL; int32_t iter = 0; while ((pIte = tSimpleHashIterate(*ppWinUpdated, pIte, &iter)) != NULL) { @@ -1404,10 +1407,12 @@ void streamIntervalReloadState(SOperatorInfo* pOperator) { void* pBuf = NULL; int32_t code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME, strlen(STREAM_INTERVAL_OP_STATE_NAME), &pBuf, &size); - TSKEY ts = *(TSKEY*)pBuf; - taosMemoryFree(pBuf); - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); - pInfo->stateStore.streamStateReloadInfo(pInfo->pState, ts); + if (code == 0) { + TSKEY ts = *(TSKEY*)pBuf; + taosMemoryFree(pBuf); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); + pInfo->stateStore.streamStateReloadInfo(pInfo->pState, ts); + } } SOperatorInfo* downstream = pOperator->pDownstream[0]; if (downstream->fpSet.reloadStreamStateFn) { @@ -1624,6 +1629,7 @@ void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uin pScanInfo->igCheckUpdate); } pScanInfo->twAggSup = *pTwSup; + pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo; } static TSKEY sesionTs(void* pKey) { @@ -1726,8 +1732,8 @@ void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT pCurWin->sessionWin.win.skey = startTs; pCurWin->sessionWin.win.ekey = endTs; int32_t size = pAggSup->resultRowSize; - int32_t code = pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, - pAggSup->gap, (void**)&pCurWin->pStatePos, &size); + int32_t code = pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, + pAggSup->gap, (void**)&pCurWin->pStatePos, &size); if (code == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->sessionWin.win)) { code = TSDB_CODE_FAILED; clearOutputBuf(pAggSup->pState, pCurWin->pStatePos, &pAggSup->pSessionAPI->stateStore); @@ -1825,9 +1831,9 @@ void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SAr } } -int32_t updateSessionWindowInfo(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t groupId, - int32_t rows, int32_t start, int64_t gap, SSHashObj* pResultRows, SSHashObj* pStUpdated, - SSHashObj* pStDeleted) { +int32_t updateSessionWindowInfo(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo, TSKEY* pStartTs, + TSKEY* pEndTs, uint64_t groupId, int32_t rows, int32_t start, int64_t gap, + SSHashObj* pResultRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted) { for (int32_t i = start; i < rows; ++i) { if (!isInWindow(pWinInfo, pStartTs[i], gap) && (!pEndTs || !isInWindow(pWinInfo, pEndTs[i], gap))) { return i - start; @@ -1859,8 +1865,8 @@ static int32_t initSessionOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pR } int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindowInfo* pCurWin, SResultRow** pResult, - int32_t startIndex, int32_t winRows, int32_t rows, int32_t numOutput, - SOperatorInfo* pOperator, int64_t winDelta) { + int32_t startIndex, int32_t winRows, int32_t rows, int32_t numOutput, + SOperatorInfo* pOperator, int64_t winDelta) { SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; int32_t code = initSessionOutputBuf(pCurWin, pResult, pSup->pCtx, numOutput, pSup->rowEntryInfoOffset); @@ -1984,9 +1990,10 @@ static void compactSessionSemiWindow(SOperatorInfo* pOperator, SResultWindowInfo } int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo) { - qDebug("===stream===try save session result skey:%" PRId64 ", ekey:%" PRId64 ".pos%d", - pWinInfo->sessionWin.win.skey, pWinInfo->sessionWin.win.ekey, pWinInfo->pStatePos->needFree); - return pAggSup->stateStore.streamStateSessionPut(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pStatePos, pAggSup->resultRowSize); + qDebug("===stream===try save session result skey:%" PRId64 ", ekey:%" PRId64 ".pos%d", pWinInfo->sessionWin.win.skey, + pWinInfo->sessionWin.win.ekey, pWinInfo->pStatePos->needFree); + return pAggSup->stateStore.streamStateSessionPut(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pStatePos, + pAggSup->resultRowSize); } static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pStUpdated, @@ -2018,7 +2025,9 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData TSKEY* endTsCols = (int64_t*)pEndTsCol->pData; for (int32_t i = 0; i < rows;) { - if (pInfo->ignoreExpiredData && isOverdue(endTsCols[i], &pInfo->twAggSup)) { + if (!IS_FINAL_SESSION_OP(pOperator) && pInfo->ignoreExpiredData && + checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo, &pInfo->twAggSup, + pSDataBlock->info.id.uid, endTsCols[i])) { i++; continue; } @@ -2048,7 +2057,8 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) { code = saveResult(winInfo, pStUpdated); if (code != TSDB_CODE_SUCCESS) { - qError("%s do stream session aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); + qError("%s do stream session aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), + tstrerror(code)); T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } } @@ -2087,8 +2097,8 @@ void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArr inline int32_t sessionKeyCompareAsc(const void* pKey1, const void* pKey2) { SResultWindowInfo* pWinInfo1 = (SResultWindowInfo*)pKey1; SResultWindowInfo* pWinInfo2 = (SResultWindowInfo*)pKey2; - SSessionKey* pWin1 = &pWinInfo1->sessionWin; - SSessionKey* pWin2 = &pWinInfo2->sessionWin; + SSessionKey* pWin1 = &pWinInfo1->sessionWin; + SSessionKey* pWin2 = &pWinInfo2->sessionWin; if (pWin1->groupId > pWin2->groupId) { return 1; @@ -2293,9 +2303,9 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) { SResultWindowInfo* pWinInfo = taosArrayGet(pGroupResInfo->pRows, i); - SRowBuffPos* pPos = pWinInfo->pStatePos; - SResultRow* pRow = NULL; - SSessionKey* pKey = (SSessionKey*) pPos->pKey; + SRowBuffPos* pPos = pWinInfo->pStatePos; + SResultRow* pRow = NULL; + SSessionKey* pKey = (SSessionKey*)pPos->pKey; if (pBlock->info.id.groupId == 0) { pBlock->info.id.groupId = pKey->groupId; @@ -2315,7 +2325,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa } } - int32_t code = pAPI->stateStore.streamStateGetByPos(pState, pPos, (void**)&pRow); + int32_t code = pAPI->stateStore.streamStateGetByPos(pState, pPos, (void**)&pRow); if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { ASSERT(pBlock->info.rows > 0); break; @@ -2328,7 +2338,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa pGroupResInfo->index += 1; continue; } - + doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); // no results, continue to check the next one if (pRow->numOfRows == 0) { @@ -2412,7 +2422,7 @@ void getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins) { return; } SResultWindowInfo* pWinInfo = taosArrayGet(pAllWins, size - 1); - SSessionKey* pSeKey = pWinInfo->pStatePos->pKey; + SSessionKey* pSeKey = pWinInfo->pStatePos->pKey; taosArrayPush(pMaxWins, pSeKey); if (pSeKey->groupId == 0) { return; @@ -2719,7 +2729,8 @@ void resetWinRange(STimeWindow* winRange) { void getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SResultWindowInfo* pWinInfo) { int32_t rowSize = pAggSup->resultRowSize; - int32_t code = pAggSup->stateStore.streamStateSessionGet(pAggSup->pState, pKey, (void**)&pWinInfo->pStatePos, &rowSize); + int32_t code = + pAggSup->stateStore.streamStateSessionGet(pAggSup->pState, pKey, (void**)&pWinInfo->pStatePos, &rowSize); if (code == TSDB_CODE_SUCCESS) { pWinInfo->sessionWin = *pKey; pWinInfo->isOutput = true; @@ -2733,16 +2744,16 @@ void getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, void streamSessionSemiReloadState(SOperatorInfo* pOperator) { SStreamSessionAggOperatorInfo* pInfo = pOperator->info; - SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; resetWinRange(&pAggSup->winRange); SResultWindowInfo winInfo = {0}; - int32_t size = 0; - void* pBuf = NULL; - int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, + int32_t size = 0; + void* pBuf = NULL; + int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); - int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); - SSessionKey* pSeKeyBuf = (SSessionKey*) pBuf; + int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); + SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); for (int32_t i = 0; i < num; i++) { SResultWindowInfo winInfo = {0}; @@ -2766,12 +2777,12 @@ void streamSessionReloadState(SOperatorInfo* pOperator) { SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; resetWinRange(&pAggSup->winRange); - int32_t size = 0; - void* pBuf = NULL; - int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, - strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); - int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); - SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; + int32_t size = 0; + void* pBuf = NULL; + int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, + strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); + int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); + SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY)); @@ -2890,7 +2901,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->recvGetAll = false; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; - // for stream + // for stream void* buff = NULL; int32_t len = 0; int32_t res = @@ -2927,7 +2938,8 @@ static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) { pInfo->streamAggSup.stateStore.streamStateSessionClear(pInfo->streamAggSup.pState); } -void deleteSessionWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate, SSHashObj* pMapDelete) { +void deleteSessionWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate, + SSHashObj* pMapDelete) { SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); doDeleteTimeWindows(pAggSup, pBlock, pWins); removeSessionResults(pAggSup, pMapUpdate, pWins); @@ -3026,7 +3038,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc); removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated); - if(pInfo->isHistoryOp) { + if (pInfo->isHistoryOp) { getMaxTsWins(pInfo->pUpdated, pInfo->historyWins); } @@ -3060,8 +3072,9 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream pOperator->operatorType = pPhyNode->type; if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { - pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionSemiAgg, NULL, - destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = + createOperatorFpSet(optrDummyOpenFn, doStreamSessionSemiAgg, NULL, destroyStreamSessionAggOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionSemiReloadState); } setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), pPhyNode->type, false, OP_NOT_OPENED, pInfo, @@ -3177,7 +3190,7 @@ void getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SS pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin); int32_t nextSize = pAggSup->resultRowSize; int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, - (void**)&pNextWin->winInfo.pStatePos, &nextSize); + (void**)&pNextWin->winInfo.pStatePos, &nextSize); if (code != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_INVALID(pNextWin->winInfo); } else { @@ -3190,8 +3203,8 @@ void getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SS pNextWin->winInfo.isOutput = true; } pAggSup->stateStore.streamStateFreeCur(pCur); - qDebug("===stream===get state next win buff. skey:%" PRId64 ", endkey:%" PRId64, pNextWin->winInfo.sessionWin.win.skey, - pNextWin->winInfo.sessionWin.win.ekey); + qDebug("===stream===get state next win buff. skey:%" PRId64 ", endkey:%" PRId64, + pNextWin->winInfo.sessionWin.win.skey, pNextWin->winInfo.sessionWin.win.ekey); } void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData, @@ -3260,13 +3273,13 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, pNextWin->winInfo.isOutput = true; } pAggSup->stateStore.streamStateFreeCur(pCur); - qDebug("===stream===set state next win buff. skey:%" PRId64 ", endkey:%" PRId64, pNextWin->winInfo.sessionWin.win.skey, - pNextWin->winInfo.sessionWin.win.ekey); + qDebug("===stream===set state next win buff. skey:%" PRId64 ", endkey:%" PRId64, + pNextWin->winInfo.sessionWin.win.skey, pNextWin->winInfo.sessionWin.win.ekey); } -int32_t updateStateWindowInfo(SStreamAggSupporter* pAggSup, SStateWindowInfo* pWinInfo, SStateWindowInfo* pNextWin, TSKEY* pTs, uint64_t groupId, - SColumnInfoData* pKeyCol, int32_t rows, int32_t start, bool* allEqual, - SSHashObj* pResultRows, SSHashObj* pSeUpdated, SSHashObj* pSeDeleted) { +int32_t updateStateWindowInfo(SStreamAggSupporter* pAggSup, SStateWindowInfo* pWinInfo, SStateWindowInfo* pNextWin, + TSKEY* pTs, uint64_t groupId, SColumnInfoData* pKeyCol, int32_t rows, int32_t start, + bool* allEqual, SSHashObj* pResultRows, SSHashObj* pSeUpdated, SSHashObj* pSeDeleted) { *allEqual = true; for (int32_t i = start; i < rows; ++i) { char* pKeyData = colDataGetData(pKeyCol, i); @@ -3330,7 +3343,8 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl blockDataEnsureCapacity(pAggSup->pScanBlock, rows); SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId); for (int32_t i = 0; i < rows; i += winRows) { - if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup) || colDataIsNull_s(pKeyColInfo, i)) { + if (pInfo->ignoreExpiredData && checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo, + &pInfo->twAggSup, pSDataBlock->info.id.uid, tsCols[i]) || colDataIsNull_s(pKeyColInfo, i)) { i++; continue; } @@ -3341,7 +3355,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl SStateWindowInfo nextWin = {0}; setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin); releaseOutputBuf(pAggSup->pState, nextWin.winInfo.pStatePos, &pAPI->stateStore); - + setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); winRows = updateStateWindowInfo(pAggSup, &curWin, &nextWin, tsCols, groupId, pKeyColInfo, rows, i, &allEqual, pAggSup->pResultRows, pSeUpdated, pStDeleted); @@ -3478,6 +3492,7 @@ void doStreamStateSaveCheckpoint(SOperatorInfo* pOperator) { len = doStreamStateEncodeOpState(&pBuf, len, pOperator, true); pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME, strlen(STREAM_STATE_OP_CHECKPOINT_NAME), buf, len); + taosMemoryFree(buf); } static SSDataBlock* buildStateResult(SOperatorInfo* pOperator) { @@ -3617,10 +3632,11 @@ void streamStateReleaseState(SOperatorInfo* pOperator) { static void compactStateWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin, SSHashObj* pStUpdated, SSHashObj* pStDeleted) { - SExprSupp* pSup = &pOperator->exprSupp; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExprSupp* pSup = &pOperator->exprSupp; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamStateAggOperatorInfo* pInfo = pOperator->info; - compactTimeWindow(pSup, &pInfo->streamAggSup, &pInfo->twAggSup, pTaskInfo, pCurWin, pNextWin, pStUpdated, pStDeleted, false); + compactTimeWindow(pSup, &pInfo->streamAggSup, &pInfo->twAggSup, pTaskInfo, pCurWin, pNextWin, pStUpdated, pStDeleted, + false); } void streamStateReloadState(SOperatorInfo* pOperator) { @@ -3632,7 +3648,7 @@ void streamStateReloadState(SOperatorInfo* pOperator) { int32_t size = 0; void* pBuf = NULL; int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_OP_STATE_NAME, - strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size); + strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size); int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); qDebug("===stream=== reload state. get result count:%d", num); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; @@ -4013,8 +4029,9 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamIntervalAgg, NULL, - destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = + createOperatorFpSet(optrDummyOpenFn, doStreamIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState); pInfo->stateStore = pTaskInfo->storageAPI.stateStore; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index db7c5e2570..0ca91f74ad 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -899,6 +899,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI SWindowRowsSup* pRowSup = &pInfo->winSup; pRowSup->numOfRows = 0; + pRowSup->startRowIndex = 0; struct SColumnDataAgg* pAgg = NULL; for (int32_t j = 0; j < pBlock->info.rows; ++j) { @@ -923,9 +924,6 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI doKeepTuple(pRowSup, tsList[j], gid); } else if (compareVal(val, &pInfo->stateKey)) { doKeepTuple(pRowSup, tsList[j], gid); - if (j == 0 && pRowSup->startRowIndex != 0) { - pRowSup->startRowIndex = 0; - } } else { // a new state window started SResultRow* pResult = NULL; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index fbabef8fd3..98fda024fa 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -3425,7 +3425,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_wstart", .type = FUNCTION_TYPE_WSTART, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_SKIP_SCAN_CHECK_FUNC, .translateFunc = translateTimePseudoColumn, .getEnvFunc = getTimePseudoFuncEnv, .initFunc = NULL, @@ -3435,7 +3435,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_wend", .type = FUNCTION_TYPE_WEND, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_SKIP_SCAN_CHECK_FUNC, .translateFunc = translateTimePseudoColumn, .getEnvFunc = getTimePseudoFuncEnv, .initFunc = NULL, @@ -3445,7 +3445,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_wduration", .type = FUNCTION_TYPE_WDURATION, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_SKIP_SCAN_CHECK_FUNC, .translateFunc = translateWduration, .getEnvFunc = getTimePseudoFuncEnv, .initFunc = NULL, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 071afe0159..25a3c509a7 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -5472,7 +5472,6 @@ int32_t blockDistFunction(SqlFunctionCtx* pCtx) { SColumnInfoData* pInputCol = pInput->pData[0]; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - STableBlockDistInfo* pDistInfo = GET_ROWCELL_INTERBUF(pResInfo); STableBlockDistInfo p1 = {0}; @@ -5481,6 +5480,7 @@ int32_t blockDistFunction(SqlFunctionCtx* pCtx) { pDistInfo->numOfBlocks += p1.numOfBlocks; pDistInfo->numOfTables += p1.numOfTables; pDistInfo->numOfInmemRows += p1.numOfInmemRows; + pDistInfo->numOfSttRows += p1.numOfSttRows; pDistInfo->totalSize += p1.totalSize; pDistInfo->totalRows += p1.totalRows; pDistInfo->numOfFiles += p1.numOfFiles; @@ -5488,7 +5488,6 @@ int32_t blockDistFunction(SqlFunctionCtx* pCtx) { pDistInfo->defMinRows = p1.defMinRows; pDistInfo->defMaxRows = p1.defMaxRows; pDistInfo->rowSize = p1.rowSize; - pDistInfo->numOfSmallBlocks = p1.numOfSmallBlocks; if (pDistInfo->minRows > p1.minRows) { pDistInfo->minRows = p1.minRows; @@ -5523,7 +5522,7 @@ int32_t tSerializeBlockDistInfo(void* buf, int32_t bufLen, const STableBlockDist if (tEncodeI32(&encoder, pInfo->defMaxRows) < 0) return -1; if (tEncodeI32(&encoder, pInfo->defMinRows) < 0) return -1; if (tEncodeU32(&encoder, pInfo->numOfInmemRows) < 0) return -1; - if (tEncodeU32(&encoder, pInfo->numOfSmallBlocks) < 0) return -1; + if (tEncodeU32(&encoder, pInfo->numOfSttRows) < 0) return -1; if (tEncodeU32(&encoder, pInfo->numOfVgroups) < 0) return -1; for (int32_t i = 0; i < tListLen(pInfo->blockRowsHisto); ++i) { @@ -5555,7 +5554,7 @@ int32_t tDeserializeBlockDistInfo(void* buf, int32_t bufLen, STableBlockDistInfo if (tDecodeI32(&decoder, &pInfo->defMaxRows) < 0) return -1; if (tDecodeI32(&decoder, &pInfo->defMinRows) < 0) return -1; if (tDecodeU32(&decoder, &pInfo->numOfInmemRows) < 0) return -1; - if (tDecodeU32(&decoder, &pInfo->numOfSmallBlocks) < 0) return -1; + if (tDecodeU32(&decoder, &pInfo->numOfSttRows) < 0) return -1; if (tDecodeU32(&decoder, &pInfo->numOfVgroups) < 0) return -1; for (int32_t i = 0; i < tListLen(pInfo->blockRowsHisto); ++i) { @@ -5589,7 +5588,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } int32_t len = sprintf(st + VARSTR_HEADER_SIZE, - "Total_Blocks=[%d] Total_Size=[%.2f KB] Average_size=[%.2f KB] Compression_Ratio=[%.2f %c]", + "Total_Blocks=[%d] Total_Size=[%.2f KiB] Average_size=[%.2f KiB] Compression_Ratio=[%.2f %c]", pData->numOfBlocks, pData->totalSize / 1024.0, averageSize / 1024.0, compRatio, '%'); varDataSetLen(st, len); @@ -5600,14 +5599,16 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { avgRows = pData->totalRows / pData->numOfBlocks; } - len = sprintf(st + VARSTR_HEADER_SIZE, - "Total_Rows=[%" PRId64 "] Inmem_Rows=[%d] MinRows=[%d] MaxRows=[%d] Average_Rows=[%" PRId64 "]", - pData->totalRows, pData->numOfInmemRows, pData->minRows, pData->maxRows, avgRows); - + len = sprintf(st + VARSTR_HEADER_SIZE, "Block_Rows=[%" PRId64 "] MinRows=[%d] MaxRows=[%d] AvgRows=[%" PRId64 "]", + pData->totalRows, pData->minRows, pData->maxRows, avgRows); varDataSetLen(st, len); colDataSetVal(pColInfo, row++, st, false); - len = sprintf(st + VARSTR_HEADER_SIZE, "Total_Tables=[%d] Total_Files=[%d] Total_Vgroups=[%d]", pData->numOfTables, + len = sprintf(st + VARSTR_HEADER_SIZE, "Inmem_Rows=[%d] Stt_Rows=[%d] ", pData->numOfInmemRows, pData->numOfSttRows); + varDataSetLen(st, len); + colDataSetVal(pColInfo, row++, st, false); + + len = sprintf(st + VARSTR_HEADER_SIZE, "Total_Tables=[%d] Total_Filesets=[%d] Total_Vgroups=[%d]", pData->numOfTables, pData->numOfFiles, pData->numOfVgroups); varDataSetLen(st, len); diff --git a/source/libs/geometry/src/geomFunc.c b/source/libs/geometry/src/geomFunc.c index 3588bf8b7d..2ac1761737 100644 --- a/source/libs/geometry/src/geomFunc.c +++ b/source/libs/geometry/src/geomFunc.c @@ -67,15 +67,19 @@ int32_t doGeomFromTextFunc(const char *input, unsigned char **output) { return TSDB_CODE_SUCCESS; } - // make input as a zero ending string - char *end = varDataVal(input) + varDataLen(input); - char endValue = *end; - *end = 0; - + char *inputGeom = NULL; unsigned char *outputGeom = NULL; size_t size = 0; - code = doGeomFromText(varDataVal(input), &outputGeom, &size); + // make a zero ending string + inputGeom = taosMemoryCalloc(1, varDataLen(input) + 1); + if (inputGeom == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(inputGeom, varDataVal(input), varDataLen(input)); + + code = doGeomFromText(inputGeom, &outputGeom, &size); if (code != TSDB_CODE_SUCCESS) { goto _exit; } @@ -92,8 +96,7 @@ int32_t doGeomFromTextFunc(const char *input, unsigned char **output) { _exit: geosFreeBuffer(outputGeom); - - *end = endValue; //recover the input string + geosFreeBuffer(inputGeom); return code; } diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 785f0bc70d..e40c470dde 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -423,6 +423,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_SCALAR_FIELD(igLastNull); COPY_SCALAR_FIELD(groupOrderScan); COPY_SCALAR_FIELD(onlyMetaCtbIdx); + COPY_SCALAR_FIELD(filesetDelimited); return TSDB_CODE_SUCCESS; } @@ -650,6 +651,7 @@ static int32_t physiTableScanCopy(const STableScanPhysiNode* pSrc, STableScanPhy COPY_SCALAR_FIELD(triggerType); COPY_SCALAR_FIELD(watermark); COPY_SCALAR_FIELD(igExpired); + COPY_SCALAR_FIELD(filesetDelimited); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 47a74a3785..0041cd4fa0 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -679,6 +679,7 @@ static const char* jkScanLogicPlanDataRequired = "DataRequired"; static const char* jkScanLogicPlanTagCond = "TagCond"; static const char* jkScanLogicPlanGroupTags = "GroupTags"; static const char* jkScanLogicPlanOnlyMetaCtbIdx = "OnlyMetaCtbIdx"; +static const char* jkScanLogicPlanFilesetDelimited = "FilesetDelimited"; static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { const SScanLogicNode* pNode = (const SScanLogicNode*)pObj; @@ -723,6 +724,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkScanLogicPlanOnlyMetaCtbIdx, pNode->onlyMetaCtbIdx); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkScanLogicPlanFilesetDelimited, pNode->filesetDelimited); + } return code; } @@ -770,7 +774,9 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkScanLogicPlanOnlyMetaCtbIdx, &pNode->onlyMetaCtbIdx); } - + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkScanLogicPlanFilesetDelimited, &pNode->filesetDelimited); + } return code; } @@ -1832,6 +1838,7 @@ static const char* jkTableScanPhysiPlanTags = "Tags"; static const char* jkTableScanPhysiPlanSubtable = "Subtable"; static const char* jkTableScanPhysiPlanAssignBlockUid = "AssignBlockUid"; static const char* jkTableScanPhysiPlanIgnoreUpdate = "IgnoreUpdate"; +static const char* jkTableScanPhysiPlanFilesetDelimited = "FilesetDelimited"; static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj; @@ -1900,6 +1907,9 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanIgnoreUpdate, pNode->igCheckUpdate); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanFilesetDelimited, pNode->filesetDelimited); + } return code; } @@ -1971,6 +1981,9 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetTinyIntValue(pJson, jkTableScanPhysiPlanIgnoreUpdate, &pNode->igCheckUpdate); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanFilesetDelimited, &pNode->filesetDelimited); + } return code; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 48ebd8d886..a2ea8a7e59 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2167,7 +2167,9 @@ static int32_t physiTableScanNodeInlineToMsg(const void* pObj, STlvEncoder* pEnc if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeValueI8(pEncoder, pNode->igCheckUpdate); } - + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueBool(pEncoder, pNode->filesetDelimited); + } return code; } @@ -2246,6 +2248,9 @@ static int32_t msgToPhysiTableScanNodeInline(STlvDecoder* pDecoder, void* pObj) if (TSDB_CODE_SUCCESS == code) { code = tlvDecodeValueI8(pDecoder, &pNode->igCheckUpdate); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvDecodeValueBool(pDecoder, &pNode->filesetDelimited); + } return code; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 22417086fe..f70cb3866e 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -889,6 +889,7 @@ void nodesDestroyNode(SNode* pNode) { taosHashCleanup(pStmt->pSubTableHashObj); taosHashCleanup(pStmt->pTableNameHashObj); taosHashCleanup(pStmt->pDbFNameHashObj); + taosHashCleanup(pStmt->pTableCxtHashObj); if (pStmt->freeHashFunc) { pStmt->freeHashFunc(pStmt->pTableBlockHashObj); } diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index ce8c2d8a3d..b20587dd43 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -50,7 +50,7 @@ void insCheckTableDataOrder(STableDataCxt *pTableCxt, TSKEY tsKey); int32_t insGetTableDataCxt(SHashObj *pHash, void *id, int32_t idLen, STableMeta *pTableMeta, SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode, bool ignoreColVals); int32_t initTableColSubmitData(STableDataCxt *pTableCxt); -int32_t insMergeTableDataCxt(SHashObj *pTableHash, SArray **pVgDataBlocks); +int32_t insMergeTableDataCxt(SHashObj *pTableHash, SArray **pVgDataBlocks, bool isRebuild); int32_t insBuildVgDataBlocks(SHashObj *pVgroupsHashObj, SArray *pVgDataBlocks, SArray **pDataBlocks); void insDestroyTableDataCxtHashMap(SHashObj *pTableCxtHash); void insDestroyVgroupDataCxt(SVgroupDataCxt *pVgCxt); diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index f2194402da..2dbba38212 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -425,7 +425,7 @@ SQuery* smlInitHandle() { int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) { SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)(handle)->pRoot; // merge according to vgId - int32_t code = insMergeTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks); + int32_t code = insMergeTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks, true); if (code != TSDB_CODE_SUCCESS) { uError("insMergeTableDataCxt failed"); return code; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 2b8516d37b..1994ddb437 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -55,6 +55,7 @@ typedef struct SInsertParseContext { bool usingDuplicateTable; bool forceUpdate; bool needTableTagVal; + bool needRequest; // whether or not request server } SInsertParseContext; typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param); @@ -267,26 +268,47 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, E return code; } -static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t* time, SMsgBuf* pMsgBuf) { - int32_t index = 0; - int64_t interval; - int64_t ts = 0; - const char* pTokenEnd = *end; - +static int parseTimestampOrInterval(const char** end, SToken* pToken, int16_t timePrec, int64_t* ts, int64_t* interval, SMsgBuf* pMsgBuf, bool* isTs) { if (pToken->type == TK_NOW) { - ts = taosGetTimestamp(timePrec); + *isTs = true; + *ts = taosGetTimestamp(timePrec); } else if (pToken->type == TK_TODAY) { - ts = taosGetTimestampToday(timePrec); + *isTs = true; + *ts = taosGetTimestampToday(timePrec); } else if (pToken->type == TK_NK_INTEGER) { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &ts)) { + *isTs = true; + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, ts)) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); } + } else if (pToken->type == TK_NK_VARIABLE) { + char unit = 0; + *isTs = false; + if (parseAbsoluteDuration(pToken->z, pToken->n, interval, &unit, timePrec) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } } else { // parse the RFC-3339/ISO-8601 timestamp format string - if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { + *isTs = true; + if (taosParseTime(pToken->z, ts, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); } + } - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; +} + +static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t* time, SMsgBuf* pMsgBuf) { + int32_t index = 0, i = 0; + int64_t interval = 0, tempInterval = 0; + int64_t ts = 0, tempTs = 0; + bool firstIsTS = false, secondIsTs = false; + const char* pTokenEnd = *end; + + if (TSDB_CODE_SUCCESS != parseTimestampOrInterval(&pTokenEnd, pToken, timePrec, &ts, &interval, pMsgBuf, &firstIsTS)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } + + if (firstIsTS) { + *time = ts; } for (int k = pToken->n; pToken->z[k] != '\0'; k++) { @@ -298,45 +320,98 @@ static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t } if (pToken->z[k] == ',') { *end = pTokenEnd; + if (!firstIsTS) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } *time = ts; - return 0; + return TSDB_CODE_SUCCESS; } - break; } - /* - * time expression: - * e.g., now+12a, now-5h - */ + while (pTokenEnd[i] != '\0') { + if (pTokenEnd[i] == ' ' || pTokenEnd[i] == '\t') { + i++; + continue; + } + else if (pTokenEnd[i] == ',' || pTokenEnd[i] == ')') { + *end = pTokenEnd + i; + if (!firstIsTS) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } + *time = ts; + return TSDB_CODE_SUCCESS; + } else { + break; + } + } + pTokenEnd = pTokenEnd + i; + index = 0; SToken token = tStrGetToken(pTokenEnd, &index, false, NULL); - pTokenEnd += index; if (token.type == TK_NK_MINUS || token.type == TK_NK_PLUS) { + pTokenEnd += index; index = 0; SToken valueToken = tStrGetToken(pTokenEnd, &index, false, NULL); pTokenEnd += index; + char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW]; + if (TK_NK_STRING == valueToken.type) { + if (valueToken.n >= TSDB_MAX_BYTES_PER_ROW) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", valueToken.z); + } + int32_t len = trimString(valueToken.z, valueToken.n, tmpTokenBuf, TSDB_MAX_BYTES_PER_ROW); + valueToken.z = tmpTokenBuf; + valueToken.n = len; + } + + if (TSDB_CODE_SUCCESS != parseTimestampOrInterval(&pTokenEnd, &valueToken, timePrec, &tempTs, &tempInterval, pMsgBuf, &secondIsTs)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } if (valueToken.n < 2) { return buildSyntaxErrMsg(pMsgBuf, "value expected in timestamp", token.z); } - char unit = 0; - if (parseAbsoluteDuration(valueToken.z, valueToken.n, &interval, &unit, timePrec) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; + if (secondIsTs) { + // not support operator between tow timestamp, such as today() + now() + if (firstIsTS) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } + ts = tempTs; + }else { + // not support operator between tow interval, such as 2h + 3s + if (!firstIsTS) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } + interval = tempInterval; } - - if (token.type == TK_NK_PLUS) { - ts += interval; + if (token.type == TK_NK_MINUS) { + // not support interval - ts,such as 2h - today() + if (secondIsTs) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } + *time = ts - interval; } else { - ts = ts - interval; + *time = ts + interval; } - *end = pTokenEnd; + for (int k = valueToken.n; valueToken.z[k] != '\0'; k++) { + if (valueToken.z[k] == ' ' || valueToken.z[k] == '\t') continue; + if (valueToken.z[k] == '(' && valueToken.z[k + 1] == ')') { // for insert NOW()/TODAY() + *end = pTokenEnd = &valueToken.z[k + 2]; + k++; + continue; + } + if (valueToken.z[k] == ',' || valueToken.z[k] == ')') { + *end = pTokenEnd; + return TSDB_CODE_SUCCESS; + } + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } } - *time = ts; + *end = pTokenEnd; return TSDB_CODE_SUCCESS; } @@ -432,7 +507,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_TINYINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid tinyint data", pToken->z); } else if (!IS_VALID_TINYINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "tinyint data overflow", pToken->z); @@ -443,7 +519,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_UTINYINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z); } else if (uv > UINT8_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z); @@ -453,7 +530,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_SMALLINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid smallint data", pToken->z); } else if (!IS_VALID_SMALLINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "smallint data overflow", pToken->z); @@ -463,7 +541,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_USMALLINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z); } else if (uv > UINT16_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z); @@ -473,7 +552,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_INT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid int data", pToken->z); } else if (!IS_VALID_INT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "int data overflow", pToken->z); @@ -483,7 +563,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_UINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z); } else if (uv > UINT32_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z); @@ -493,7 +574,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_BIGINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); } val->i64 = iv; @@ -501,7 +583,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_UBIGINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z); } *(uint64_t*)(&val->i64) = uv; @@ -510,11 +593,11 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, case TSDB_DATA_TYPE_FLOAT: { double dv; - if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); } - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || - isnan(dv)) { + if (dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); } *(float*)(&val->i64) = dv; @@ -523,8 +606,9 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, case TSDB_DATA_TYPE_DOUBLE: { double dv; - if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { - return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); + code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv); + if (TSDB_CODE_SUCCESS != code) { + return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); } if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); @@ -625,7 +709,8 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStm static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, SSchema* pTagSchema, SToken* pToken, SArray* pTagName, SArray* pTagVals, STag** pTag) { - if (!isNullValue(pTagSchema->type, pToken)) { + bool isNull = isNullValue(pTagSchema->type, pToken); + if (!isNull) { taosArrayPush(pTagName, pTagSchema->name); } @@ -634,13 +719,15 @@ static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStm return buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", pToken->z); } - if (isNullValue(pTagSchema->type, pToken)) { + if (isNull) { return tTagNew(pTagVals, 1, true, pTag); } else { return parseJsontoTagData(pToken->z, pTagVals, pTag, &pCxt->msg); } } + if (isNull) return 0; + STagVal val = {0}; int32_t code = parseTagToken(ppSql, pToken, pTagSchema, pStmt->pTableMeta->tableInfo.precision, &val, &pCxt->msg); @@ -652,6 +739,10 @@ static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStm } static int32_t buildCreateTbReq(SVnodeModifyOpStmt* pStmt, STag* pTag, SArray* pTagName) { + if (pStmt->pCreateTblReq) { + tdDestroySVCreateTbReq(pStmt->pCreateTblReq); + taosMemoryFreeClear(pStmt->pCreateTblReq); + } pStmt->pCreateTblReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); if (NULL == pStmt->pCreateTblReq) { return TSDB_CODE_OUT_OF_MEMORY; @@ -666,7 +757,7 @@ static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMs if ((pToken->type != TK_NOW && pToken->type != TK_TODAY && pToken->type != TK_NK_INTEGER && pToken->type != TK_NK_STRING && pToken->type != TK_NK_FLOAT && pToken->type != TK_NK_BOOL && pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT && - pToken->type != TK_NK_BIN) || + pToken->type != TK_NK_BIN && pToken->type != TK_NK_VARIABLE) || (pToken->n == 0) || (pToken->type == TK_NK_RP)) { return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z); } @@ -845,6 +936,9 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt SSchema* pTagSchema = &pSchema[pCxt->tags.pColIndex[i]]; isJson = pTagSchema->type == TSDB_DATA_TYPE_JSON; code = checkAndTrimValue(&token, pCxt->tmpTokenBuf, &pCxt->msg); + if (TK_NK_VARIABLE == token.type) { + code = buildSyntaxErrMsg(&pCxt->msg, "not expected tags values ", token.z); + } if (TSDB_CODE_SUCCESS == code) { code = parseTagValue(pCxt, pStmt, &pStmt->pSql, pTagSchema, &token, pTagName, pTagVals, &pTag); } @@ -1346,7 +1440,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_TINYINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z); } else if (!IS_VALID_TINYINT(pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "tinyint data overflow", pToken->z); @@ -1354,7 +1449,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_UTINYINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z); } else if (pVal->value.val > UINT8_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned tinyint data overflow", pToken->z); @@ -1362,7 +1458,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_SMALLINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z); } else if (!IS_VALID_SMALLINT(pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "smallint data overflow", pToken->z); @@ -1370,7 +1467,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_USMALLINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z); } else if (pVal->value.val > UINT16_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned smallint data overflow", pToken->z); @@ -1378,7 +1476,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_INT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z); } else if (!IS_VALID_INT(pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "int data overflow", pToken->z); @@ -1386,7 +1485,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_UINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); } else if (pVal->value.val > UINT32_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned int data overflow", pToken->z); @@ -1394,25 +1494,26 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_BIGINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z); } break; } case TSDB_DATA_TYPE_UBIGINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z); } break; } case TSDB_DATA_TYPE_FLOAT: { - char* endptr = NULL; double dv; - if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + int32_t code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || - isnan(dv)) { + if (dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } float f = dv; @@ -1420,12 +1521,12 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_DOUBLE: { - char* endptr = NULL; double dv; - if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { - return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); + int32_t code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv); + if (TSDB_CODE_SUCCESS != code) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { + if (isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); } pVal->value.val = *(int64_t*)&dv; @@ -1535,6 +1636,10 @@ static int32_t parseValueToken(SInsertParseContext* pCxt, const char** pSql, STo if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { return buildSyntaxErrMsg(&pCxt->msg, "primary timestamp should not be null", pToken->z); } + + if (TK_NK_VARIABLE == pToken->type && pSchema->type != TSDB_DATA_TYPE_TIMESTAMP) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid values", pToken->z); + } pVal->flag = CV_FLAG_NULL; return TSDB_CODE_SUCCESS; } @@ -1587,6 +1692,9 @@ typedef union SRowsDataContext{ static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext* pStbRowsCxt, SToken* pToken, bool* pFoundCtbName) { *pFoundCtbName = false; int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg); + if (TK_NK_VARIABLE == pToken->type) { + code = buildInvalidOperationMsg(&pCxt->msg, "not expected tbname"); + } if (code == TSDB_CODE_SUCCESS){ if (isNullValue(TSDB_DATA_TYPE_BINARY, pToken)) { return buildInvalidOperationMsg(&pCxt->msg, "tbname can not be null value"); @@ -1624,6 +1732,10 @@ static int32_t processCtbTagsAfterCtbName(SInsertParseContext* pCxt, SVnodeModif SToken* pTagToken = (SToken*)(tagTokens + i); SSchema* pTagSchema = tagSchemas[i]; code = checkAndTrimValue(pTagToken, pCxt->tmpTokenBuf, &pCxt->msg); + if (TK_NK_VARIABLE == pTagToken->type) { + code = buildInvalidOperationMsg(&pCxt->msg, "not expected tag"); + } + if (code == TSDB_CODE_SUCCESS) { code = parseTagValue(pCxt, pStmt, NULL, pTagSchema, pTagToken, pStbRowsCxt->aTagNames, pStbRowsCxt->aTagVals, &pStbRowsCxt->pTag); @@ -1668,6 +1780,9 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* const SSchema* pSchema = &pSchemas[pCols->pColIndex[i]]; SColVal* pVal = taosArrayGet(pStbRowsCxt->aColVals, pCols->pColIndex[i]); code = parseValueToken(pCxt, ppSql, pToken, (SSchema*)pSchema, getTableInfo(pStbRowsCxt->pStbMeta).precision, pVal); + if (TK_NK_VARIABLE == pToken->type) { + code = buildInvalidOperationMsg(&pCxt->msg, "not expected row value"); + } } else if (pCols->pColIndex[i] < tbnameIdx) { const SSchema* pTagSchema = &pSchemas[pCols->pColIndex[i]]; if (canParseTagsAfter) { @@ -1676,6 +1791,9 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* ++(*pNumOfTagTokens); } else { code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg); + if (TK_NK_VARIABLE == pToken->type) { + code = buildInvalidOperationMsg(&pCxt->msg, "not expected row value"); + } if (code == TSDB_CODE_SUCCESS) { code = parseTagValue(pCxt, pStmt, ppSql, (SSchema*)pTagSchema, pToken, pTagNames, pTagVals, &pStbRowsCxt->pTag); } @@ -1797,9 +1915,10 @@ static void clearStbRowsDataContext(SStbRowsDataContext* pStbRowsCxt) { taosMemoryFreeClear(pStbRowsCxt->pCreateCtbReq); } -static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, - SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, SToken* pToken) { - bool bFirstTable = false; +static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, + SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, SToken* pToken, + STableDataCxt** ppTableDataCxt) { + bool bFirstTable = false; int32_t code = getStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pGotRow, pToken, &bFirstTable); if (code != TSDB_CODE_SUCCESS || !*pGotRow) { return code; @@ -1809,15 +1928,14 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt code = processCtbAutoCreationAndCtbMeta(pCxt, pStmt, pStbRowsCxt); } - STableDataCxt* pTableDataCxt = NULL; code = insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStbRowsCxt->pCtbMeta->uid, sizeof(pStbRowsCxt->pCtbMeta->uid), - pStbRowsCxt->pCtbMeta, &pStbRowsCxt->pCreateCtbReq, &pTableDataCxt, false, true); - initTableColSubmitData(pTableDataCxt); + pStbRowsCxt->pCtbMeta, &pStbRowsCxt->pCreateCtbReq, ppTableDataCxt, false, true); + initTableColSubmitData(*ppTableDataCxt); if (code == TSDB_CODE_SUCCESS) { - SRow** pRow = taosArrayReserve(pTableDataCxt->pData->aRowP, 1); - code = tRowBuild(pStbRowsCxt->aColVals, pTableDataCxt->pSchema, pRow); + SRow** pRow = taosArrayReserve((*ppTableDataCxt)->pData->aRowP, 1); + code = tRowBuild(pStbRowsCxt->aColVals, (*ppTableDataCxt)->pSchema, pRow); if (TSDB_CODE_SUCCESS == code) { - insCheckTableDataOrder(pTableDataCxt, TD_ROW_KEY(*pRow)); + insCheckTableDataOrder(*ppTableDataCxt, TD_ROW_KEY(*pRow)); } } @@ -1915,7 +2033,8 @@ static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, if (!pStmt->stbSyntax) { code = parseOneRow(pCxt, &pStmt->pSql, rowsDataCxt.pTableDataCxt, &gotRow, pToken); } else { - code = parseOneStbRow(pCxt, pStmt, &pStmt->pSql, rowsDataCxt.pStbRowsCxt, &gotRow, pToken); + STableDataCxt* pTableDataCxt = NULL; + code = parseOneStbRow(pCxt, pStmt, &pStmt->pSql, rowsDataCxt.pStbRowsCxt, &gotRow, pToken, &pTableDataCxt); } } @@ -1979,7 +2098,14 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt if (!pStmt->stbSyntax) { code = parseOneRow(pCxt, (const char**)&pRow, rowsDataCxt.pTableDataCxt, &gotRow, &token); } else { - code = parseOneStbRow(pCxt, pStmt, (const char**)&pRow, rowsDataCxt.pStbRowsCxt, &gotRow, &token); + STableDataCxt* pTableDataCxt = NULL; + code = parseOneStbRow(pCxt, pStmt, (const char**)&pRow, rowsDataCxt.pStbRowsCxt, &gotRow, &token, &pTableDataCxt); + if (code == TSDB_CODE_SUCCESS) { + SStbRowsDataContext* pStbRowsCxt = rowsDataCxt.pStbRowsCxt; + void* pData = pTableDataCxt; + taosHashPut(pStmt->pTableCxtHashObj, &pStbRowsCxt->pCtbMeta->uid, sizeof(pStbRowsCxt->pCtbMeta->uid), &pData, + POINTER_BYTES); + } } if (code && firstLine) { firstLine = false; @@ -1992,7 +2118,7 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt (*pNumOfRows)++; } - if (TSDB_CODE_SUCCESS == code && (*pNumOfRows) > tsMaxInsertBatchRows) { + if (TSDB_CODE_SUCCESS == code && (*pNumOfRows) >= tsMaxInsertBatchRows) { pStmt->fileProcessing = true; break; } @@ -2003,7 +2129,7 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt parserDebug("0x%" PRIx64 " %d rows have been parsed", pCxt->pComCxt->requestId, *pNumOfRows); - if (TSDB_CODE_SUCCESS == code && 0 == (*pNumOfRows) && + if (TSDB_CODE_SUCCESS == code && 0 == (*pNumOfRows) && 0 == pStmt->totalRowsNum && (!TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) && !pStmt->fileProcessing) { code = buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); } @@ -2011,6 +2137,12 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt } static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SRowsDataContext rowsDataCxt) { + // init only for file + if (NULL == pStmt->pTableCxtHashObj) { + pStmt->pTableCxtHashObj = + taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + } + int32_t numOfRows = 0; int32_t code = parseCsvFile(pCxt, pStmt, rowsDataCxt, &numOfRows); if (TSDB_CODE_SUCCESS == code) { @@ -2022,7 +2154,18 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpSt } else { parserDebug("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId); } + if (pStmt->insertType != TSDB_QUERY_TYPE_FILE_INSERT) { + return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is exclusive", NULL); + } } + + // just record pTableCxt whose data come from file + if (!pStmt->stbSyntax && numOfRows > 0) { + void* pData = rowsDataCxt.pTableDataCxt; + taosHashPut(pStmt->pTableCxtHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), &pData, + POINTER_BYTES); + } + return code; } @@ -2061,6 +2204,9 @@ static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS NEXT_TOKEN(pStmt->pSql, token); switch (token.type) { case TK_VALUES: + if (TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_FILE_INSERT)) { + return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is exclusive", token.z); + } return parseValuesClause(pCxt, pStmt, rowsDataCxt, &token); case TK_FILE: return parseFileClause(pCxt, pStmt, rowsDataCxt, &token); @@ -2275,8 +2421,25 @@ static int32_t parseInsertBodyBottom(SInsertParseContext* pCxt, SVnodeModifyOpSt return setStmtInfo(pCxt, pStmt); } + // release old array alloced by merge + pStmt->freeArrayFunc(pStmt->pVgDataBlocks); + pStmt->pVgDataBlocks = NULL; + + bool fileOnly = (pStmt->insertType == TSDB_QUERY_TYPE_FILE_INSERT); + if (fileOnly) { + // none data, skip merge & buildvgdata + if (0 == taosHashGetSize(pStmt->pTableCxtHashObj)) { + pCxt->needRequest = false; + return TSDB_CODE_SUCCESS; + } + } + // merge according to vgId - int32_t code = insMergeTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks); + int32_t code = insMergeTableDataCxt(fileOnly ? pStmt->pTableCxtHashObj : pStmt->pTableBlockHashObj, + &pStmt->pVgDataBlocks, pStmt->fileProcessing); + // clear tmp hashobj only + taosHashClear(pStmt->pTableCxtHashObj); + if (TSDB_CODE_SUCCESS == code) { code = insBuildVgDataBlocks(pStmt->pVgroupsHashObj, pStmt->pVgDataBlocks, &pStmt->pDataBlocks); } @@ -2718,6 +2881,7 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal .msg = {.buf = pCxt->pMsg, .len = pCxt->msgLen}, .missCache = false, .usingDuplicateTable = false, + .needRequest = true, .forceUpdate = (NULL != pCatalogReq ? pCatalogReq->forceUpdate : false)}; int32_t code = initInsertQuery(&context, pCatalogReq, pMetaData, pQuery); @@ -2732,5 +2896,10 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal code = setRefreshMeta(*pQuery); } insDestroyBoundColInfo(&context.tags); + + // if no data to insert, set emptyMode to avoid request server + if (!context.needRequest) { + (*pQuery)->execMode = QUERY_EXEC_MODE_EMPTY_RESULT; + } return code; } diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 5137deca2e..a88aec20b3 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -58,7 +58,7 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash // merge according to vgId if (taosHashGetSize(pBlockHash) > 0) { - code = insMergeTableDataCxt(pBlockHash, &pVgDataBlocks); + code = insMergeTableDataCxt(pBlockHash, &pVgDataBlocks, true); } if (TSDB_CODE_SUCCESS == code) { code = insBuildVgDataBlocks(pVgHash, pVgDataBlocks, &pStmt->pDataBlocks); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 21b093c76c..a924ed68b0 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -289,6 +289,14 @@ static int32_t rebuildTableData(SSubmitTbData* pSrc, SSubmitTbData** pDst) { pTmp->uid = pSrc->uid; pTmp->sver = pSrc->sver; pTmp->pCreateTbReq = NULL; + if (pTmp->flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { + if (pSrc->pCreateTbReq) { + cloneSVreateTbReq(pSrc->pCreateTbReq, &pTmp->pCreateTbReq); + } else { + pTmp->flags &= ~SUBMIT_REQ_AUTO_CREATE_TABLE; + } + } + if (pTmp->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { pTmp->aCol = taosArrayInit(128, sizeof(SColData)); if (NULL == pTmp->aCol) { @@ -416,15 +424,21 @@ void insDestroyTableDataCxtHashMap(SHashObj* pTableCxtHash) { taosHashCleanup(pTableCxtHash); } -static int32_t fillVgroupDataCxt(STableDataCxt* pTableCxt, SVgroupDataCxt* pVgCxt) { +static int32_t fillVgroupDataCxt(STableDataCxt* pTableCxt, SVgroupDataCxt* pVgCxt, bool isRebuild) { if (NULL == pVgCxt->pData->aSubmitTbData) { pVgCxt->pData->aSubmitTbData = taosArrayInit(128, sizeof(SSubmitTbData)); if (NULL == pVgCxt->pData->aSubmitTbData) { return TSDB_CODE_OUT_OF_MEMORY; } } + + // push data to submit, rebuild empty data for next submit taosArrayPush(pVgCxt->pData->aSubmitTbData, pTableCxt->pData); - rebuildTableData(pTableCxt->pData, &pTableCxt->pData); + if (isRebuild) { + rebuildTableData(pTableCxt->pData, &pTableCxt->pData); + } else { + taosMemoryFreeClear(pTableCxt->pData); + } qDebug("add tableDataCxt uid:%" PRId64 " to vgId:%d", pTableCxt->pMeta->uid, pVgCxt->vgId); @@ -467,7 +481,7 @@ int insColDataComp(const void* lp, const void* rp) { return 0; } -int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks) { +int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks, bool isRebuild) { SHashObj* pVgroupHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); SArray* pVgroupList = taosArrayInit(8, POINTER_BYTES); if (NULL == pVgroupHash || NULL == pVgroupList) { @@ -502,6 +516,13 @@ int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks) { tColDataSortMerge(pTableCxt->pData->aCol); } else { + // skip the table has no data to insert + // eg: import a csv without valid data + // if (0 == taosArrayGetSize(pTableCxt->pData->aRowP)) { + // qWarn("no row in tableDataCxt uid:%" PRId64 " ", pTableCxt->pMeta->uid); + // p = taosHashIterate(pTableHash, p); + // continue; + // } if (!pTableCxt->ordered) { code = tRowSort(pTableCxt->pData->aRowP); } @@ -520,7 +541,7 @@ int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks) { pVgCxt = *(SVgroupDataCxt**)pp; } if (TSDB_CODE_SUCCESS == code) { - code = fillVgroupDataCxt(pTableCxt, pVgCxt); + code = fillVgroupDataCxt(pTableCxt, pVgCxt, isRebuild); } } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 5193bdc47f..5988c68e17 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -609,6 +609,11 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) { break; } + // support float with no decimal part after the decimal point + if (z[i] == '.' && seg == 1) { + *tokenId = TK_NK_FLOAT; + i++; + } if ((z[i] == 'e' || z[i] == 'E') && (isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) { i += 2; @@ -751,7 +756,7 @@ SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr, bool* pIgnoreC // support parse the -/+number format if ((isPrevOptr) && (t0.type == TK_NK_MINUS || t0.type == TK_NK_PLUS)) { len = tGetToken(&str[*i + t0.n], &type); - if (type == TK_NK_INTEGER || type == TK_NK_FLOAT) { + if (type == TK_NK_INTEGER || type == TK_NK_FLOAT || type == TK_NK_BIN || type == TK_NK_HEX) { t0.type = type; t0.n += len; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 5c30384a6b..c5ac8a3dd6 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4622,11 +4622,15 @@ static int32_t addOrderByPrimaryKeyToQueryImpl(STranslateContext* pCxt, SNode* p return TSDB_CODE_OUT_OF_MEMORY; } ((SExprNode*)pOrderByExpr->pExpr)->orderAlias = true; - NODES_DESTORY_LIST(*pOrderByList); + // NODES_DESTORY_LIST(*pOrderByList); return nodesListMakeStrictAppend(pOrderByList, (SNode*)pOrderByExpr); } static int32_t addOrderByPrimaryKeyToQuery(STranslateContext* pCxt, SNode* pPrimaryKeyExpr, SNode* pStmt) { + SNodeList* pOrederList = ((SSelectStmt*)pStmt)->pOrderByList; + if (pOrederList && pOrederList->length > 0) { + return TSDB_CODE_SUCCESS; + } if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { return addOrderByPrimaryKeyToQueryImpl(pCxt, pPrimaryKeyExpr, &((SSelectStmt*)pStmt)->pOrderByList); } @@ -7388,6 +7392,13 @@ static int32_t subtableExprHasColumnOrPseudoColumn(SNode* pNode) { static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt) { SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; + if ( (SRealTableNode*)pSelect->pFromTable && ((SRealTableNode*)pSelect->pFromTable)->pMeta + && TSDB_SUPER_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType + && !hasPartitionByTbname(pSelect->pPartitionByList) + && pSelect->pWindow != NULL && pSelect->pWindow->type == QUERY_NODE_EVENT_WINDOW) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "Event window for stream on super table must patitioned by table name"); + } if (TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type || !isTimeLineQuery(pStmt->pQuery) || crossTableWithoutAggOper(pSelect) || NULL != pSelect->pOrderByList || crossTableWithUdaf(pSelect) || hasJsonTypeProjection(pSelect)) { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index e71e18d37d..aa3181e166 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1400,6 +1400,7 @@ static int32_t sortPriKeyOptApply(SOptimizeContext* pCxt, SLogicSubplan* pLogicS pScan->node.outputTsOrder = order; if (TSDB_SUPER_TABLE == pScan->tableType) { pScan->scanType = SCAN_TYPE_TABLE_MERGE; + pScan->filesetDelimited = true; pScan->node.resultDataOrder = DATA_ORDER_LEVEL_GLOBAL; pScan->node.requireDataOrder = DATA_ORDER_LEVEL_GLOBAL; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index fac93948f5..60b04bb1d1 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -622,6 +622,7 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp pTableScan->igExpired = pScanLogicNode->igExpired; pTableScan->igCheckUpdate = pScanLogicNode->igCheckUpdate; pTableScan->assignBlockUid = pCxt->pPlanCxt->rSmaQuery ? true : false; + pTableScan->filesetDelimited = pScanLogicNode->filesetDelimited; int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 25d575a2ca..1f9c4bcae9 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -726,6 +726,7 @@ static void stbSplSetTableMergeScan(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { SScanLogicNode* pScan = (SScanLogicNode*)pNode; pScan->scanType = SCAN_TYPE_TABLE_MERGE; + pScan->filesetDelimited = true; if (NULL != pScan->pGroupTags) { pScan->groupSort = true; } @@ -1067,6 +1068,7 @@ static SNode* stbSplCreateColumnNode(SExprNode* pExpr) { strcpy(pCol->colName, pExpr->aliasName); } strcpy(pCol->node.aliasName, pExpr->aliasName); + strcpy(pCol->node.userAlias, pExpr->userAlias); pCol->node.resType = pExpr->resType; return (SNode*)pCol; } @@ -1096,8 +1098,9 @@ static int32_t stbSplCreateMergeKeys(SNodeList* pSortKeys, SNodeList* pTargets, SNode* pTarget = NULL; bool found = false; FOREACH(pTarget, pTargets) { - if ((QUERY_NODE_COLUMN == nodeType(pSortExpr) && nodesEqualNode((SNode*)pSortExpr, pTarget)) || - (0 == strcmp(pSortExpr->aliasName, ((SColumnNode*)pTarget)->colName))) { + if ((QUERY_NODE_COLUMN == nodeType(pSortExpr) && nodesEqualNode((SNode*)pSortExpr, pTarget)) + // || (0 == strcmp(pSortExpr->aliasName, ((SColumnNode*)pTarget)->colName)) + ) { code = nodesListMakeStrictAppend(&pMergeKeys, stbSplCreateOrderByExpr(pSortKey, pTarget)); if (TSDB_CODE_SUCCESS != code) { break; @@ -1275,6 +1278,7 @@ static int32_t stbSplCreateMergeScanNode(SScanLogicNode* pScan, SLogicNode** pOu SNodeList* pMergeKeys = NULL; if (TSDB_CODE_SUCCESS == code) { pMergeScan->scanType = SCAN_TYPE_TABLE_MERGE; + pMergeScan->filesetDelimited = true; pMergeScan->node.pChildren = pChildren; splSetParent((SLogicNode*)pMergeScan); code = stbSplCreateMergeKeysByPrimaryKey(stbSplFindPrimaryKeyFromScan(pMergeScan), diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index 1c7c937b7f..2da270e42d 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -164,6 +164,7 @@ static int32_t adjustScanDataRequirement(SScanLogicNode* pScan, EDataOrderLevel pScan->scanType = SCAN_TYPE_TABLE; } else if (TSDB_SUPER_TABLE == pScan->tableType) { pScan->scanType = SCAN_TYPE_TABLE_MERGE; + pScan->filesetDelimited = true; } if (TSDB_NORMAL_TABLE != pScan->tableType && TSDB_CHILD_TABLE != pScan->tableType) { diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 3eadea3cdd..bed0f79f02 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -17,6 +17,7 @@ #define _STREAM_BACKEDN_ROCKSDB_H_ #include "rocksdb/c.h" +//#include "streamInt.h" #include "streamState.h" #include "tcoding.h" #include "tcommon.h" @@ -42,15 +43,110 @@ typedef struct { TdThreadMutex cfMutex; SHashObj* cfInst; int64_t defaultCfInit; + } SBackendWrapper; +typedef struct { + void* tableOpt; +} RocksdbCfParam; + +typedef struct { + rocksdb_t* db; + rocksdb_writeoptions_t* writeOpt; + rocksdb_readoptions_t* readOpt; + rocksdb_options_t* dbOpt; + rocksdb_env_t* env; + rocksdb_cache_t* cache; + + rocksdb_column_family_handle_t** pCf; + rocksdb_comparator_t** pCompares; + rocksdb_options_t** pCfOpts; + RocksdbCfParam* pCfParams; + + rocksdb_compactionfilterfactory_t* filterFactory; + TdThreadMutex mutex; + char* idstr; + char* path; + int64_t refId; + + void* pTask; + int64_t streamId; + int64_t taskId; + int64_t chkpId; + SArray* chkpSaved; + SArray* chkpInUse; + int32_t chkpCap; + TdThreadRwlock chkpDirLock; + int64_t dataWritten; + +} STaskDbWrapper; + +typedef struct SDbChkp { + int8_t init; + char* pCurrent; + char* pManifest; + SArray* pSST; + int64_t preCkptId; + int64_t curChkpId; + char* path; + + char* buf; + int32_t len; + + // ping-pong buf + SHashObj* pSstTbl[2]; + int8_t idx; + + SArray* pAdd; + SArray* pDel; + int8_t update; + + TdThreadRwlock rwLock; +} SDbChkp; +typedef struct { + int8_t init; + char* pCurrent; + char* pManifest; + SArray* pSST; + int64_t preCkptId; + int64_t curChkpId; + char* path; + + char* buf; + int32_t len; + + // ping-pong buf + SHashObj* pSstTbl[2]; + int8_t idx; + + SArray* pAdd; + SArray* pDel; + int8_t update; + + SHashObj* pDbChkpTbl; + + TdThreadRwlock rwLock; +} SBkdMgt; + +bool streamBackendDataIsExist(const char* path, int64_t chkpId, int32_t vgId); void* streamBackendInit(const char* path, int64_t chkpId, int32_t vgId); void streamBackendCleanup(void* arg); void streamBackendHandleCleanup(void* arg); int32_t streamBackendLoadCheckpointInfo(void* pMeta); -int32_t streamBackendDoCheckpoint(void* pMeta, uint64_t checkpointId); +int32_t streamBackendDoCheckpoint(void* pMeta, int64_t checkpointId); SListNode* streamBackendAddCompare(void* backend, void* arg); void streamBackendDelCompare(void* backend, void* arg); +int32_t streamStateCvtDataFormat(char* path, char* key, void* cfInst); + +STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId); +void taskDbDestroy(void* pBackend, bool flush); +void taskDbDestroy2(void* pBackend); +int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId); + +void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId); + +void* taskDbAddRef(void* pTaskDb); +void taskDbRemoveRef(void* pTaskDb); int streamStateOpenBackend(void* backend, SStreamState* pState); void streamStateCloseBackend(SStreamState* pState, bool remove); @@ -122,7 +218,7 @@ int32_t streamDefaultGet_rocksdb(SStreamState* pState, const void* key, void** p int32_t streamDefaultDel_rocksdb(SStreamState* pState, const void* key); int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, const void* end, SArray* result); void* streamDefaultIterCreate_rocksdb(SStreamState* pState); -bool streamDefaultIterValid_rocksdb(void* iter); +bool streamDefaultIterValid_rocksdb(void* iter); void streamDefaultIterSeek_rocksdb(void* iter, const char* key); void streamDefaultIterNext_rocksdb(void* iter); char* streamDefaultIterKey_rocksdb(void* iter, int32_t* len); @@ -146,5 +242,20 @@ int32_t streamBackendTriggerChkp(void* pMeta, char* dst); int32_t streamBackendAddInUseChkp(void* arg, int64_t chkpId); int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId); +int32_t taskDbBuildSnap(void* arg, SArray* pSnap); + // int32_t streamDefaultIter_rocksdb(SStreamState* pState, const void* start, const void* end, SArray* result); + +// STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId); +// void taskDbDestroy(void* pDb, bool flush); + +int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId); + +SBkdMgt* bkdMgtCreate(char* path); +int32_t bkdMgtAddChkp(SBkdMgt* bm, char* task, char* path); +int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list, char* name); +int32_t bkdMgtDumpTo(SBkdMgt* bm, char* taskId, char* dname); +void bkdMgtDestroy(SBkdMgt* bm); + +int32_t taskDbGenChkpUploadData(void* arg, void* bkdMgt, int64_t chkpId, int8_t type, char** path, SArray* list); #endif \ No newline at end of file diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index afb6b9227d..0df36ec391 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -57,24 +57,25 @@ typedef struct { SSDataBlock* pBlock; } SStreamTrigger; -typedef struct SStreamGlobalEnv { - int8_t inited; - void* timer; -} SStreamGlobalEnv; - typedef struct SStreamContinueExecInfo { SEpSet epset; int32_t taskId; SRpcMsg msg; } SStreamContinueExecInfo; +typedef struct { + int64_t streamId; + int64_t taskId; + int64_t chkpId; + char* dbPrefixPath; +} SStreamTaskSnap; struct STokenBucket { - int32_t numCapacity; // total capacity, available token per second - int32_t numOfToken; // total available tokens - int32_t numRate; // number of token per second - double quotaCapacity; // available capacity for maximum input size, KiloBytes per Second - double quotaRemain; // not consumed bytes per second - double quotaRate; // number of token per second + int32_t numCapacity; // total capacity, available token per second + int32_t numOfToken; // total available tokens + int32_t numRate; // number of token per second + double quotaCapacity; // available capacity for maximum input size, KiloBytes per Second + double quotaRemain; // not consumed bytes per second + double quotaRate; // number of token per second int64_t tokenFillTimestamp; // fill timestamp int64_t quotaFillTimestamp; // fill timestamp }; @@ -86,9 +87,10 @@ struct SStreamQueue { int8_t status; }; -extern SStreamGlobalEnv streamEnv; +extern void* streamTimer; extern int32_t streamBackendId; extern int32_t streamBackendCfWrapperId; +extern int32_t taskDbWrapperId; void streamRetryDispatchData(SStreamTask* pTask, int64_t waitDuration); int32_t streamDispatchStreamBlock(SStreamTask* pTask); @@ -106,6 +108,8 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq); +int32_t streamSaveTaskCheckpointInfo(SStreamTask* p, int64_t checkpointId); +int32_t streamTaskBuildCheckpoint(SStreamTask* pTask); int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, int64_t checkpointId); int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet); @@ -117,7 +121,8 @@ int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t STaskId streamTaskExtractKey(const SStreamTask* pTask); void streamTaskInitForLaunchHTask(SHistoryTaskInfo* pInfo); void streamTaskSetRetryInfoForLaunch(SHistoryTaskInfo* pInfo); -int32_t streamTaskBuildScanhistoryRspMsg(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq, void** pBuffer, int32_t* pLen); +int32_t streamTaskBuildScanhistoryRspMsg(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq, void** pBuffer, + int32_t* pLen); int32_t streamTaskFillHistoryFinished(SStreamTask* pTask); void streamClearChkptReadyMsg(SStreamTask* pTask); @@ -132,6 +137,17 @@ int32_t streamAddEndScanHistoryMsg(SStreamTask* pTask, SRpcHandleInfo* pRpcInfo, int32_t streamNotifyUpstreamContinue(SStreamTask* pTask); int32_t streamTransferStateToStreamTask(SStreamTask* pTask); +// <<<<<<< HEAD +// void streamClearChkptReadyMsg(SStreamTask* pTask); + +// int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate, const +// char*); STaskId streamTaskExtractKey(const SStreamTask* pTask); void streamTaskInitForLaunchHTask(SHistoryTaskInfo* +// pInfo); void streamTaskSetRetryInfoForLaunch(SHistoryTaskInfo* pInfo); + +// void streamMetaResetStartInfo(STaskStartInfo* pMeta); + +// ======= +// >>>>>>> 3.0 SStreamQueue* streamQueueOpen(int64_t cap); void streamQueueClose(SStreamQueue* pQueue, int32_t taskId); void streamQueueProcessSuccess(SStreamQueue* queue); @@ -151,10 +167,14 @@ int uploadCheckpoint(char* id, char* path); int downloadCheckpoint(char* id, char* path); int deleteCheckpoint(char* id); int deleteCheckpointFile(char* id, char* name); +int downloadCheckpointByName(char* id, char* fname, char* dstName); int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask); int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask); +typedef int32_t (*__stream_async_exec_fn_t)(void* param); + +int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, void* param, int32_t* code); #ifdef __cplusplus } #endif diff --git a/source/libs/stream/inc/streamsm.h b/source/libs/stream/inc/streamsm.h index be3665fde7..7be655fbed 100644 --- a/source/libs/stream/inc/streamsm.h +++ b/source/libs/stream/inc/streamsm.h @@ -32,8 +32,8 @@ typedef int32_t (*__state_trans_fn)(SStreamTask*); typedef int32_t (*__state_trans_succ_fn)(SStreamTask*); typedef struct SAttachedEventInfo { - ETaskStatus status; // required status that this event can be handled - EStreamTaskEvent event; // the delayed handled event + ETaskStatus status; // required status that this event can be handled + EStreamTaskEvent event; // the delayed handled event } SAttachedEventInfo; typedef struct STaskStateTrans { @@ -64,8 +64,8 @@ typedef struct SStreamEventInfo { const char* name; } SStreamEventInfo; -SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask); -void* streamDestroyStateMachine(SStreamTaskSM* pSM); +// SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask); +// void* streamDestroyStateMachine(SStreamTaskSM* pSM); #ifdef __cplusplus } #endif diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 1c874f34de..1bef42bf14 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -16,38 +16,18 @@ #include "streamInt.h" #include "ttimer.h" -SStreamGlobalEnv streamEnv; +void* streamTimer = NULL; -int32_t streamInit() { - int8_t old; - while (1) { - old = atomic_val_compare_exchange_8(&streamEnv.inited, 0, 2); - if (old != 2) break; +int32_t streamTimerInit() { + streamTimer = taosTmrInit(1000, 100, 10000, "STREAM"); + if (streamTimer == NULL) { + return -1; } - - if (old == 0) { - streamEnv.timer = taosTmrInit(1000, 100, 10000, "STREAM"); - if (streamEnv.timer == NULL) { - atomic_store_8(&streamEnv.inited, 0); - return -1; - } - atomic_store_8(&streamEnv.inited, 1); - } - return 0; } -void streamCleanUp() { - int8_t old; - while (1) { - old = atomic_val_compare_exchange_8(&streamEnv.inited, 1, 2); - if (old != 2) break; - } - - if (old == 1) { - taosTmrCleanUp(streamEnv.timer); - atomic_store_8(&streamEnv.inited, 0); - } +void streamTimerCleanUp() { + taosTmrCleanUp(streamTimer); } char* createStreamTaskIdStr(int64_t streamId, int32_t taskId) { @@ -77,7 +57,7 @@ static void streamSchedByTimer(void* param, void* tmrId) { if (pTrigger == NULL) { stError("s-task:%s failed to prepare retrieve data trigger, code:%s, try again in %dms", id, "out of memory", nextTrigger); - taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamEnv.timer, &pTask->schedInfo.pTimer); + taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pTimer); return; } @@ -88,7 +68,7 @@ static void streamSchedByTimer(void* param, void* tmrId) { stError("s-task:%s failed to prepare retrieve data trigger, code:%s, try again in %dms", id, "out of memory", nextTrigger); - taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamEnv.timer, &pTask->schedInfo.pTimer); + taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pTimer); return; } @@ -97,7 +77,7 @@ static void streamSchedByTimer(void* param, void* tmrId) { int32_t code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTrigger); if (code != TSDB_CODE_SUCCESS) { - taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamEnv.timer, &pTask->schedInfo.pTimer); + taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pTimer); return; } @@ -105,7 +85,7 @@ static void streamSchedByTimer(void* param, void* tmrId) { } } - taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamEnv.timer, &pTask->schedInfo.pTimer); + taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pTimer); } int32_t streamSetupScheduleTrigger(SStreamTask* pTask) { @@ -115,7 +95,7 @@ int32_t streamSetupScheduleTrigger(SStreamTask* pTask) { stDebug("s-task:%s setup scheduler trigger, delay:%" PRId64 " ms", pTask->id.idStr, pTask->info.triggerParam); - pTask->schedInfo.pTimer = taosTmrStart(streamSchedByTimer, (int32_t)pTask->info.triggerParam, pTask, streamEnv.timer); + pTask->schedInfo.pTimer = taosTmrStart(streamSchedByTimer, (int32_t)pTask->info.triggerParam, pTask, streamTimer); pTask->schedInfo.status = TASK_TRIGGER_STATUS__INACTIVE; } diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 9699386fd4..630650025d 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -20,34 +20,10 @@ #include "tcommon.h" #include "tref.h" -typedef struct { - int8_t init; - char* pCurrent; - char* pManifest; - SArray* pSST; - int64_t preCkptId; - int64_t curChkpId; - char* path; - - char* buf; - int32_t len; - - // ping-pong buf - SHashObj* pSstTbl[2]; - int8_t idx; - - SArray* pAdd; - SArray* pDel; - int8_t update; -} SBackendManager; - typedef struct SCompactFilteFactory { void* status; } SCompactFilteFactory; -typedef struct { - void* tableOpt; -} RocksdbCfParam; typedef struct { rocksdb_t* db; rocksdb_column_family_handle_t** pHandle; @@ -61,46 +37,65 @@ typedef struct { rocksdb_comparator_t** pCompares; } RocksdbCfInst; -uint32_t nextPow2(uint32_t x); - int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t nCf); -void destroyRocksdbCfInst(RocksdbCfInst* inst); +void destroyRocksdbCfInst(RocksdbCfInst* inst); +int32_t getCfIdx(const char* cfName); +STaskDbWrapper* taskDbOpenImpl(char* key, char* statePath, char* dbPath); + +void destroyCompactFilteFactory(void* arg); +void destroyCompactFilte(void* arg); +const char* compactFilteFactoryName(void* arg); +const char* compactFilteFactoryNameSess(void* arg); +const char* compactFilteFactoryNameState(void* arg); +const char* compactFilteFactoryNameFunc(void* arg); +const char* compactFilteFactoryNameFill(void* arg); + +const char* compactFilteName(void* arg); +const char* compactFilteNameSess(void* arg); +const char* compactFilteNameState(void* arg); +const char* compactFilteNameFill(void* arg); +const char* compactFilteNameFunc(void* arg); -void destroyCompactFilteFactory(void* arg); -void destroyCompactFilte(void* arg); -const char* compactFilteFactoryName(void* arg); -const char* compactFilteName(void* arg); unsigned char compactFilte(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, char** newval, size_t* newvlen, unsigned char* value_changed); rocksdb_compactionfilter_t* compactFilteFactoryCreateFilter(void* arg, rocksdb_compactionfiltercontext_t* ctx); +rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterSess(void* arg, rocksdb_compactionfiltercontext_t* ctx); +rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterState(void* arg, rocksdb_compactionfiltercontext_t* ctx); +rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFunc(void* arg, rocksdb_compactionfiltercontext_t* ctx); +rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFill(void* arg, rocksdb_compactionfiltercontext_t* ctx); -const char* cfName[] = {"default", "state", "fill", "sess", "func", "parname", "partag"}; +typedef int (*__db_key_encode_fn_t)(void* key, char* buf); +typedef int (*__db_key_decode_fn_t)(void* key, char* buf); +typedef int (*__db_key_tostr_fn_t)(void* key, char* buf); +typedef const char* (*__db_key_cmpname_fn_t)(void* statue); +typedef int (*__db_key_cmp_fn_t)(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen); +typedef void (*__db_key_cmp_destroy_fn_t)(void* state); +typedef int32_t (*__db_value_encode_fn_t)(void* value, int32_t vlen, int64_t ttl, char** dest); +typedef int32_t (*__db_value_decode_fn_t)(void* value, int32_t vlen, int64_t* ttl, char** dest); -typedef int (*EncodeFunc)(void* key, char* buf); -typedef int (*DecodeFunc)(void* key, char* buf); -typedef int (*ToStringFunc)(void* key, char* buf); -typedef const char* (*CompareName)(void* statue); -typedef int (*BackendCmpFunc)(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen); -typedef void (*DestroyFunc)(void* state); -typedef int32_t (*EncodeValueFunc)(void* value, int32_t vlen, int64_t ttl, char** dest); -typedef int32_t (*DecodeValueFunc)(void* value, int32_t vlen, int64_t* ttl, char** dest); +typedef rocksdb_compactionfilter_t* (*__db_factory_create_fn_t)(void* arg, rocksdb_compactionfiltercontext_t* ctx); +typedef const char* (*__db_factory_name_fn_t)(void* arg); +typedef void (*__db_factory_destroy_fn_t)(void* arg); typedef struct { - const char* key; - int32_t len; - int idx; - BackendCmpFunc cmpFunc; - EncodeFunc enFunc; - DecodeFunc deFunc; - ToStringFunc toStrFunc; - CompareName cmpName; - DestroyFunc detroyFunc; - EncodeValueFunc enValueFunc; - DecodeValueFunc deValueFunc; + const char* key; + int32_t len; + int idx; + __db_key_cmp_fn_t cmpKey; + __db_key_encode_fn_t enFunc; + __db_key_decode_fn_t deFunc; + __db_key_tostr_fn_t toStrFunc; + __db_key_cmpname_fn_t cmpName; + __db_key_cmp_destroy_fn_t destroyCmp; + __db_value_encode_fn_t enValueFunc; + __db_value_decode_fn_t deValueFunc; + + __db_factory_create_fn_t createFilter; + __db_factory_destroy_fn_t destroyFilter; + __db_factory_name_fn_t funcName; } SCfInit; -#define GEN_COLUMN_FAMILY_NAME(name, idstr, SUFFIX) sprintf(name, "%s_%s", idstr, (SUFFIX)); const char* compareDefaultName(void* name); const char* compareStateName(void* name); const char* compareWinKeyName(void* name); @@ -139,296 +134,71 @@ int parKeyEncode(void* k, char* buf); int parKeyDecode(void* k, char* buf); int parKeyToString(void* k, char* buf); -int stremaValueEncode(void* k, char* buf); -int streamValueDecode(void* k, char* buf); -int32_t streamValueToString(void* k, char* buf); -int32_t streaValueIsStale(void* k, int64_t ts); -void destroyFunc(void* arg); +int32_t valueEncode(void* value, int32_t vlen, int64_t ttl, char** dest); +int32_t valueDecode(void* value, int32_t vlen, int64_t* ttl, char** dest); +int32_t valueToString(void* k, char* buf); +int32_t valueIsStale(void* k, int64_t ts); -int32_t encodeValueFunc(void* value, int32_t vlen, int64_t ttl, char** dest); -int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest); +void destroyCompare(void* arg); -SBackendManager* bkdMgtCreate(char* path) { - SBackendManager* p = taosMemoryCalloc(1, sizeof(SBackendManager)); - p->curChkpId = 0; - p->preCkptId = 0; - p->pSST = taosArrayInit(64, sizeof(void*)); - p->path = taosStrdup(path); - p->len = strlen(path) + 128; - p->buf = taosMemoryCalloc(1, p->len); +static bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len); +static rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, + rocksdb_snapshot_t** snapshot, rocksdb_readoptions_t** readOpt); - p->idx = 0; - p->pSstTbl[0] = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); - p->pSstTbl[1] = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); - - p->pAdd = taosArrayInit(64, sizeof(void*)); - p->pDel = taosArrayInit(64, sizeof(void*)); - p->update = 0; - return p; -} -void bkdMgtDestroy(SBackendManager* bm) { - if (bm == NULL) return; - - taosMemoryFree(bm->buf); - taosMemoryFree(bm->path); - - taosArrayDestroyP(bm->pSST, taosMemoryFree); - taosArrayDestroyP(bm->pAdd, taosMemoryFree); - taosArrayDestroyP(bm->pDel, taosMemoryFree); - - taosHashCleanup(bm->pSstTbl[0]); - taosHashCleanup(bm->pSstTbl[1]); - - taosMemoryFree(bm->pCurrent); - taosMemoryFree(bm->pManifest); - - taosMemoryFree(bm); -} - -int32_t compareHashTableImpl(SHashObj* p1, SHashObj* p2, SArray* diff) { - int32_t code = 0; - size_t len = 0; - void* pIter = taosHashIterate(p2, NULL); - while (pIter) { - char* name = taosHashGetKey(pIter, &len); - if (!taosHashGet(p1, name, len)) { - char* p = taosStrdup(name); - taosArrayPush(diff, &p); - } - pIter = taosHashIterate(p2, pIter); - } - return code; -} -int32_t compareHashTable(SHashObj* p1, SHashObj* p2, SArray* add, SArray* del) { - int32_t code = 0; - - code = compareHashTableImpl(p1, p2, add); - code = compareHashTableImpl(p2, p1, del); - - return code; -} -int32_t bkdMgtGetDelta(SBackendManager* bm, int64_t chkpId, SArray* list) { - const char* pCurrent = "CURRENT"; - int32_t currLen = strlen(pCurrent); - - const char* pManifest = "MANIFEST-"; - int32_t maniLen = strlen(pManifest); - - const char* pSST = ".sst"; - int32_t sstLen = strlen(pSST); - - memset(bm->buf, 0, bm->len); - sprintf(bm->buf, "%s%scheckpoint%" PRId64 "", bm->path, TD_DIRSEP, chkpId); - - taosArrayClearP(bm->pAdd, taosMemoryFree); - taosArrayClearP(bm->pDel, taosMemoryFree); - - TdDirPtr pDir = taosOpenDir(bm->buf); - TdDirEntryPtr de = NULL; - int8_t dummy = 0; - while ((de = taosReadDir(pDir)) != NULL) { - char* name = taosGetDirEntryName(de); - if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue; - if (strlen(name) == currLen && strcmp(name, pCurrent) == 0) { - taosMemoryFreeClear(bm->pCurrent); - bm->pCurrent = taosStrdup(name); - taosHashPut(bm->pSstTbl[1 - bm->idx], name, strlen(name), &dummy, sizeof(dummy)); - continue; - } - - if (strlen(name) >= maniLen && strncmp(name, pManifest, maniLen) == 0) { - taosMemoryFreeClear(bm->pManifest); - bm->pManifest = taosStrdup(name); - taosHashPut(bm->pSstTbl[1 - bm->idx], name, strlen(name), &dummy, sizeof(dummy)); - continue; - } - if (strlen(name) >= sstLen && strncmp(name + strlen(name) - 4, pSST, sstLen) == 0) { - // char* p = taosStrdup(name); - taosHashPut(bm->pSstTbl[1 - bm->idx], name, strlen(name), &dummy, sizeof(dummy)); - continue; - } - } - if (bm->init == 0) { - bm->preCkptId = -1; - bm->curChkpId = chkpId; - bm->init = 1; - - void* pIter = taosHashIterate(bm->pSstTbl[1 - bm->idx], NULL); - while (pIter) { - size_t len; - char* name = taosHashGetKey(pIter, &len); - if (name != NULL && len != 0) { - taosArrayPush(bm->pAdd, &name); - } - pIter = taosHashIterate(bm->pSstTbl[1 - bm->idx], pIter); - } - if (taosArrayGetSize(bm->pAdd) > 0) bm->update = 1; - } else { - int32_t code = compareHashTable(bm->pSstTbl[bm->idx], bm->pSstTbl[1 - bm->idx], bm->pAdd, bm->pDel); - if (code != 0) { - // dead code - taosArrayClearP(bm->pAdd, taosMemoryFree); - taosArrayClearP(bm->pDel, taosMemoryFree); - taosHashClear(bm->pSstTbl[1 - bm->idx]); - bm->update = 0; - taosCloseDir(&pDir); - return code; - } - - bm->preCkptId = bm->curChkpId; - bm->curChkpId = chkpId; - if (taosArrayGetSize(bm->pAdd) == 0 && taosArrayGetSize(bm->pDel) == 0) { - bm->update = 0; - } - } - taosHashClear(bm->pSstTbl[bm->idx]); - bm->idx = 1 - bm->idx; - - taosCloseDir(&pDir); - - return 0; -} - -int32_t bkdMgtDumpTo(SBackendManager* bm, char* dname) { - int32_t code = 0; - int32_t len = bm->len + 128; - - char* srcBuf = taosMemoryCalloc(1, len); - char* dstBuf = taosMemoryCalloc(1, len); - - char* srcDir = taosMemoryCalloc(1, len); - char* dstDir = taosMemoryCalloc(1, len); - - sprintf(srcDir, "%s%s%s%" PRId64 "", bm->path, TD_DIRSEP, "checkpoint", bm->curChkpId); - sprintf(dstDir, "%s%s%s", bm->path, TD_DIRSEP, dname); - - if (!taosDirExist(srcDir)) { - stError("failed to dump srcDir %s, reason: not exist such dir", srcDir); - code = -1; - goto _ERROR; - } - - code = taosMkDir(dstDir); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - stError("failed to mkdir srcDir %s, reason: %s", dstDir, terrstr()); - goto _ERROR; - } - - // clear current file - memset(dstBuf, 0, len); - sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, bm->pCurrent); - taosRemoveFile(dstBuf); - - memset(dstBuf, 0, len); - sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, bm->pManifest); - taosRemoveFile(dstBuf); - - // add file to $name dir - for (int i = 0; i < taosArrayGetSize(bm->pAdd); i++) { - memset(dstBuf, 0, len); - memset(srcBuf, 0, len); - - char* filename = taosArrayGetP(bm->pAdd, i); - sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, filename); - sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, filename); - - taosCopyFile(srcBuf, dstBuf); - } - // del file in $name - for (int i = 0; i < taosArrayGetSize(bm->pDel); i++) { - memset(dstBuf, 0, len); - memset(srcBuf, 0, len); - - char* filename = taosArrayGetP(bm->pDel, i); - sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, filename); - taosRemoveFile(dstBuf); - } - - // copy current file to dst dir - memset(srcBuf, 0, len); - memset(dstBuf, 0, len); - sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, bm->pCurrent); - sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, bm->pCurrent); - taosCopyFile(srcBuf, dstBuf); - - // copy manifest file to dst dir - memset(srcBuf, 0, len); - memset(dstBuf, 0, len); - sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, bm->pManifest); - sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, bm->pManifest); - taosCopyFile(srcBuf, dstBuf); - - // clear delta data buf - taosArrayClearP(bm->pAdd, taosMemoryFree); - taosArrayClearP(bm->pDel, taosMemoryFree); - -_ERROR: - taosMemoryFree(srcBuf); - taosMemoryFree(dstBuf); - taosMemoryFree(srcDir); - taosMemoryFree(dstDir); - return code; -} +#define GEN_COLUMN_FAMILY_NAME(name, idstr, SUFFIX) sprintf(name, "%s_%s", idstr, (SUFFIX)); +int32_t copyFiles(const char* src, const char* dst); +uint32_t nextPow2(uint32_t x); SCfInit ginitDict[] = { {"default", 7, 0, defaultKeyComp, defaultKeyEncode, defaultKeyDecode, defaultKeyToString, compareDefaultName, - destroyFunc, encodeValueFunc, decodeValueFunc}, - {"state", 5, 1, stateKeyDBComp, stateKeyEncode, stateKeyDecode, stateKeyToString, compareStateName, destroyFunc, - encodeValueFunc, decodeValueFunc}, - {"fill", 4, 2, winKeyDBComp, winKeyEncode, winKeyDecode, winKeyToString, compareWinKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc}, + destroyCompare, valueEncode, valueDecode, compactFilteFactoryCreateFilter, destroyCompactFilteFactory, + compactFilteFactoryName}, + + {"state", 5, 1, stateKeyDBComp, stateKeyEncode, stateKeyDecode, stateKeyToString, compareStateName, destroyCompare, + valueEncode, valueDecode, compactFilteFactoryCreateFilterState, destroyCompactFilteFactory, + compactFilteFactoryNameState}, + + {"fill", 4, 2, winKeyDBComp, winKeyEncode, winKeyDecode, winKeyToString, compareWinKeyName, destroyCompare, + valueEncode, valueDecode, compactFilteFactoryCreateFilterFill, destroyCompactFilteFactory, + compactFilteFactoryNameFill}, + {"sess", 4, 3, stateSessionKeyDBComp, stateSessionKeyEncode, stateSessionKeyDecode, stateSessionKeyToString, - compareSessionKeyName, destroyFunc, encodeValueFunc, decodeValueFunc}, - {"func", 4, 4, tupleKeyDBComp, tupleKeyEncode, tupleKeyDecode, tupleKeyToString, compareFuncKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc}, - {"parname", 7, 5, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, compareParKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc}, - {"partag", 6, 6, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, comparePartagKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc}, + compareSessionKeyName, destroyCompare, valueEncode, valueDecode, compactFilteFactoryCreateFilterSess, + destroyCompactFilteFactory, compactFilteFactoryNameSess}, + + {"func", 4, 4, tupleKeyDBComp, tupleKeyEncode, tupleKeyDecode, tupleKeyToString, compareFuncKeyName, destroyCompare, + valueEncode, valueDecode, compactFilteFactoryCreateFilterFunc, destroyCompactFilteFactory, + compactFilteFactoryNameFunc}, + + {"parname", 7, 5, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, compareParKeyName, destroyCompare, + valueEncode, valueDecode, compactFilteFactoryCreateFilter, destroyCompactFilteFactory, compactFilteFactoryName}, + + {"partag", 6, 6, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, comparePartagKeyName, destroyCompare, + valueEncode, valueDecode, compactFilteFactoryCreateFilter, destroyCompactFilteFactory, compactFilteFactoryName}, }; -bool isValidCheckpoint(const char* dir) { return true; } - -int32_t copyFiles(const char* src, const char* dst) { - int32_t code = 0; - // opt later, just hard link - int32_t sLen = strlen(src); - int32_t dLen = strlen(dst); - char* srcName = taosMemoryCalloc(1, sLen + 64); - char* dstName = taosMemoryCalloc(1, dLen + 64); - - TdDirPtr pDir = taosOpenDir(src); - if (pDir == NULL) { - taosMemoryFree(srcName); - taosMemoryFree(dstName); - return -1; - } - - TdDirEntryPtr de = NULL; - while ((de = taosReadDir(pDir)) != NULL) { - char* name = taosGetDirEntryName(de); - if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue; - - sprintf(srcName, "%s%s%s", src, TD_DIRSEP, name); - sprintf(dstName, "%s%s%s", dst, TD_DIRSEP, name); - if (!taosDirEntryIsDir(de)) { - code = taosCopyFile(srcName, dstName); - if (code == -1) { - goto _err; - } +int32_t getCfIdx(const char* cfName) { + int idx = -1; + size_t len = strlen(cfName); + for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) { + if (len == ginitDict[i].len && strncmp(cfName, ginitDict[i].key, strlen(cfName)) == 0) { + idx = i; + break; } - - memset(srcName, 0, sLen + 64); - memset(dstName, 0, dLen + 64); } - -_err: - taosMemoryFreeClear(srcName); - taosMemoryFreeClear(dstName); - taosCloseDir(&pDir); - return code >= 0 ? 0 : -1; + return idx; } + +bool isValidCheckpoint(const char* dir) { + return true; + STaskDbWrapper* pDb = taskDbOpenImpl(NULL, NULL, (char*)dir); + if (pDb == NULL) { + return false; + } + taskDbDestroy(pDb, false); + return true; +} + int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) { // impl later int32_t code = 0; @@ -451,6 +221,7 @@ int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) { } taosMkDir(state); code = copyFiles(chkp, state); + stInfo("copy snap file from %s to %s", chkp, state); if (code != 0) { stError("failed to restart stream backend from %s, reason: %s", chkp, tstrerror(TAOS_SYSTEM_ERROR(errno))); } else { @@ -468,7 +239,246 @@ int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) { return 0; } +int32_t remoteChkp_readMetaData(char* path, SArray* list) { + char* metaPath = taosMemoryCalloc(1, strlen(path)); + sprintf(metaPath, "%s%s%s", path, TD_DIRSEP, "META"); + TdFilePtr pFile = taosOpenFile(path, TD_FILE_READ); + + char buf[128] = {0}; + if (taosReadFile(pFile, buf, sizeof(buf)) <= 0) { + taosMemoryFree(metaPath); + taosCloseFile(&pFile); + return -1; + } + int32_t len = strlen(buf); + for (int i = 0; i < len; i++) { + if (buf[i] == '\n') { + char* item = taosMemoryCalloc(1, i + 1); + memcpy(item, buf, i); + taosArrayPush(list, &item); + + item = taosMemoryCalloc(1, len - i); + memcpy(item, buf + i + 1, len - i - 1); + taosArrayPush(list, &item); + } + } + + taosCloseFile(&pFile); + taosMemoryFree(metaPath); + return 0; +} +int32_t remoteChkp_validMetaFile(char* name, char* prename, int64_t chkpId) { + int8_t valid = 0; + for (int i = 0; i < strlen(name); i++) { + if (name[i] == '_') { + memcpy(prename, name, i); + if (taosStr2int64(name + i + 1) != chkpId) { + break; + } else { + valid = 1; + } + } + } + return valid; +} +int32_t remoteChkp_validAndCvtMeta(char* path, SArray* list, int64_t chkpId) { + int32_t complete = 1; + int32_t len = strlen(path) + 32; + char* src = taosMemoryCalloc(1, len); + char* dst = taosMemoryCalloc(1, len); + + int8_t count = 0; + for (int i = 0; i < taosArrayGetSize(list); i++) { + char* p = taosArrayGetP(list, i); + sprintf(src, "%s%s%s", path, TD_DIRSEP, p); + + // check file exist + if (taosStatFile(src, NULL, NULL, NULL) != 0) { + complete = 0; + break; + } + + // check file name + char temp[64] = {0}; + if (remoteChkp_validMetaFile(p, temp, chkpId)) { + count++; + } + + // rename file + sprintf(dst, "%s%s%s", path, TD_DIRSEP, temp); + taosRenameFile(src, dst); + + memset(src, 0, len); + memset(dst, 0, len); + } + if (count != taosArrayGetSize(list)) { + complete = 0; + } + + taosMemoryFree(src); + taosMemoryFree(dst); + + return complete == 1 ? 0 : -1; +} + +int32_t rebuildFromRemoteChkp_rsync(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { + // impl later + int32_t code = 0; + if (taosIsDir(chkpPath)) { + taosRemoveDir(chkpPath); + } + if (taosIsDir(defaultPath)) { + taosRemoveDir(defaultPath); + } + + code = downloadCheckpoint(key, chkpPath); + if (code != 0) { + return code; + } + code = copyFiles(chkpPath, defaultPath); + + return code; +} +int32_t rebuildFromRemoteChkp_s3(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { + int32_t code = downloadCheckpoint(key, chkpPath); + if (code != 0) { + return code; + } + + int32_t len = strlen(defaultPath) + 32; + char* tmp = taosMemoryCalloc(1, len); + sprintf(tmp, "%s%s", defaultPath, "_tmp"); + if (taosIsDir(tmp)) taosRemoveDir(tmp); + if (taosIsDir(defaultPath)) taosRenameFile(defaultPath, tmp); + + SArray* list = taosArrayInit(2, sizeof(void*)); + code = remoteChkp_readMetaData(chkpPath, list); + if (code == 0) { + code = remoteChkp_validAndCvtMeta(chkpPath, list, chkpId); + } + taosArrayDestroyP(list, taosMemoryFree); + + if (code == 0) { + taosMkDir(defaultPath); + code = copyFiles(chkpPath, defaultPath); + } + + if (code != 0) { + if (taosIsDir(defaultPath)) taosRemoveDir(defaultPath); + if (taosIsDir(tmp)) taosRenameFile(tmp, defaultPath); + } else { + taosRemoveDir(tmp); + } + + taosMemoryFree(tmp); + return code; +} +int32_t rebuildFromRemoteChkp(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { + UPLOAD_TYPE type = getUploadType(); + if (type == UPLOAD_S3) { + return rebuildFromRemoteChkp_s3(key, chkpPath, chkpId, defaultPath); + } else if (type == UPLOAD_RSYNC) { + return rebuildFromRemoteChkp_rsync(key, chkpPath, chkpId, defaultPath); + } + return -1; +} + +int32_t rebuildFromLocalChkp(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { + int32_t code = -1; + int32_t len = strlen(defaultPath) + 32; + char* tmp = taosMemoryCalloc(1, len); + sprintf(tmp, "%s%s", defaultPath, "_tmp"); + + if (taosIsDir(tmp)) taosRemoveDir(tmp); + if (taosIsDir(defaultPath)) taosRenameFile(defaultPath, tmp); + + if (taosIsDir(chkpPath) && isValidCheckpoint(chkpPath)) { + if (taosIsDir(tmp)) { + taosRemoveDir(tmp); + } + taosMkDir(defaultPath); + code = copyFiles(chkpPath, defaultPath); + if (code != 0) { + stError("failed to restart stream backend from %s, reason: %s", chkpPath, tstrerror(TAOS_SYSTEM_ERROR(errno))); + } else { + stInfo("start to restart stream backend at checkpoint path: %s", chkpPath); + } + } + if (code != 0) { + if (taosIsDir(defaultPath)) taosRemoveDir(defaultPath); + if (taosIsDir(tmp)) taosRenameFile(tmp, defaultPath); + } else { + taosRemoveDir(tmp); + } + + taosMemoryFree(tmp); + return code; +} + +int32_t rebuildFromlocalDefault(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { + int32_t code = 0; + return code; +} + +int32_t rebuildDirFromChkp2(const char* path, char* key, int64_t chkpId, char** dbPrefixPath, char** dbPath) { + // impl later + int32_t code = 0; + + char* prefixPath = taosMemoryCalloc(1, strlen(path) + 128); + sprintf(prefixPath, "%s%s%s", path, TD_DIRSEP, key); + + if (!taosIsDir(prefixPath)) { + code = taosMkDir(prefixPath); + ASSERT(code == 0); + } + + char* defaultPath = taosMemoryCalloc(1, strlen(path) + 256); + sprintf(defaultPath, "%s%s%s", prefixPath, TD_DIRSEP, "state"); + if (!taosIsDir(defaultPath)) { + taosMulMkDir(defaultPath); + } + + char* chkpPath = taosMemoryCalloc(1, strlen(path) + 256); + if (chkpId != 0) { + sprintf(chkpPath, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", chkpId); + code = rebuildFromLocalChkp(key, chkpPath, chkpId, defaultPath); + if (code != 0) { + code = rebuildFromRemoteChkp(key, chkpPath, chkpId, defaultPath); + } + + if (code != 0) { + stInfo("failed to start stream backend at %s, reason: %s, restart from default defaultPath dir:%s", chkpPath, + tstrerror(TAOS_SYSTEM_ERROR(errno)), defaultPath); + code = taosMkDir(defaultPath); + } + } else { + sprintf(chkpPath, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", + (int64_t)-1); + + code = rebuildFromLocalChkp(key, chkpPath, -1, defaultPath); + if (code != 0) { + code = taosMkDir(defaultPath); + } + } + taosMemoryFree(chkpPath); + + *dbPath = defaultPath; + *dbPrefixPath = prefixPath; + + return code; +} + +bool streamBackendDataIsExist(const char* path, int64_t chkpId, int32_t vgId) { + bool exist = true; + char* state = taosMemoryCalloc(1, strlen(path) + 32); + sprintf(state, "%s%s%s", path, TD_DIRSEP, "state"); + if (!taosDirExist(state)) { + exist = false; + } + taosMemoryFree(state); + return exist; +} void* streamBackendInit(const char* streamPath, int64_t chkpId, int32_t vgId) { char* backendPath = NULL; int32_t code = rebuildDirFromCheckpoint(streamPath, chkpId, &backendPath); @@ -648,7 +658,7 @@ void streamBackendHandleCleanup(void* arg) { taosThreadRwlockDestroy(&wrapper->rwLock); wrapper->rocksdb = NULL; - taosReleaseRef(streamBackendId, wrapper->backendId); + // taosReleaseRef(streamBackendId, wrapper->backendId); stDebug("end to do-close backendwrapper %p, %s", wrapper, wrapper->idstr); taosMemoryFree(wrapper); @@ -728,8 +738,65 @@ int32_t delObsoleteCheckpoint(void* arg, const char* path) { taosArrayDestroy(chkpDel); return 0; } +/* + * checkpointSave |--cp1--|--cp2--|--cp3--|--cp4--|--cp5--| + * chkpInUse: |--cp2--|--cp4--| + * chkpInUse is doing translation, cannot del until + * replication is finished + */ +int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) { + STaskDbWrapper* pBackend = arg; -static int32_t compareCheckpoint(const void* a, const void* b) { + taosThreadRwlockWrlock(&pBackend->chkpDirLock); + + taosArrayPush(pBackend->chkpSaved, &chkpId); + + SArray* chkpDel = taosArrayInit(8, sizeof(int64_t)); + SArray* chkpDup = taosArrayInit(8, sizeof(int64_t)); + + int64_t firsId = 0; + if (taosArrayGetSize(pBackend->chkpInUse) >= 1) { + firsId = *(int64_t*)taosArrayGet(pBackend->chkpInUse, 0); + + for (int i = 0; i < taosArrayGetSize(pBackend->chkpSaved); i++) { + int64_t id = *(int64_t*)taosArrayGet(pBackend->chkpSaved, i); + if (id >= firsId) { + taosArrayPush(chkpDup, &id); + } else { + taosArrayPush(chkpDel, &id); + } + } + } else { + int32_t sz = taosArrayGetSize(pBackend->chkpSaved); + int32_t dsz = sz - pBackend->chkpCap; // del size + + for (int i = 0; i < dsz; i++) { + int64_t id = *(int64_t*)taosArrayGet(pBackend->chkpSaved, i); + taosArrayPush(chkpDel, &id); + } + for (int i = dsz < 0 ? 0 : dsz; i < sz; i++) { + int64_t id = *(int64_t*)taosArrayGet(pBackend->chkpSaved, i); + taosArrayPush(chkpDup, &id); + } + } + taosArrayDestroy(pBackend->chkpSaved); + pBackend->chkpSaved = chkpDup; + + taosThreadRwlockUnlock(&pBackend->chkpDirLock); + + for (int i = 0; i < taosArrayGetSize(chkpDel); i++) { + int64_t id = *(int64_t*)taosArrayGet(chkpDel, i); + char tbuf[256] = {0}; + sprintf(tbuf, "%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, id); + if (taosIsDir(tbuf)) { + taosRemoveDir(tbuf); + } + } + taosArrayDestroy(chkpDel); + return 0; +} + +static int32_t chkpIdComp(const void* a, const void* b) { int64_t x = *(int64_t*)a; int64_t y = *(int64_t*)b; return x < y ? -1 : 1; @@ -774,7 +841,7 @@ int32_t streamBackendLoadCheckpointInfo(void* arg) { continue; } } - taosArraySort(suffix, compareCheckpoint); + taosArraySort(suffix, chkpIdComp); // free previous chkpSaved taosArrayClear(pMeta->chkpSaved); for (int i = 0; i < taosArrayGetSize(suffix); i++) { @@ -789,30 +856,50 @@ int32_t streamBackendLoadCheckpointInfo(void* arg) { } int32_t chkpGetAllDbCfHandle(SStreamMeta* pMeta, rocksdb_column_family_handle_t*** ppHandle, SArray* refs) { - SArray* pHandle = taosArrayInit(16, POINTER_BYTES); - void* pIter = taosHashIterate(pMeta->pTaskBackendUnique, NULL); - while (pIter) { - int64_t id = *(int64_t*)pIter; + return 0; + // SArray* pHandle = taosArrayInit(16, POINTER_BYTES); + // void* pIter = taosHashIterate(pMeta->pTaskDbUnique, NULL); + // while (pIter) { + // int64_t id = *(int64_t*)pIter; - SBackendCfWrapper* wrapper = taosAcquireRef(streamBackendCfWrapperId, id); - if (wrapper == NULL) { - pIter = taosHashIterate(pMeta->pTaskBackendUnique, pIter); - continue; + // SBackendCfWrapper* wrapper = taosAcquireRef(streamBackendCfWrapperId, id); + // if (wrapper == NULL) { + // pIter = taosHashIterate(pMeta->pTaskDbUnique, pIter); + // continue; + // } + + // taosThreadRwlockRdlock(&wrapper->rwLock); + // for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) { + // if (wrapper->pHandle[i]) { + // rocksdb_column_family_handle_t* p = wrapper->pHandle[i]; + // taosArrayPush(pHandle, &p); + // } + // } + // taosThreadRwlockUnlock(&wrapper->rwLock); + + // taosArrayPush(refs, &id); + // } + + // int32_t nCf = taosArrayGetSize(pHandle); + + // rocksdb_column_family_handle_t** ppCf = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*)); + // for (int i = 0; i < nCf; i++) { + // ppCf[i] = taosArrayGetP(pHandle, i); + // } + // taosArrayDestroy(pHandle); + + // *ppHandle = ppCf; + // return nCf; +} + +int32_t chkpGetAllDbCfHandle2(STaskDbWrapper* pBackend, rocksdb_column_family_handle_t*** ppHandle) { + SArray* pHandle = taosArrayInit(8, POINTER_BYTES); + for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) { + if (pBackend->pCf[i]) { + rocksdb_column_family_handle_t* p = pBackend->pCf[i]; + taosArrayPush(pHandle, &p); } - - taosThreadRwlockRdlock(&wrapper->rwLock); - for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) { - if (wrapper->pHandle[i]) { - rocksdb_column_family_handle_t* p = wrapper->pHandle[i]; - taosArrayPush(pHandle, &p); - } - } - taosThreadRwlockUnlock(&wrapper->rwLock); - - taosArrayPush(refs, &id); - pIter = taosHashIterate(pMeta->pTaskBackendUnique, pIter); } - int32_t nCf = taosArrayGetSize(pHandle); if (nCf == 0) { taosArrayDestroy(pHandle); @@ -828,6 +915,7 @@ int32_t chkpGetAllDbCfHandle(SStreamMeta* pMeta, rocksdb_column_family_handle_t* *ppHandle = ppCf; return nCf; } + int32_t chkpDoDbCheckpoint(rocksdb_t* db, char* path) { int32_t code = -1; char* err = NULL; @@ -866,7 +954,8 @@ int32_t chkpPreFlushDb(rocksdb_t* db, rocksdb_column_family_handle_t** cf, int32 rocksdb_flushoptions_destroy(flushOpt); return code; } -int32_t chkpPreCheckDir(char* path, int64_t chkpId, char** chkpDir, char** chkpIdDir) { + +int32_t chkpPreBuildDir(char* path, int64_t chkpId, char** chkpDir, char** chkpIdDir) { int32_t code = 0; char* pChkpDir = taosMemoryCalloc(1, 256); char* pChkpIdDir = taosMemoryCalloc(1, 256); @@ -881,145 +970,111 @@ int32_t chkpPreCheckDir(char* path, int64_t chkpId, char** chkpDir, char** chkpI return code; } - sprintf(pChkpIdDir, "%s%scheckpoint%" PRId64, pChkpDir, TD_DIRSEP, chkpId); + sprintf(pChkpIdDir, "%s%s%s%" PRId64, pChkpDir, TD_DIRSEP, "checkpoint", chkpId); if (taosIsDir(pChkpIdDir)) { stInfo("stream rm exist checkpoint%s", pChkpIdDir); - taosRemoveFile(pChkpIdDir); + taosRemoveDir(pChkpIdDir); } *chkpDir = pChkpDir; *chkpIdDir = pChkpIdDir; return 0; } - -int32_t streamBackendTriggerChkp(void* arg, char* dst) { +int32_t taskDbBuildSnap(void* arg, SArray* pSnap) { SStreamMeta* pMeta = arg; - int64_t backendRid = pMeta->streamBackendRid; - int32_t code = -1; + void* pIter = taosHashIterate(pMeta->pTaskDbUnique, NULL); + int32_t code = 0; - SArray* refs = taosArrayInit(16, sizeof(int64_t)); - rocksdb_column_family_handle_t** ppCf = NULL; + while (pIter) { + STaskDbWrapper* pTaskDb = *(STaskDbWrapper**)pIter; + taskDbAddRef(pTaskDb); - int64_t st = taosGetTimestampMs(); - SBackendWrapper* pHandle = taosAcquireRef(streamBackendId, backendRid); + code = taskDbDoCheckpoint(pTaskDb, pTaskDb->chkpId); + taskDbRemoveRef(pTaskDb); - if (pHandle == NULL || pHandle->db == NULL) { - goto _ERROR; + SStreamTask* pTask = pTaskDb->pTask; + SStreamTaskSnap snap = {.streamId = pTask->id.streamId, + .taskId = pTask->id.taskId, + .chkpId = pTaskDb->chkpId, + .dbPrefixPath = taosStrdup(pTaskDb->path)}; + taosArrayPush(pSnap, &snap); + pIter = taosHashIterate(pMeta->pTaskDbUnique, pIter); } - int32_t nCf = chkpGetAllDbCfHandle(pMeta, &ppCf, refs); - stDebug("stream backend:%p start to do checkpoint at:%s, cf num: %d ", pHandle, dst, nCf); - - code = chkpPreFlushDb(pHandle->db, ppCf, nCf); - if (code == 0) { - code = chkpDoDbCheckpoint(pHandle->db, dst); - if (code != 0) { - stError("stream backend:%p failed to do checkpoint at:%s", pHandle, dst); - } else { - stDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pHandle, dst, - taosGetTimestampMs() - st); - } - } else { - stError("stream backend:%p failed to flush db at:%s", pHandle, dst); - } - - // release all ref to cfWrapper; - for (int i = 0; i < taosArrayGetSize(refs); i++) { - int64_t id = *(int64_t*)taosArrayGet(refs, i); - taosReleaseRef(streamBackendCfWrapperId, id); - } - -_ERROR: - taosMemoryFree(ppCf); - taosReleaseRef(streamBackendId, backendRid); - taosArrayDestroy(refs); return code; } int32_t streamBackendAddInUseChkp(void* arg, int64_t chkpId) { - if (arg == NULL) return 0; + // if (arg == NULL) return 0; - SStreamMeta* pMeta = arg; - taosWLockLatch(&pMeta->chkpDirLock); - taosArrayPush(pMeta->chkpInUse, &chkpId); - taosWUnLockLatch(&pMeta->chkpDirLock); + // SStreamMeta* pMeta = arg; + // taosWLockLatch(&pMeta->chkpDirLock); + // taosArrayPush(pMeta->chkpInUse, &chkpId); + // taosWUnLockLatch(&pMeta->chkpDirLock); return 0; } int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId) { - if (arg == NULL) return 0; - - SStreamMeta* pMeta = arg; - taosWLockLatch(&pMeta->chkpDirLock); - if (taosArrayGetSize(pMeta->chkpInUse) > 0) { - int64_t id = *(int64_t*)taosArrayGet(pMeta->chkpInUse, 0); - if (id == chkpId) { - taosArrayPopFrontBatch(pMeta->chkpInUse, 1); - } - } - taosWUnLockLatch(&pMeta->chkpDirLock); return 0; + // if (arg == NULL) return 0; + + // SStreamMeta* pMeta = arg; + // taosWLockLatch(&pMeta->chkpDirLock); + // if (taosArrayGetSize(pMeta->chkpInUse) > 0) { + // int64_t id = *(int64_t*)taosArrayGet(pMeta->chkpInUse, 0); + // if (id == chkpId) { + // taosArrayPopFrontBatch(pMeta->chkpInUse, 1); + // } + // } + // taosWUnLockLatch(&pMeta->chkpDirLock); } -int32_t streamBackendDoCheckpoint(void* arg, uint64_t checkpointId) { - SStreamMeta* pMeta = arg; - int64_t backendRid = pMeta->streamBackendRid; - int64_t st = taosGetTimestampMs(); - int32_t code = -1; +/* + 0 +*/ +int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId) { + STaskDbWrapper* pTaskDb = arg; + int64_t st = taosGetTimestampMs(); + int32_t code = -1; + int64_t refId = pTaskDb->refId; - SArray* refs = taosArrayInit(16, sizeof(int64_t)); - - rocksdb_column_family_handle_t** ppCf = NULL; + if (taosAcquireRef(taskDbWrapperId, refId) == NULL) { + return -1; + } char* pChkpDir = NULL; char* pChkpIdDir = NULL; - if (chkpPreCheckDir(pMeta->path, checkpointId, &pChkpDir, &pChkpIdDir) != 0) { - taosArrayDestroy(refs); - return code; + if (chkpPreBuildDir(pTaskDb->path, chkpId, &pChkpDir, &pChkpIdDir) != 0) { + code = -1; + goto _EXIT; } - - SBackendWrapper* pHandle = taosAcquireRef(streamBackendId, backendRid); - if (pHandle == NULL || pHandle->db == NULL) { - stError("failed to acquire state-backend handle"); - goto _ERROR; - } - // Get all cf and acquire cfWrappter - int32_t nCf = chkpGetAllDbCfHandle(pMeta, &ppCf, refs); - stDebug("stream backend:%p start to do checkpoint at:%s, cf num: %d ", pHandle, pChkpIdDir, nCf); + rocksdb_column_family_handle_t** ppCf = NULL; - code = chkpPreFlushDb(pHandle->db, ppCf, nCf); - if (code == 0) { - code = chkpDoDbCheckpoint(pHandle->db, pChkpIdDir); - if (code != 0) { - stError("stream backend:%p failed to do checkpoint at:%s", pHandle, pChkpIdDir); + int32_t nCf = chkpGetAllDbCfHandle2(pTaskDb, &ppCf); + qDebug("stream backend:%p start to do checkpoint at:%s, cf num: %d ", pTaskDb, pChkpIdDir, nCf); + + if ((code = chkpPreFlushDb(pTaskDb->db, ppCf, nCf)) == 0) { + if ((code = chkpDoDbCheckpoint(pTaskDb->db, pChkpIdDir)) != 0) { + stError("stream backend:%p failed to do checkpoint at:%s", pTaskDb, pChkpIdDir); } else { - stDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pHandle, pChkpIdDir, - taosGetTimestampMs() - st); + qDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pTaskDb, pChkpIdDir, + taosGetTimestampMs() - st); } } else { - stError("stream backend:%p failed to flush db at:%s", pHandle, pChkpIdDir); - } - // release all ref to cfWrapper; - for (int i = 0; i < taosArrayGetSize(refs); i++) { - int64_t id = *(int64_t*)taosArrayGet(refs, i); - taosReleaseRef(streamBackendCfWrapperId, id); - } - if (code == 0) { - taosWLockLatch(&pMeta->chkpDirLock); - taosArrayPush(pMeta->chkpSaved, &checkpointId); - taosWUnLockLatch(&pMeta->chkpDirLock); - - // delete obsolte checkpoint - delObsoleteCheckpoint(arg, pChkpDir); - pMeta->chkpId = checkpointId; + stError("stream backend:%p failed to flush db at:%s", pTaskDb, pChkpIdDir); } -_ERROR: - taosReleaseRef(streamBackendId, backendRid); - taosArrayDestroy(refs); - taosMemoryFree(ppCf); + code = chkpMayDelObsolete(pTaskDb, chkpId, pChkpDir); + pTaskDb->dataWritten = 0; + + pTaskDb->chkpId = chkpId; + +_EXIT: taosMemoryFree(pChkpDir); taosMemoryFree(pChkpIdDir); + taosReleaseRef(taskDbWrapperId, refId); + taosMemoryFree(ppCf); return code; } +int32_t streamBackendDoCheckpoint(void* arg, int64_t chkpId) { return taskDbDoCheckpoint(arg, chkpId); } SListNode* streamBackendAddCompare(void* backend, void* arg) { SBackendWrapper* pHandle = (SBackendWrapper*)backend; @@ -1040,15 +1095,33 @@ void streamBackendDelCompare(void* backend, void* arg) { taosMemoryFree(node); } } -void streamStateDestroy_rocksdb(SStreamState* pState, bool remove) { streamStateCloseBackend(pState, remove); } -static bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len); +void streamStateDestroy_rocksdb(SStreamState* pState, bool remove) { streamStateCloseBackend(pState, remove); } +void destroyRocksdbCfInst(RocksdbCfInst* inst) { + int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); + if (inst->pHandle) { + for (int i = 0; i < cfLen; i++) { + if (inst->pHandle[i]) rocksdb_column_family_handle_destroy((inst->pHandle)[i]); + } + taosMemoryFree(inst->pHandle); + } + + if (inst->cfOpt) { + for (int i = 0; i < cfLen; i++) { + rocksdb_options_destroy(inst->cfOpt[i]); + rocksdb_block_based_options_destroy(((RocksdbCfParam*)inst->param)[i].tableOpt); + } + taosMemoryFreeClear(inst->cfOpt); + taosMemoryFreeClear(inst->param); + } + if (inst->wOpt) rocksdb_writeoptions_destroy(inst->wOpt); + if (inst->rOpt) rocksdb_readoptions_destroy(inst->rOpt); + + taosMemoryFree(inst); +} // |key|-----value------| // |key|ttl|len|userData| -static rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, - rocksdb_snapshot_t** snapshot, rocksdb_readoptions_t** readOpt); - int defaultKeyComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen) { int len = aLen < bLen ? aLen : bLen; int ret = memcmp(aBuf, bBuf, len); @@ -1331,23 +1404,7 @@ int parKeyToString(void* k, char* buf) { n = sprintf(buf + n, "[groupId:%" PRIi64 "]", *key); return n; } -int stremaValueEncode(void* k, char* buf) { - int len = 0; - SStreamValue* key = k; - len += taosEncodeFixedI64((void**)&buf, key->unixTimestamp); - len += taosEncodeFixedI32((void**)&buf, key->len); - len += taosEncodeBinary((void**)&buf, key->data, key->len); - return len; -} -int streamValueDecode(void* k, char* buf) { - SStreamValue* key = k; - char* p = buf; - p = taosDecodeFixedI64(p, &key->unixTimestamp); - p = taosDecodeFixedI32(p, &key->len); - p = taosDecodeBinary(p, (void**)&key->data, key->len); - return p - buf; -} -int32_t streamValueToString(void* k, char* buf) { +int32_t valueToString(void* k, char* buf) { SStreamValue* key = k; int n = 0; n += sprintf(buf + n, "[unixTimestamp:%" PRIi64 ",", key->unixTimestamp); @@ -1357,7 +1414,7 @@ int32_t streamValueToString(void* k, char* buf) { } /*1: stale, 0: no stale*/ -int32_t streaValueIsStale(void* k, int64_t ts) { +int32_t valueIsStale(void* k, int64_t ts) { SStreamValue* key = k; if (key->unixTimestamp < ts) { return 1; @@ -1365,12 +1422,12 @@ int32_t streaValueIsStale(void* k, int64_t ts) { return 0; } -void destroyFunc(void* arg) { +void destroyCompare(void* arg) { (void)arg; return; } -int32_t encodeValueFunc(void* value, int32_t vlen, int64_t ttl, char** dest) { +int32_t valueEncode(void* value, int32_t vlen, int64_t ttl, char** dest) { SStreamValue key = {.unixTimestamp = ttl, .len = vlen, .data = (char*)(value)}; int32_t len = 0; if (*dest == NULL) { @@ -1392,7 +1449,7 @@ int32_t encodeValueFunc(void* value, int32_t vlen, int64_t ttl, char** dest) { * ret >= 0 : found valid value * ret < 0 : error or timeout */ -int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest) { +int32_t valueDecode(void* value, int32_t vlen, int64_t* ttl, char** dest) { SStreamValue key = {0}; char* p = value; if (streamStateValueIsStale(p)) { @@ -1449,7 +1506,23 @@ void destroyCompactFilteFactory(void* arg) { } const char* compactFilteFactoryName(void* arg) { SCompactFilteFactory* state = arg; - return "stream_compact_filter"; + return "stream_compact_factory_filter_default"; +} +const char* compactFilteFactoryNameSess(void* arg) { + SCompactFilteFactory* state = arg; + return "stream_compact_factory_filter_sess"; +} +const char* compactFilteFactoryNameState(void* arg) { + SCompactFilteFactory* state = arg; + return "stream_compact_factory_filter_state"; +} +const char* compactFilteFactoryNameFill(void* arg) { + SCompactFilteFactory* state = arg; + return "stream_compact_factory_filter_fill"; +} +const char* compactFilteFactoryNameFunc(void* arg) { + SCompactFilteFactory* state = arg; + return "stream_compact_factory_filter_func"; } void destroyCompactFilte(void* arg) { (void)arg; } @@ -1457,38 +1530,499 @@ unsigned char compactFilte(void* arg, int level, const char* key, size_t klen, c char** newval, size_t* newvlen, unsigned char* value_changed) { return streamStateValueIsStale((char*)val) ? 1 : 0; } -const char* compactFilteName(void* arg) { return "stream_filte"; } +const char* compactFilteName(void* arg) { return "stream_filte_default"; } +const char* compactFilteNameSess(void* arg) { return "stream_filte_sess"; } +const char* compactFilteNameState(void* arg) { return "stream_filte_state"; } +const char* compactFilteNameFill(void* arg) { return "stream_filte_fill"; } +const char* compactFilteNameFunc(void* arg) { return "stream_filte_func"; } + +unsigned char compactFilteSess(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, + char** newval, size_t* newvlen, unsigned char* value_changed) { + // not impl yet + return 0; +} + +unsigned char compactFilteState(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, + char** newval, size_t* newvlen, unsigned char* value_changed) { + // not impl yet + return 0; +} + +unsigned char compactFilteFill(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, + char** newval, size_t* newvlen, unsigned char* value_changed) { + // not impl yet + return 0; +} + +unsigned char compactFilteFunc(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, + char** newval, size_t* newvlen, unsigned char* value_changed) { + // not impl yet + return 0; + // return streamStateValueIsStale((char*)val) ? 1 : 0; +} rocksdb_compactionfilter_t* compactFilteFactoryCreateFilter(void* arg, rocksdb_compactionfiltercontext_t* ctx) { SCompactFilteFactory* state = arg; rocksdb_compactionfilter_t* filter = - rocksdb_compactionfilter_create(NULL, destroyCompactFilte, compactFilte, compactFilteName); + rocksdb_compactionfilter_create(state, destroyCompactFilte, compactFilte, compactFilteName); + return filter; +} +rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterSess(void* arg, rocksdb_compactionfiltercontext_t* ctx) { + SCompactFilteFactory* state = arg; + rocksdb_compactionfilter_t* filter = + rocksdb_compactionfilter_create(state, destroyCompactFilte, compactFilteSess, compactFilteNameSess); + return filter; +} +rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterState(void* arg, rocksdb_compactionfiltercontext_t* ctx) { + SCompactFilteFactory* state = arg; + rocksdb_compactionfilter_t* filter = + rocksdb_compactionfilter_create(state, destroyCompactFilte, compactFilteState, compactFilteNameState); + return filter; +} +rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFill(void* arg, rocksdb_compactionfiltercontext_t* ctx) { + SCompactFilteFactory* state = arg; + rocksdb_compactionfilter_t* filter = + rocksdb_compactionfilter_create(state, destroyCompactFilte, compactFilteFill, compactFilteNameFill); + return filter; +} +rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFunc(void* arg, rocksdb_compactionfiltercontext_t* ctx) { + SCompactFilteFactory* state = arg; + rocksdb_compactionfilter_t* filter = + rocksdb_compactionfilter_create(state, destroyCompactFilte, compactFilteFunc, compactFilteNameFunc); return filter; } -void destroyRocksdbCfInst(RocksdbCfInst* inst) { - int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); - if (inst->pHandle) { - for (int i = 0; i < cfLen; i++) { - if (inst->pHandle[i]) rocksdb_column_family_handle_destroy((inst->pHandle)[i]); - } - taosMemoryFree(inst->pHandle); +int32_t taskDbOpenCfs(STaskDbWrapper* pTask, char* path, char** pCfNames, int32_t nCf) { + int32_t code = -1; + char* err = NULL; + + rocksdb_options_t** cfOpts = taosMemoryCalloc(nCf, sizeof(rocksdb_options_t*)); + rocksdb_column_family_handle_t** cfHandle = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*)); + + for (int i = 0; i < nCf; i++) { + int32_t idx = getCfIdx(pCfNames[i]); + cfOpts[i] = pTask->pCfOpts[idx]; } - if (inst->cfOpt) { - for (int i = 0; i < cfLen; i++) { - rocksdb_options_destroy(inst->cfOpt[i]); - rocksdb_block_based_options_destroy(((RocksdbCfParam*)inst->param)[i].tableOpt); - } - taosMemoryFreeClear(inst->cfOpt); - taosMemoryFreeClear(inst->param); - } - if (inst->wOpt) rocksdb_writeoptions_destroy(inst->wOpt); - if (inst->rOpt) rocksdb_readoptions_destroy(inst->rOpt); + rocksdb_t* db = rocksdb_open_column_families(pTask->dbOpt, path, nCf, (const char* const*)pCfNames, + (const rocksdb_options_t* const*)cfOpts, cfHandle, &err); - taosMemoryFree(inst); + if (err != NULL) { + stError("failed to open cf path: %s", err); + taosMemoryFree(err); + goto _EXIT; + } + + for (int i = 0; i < nCf; i++) { + int32_t idx = getCfIdx(pCfNames[i]); + pTask->pCf[idx] = cfHandle[i]; + } + + pTask->db = db; + code = 0; + +_EXIT: + taosMemoryFree(cfOpts); + taosMemoryFree(cfHandle); + return code; +} +void* taskDbAddRef(void* pTaskDb) { + STaskDbWrapper* pBackend = pTaskDb; + return taosAcquireRef(taskDbWrapperId, pBackend->refId); +} +void taskDbRemoveRef(void* pTaskDb) { + if (pTaskDb == NULL) return; + STaskDbWrapper* pBackend = pTaskDb; + taosReleaseRef(taskDbWrapperId, pBackend->refId); } +void taskDbInitOpt(STaskDbWrapper* pTaskDb) { + rocksdb_env_t* env = rocksdb_create_default_env(); + + rocksdb_cache_t* cache = rocksdb_cache_create_lru(256); + rocksdb_options_t* opts = rocksdb_options_create(); + rocksdb_options_set_env(opts, env); + rocksdb_options_set_create_if_missing(opts, 1); + rocksdb_options_set_create_missing_column_families(opts, 1); + // rocksdb_options_set_max_total_wal_size(opts, dbMemLimit); + rocksdb_options_set_recycle_log_file_num(opts, 6); + rocksdb_options_set_max_write_buffer_number(opts, 3); + rocksdb_options_set_info_log_level(opts, 1); + rocksdb_options_set_db_write_buffer_size(opts, 64 << 20); + rocksdb_options_set_write_buffer_size(opts, 32 << 20); + rocksdb_options_set_atomic_flush(opts, 1); + + pTaskDb->dbOpt = opts; + pTaskDb->env = env; + pTaskDb->cache = cache; + pTaskDb->filterFactory = rocksdb_compactionfilterfactory_create( + NULL, destroyCompactFilteFactory, compactFilteFactoryCreateFilter, compactFilteFactoryName); + rocksdb_options_set_compaction_filter_factory(pTaskDb->dbOpt, pTaskDb->filterFactory); + pTaskDb->readOpt = rocksdb_readoptions_create(); + pTaskDb->writeOpt = rocksdb_writeoptions_create(); + + size_t nCf = sizeof(ginitDict) / sizeof(ginitDict[0]); + pTaskDb->pCf = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*)); + pTaskDb->pCfParams = taosMemoryCalloc(nCf, sizeof(RocksdbCfParam)); + pTaskDb->pCfOpts = taosMemoryCalloc(nCf, sizeof(rocksdb_options_t*)); + pTaskDb->pCompares = taosMemoryCalloc(nCf, sizeof(rocksdb_comparator_t*)); + + for (int i = 0; i < nCf; i++) { + rocksdb_options_t* opt = rocksdb_options_create_copy(pTaskDb->dbOpt); + rocksdb_block_based_table_options_t* tableOpt = rocksdb_block_based_options_create(); + rocksdb_block_based_options_set_block_cache(tableOpt, pTaskDb->cache); + rocksdb_block_based_options_set_partition_filters(tableOpt, 1); + + rocksdb_filterpolicy_t* filter = rocksdb_filterpolicy_create_bloom(15); + rocksdb_block_based_options_set_filter_policy(tableOpt, filter); + + rocksdb_options_set_block_based_table_factory((rocksdb_options_t*)opt, tableOpt); + + SCfInit* cfPara = &ginitDict[i]; + + rocksdb_comparator_t* compare = + rocksdb_comparator_create(NULL, cfPara->destroyCmp, cfPara->cmpKey, cfPara->cmpName); + rocksdb_options_set_comparator((rocksdb_options_t*)opt, compare); + + rocksdb_compactionfilterfactory_t* filterFactory = + rocksdb_compactionfilterfactory_create(NULL, cfPara->destroyFilter, cfPara->createFilter, cfPara->funcName); + rocksdb_options_set_compaction_filter_factory(opt, filterFactory); + + pTaskDb->pCompares[i] = compare; + pTaskDb->pCfOpts[i] = opt; + pTaskDb->pCfParams[i].tableOpt = tableOpt; + } + return; +} +void taskDbInitChkpOpt(STaskDbWrapper* pTaskDb) { + pTaskDb->chkpId = -1; + pTaskDb->chkpCap = 4; + pTaskDb->chkpSaved = taosArrayInit(4, sizeof(int64_t)); + pTaskDb->chkpInUse = taosArrayInit(4, sizeof(int64_t)); + + taosThreadRwlockInit(&pTaskDb->chkpDirLock, NULL); +} + +void taskDbDestroyChkpOpt(STaskDbWrapper* pTaskDb) { + taosArrayDestroy(pTaskDb->chkpSaved); + taosArrayDestroy(pTaskDb->chkpInUse); + taosThreadRwlockDestroy(&pTaskDb->chkpDirLock); +} + +int32_t taskDbBuildFullPath(char* path, char* key, char** dbFullPath, char** stateFullPath) { + int32_t code = 0; + + char* statePath = taosMemoryCalloc(1, strlen(path) + 128); + sprintf(statePath, "%s%s%s", path, TD_DIRSEP, key); + if (!taosDirExist(statePath)) { + code = taosMulMkDir(statePath); + if (code != 0) { + stError("failed to create dir: %s, reason:%s", statePath, tstrerror(code)); + taosMemoryFree(statePath); + return code; + } + } + + char* dbPath = taosMemoryCalloc(1, strlen(statePath) + 128); + sprintf(dbPath, "%s%s%s", statePath, TD_DIRSEP, "state"); + if (!taosDirExist(dbPath)) { + code = taosMulMkDir(dbPath); + if (code != 0) { + stError("failed to create dir: %s, reason:%s", dbPath, tstrerror(code)); + taosMemoryFree(statePath); + taosMemoryFree(dbPath); + return code; + } + } + + *dbFullPath = dbPath; + *stateFullPath = statePath; + return 0; +} +void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId) { + STaskDbWrapper* p = pTaskDb; + taosThreadMutexLock(&p->mutex); + p->chkpId = chkpId; + taosThreadMutexUnlock(&p->mutex); +} + +STaskDbWrapper* taskDbOpenImpl(char* key, char* statePath, char* dbPath) { + char* err = NULL; + char** cfNames = NULL; + size_t nCf = 0; + + STaskDbWrapper* pTaskDb = taosMemoryCalloc(1, sizeof(STaskDbWrapper)); + pTaskDb->idstr = key ? taosStrdup(key) : NULL; + pTaskDb->path = statePath ? taosStrdup(statePath) : NULL; + + taosThreadMutexInit(&pTaskDb->mutex, NULL); + taskDbInitChkpOpt(pTaskDb); + taskDbInitOpt(pTaskDb); + + cfNames = rocksdb_list_column_families(pTaskDb->dbOpt, dbPath, &nCf, &err); + if (nCf == 0) { + stInfo("newly create db, need to restart"); + // pre create db + pTaskDb->db = rocksdb_open(pTaskDb->pCfOpts[0], dbPath, &err); + rocksdb_close(pTaskDb->db); + + if (cfNames != NULL) { + rocksdb_list_column_families_destroy(cfNames, nCf); + } + taosMemoryFree(err); + err = NULL; + + cfNames = rocksdb_list_column_families(pTaskDb->dbOpt, dbPath, &nCf, &err); + ASSERT(err == NULL); + } + + if (taskDbOpenCfs(pTaskDb, dbPath, cfNames, nCf) != 0) { + goto _EXIT; + } + + if (cfNames != NULL) { + rocksdb_list_column_families_destroy(cfNames, nCf); + cfNames = NULL; + } + + qDebug("succ to init stream backend at %s, backend:%p", dbPath, pTaskDb); + return pTaskDb; +_EXIT: + + taskDbDestroy(pTaskDb, false); + if (err) taosMemoryFree(err); + if (cfNames) rocksdb_list_column_families_destroy(cfNames, nCf); + return NULL; +} +STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId) { + char* statePath = NULL; + char* dbPath = NULL; + + if (rebuildDirFromChkp2(path, key, chkpId, &statePath, &dbPath) != 0) { + return NULL; + } + + STaskDbWrapper* pTaskDb = taskDbOpenImpl(key, statePath, dbPath); + taosMemoryFree(dbPath); + taosMemoryFree(statePath); + return pTaskDb; +} + +void taskDbDestroy(void* pDb, bool flush) { + STaskDbWrapper* wrapper = pDb; + qDebug("succ to destroy stream backend:%p", wrapper); + + int8_t nCf = sizeof(ginitDict) / sizeof(ginitDict[0]); + + if (wrapper == NULL) return; + + if (flush) { + if (wrapper->db && wrapper->pCf) { + rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); + rocksdb_flushoptions_set_wait(flushOpt, 1); + + char* err = NULL; + rocksdb_column_family_handle_t** cfs = taosMemoryCalloc(1, sizeof(rocksdb_column_family_handle_t*) * nCf); + int numOfFlushCf = 0; + for (int i = 0; i < nCf; i++) { + if (wrapper->pCf[i] != NULL) { + cfs[numOfFlushCf++] = wrapper->pCf[i]; + } + } + if (numOfFlushCf != 0) { + rocksdb_flush_cfs(wrapper->db, flushOpt, cfs, numOfFlushCf, &err); + if (err != NULL) { + stError("failed to flush all cfs, reason:%s", err); + taosMemoryFreeClear(err); + } + } + taosMemoryFree(cfs); + rocksdb_flushoptions_destroy(flushOpt); + } + } + for (int i = 0; i < nCf; i++) { + if (wrapper->pCf[i] != NULL) { + rocksdb_column_family_handle_destroy(wrapper->pCf[i]); + } + } + + if (wrapper->db) rocksdb_close(wrapper->db); + + rocksdb_options_destroy(wrapper->dbOpt); + rocksdb_readoptions_destroy(wrapper->readOpt); + rocksdb_writeoptions_destroy(wrapper->writeOpt); + rocksdb_env_destroy(wrapper->env); + rocksdb_cache_destroy(wrapper->cache); + + taosMemoryFree(wrapper->pCf); + for (int i = 0; i < nCf; i++) { + rocksdb_options_t* opt = wrapper->pCfOpts[i]; + rocksdb_comparator_t* compare = wrapper->pCompares[i]; + rocksdb_block_based_table_options_t* tblOpt = wrapper->pCfParams[i].tableOpt; + + rocksdb_options_destroy(opt); + rocksdb_comparator_destroy(compare); + rocksdb_block_based_options_destroy(tblOpt); + } + taosMemoryFree(wrapper->pCompares); + taosMemoryFree(wrapper->pCfOpts); + taosMemoryFree(wrapper->pCfParams); + + taosThreadMutexDestroy(&wrapper->mutex); + + taskDbDestroyChkpOpt(wrapper); + + taosMemoryFree(wrapper->idstr); + taosMemoryFree(wrapper->path); + taosMemoryFree(wrapper); + + return; +} + +void taskDbDestroy2(void* pDb) { taskDbDestroy(pDb, true); } + +int32_t taskDbGenChkpUploadData__rsync(STaskDbWrapper* pDb, int64_t chkpId, char** path) { + int64_t st = taosGetTimestampMs(); + int32_t code = -1; + int64_t refId = pDb->refId; + + if (taosAcquireRef(taskDbWrapperId, refId) == NULL) { + return -1; + } + + char* buf = taosMemoryCalloc(1, strlen(pDb->path) + 128); + sprintf(buf, "%s%s%s%s%s%" PRId64 "", pDb->path, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", chkpId); + if (taosIsDir(buf)) { + code = 0; + *path = buf; + } else { + taosMemoryFree(buf); + } + + taosReleaseRef(taskDbWrapperId, refId); + return code; +} + +int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64_t chkpId, char** path, SArray* list) { + int32_t code = 0; + SBkdMgt* p = (SBkdMgt*)bkdChkpMgt; + + char* temp = taosMemoryCalloc(1, strlen(pDb->path) + 32); + sprintf(temp, "%s%s%s%" PRId64 "", pDb->path, TD_DIRSEP, "tmp", chkpId); + + if (taosDirExist(temp)) { + taosRemoveDir(temp); + taosMkDir(temp); + } else { + taosMkDir(temp); + } + code = bkdMgtGetDelta(p, pDb->idstr, chkpId, list, temp); + + *path = temp; + + return code; +} +int32_t taskDbGenChkpUploadData(void* arg, void* mgt, int64_t chkpId, int8_t type, char** path, SArray* list) { + STaskDbWrapper* pDb = arg; + UPLOAD_TYPE utype = type; + + if (utype == UPLOAD_RSYNC) { + return taskDbGenChkpUploadData__rsync(pDb, chkpId, path); + } else if (utype == UPLOAD_S3) { + return taskDbGenChkpUploadData__s3(pDb, mgt, chkpId, path, list); + } + return -1; +} + +int32_t taskDbOpenCfByKey(STaskDbWrapper* pDb, const char* key) { + int32_t code = 0; + char* err = NULL; + int8_t idx = getCfIdx(key); + + if (idx == -1) return -1; + + if (pDb->pCf[idx] != NULL) return code; + + rocksdb_column_family_handle_t* cf = + rocksdb_create_column_family(pDb->db, pDb->pCfOpts[idx], ginitDict[idx].key, &err); + if (err != NULL) { + stError("failed to open cf, key:%s, reason: %s", key, err); + taosMemoryFree(err); + code = -1; + return code; + } + + pDb->pCf[idx] = cf; + return code; +} +int32_t copyDataAt(RocksdbCfInst* pSrc, STaskDbWrapper* pDst, int8_t i) { + int32_t WRITE_BATCH = 1024; + char* err = NULL; + int code = 0; + + rocksdb_readoptions_t* pRdOpt = rocksdb_readoptions_create(); + + rocksdb_writebatch_t* wb = rocksdb_writebatch_create(); + rocksdb_iterator_t* pIter = rocksdb_create_iterator_cf(pSrc->db, pRdOpt, pSrc->pHandle[i]); + rocksdb_iter_seek_to_first(pIter); + while (rocksdb_iter_valid(pIter)) { + if (rocksdb_writebatch_count(wb) >= WRITE_BATCH) { + rocksdb_write(pDst->db, pDst->writeOpt, wb, &err); + if (err != NULL) { + code = -1; + goto _EXIT; + } + rocksdb_writebatch_clear(wb); + } + + size_t klen = 0, vlen = 0; + char* key = (char*)rocksdb_iter_key(pIter, &klen); + char* val = (char*)rocksdb_iter_value(pIter, &vlen); + + rocksdb_writebatch_put_cf(wb, pDst->pCf[i], key, klen, val, vlen); + rocksdb_iter_next(pIter); + } + + if (rocksdb_writebatch_count(wb) > 0) { + rocksdb_write(pDst->db, pDst->writeOpt, wb, &err); + if (err != NULL) { + code = -1; + goto _EXIT; + } + } + +_EXIT: + rocksdb_iter_destroy(pIter); + rocksdb_readoptions_destroy(pRdOpt); + taosMemoryFree(err); + + return code; +} + +int32_t streamStateCvtDataFormat(char* path, char* key, void* pCfInst) { + int nCf = sizeof(ginitDict) / sizeof(ginitDict[0]); + + int32_t code = 0; + + STaskDbWrapper* pTaskDb = taskDbOpen(path, key, 0); + RocksdbCfInst* pSrcBackend = pCfInst; + + for (int i = 0; i < nCf; i++) { + rocksdb_column_family_handle_t* pSrcCf = pSrcBackend->pHandle[i]; + if (pSrcCf == NULL) continue; + + code = taskDbOpenCfByKey(pTaskDb, ginitDict[i].key); + if (code != 0) goto _EXIT; + + code = copyDataAt(pSrcBackend, pTaskDb, i); + if (code != 0) goto _EXIT; + } + +_EXIT: + taskDbDestroy(pTaskDb, true); + + return code; +} int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t nCf) { SBackendWrapper* handle = backend; char* err = NULL; @@ -1525,7 +2059,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t SCfInit* cfPara = &ginitDict[idx]; rocksdb_comparator_t* compare = - rocksdb_comparator_create(NULL, cfPara->detroyFunc, cfPara->cmpFunc, cfPara->cmpName); + rocksdb_comparator_create(NULL, cfPara->destroyCmp, cfPara->cmpKey, cfPara->cmpName); rocksdb_options_set_comparator((rocksdb_options_t*)cfOpts[i], compare); pCompare[i] = compare; } @@ -1609,7 +2143,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t SCfInit* cfPara = &ginitDict[i]; rocksdb_comparator_t* compare = - rocksdb_comparator_create(NULL, cfPara->detroyFunc, cfPara->cmpFunc, cfPara->cmpName); + rocksdb_comparator_create(NULL, cfPara->destroyCmp, cfPara->cmpKey, cfPara->cmpName); rocksdb_options_set_comparator((rocksdb_options_t*)opt, compare); inst->pCompares[i] = compare; @@ -1689,7 +2223,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { for (int i = 0; i < cfLen; i++) { SCfInit* cf = &ginitDict[i]; - rocksdb_comparator_t* compare = rocksdb_comparator_create(NULL, cf->detroyFunc, cf->cmpFunc, cf->cmpName); + rocksdb_comparator_t* compare = rocksdb_comparator_create(NULL, cf->destroyCmp, cf->cmpKey, cf->cmpName); rocksdb_options_set_comparator((rocksdb_options_t*)cfOpt[i], compare); pCompare[i] = compare; } @@ -1755,26 +2289,25 @@ int streamStateGetCfIdx(SStreamState* pState, const char* funcName) { } } if (pState != NULL && idx != -1) { - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; rocksdb_column_family_handle_t* cf = NULL; - taosThreadRwlockWrlock(&wrapper->rwLock); - cf = wrapper->pHandle[idx]; - if (cf == NULL) { - char buf[128] = {0}; - GEN_COLUMN_FAMILY_NAME(buf, wrapper->idstr, ginitDict[idx].key); - char* err = NULL; - cf = rocksdb_create_column_family(wrapper->rocksdb, wrapper->cfOpts[idx], buf, &err); + taosThreadMutexLock(&wrapper->mutex); + + cf = wrapper->pCf[idx]; + if (cf == NULL) { + char* err = NULL; + cf = rocksdb_create_column_family(wrapper->db, wrapper->pCfOpts[idx], ginitDict[idx].key, &err); if (err != NULL) { idx = -1; stError("failed to open cf, %p %s_%s, reason:%s", pState, wrapper->idstr, funcName, err); taosMemoryFree(err); } else { - stDebug("succ to to open cf, %p %s_%s", pState, wrapper->idstr, funcName); - wrapper->pHandle[idx] = cf; + qDebug("succ to open cf, %p %s_%s", pState, wrapper->idstr, funcName); + wrapper->pCf[idx] = cf; } } - taosThreadRwlockUnlock(&wrapper->rwLock); + taosThreadMutexUnlock(&wrapper->mutex); } return idx; @@ -1795,118 +2328,120 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe *readOpt = rocksdb_readoptions_create(); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; if (snapshot != NULL) { - *snapshot = (rocksdb_snapshot_t*)rocksdb_create_snapshot(wrapper->rocksdb); + *snapshot = (rocksdb_snapshot_t*)rocksdb_create_snapshot(wrapper->db); rocksdb_readoptions_set_snapshot(*readOpt, *snapshot); rocksdb_readoptions_set_fill_cache(*readOpt, 0); } - return rocksdb_create_iterator_cf(wrapper->rocksdb, *readOpt, - ((rocksdb_column_family_handle_t**)wrapper->pHandle)[idx]); + return rocksdb_create_iterator_cf(wrapper->db, *readOpt, ((rocksdb_column_family_handle_t**)wrapper->pCf)[idx]); } -#define STREAM_STATE_PUT_ROCKSDB(pState, funcname, key, value, vLen) \ - do { \ - code = 0; \ - char buf[128] = {0}; \ - char* err = NULL; \ - int i = streamStateGetCfIdx(pState, funcname); \ - if (i < 0) { \ - stWarn("streamState failed to get cf name: %s", funcname); \ - code = -1; \ - break; \ - } \ - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; \ - char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ - int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pHandle)[ginitDict[i].idx]; \ - rocksdb_t* db = wrapper->rocksdb; \ - rocksdb_writeoptions_t* opts = wrapper->writeOpts; \ - char* ttlV = NULL; \ - int32_t ttlVLen = ginitDict[i].enValueFunc((char*)value, vLen, 0, &ttlV); \ - rocksdb_put_cf(db, opts, pHandle, (const char*)buf, klen, (const char*)ttlV, (size_t)ttlVLen, &err); \ - if (err != NULL) { \ - stError("streamState str: %s failed to write to %s, err: %s", toString, funcname, err); \ - taosMemoryFree(err); \ - code = -1; \ - } else { \ - stTrace("streamState str:%s succ to write to %s, rowValLen:%d, ttlValLen:%d", toString, funcname, vLen, \ - ttlVLen); \ - } \ - taosMemoryFree(ttlV); \ +#define STREAM_STATE_PUT_ROCKSDB(pState, funcname, key, value, vLen) \ + do { \ + code = 0; \ + char buf[128] = {0}; \ + char* err = NULL; \ + int i = streamStateGetCfIdx(pState, funcname); \ + if (i < 0) { \ + qWarn("streamState failed to get cf name: %s", funcname); \ + code = -1; \ + break; \ + } \ + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ + wrapper->dataWritten += 1; \ + char toString[128] = {0}; \ + if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ + rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ + rocksdb_writeoptions_t* opts = wrapper->writeOpt; \ + rocksdb_t* db = wrapper->db; \ + char* ttlV = NULL; \ + int32_t ttlVLen = ginitDict[i].enValueFunc((char*)value, vLen, 0, &ttlV); \ + rocksdb_put_cf(db, opts, pHandle, (const char*)buf, klen, (const char*)ttlV, (size_t)ttlVLen, &err); \ + if (err != NULL) { \ + stError("streamState str: %s failed to write to %s, err: %s", toString, funcname, err); \ + taosMemoryFree(err); \ + code = -1; \ + } else { \ + qTrace("streamState str:%s succ to write to %s, rowValLen:%d, ttlValLen:%d, %p", toString, funcname, vLen, \ + ttlVLen, wrapper); \ + } \ + taosMemoryFree(ttlV); \ } while (0); -#define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \ - do { \ - code = 0; \ - char buf[128] = {0}; \ - char* err = NULL; \ - int i = streamStateGetCfIdx(pState, funcname); \ - if (i < 0) { \ - stWarn("streamState failed to get cf name: %s", funcname); \ - code = -1; \ - break; \ - } \ - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; \ - char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ - int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pHandle)[ginitDict[i].idx]; \ - rocksdb_t* db = wrapper->rocksdb; \ - rocksdb_readoptions_t* opts = wrapper->readOpts; \ - size_t len = 0; \ - char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ - if (val == NULL || len == 0) { \ - if (err == NULL) { \ - stTrace("streamState str: %s failed to read from %s_%s, err: not exist", toString, wrapper->idstr, funcname); \ - } else { \ - stError("streamState str: %s failed to read from %s_%s, err: %s", toString, wrapper->idstr, funcname, err); \ - taosMemoryFreeClear(err); \ - } \ - code = -1; \ - } else { \ - char* p = NULL; \ - int32_t tlen = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ - if (tlen <= 0) { \ - stError("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, wrapper->idstr, \ - funcname); \ - code = -1; \ - } else { \ - stTrace("streamState str: %s succ to read from %s_%s, valLen:%d", toString, wrapper->idstr, funcname, tlen); \ - } \ - taosMemoryFree(val); \ - if (vLen != NULL) *vLen = tlen; \ - } \ +#define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \ + do { \ + code = 0; \ + char buf[128] = {0}; \ + char* err = NULL; \ + int i = streamStateGetCfIdx(pState, funcname); \ + if (i < 0) { \ + qWarn("streamState failed to get cf name: %s", funcname); \ + code = -1; \ + break; \ + } \ + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ + char toString[128] = {0}; \ + if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ + rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ + rocksdb_t* db = wrapper->db; \ + rocksdb_readoptions_t* opts = wrapper->readOpt; \ + size_t len = 0; \ + char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ + if (val == NULL || len == 0) { \ + if (err == NULL) { \ + qTrace("streamState str: %s failed to read from %s_%s, err: not exist", toString, wrapper->idstr, funcname); \ + } else { \ + stError("streamState str: %s failed to read from %s_%s, err: %s", toString, wrapper->idstr, funcname, err); \ + taosMemoryFreeClear(err); \ + } \ + code = -1; \ + } else { \ + char* p = NULL; \ + int32_t tlen = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ + if (tlen <= 0) { \ + stError("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, wrapper->idstr, \ + funcname); \ + code = -1; \ + } else { \ + qTrace("streamState str: %s succ to read from %s_%s, valLen:%d, %p", toString, wrapper->idstr, funcname, tlen, \ + wrapper); \ + } \ + taosMemoryFree(val); \ + if (vLen != NULL) *vLen = tlen; \ + } \ } while (0); -#define STREAM_STATE_DEL_ROCKSDB(pState, funcname, key) \ - do { \ - code = 0; \ - char buf[128] = {0}; \ - char* err = NULL; \ - int i = streamStateGetCfIdx(pState, funcname); \ - if (i < 0) { \ - stWarn("streamState failed to get cf name: %s_%s", pState->pTdbState->idstr, funcname); \ - code = -1; \ - break; \ - } \ - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; \ - char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ - int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pHandle)[ginitDict[i].idx]; \ - rocksdb_t* db = wrapper->rocksdb; \ - rocksdb_writeoptions_t* opts = wrapper->writeOpts; \ - rocksdb_delete_cf(db, opts, pHandle, (const char*)buf, klen, &err); \ - if (err != NULL) { \ - stError("streamState str: %s failed to del from %s_%s, err: %s", toString, wrapper->idstr, funcname, err); \ - taosMemoryFree(err); \ - code = -1; \ - } else { \ - stTrace("streamState str: %s succ to del from %s_%s", toString, wrapper->idstr, funcname); \ - } \ +#define STREAM_STATE_DEL_ROCKSDB(pState, funcname, key) \ + do { \ + code = 0; \ + char buf[128] = {0}; \ + char* err = NULL; \ + int i = streamStateGetCfIdx(pState, funcname); \ + if (i < 0) { \ + qWarn("streamState failed to get cf name: %s_%s", pState->pTdbState->idstr, funcname); \ + code = -1; \ + break; \ + } \ + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ + wrapper->dataWritten += 1; \ + char toString[128] = {0}; \ + if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ + rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ + rocksdb_t* db = wrapper->db; \ + rocksdb_writeoptions_t* opts = wrapper->writeOpt; \ + rocksdb_delete_cf(db, opts, pHandle, (const char*)buf, klen, &err); \ + if (err != NULL) { \ + stError("streamState str: %s failed to del from %s_%s, err: %s", toString, wrapper->idstr, funcname, err); \ + taosMemoryFree(err); \ + code = -1; \ + } else { \ + qTrace("streamState str: %s succ to del from %s_%s", toString, wrapper->idstr, funcname); \ + } \ } while (0); // state cf @@ -1932,7 +2467,8 @@ int32_t streamStateDel_rocksdb(SStreamState* pState, const SWinKey* key) { int32_t streamStateClear_rocksdb(SStreamState* pState) { stDebug("streamStateClear_rocksdb"); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + wrapper->dataWritten += 1; char sKeyStr[128] = {0}; char eKeyStr[128] = {0}; @@ -1942,10 +2478,9 @@ int32_t streamStateClear_rocksdb(SStreamState* pState) { int sLen = stateKeyEncode(&sKey, sKeyStr); int eLen = stateKeyEncode(&eKey, eKeyStr); - if (wrapper->pHandle[1] != NULL) { + if (wrapper->pCf[1] != NULL) { char* err = NULL; - rocksdb_delete_range_cf(wrapper->rocksdb, wrapper->writeOpts, wrapper->pHandle[1], sKeyStr, sLen, eKeyStr, eLen, - &err); + rocksdb_delete_range_cf(wrapper->db, wrapper->writeOpt, wrapper->pCf[1], sKeyStr, sLen, eKeyStr, eLen, &err); if (err != NULL) { char toStringStart[128] = {0}; char toStringEnd[128] = {0}; @@ -1955,7 +2490,7 @@ int32_t streamStateClear_rocksdb(SStreamState* pState) { stWarn("failed to delete range cf(state) start: %s, end:%s, reason:%s", toStringStart, toStringEnd, err); taosMemoryFree(err); } else { - rocksdb_compact_range_cf(wrapper->rocksdb, wrapper->pHandle[1], sKeyStr, sLen, eKeyStr, eLen); + rocksdb_compact_range_cf(wrapper->db, wrapper->pCf[1], sKeyStr, sLen, eKeyStr, eLen); } } @@ -2029,7 +2564,7 @@ int32_t streamStateGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, cons if (pVLen != NULL) { size_t vlen = 0; const char* valStr = rocksdb_iter_value(pCur->iter, &vlen); - *pVLen = decodeValueFunc((void*)valStr, vlen, NULL, (char**)pVal); + *pVLen = valueDecode((void*)valStr, vlen, NULL, (char**)pVal); } *pKey = pKtmp->key; @@ -2052,9 +2587,9 @@ SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWin if (pCur == NULL) { return NULL; } - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; pCur->number = pState->number; - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); @@ -2104,7 +2639,7 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState) { if (pCur == NULL) return NULL; pCur->number = pState->number; - pCur->db = ((SBackendCfWrapper*)pState->pTdbState->pBackendCfWrapper)->rocksdb; + pCur->db = ((STaskDbWrapper*)pState->pTdbState->pOwner->pBackend)->db; pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); @@ -2126,12 +2661,13 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState) { } SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + qDebug("streamStateGetCur_rocksdb"); + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) return NULL; - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); pCur->number = pState->number; @@ -2178,6 +2714,9 @@ int32_t streamStateFuncDel_rocksdb(SStreamState* pState, const STupleKey* key) { int32_t streamStateSessionPut_rocksdb(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen) { int code = 0; SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; + if (value == NULL || vLen == 0) { + stError("streamStateSessionPut_rocksdb val: %p, len: %d", value, vLen); + } STREAM_STATE_PUT_ROCKSDB(pState, "sess", &sKey, value, vLen); return code; } @@ -2226,11 +2765,11 @@ SStreamStateCur* streamStateSessionSeekToLast_rocksdb(SStreamState* pState) { if (code != 0) { return NULL; } + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - SStreamStateCur* pCur = createStreamStateCursor(); + SStreamStateCur* pCur = createStreamStateCursor(); pCur->number = pState->number; - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); @@ -2262,13 +2801,14 @@ int32_t streamStateSessionCurPrev_rocksdb(SStreamStateCur* pCur) { SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pState, const SSessionKey* key) { stDebug("streamStateSessionSeekKeyCurrentPrev_rocksdb"); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - SStreamStateCur* pCur = createStreamStateCursor(); + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { return NULL; } + pCur->number = pState->number; - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); @@ -2301,13 +2841,13 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta return pCur; } SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pState, SSessionKey* key) { - stDebug("streamStateSessionSeekKeyCurrentNext_rocksdb"); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - SStreamStateCur* pCur = createStreamStateCursor(); + qDebug("streamStateSessionSeekKeyCurrentNext_rocksdb"); + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { return NULL; } - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); pCur->number = pState->number; @@ -2339,13 +2879,13 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pSta } SStreamStateCur* streamStateSessionSeekKeyNext_rocksdb(SStreamState* pState, const SSessionKey* key) { - stDebug("streamStateSessionSeekKeyNext_rocksdb"); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - SStreamStateCur* pCur = createStreamStateCursor(); + qDebug("streamStateSessionSeekKeyNext_rocksdb"); + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { return NULL; } - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); pCur->number = pState->number; @@ -2398,7 +2938,7 @@ int32_t streamStateSessionGetKVByCur_rocksdb(SStreamStateCur* pCur, SSessionKey* SStateSessionKey* pKTmp = &ktmp; const char* vval = rocksdb_iter_value(pCur->iter, (size_t*)&vLen); char* val = NULL; - int32_t len = decodeValueFunc((void*)vval, vLen, NULL, &val); + int32_t len = valueDecode((void*)vval, vLen, NULL, &val); if (len < 0) { taosMemoryFree(val); return -1; @@ -2443,13 +2983,13 @@ int32_t streamStateFillDel_rocksdb(SStreamState* pState, const SWinKey* key) { } SStreamStateCur* streamStateFillGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { - stDebug("streamStateFillGetCur_rocksdb"); - SStreamStateCur* pCur = createStreamStateCursor(); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + qDebug("streamStateFillGetCur_rocksdb"); + SStreamStateCur* pCur = createStreamStateCursor(); + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; if (pCur == NULL) return NULL; - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "fill", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); pCur->number = pState->number; @@ -2492,7 +3032,7 @@ int32_t streamStateFillGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, winKeyDecode(&winKey, keyStr); const char* valStr = rocksdb_iter_value(pCur->iter, &vlen); - int32_t len = decodeValueFunc((void*)valStr, vlen, NULL, (char**)pVal); + int32_t len = valueDecode((void*)valStr, vlen, NULL, (char**)pVal); if (len < 0) { return -1; } @@ -2503,14 +3043,14 @@ int32_t streamStateFillGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, } SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key) { - stDebug("streamStateFillSeekKeyNext_rocksdb"); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - SStreamStateCur* pCur = createStreamStateCursor(); + qDebug("streamStateFillSeekKeyNext_rocksdb"); + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + SStreamStateCur* pCur = createStreamStateCursor(); if (!pCur) { return NULL; } - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "fill", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); pCur->number = pState->number; @@ -2541,14 +3081,14 @@ SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const return NULL; } SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const SWinKey* key) { - stDebug("streamStateFillSeekKeyPrev_rocksdb"); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - SStreamStateCur* pCur = createStreamStateCursor(); + qDebug("streamStateFillSeekKeyPrev_rocksdb"); + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { return NULL; } - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "fill", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); pCur->number = pState->number; @@ -2580,12 +3120,12 @@ SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const } int32_t streamStateSessionGetKeyByRange_rocksdb(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey) { stDebug("streamStateSessionGetKeyByRange_rocksdb"); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - SStreamStateCur* pCur = createStreamStateCursor(); + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { return -1; } - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); pCur->number = pState->number; @@ -2812,7 +3352,7 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co int code = 0; char* err = NULL; - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; rocksdb_snapshot_t* snapshot = NULL; rocksdb_readoptions_t* readopts = NULL; rocksdb_iterator_t* pIter = streamStateIterCreate(pState, "default", &snapshot, &readopts); @@ -2826,7 +3366,7 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co int32_t vlen = 0; const char* vval = rocksdb_iter_value(pIter, (size_t*)&vlen); char* val = NULL; - int32_t len = decodeValueFunc((void*)vval, vlen, NULL, NULL); + int32_t len = valueDecode((void*)vval, vlen, NULL, NULL); if (len < 0) { rocksdb_iter_next(pIter); continue; @@ -2845,16 +3385,16 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co } rocksdb_iter_next(pIter); } - rocksdb_release_snapshot(wrapper->rocksdb, snapshot); + rocksdb_release_snapshot(wrapper->db, snapshot); rocksdb_readoptions_destroy(readopts); rocksdb_iter_destroy(pIter); return code; } void* streamDefaultIterCreate_rocksdb(SStreamState* pState) { - SStreamStateCur* pCur = createStreamStateCursor(); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + SStreamStateCur* pCur = createStreamStateCursor(); + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "default", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); pCur->number = pState->number; @@ -2885,7 +3425,7 @@ char* streamDefaultIterVal_rocksdb(void* iter, int32_t* len) { int32_t vlen = 0; const char* val = rocksdb_iter_value(pCur->iter, (size_t*)&vlen); - *len = decodeValueFunc((void*)val, vlen, NULL, &ret); + *len = valueDecode((void*)val, vlen, NULL, &ret); if (*len < 0) { taosMemoryFree(ret); return NULL; @@ -2907,7 +3447,8 @@ void streamStateClearBatch(void* pBatch) { rocksdb_writebatch_clear((rocksdb_ void streamStateDestroyBatch(void* pBatch) { rocksdb_writebatch_destroy((rocksdb_writebatch_t*)pBatch); } int32_t streamStatePutBatch(SStreamState* pState, const char* cfKeyName, rocksdb_writebatch_t* pBatch, void* key, void* val, int32_t vlen, int64_t ttl) { - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + wrapper->dataWritten += 1; int i = streamStateGetCfIdx(pState, cfKeyName); if (i < 0) { @@ -2921,7 +3462,7 @@ int32_t streamStatePutBatch(SStreamState* pState, const char* cfKeyName, rocksdb char* ttlV = NULL; int32_t ttlVLen = ginitDict[i].enValueFunc(val, vlen, ttl, &ttlV); - rocksdb_column_family_handle_t* pCf = wrapper->pHandle[ginitDict[i].idx]; + rocksdb_column_family_handle_t* pCf = wrapper->pCf[ginitDict[i].idx]; rocksdb_writebatch_put_cf((rocksdb_writebatch_t*)pBatch, pCf, buf, (size_t)klen, ttlV, (size_t)ttlVLen); taosMemoryFree(ttlV); @@ -2940,8 +3481,10 @@ int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb char* ttlV = tmpBuf; int32_t ttlVLen = ginitDict[cfIdx].enValueFunc(val, vlen, ttl, &ttlV); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - rocksdb_column_family_handle_t* pCf = wrapper->pHandle[ginitDict[cfIdx].idx]; + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + wrapper->dataWritten += 1; + + rocksdb_column_family_handle_t* pCf = wrapper->pCf[ginitDict[cfIdx].idx]; rocksdb_writebatch_put_cf((rocksdb_writebatch_t*)pBatch, pCf, buf, (size_t)klen, ttlV, (size_t)ttlVLen); if (tmpBuf == NULL) { @@ -2956,15 +3499,16 @@ int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb return 0; } int32_t streamStatePutBatch_rocksdb(SStreamState* pState, void* pBatch) { - char* err = NULL; - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - rocksdb_write(wrapper->rocksdb, wrapper->writeOpts, (rocksdb_writebatch_t*)pBatch, &err); + char* err = NULL; + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + wrapper->dataWritten += 1; + rocksdb_write(wrapper->db, wrapper->writeOpt, (rocksdb_writebatch_t*)pBatch, &err); if (err != NULL) { stError("streamState failed to write batch, err:%s", err); taosMemoryFree(err); return -1; } else { - stDebug("write batch to backend:%p", wrapper->pBackend); + stDebug("write batch to backend:%p", wrapper->db); } return 0; } @@ -2978,3 +3522,446 @@ uint32_t nextPow2(uint32_t x) { x = x | (x >> 16); return x + 1; } +int32_t copyFiles(const char* src, const char* dst) { + int32_t code = 0; + // opt later, just hard link + int32_t sLen = strlen(src); + int32_t dLen = strlen(dst); + char* srcName = taosMemoryCalloc(1, sLen + 64); + char* dstName = taosMemoryCalloc(1, dLen + 64); + + TdDirPtr pDir = taosOpenDir(src); + if (pDir == NULL) { + taosMemoryFree(srcName); + taosMemoryFree(dstName); + return -1; + } + + TdDirEntryPtr de = NULL; + while ((de = taosReadDir(pDir)) != NULL) { + char* name = taosGetDirEntryName(de); + if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue; + + sprintf(srcName, "%s%s%s", src, TD_DIRSEP, name); + sprintf(dstName, "%s%s%s", dst, TD_DIRSEP, name); + if (!taosDirEntryIsDir(de)) { + code = taosCopyFile(srcName, dstName); + if (code == -1) { + goto _err; + } + } + + memset(srcName, 0, sLen + 64); + memset(dstName, 0, dLen + 64); + } + +_err: + taosMemoryFreeClear(srcName); + taosMemoryFreeClear(dstName); + taosCloseDir(&pDir); + return code >= 0 ? 0 : -1; +} + +int32_t isBkdDataMeta(char* name, int32_t len) { + const char* pCurrent = "CURRENT"; + int32_t currLen = strlen(pCurrent); + + const char* pManifest = "MANIFEST-"; + int32_t maniLen = strlen(pManifest); + + if (len >= maniLen && strncmp(name, pManifest, maniLen) == 0) { + return 1; + } else if (len == currLen && strcmp(name, pCurrent) == 0) { + return 1; + } + return 0; +} +int32_t compareHashTableImpl(SHashObj* p1, SHashObj* p2, SArray* diff) { + int32_t code = 0; + size_t len = 0; + void* pIter = taosHashIterate(p2, NULL); + while (pIter) { + char* name = taosHashGetKey(pIter, &len); + if (!isBkdDataMeta(name, len) && !taosHashGet(p1, name, len)) { + char* fname = taosMemoryCalloc(1, len + 1); + strncpy(fname, name, len); + taosArrayPush(diff, &fname); + } + pIter = taosHashIterate(p2, pIter); + } + return code; +} +int32_t compareHashTable(SHashObj* p1, SHashObj* p2, SArray* add, SArray* del) { + int32_t code = 0; + + code = compareHashTableImpl(p1, p2, add); + code = compareHashTableImpl(p2, p1, del); + + return code; +} + +void hashTableToDebug(SHashObj* pTbl, char** buf) { + size_t sz = taosHashGetSize(pTbl); + int32_t total = 0; + char* p = taosMemoryCalloc(1, sz * 16 + 4); + void* pIter = taosHashIterate(pTbl, NULL); + while (pIter) { + size_t len = 0; + char* name = taosHashGetKey(pIter, &len); + char* tname = taosMemoryCalloc(1, len + 1); + memcpy(tname, name, len); + total += sprintf(p + total, "%s,", tname); + + pIter = taosHashIterate(pTbl, pIter); + taosMemoryFree(tname); + } + if (total > 0) { + p[total - 1] = 0; + } + *buf = p; +} +void strArrayDebugInfo(SArray* pArr, char** buf) { + int32_t sz = taosArrayGetSize(pArr); + if (sz <= 0) return; + + char* p = (char*)taosMemoryCalloc(1, 64 + sz * 64); + int32_t total = 0; + + for (int i = 0; i < sz; i++) { + char* name = taosArrayGetP(pArr, i); + total += sprintf(p + total, "%s,", name); + } + p[total - 1] = 0; + + *buf = p; +} +void dbChkpDebugInfo(SDbChkp* pDb) { + // stTrace("chkp get file list: curr"); + char* p[4] = {NULL}; + + hashTableToDebug(pDb->pSstTbl[pDb->idx], &p[0]); + stTrace("chkp previous file: [%s]", p[0]); + + hashTableToDebug(pDb->pSstTbl[1 - pDb->idx], &p[1]); + stTrace("chkp curr file: [%s]", p[1]); + + strArrayDebugInfo(pDb->pAdd, &p[2]); + stTrace("chkp newly addded file: [%s]", p[2]); + + strArrayDebugInfo(pDb->pDel, &p[3]); + stTrace("chkp newly deleted file: [%s]", p[3]); + + for (int i = 0; i < 4; i++) { + taosMemoryFree(p[i]); + } +} +int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { + taosThreadRwlockWrlock(&p->rwLock); + + p->preCkptId = p->curChkpId; + p->curChkpId = chkpId; + const char* pCurrent = "CURRENT"; + int32_t currLen = strlen(pCurrent); + + const char* pManifest = "MANIFEST-"; + int32_t maniLen = strlen(pManifest); + + const char* pSST = ".sst"; + int32_t sstLen = strlen(pSST); + + memset(p->buf, 0, p->len); + sprintf(p->buf, "%s%s%s%scheckpoint%" PRId64 "", p->path, TD_DIRSEP, "checkpoints", TD_DIRSEP, chkpId); + + taosArrayClearP(p->pAdd, taosMemoryFree); + taosArrayClearP(p->pDel, taosMemoryFree); + taosHashClear(p->pSstTbl[1 - p->idx]); + + TdDirPtr pDir = taosOpenDir(p->buf); + TdDirEntryPtr de = NULL; + int8_t dummy = 0; + while ((de = taosReadDir(pDir)) != NULL) { + char* name = taosGetDirEntryName(de); + if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue; + if (strlen(name) == currLen && strcmp(name, pCurrent) == 0) { + taosMemoryFreeClear(p->pCurrent); + p->pCurrent = taosStrdup(name); + // taosHashPut(p->pSstTbl[1 - p->idx], name, strlen(name), &dummy, sizeof(dummy)); + continue; + } + + if (strlen(name) >= maniLen && strncmp(name, pManifest, maniLen) == 0) { + taosMemoryFreeClear(p->pManifest); + p->pManifest = taosStrdup(name); + // taosHashPut(p->pSstTbl[1 - p->idx], name, strlen(name), &dummy, sizeof(dummy)); + continue; + } + if (strlen(name) >= sstLen && strncmp(name + strlen(name) - 4, pSST, sstLen) == 0) { + taosHashPut(p->pSstTbl[1 - p->idx], name, strlen(name), &dummy, sizeof(dummy)); + continue; + } + } + taosCloseDir(&pDir); + + if (p->init == 0) { + void* pIter = taosHashIterate(p->pSstTbl[1 - p->idx], NULL); + while (pIter) { + size_t len = 0; + char* name = taosHashGetKey(pIter, &len); + if (name != NULL && !isBkdDataMeta(name, len)) { + char* fname = taosMemoryCalloc(1, len + 1); + strncpy(fname, name, len); + taosArrayPush(p->pAdd, &fname); + } + pIter = taosHashIterate(p->pSstTbl[1 - p->idx], pIter); + } + if (taosArrayGetSize(p->pAdd) > 0) p->update = 1; + + p->init = 1; + p->preCkptId = -1; + p->curChkpId = chkpId; + } else { + int32_t code = compareHashTable(p->pSstTbl[p->idx], p->pSstTbl[1 - p->idx], p->pAdd, p->pDel); + if (code != 0) { + // dead code + taosArrayClearP(p->pAdd, taosMemoryFree); + taosArrayClearP(p->pDel, taosMemoryFree); + taosHashClear(p->pSstTbl[1 - p->idx]); + p->update = 0; + return code; + } + + if (taosArrayGetSize(p->pAdd) == 0 && taosArrayGetSize(p->pDel) == 0) { + p->update = 0; + } + + p->preCkptId = p->curChkpId; + p->curChkpId = chkpId; + } + + dbChkpDebugInfo(p); + + p->idx = 1 - p->idx; + + taosThreadRwlockUnlock(&p->rwLock); + + return 0; +} + +SDbChkp* dbChkpCreate(char* path, int64_t initChkpId) { + SDbChkp* p = taosMemoryCalloc(1, sizeof(SDbChkp)); + p->curChkpId = initChkpId; + p->preCkptId = -1; + p->pSST = taosArrayInit(64, sizeof(void*)); + p->path = path; + p->len = strlen(path) + 128; + p->buf = taosMemoryCalloc(1, p->len); + + p->idx = 0; + p->pSstTbl[0] = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + p->pSstTbl[1] = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + + p->pAdd = taosArrayInit(64, sizeof(void*)); + p->pDel = taosArrayInit(64, sizeof(void*)); + p->update = 0; + taosThreadRwlockInit(&p->rwLock, NULL); + + SArray* list = NULL; + int32_t code = dbChkpGetDelta(p, initChkpId, list); + + return p; +} + +void dbChkpDestroy(SDbChkp* pChkp) { + taosMemoryFree(pChkp->buf); + taosMemoryFree(pChkp->path); + + taosArrayDestroyP(pChkp->pSST, taosMemoryFree); + taosArrayDestroyP(pChkp->pAdd, taosMemoryFree); + taosArrayDestroyP(pChkp->pDel, taosMemoryFree); + + taosHashCleanup(pChkp->pSstTbl[0]); + taosHashCleanup(pChkp->pSstTbl[1]); + + taosMemoryFree(pChkp->pCurrent); + taosMemoryFree(pChkp->pManifest); + taosMemoryFree(pChkp); +} + +int32_t dbChkpInit(SDbChkp* p) { + if (p == NULL) return 0; + return 0; +} +int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) { + taosThreadRwlockRdlock(&p->rwLock); + int32_t code = -1; + int32_t len = p->len + 128; + + char* srcBuf = taosMemoryCalloc(1, len); + char* dstBuf = taosMemoryCalloc(1, len); + + char* srcDir = taosMemoryCalloc(1, len); + char* dstDir = taosMemoryCalloc(1, len); + + sprintf(srcDir, "%s%s%s%s%s%" PRId64 "", p->path, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", p->curChkpId); + sprintf(dstDir, "%s", dname); + + if (!taosDirExist(srcDir)) { + stError("failed to dump srcDir %s, reason: not exist such dir", srcDir); + goto _ERROR; + } + + // add file to $name dir + for (int i = 0; i < taosArrayGetSize(p->pAdd); i++) { + memset(srcBuf, 0, len); + memset(dstBuf, 0, len); + + char* filename = taosArrayGetP(p->pAdd, i); + sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, filename); + sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, filename); + + if (taosCopyFile(srcBuf, dstBuf) < 0) { + stError("failed to copy file from %s to %s", srcBuf, dstBuf); + goto _ERROR; + } + } + // del file in $name + for (int i = 0; i < taosArrayGetSize(p->pDel); i++) { + char* filename = taosArrayGetP(p->pDel, i); + char* p = taosStrdup(filename); + taosArrayPush(list, &p); + } + + // copy current file to dst dir + memset(srcBuf, 0, len); + memset(dstBuf, 0, len); + sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, p->pCurrent); + sprintf(dstBuf, "%s%s%s_%" PRId64 "", dstDir, TD_DIRSEP, p->pCurrent, p->curChkpId); + if (taosCopyFile(srcBuf, dstBuf) < 0) { + stError("failed to copy file from %s to %s", srcBuf, dstBuf); + goto _ERROR; + } + + // copy manifest file to dst dir + memset(srcBuf, 0, len); + memset(dstBuf, 0, len); + sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, p->pManifest); + sprintf(dstBuf, "%s%s%s_%" PRId64 "", dstDir, TD_DIRSEP, p->pManifest, p->curChkpId); + if (taosCopyFile(srcBuf, dstBuf) < 0) { + stError("failed to copy file from %s to %s", srcBuf, dstBuf); + goto _ERROR; + } + + static char* chkpMeta = "META"; + memset(dstBuf, 0, len); + sprintf(dstDir, "%s%s%s", dstDir, TD_DIRSEP, chkpMeta); + + TdFilePtr pFile = taosOpenFile(dstDir, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + stError("chkp failed to create meta file: %s", dstDir); + goto _ERROR; + } + char content[128] = {0}; + snprintf(content, sizeof(content), "%s_%" PRId64 "\n%s_%" PRId64 "", p->pCurrent, p->curChkpId, p->pManifest, + p->curChkpId); + if (taosWriteFile(pFile, content, strlen(content)) <= 0) { + stError("chkp failed to write meta file: %s", dstDir); + taosCloseFile(&pFile); + goto _ERROR; + } + taosCloseFile(&pFile); + + // clear delta data buf + taosArrayClearP(p->pAdd, taosMemoryFree); + taosArrayClearP(p->pDel, taosMemoryFree); + code = 0; + +_ERROR: + taosThreadRwlockUnlock(&p->rwLock); + taosMemoryFree(srcBuf); + taosMemoryFree(dstBuf); + taosMemoryFree(srcDir); + taosMemoryFree(dstDir); + return code; +} +SBkdMgt* bkdMgtCreate(char* path) { + SBkdMgt* p = taosMemoryCalloc(1, sizeof(SBkdMgt)); + p->pDbChkpTbl = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + p->path = taosStrdup(path); + taosThreadRwlockInit(&p->rwLock, NULL); + return p; +} + +void bkdMgtDestroy(SBkdMgt* bm) { + if (bm == NULL) return; + void* pIter = taosHashIterate(bm->pDbChkpTbl, NULL); + while (pIter) { + SDbChkp* pChkp = *(SDbChkp**)(pIter); + dbChkpDestroy(pChkp); + + pIter = taosHashIterate(bm->pDbChkpTbl, pIter); + } + + taosThreadRwlockDestroy(&bm->rwLock); + taosMemoryFree(bm->path); + taosHashCleanup(bm->pDbChkpTbl); + + taosMemoryFree(bm); +} +int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list, char* dname) { + int32_t code = 0; + + taosThreadRwlockWrlock(&bm->rwLock); + SDbChkp** ppChkp = taosHashGet(bm->pDbChkpTbl, taskId, strlen(taskId)); + SDbChkp* pChkp = ppChkp != NULL ? *ppChkp : NULL; + + if (pChkp == NULL) { + char* path = taosMemoryCalloc(1, strlen(bm->path) + 64); + sprintf(path, "%s%s%s", bm->path, TD_DIRSEP, taskId); + + SDbChkp* p = dbChkpCreate(path, chkpId); + taosHashPut(bm->pDbChkpTbl, taskId, strlen(taskId), &p, sizeof(void*)); + + pChkp = p; + + code = dbChkpDumpTo(pChkp, dname, list); + taosThreadRwlockUnlock(&bm->rwLock); + return code; + } + + code = dbChkpGetDelta(pChkp, chkpId, NULL); + code = dbChkpDumpTo(pChkp, dname, list); + + taosThreadRwlockUnlock(&bm->rwLock); + return code; +} + +int32_t bkdMgtAddChkp(SBkdMgt* bm, char* task, char* path) { + int32_t code = -1; + + taosThreadRwlockWrlock(&bm->rwLock); + SDbChkp** pp = taosHashGet(bm->pDbChkpTbl, task, strlen(task)); + if (pp == NULL) { + SDbChkp* p = dbChkpCreate(path, 0); + if (p != NULL) { + taosHashPut(bm->pDbChkpTbl, task, strlen(task), &p, sizeof(void*)); + code = 0; + } + } else { + stError("task chkp already exists"); + } + + taosThreadRwlockUnlock(&bm->rwLock); + + return code; +} + +int32_t bkdMgtDumpTo(SBkdMgt* bm, char* taskId, char* dname) { + int32_t code = 0; + taosThreadRwlockRdlock(&bm->rwLock); + + SDbChkp* p = taosHashGet(bm->pDbChkpTbl, taskId, strlen(taskId)); + code = dbChkpDumpTo(p, dname, NULL); + + taosThreadRwlockUnlock(&bm->rwLock); + return code; +} \ No newline at end of file diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 9391cbcd9b..e2561de841 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -15,8 +15,16 @@ #include "cos.h" #include "rsync.h" +#include "streamBackendRocksdb.h" #include "streamInt.h" +typedef struct { + UPLOAD_TYPE type; + char* taskId; + int64_t chkpId; + + SStreamTask* pTask; +} SAsyncUploadArg; int32_t tEncodeStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; @@ -95,12 +103,12 @@ int32_t tDecodeStreamCheckpointReadyMsg(SDecoder* pDecoder, SStreamCheckpointRea static int32_t streamAlignCheckpoint(SStreamTask* pTask) { int32_t num = taosArrayGetSize(pTask->upstreamInfo.pList); - int64_t old = atomic_val_compare_exchange_32(&pTask->checkpointAlignCnt, 0, num); + int64_t old = atomic_val_compare_exchange_32(&pTask->chkInfo.downstreamAlignNum, 0, num); if (old == 0) { stDebug("s-task:%s set initial align upstream num:%d", pTask->id.idStr, num); } - return atomic_sub_fetch_32(&pTask->checkpointAlignCnt, 1); + return atomic_sub_fetch_32(&pTask->chkInfo.downstreamAlignNum, 1); } static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType) { @@ -118,7 +126,7 @@ static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpoint } pBlock->info.type = STREAM_CHECKPOINT; - pBlock->info.version = pTask->checkpointingId; + pBlock->info.version = pTask->chkInfo.checkpointingId; pBlock->info.rows = 1; pBlock->info.childId = pTask->info.selfChildId; @@ -141,13 +149,12 @@ int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSo // 1. set task status to be prepared for check point, no data are allowed to put into inputQ. streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT); - pTask->checkpointingId = pReq->checkpointId; - pTask->checkpointNotReadyTasks = streamTaskGetNumOfDownstream(pTask); + pTask->chkInfo.checkpointingId = pReq->checkpointId; + pTask->chkInfo.checkpointNotReadyTasks = streamTaskGetNumOfDownstream(pTask); pTask->chkInfo.startTs = taosGetTimestampMs(); pTask->execInfo.checkpoint += 1; // 2. Put the checkpoint block into inputQ, to make sure all blocks with less version have been handled by this task - // already. int32_t code = appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER); return code; } @@ -171,13 +178,12 @@ static int32_t continueDispatchCheckpointBlock(SStreamDataBlock* pBlock, SStream int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { SSDataBlock* pDataBlock = taosArrayGet(pBlock->blocks, 0); int64_t checkpointId = pDataBlock->info.version; - - const char* id = pTask->id.idStr; - int32_t code = TSDB_CODE_SUCCESS; + const char* id = pTask->id.idStr; + int32_t code = TSDB_CODE_SUCCESS; // set task status if (streamTaskGetStatus(pTask, NULL) != TASK_STATUS__CK) { - pTask->checkpointingId = checkpointId; + pTask->chkInfo.checkpointingId = checkpointId; code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT); if (code != TSDB_CODE_SUCCESS) { stError("s-task:%s handle checkpoint-trigger block failed, code:%s", id, tstrerror(code)); @@ -185,26 +191,15 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc } } - { // todo: remove this when the pipeline checkpoint generating is used. - SStreamMeta* pMeta = pTask->pMeta; - streamMetaWLock(pMeta); - - if (pMeta->chkptNotReadyTasks == 0) { - pMeta->chkptNotReadyTasks = pMeta->numOfStreamTasks; - } - - streamMetaWUnLock(pMeta); - } - // todo fix race condition: set the status and append checkpoint block int32_t taskLevel = pTask->info.taskLevel; if (taskLevel == TASK_LEVEL__SOURCE) { - if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH || - pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { + int8_t type = pTask->outputInfo.type; + if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH) { stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId); continueDispatchCheckpointBlock(pBlock, pTask); } else { // only one task exists, no need to dispatch downstream info - atomic_add_fetch_32(&pTask->checkpointNotReadyTasks, 1); + atomic_add_fetch_32(&pTask->chkInfo.checkpointNotReadyTasks, 1); streamProcessCheckpointReadyMsg(pTask); streamFreeQitem((SStreamQueueItem*)pBlock); } @@ -233,15 +228,13 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc id, num); streamFreeQitem((SStreamQueueItem*)pBlock); streamTaskBuildCheckpoint(pTask); - } else { - stDebug( - "s-task:%s process checkpoint block, all %d upstreams sent checkpoint msgs, dispatch checkpoint msg " - "downstream", - id, num); + } else { // source & agg tasks need to forward the checkpoint msg downwards + stDebug("s-task:%s process checkpoint block, all %d upstreams sent checkpoint msgs, continue forwards msg", id, + num); // set the needed checked downstream tasks, only when all downstream tasks do checkpoint complete, this task // can start local checkpoint procedure - pTask->checkpointNotReadyTasks = streamTaskGetNumOfDownstream(pTask); + pTask->chkInfo.checkpointNotReadyTasks = streamTaskGetNumOfDownstream(pTask); // Put the checkpoint block into inputQ, to make sure all blocks with less version have been handled by this task // already. And then, dispatch check point msg to all downstream tasks @@ -260,7 +253,7 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask) { ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE || pTask->info.taskLevel == TASK_LEVEL__AGG); // only when all downstream tasks are send checkpoint rsp, we can start the checkpoint procedure for the agg task - int32_t notReady = atomic_sub_fetch_32(&pTask->checkpointNotReadyTasks, 1); + int32_t notReady = atomic_sub_fetch_32(&pTask->chkInfo.checkpointNotReadyTasks, 1); ASSERT(notReady >= 0); if (notReady == 0) { @@ -276,11 +269,11 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask) { } void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) { - pTask->checkpointingId = 0; // clear the checkpoint id + pTask->chkInfo.checkpointingId = 0; // clear the checkpoint id pTask->chkInfo.failedId = 0; pTask->chkInfo.startTs = 0; // clear the recorded start time - pTask->checkpointNotReadyTasks = 0; - pTask->checkpointAlignCnt = 0; + pTask->chkInfo.checkpointNotReadyTasks = 0; + // pTask->chkInfo.checkpointAlignCnt = 0; pTask->chkInfo.dispatchCheckpointTrigger = false; streamTaskOpenAllUpstreamInput(pTask); // open inputQ for all upstream tasks if (clearChkpReadyMsg) { @@ -288,100 +281,217 @@ void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) { } } -int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, int64_t checkpointId) { - int32_t vgId = pMeta->vgId; - int32_t code = 0; +int32_t streamSaveTaskCheckpointInfo(SStreamTask* p, int64_t checkpointId) { + SStreamMeta* pMeta = p->pMeta; + int32_t vgId = pMeta->vgId; + const char* id = p->id.idStr; + int32_t code = 0; - streamMetaWLock(pMeta); + if (p->info.fillHistory == 1) { + return code; + } - for (int32_t i = 0; i < taosArrayGetSize(pMeta->pTaskList); ++i) { - STaskId* pId = taosArrayGet(pMeta->pTaskList, i); - SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, pId, sizeof(*pId)); - if (ppTask == NULL) { - continue; - } + if (p->info.taskLevel > TASK_LEVEL__SINK) { + return code; + } - SStreamTask* p = *ppTask; - if (p->info.fillHistory == 1) { - continue; - } + taosThreadMutexLock(&p->lock); - ASSERT(p->chkInfo.checkpointId <= p->checkpointingId && p->checkpointingId == checkpointId && - p->chkInfo.checkpointVer <= p->chkInfo.processedVer); + ASSERT(p->chkInfo.checkpointId <= p->chkInfo.checkpointingId && p->chkInfo.checkpointingId == checkpointId && + p->chkInfo.checkpointVer <= p->chkInfo.processedVer); + p->chkInfo.checkpointId = p->chkInfo.checkpointingId; + p->chkInfo.checkpointVer = p->chkInfo.processedVer; - p->chkInfo.checkpointId = p->checkpointingId; - p->chkInfo.checkpointVer = p->chkInfo.processedVer; + streamTaskClearCheckInfo(p, false); + char* str = NULL; + streamTaskGetStatus(p, &str); - streamTaskClearCheckInfo(p, false); + code = streamTaskHandleEvent(p->status.pSM, TASK_EVENT_CHECKPOINT_DONE); + taosThreadMutexUnlock(&p->lock); - char* str = NULL; - streamTaskGetStatus(p, &str); + if (code != TSDB_CODE_SUCCESS) { + stDebug("s-task:%s vgId:%d handle event:checkpoint-done failed", id, vgId); + return -1; + } - code = streamTaskHandleEvent(p->status.pSM, TASK_EVENT_CHECKPOINT_DONE); + stDebug("vgId:%d s-task:%s level:%d open upstream inputQ, save status after checkpoint, checkpointId:%" PRId64 + ", Ver(saved):%" PRId64 " currentVer:%" PRId64 ", status: normal, prev:%s", + vgId, id, p->info.taskLevel, checkpointId, p->chkInfo.checkpointVer, p->chkInfo.nextProcessVer, str); + + // save the task if not sink task + if (p->info.taskLevel != TASK_LEVEL__SINK) { + streamMetaWLock(pMeta); + + code = streamMetaSaveTask(pMeta, p); if (code != TSDB_CODE_SUCCESS) { - stDebug("s-task:%s vgId:%d save task status failed, since handle event failed", p->id.idStr, vgId); streamMetaWUnLock(pMeta); - return -1; - } else { // save the task - streamMetaSaveTask(pMeta, p); + stError("s-task:%s vgId:%d failed to save task info after do checkpoint, checkpointId:%" PRId64 ", since %s", id, + vgId, checkpointId, terrstr()); + return code; } - stDebug( - "vgId:%d s-task:%s level:%d open upstream inputQ, commit task status after checkpoint completed, " - "checkpointId:%" PRId64 ", Ver(saved):%" PRId64 " currentVer:%" PRId64 ", status to be normal, prev:%s", - pMeta->vgId, p->id.idStr, p->info.taskLevel, checkpointId, p->chkInfo.checkpointVer, p->chkInfo.nextProcessVer, - str); - } + code = streamMetaCommit(pMeta); + if (code != TSDB_CODE_SUCCESS) { + stError("s-task:%s vgId:%d failed to commit stream meta after do checkpoint, checkpointId:%" PRId64 ", since %s", + id, vgId, checkpointId, terrstr()); + } - code = streamMetaCommit(pMeta); - if (code < 0) { - stError("vgId:%d failed to commit stream meta after do checkpoint, checkpointId:%" PRId64 ", since %s", pMeta->vgId, - checkpointId, terrstr()); - } else { - stInfo("vgId:%d commit stream meta after do checkpoint, checkpointId:%" PRId64 " DONE", pMeta->vgId, checkpointId); + streamMetaWUnLock(pMeta); } - - streamMetaWUnLock(pMeta); return code; } -int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { - int32_t code = 0; +void streamTaskSetFailedId(SStreamTask* pTask) { + pTask->chkInfo.failedId = pTask->chkInfo.checkpointingId; + pTask->chkInfo.checkpointId = pTask->chkInfo.checkpointingId; +} - // check for all tasks, and do generate the vnode-wide checkpoint data. - SStreamMeta* pMeta = pTask->pMeta; - int32_t remain = atomic_sub_fetch_32(&pMeta->chkptNotReadyTasks, 1); - ASSERT(remain >= 0); - - double el = (taosGetTimestampMs() - pTask->chkInfo.startTs) / 1000.0; - if (remain == 0) { // all tasks are ready - stDebug("s-task:%s all downstreams are ready, ready for do checkpoint", pTask->id.idStr); - streamBackendDoCheckpoint(pMeta, pTask->checkpointingId); - streamSaveAllTaskStatus(pMeta, pTask->checkpointingId); - stInfo( - "vgId:%d vnode wide checkpoint completed, save all tasks status, last:%s, level:%d elapsed time:%.2f Sec " - "checkpointId:%" PRId64, - pMeta->vgId, pTask->id.idStr, pTask->info.taskLevel, el, pTask->checkpointingId); +int32_t getChkpMeta(char* id, char* path, SArray* list) { + char* file = taosMemoryCalloc(1, strlen(path) + 32); + sprintf(file, "%s%s%s", path, TD_DIRSEP, "META_TMP"); + int32_t code = downloadCheckpointByName(id, "META", file); + if (code != 0) { + stDebug("chkp failed to download meta file:%s", file); + taosMemoryFree(file); + return code; + } + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + char buf[128] = {0}; + if (taosReadFile(pFile, buf, sizeof(buf)) <= 0) { + stError("chkp failed to read meta file:%s", file); + code = -1; } else { - stInfo( - "vgId:%d vnode wide tasks not reach checkpoint ready status, ready s-task:%s, level:%d elapsed time:%.2f Sec " - "not ready:%d/%d", - pMeta->vgId, pTask->id.idStr, pTask->info.taskLevel, el, remain, pMeta->numOfStreamTasks); + int32_t len = strlen(buf); + for (int i = 0; i < len; i++) { + if (buf[i] == '\n') { + char* item = taosMemoryCalloc(1, i + 1); + memcpy(item, buf, i); + taosArrayPush(list, &item); + + item = taosMemoryCalloc(1, len - i); + memcpy(item, buf + i + 1, len - i - 1); + taosArrayPush(list, &item); + } + } + } + taosCloseFile(&pFile); + taosRemoveFile(file); + taosMemoryFree(file); + return code; +} +int32_t doUploadChkp(void* param) { + SAsyncUploadArg* arg = param; + char* path = NULL; + int32_t code = 0; + SArray* toDelFiles = taosArrayInit(4, sizeof(void*)); + + if ((code = taskDbGenChkpUploadData(arg->pTask->pBackend, arg->pTask->pMeta->bkdChkptMgt, arg->chkpId, + (int8_t)(arg->type), &path, toDelFiles)) != 0) { + stError("s-task:%s failed to gen upload checkpoint:%" PRId64 "", arg->pTask->id.idStr, arg->chkpId); + } + if (arg->type == UPLOAD_S3) { + if (code == 0 && (code = getChkpMeta(arg->taskId, path, toDelFiles)) != 0) { + stError("s-task:%s failed to get checkpoint:%" PRId64 " meta", arg->pTask->id.idStr, arg->chkpId); + } + } + + if (code == 0 && (code = uploadCheckpoint(arg->taskId, path)) != 0) { + stError("s-task:%s failed to upload checkpoint:%" PRId64, arg->pTask->id.idStr, arg->chkpId); + } + + if (code == 0) { + for (int i = 0; i < taosArrayGetSize(toDelFiles); i++) { + char* p = taosArrayGetP(toDelFiles, i); + code = deleteCheckpointFile(arg->taskId, p); + stDebug("s-task:%s try to del file: %s", arg->pTask->id.idStr, p); + if (code != 0) { + break; + } + } + } + + taosArrayDestroyP(toDelFiles, taosMemoryFree); + + taosRemoveDir(path); + taosMemoryFree(path); + + taosMemoryFree(arg->taskId); + taosMemoryFree(arg); + return code; +} +int32_t streamTaskUploadChkp(SStreamTask* pTask, int64_t chkpId, char* taskId) { + // async upload + UPLOAD_TYPE type = getUploadType(); + if (type == UPLOAD_DISABLE) { + return 0; + } + SAsyncUploadArg* arg = taosMemoryCalloc(1, sizeof(SAsyncUploadArg)); + arg->type = type; + arg->taskId = taosStrdup(taskId); + arg->chkpId = chkpId; + arg->pTask = pTask; + + return streamMetaAsyncExec(pTask->pMeta, doUploadChkp, arg, NULL); +} +int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { + int32_t code = TSDB_CODE_SUCCESS; + int64_t startTs = pTask->chkInfo.startTs; + int64_t ckId = pTask->chkInfo.checkpointingId; + + // sink task do not need to save the status, and generated the checkpoint + if (pTask->info.taskLevel != TASK_LEVEL__SINK) { + stDebug("s-task:%s level:%d start gen checkpoint", pTask->id.idStr, pTask->info.taskLevel); + code = streamBackendDoCheckpoint(pTask->pBackend, ckId); + if (code != TSDB_CODE_SUCCESS) { + stError("s-task:%s gen checkpoint:%" PRId64 " failed, code:%s", pTask->id.idStr, ckId, tstrerror(terrno)); + } } // send check point response to upstream task - if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { - code = streamTaskSendCheckpointSourceRsp(pTask); - } else { - code = streamTaskSendCheckpointReadyMsg(pTask); + if (code == TSDB_CODE_SUCCESS) { + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + code = streamTaskSendCheckpointSourceRsp(pTask); + } else { + code = streamTaskSendCheckpointReadyMsg(pTask); + } + + if (code != TSDB_CODE_SUCCESS) { + // todo: let's retry send rsp to upstream/mnode + stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%" PRId64 ", code:%s", pTask->id.idStr, + ckId, tstrerror(code)); + } } - if (code != TSDB_CODE_SUCCESS) { - // todo: let's retry send rsp to upstream/mnode - stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%" PRId64 ", code:%s", pTask->id.idStr, - pTask->checkpointingId, tstrerror(code)); + // clear the checkpoint info, and commit the newest checkpoint info if all works are done successfully + if (code == TSDB_CODE_SUCCESS) { + code = streamSaveTaskCheckpointInfo(pTask, ckId); + if (code != TSDB_CODE_SUCCESS) { + stError("s-task:%s commit taskInfo failed, checkpoint:%" PRId64 " failed, code:%s", pTask->id.idStr, ckId, + tstrerror(terrno)); + } else { + code = streamTaskUploadChkp(pTask, ckId, (char*)pTask->id.idStr); + if (code != 0) { + stError("s-task:%s failed to upload checkpoint:%" PRId64 " failed", pTask->id.idStr, ckId); + } + } } + if (code != TSDB_CODE_SUCCESS) { // clear the checkpoint info if failed + taosThreadMutexLock(&pTask->lock); + streamTaskClearCheckInfo(pTask, false); + code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_CHECKPOINT_DONE); + taosThreadMutexUnlock(&pTask->lock); + + streamTaskSetFailedId(pTask); + stDebug("s-task:%s clear checkpoint flag since gen checkpoint failed, checkpointId:%" PRId64, pTask->id.idStr, + ckId); + } + + double el = (taosGetTimestampMs() - startTs) / 1000.0; + stInfo("s-task:%s vgId:%d level:%d, checkpointId:%" PRId64 " ver:%" PRId64 " elapsed time:%.2f Sec, %s ", + pTask->id.idStr, pTask->pMeta->vgId, pTask->info.taskLevel, ckId, pTask->chkInfo.checkpointVer, el, + (code == TSDB_CODE_SUCCESS) ? "succ" : "failed"); + return code; } @@ -390,6 +500,7 @@ static int uploadCheckpointToS3(char* id, char* path) { if (pDir == NULL) return -1; TdDirEntryPtr de = NULL; + s3Init(); while ((de = taosReadDir(pDir)) != NULL) { char* name = taosGetDirEntryName(de); if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0 || taosDirEntryIsDir(de)) continue; @@ -409,12 +520,24 @@ static int uploadCheckpointToS3(char* id, char* path) { return -1; } stDebug("[s3] upload checkpoint:%s", filename); + // break; } taosCloseDir(&pDir); return 0; } +static int downloadCheckpointByNameS3(char* id, char* fname, char* dstName) { + int code = 0; + char* buf = taosMemoryCalloc(1, strlen(id) + strlen(dstName) + 4); + sprintf(buf, "%s/%s", id, fname); + if (s3GetObjectToFile(buf, dstName) != 0) { + code = -1; + } + taosMemoryFree(buf); + return code; +} + UPLOAD_TYPE getUploadType() { if (strlen(tsSnodeAddress) != 0) { return UPLOAD_RSYNC; @@ -438,6 +561,20 @@ int uploadCheckpoint(char* id, char* path) { return 0; } +// fileName: CURRENT +int downloadCheckpointByName(char* id, char* fname, char* dstName) { + if (id == NULL || fname == NULL || strlen(id) == 0 || strlen(fname) == 0 || strlen(fname) >= PATH_MAX) { + stError("uploadCheckpointByName parameters invalid"); + return -1; + } + if (strlen(tsSnodeAddress) != 0) { + return 0; + } else if (tsS3StreamEnabled) { + return downloadCheckpointByNameS3(id, fname, dstName); + } + return 0; +} + int downloadCheckpoint(char* id, char* path) { if (id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX) { stError("downloadCheckpoint parameters invalid"); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index b0ba78ca6e..1a67b08749 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -506,9 +506,9 @@ void streamRetryDispatchData(SStreamTask* pTask, int64_t waitDuration) { waitDuration, pTask->execInfo.dispatch, pTask->msgInfo.retryCount); if (pTask->msgInfo.pTimer != NULL) { - taosTmrReset(doRetryDispatchData, waitDuration, pTask, streamEnv.timer, &pTask->msgInfo.pTimer); + taosTmrReset(doRetryDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pTimer); } else { - pTask->msgInfo.pTimer = taosTmrStart(doRetryDispatchData, waitDuration, pTask, streamEnv.timer); + pTask->msgInfo.pTimer = taosTmrStart(doRetryDispatchData, waitDuration, pTask, streamTimer); } } @@ -1167,10 +1167,10 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i pTask->info.taskLevel == TASK_LEVEL__SOURCE) { stError("s-task:%s failed to dispatch checkpoint-trigger msg, checkpointId:%" PRId64 ", set the current checkpoint failed, and send rsp to mnode", - id, pTask->checkpointingId); + id, pTask->chkInfo.checkpointingId); { // send checkpoint failure msg to mnode directly - pTask->chkInfo.failedId = pTask->checkpointingId; // record the latest failed checkpoint id - pTask->checkpointingId = pTask->checkpointingId; + pTask->chkInfo.failedId = pTask->chkInfo.checkpointingId; // record the latest failed checkpoint id + pTask->chkInfo.checkpointingId = pTask->chkInfo.checkpointingId; streamTaskSendCheckpointSourceRsp(pTask); } } else { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 8b14846414..25f32195be 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -252,14 +252,18 @@ static void streamScanHistoryDataImpl(SStreamTask* pTask, SArray* pRes, int32_t* SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) { ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE); - void* exec = pTask->exec.pExecutor; - bool finished = false; + void* exec = pTask->exec.pExecutor; + bool finished = false; + const char* id = pTask->id.idStr; - qSetStreamOpOpen(exec); + if (!pTask->hTaskInfo.operatorOpen) { + qSetStreamOpOpen(exec); + pTask->hTaskInfo.operatorOpen = true; + } while (1) { if (streamTaskShouldPause(pTask)) { - stDebug("s-task:%s paused from the scan-history task", pTask->id.idStr); + stDebug("s-task:%s paused from the scan-history task", id); // quit from step1, not continue to handle the step2 return (SScanhistoryDataInfo){TASK_SCANHISTORY_QUIT, 0}; } @@ -267,8 +271,7 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) { SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); if (pRes == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - stError("s-task:%s scan-history prepare result block failed, code:%s, retry later", pTask->id.idStr, - tstrerror(terrno)); + stError("s-task:%s scan-history prepare result block failed, code:%s, retry later", id, tstrerror(terrno)); continue; } @@ -281,12 +284,12 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) { } // dispatch the generated results - int32_t code = handleResultBlocks(pTask, pRes, size); + /*int32_t code = */handleResultBlocks(pTask, pRes, size); int64_t el = taosGetTimestampMs() - st; // downstream task input queue is full, try in 5sec - if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) { + if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED && (pTask->info.fillHistory == 1)) { return (SScanhistoryDataInfo){TASK_SCANHISTORY_REXEC, 5000}; } @@ -294,9 +297,9 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) { return (SScanhistoryDataInfo){TASK_SCANHISTORY_CONT, 0}; } - if (el >= STREAM_SCAN_HISTORY_TIMESLICE) { - stDebug("s-task:%s fill-history:%d time slice exhausted, elapsed time:%.2fs, retry in 100ms", - pTask->id.idStr, pTask->info.fillHistory, el / 1000.0); + if (el >= STREAM_SCAN_HISTORY_TIMESLICE && (pTask->info.fillHistory == 1)) { + stDebug("s-task:%s fill-history:%d time slice exhausted, elapsed time:%.2fs, retry in 100ms", id, + pTask->info.fillHistory, el / 1000.0); return (SScanhistoryDataInfo){TASK_SCANHISTORY_REXEC, 100}; } } @@ -543,7 +546,7 @@ int32_t streamProcessTranstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock * todo: the batch of blocks should be tuned dynamic, according to the total elapsed time of each batch of blocks, the * appropriate batch of blocks should be handled in 5 to 10 sec. */ -int32_t streamExecForAll(SStreamTask* pTask) { +int32_t doStreamExecTask(SStreamTask* pTask) { const char* id = pTask->id.idStr; // merge multiple input data if possible in the input queue. @@ -654,7 +657,7 @@ int32_t streamExecTask(SStreamTask* pTask) { int8_t schedStatus = streamTaskSetSchedStatusActive(pTask); if (schedStatus == TASK_SCHED_STATUS__WAITING) { while (1) { - int32_t code = streamExecForAll(pTask); + int32_t code = doStreamExecTask(pTask); if (code < 0) { // todo this status should be removed atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__FAILED); return -1; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 00b292a69a..807f120cb7 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -18,6 +18,7 @@ #include "streamInt.h" #include "tmisce.h" #include "tref.h" +#include "tsched.h" #include "tstream.h" #include "ttimer.h" #include "wal.h" @@ -27,6 +28,7 @@ static TdThreadOnce streamMetaModuleInit = PTHREAD_ONCE_INIT; int32_t streamBackendId = 0; int32_t streamBackendCfWrapperId = 0; int32_t streamMetaId = 0; +int32_t taskDbWrapperId = 0; static void metaHbToMnode(void* param, void* tmrId); static void streamMetaClear(SStreamMeta* pMeta); @@ -55,6 +57,7 @@ int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid); static void streamMetaEnvInit() { streamBackendId = taosOpenRef(64, streamBackendCleanup); streamBackendCfWrapperId = taosOpenRef(64, streamBackendHandleCleanup); + taskDbWrapperId = taosOpenRef(64, taskDbDestroy2); streamMetaId = taosOpenRef(64, streamMetaCloseImpl); @@ -62,6 +65,7 @@ static void streamMetaEnvInit() { } void streamMetaInit() { taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); } + void streamMetaCleanup() { taosCloseRef(streamBackendId); taosCloseRef(streamBackendCfWrapperId); @@ -106,6 +110,174 @@ int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid) { return 0; } +typedef struct { + int64_t chkpId; + char* path; + char* taskId; + + SArray* pChkpSave; + SArray* pChkpInUse; + int8_t chkpCap; + void* backend; + +} StreamMetaTaskState; + +int32_t streamMetaOpenTdb(SStreamMeta* pMeta) { + if (tdbOpen(pMeta->path, 16 * 1024, 1, &pMeta->db, 0) < 0) { + return -1; + // goto _err; + } + + if (tdbTbOpen("task.db", STREAM_TASK_KEY_LEN, -1, NULL, pMeta->db, &pMeta->pTaskDb, 0) < 0) { + return -1; + } + + if (tdbTbOpen("checkpoint.db", sizeof(int32_t), -1, NULL, pMeta->db, &pMeta->pCheckpointDb, 0) < 0) { + return -1; + } + return 0; +} + +// +// impl later +// +enum STREAM_STATE_VER { + STREAM_STATA_NO_COMPATIBLE, + STREAM_STATA_COMPATIBLE, + STREAM_STATA_NEED_CONVERT, +}; + +int32_t streamMetaCheckBackendCompatible(SStreamMeta* pMeta) { + int8_t ret = STREAM_STATA_COMPATIBLE; + TBC* pCur = NULL; + + if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) { + // no task info, no stream + return ret; + } + void* pKey = NULL; + int32_t kLen = 0; + void* pVal = NULL; + int32_t vLen = 0; + + tdbTbcMoveToFirst(pCur); + while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { + if (pVal == NULL || vLen == 0) { + break; + } + SDecoder decoder; + SCheckpointInfo info; + tDecoderInit(&decoder, (uint8_t*)pVal, vLen); + if (tDecodeStreamTaskChkInfo(&decoder, &info) < 0) { + continue; + } + if (info.msgVer <= SSTREAM_TASK_INCOMPATIBLE_VER) { + ret = STREAM_STATA_NO_COMPATIBLE; + } else if (info.msgVer == SSTREAM_TASK_NEED_CONVERT_VER) { + ret = STREAM_STATA_NEED_CONVERT; + } else if (info.msgVer == SSTREAM_TASK_VER) { + ret = STREAM_STATA_COMPATIBLE; + } + tDecoderClear(&decoder); + break; + } + tdbFree(pKey); + tdbFree(pVal); + tdbTbcClose(pCur); + return ret; +} + +int32_t streamMetaCvtDbFormat(SStreamMeta* pMeta) { + int32_t code = 0; + int64_t chkpId = streamMetaGetLatestCheckpointId(pMeta); + + bool exist = streamBackendDataIsExist(pMeta->path, chkpId, pMeta->vgId); + if (exist == false) { + return code; + } + SBackendWrapper* pBackend = streamBackendInit(pMeta->path, chkpId, pMeta->vgId); + + void* pIter = taosHashIterate(pBackend->cfInst, NULL); + while (pIter) { + void* key = taosHashGetKey(pIter, NULL); + code = streamStateCvtDataFormat(pMeta->path, key, *(void**)pIter); + if (code != 0) { + qError("failed to cvt data"); + goto _EXIT; + } + + pIter = taosHashIterate(pBackend->cfInst, pIter); + } + +_EXIT: + streamBackendCleanup((void*)pBackend); + + if (code == 0) { + char* state = taosMemoryCalloc(1, strlen(pMeta->path) + 32); + sprintf(state, "%s%s%s", pMeta->path, TD_DIRSEP, "state"); + taosRemoveDir(state); + taosMemoryFree(state); + } + + return code; +} +int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) { + int8_t compatible = streamMetaCheckBackendCompatible(pMeta); + if (compatible == STREAM_STATA_COMPATIBLE) { + return 0; + } else if (compatible == STREAM_STATA_NEED_CONVERT) { + qInfo("stream state need covert backend format"); + + return streamMetaCvtDbFormat(pMeta); + } else if (compatible == STREAM_STATA_NO_COMPATIBLE) { + qError( + "stream read incompatible data, rm %s/vnode/vnode*/tq/stream if taosd cannot start, and rebuild stream " + "manually", + tsDataDir); + + return -1; + } + return 0; +} + +int32_t streamTaskSetDb(SStreamMeta* pMeta, void* arg, char* key) { + SStreamTask* pTask = arg; + + int64_t chkpId = pTask->chkInfo.checkpointId; + + taosThreadMutexLock(&pMeta->backendMutex); + void** ppBackend = taosHashGet(pMeta->pTaskDbUnique, key, strlen(key)); + if (ppBackend != NULL && *ppBackend != NULL) { + taskDbAddRef(*ppBackend); + + STaskDbWrapper* pBackend = *ppBackend; + + pTask->backendRefId = pBackend->refId; + pTask->pBackend = pBackend; + taosThreadMutexUnlock(&pMeta->backendMutex); + + stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend); + return 0; + } + + STaskDbWrapper* pBackend = taskDbOpen(pMeta->path, key, chkpId); + if (pBackend == NULL) { + taosThreadMutexUnlock(&pMeta->backendMutex); + return -1; + } + + int64_t tref = taosAddRef(taskDbWrapperId, pBackend); + pTask->backendRefId = tref; + pTask->pBackend = pBackend; + pBackend->refId = tref; + pBackend->pTask = pTask; + + taosHashPut(pMeta->pTaskDbUnique, key, strlen(key), &pBackend, sizeof(void*)); + taosThreadMutexUnlock(&pMeta->backendMutex); + + stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend); + return 0; +} SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage) { int32_t code = -1; SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta)); @@ -121,15 +293,11 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF sprintf(tpath, "%s%s%s", path, TD_DIRSEP, "stream"); pMeta->path = tpath; - if (tdbOpen(pMeta->path, 16 * 1024, 1, &pMeta->db, 0) < 0) { + if (streamMetaOpenTdb(pMeta) < 0) { goto _err; } - if (tdbTbOpen("task.db", STREAM_TASK_KEY_LEN, -1, NULL, pMeta->db, &pMeta->pTaskDb, 0) < 0) { - goto _err; - } - - if (tdbTbOpen("checkpoint.db", sizeof(int32_t), -1, NULL, pMeta->db, &pMeta->pCheckpointDb, 0) < 0) { + if (streamMetaMayCvtDbFormat(pMeta) < 0) { goto _err; } if (streamMetaBegin(pMeta) < 0) { @@ -176,47 +344,41 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->stage = stage; pMeta->role = (vgId == SNODE_HANDLE) ? NODE_ROLE_LEADER : NODE_ROLE_UNINIT; - // send heartbeat every 5sec. - pMeta->rid = taosAddRef(streamMetaId, pMeta); - int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); - *pRid = pMeta->rid; + pMeta->pTaskDbUnique = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); - metaRefMgtAdd(pMeta->vgId, pRid); - - pMeta->pHbInfo->hbTmr = taosTmrStart(metaHbToMnode, META_HB_CHECK_INTERVAL, pRid, streamEnv.timer); - pMeta->pHbInfo->tickCounter = 0; - pMeta->pHbInfo->stopFlag = 0; - - pMeta->pTaskBackendUnique = - taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); - pMeta->chkpSaved = taosArrayInit(4, sizeof(int64_t)); - pMeta->chkpInUse = taosArrayInit(4, sizeof(int64_t)); - pMeta->chkpCap = 2; - taosInitRWLatch(&pMeta->chkpDirLock); - - pMeta->chkpId = streamMetaGetLatestCheckpointId(pMeta); - pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId, pMeta->vgId); - while (pMeta->streamBackend == NULL) { - taosMsleep(100); - pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId, vgId); - if (pMeta->streamBackend == NULL) { - stInfo("vgId:%d failed to init stream backend, retry in 100ms", pMeta->vgId); - } - } - pMeta->streamBackendRid = taosAddRef(streamBackendId, pMeta->streamBackend); - - code = streamBackendLoadCheckpointInfo(pMeta); - - taosInitRWLatch(&pMeta->lock); - taosThreadMutexInit(&pMeta->backendMutex, NULL); + // pMeta->chkpId = streamGetLatestCheckpointId(pMeta); + // pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId); + // while (pMeta->streamBackend == NULL) { + // qError("vgId:%d failed to init stream backend", pMeta->vgId); + // taosMsleep(2 * 1000); + // qInfo("vgId:%d retry to init stream backend", pMeta->vgId); + // pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId); + // if (pMeta->streamBackend == NULL) { + // } + // } + // pMeta->streamBackendRid = taosAddRef(streamBackendId, pMeta->streamBackend); pMeta->numOfPausedTasks = 0; pMeta->numOfStreamTasks = 0; stInfo("vgId:%d open stream meta successfully, latest checkpoint:%" PRId64 ", stage:%" PRId64, vgId, pMeta->chkpId, stage); + + pMeta->rid = taosAddRef(streamMetaId, pMeta); + + int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); + memcpy(pRid, &pMeta->rid, sizeof(pMeta->rid)); + metaRefMgtAdd(pMeta->vgId, pRid); + + pMeta->pHbInfo->hbTmr = taosTmrStart(metaHbToMnode, META_HB_CHECK_INTERVAL, pRid, streamTimer); + pMeta->pHbInfo->tickCounter = 0; + pMeta->pHbInfo->stopFlag = 0; + pMeta->qHandle = taosInitScheduler(32, 1, "stream-chkp", NULL); + + pMeta->bkdChkptMgt = bkdMgtCreate(tpath); + return pMeta; - _err: +_err: taosMemoryFree(pMeta->path); if (pMeta->pTasksMap) taosHashCleanup(pMeta->pTasksMap); if (pMeta->pTaskList) taosArrayDestroy(pMeta->pTaskList); @@ -311,14 +473,13 @@ void streamMetaClear(SStreamMeta* pMeta) { taosRemoveRef(streamBackendId, pMeta->streamBackendRid); taosHashClear(pMeta->pTasksMap); - taosHashClear(pMeta->pTaskBackendUnique); + taosHashClear(pMeta->pTaskDbUnique); taosArrayClear(pMeta->pTaskList); taosArrayClear(pMeta->chkpSaved); taosArrayClear(pMeta->chkpInUse); pMeta->numOfStreamTasks = 0; pMeta->numOfPausedTasks = 0; - pMeta->chkptNotReadyTasks = 0; // the willrestart/starting flag can NOT be cleared taosHashClear(pMeta->startInfo.pReadyTaskSet); @@ -360,7 +521,9 @@ void streamMetaCloseImpl(void* arg) { taosArrayDestroy(pMeta->chkpInUse); taosHashCleanup(pMeta->pTasksMap); - taosHashCleanup(pMeta->pTaskBackendUnique); + taosHashCleanup(pMeta->pTaskDbUnique); + taosHashCleanup(pMeta->pUpdateTaskSet); + // taosHashCleanup(pMeta->pTaskBackendUnique); taosHashCleanup(pMeta->updateInfo.pTasks); taosHashCleanup(pMeta->startInfo.pReadyTaskSet); taosHashCleanup(pMeta->startInfo.pFailedTaskSet); @@ -369,6 +532,11 @@ void streamMetaCloseImpl(void* arg) { taosMemoryFree(pMeta->path); taosThreadMutexDestroy(&pMeta->backendMutex); + taosCleanUpScheduler(pMeta->qHandle); + taosMemoryFree(pMeta->qHandle); + + bkdMgtDestroy(pMeta->bkdChkptMgt); + pMeta->role = NODE_ROLE_UNINIT; taosMemoryFree(pMeta); stDebug("end to close stream meta"); @@ -661,6 +829,11 @@ static void doClear(void* pKey, void* pVal, TBC* pCur, SArray* pRecycleList) { taosArrayDestroy(pRecycleList); } +int32_t streamMetaReloadAllTasks(SStreamMeta* pMeta) { + if (pMeta == NULL) return 0; + + return streamMetaLoadAllTasks(pMeta); +} int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta) { TBC* pCur = NULL; int32_t vgId = pMeta->vgId; @@ -728,8 +901,6 @@ int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta) { } else { // todo this should replace the existed object put by replay creating stream task msg from mnode stError("s-task:0x%x already added into table meta by replaying WAL, need check", pTask->id.taskId); - tdbFree(pKey); - tdbFree(pVal); taosMemoryFree(pTask); continue; } @@ -928,7 +1099,7 @@ void metaHbToMnode(void* param, void* tmrId) { } if (!waitForEnoughDuration(pMeta->pHbInfo)) { - taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamEnv.timer, &pMeta->pHbInfo->hbTmr); + taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); taosReleaseRef(streamMetaId, rid); return; } @@ -983,9 +1154,9 @@ void metaHbToMnode(void* param, void* tmrId) { entry.sinkDataSize = SIZE_IN_MiB((*pTask)->execInfo.sink.dataSize); } - if ((*pTask)->checkpointingId != 0) { - entry.checkpointFailed = ((*pTask)->chkInfo.failedId >= (*pTask)->checkpointingId); - entry.activeCheckpointId = (*pTask)->checkpointingId; + if ((*pTask)->chkInfo.checkpointingId != 0) { + entry.checkpointFailed = ((*pTask)->chkInfo.failedId >= (*pTask)->chkInfo.checkpointingId); + entry.activeCheckpointId = (*pTask)->chkInfo.checkpointingId; } if ((*pTask)->exec.pWalReader != NULL) { @@ -1028,7 +1199,9 @@ void metaHbToMnode(void* param, void* tmrId) { } tEncoderClear(&encoder); - SRpcMsg msg = {.info.noResp = 1,}; + SRpcMsg msg = { + .info.noResp = 1, + }; initRpcMsg(&msg, TDMT_MND_STREAM_HEARTBEAT, buf, tlen); pMeta->pHbInfo->hbCount += 1; @@ -1040,9 +1213,9 @@ void metaHbToMnode(void* param, void* tmrId) { stDebug("vgId:%d no tasks and no mnd epset, not send stream hb to mnode", pMeta->vgId); } - _end: +_end: clearHbMsg(&hbMsg, pIdList); - taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamEnv.timer, &pMeta->pHbInfo->hbTmr); + taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); taosReleaseRef(streamMetaId, rid); } @@ -1070,8 +1243,8 @@ bool streamMetaTaskInTimer(SStreamMeta* pMeta) { void streamMetaNotifyClose(SStreamMeta* pMeta) { int32_t vgId = pMeta->vgId; - stDebug("vgId:%d notify all stream tasks that the vnode is closing. isLeader:%d startHb:%" PRId64 ", totalHb:%d", vgId, - (pMeta->role == NODE_ROLE_LEADER), pMeta->pHbInfo->hbStart, pMeta->pHbInfo->hbCount); + stDebug("vgId:%d notify all stream tasks that the vnode is closing. isLeader:%d startHb:%" PRId64 ", totalHb:%d", + vgId, (pMeta->role == NODE_ROLE_LEADER), pMeta->pHbInfo->hbStart, pMeta->pHbInfo->hbCount); streamMetaWLock(pMeta); @@ -1142,4 +1315,19 @@ void streamMetaWUnLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-wunlock", pMeta->vgId); taosWUnLockLatch(&pMeta->lock); } +static void execHelper(struct SSchedMsg* pSchedMsg) { + __async_exec_fn_t execFn = (__async_exec_fn_t)pSchedMsg->ahandle; + int32_t code = execFn(pSchedMsg->thandle); + if (code != 0 && pSchedMsg->msg != NULL) { + *(int32_t*)pSchedMsg->msg = code; + } +} +int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, void* param, int32_t* code) { + SSchedMsg schedMsg = {0}; + schedMsg.fp = execHelper; + schedMsg.ahandle = fn; + schedMsg.thandle = param; + schedMsg.msg = code; + return taosScheduleTask(pMeta->qHandle, &schedMsg); +} diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 5893bc14f1..e29f2ba7de 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -32,6 +32,7 @@ typedef struct SBackendFileItem { char* name; int8_t type; int64_t size; + int8_t ref; } SBackendFileItem; typedef struct SBackendFile { char* pCurrent; @@ -40,7 +41,28 @@ typedef struct SBackendFile { SArray* pSst; char* pCheckpointMeta; char* path; + } SBanckendFile; + +typedef struct SBackendSnapFiles2 { + char* pCurrent; + char* pMainfest; + char* pOptions; + SArray* pSst; + char* pCheckpointMeta; + char* path; + + int64_t checkpointId; + int64_t seraial; + int64_t offset; + TdFilePtr fd; + int8_t filetype; + SArray* pFileList; + int32_t currFileIdx; + SStreamTaskSnap snapInfo; + int8_t inited; + +} SBackendSnapFile2; struct SStreamSnapHandle { void* handle; SBanckendFile* pBackendFile; @@ -51,16 +73,23 @@ struct SStreamSnapHandle { int8_t filetype; SArray* pFileList; int32_t currFileIdx; - int8_t delFlag; // 0 : not del, 1: del + char* metaPath; + + SArray* pDbSnapSet; + int32_t currIdx; + int8_t delFlag; // 0 : not del, 1: del }; struct SStreamSnapBlockHdr { int8_t type; int8_t flag; int64_t index; - char name[128]; - int64_t totalSize; - int64_t size; - uint8_t data[]; + // int64_t streamId; + // int64_t taskId; + SStreamTaskSnap snapInfo; + char name[128]; + int64_t totalSize; + int64_t size; + uint8_t data[]; }; struct SStreamSnapReader { void* pMeta; @@ -82,7 +111,7 @@ const char* ROCKSDB_CURRENT = "CURRENT"; const char* ROCKSDB_CHECKPOINT_META = "CHECKPOINT"; static int64_t kBlockSize = 64 * 1024; -int32_t streamSnapHandleInit(SStreamSnapHandle* handle, char* path, int64_t chkpId, void* pMeta); +int32_t streamSnapHandleInit(SStreamSnapHandle* handle, char* path, void* pMeta); void streamSnapHandleDestroy(SStreamSnapHandle* handle); // static void streamBuildFname(char* path, char* file, char* fullname) @@ -106,195 +135,205 @@ int32_t streamGetFileSize(char* path, char* name, int64_t* sz) { TdFilePtr streamOpenFile(char* path, char* name, int32_t opt) { char fullname[256] = {0}; + STREAM_ROCKSDB_BUILD_FULLNAME(path, name, fullname); return taosOpenFile(fullname, opt); } -int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chkpId, void* pMeta) { - // impl later - int len = strlen(path); - char* tdir = taosMemoryCalloc(1, len + 256); - memcpy(tdir, path, len); +int32_t streamTaskDbGetSnapInfo(void* arg, char* path, SArray* pSnap) { return taskDbBuildSnap(arg, pSnap); } - int32_t code = 0; +void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) { + if (qDebugFlag & DEBUG_DEBUG) { + char* buf = taosMemoryCalloc(1, 512); + sprintf(buf + strlen(buf), "["); - int8_t validChkp = 0; - if (chkpId != 0) { - sprintf(tdir, "%s%s%s%s%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, "stream", TD_DIRSEP, "checkpoints", TD_DIRSEP, - chkpId); - if (taosIsDir(tdir)) { - validChkp = 1; - stInfo("%s start to read snap %s", STREAM_STATE_TRANSFER, tdir); - streamBackendAddInUseChkp(pMeta, chkpId); - } else { - stWarn("%s failed to read from %s, reason: dir not exist,retry to default state dir", STREAM_STATE_TRANSFER, - tdir); + if (pSnapFile->pCurrent) sprintf(buf, "current: %s,", pSnapFile->pCurrent); + if (pSnapFile->pMainfest) sprintf(buf + strlen(buf), "MANIFEST: %s,", pSnapFile->pMainfest); + if (pSnapFile->pOptions) sprintf(buf + strlen(buf), "options: %s,", pSnapFile->pOptions); + if (pSnapFile->pSst) { + for (int i = 0; i < taosArrayGetSize(pSnapFile->pSst); i++) { + char* name = taosArrayGetP(pSnapFile->pSst, i); + sprintf(buf + strlen(buf), "%s,", name); + } } + sprintf(buf + strlen(buf) - 1, "]"); + + qInfo("%s %" PRId64 "-%" PRId64 " get file list: %s", STREAM_STATE_TRANSFER, pSnapFile->snapInfo.streamId, + pSnapFile->snapInfo.taskId, buf); + taosMemoryFree(buf); } +} - // no checkpoint specified or not exists invalid checkpoint, do checkpoint at default path and translate it - if (validChkp == 0) { - sprintf(tdir, "%s%s%s%s%s", path, TD_DIRSEP, "stream", TD_DIRSEP, "state"); - char* chkpdir = taosMemoryCalloc(1, len + 256); - sprintf(chkpdir, "%s%s%s", tdir, TD_DIRSEP, "tmp"); - taosMemoryFree(tdir); +int32_t snapFileGenMeta(SBackendSnapFile2* pSnapFile) { + SBackendFileItem item = {0}; + item.ref = 1; + // current + item.name = pSnapFile->pCurrent; + item.type = ROCKSDB_CURRENT_TYPE; + streamGetFileSize(pSnapFile->path, item.name, &item.size); + taosArrayPush(pSnapFile->pFileList, &item); - tdir = chkpdir; - stInfo("%s start to trigger checkpoint on %s", STREAM_STATE_TRANSFER, tdir); + // mainfest + item.name = pSnapFile->pMainfest; + item.type = ROCKSDB_MAINFEST_TYPE; + streamGetFileSize(pSnapFile->path, item.name, &item.size); + taosArrayPush(pSnapFile->pFileList, &item); - code = streamBackendTriggerChkp(pMeta, tdir); - if (code != 0) { - stError("%s failed to trigger chekckpoint at %s", STREAM_STATE_TRANSFER, tdir); - taosMemoryFree(tdir); - return code; - } - pHandle->delFlag = 1; - chkpId = 0; + // options + item.name = pSnapFile->pOptions; + item.type = ROCKSDB_OPTIONS_TYPE; + streamGetFileSize(pSnapFile->path, item.name, &item.size); + taosArrayPush(pSnapFile->pFileList, &item); + // sst + for (int i = 0; i < taosArrayGetSize(pSnapFile->pSst); i++) { + char* sst = taosArrayGetP(pSnapFile->pSst, i); + item.name = sst; + item.type = ROCKSDB_SST_TYPE; + streamGetFileSize(pSnapFile->path, item.name, &item.size); + taosArrayPush(pSnapFile->pFileList, &item); } - - stInfo("%s start to read dir: %s", STREAM_STATE_TRANSFER, tdir); - - TdDirPtr pDir = taosOpenDir(tdir); + // meta + item.name = pSnapFile->pCheckpointMeta; + item.type = ROCKSDB_CHECKPOINT_META_TYPE; + if (streamGetFileSize(pSnapFile->path, item.name, &item.size) == 0) { + taosArrayPush(pSnapFile->pFileList, &item); + } + return 0; +} +int32_t snapFileReadMeta(SBackendSnapFile2* pSnapFile) { + TdDirPtr pDir = taosOpenDir(pSnapFile->path); if (NULL == pDir) { - stError("%s failed to open %s", STREAM_STATE_TRANSFER, tdir); - goto _err; + qError("%s failed to open %s", STREAM_STATE_TRANSFER, pSnapFile->path); + return -1; } - SBanckendFile* pFile = taosMemoryCalloc(1, sizeof(SBanckendFile)); - pHandle->pBackendFile = pFile; - pHandle->checkpointId = chkpId; - pHandle->seraial = 0; - - pFile->path = tdir; - pFile->pSst = taosArrayInit(16, sizeof(void*)); - TdDirEntryPtr pDirEntry; while ((pDirEntry = taosReadDir(pDir)) != NULL) { char* name = taosGetDirEntryName(pDirEntry); if (strlen(name) >= strlen(ROCKSDB_CURRENT) && 0 == strncmp(name, ROCKSDB_CURRENT, strlen(ROCKSDB_CURRENT))) { - pFile->pCurrent = taosStrdup(name); + pSnapFile->pCurrent = taosStrdup(name); continue; } if (strlen(name) >= strlen(ROCKSDB_MAINFEST) && 0 == strncmp(name, ROCKSDB_MAINFEST, strlen(ROCKSDB_MAINFEST))) { - pFile->pMainfest = taosStrdup(name); + pSnapFile->pMainfest = taosStrdup(name); continue; } if (strlen(name) >= strlen(ROCKSDB_OPTIONS) && 0 == strncmp(name, ROCKSDB_OPTIONS, strlen(ROCKSDB_OPTIONS))) { - pFile->pOptions = taosStrdup(name); + pSnapFile->pOptions = taosStrdup(name); continue; } if (strlen(name) >= strlen(ROCKSDB_CHECKPOINT_META) && 0 == strncmp(name, ROCKSDB_CHECKPOINT_META, strlen(ROCKSDB_CHECKPOINT_META))) { - pFile->pCheckpointMeta = taosStrdup(name); + pSnapFile->pCheckpointMeta = taosStrdup(name); continue; } if (strlen(name) >= strlen(ROCKSDB_SST) && 0 == strncmp(name + strlen(name) - strlen(ROCKSDB_SST), ROCKSDB_SST, strlen(ROCKSDB_SST))) { char* sst = taosStrdup(name); - taosArrayPush(pFile->pSst, &sst); + taosArrayPush(pSnapFile->pSst, &sst); } } - if (qDebugFlag & DEBUG_TRACE) { - char* buf = taosMemoryCalloc(1, 128 + taosArrayGetSize(pFile->pSst) * 64); - sprintf(buf, "[current: %s,", pFile->pCurrent); - sprintf(buf + strlen(buf), "MANIFEST: %s,", pFile->pMainfest); - sprintf(buf + strlen(buf), "options: %s,", pFile->pOptions); - - for (int i = 0; i < taosArrayGetSize(pFile->pSst); i++) { - char* name = taosArrayGetP(pFile->pSst, i); - sprintf(buf + strlen(buf), "%s,", name); - } - sprintf(buf + strlen(buf) - 1, "]"); - - stInfo("%s get file list: %s", STREAM_STATE_TRANSFER, buf); - taosMemoryFree(buf); - } - taosCloseDir(&pDir); - - if (pFile->pCurrent == NULL) { - stError("%s failed to open %s, reason: no valid file", STREAM_STATE_TRANSFER, tdir); - code = -1; - tdir = NULL; - goto _err; - } - SArray* list = taosArrayInit(64, sizeof(SBackendFileItem)); - - SBackendFileItem item; - // current - item.name = pFile->pCurrent; - item.type = ROCKSDB_CURRENT_TYPE; - streamGetFileSize(pFile->path, item.name, &item.size); - taosArrayPush(list, &item); - - // mainfest - item.name = pFile->pMainfest; - item.type = ROCKSDB_MAINFEST_TYPE; - streamGetFileSize(pFile->path, item.name, &item.size); - taosArrayPush(list, &item); - - // options - item.name = pFile->pOptions; - item.type = ROCKSDB_OPTIONS_TYPE; - streamGetFileSize(pFile->path, item.name, &item.size); - taosArrayPush(list, &item); - // sst - for (int i = 0; i < taosArrayGetSize(pFile->pSst); i++) { - char* sst = taosArrayGetP(pFile->pSst, i); - item.name = sst; - item.type = ROCKSDB_SST_TYPE; - streamGetFileSize(pFile->path, item.name, &item.size); - taosArrayPush(list, &item); - } - // meta - item.name = pFile->pCheckpointMeta; - item.type = ROCKSDB_CHECKPOINT_META_TYPE; - if (streamGetFileSize(pFile->path, item.name, &item.size) == 0) { - taosArrayPush(list, &item); - } - - pHandle->pBackendFile = pFile; - - pHandle->currFileIdx = 0; - pHandle->pFileList = list; - pHandle->seraial = 0; - pHandle->offset = 0; - pHandle->handle = pMeta; return 0; +} +int32_t streamBackendSnapInitFile(char* metaPath, SStreamTaskSnap* pSnap, SBackendSnapFile2* pSnapFile) { + int32_t code = -1; + + char* path = taosMemoryCalloc(1, strlen(pSnap->dbPrefixPath) + 256); + // char idstr[64] = {0}; + sprintf(path, "%s%s%s%s%s%" PRId64 "", pSnap->dbPrefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", + pSnap->chkpId); + if (!taosIsDir(path)) { + goto _ERROR; + } + + pSnapFile->pSst = taosArrayInit(16, sizeof(void*)); + pSnapFile->pFileList = taosArrayInit(64, sizeof(SBackendFileItem)); + pSnapFile->path = path; + pSnapFile->snapInfo = *pSnap; + if ((code = snapFileReadMeta(pSnapFile)) != 0) { + goto _ERROR; + } + if ((code = snapFileGenMeta(pSnapFile)) != 0) { + goto _ERROR; + } + + snapFileDebugInfo(pSnapFile); + path = NULL; + code = 0; + +_ERROR: + taosMemoryFree(path); + return code; +} +void snapFileDestroy(SBackendSnapFile2* pSnap) { + taosMemoryFree(pSnap->pCheckpointMeta); + taosMemoryFree(pSnap->pCurrent); + taosMemoryFree(pSnap->pMainfest); + taosMemoryFree(pSnap->pOptions); + taosMemoryFree(pSnap->path); + for (int i = 0; i < taosArrayGetSize(pSnap->pSst); i++) { + char* sst = taosArrayGetP(pSnap->pSst, i); + taosMemoryFree(sst); + } + // unite read/write snap file + for (int i = 0; i < taosArrayGetSize(pSnap->pFileList); i++) { + SBackendFileItem* pItem = taosArrayGet(pSnap->pFileList, i); + if (pItem->ref == 0) { + taosMemoryFree(pItem->name); + } + } + taosArrayDestroy(pSnap->pFileList); + taosArrayDestroy(pSnap->pSst); + taosCloseFile(&pSnap->fd); + + return; +} +int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, void* pMeta) { + // impl later + SArray* pSnapSet = taosArrayInit(4, sizeof(SStreamTaskSnap)); + int32_t code = streamTaskDbGetSnapInfo(pMeta, path, pSnapSet); + if (code != 0) { + return -1; + } + + SArray* pDbSnapSet = taosArrayInit(8, sizeof(SBackendSnapFile2)); + + for (int i = 0; i < taosArrayGetSize(pSnapSet); i++) { + SStreamTaskSnap* pSnap = taosArrayGet(pSnapSet, i); + + SBackendSnapFile2 snapFile = {0}; + code = streamBackendSnapInitFile(path, pSnap, &snapFile); + ASSERT(code == 0); + taosArrayPush(pDbSnapSet, &snapFile); + } + for (int i = 0; i < taosArrayGetSize(pSnapSet); i++) { + SStreamTaskSnap* pSnap = taosArrayGet(pSnapSet, i); + taosMemoryFree(pSnap->dbPrefixPath); + } + taosArrayDestroy(pSnapSet); + + pHandle->pDbSnapSet = pDbSnapSet; + pHandle->currIdx = 0; + return 0; + _err: streamSnapHandleDestroy(pHandle); - taosMemoryFreeClear(tdir); code = -1; return code; } void streamSnapHandleDestroy(SStreamSnapHandle* handle) { - SBanckendFile* pFile = handle->pBackendFile; - - if (handle->checkpointId == 0) { - // del tmp dir - if (pFile && taosIsDir(pFile->path)) { - if (handle->delFlag) taosRemoveDir(pFile->path); + if (handle->pDbSnapSet) { + for (int i = 0; i < taosArrayGetSize(handle->pDbSnapSet); i++) { + SBackendSnapFile2* pSnapFile = taosArrayGet(handle->pDbSnapSet, i); + snapFileDebugInfo(pSnapFile); + snapFileDestroy(pSnapFile); } - } else { - streamBackendDelInUseChkp(handle->handle, handle->checkpointId); + taosArrayDestroy(handle->pDbSnapSet); } - if (pFile) { - taosMemoryFree(pFile->pCheckpointMeta); - taosMemoryFree(pFile->pCurrent); - taosMemoryFree(pFile->pMainfest); - taosMemoryFree(pFile->pOptions); - taosMemoryFree(pFile->path); - for (int i = 0; i < taosArrayGetSize(pFile->pSst); i++) { - char* sst = taosArrayGetP(pFile->pSst, i); - taosMemoryFree(sst); - } - taosArrayDestroy(pFile->pSst); - taosMemoryFree(pFile); - } - taosArrayDestroy(handle->pFileList); - taosCloseFile(&handle->fd); + taosMemoryFree(handle->metaPath); return; } @@ -305,7 +344,7 @@ int32_t streamSnapReaderOpen(void* pMeta, int64_t sver, int64_t chkpId, char* pa return TSDB_CODE_OUT_OF_MEMORY; } - if (streamSnapHandleInit(&pReader->handle, (char*)path, chkpId, pMeta) < 0) { + if (streamSnapHandleInit(&pReader->handle, (char*)path, pMeta) < 0) { taosMemoryFree(pReader); return -1; } @@ -321,34 +360,50 @@ int32_t streamSnapReaderClose(SStreamSnapReader* pReader) { taosMemoryFree(pReader); return 0; } + int32_t streamSnapRead(SStreamSnapReader* pReader, uint8_t** ppData, int64_t* size) { // impl later int32_t code = 0; SStreamSnapHandle* pHandle = &pReader->handle; - SBanckendFile* pFile = pHandle->pBackendFile; + int32_t idx = pHandle->currIdx; - SBackendFileItem* item = taosArrayGet(pHandle->pFileList, pHandle->currFileIdx); + SBackendSnapFile2* pSnapFile = taosArrayGet(pHandle->pDbSnapSet, idx); + if (pSnapFile == NULL) { + return 0; + } + SBackendFileItem* item = NULL; + +_NEXT: + + if (pSnapFile->fd == NULL) { + if (pSnapFile->currFileIdx >= taosArrayGetSize(pSnapFile->pFileList)) { + if (pHandle->currIdx + 1 < taosArrayGetSize(pHandle->pDbSnapSet)) { + pHandle->currIdx += 1; + + pSnapFile = taosArrayGet(pHandle->pDbSnapSet, pHandle->currIdx); + goto _NEXT; + } else { + *ppData = NULL; + *size = 0; + return 0; + } - if (pHandle->fd == NULL) { - if (pHandle->currFileIdx >= taosArrayGetSize(pHandle->pFileList)) { - // finish - *ppData = NULL; - *size = 0; - return 0; } else { - pHandle->fd = streamOpenFile(pFile->path, item->name, TD_FILE_READ); + item = taosArrayGet(pSnapFile->pFileList, pSnapFile->currFileIdx); + pSnapFile->fd = streamOpenFile(pSnapFile->path, item->name, TD_FILE_READ); stDebug("%s open file %s, current offset:%" PRId64 ", size:% " PRId64 ", file no.%d", STREAM_STATE_TRANSFER, - item->name, (int64_t)pHandle->offset, item->size, pHandle->currFileIdx); + item->name, (int64_t)pSnapFile->offset, item->size, pSnapFile->currFileIdx); } } + item = taosArrayGet(pSnapFile->pFileList, pSnapFile->currFileIdx); + + qDebug("%s start to read file %s, current offset:%" PRId64 ", size:%" PRId64 + ", file no.%d, total set:%d, current set idx: %d", + STREAM_STATE_TRANSFER, item->name, (int64_t)pSnapFile->offset, item->size, pSnapFile->currFileIdx, + (int)taosArrayGetSize(pHandle->pDbSnapSet), pHandle->currIdx); - stDebug("%s start to read file %s, current offset:%" PRId64 ", size:%" PRId64 ", file no.%d", STREAM_STATE_TRANSFER, - item->name, (int64_t)pHandle->offset, item->size, pHandle->currFileIdx); uint8_t* buf = taosMemoryCalloc(1, sizeof(SStreamSnapBlockHdr) + kBlockSize); - if (buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - int64_t nread = taosPReadFile(pHandle->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pHandle->offset); + int64_t nread = taosPReadFile(pSnapFile->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pSnapFile->offset); if (nread == -1) { taosMemoryFree(buf); code = TAOS_SYSTEM_ERROR(terrno); @@ -358,44 +413,51 @@ int32_t streamSnapRead(SStreamSnapReader* pReader, uint8_t** ppData, int64_t* si } else if (nread > 0 && nread <= kBlockSize) { // left bytes less than kBlockSize stDebug("%s read file %s, current offset:%" PRId64 ",size:% " PRId64 ", file no.%d", STREAM_STATE_TRANSFER, - item->name, (int64_t)pHandle->offset, item->size, pHandle->currFileIdx); - pHandle->offset += nread; - if (pHandle->offset >= item->size || nread < kBlockSize) { - taosCloseFile(&pHandle->fd); - pHandle->offset = 0; - pHandle->currFileIdx += 1; + item->name, (int64_t)pSnapFile->offset, item->size, pSnapFile->currFileIdx); + pSnapFile->offset += nread; + if (pSnapFile->offset >= item->size || nread < kBlockSize) { + taosCloseFile(&pSnapFile->fd); + pSnapFile->offset = 0; + pSnapFile->currFileIdx += 1; } } else { stDebug("%s no data read, close file no.%d, move to next file, open and read", STREAM_STATE_TRANSFER, - pHandle->currFileIdx); - taosCloseFile(&pHandle->fd); - pHandle->offset = 0; - pHandle->currFileIdx += 1; + pSnapFile->currFileIdx); + taosCloseFile(&pSnapFile->fd); + pSnapFile->offset = 0; + pSnapFile->currFileIdx += 1; - if (pHandle->currFileIdx >= taosArrayGetSize(pHandle->pFileList)) { + if (pSnapFile->currFileIdx >= taosArrayGetSize(pSnapFile->pFileList)) { // finish - *ppData = NULL; - *size = 0; - taosMemoryFree(buf); - return 0; + if (pHandle->currIdx + 1 < taosArrayGetSize(pHandle->pDbSnapSet)) { + // skip to next snap set + pHandle->currIdx += 1; + pSnapFile = taosArrayGet(pHandle->pDbSnapSet, pHandle->currIdx); + goto _NEXT; + } else { + *ppData = NULL; + *size = 0; + return 0; + } } - item = taosArrayGet(pHandle->pFileList, pHandle->currFileIdx); - pHandle->fd = streamOpenFile(pFile->path, item->name, TD_FILE_READ); + item = taosArrayGet(pSnapFile->pFileList, pSnapFile->currFileIdx); + pSnapFile->fd = streamOpenFile(pSnapFile->path, item->name, TD_FILE_READ); - nread = taosPReadFile(pHandle->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pHandle->offset); - pHandle->offset += nread; + nread = taosPReadFile(pSnapFile->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pSnapFile->offset); + pSnapFile->offset += nread; stDebug("%s open file and read file %s, current offset:%" PRId64 ", size:% " PRId64 ", file no.%d", - STREAM_STATE_TRANSFER, item->name, (int64_t)pHandle->offset, item->size, pHandle->currFileIdx); + STREAM_STATE_TRANSFER, item->name, (int64_t)pSnapFile->offset, item->size, pSnapFile->currFileIdx); } SStreamSnapBlockHdr* pHdr = (SStreamSnapBlockHdr*)buf; pHdr->size = nread; pHdr->type = item->type; pHdr->totalSize = item->size; + pHdr->snapInfo = pSnapFile->snapInfo; memcpy(pHdr->name, item->name, strlen(item->name)); - pHandle->seraial += nread; + pSnapFile->seraial += nread; *ppData = buf; *size = sizeof(SStreamSnapBlockHdr) + nread; @@ -408,101 +470,133 @@ int32_t streamSnapWriterOpen(void* pMeta, int64_t sver, int64_t ever, char* path if (pWriter == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } + SStreamSnapHandle* pHandle = &pWriter->handle; + pHandle->currIdx = 0; + pHandle->metaPath = taosStrdup(path); + pHandle->pDbSnapSet = taosArrayInit(8, sizeof(SBackendSnapFile2)); - SBanckendFile* pFile = taosMemoryCalloc(1, sizeof(SBanckendFile)); - pFile->path = taosStrdup(path); - SArray* list = taosArrayInit(64, sizeof(SBackendFileItem)); - - SBackendFileItem item; - item.name = taosStrdup((char*)ROCKSDB_CURRENT); - item.type = ROCKSDB_CURRENT_TYPE; - taosArrayPush(list, &item); - - pHandle->pBackendFile = pFile; - - pHandle->pFileList = list; - pHandle->currFileIdx = 0; - pHandle->offset = 0; - pHandle->delFlag = 0; + SBackendSnapFile2 snapFile = {0}; + taosArrayPush(pHandle->pDbSnapSet, &snapFile); *ppWriter = pWriter; return 0; } +int32_t snapInfoEqual(SStreamTaskSnap* a, SStreamTaskSnap* b) { + if (a->streamId != b->streamId || a->taskId != b->taskId || a->chkpId != b->chkpId) { + return 0; + } + return 1; +} + +int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nData, SBackendSnapFile2* pSnapFile) { + int code = -1; + SStreamSnapBlockHdr* pHdr = (SStreamSnapBlockHdr*)pData; + SStreamSnapHandle* pHandle = &pWriter->handle; + SStreamTaskSnap snapInfo = pHdr->snapInfo; + + SStreamTaskSnap* pSnapInfo = &pSnapFile->snapInfo; + + SBackendFileItem* pItem = taosArrayGet(pSnapFile->pFileList, pSnapFile->currFileIdx); + + if (pSnapFile->fd == 0) { + pSnapFile->fd = streamOpenFile(pSnapFile->path, pItem->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pSnapFile->fd == NULL) { + code = TAOS_SYSTEM_ERROR(terrno); + stError("%s failed to open file name:%s%s%s, reason:%s", STREAM_STATE_TRANSFER, pHandle->metaPath, TD_DIRSEP, + pHdr->name, tstrerror(code)); + } + } + if (strlen(pHdr->name) == strlen(pItem->name) && strcmp(pHdr->name, pItem->name) == 0) { + int64_t bytes = taosPWriteFile(pSnapFile->fd, pHdr->data, pHdr->size, pSnapFile->offset); + if (bytes != pHdr->size) { + code = TAOS_SYSTEM_ERROR(terrno); + stError("%s failed to write snap, file name:%s, reason:%s", STREAM_STATE_TRANSFER, pHdr->name, tstrerror(code)); + return code; + } else { + qInfo("succ to write data %s", pItem->name); + } + pSnapFile->offset += bytes; + } else { + taosCloseFile(&pSnapFile->fd); + pSnapFile->offset = 0; + pSnapFile->currFileIdx += 1; + + SBackendFileItem item = {0}; + item.name = taosStrdup(pHdr->name); + item.type = pHdr->type; + + taosArrayPush(pSnapFile->pFileList, &item); + + SBackendFileItem* pItem = taosArrayGet(pSnapFile->pFileList, pSnapFile->currFileIdx); + pSnapFile->fd = streamOpenFile(pSnapFile->path, pItem->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pSnapFile->fd == NULL) { + code = TAOS_SYSTEM_ERROR(terrno); + stError("%s failed to open file name:%s%s%s, reason:%s", STREAM_STATE_TRANSFER, pSnapFile->path, TD_DIRSEP, + pHdr->name, tstrerror(code)); + } + + taosPWriteFile(pSnapFile->fd, pHdr->data, pHdr->size, pSnapFile->offset); + qInfo("succ to write data %s", pItem->name); + pSnapFile->offset += pHdr->size; + } + code = 0; +_EXIT: + return code; +} + int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; SStreamSnapBlockHdr* pHdr = (SStreamSnapBlockHdr*)pData; SStreamSnapHandle* pHandle = &pWriter->handle; - SBanckendFile* pFile = pHandle->pBackendFile; - SBackendFileItem* pItem = taosArrayGet(pHandle->pFileList, pHandle->currFileIdx); + SStreamTaskSnap snapInfo = pHdr->snapInfo; - if (pHandle->fd == NULL) { - pHandle->fd = streamOpenFile(pFile->path, pItem->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); - if (pHandle->fd == NULL) { - code = TAOS_SYSTEM_ERROR(terrno); - stError("%s failed to open file name:%s%s%s, reason:%s", STREAM_STATE_TRANSFER, pFile->path, TD_DIRSEP, - pHdr->name, tstrerror(code)); - } - } + SBackendSnapFile2* pDbSnapFile = taosArrayGet(pHandle->pDbSnapSet, pHandle->currIdx); + if (pDbSnapFile->inited == 0) { + char idstr[64] = {0}; + sprintf(idstr, "0x%" PRIx64 "-0x%x", snapInfo.streamId, (int32_t)(snapInfo.taskId)); - if (strlen(pHdr->name) == strlen(pItem->name) && strcmp(pHdr->name, pItem->name) == 0) { - int64_t bytes = taosPWriteFile(pHandle->fd, pHdr->data, pHdr->size, pHandle->offset); - if (bytes != pHdr->size) { - code = TAOS_SYSTEM_ERROR(terrno); - stError("%s failed to write snap, file name:%s, reason:%s", STREAM_STATE_TRANSFER, pHdr->name, tstrerror(code)); - return code; + char* path = taosMemoryCalloc(1, strlen(pHandle->metaPath) + 256); + sprintf(path, "%s%s%s%s%s%s%s%" PRId64 "", pHandle->metaPath, TD_DIRSEP, idstr, TD_DIRSEP, "checkpoints", TD_DIRSEP, + "checkpoint", snapInfo.chkpId); + if (!taosIsDir(path)) { + code = taosMulMkDir(path); + qInfo("%s mkdir %s", STREAM_STATE_TRANSFER, path); + ASSERT(code == 0); } - pHandle->offset += bytes; + + pDbSnapFile->path = path; + pDbSnapFile->snapInfo = snapInfo; + pDbSnapFile->pFileList = taosArrayInit(64, sizeof(SBackendFileItem)); + pDbSnapFile->currFileIdx = 0; + pDbSnapFile->offset = 0; + + SBackendFileItem item = {0}; + item.name = taosStrdup((char*)ROCKSDB_CURRENT); + item.type = ROCKSDB_CURRENT_TYPE; + + taosArrayPush(pDbSnapFile->pFileList, &item); + + pDbSnapFile->inited = 1; + return streamSnapWriteImpl(pWriter, pData, nData, pDbSnapFile); } else { - taosCloseFile(&pHandle->fd); - pHandle->offset = 0; - pHandle->currFileIdx += 1; + if (snapInfoEqual(&snapInfo, &pDbSnapFile->snapInfo)) { + return streamSnapWriteImpl(pWriter, pData, nData, pDbSnapFile); + } else { + SBackendSnapFile2 snapFile = {0}; + taosArrayPush(pHandle->pDbSnapSet, &snapFile); + pHandle->currIdx += 1; - SBackendFileItem item; - item.name = taosStrdup(pHdr->name); - item.type = pHdr->type; - taosArrayPush(pHandle->pFileList, &item); - - SBackendFileItem* pItem = taosArrayGet(pHandle->pFileList, pHandle->currFileIdx); - pHandle->fd = streamOpenFile(pFile->path, pItem->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); - if (pHandle->fd == NULL) { - code = TAOS_SYSTEM_ERROR(terrno); - stError("%s failed to open file name:%s%s%s, reason:%s", STREAM_STATE_TRANSFER, pFile->path, TD_DIRSEP, - pHdr->name, tstrerror(code)); + return streamSnapWrite(pWriter, pData, nData); } - - taosPWriteFile(pHandle->fd, pHdr->data, pHdr->size, pHandle->offset); - pHandle->offset += pHdr->size; } - - // impl later - return 0; + return code; } int32_t streamSnapWriterClose(SStreamSnapWriter* pWriter, int8_t rollback) { - SStreamSnapHandle* handle = &pWriter->handle; - if (qDebugFlag & DEBUG_TRACE) { - char* buf = (char*)taosMemoryMalloc(128 + taosArrayGetSize(handle->pFileList) * 64); - int n = sprintf(buf, "["); - for (int i = 0; i < taosArrayGetSize(handle->pFileList); i++) { - SBackendFileItem* item = taosArrayGet(handle->pFileList, i); - if (i != taosArrayGetSize(handle->pFileList) - 1) { - n += sprintf(buf + n, "%s %" PRId64 ",", item->name, item->size); - } else { - n += sprintf(buf + n, "%s %" PRId64 "]", item->name, item->size); - } - } - stDebug("%s snap get file list, %s", STREAM_STATE_TRANSFER, buf); - taosMemoryFree(buf); - } - - for (int i = 0; i < taosArrayGetSize(handle->pFileList); i++) { - SBackendFileItem* item = taosArrayGet(handle->pFileList, i); - taosMemoryFree(item->name); - } - - streamSnapHandleDestroy(handle); + if (pWriter == NULL) return 0; + streamSnapHandleDestroy(&pWriter->handle); taosMemoryFree(pWriter); return 0; diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 71f1e9b45b..309f377621 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -67,7 +67,8 @@ int32_t streamTaskSetReady(SStreamTask* pTask) { stDebug("s-task:%s all %d downstream ready, init completed, elapsed time:%" PRId64 "ms, task status:%s", pTask->id.idStr, numOfDowns, el, p); - streamMetaUpdateTaskDownstreamStatus(pTask, pTask->execInfo.init, pTask->execInfo.start, true); + streamMetaUpdateTaskDownstreamStatus(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, pTask->execInfo.init, + pTask->execInfo.start, true); return TSDB_CODE_SUCCESS; } @@ -114,7 +115,7 @@ static void doReExecScanhistory(void* param, void* tmrId) { // release the task. streamMetaReleaseTask(pTask->pMeta, pTask); } else { - taosTmrReset(doReExecScanhistory, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamEnv.timer, + taosTmrReset(doReExecScanhistory, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, &pTask->schedHistoryInfo.pTimer); } } @@ -137,9 +138,9 @@ int32_t streamReExecScanHistoryFuture(SStreamTask* pTask, int32_t idleDuration) stDebug("s-task:%s scan-history resumed in %.2fs, ref:%d", pTask->id.idStr, numOfTicks*0.1, ref); if (pTask->schedHistoryInfo.pTimer == NULL) { - pTask->schedHistoryInfo.pTimer = taosTmrStart(doReExecScanhistory, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamEnv.timer); + pTask->schedHistoryInfo.pTimer = taosTmrStart(doReExecScanhistory, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer); } else { - taosTmrReset(doReExecScanhistory, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamEnv.timer, &pTask->schedHistoryInfo.pTimer); + taosTmrReset(doReExecScanhistory, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, &pTask->schedHistoryInfo.pTimer); } return TSDB_CODE_SUCCESS; @@ -469,15 +470,25 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs addIntoNodeUpdateList(pTask, pRsp->downstreamNodeId); } - streamMetaUpdateTaskDownstreamStatus(pTask, pTask->execInfo.init, taosGetTimestampMs(), false); + streamMetaUpdateTaskDownstreamStatus(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, pTask->execInfo.init, + taosGetTimestampMs(), false); + // automatically set the related fill-history task to be failed. + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + STaskId* pId = &pTask->hTaskInfo.id; + + SStreamTask* pHTask = streamMetaAcquireTask(pTask->pMeta, pId->streamId, pId->taskId); + streamMetaUpdateTaskDownstreamStatus(pHTask->pMeta, pId->streamId, pId->taskId, pHTask->execInfo.init, + taosGetTimestampMs(), false); + streamMetaReleaseTask(pTask->pMeta, pHTask); + } } else { // TASK_DOWNSTREAM_NOT_READY, let's retry in 100ms STaskRecheckInfo* pInfo = createRecheckInfo(pTask, pRsp); int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s downstream taskId:0x%x (vgId:%d) not ready, stage:%"PRId64", retry in 100ms, ref:%d ", id, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->oldStage, ref); - pInfo->checkTimer = taosTmrStart(recheckDownstreamTasks, CHECK_DOWNSTREAM_INTERVAL, pInfo, streamEnv.timer); + pInfo->checkTimer = taosTmrStart(recheckDownstreamTasks, CHECK_DOWNSTREAM_INTERVAL, pInfo, streamTimer); } } @@ -718,7 +729,7 @@ static void tryLaunchHistoryTask(void* param, void* tmrId) { pHTaskInfo->tickCount -= 1; if (pHTaskInfo->tickCount > 0) { - taosTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamEnv.timer, &pHTaskInfo->pTimer); + taosTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamTimer, &pHTaskInfo->pTimer); streamMetaReleaseTask(pMeta, pTask); return; } @@ -746,7 +757,7 @@ static void tryLaunchHistoryTask(void* param, void* tmrId) { stDebug("s-task:%s status:%s failed to launch fill-history task:0x%x, retry launch:%dms, retryCount:%d", pTask->id.idStr, p, hTaskId, pHTaskInfo->waitInterval, pHTaskInfo->retryTimes); - taosTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamEnv.timer, &pHTaskInfo->pTimer); + taosTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamTimer, &pHTaskInfo->pTimer); streamMetaReleaseTask(pMeta, pTask); return; } @@ -807,7 +818,7 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) { streamTaskInitForLaunchHTask(&pTask->hTaskInfo); if (pTask->hTaskInfo.pTimer == NULL) { int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); - pTask->hTaskInfo.pTimer = taosTmrStart(tryLaunchHistoryTask, WAIT_FOR_MINIMAL_INTERVAL, pInfo, streamEnv.timer); + pTask->hTaskInfo.pTimer = taosTmrStart(tryLaunchHistoryTask, WAIT_FOR_MINIMAL_INTERVAL, pInfo, streamTimer); if (pTask->hTaskInfo.pTimer == NULL) { atomic_sub_fetch_32(&pTask->status.timerActive, 1); stError("s-task:%s failed to start timer, related fill-history task not launched, ref:%d", pTask->id.idStr, @@ -820,7 +831,7 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) { } else { // timer exists ASSERT(pTask->status.timerActive >= 1); stDebug("s-task:%s set timer active flag, task timer not null", pTask->id.idStr); - taosTmrReset(tryLaunchHistoryTask, WAIT_FOR_MINIMAL_INTERVAL, pInfo, streamEnv.timer, &pTask->hTaskInfo.pTimer); + taosTmrReset(tryLaunchHistoryTask, WAIT_FOR_MINIMAL_INTERVAL, pInfo, streamTimer, &pTask->hTaskInfo.pTimer); } return TSDB_CODE_SUCCESS; @@ -1058,35 +1069,35 @@ static void displayStatusInfo(SStreamMeta* pMeta, SHashObj* pTaskSet, bool succ) } } -int32_t streamMetaUpdateTaskDownstreamStatus(SStreamTask* pTask, int64_t startTs, int64_t endTs, bool ready) { - SStreamMeta* pMeta = pTask->pMeta; +int32_t streamMetaUpdateTaskDownstreamStatus(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, int64_t startTs, + int64_t endTs, bool ready) { + STaskStartInfo* pStartInfo = &pMeta->startInfo; + STaskId id = {.streamId = streamId, .taskId = taskId}; streamMetaWLock(pMeta); - - STaskId id = streamTaskExtractKey(pTask); - STaskStartInfo* pStartInfo = &pMeta->startInfo; - - SHashObj* pDst = ready? pStartInfo->pReadyTaskSet:pStartInfo->pFailedTaskSet; + SHashObj* pDst = ready ? pStartInfo->pReadyTaskSet : pStartInfo->pFailedTaskSet; STaskInitTs initTs = {.start = startTs, .end = endTs, .success = ready}; taosHashPut(pDst, &id, sizeof(id), &initTs, sizeof(STaskInitTs)); int32_t numOfTotal = streamMetaGetNumOfTasks(pMeta); + int32_t numOfRecv = taosHashGetSize(pStartInfo->pReadyTaskSet) + taosHashGetSize(pStartInfo->pFailedTaskSet); - if (taosHashGetSize(pStartInfo->pReadyTaskSet) + taosHashGetSize(pStartInfo->pFailedTaskSet) == numOfTotal) { + if (numOfRecv == numOfTotal) { pStartInfo->readyTs = taosGetTimestampMs(); pStartInfo->elapsedTime = (pStartInfo->startTs != 0) ? pStartInfo->readyTs - pStartInfo->startTs : 0; - stDebug("vgId:%d all %d task(s) check downstream completed, last completed task:%s level:%d, startTs:%" PRId64 + stDebug("vgId:%d all %d task(s) check downstream completed, last completed task:0x%x startTs:%" PRId64 ", readyTs:%" PRId64 " total elapsed time:%.2fs", - pMeta->vgId, numOfTotal, pTask->id.idStr, pTask->info.taskLevel, pStartInfo->startTs, pStartInfo->readyTs, + pMeta->vgId, numOfTotal, taskId, pStartInfo->startTs, pStartInfo->readyTs, pStartInfo->elapsedTime / 1000.0); // print the initialization elapsed time and info displayStatusInfo(pMeta, pStartInfo->pReadyTaskSet, true); displayStatusInfo(pMeta, pStartInfo->pFailedTaskSet, false); - streamMetaResetStartInfo(pStartInfo); + } else { + stDebug("vgId:%d recv check down results:%d, total:%d", pMeta->vgId, numOfRecv, numOfTotal); } streamMetaWUnLock(pMeta); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 0f32fd6879..276ed08785 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -106,51 +106,21 @@ SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t sz } SStreamTask* pStreamTask = pTask; - char statePath[1024]; - if (!specPath) { - sprintf(statePath, "%s%s%d", path, TD_DIRSEP, pStreamTask->id.taskId); - } else { - memset(statePath, 0, 1024); - tstrncpy(statePath, path, 1024); - } - pState->taskId = pStreamTask->id.taskId; pState->streamId = pStreamTask->id.streamId; - sprintf(pState->pTdbState->idstr, "0x%" PRIx64 "-%d", pState->streamId, pState->taskId); + sprintf(pState->pTdbState->idstr, "0x%" PRIx64 "-0x%x", pState->streamId, pState->taskId); + + streamTaskSetDb(pStreamTask->pMeta, pTask, pState->pTdbState->idstr); #ifdef USE_ROCKSDB SStreamMeta* pMeta = pStreamTask->pMeta; - pState->streamBackendRid = pMeta->streamBackendRid; - // streamMetaWLock(pMeta); - taosThreadMutexLock(&pMeta->backendMutex); - void* uniqueId = - taosHashGet(pMeta->pTaskBackendUnique, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); - if (uniqueId == NULL) { - int code = streamStateOpenBackend(pMeta->streamBackend, pState); - if (code == -1) { - taosThreadMutexUnlock(&pMeta->backendMutex); - taosMemoryFree(pState); - return NULL; - } - taosHashPut(pMeta->pTaskBackendUnique, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1, - &pState->pTdbState->backendCfWrapperId, sizeof(pState->pTdbState->backendCfWrapperId)); - } else { - int64_t id = *(int64_t*)uniqueId; - pState->pTdbState->backendCfWrapperId = id; - pState->pTdbState->pBackendCfWrapper = taosAcquireRef(streamBackendCfWrapperId, id); - // already exist stream task for - stInfo("already exist stream-state for %s", pState->pTdbState->idstr); - // taosAcquireRef(streamBackendId, pState->streamBackendRid); - } - taosThreadMutexUnlock(&pMeta->backendMutex); - pState->pTdbState->pOwner = pTask; pState->pFileState = NULL; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT); pState->parNameMap = tSimpleHashInit(1024, hashFn); stInfo("succ to open state %p on backend %p 0x%" PRIx64 "-%d", pState, pMeta->streamBackend, pState->streamId, - pState->taskId); + pState->taskId); return pState; #else @@ -237,6 +207,12 @@ _err: #endif } +int32_t streamStateDelTaskDb(SStreamState* pState) { + SStreamTask* pTask = pState->pTdbState->pOwner; + taskDbRemoveRef(pTask->pBackend); + taosMemoryFree(pTask); + return 0; +} void streamStateClose(SStreamState* pState, bool remove) { SStreamTask* pTask = pState->pTdbState->pOwner; #ifdef USE_ROCKSDB @@ -692,8 +668,7 @@ void streamStateResetCur(SStreamStateCur* pCur) { } void streamStateFreeCur(SStreamStateCur* pCur) { - if (!pCur || pCur->buffIndex >= 0) { - taosMemoryFree(pCur); + if (!pCur) { return; } qDebug("streamStateFreeCur"); @@ -722,7 +697,7 @@ int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void streamStateReleaseBuf(pState, pos, true); putFreeBuff(pState->pFileState, pos); stDebug("===stream===save skey:%" PRId64 ", ekey:%" PRId64 ", groupId:%" PRIu64 ".code:%d", key->win.skey, - key->win.ekey, key->groupId, code); + key->win.ekey, key->groupId, code); } else { code = putSessionWinResultBuff(pState->pFileState, value); } @@ -768,7 +743,7 @@ int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVa int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key) { #ifdef USE_ROCKSDB - stDebug("===stream===delete skey:%" PRId64 ", ekey:%" PRId64 ", groupId:%" PRIu64, key->win.skey, key->win.ekey, + qDebug("===stream===delete skey:%" PRId64 ", ekey:%" PRId64 ", groupId:%" PRIu64, key->win.skey, key->win.ekey, key->groupId); return deleteRowBuff(pState->pFileState, key, sizeof(SSessionKey)); #else @@ -1122,7 +1097,7 @@ int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal void streamStateDestroy(SStreamState* pState, bool remove) { #ifdef USE_ROCKSDB streamFileStateDestroy(pState->pFileState); - streamStateDestroy_rocksdb(pState, remove); + // streamStateDestroy_rocksdb(pState, remove); tSimpleHashCleanup(pState->parNameMap); // do nothong #endif @@ -1232,4 +1207,4 @@ char* streamStateIntervalDump(SStreamState* pState) { streamStateFreeCur(pCur); return dumpBuf; } -#endif +#endif \ No newline at end of file diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 2f0bfc8dbc..db0217f000 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -250,9 +250,8 @@ int32_t tDecodeStreamTaskChkInfo(SDecoder* pDecoder, SCheckpointInfo* pChkpInfo) SEpSet epSet; if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeI64(pDecoder, &ver) < 0) return -1; - - if (ver != SSTREAM_TASK_VER) return -1; + if (tDecodeI64(pDecoder, &pChkpInfo->msgVer) < 0) return -1; + // if (ver != SSTREAM_TASK_VER) return -1; if (tDecodeI64(pDecoder, &skip64) < 0) return -1; if (tDecodeI32(pDecoder, &skip32) < 0) return -1; @@ -379,6 +378,8 @@ void tFreeStreamTask(SStreamTask* pTask) { if (pTask->pState) { stDebug("s-task:0x%x start to free task state", taskId); streamStateClose(pTask->pState, status == TASK_STATUS__DROPPING); + taskDbRemoveRef(pTask->pBackend); + } if (pTask->id.idStr != NULL) { @@ -467,6 +468,14 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i } taosThreadMutexInit(&pTask->lock, &attr); + // if (pTask->info.fillHistory == 1) { + // // + // } else { + + // } + // if (streamTaskSetDb(pMeta, pTask) != 0) { + // return -1; + // } streamTaskOpenAllUpstreamInput(pTask); pTask->outputInfo.pDownstreamUpdateList = taosArrayInit(4, sizeof(SDownstreamTaskEpset)); diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 858667b563..f78f6f4df1 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -129,9 +129,9 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma } pInfo->numBuckets = DEFAULT_BUCKET_SIZE; pInfo->pCloseWinSBF = NULL; - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT); - pInfo->pMap = taosHashInit(DEFAULT_MAP_CAPACITY, hashFn, true, HASH_NO_LOCK); } + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT); + pInfo->pMap = taosHashInit(DEFAULT_MAP_CAPACITY, hashFn, true, HASH_NO_LOCK); pInfo->maxDataVersion = 0; return pInfo; } @@ -384,3 +384,14 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { tDecoderClear(&decoder); return 0; } + +bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) { + TSKEY *pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); + bool res = true; + if ( pMapMaxTs && ts < *pMapMaxTs ) { + res = false; + } else { + taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), &ts, sizeof(TSKEY)); + } + return res; +} diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h index 04456b2454..ecd2b5163e 100644 --- a/source/libs/sync/inc/syncReplication.h +++ b/source/libs/sync/inc/syncReplication.h @@ -56,6 +56,10 @@ int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode); int32_t syncNodeSendAppendEntries(SSyncNode* pNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg); +int32_t syncSnapSendMsg(SSyncSnapshotSender* pSender, int32_t seq, void* pBlock, int32_t len, int32_t typ); +int32_t syncSnapSendRsp(SSyncSnapshotReceiver* pReceiver, SyncSnapshotSend* pMsg, void* pBlock, int32_t len, + int32_t typ, int32_t code); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 95952c960e..93e81fd8e2 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -23,8 +23,6 @@ #include "syncReplication.h" #include "syncUtil.h" -int32_t syncSnapSendMsg(SSyncSnapshotSender *pSender, int32_t seq, void *pBlock, int32_t len, int32_t typ); - static void syncSnapBufferReset(SSyncSnapBuffer *pBuf) { taosThreadMutexLock(&pBuf->mutex); for (int64_t i = pBuf->start; i < pBuf->end; ++i) { @@ -123,6 +121,11 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { if (pSender->pSndBuf) { syncSnapBufferDestroy(&pSender->pSndBuf); } + + if (pSender->snapshotParam.data) { + taosMemoryFree(pSender->snapshotParam.data); + pSender->snapshotParam.data = NULL; + } // free sender taosMemoryFree(pSender); } @@ -153,7 +156,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { pSender->lastSendTime = taosGetTimestampMs(); pSender->finish = false; - // Get full snapshot info + // Get snapshot info SSyncNode *pSyncNode = pSender->pSyncNode; SSnapshot snapInfo = {.type = TDMT_SYNC_PREP_SNAPSHOT}; if (pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapInfo) != 0) { @@ -161,11 +164,10 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { goto _out; } - int dataLen = 0; void *pData = snapInfo.data; - int32_t type = 0; + int32_t type = (pData) ? snapInfo.type : 0; + int32_t dataLen = 0; if (pData) { - type = snapInfo.type; SSyncTLV *datHead = pData; if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT) { sSError(pSender, "unexpected data typ in data of snapshot info. typ: %d", datHead->typ); @@ -347,9 +349,6 @@ _out:; return code; } -// return 0, start ok -// return 1, last snapshot finish ok -// return -1, error int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) { SSyncSnapshotSender *pSender = syncNodeGetSnapshotSender(pSyncNode, pDestId); if (pSender == NULL) { @@ -380,6 +379,7 @@ int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) { return 0; } +// receiver SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId) { bool condition = (pSyncNode->pFsm->FpSnapshotStartWrite != NULL) && (pSyncNode->pFsm->FpSnapshotStopWrite != NULL) && (pSyncNode->pFsm->FpSnapshotDoWrite != NULL); @@ -509,8 +509,6 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *p sRInfo(pReceiver, "snapshot receiver start, from dnode:%d.", DID(&pReceiver->fromId)); } -// just set start = false -// FpSnapshotStopWrite should not be called void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { sRDebug(pReceiver, "snapshot receiver stop, not apply, writer:%p", pReceiver->pWriter); @@ -531,7 +529,6 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { syncSnapBufferReset(pReceiver->pRcvBuf); } -// when recv last snapshot block, apply data into snapshot static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) { int32_t code = 0; if (pReceiver->pWriter != NULL) { @@ -590,8 +587,6 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap return 0; } -// apply data block -// update progress static int32_t snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) { if (pMsg->seq != pReceiver->ack + 1) { sRError(pReceiver, "snapshot receiver invalid seq, ack:%d seq:%d", pReceiver->ack, pMsg->seq); @@ -644,6 +639,50 @@ SyncIndex syncNodeGetSnapBeginIndex(SSyncNode *ths) { return snapStart; } +static int32_t syncSnapReceiverExchgSnapInfo(SSyncNode *pSyncNode, SSyncSnapshotReceiver *pReceiver, + SyncSnapshotSend *pMsg, SSnapshot *pInfo) { + ASSERT(pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT); + int32_t code = 0; + + // copy snap info from leader + void *data = taosMemoryCalloc(1, pMsg->dataLen); + if (data == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; + goto _out; + } + pInfo->data = data; + data = NULL; + memcpy(pInfo->data, pMsg->data, pMsg->dataLen); + + // exchange snap info + pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, pInfo); + SSyncTLV *datHead = pInfo->data; + if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { + sRError(pReceiver, "unexpected data typ in data of snapshot info. typ: %d", datHead->typ); + code = TSDB_CODE_INVALID_DATA_FMT; + goto _out; + } + int32_t dataLen = sizeof(SSyncTLV) + datHead->len; + + // save exchanged snap info + SSnapshotParam *pParam = &pReceiver->snapshotParam; + data = taosMemoryRealloc(pParam->data, dataLen); + if (data == NULL) { + sError("vgId:%d, failed to realloc memory for snapshot prep due to %s. dataLen:%d", pSyncNode->vgId, + strerror(errno), dataLen); + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; + goto _out; + } + pParam->data = data; + data = NULL; + memcpy(pParam->data, pInfo->data, dataLen); + +_out: + return code; +} + static int32_t syncNodeOnSnapshotPrep(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; int64_t timeNow = taosGetTimestampMs(); @@ -686,77 +725,27 @@ _START_RECEIVER: snapshotReceiverStop(pReceiver); } - snapshotReceiverStart(pReceiver, pMsg); // set start-time same with sender + snapshotReceiverStart(pReceiver, pMsg); -_SEND_REPLY: - // build msg - ; // make complier happy +_SEND_REPLY:; SSnapshot snapInfo = {.type = TDMT_SYNC_PREP_SNAPSHOT_REPLY}; int32_t dataLen = 0; - if (pMsg->dataLen > 0) { - void *data = taosMemoryCalloc(1, pMsg->dataLen); - if (data == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = terrno; + if (pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT) { + if (syncSnapReceiverExchgSnapInfo(pSyncNode, pReceiver, pMsg, &snapInfo) != 0) { goto _out; } - memcpy(data, pMsg->data, pMsg->dataLen); - snapInfo.data = data; - data = NULL; - pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapInfo); - SSyncTLV *datHead = snapInfo.data; - if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { - sRError(pReceiver, "unexpected data typ in data of snapshot info. typ: %d", datHead->typ); - code = TSDB_CODE_INVALID_DATA_FMT; - goto _out; - } dataLen = sizeof(SSyncTLV) + datHead->len; } - SRpcMsg rpcMsg = {0}; - if (syncBuildSnapshotSendRsp(&rpcMsg, dataLen, pSyncNode->vgId) != 0) { - sRError(pReceiver, "snapshot receiver failed to build resp since %s", terrstr()); + // send response + int32_t type = (snapInfo.data) ? snapInfo.type : 0; + if (syncSnapSendRsp(pReceiver, pMsg, snapInfo.data, dataLen, type, code) != 0) { code = terrno; goto _out; } - SyncSnapshotRsp *pRspMsg = rpcMsg.pCont; - pRspMsg->srcId = pSyncNode->myRaftId; - pRspMsg->destId = pMsg->srcId; - pRspMsg->term = raftStoreGetTerm(pSyncNode); - pRspMsg->lastIndex = pMsg->lastIndex; - pRspMsg->lastTerm = pMsg->lastTerm; - pRspMsg->startTime = pMsg->startTime; - pRspMsg->ack = pMsg->seq; // receiver maybe already closed - pRspMsg->code = code; - pRspMsg->snapBeginIndex = syncNodeGetSnapBeginIndex(pSyncNode); - - if (snapInfo.data) { - pRspMsg->payloadType = snapInfo.type; - memcpy(pRspMsg->data, snapInfo.data, dataLen); - - // save snapshot info - SSnapshotParam *pParam = &pReceiver->snapshotParam; - void *data = taosMemoryRealloc(pParam->data, dataLen); - if (data == NULL) { - sError("vgId:%d, failed to realloc memory for snapshot prep due to %s. dataLen:%d", pSyncNode->vgId, - strerror(errno), dataLen); - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = terrno; - goto _out; - } - pParam->data = data; - memcpy(pParam->data, snapInfo.data, dataLen); - } - - // send msg - if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) { - sRError(pReceiver, "failed to send resp since %s", terrstr()); - code = terrno; - } - _out: if (snapInfo.data) { taosMemoryFree(snapInfo.data); @@ -793,38 +782,20 @@ _SEND_REPLY: code = terrno; } - // build msg - SRpcMsg rpcMsg = {0}; - if (syncBuildSnapshotSendRsp(&rpcMsg, 0, pSyncNode->vgId) != 0) { - sRError(pReceiver, "failed to build snapshot receiver resp since %s", terrstr()); - return -1; - } - - SyncSnapshotRsp *pRspMsg = rpcMsg.pCont; - pRspMsg->srcId = pSyncNode->myRaftId; - pRspMsg->destId = pMsg->srcId; - pRspMsg->term = raftStoreGetTerm(pSyncNode); - pRspMsg->lastIndex = pMsg->lastIndex; - pRspMsg->lastTerm = pMsg->lastTerm; - pRspMsg->startTime = pMsg->startTime; - pRspMsg->ack = pReceiver->ack; // receiver maybe already closed - pRspMsg->code = code; - pRspMsg->snapBeginIndex = pReceiver->snapshotParam.start; - - // send msg - if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) { - sRError(pReceiver, "failed to send snapshot receiver resp since %s", terrstr()); + // send response + if (syncSnapSendRsp(pReceiver, pMsg, NULL, 0, 0, code) != 0) { return -1; } return code; } -static int32_t syncSnapSendRsp(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg, int32_t code) { +int32_t syncSnapSendRsp(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg, void *pBlock, int32_t blockLen, + int32_t type, int32_t code) { SSyncNode *pSyncNode = pReceiver->pSyncNode; // build msg SRpcMsg rpcMsg = {0}; - if (syncBuildSnapshotSendRsp(&rpcMsg, 0, pSyncNode->vgId)) { + if (syncBuildSnapshotSendRsp(&rpcMsg, blockLen, pSyncNode->vgId)) { sRError(pReceiver, "failed to build snapshot receiver resp since %s", terrstr()); return -1; } @@ -832,13 +803,18 @@ static int32_t syncSnapSendRsp(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSen SyncSnapshotRsp *pRspMsg = rpcMsg.pCont; pRspMsg->srcId = pSyncNode->myRaftId; pRspMsg->destId = pMsg->srcId; - pRspMsg->term = raftStoreGetTerm(pSyncNode); + pRspMsg->term = pMsg->term; pRspMsg->lastIndex = pMsg->lastIndex; pRspMsg->lastTerm = pMsg->lastTerm; pRspMsg->startTime = pMsg->startTime; pRspMsg->ack = pMsg->seq; pRspMsg->code = code; pRspMsg->snapBeginIndex = pReceiver->snapshotParam.start; + pRspMsg->payloadType = type; + + if (pBlock != NULL && blockLen > 0) { + memcpy(pRspMsg->data, pBlock, blockLen); + } // send msg if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) { @@ -872,7 +848,7 @@ static int32_t syncSnapBufferRecv(SSyncSnapshotReceiver *pReceiver, SyncSnapshot ppMsg[0] = NULL; pRcvBuf->end = TMAX(pMsg->seq + 1, pRcvBuf->end); } else if (pMsg->seq < pRcvBuf->start) { - syncSnapSendRsp(pReceiver, pMsg, code); + syncSnapSendRsp(pReceiver, pMsg, NULL, 0, 0, code); goto _out; } @@ -892,7 +868,7 @@ static int32_t syncSnapBufferRecv(SSyncSnapshotReceiver *pReceiver, SyncSnapshot } } pRcvBuf->start = seq + 1; - syncSnapSendRsp(pReceiver, pRcvBuf->entries[seq % pRcvBuf->size], code); + syncSnapSendRsp(pReceiver, pRcvBuf->entries[seq % pRcvBuf->size], NULL, 0, 0, code); pRcvBuf->entryDeleteCb(pRcvBuf->entries[seq % pRcvBuf->size]); pRcvBuf->entries[seq % pRcvBuf->size] = NULL; if (code) goto _out; @@ -915,7 +891,7 @@ static int32_t syncNodeOnSnapshotReceive(SSyncNode *pSyncNode, SyncSnapshotSend if (snapshotReceiverSignatureCmp(pReceiver, pMsg) != 0) { terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; sRError(pReceiver, "failed to receive snapshot data since %s.", terrstr()); - return syncSnapSendRsp(pReceiver, pMsg, terrno); + return syncSnapSendRsp(pReceiver, pMsg, NULL, 0, 0, terrno); } return syncSnapBufferRecv(pReceiver, ppMsg); @@ -971,26 +947,6 @@ _SEND_REPLY:; return code; } -// receiver on message -// -// condition 1, recv SYNC_SNAPSHOT_SEQ_PREP -// if receiver already start -// if sender.start-time > receiver.start-time, restart receiver(reply snapshot start) -// if sender.start-time = receiver.start-time, maybe duplicate msg -// if sender.start-time < receiver.start-time, ignore -// else -// waiting for clock match -// start receiver(reply snapshot start) -// -// condition 2, recv SYNC_SNAPSHOT_SEQ_BEGIN -// a. create writer with -// -// condition 3, recv SYNC_SNAPSHOT_SEQ_END, finish receiver(apply snapshot data, update commit index, maybe reconfig) -// -// condition 4, recv SYNC_SNAPSHOT_SEQ_FORCE_CLOSE, force close -// -// condition 5, got data, update ack -// int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { SyncSnapshotSend **ppMsg = (SyncSnapshotSend **)&pRpcMsg->pCont; SyncSnapshotSend *pMsg = ppMsg[0]; @@ -1074,6 +1030,32 @@ _out:; return code; } +static int32_t syncSnapSenderExchgSnapInfo(SSyncNode *pSyncNode, SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) { + ASSERT(pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT_REPLY); + + SSyncTLV *datHead = (void *)pMsg->data; + if (datHead->typ != pMsg->payloadType) { + sSError(pSender, "unexpected data type in data of SyncSnapshotRsp. typ: %d", datHead->typ); + terrno = TSDB_CODE_INVALID_DATA_FMT; + return -1; + } + int32_t dataLen = sizeof(SSyncTLV) + datHead->len; + + SSnapshotParam *pParam = &pSender->snapshotParam; + void *data = taosMemoryRealloc(pParam->data, dataLen); + if (data == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + memcpy(data, pMsg->data, dataLen); + + pParam->data = data; + data = NULL; + sSInfo(pSender, "data of snapshot param. len: %d", datHead->len); + return 0; +} + +// sender static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) { SSnapshot snapshot = {0}; pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); @@ -1090,14 +1072,9 @@ static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSend // start reader if (pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { - SSyncTLV *datHead = (void *)pMsg->data; - if (datHead->typ != pMsg->payloadType) { - sSError(pSender, "unexpected data type in data of SyncSnapshotRsp. typ: %d", datHead->typ); - terrno = TSDB_CODE_INVALID_DATA_FMT; + if (syncSnapSenderExchgSnapInfo(pSyncNode, pSender, pMsg) != 0) { return -1; } - pSender->snapshotParam.data = (void *)pMsg->data; - sSInfo(pSender, "data of snapshot param. len: %d", datHead->len); } int32_t code = pSyncNode->pFsm->FpSnapshotStartRead(pSyncNode->pFsm, &pSender->snapshotParam, &pSender->pReader); @@ -1131,7 +1108,7 @@ static int32_t syncSnapBufferSend(SSyncSnapshotSender *pSender, SyncSnapshotRsp goto _out; } - if (pSender->pReader == NULL || pSender->finish) { + if (pSender->pReader == NULL || pSender->finish || !snapshotSenderIsStart(pSender)) { code = terrno = TSDB_CODE_SYN_INTERNAL_ERROR; goto _out; } @@ -1182,12 +1159,6 @@ _out: return code; } -// sender on message -// -// condition 1 sender receives SYNC_SNAPSHOT_SEQ_END, close sender -// condition 2 sender receives ack, set seq = ack + 1, send msg from seq -// condition 3 sender receives error msg, just print error log -// int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { SyncSnapshotRsp **ppMsg = (SyncSnapshotRsp **)&pRpcMsg->pCont; SyncSnapshotRsp *pMsg = ppMsg[0]; diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 06847c081c..2ce56af946 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -487,3 +487,13 @@ void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteRepl sNInfo(pSyncNode, "send sync-request-vote-reply to dnode:%d {term:%" PRId64 ", grant:%d}, %s", DID(&pMsg->destId), pMsg->term, pMsg->voteGranted, s); } + +int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size) { + void* data = taosMemoryRealloc(pSnap->data, size); + if (data == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pSnap->data = data; + return 0; +} diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 73427446e6..19b2dfc300 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -23,6 +23,7 @@ extern "C" { #include "taoserror.h" #include "theap.h" #include "tmisce.h" +#include "tmsg.h" #include "transLog.h" #include "transportInt.h" #include "trpc.h" diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index ef60c8a94e..936d11c151 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -957,8 +957,8 @@ static void cliSendCb(uv_write_t* req, int status) { SCliMsg* pMsg = !transQueueEmpty(&pConn->cliMsgs) ? transQueueGet(&pConn->cliMsgs, 0) : NULL; if (pMsg != NULL) { int64_t cost = taosGetTimestampUs() - pMsg->st; - if (cost > 1000 * 20) { - tWarn("%s conn %p send cost:%dus, send exception", CONN_GET_INST_LABEL(pConn), pConn, (int)cost); + if (cost > 1000 * 50) { + tTrace("%s conn %p send cost:%dus ", CONN_GET_INST_LABEL(pConn), pConn, (int)cost); } } @@ -1468,7 +1468,7 @@ static FORCE_INLINE uint32_t cliGetIpFromFqdnCache(SHashObj* cache, char* fqdn) if (v == NULL) { addr = taosGetIpv4FromFqdn(fqdn); if (addr == 0xffffffff) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TSDB_CODE_RPC_FQDN_ERROR; tError("failed to get ip from fqdn:%s since %s", fqdn, terrstr()); return addr; } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index bf73c253bc..017969b4e5 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -159,7 +159,7 @@ static void uvStartSendResp(SSvrMsg* msg); static void uvNotifyLinkBrokenToApp(SSvrConn* conn); -static FORCE_INLINE void destroySmsg(SSvrMsg* smsg); +static FORCE_INLINE void destroySmsg(SSvrMsg* smsg); static FORCE_INLINE SSvrConn* createConn(void* hThrd); static FORCE_INLINE void destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/); static FORCE_INLINE void destroyConnRegArg(SSvrConn* conn); @@ -1499,6 +1499,7 @@ int transSendResponse(const STransMsg* msg) { } SExHandle* exh = msg->info.handle; if (exh == NULL) { + rpcFreeCont(msg->pCont); return 0; } int64_t refId = msg->info.refId; diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 33d8d34514..341d989f8f 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -327,15 +327,19 @@ int32_t walEndSnapshot(SWal *pWal) { // iterate files, until the searched result // delete according to file size or close time + SWalFileInfo *pUntil = NULL; for (SWalFileInfo *iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { if ((pWal->cfg.retentionSize > 0 && newTotSize > pWal->cfg.retentionSize) || (pWal->cfg.retentionPeriod == 0 || pWal->cfg.retentionPeriod > 0 && iter->closeTs >= 0 && iter->closeTs + pWal->cfg.retentionPeriod < ts)) { - deleteCnt++; newTotSize -= iter->fileSize; - taosArrayPush(pWal->toDeleteFiles, iter); + pUntil = iter; } } + for (SWalFileInfo *iter = pWal->fileInfoSet->pData; iter <= pUntil; iter++) { + deleteCnt++; + taosArrayPush(pWal->toDeleteFiles, iter); + } // make new array, remove files taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt); diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 15aca85fc2..39de117339 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -348,15 +348,15 @@ HANDLE taosOpenFileNotStream(const char *path, int32_t tdFileOptions) { if (h != INVALID_HANDLE_VALUE && (tdFileOptions & TD_FILE_APPEND) && (tdFileOptions & TD_FILE_WRITE)) { SetFilePointer(h, 0, NULL, FILE_END); } - if (h == INVALID_HANDLE_VALUE) { - DWORD dwError = GetLastError(); - LPVOID lpMsgBuf; - FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL, dwError, - 0, - (LPTSTR)&lpMsgBuf, 0, NULL); - printf("CreateFile failed with error %d: %s", dwError, (char*)lpMsgBuf); - LocalFree(lpMsgBuf); - } + // if (h == INVALID_HANDLE_VALUE) { + // DWORD dwError = GetLastError(); + // LPVOID lpMsgBuf; + // FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL, dwError, + // 0, + // (LPTSTR)&lpMsgBuf, 0, NULL); + // printf("CreateFile failed with error %d: %s", dwError, (char*)lpMsgBuf); + // LocalFree(lpMsgBuf); + // } return h; } diff --git a/source/os/test/osTests.cpp b/source/os/test/osTests.cpp index e2185aeac2..aad3eda9aa 100644 --- a/source/os/test/osTests.cpp +++ b/source/os/test/osTests.cpp @@ -32,6 +32,35 @@ #ifdef WINDOWS #include +#else + +#include + +TEST(osTest, osFQDNSuccess) { + char fqdn[1024]; + char ipString[INET_ADDRSTRLEN]; + int code = taosGetFqdn(fqdn); + uint32_t ipv4 = taosGetIpv4FromFqdn(fqdn); + ASSERT_NE(ipv4, 0xffffffff); + + struct in_addr addr; + addr.s_addr = htonl(ipv4); + snprintf(ipString, INET_ADDRSTRLEN, "%u.%u.%u.%u", (unsigned int)(addr.s_addr >> 24) & 0xFF, + (unsigned int)(addr.s_addr >> 16) & 0xFF, (unsigned int)(addr.s_addr >> 8) & 0xFF, + (unsigned int)(addr.s_addr) & 0xFF); + printf("fqdn:%s ip:%s\n", fqdn, ipString); +} + +TEST(osTest, osFQDNFailed) { + char fqdn[1024] = "fqdn_test_not_found"; + char ipString[24]; + uint32_t ipv4 = taosGetIpv4FromFqdn(fqdn); + ASSERT_EQ(ipv4, 0xffffffff); + + terrno = TSDB_CODE_RPC_FQDN_ERROR; + printf("fqdn:%s transfer to ip failed!\n", fqdn); +} + #endif // WINDOWS TEST(osTest, osSystem) { @@ -314,6 +343,6 @@ TEST(osTest, osFilePerformance) { printf("Test OpenForRead & Close file %d times, cost: %" PRId64 "us\n", TESTTIMES, OpenForReadCloseFileCost); } -#endif OSFILE_PERFORMANCE_TEST +#endif // OSFILE_PERFORMANCE_TEST #pragma GCC diagnostic pop diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 09f50d0f6d..3cc00ddc7f 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -538,66 +538,71 @@ int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelement memcpy(output, input + 1, nelements * longBytes); return nelements * longBytes; } else if (input[0] == 1) { // Decompress - int64_t *ostream = (int64_t *)output; + if (tsSIMDEnable && tsAVX512Enable) { + tsDecompressTimestampAvx512(input, nelements, output, false); + } else if (tsSIMDEnable && tsAVX2Enable) { + tsDecompressTimestampAvx2(input, nelements, output, false); + } else { + int64_t *ostream = (int64_t *)output; - int32_t ipos = 1, opos = 0; - int8_t nbytes = 0; - int64_t prev_value = 0; - int64_t prev_delta = 0; - int64_t delta_of_delta = 0; + int32_t ipos = 1, opos = 0; + int8_t nbytes = 0; + int64_t prev_value = 0; + int64_t prev_delta = 0; + int64_t delta_of_delta = 0; - while (1) { - uint8_t flags = input[ipos++]; - // Decode dd1 - uint64_t dd1 = 0; - nbytes = flags & INT8MASK(4); - if (nbytes == 0) { - delta_of_delta = 0; - } else { - if (is_bigendian()) { - memcpy(((char *)(&dd1)) + longBytes - nbytes, input + ipos, nbytes); + while (1) { + uint8_t flags = input[ipos++]; + // Decode dd1 + uint64_t dd1 = 0; + nbytes = flags & INT8MASK(4); + if (nbytes == 0) { + delta_of_delta = 0; } else { - memcpy(&dd1, input + ipos, nbytes); + if (is_bigendian()) { + memcpy(((char *)(&dd1)) + longBytes - nbytes, input + ipos, nbytes); + } else { + memcpy(&dd1, input + ipos, nbytes); + } + delta_of_delta = ZIGZAG_DECODE(int64_t, dd1); } - delta_of_delta = ZIGZAG_DECODE(int64_t, dd1); - } - ipos += nbytes; - if (opos == 0) { - prev_value = delta_of_delta; - prev_delta = 0; - ostream[opos++] = delta_of_delta; - } else { + + ipos += nbytes; + if (opos == 0) { + prev_value = delta_of_delta; + prev_delta = 0; + ostream[opos++] = delta_of_delta; + } else { + prev_delta = delta_of_delta + prev_delta; + prev_value = prev_value + prev_delta; + ostream[opos++] = prev_value; + } + if (opos == nelements) return nelements * longBytes; + + // Decode dd2 + uint64_t dd2 = 0; + nbytes = (flags >> 4) & INT8MASK(4); + if (nbytes == 0) { + delta_of_delta = 0; + } else { + if (is_bigendian()) { + memcpy(((char *)(&dd2)) + longBytes - nbytes, input + ipos, nbytes); + } else { + memcpy(&dd2, input + ipos, nbytes); + } + // zigzag_decoding + delta_of_delta = ZIGZAG_DECODE(int64_t, dd2); + } + ipos += nbytes; prev_delta = delta_of_delta + prev_delta; prev_value = prev_value + prev_delta; ostream[opos++] = prev_value; + if (opos == nelements) return nelements * longBytes; } - if (opos == nelements) return nelements * longBytes; - - // Decode dd2 - uint64_t dd2 = 0; - nbytes = (flags >> 4) & INT8MASK(4); - if (nbytes == 0) { - delta_of_delta = 0; - } else { - if (is_bigendian()) { - memcpy(((char *)(&dd2)) + longBytes - nbytes, input + ipos, nbytes); - } else { - memcpy(&dd2, input + ipos, nbytes); - } - // zigzag_decoding - delta_of_delta = ZIGZAG_DECODE(int64_t, dd2); - } - ipos += nbytes; - prev_delta = delta_of_delta + prev_delta; - prev_value = prev_value + prev_delta; - ostream[opos++] = prev_value; - if (opos == nelements) return nelements * longBytes; } - - } else { - ASSERT(0); - return -1; } + + return nelements * longBytes; } /* --------------------------------------------Double Compression ---------------------------------------------- */ diff --git a/source/util/src/tdecompress.c b/source/util/src/tdecompress.c index 68841941db..f212bf5231 100644 --- a/source/util/src/tdecompress.c +++ b/source/util/src/tdecompress.c @@ -53,11 +53,8 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, int64_t prevValue = 0; #if __AVX2__ - while (1) { - if (_pos == nelements) break; - - uint64_t w = 0; - memcpy(&w, ip, LONG_BYTES); + while (_pos < nelements) { + uint64_t w = *(uint64_t*) ip; char selector = (char)(w & INT64MASK(4)); // selector = 4 char bit = bit_per_integer[(int32_t)selector]; // bit = 3 @@ -114,7 +111,7 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, __m256i signmask = _mm256_and_si256(_mm256_set1_epi64x(1), zigzagVal); signmask = _mm256_sub_epi64(_mm256_setzero_si256(), signmask); - // get the four zigzag values here + // get four zigzag values here __m256i delta = _mm256_xor_si256(_mm256_srli_epi64(zigzagVal, 1), signmask); // calculate the cumulative sum (prefix sum) for each number @@ -250,73 +247,264 @@ int32_t tsDecompressFloatImplAvx2(const char *const input, const int32_t nelemen return 0; } -int32_t tsDecompressTimestampAvx2(const char* const input, const int32_t nelements, char *const output, bool bigEndian) { +int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, char *const output, + bool bigEndian) { +#if 0 int64_t *ostream = (int64_t *)output; int32_t ipos = 1, opos = 0; - int8_t nbytes = 0; - - int64_t prevValue = 0; - int64_t prevDelta = 0; - - int64_t deltaOfDelta = 0; - int32_t longBytes = LONG_BYTES; + __m128i prevVal = _mm_setzero_si128(); + __m128i prevDelta = _mm_setzero_si128(); #if __AVX2__ + int32_t batch = nelements >> 1; + int32_t remainder = nelements & 0x01; + __mmask16 mask2[16] = {0, 0x0001, 0x0003, 0x0007, 0x000f, 0x001f, 0x003f, 0x007f, 0x00ff}; - int32_t batch = nelements >> 2; - int32_t remainder = nelements & 0x1; - - while (1) { + int32_t i = 0; + if (batch > 1) { + // first loop uint8_t flags = input[ipos++]; - // Decode dd1 - uint64_t dd1 = 0; - nbytes = flags & INT8MASK(4); + int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7 + int8_t nbytes2 = (flags >> 4) & INT8MASK(4); + __m128i data1; + if (nbytes1 == 0) { + data1 = _mm_setzero_si128(); + } else { + memcpy(&data1, (const void*) (input + ipos), nbytes1); + } + + __m128i data2; + if (nbytes2 == 0) { + data2 = _mm_setzero_si128(); + } else { + memcpy(&data2, (const void*) (input + ipos + nbytes1), nbytes2); + } + + data2 = _mm_broadcastq_epi64(data2); + __m128i zzVal = _mm_blend_epi32(data2, data1, 0x03); + + // ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) + __m128i signmask = _mm_and_si128(_mm_set1_epi64x(1), zzVal); + signmask = _mm_sub_epi64(_mm_setzero_si128(), signmask); + + // get two zigzag values here + __m128i deltaOfDelta = _mm_xor_si128(_mm_srli_epi64(zzVal, 1), signmask); + + __m128i deltaCurrent = _mm_add_epi64(deltaOfDelta, prevDelta); + deltaCurrent = _mm_add_epi64(_mm_slli_si128(deltaCurrent, 8), deltaCurrent); + + __m128i val = _mm_add_epi64(deltaCurrent, prevVal); + _mm_storeu_si128((__m128i *)&ostream[opos], val); + + // keep the previous value + prevVal = _mm_shuffle_epi32 (val, 0xEE); + + // keep the previous delta of delta, for the first item + prevDelta = _mm_shuffle_epi32(deltaOfDelta, 0xEE); + + opos += 2; + ipos += nbytes1 + nbytes2; + i += 1; + } + + // the remain + for(; i < batch; ++i) { + uint8_t flags = input[ipos++]; + + int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7 + int8_t nbytes2 = (flags >> 4) & INT8MASK(4); + +// __m128i data1 = _mm_maskz_loadu_epi8(mask2[nbytes1], (const void*)(input + ipos)); +// __m128i data2 = _mm_maskz_loadu_epi8(mask2[nbytes2], (const void*)(input + ipos + nbytes1)); + __m128i data1; + if (nbytes1 == 0) { + data1 = _mm_setzero_si128(); + } else { + int64_t dd = 0; + memcpy(&dd, (const void*) (input + ipos), nbytes1); + data1 = _mm_loadu_si64(&dd); + } + + __m128i data2; + if (nbytes2 == 0) { + data2 = _mm_setzero_si128(); + } else { + int64_t dd = 0; + memcpy(&dd, (const void*) (input + ipos + nbytes1), nbytes2); + data2 = _mm_loadu_si64(&dd); + } + + data2 = _mm_broadcastq_epi64(data2); + + __m128i zzVal = _mm_blend_epi32(data2, data1, 0x03); + + // ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) + __m128i signmask = _mm_and_si128(_mm_set1_epi64x(1), zzVal); + signmask = _mm_sub_epi64(_mm_setzero_si128(), signmask); + + // get two zigzag values here + __m128i deltaOfDelta = _mm_xor_si128(_mm_srli_epi64(zzVal, 1), signmask); + + __m128i deltaCurrent = _mm_add_epi64(deltaOfDelta, prevDelta); + deltaCurrent = _mm_add_epi64(_mm_slli_si128(deltaCurrent, 8), deltaCurrent); + + __m128i val = _mm_add_epi64(deltaCurrent, prevVal); + _mm_storeu_si128((__m128i *)&ostream[opos], val); + + // keep the previous value + prevVal = _mm_shuffle_epi32 (val, 0xEE); + + // keep the previous delta of delta + __m128i delta = _mm_add_epi64(_mm_slli_si128(deltaOfDelta, 8), deltaOfDelta); + prevDelta = _mm_shuffle_epi32(_mm_add_epi64(delta, prevDelta), 0xEE); + + opos += 2; + ipos += nbytes1 + nbytes2; + } + + if (remainder > 0) { + uint64_t dd = 0; + uint8_t flags = input[ipos++]; + + int32_t nbytes = flags & INT8MASK(4); + int64_t deltaOfDelta = 0; if (nbytes == 0) { deltaOfDelta = 0; } else { - if (bigEndian) { - memcpy(((char *)(&dd1)) + longBytes - nbytes, input + ipos, nbytes); - } else { - memcpy(&dd1, input + ipos, nbytes); - } - deltaOfDelta = ZIGZAG_DECODE(int64_t, dd1); + // if (is_bigendian()) { + // memcpy(((char *)(&dd1)) + longBytes - nbytes, input + ipos, nbytes); + // } else { + memcpy(&dd, input + ipos, nbytes); + // } + deltaOfDelta = ZIGZAG_DECODE(int64_t, dd); } ipos += nbytes; - prevDelta += deltaOfDelta; - prevValue += prevDelta; - ostream[opos++] = prevValue; - - if (opos == nelements) { - return nelements * longBytes; - } - - // Decode dd2 - uint64_t dd2 = 0; - nbytes = (flags >> 4) & INT8MASK(4); - if (nbytes == 0) { - deltaOfDelta = 0; + if (opos == 0) { + ostream[opos++] = deltaOfDelta; } else { - if (bigEndian) { - memcpy(((char *)(&dd2)) + longBytes - nbytes, input + ipos, nbytes); - } else { - memcpy(&dd2, input + ipos, nbytes); - } - // zigzag_decoding - deltaOfDelta = ZIGZAG_DECODE(int64_t, dd2); - } - - ipos += nbytes; - prevDelta += deltaOfDelta; - prevValue += prevDelta; - ostream[opos++] = prevValue; - - if (opos == nelements) { - return nelements * longBytes; + int64_t prevDeltaX = deltaOfDelta + prevDelta[1]; + ostream[opos++] = prevVal[1] + prevDeltaX; } } +#endif +#endif + return 0; +} + +int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output, + bool UNUSED_PARAM(bigEndian)) { + int64_t *ostream = (int64_t *)output; + int32_t ipos = 1, opos = 0; + +#if __AVX512VL__ + + __m128i prevVal = _mm_setzero_si128(); + __m128i prevDelta = _mm_setzero_si128(); + + int32_t numOfBatch = nelements >> 1; + int32_t remainder = nelements & 0x01; + __mmask16 mask2[16] = {0, 0x0001, 0x0003, 0x0007, 0x000f, 0x001f, 0x003f, 0x007f, 0x00ff}; + + int32_t i = 0; + if (numOfBatch > 1) { + // first loop + uint8_t flags = input[ipos++]; + + int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7 + int8_t nbytes2 = (flags >> 4) & INT8MASK(4); + + __m128i data1 = _mm_maskz_loadu_epi8(mask2[nbytes1], (const void*)(input + ipos)); + __m128i data2 = _mm_maskz_loadu_epi8(mask2[nbytes2], (const void*)(input + ipos + nbytes1)); + data2 = _mm_broadcastq_epi64(data2); + + __m128i zzVal = _mm_blend_epi32(data2, data1, 0x03); + + // ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) + __m128i signmask = _mm_and_si128(_mm_set1_epi64x(1), zzVal); + signmask = _mm_sub_epi64(_mm_setzero_si128(), signmask); + + // get two zigzag values here + __m128i deltaOfDelta = _mm_xor_si128(_mm_srli_epi64(zzVal, 1), signmask); + + __m128i deltaCurrent = _mm_add_epi64(deltaOfDelta, prevDelta); + deltaCurrent = _mm_add_epi64(_mm_slli_si128(deltaCurrent, 8), deltaCurrent); + + __m128i val = _mm_add_epi64(deltaCurrent, prevVal); + _mm_storeu_si128((__m128i *)&ostream[opos], val); + + // keep the previous value + prevVal = _mm_shuffle_epi32 (val, 0xEE); + + // keep the previous delta of delta, for the first item + prevDelta = _mm_shuffle_epi32(deltaOfDelta, 0xEE); + + opos += 2; + ipos += nbytes1 + nbytes2; + i += 1; + } + + // the remain + for(; i < numOfBatch; ++i) { + uint8_t flags = input[ipos++]; + + int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7 + int8_t nbytes2 = (flags >> 4) & INT8MASK(4); + + __m128i data1 = _mm_maskz_loadu_epi8(mask2[nbytes1], (const void*)(input + ipos)); + __m128i data2 = _mm_maskz_loadu_epi8(mask2[nbytes2], (const void*)(input + ipos + nbytes1)); + data2 = _mm_broadcastq_epi64(data2); + + __m128i zzVal = _mm_blend_epi32(data2, data1, 0x03); + + // ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) + __m128i signmask = _mm_and_si128(_mm_set1_epi64x(1), zzVal); + signmask = _mm_sub_epi64(_mm_setzero_si128(), signmask); + + // get two zigzag values here + __m128i deltaOfDelta = _mm_xor_si128(_mm_srli_epi64(zzVal, 1), signmask); + + __m128i deltaCurrent = _mm_add_epi64(deltaOfDelta, prevDelta); + deltaCurrent = _mm_add_epi64(_mm_slli_si128(deltaCurrent, 8), deltaCurrent); + + __m128i val = _mm_add_epi64(deltaCurrent, prevVal); + _mm_storeu_si128((__m128i *)&ostream[opos], val); + + // keep the previous value + prevVal = _mm_shuffle_epi32 (val, 0xEE); + + // keep the previous delta of delta + __m128i delta = _mm_add_epi64(_mm_slli_si128(deltaOfDelta, 8), deltaOfDelta); + prevDelta = _mm_shuffle_epi32(_mm_add_epi64(delta, prevDelta), 0xEE); + + opos += 2; + ipos += nbytes1 + nbytes2; + } + + if (remainder > 0) { + uint64_t dd = 0; + uint8_t flags = input[ipos++]; + + int32_t nbytes = flags & INT8MASK(4); + int64_t deltaOfDelta = 0; + if (nbytes == 0) { + deltaOfDelta = 0; + } else { + memcpy(&dd, input + ipos, nbytes); + deltaOfDelta = ZIGZAG_DECODE(int64_t, dd); + } + + ipos += nbytes; + if (opos == 0) { + ostream[opos++] = deltaOfDelta; + } else { + int64_t prevDeltaX = deltaOfDelta + prevDelta[1]; + ostream[opos++] = prevVal[1] + prevDeltaX; + } + } + #endif return 0; } \ No newline at end of file diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 8847b7d894..a869af7d5d 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -247,7 +247,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST, "Mnode already exists" TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_NOT_EXIST, "Mnode not there") TAOS_DEFINE_ERROR(TSDB_CODE_MND_QNODE_ALREADY_EXIST, "Qnode already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_QNODE_NOT_EXIST, "Qnode not there") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_SNODE_ALREADY_EXIST, "Snode already exists") +//TAOS_DEFINE_ERROR(TSDB_CODE_MND_SNODE_ALREADY_EXIST, "Snode already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SNODE_ALREADY_EXIST, "Snode can only be created 1") TAOS_DEFINE_ERROR(TSDB_CODE_MND_SNODE_NOT_EXIST, "Snode not there") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_FEW_MNODES, "The replica of mnode cannot less than 1") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_MNODES, "The replica of mnode cannot exceed 3") @@ -445,7 +446,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_DEC_IVLD_KLEN, "Invalid klen to decod TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_IVLD_KEY, "Invalid key to gen active code") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_APP_LIMIT, "Limited app num to gen active code") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_ENC_IVLD_KLEN, "Invalid klen to encode active code") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_IVLD_DIST, "Invalid dist to parse active code") // sync TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT, "Sync timeout") diff --git a/source/util/test/decompressTest.cpp b/source/util/test/decompressTest.cpp new file mode 100644 index 0000000000..caf8df3ba8 --- /dev/null +++ b/source/util/test/decompressTest.cpp @@ -0,0 +1,94 @@ +#include +#include +#include +#include + +namespace {} // namespace + +TEST(utilTest, decompress_test) { + int64_t tsList[10] = {1700000000, 1700000100, 1700000200, 1700000300, 1700000400, + 1700000500, 1700000600, 1700000700, 1700000800, 1700000900}; + + char* pOutput[10 * sizeof(int64_t)] = {0}; + int32_t len = tsCompressTimestamp(tsList, sizeof(tsList), sizeof(tsList) / sizeof(tsList[0]), pOutput, 10, ONE_STAGE_COMP, NULL, 0); + + char* decompOutput[10 * 8] = {0}; + tsDecompressTimestamp(pOutput, len, 10, decompOutput, sizeof(int64_t)*10, ONE_STAGE_COMP, NULL, 0); + + for(int32_t i = 0; i < 10; ++i) { + std::cout<< ((int64_t*)decompOutput)[i] << std::endl; + } + + memset(decompOutput, 0, 10*8); + tsDecompressTimestampAvx512(reinterpret_cast(pOutput), 10, + reinterpret_cast(decompOutput), false); + + for(int32_t i = 0; i < 10; ++i) { + std::cout<<((int64_t*)decompOutput)[i] << std::endl; + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + int64_t tsList1[7] = {1700000000, 1700000000, 1700000000, 1700000000, 1700000000, 1700000000, 1700000900}; + int32_t len1 = tsCompressTimestamp(tsList1, sizeof(tsList1), sizeof(tsList1) / sizeof(tsList1[0]), pOutput, 7, ONE_STAGE_COMP, NULL, 0); + + memset(decompOutput, 0, 10*8); + tsDecompressTimestampAvx512(reinterpret_cast(pOutput), 7, + reinterpret_cast(decompOutput), false); + + for(int32_t i = 0; i < 7; ++i) { + std::cout<<((int64_t*)decompOutput)[i] << std::endl; + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + int64_t tsList2[1] = {1700000000}; + int32_t len2 = tsCompressTimestamp(tsList2, sizeof(tsList2), sizeof(tsList2) / sizeof(tsList2[0]), pOutput, 1, ONE_STAGE_COMP, NULL, 0); + + memset(decompOutput, 0, 10*8); + tsDecompressTimestampAvx512(reinterpret_cast(pOutput), 1, + reinterpret_cast(decompOutput), false); + + for(int32_t i = 0; i < 1; ++i) { + std::cout<<((int64_t*)decompOutput)[i] << std::endl; + } +} + +TEST(utilTest, decompress_perf_test) { + int32_t num = 10000; + + int64_t* pList = static_cast(taosMemoryCalloc(num, sizeof(int64_t))); + int64_t iniVal = 1700000000; + + uint32_t v = 100; + + for(int32_t i = 0; i < num; ++i) { + iniVal += taosRandR(&v)%10; + pList[i] = iniVal; + } + + char* px = static_cast(taosMemoryMalloc(num * sizeof(int64_t))); + int32_t len = tsCompressTimestamp(pList, num * sizeof(int64_t), num, px, num, ONE_STAGE_COMP, NULL, 0); + + char* pOutput = static_cast(taosMemoryMalloc(num * sizeof(int64_t))); + + int64_t st = taosGetTimestampUs(); + for(int32_t k = 0; k < 10000; ++k) { + tsDecompressTimestamp(px, len, num, pOutput, sizeof(int64_t) * num, ONE_STAGE_COMP, NULL, 0); + } + + int64_t el1 = taosGetTimestampUs() - st; + std::cout << "soft decompress elapsed time:" << el1 << " us" << std::endl; + + memset(pOutput, 0, num * sizeof(int64_t)); + st = taosGetTimestampUs(); + for(int32_t k = 0; k < 10000; ++k) { + tsDecompressTimestampAvx512(px, num, pOutput, false); + } + + int64_t el2 = taosGetTimestampUs() - st; + std::cout << "SIMD decompress elapsed time:" << el2 << " us" << std::endl; + + taosMemoryFree(pList); + taosMemoryFree(pOutput); + taosMemoryFree(px); +} + diff --git a/tests/docs-examples-test/python.sh b/tests/docs-examples-test/python.sh index 5de7261e02..84f0771ec5 100644 --- a/tests/docs-examples-test/python.sh +++ b/tests/docs-examples-test/python.sh @@ -86,7 +86,7 @@ pip3 install kafka-python python3 kafka_example_consumer.py # 21 -pip3 install taos-ws-py==0.2.6 +pip3 install taos-ws-py==0.3.1 python3 conn_websocket_pandas.py # 22 diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 7a47df97a9..549e24b0a6 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -6,6 +6,7 @@ ,,y,unit-test,bash test.sh #system test +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/stream_basic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/scalar_function.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/at_once_interval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/at_once_session.py @@ -22,7 +23,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/pause_resume_test.py #,,n,system-test,python3 ./test.py -f 8-stream/vnode_restart.py -N 4 #,,n,system-test,python3 ./test.py -f 8-stream/snode_restart.py -N 4 -,,n,system-test,python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 +#,,n,system-test,python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py @@ -249,6 +250,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/delete_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_hot_refresh_configurations.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_double.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py @@ -300,8 +302,10 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/precisionUS.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/precisionNS.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_ts4219.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/ts-4272.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_ts4295.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_td27388.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_timestamp.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show_tag_index.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/information_schema.py @@ -855,6 +859,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/odbc.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/fill_with_group.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/state_window.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py ,,n,system-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insertMix.py -N 3 @@ -879,7 +884,7 @@ e ,,y,script,./test.sh -f tsim/dnode/balance2.sim ,,y,script,./test.sh -f tsim/vnode/replica3_repeat.sim ,,y,script,./test.sh -f tsim/parser/col_arithmetic_operation.sim -,,y,script,./test.sh -f tsim/trans/create_db.sim +#,,y,script,./test.sh -f tsim/trans/create_db.sim ,,y,script,./test.sh -f tsim/dnode/balance3.sim ,,y,script,./test.sh -f tsim/vnode/replica3_many.sim ,,y,script,./test.sh -f tsim/stable/metrics_idx.sim @@ -1128,6 +1133,9 @@ e ,,y,script,./test.sh -f tsim/stream/distributeIntervalRetrive0.sim ,,y,script,./test.sh -f tsim/stream/distributeSession0.sim ,,y,script,./test.sh -f tsim/stream/drop_stream.sim +,,y,script,./test.sh -f tsim/stream/event0.sim +,,y,script,./test.sh -f tsim/stream/event1.sim +,,y,script,./test.sh -f tsim/stream/event2.sim ,,y,script,./test.sh -f tsim/stream/fillHistoryBasic1.sim ,,y,script,./test.sh -f tsim/stream/fillHistoryBasic2.sim ,,y,script,./test.sh -f tsim/stream/fillHistoryBasic3.sim @@ -1336,7 +1344,7 @@ e #docs-examples test ,,n,docs-examples-test,bash python.sh -,,n,docs-examples-test,bash node.sh +#,,n,docs-examples-test,bash node.sh ,,n,docs-examples-test,bash csharp.sh ,,n,docs-examples-test,bash jdbc.sh ,,n,docs-examples-test,bash go.sh diff --git a/tests/script/tsim/insert/insert_stb.sim b/tests/script/tsim/insert/insert_stb.sim index 4c0797e2a7..14ff2261c2 100644 --- a/tests/script/tsim/insert/insert_stb.sim +++ b/tests/script/tsim/insert/insert_stb.sim @@ -66,6 +66,29 @@ if $rows != 1 then return -1 endi +sql insert into st(tbname, ts, f, t) values('ct2',now,20,NULL)('ct3',now,30,NULL) +sql insert into st(tbname, t, ts, f) values('ct4',NULL, now,20)('ct5',NULL, now,30) +sql show create table ct2 +sql show create table ct3 +sql show create table ct4 +sql show create table ct5 +sql show tags from ct2 +if $data05 != NULL then + return -1 +endi +sql show tags from ct3 +if $data05 != NULL then + return -1 +endi +sql show tags from ct4 +if $data05 != NULL then + return -1 +endi +sql show tags from ct5 +if $data05 != NULL then + return -1 +endi + sql_error insert into d2.st values(now, 1, 1) sql_error insert into d2.st(ts, f) values(now, 1); sql_error insert into d2.st(ts, f, tbname) values(now, 1); diff --git a/tests/script/tsim/parser/columnValue_bigint.sim b/tests/script/tsim/parser/columnValue_bigint.sim index 0a024029a5..056855eea2 100644 --- a/tests/script/tsim/parser/columnValue_bigint.sim +++ b/tests/script/tsim/parser/columnValue_bigint.sim @@ -17,6 +17,7 @@ sql create table mt_bigint (ts timestamp, c bigint) tags (tagname bigint) ## case 00: static create table for test tag values sql create table st_bigint_0 using mt_bigint tags (NULL) +sql show create table st_bigint_0 sql show tags from st_bigint_0 if $data05 != NULL then return -1 @@ -173,6 +174,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_bigint_16 using mt_bigint tags (NULL) values (now, NULL) +sql show create table st_bigint_16 sql show tags from st_bigint_16 if $data05 != NULL then return -1 diff --git a/tests/script/tsim/parser/columnValue_bool.sim b/tests/script/tsim/parser/columnValue_bool.sim index ad4232d884..2553e6805a 100644 --- a/tests/script/tsim/parser/columnValue_bool.sim +++ b/tests/script/tsim/parser/columnValue_bool.sim @@ -17,6 +17,7 @@ sql create table mt_bool (ts timestamp, c bool) tags (tagname bool) ## case 00: static create table for test tag values sql create table st_bool_0 using mt_bool tags (NULL) +sql show create table st_bool_0 sql show tags from st_bool_0 if $data05 != NULL then print ==1== expect: NULL, actually: $data05 @@ -291,6 +292,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_bool_16 using mt_bool tags (NULL) values (now, NULL) +sql show create table st_bool_16 sql show tags from st_bool_16 if $data05 != NULL then print ==33== expect: NULL, actually: $data00 diff --git a/tests/script/tsim/parser/columnValue_float.sim b/tests/script/tsim/parser/columnValue_float.sim index b2db7dff2b..4dcda33224 100644 --- a/tests/script/tsim/parser/columnValue_float.sim +++ b/tests/script/tsim/parser/columnValue_float.sim @@ -17,6 +17,7 @@ sql create table mt_float (ts timestamp, c float) tags (tagname float) ## case 00: static create table for test tag values sql create table st_float_0 using mt_float tags (NULL) +sql show create table st_float_0 sql show tags from st_float_0 if $data05 != NULL then return -1 @@ -299,6 +300,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_float_16 using mt_float tags (NULL) values (now, NULL) +sql show create table st_float_16 sql show tags from st_float_16 if $data05 != NULL then return -1 diff --git a/tests/script/tsim/parser/columnValue_int.sim b/tests/script/tsim/parser/columnValue_int.sim index 4a3b8ebd0b..e68ae6f13f 100644 --- a/tests/script/tsim/parser/columnValue_int.sim +++ b/tests/script/tsim/parser/columnValue_int.sim @@ -17,6 +17,7 @@ sql create table mt_int (ts timestamp, c int) tags (tagname int) ## case 00: static create table for test tag values sql create table st_int_0 using mt_int tags (NULL) +sql show create table st_int_0 sql show tags from st_int_0 if $data05 != NULL then return -1 @@ -172,6 +173,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_int_16 using mt_int tags (NULL) values (now, NULL) +sql show create table st_int_16 sql show tags from st_int_16 if $data05 != NULL then return -1 diff --git a/tests/script/tsim/parser/columnValue_smallint.sim b/tests/script/tsim/parser/columnValue_smallint.sim index eb364f3630..f9be6ebd52 100644 --- a/tests/script/tsim/parser/columnValue_smallint.sim +++ b/tests/script/tsim/parser/columnValue_smallint.sim @@ -20,6 +20,7 @@ sql create table mt_smallint (ts timestamp, c smallint) tags (tagname smallint) ## case 00: static create table for test tag values sql create table st_smallint_0 using mt_smallint tags (NULL) +sql show create table st_smallint_0 sql show tags from st_smallint_0 if $data05 != NULL then return -1 @@ -175,6 +176,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_smallint_16 using mt_smallint tags (NULL) values (now, NULL) +sql show create table st_smallint_16 sql show tags from st_smallint_16 if $data05 != NULL then return -1 diff --git a/tests/script/tsim/parser/columnValue_tinyint.sim b/tests/script/tsim/parser/columnValue_tinyint.sim index d7938aa739..7d0f10a30d 100644 --- a/tests/script/tsim/parser/columnValue_tinyint.sim +++ b/tests/script/tsim/parser/columnValue_tinyint.sim @@ -17,6 +17,7 @@ sql create table mt_tinyint (ts timestamp, c tinyint) tags (tagname tinyint) ## case 00: static create table for test tag values sql create table st_tinyint_0 using mt_tinyint tags (NULL) +sql show create table st_tinyint_0 sql show tags from st_tinyint_0 if $data05 != NULL then print expect NULL, actually: $data05 @@ -173,6 +174,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_tinyint_16 using mt_tinyint tags (NULL) values (now, NULL) +sql show create table st_tinyint_16 sql show tags from st_tinyint_16 if $data05 != NULL then return -1 diff --git a/tests/script/tsim/parser/null_char.sim b/tests/script/tsim/parser/null_char.sim index 098f4061c1..9d476b5227 100644 --- a/tests/script/tsim/parser/null_char.sim +++ b/tests/script/tsim/parser/null_char.sim @@ -13,6 +13,7 @@ sql use $db #### case 0: field NULL, or 'NULL' sql create table mt1 (ts timestamp, col1 int, col2 bigint, col3 float, col4 double, col5 binary(8), col6 bool, col7 smallint, col8 tinyint, col9 nchar(8)) tags (tag1 binary(8), tag2 nchar(8), tag3 int, tag4 bigint, tag5 bool, tag6 float) sql create table st1 using mt1 tags (NULL, 'NULL', 100, 1000, 'false', 9.123) +sql show create table st1 sql insert into st1 values ('2019-01-01 09:00:00.000', 123, -123, 3.0, 4.0, 'binary', true, 1000, 121, 'nchar') sql insert into st1 values ('2019-01-01 09:00:01.000', '456', '456', '3.33', '4.444', 'binary', 'true', '1001', '122', 'nchar') sql insert into st1 values ('2019-01-01 09:00:02.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) @@ -117,6 +118,7 @@ if $rows != 0 then endi sql create table st3 using mt2 tags (NULL, 'ABC', 103, 'FALSE') +sql show create table st3 sql insert into st3 (ts, col1) values(now, 1) sql select tag1, tag2, tag3, tag5 from st3 if $rows != 1 then diff --git a/tests/script/tsim/snode/basic1.sim b/tests/script/tsim/snode/basic1.sim index 86072215f7..e7346b75a0 100644 --- a/tests/script/tsim/snode/basic1.sim +++ b/tests/script/tsim/snode/basic1.sim @@ -113,11 +113,7 @@ sql_error drop snode on dnode 2 print =============== create drop snodes sql create snode on dnode 1 -sql create snode on dnode 2 -sql show snodes -if $rows != 2 then - return -1 -endi +sql_error create snode on dnode 2 print =============== restart system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -127,7 +123,7 @@ system sh/exec.sh -n dnode2 -s start sleep 2000 sql show snodes -if $rows != 2 then +if $rows != 1 then return -1 endi diff --git a/tests/script/tsim/stream/ignoreExpiredData.sim b/tests/script/tsim/stream/ignoreExpiredData.sim index 884b7cbb5f..864f8caea5 100644 --- a/tests/script/tsim/stream/ignoreExpiredData.sim +++ b/tests/script/tsim/stream/ignoreExpiredData.sim @@ -107,7 +107,7 @@ sql select * from information_schema.ins_databases print ======database=$rows -sql use test1 +sql use test1; sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); sql create table ts1 using st tags(1,1,1); @@ -118,9 +118,9 @@ sql insert into ts1 values(1648791211000,1,2,3); sleep 1000 sql insert into ts1 values(1648791222001,2,2,3); sleep 1000 -sql insert into ts2 values(1648791211000,1,2,3); -sleep 1000 sql insert into ts2 values(1648791222001,2,2,3); +sleep 1000 +sql insert into ts2 values(1648791211000,1,2,3); $loop_count = 0 loop4: @@ -132,16 +132,26 @@ if $loop_count == 10 then return -1 endi -if $data01 != 2 then +if $data01 != 1 then print =====data01=$data01 goto loop4 endi -if $data02 != 2 then +if $data02 != 1 then print =====data02=$data02 goto loop4 endi +if $data11 != 2 then + print =====data11=$data11 + goto loop4 +endi + +if $data12 != 2 then + print =====data12=$data12 + goto loop4 +endi + $loop_count = 0 loop5: sleep 1000 @@ -162,4 +172,148 @@ if $data02 != 1 then goto loop5 endi +if $data11 != 2 then + print =====data11=$data11 + goto loop4 +endi + +if $data12 != 2 then + print =====data12=$data12 + goto loop4 +endi + + +print =============== create database test2 +sql create database test2 vgroups 4 +sql select * from information_schema.ins_databases + +print ======database=$rows + +sql use test2; + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); +sql create table ts3 using st tags(3,3,3); +sql create table ts4 using st tags(4,4,4); +sql create stream streams_21 trigger at_once IGNORE EXPIRED 1 into streamt_21 as select _wstart, count(*) c1 from st interval(10s) ; +sleep 1000 + +sql insert into ts1 values(1648791211000,1,2,3); +sql insert into ts1 values(1648791211001,2,2,3); +sql insert into ts1 values(1648791211002,2,2,3); +sql insert into ts1 values(1648791211003,2,2,3); +sql insert into ts1 values(1648791211004,2,2,3); + +sleep 1000 +sql insert into ts2 values(1648791201000,1,2,3); +sql insert into ts2 values(1648791201001,2,2,3); +sql insert into ts2 values(1648791201002,2,2,3); +sql insert into ts2 values(1648791201003,2,2,3); +sql insert into ts2 values(1648791201004,2,2,3); + +sleep 1000 +sql insert into ts2 values(1648791101000,1,2,3); +sql insert into ts2 values(1648791101001,2,2,3); +sql insert into ts2 values(1648791101002,2,2,3); +sql insert into ts2 values(1648791101003,2,2,3); +sql insert into ts2 values(1648791101004,2,2,3); + + +$loop_count = 0 +loop6: +sleep 1000 +print 1 select * from streamt_21; +sql select * from streamt_21; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop6 +endi + +if $data01 != 5 then + print =====data01=$data01 + goto loop6 +endi + +if $data11 != 5 then + print =====data11=$data11 + goto loop6 +endi + +sleep 1000 +sql insert into ts3 values(1648791241000,1,2,3); + +sleep 1000 +sql insert into ts3 values(1648791231001,2,2,3); +sql insert into ts3 values(1648791231002,2,2,3); +sql insert into ts3 values(1648791231003,2,2,3); +sql insert into ts3 values(1648791231004,2,2,3); + +$loop_count = 0 +loop7: +sleep 1000 +print 2 select * from streamt_21; +sql select * from streamt_21; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 3 then + print =====rows=$rows + goto loop7 +endi + +if $data21 != 1 then + print =====data21=$data21 + goto loop7 +endi + +sleep 1000 +sql insert into ts4 values(1648791231001,2,2,3); +sql insert into ts4 values(1648791231002,2,2,3); +sql insert into ts4 values(1648791231003,2,2,3); +sql insert into ts4 values(1648791231004,2,2,3); + +sleep 1000 +sql insert into ts4 values(1648791211001,2,2,3); +sql insert into ts4 values(1648791211002,2,2,3); +sql insert into ts4 values(1648791211003,2,2,3); +sql insert into ts4 values(1648791211004,2,2,3); + +$loop_count = 0 +loop8: +sleep 1000 +print 3 select * from streamt_21; +sql select * from streamt_21; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 4 then + print =====rows=$rows + goto loop8 +endi + +if $data21 != 4 then + print =====data21=$data21 + goto loop8 +endi + +if $data31 != 1 then + print =====data31=$data31 + goto loop8 +endi + +print ============================end + system sh/stop_dnodes.sh diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 83bfb2bed7..d54c676c0d 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -90,7 +90,10 @@ class TDTestCase: packagePath = "/usr/local/src/" dataPath = cPath + "/../data/" - packageName = "TDengine-server-"+ BASEVERSION + "-Linux-x64.tar.gz" + if platform.system() == "Linux" and platform.machine() == "aarch64": + packageName = "TDengine-server-"+ BASEVERSION + "-Linux-arm64.tar.gz" + else: + packageName = "TDengine-server-"+ BASEVERSION + "-Linux-x64.tar.gz" packageTPath = packageName.split("-Linux-")[0] my_file = Path(f"{packagePath}/{packageName}") if not my_file.exists(): diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index 544a966960..2bfe33d0af 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -247,10 +247,7 @@ class TDTestCase: tdSql.error('alter all dnodes "activeCode" "' + self.str510 + '"') tdSql.query(f'select * from information_schema.ins_dnodes') tdSql.checkEqual(tdSql.queryResult[0][8],"") - tdSql.error('alter dnode 1 "activeCode" ""') - tdSql.error('alter dnode 1 "activeCode"') - tdSql.execute('alter all dnodes "activeCode" ""') - tdSql.execute('alter all dnodes "activeCode"') + tdSql.execute('alter dnode 1 "activeCode" ""') tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes') tdSql.checkEqual(tdSql.queryResult[0][0],"") tdSql.checkEqual(tdSql.queryResult[0][1],'') @@ -262,10 +259,6 @@ class TDTestCase: tdSql.error('alter all dnodes "cActiveCode" "' + self.str257 + '"') tdSql.error('alter all dnodes "cActiveCode" "' + self.str254 + '"') tdSql.error('alter dnode 1 "cActiveCode" "' + self.str510 + '"') - tdSql.error('alter dnode 1 "cActiveCode" ""') - tdSql.error('alter dnode 1 "cActiveCode"') - tdSql.execute('alter all dnodes "cActiveCode" ""') - tdSql.execute('alter all dnodes "cActiveCode"') tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes') tdSql.checkEqual(tdSql.queryResult[0][0],"") tdSql.checkEqual(tdSql.queryResult[0][1],"") diff --git a/tests/system-test/0-others/test_create_same_name_db.py b/tests/system-test/0-others/test_create_same_name_db.py new file mode 100644 index 0000000000..2b2c63af53 --- /dev/null +++ b/tests/system-test/0-others/test_create_same_name_db.py @@ -0,0 +1,38 @@ +import time +import os +import platform +import taos +import threading +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * + + +class TDTestCase: + """This test case is used to veirfy TD-25762 + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.db_name = "db" + + def run(self): + try: + # create same name database multiple times + for i in range(100): + tdLog.debug(f"round {str(i+1)} create database {self.db_name}") + tdSql.execute(f"create database {self.db_name}") + tdLog.debug(f"round {str(i+1)} drop database {self.db_name}") + tdSql.execute(f"drop database {self.db_name}") + except Exception as ex: + tdLog.exit(str(ex)) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/test_hot_refresh_configurations.py b/tests/system-test/0-others/test_hot_refresh_configurations.py index 7aed7274a4..cbde8c060e 100644 --- a/tests/system-test/0-others/test_hot_refresh_configurations.py +++ b/tests/system-test/0-others/test_hot_refresh_configurations.py @@ -2,7 +2,7 @@ import subprocess import random import time import os - +import platform from util.log import * from util.sql import * from util.cases import * @@ -190,6 +190,8 @@ class TDTestCase: for v in values: dnode = random.choice(p_list) tdSql.execute(f'alter {dnode} "{name} {v}";') + if platform.system() == "Linux" and platform.machine() == "aarch64": + continue value = self.get_param_value_with_gdb(alias, "taosd") if value: tdLog.debug(f"value: {value}") diff --git a/tests/system-test/0-others/user_privilege.py b/tests/system-test/0-others/user_privilege.py index d1b93f6942..a731e85ddb 100644 --- a/tests/system-test/0-others/user_privilege.py +++ b/tests/system-test/0-others/user_privilege.py @@ -27,6 +27,7 @@ class TDTestCase: tdSql.init(conn.cursor()) self.setsql = TDSetSql() self.stbname = 'stb' + self.user_name = 'test' self.binary_length = 20 # the length of binary for column_dict self.nchar_length = 20 # the length of nchar for column_dict self.dbnames = ['db1', 'db2'] @@ -54,12 +55,12 @@ class TDTestCase: ] self.tbnum = 4 + self.stbnum_grant = 200 def create_user(self): - user_name = 'test' - tdSql.execute(f'create user {user_name} pass "test"') - tdSql.execute(f'grant read on {self.dbnames[0]}.{self.stbname} with t2 = "Beijing" to {user_name}') - tdSql.execute(f'grant write on {self.dbnames[1]}.{self.stbname} with t1 = 2 to {user_name}') + tdSql.execute(f'create user {self.user_name} pass "test"') + tdSql.execute(f'grant read on {self.dbnames[0]}.{self.stbname} with t2 = "Beijing" to {self.user_name}') + tdSql.execute(f'grant write on {self.dbnames[1]}.{self.stbname} with t1 = 2 to {self.user_name}') def prepare_data(self): for db in self.dbnames: @@ -70,6 +71,8 @@ class TDTestCase: tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({self.tag_list[i]})') for j in self.values_list: tdSql.execute(f'insert into {self.stbname}_{i} values({j})') + for i in range(self.stbnum_grant): + tdSql.execute(f'create table {self.stbname}_grant_{i} (ts timestamp, c0 int) tags(t0 int)') def user_read_privilege_check(self, dbname): testconn = taos.connect(user='test', password='test') @@ -128,12 +131,20 @@ class TDTestCase: tdLog.exit(f"{caller.filename}({caller.lineno}) failed: sql:{sql}, expect error not occured") pass + def user_privilege_grant_check(self): + for db in self.dbnames: + tdSql.execute(f"use {db}") + for i in range(self.stbnum_grant): + tdSql.execute(f'grant read on {db}.{self.stbname}_grant_{i} to {self.user_name}') + tdSql.execute(f'grant write on {db}.{self.stbname}_grant_{i} to {self.user_name}') + def run(self): self.prepare_data() self.create_user() self.user_read_privilege_check(self.dbnames[0]) self.user_write_privilege_check(self.dbnames[1]) self.user_privilege_error_check() + self.user_privilege_grant_check() def stop(self): tdSql.close() diff --git a/tests/system-test/1-insert/insert_double.py b/tests/system-test/1-insert/insert_double.py new file mode 100644 index 0000000000..b47b22ab44 --- /dev/null +++ b/tests/system-test/1-insert/insert_double.py @@ -0,0 +1,136 @@ +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * +import random + + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + self.database = "db1" + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def prepare_db(self): + tdSql.execute(f"drop database if exists {self.database}") + tdSql.execute(f"create database {self.database}") + tdSql.execute(f"use {self.database}") + + def test_value(self, table_name, dtype, bits): + tdSql.execute(f"drop table if exists {table_name}") + tdSql.execute(f"create table {table_name}(ts timestamp, i1 {dtype}, i2 {dtype} unsigned)") + + tdSql.execute(f"insert into {table_name} values(now, -16, +6)") + tdSql.execute(f"insert into {table_name} values(now, 80.99, +0042)") + tdSql.execute(f"insert into {table_name} values(now, -0042, +80.99)") + tdSql.execute(f"insert into {table_name} values(now, 52.34354, 18.6)") + tdSql.execute(f"insert into {table_name} values(now, -12., +3.)") + tdSql.execute(f"insert into {table_name} values(now, -0.12, +3.0)") + tdSql.execute(f"insert into {table_name} values(now, -2.3e1, +2.324e2)") + tdSql.execute(f"insert into {table_name} values(now, -2e1, +2e2)") + tdSql.execute(f"insert into {table_name} values(now, -2.e1, +2.e2)") + tdSql.execute(f"insert into {table_name} values(now, -0x40, +0b10000)") + tdSql.execute(f"insert into {table_name} values(now, -0b10000, +0x40)") + + # str support + tdSql.execute(f"insert into {table_name} values(now, '-16', '+6')") + tdSql.execute(f"insert into {table_name} values(now, ' -80.99', ' +0042')") + tdSql.execute(f"insert into {table_name} values(now, ' -0042', ' +80.99')") + tdSql.execute(f"insert into {table_name} values(now, '52.34354', '18.6')") + tdSql.execute(f"insert into {table_name} values(now, '-12.', '+5.')") + tdSql.execute(f"insert into {table_name} values(now, '-.12', '+.5')") + tdSql.execute(f"insert into {table_name} values(now, '-2.e1', '+2.e2')") + tdSql.execute(f"insert into {table_name} values(now, '-2e1', '+2e2')") + tdSql.execute(f"insert into {table_name} values(now, '-2.3e1', '+2.324e2')") + tdSql.execute(f"insert into {table_name} values(now, '-0x40', '+0b10010')") + tdSql.execute(f"insert into {table_name} values(now, '-0b10010', '+0x40')") + + tdSql.query(f"select * from {table_name}") + tdSql.checkRows(22) + + baseval = 2**(bits/2) + negval = -baseval + 1.645 + posval = baseval + 4.323 + bigval = 2**(bits-1) + max_i = bigval - 1 + min_i = -bigval + max_u = 2*bigval - 1 + min_u = 0 + print("val:", baseval, negval, posval, max_i) + + tdSql.execute(f"insert into {table_name} values(now, {negval}, {posval})") + tdSql.execute(f"insert into {table_name} values(now, -{baseval}, {baseval})") + tdSql.execute(f"insert into {table_name} values(now, {max_i}, {max_u})") + tdSql.execute(f"insert into {table_name} values(now, {min_i}, {min_u})") + + tdSql.query(f"select * from {table_name}") + tdSql.checkRows(26) + + # error case + tdSql.error(f"insert into {table_name} values(now, 0, {max_u+1})") + tdSql.error(f"insert into {table_name} values(now, 0, -1)") + tdSql.error(f"insert into {table_name} values(now, 0, -2.0)") + tdSql.error(f"insert into {table_name} values(now, 0, '-2.0')") + tdSql.error(f"insert into {table_name} values(now, {max_i+1}, 0)") + tdSql.error(f"insert into {table_name} values(now, {min_i-1}, 0)") + tdSql.error(f"insert into {table_name} values(now, '{min_i-1}', 0)") + + def test_tags(self, stable_name, dtype, bits): + tdSql.execute(f"create stable {stable_name}(ts timestamp, i1 {dtype}, i2 {dtype} unsigned) tags(id {dtype})") + + baseval = 2**(bits/2) + negval = -baseval + 1.645 + posval = baseval + 4.323 + bigval = 2**(bits-1) + max_i = bigval - 1 + min_i = -bigval + max_u = 2*bigval - 1 + min_u = 0 + + tdSql.execute(f"insert into {stable_name}_1 using {stable_name} tags('{negval}') values(now, {negval}, {posval})") + tdSql.execute(f"insert into {stable_name}_2 using {stable_name} tags({posval}) values(now, -{baseval} , {baseval})") + tdSql.execute(f"insert into {stable_name}_3 using {stable_name} tags('0x40') values(now, {max_i}, {max_u})") + tdSql.execute(f"insert into {stable_name}_4 using {stable_name} tags(0b10000) values(now, {min_i}, {min_u})") + + tdSql.execute(f"insert into {stable_name}_5 using {stable_name} tags({max_i}) values(now, '{negval}', '{posval}')") + tdSql.execute(f"insert into {stable_name}_6 using {stable_name} tags('{min_i}') values(now, '-{baseval}' , '{baseval}')") + tdSql.execute(f"insert into {stable_name}_7 using {stable_name} tags(-0x40) values(now, '{max_i}', '{max_u}')") + tdSql.execute(f"insert into {stable_name}_8 using {stable_name} tags('-0b10000') values(now, '{min_i}', '{min_u}')") + + tdSql.execute(f"insert into {stable_name}_9 using {stable_name} tags(12.) values(now, {negval}, {posval})") + tdSql.execute(f"insert into {stable_name}_10 using {stable_name} tags('-8.3') values(now, -{baseval} , {baseval})") + tdSql.execute(f"insert into {stable_name}_11 using {stable_name} tags(2.e1) values(now, {max_i}, {max_u})") + tdSql.execute(f"insert into {stable_name}_12 using {stable_name} tags('-2.3e1') values(now, {min_i}, {min_u})") + + tdSql.query(f"select * from {stable_name}") + tdSql.checkRows(12) + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare(replica = self.replicaVar) + self.prepare_db() + + self.test_value("t1", "bigint", 64) + self.test_value("t2", "int", 32) + self.test_value("t3", "smallint", 16) + self.test_value("t4", "tinyint", 8) + tdLog.printNoPrefix("==========end case1 run ...............") + + self.test_tags("t_big", "bigint", 64) + self.test_tags("t_int", "int", 32) + self.test_tags("t_small", "smallint", 16) + self.test_tags("t_tiny", "tinyint", 8) + tdLog.printNoPrefix("==========end case2 run ...............") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/insert_perf.py b/tests/system-test/1-insert/insert_perf.py new file mode 100644 index 0000000000..d96a031458 --- /dev/null +++ b/tests/system-test/1-insert/insert_perf.py @@ -0,0 +1,146 @@ +import taos +import sys +import random +import time +import csv + +from datetime import datetime + +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + + self.testcasePath = os.path.split(__file__)[0] + self.testcasefilename = os.path.split(__file__)[-1] + self.file1 = f"{self.testcasePath}/int.csv" + self.file2 = f"{self.testcasePath}/double.csv" + self.file3 = f"{self.testcasePath}/d+.csv" + self.file4 = f"{self.testcasePath}/uint.csv" + self.ts = 1700638570000 # 2023-11-22T07:36:10.000Z + self.database = "db1" + self.tb1 = "t1" + self.tb2 = "t2" + self.tb3 = "t3" + self.once = 1000 + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def prepare_db(self): + tdSql.execute(f"drop database if exists {self.database}") + tdSql.execute(f"create database {self.database}") + tdSql.execute(f"use {self.database}") + tdSql.execute(f"create table {self.tb1} (ts timestamp, i0 bigint , i1 bigint, i2 bigint, i3 bigint, i4 bigint, i5 bigint, i6 bigint, i7 bigint, i8 bigint, i9 bigint)") + tdSql.execute(f"create table {self.tb2} (ts timestamp, f0 double, f1 double, f2 double, f3 double, f4 double, f5 double, f6 double, f7 double, f8 double, f9 double)") + tdSql.execute(f"create table {self.tb3} (ts timestamp, i0 int unsigned , i1 int unsigned, i2 int unsigned, i3 int unsigned, i4 int unsigned, i5 int unsigned, i6 int unsigned, i7 int unsigned, i8 int unsigned, i9 int unsigned)") + + def make_csv(self, once, intype): + filepath = self.file1 + if intype == 2: + filepath = self.file2 + elif intype == 3: + filepath = self.file3 + elif intype == 4: + filepath = self.file4 + + f = open(filepath, 'w') + with f: + writer = csv.writer(f) + rows = [] + for i in range(once): + r = [] + if intype == 1: + for k in range(10): + r.append(random.randint(-2147483648, 2147483647)) + elif intype == 2: + for k in range(10): + r.append(random.randint(-2147483648, 2147483646) + random.random()) + elif intype == 3: + for k in range(10): + r.append(random.randint(0, 4294967294) + random.random()) + else: + for k in range(10): + r.append(random.randint(0, 4294967295)) + rows.append(r) + writer.writerows(rows) + f.close() + print(f"{filepath} ready!") + + def test_insert(self, tbname, qtime, startts, intype, outtype): + filepath = self.file1 + dinfo = "int" + if intype == 2: + filepath = self.file2 + dinfo = "double" + elif intype == 3: + filepath = self.file3 + dinfo = "+double" + elif intype == 4: + filepath = self.file4 + dinfo = "uint" + + f = open(filepath, 'r') + rows = [] + with f: + reader = csv.reader(f, delimiter=',', quotechar='|') + for row in reader: + rows.append(row) + f.close() + self.once = len(rows) + + sum = 0 + for j in range(qtime): + offset = j * self.once + ts = startts + offset + sql = f"insert into {self.database}.{tbname} values" + for i in range(self.once): + r = rows[i] + sql +=f"({ts + i},'{r[0]}','{r[1]}','{r[2]}','{r[3]}','{r[4]}','{r[5]}','{r[6]}','{r[7]}','{r[8]}','{r[9]}')" + + t1 = time.time() + tdSql.execute(f"{sql};", 1) + t2 = time.time() + #print(f"{t2} insert test {j}.") + #print(sql) + sum += t2 - t1 + + sum = sum + tbinfo = "10 bigint col/per row" + if outtype == 2: + tbinfo = "10 double col/per row" + elif outtype == 3: + tbinfo = "10 uint col/per row" + print(f" insert {self.once} * {qtime} rows: {sum} s, {dinfo} -> {tbinfo}") + + # tdSql.query(f"select count(*) from {self.database}.{tbname};") + # tdSql.checkData(0, 0, once*qtime) + + def run(self): + tdSql.prepare(replica = self.replicaVar) + # self.make_csv(self.once, 1) + # self.make_csv(self.once, 2) + # self.make_csv(self.once, 3) + # self.make_csv(self.once, 4) + + self.prepare_db() + self.test_insert(self.tb1, 1000, self.ts-10000000, 1, 1) + self.test_insert(self.tb2, 1000, self.ts-10000000, 2, 2) + self.test_insert(self.tb3, 1000, self.ts-10000000, 4, 3) + self.test_insert(self.tb2, 1000, self.ts, 1, 2) + + self.test_insert(self.tb1, 1000, self.ts, 2, 1) + self.test_insert(self.tb3, 1000, self.ts, 3, 3) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/insert_timestamp.py b/tests/system-test/1-insert/insert_timestamp.py new file mode 100644 index 0000000000..621912f664 --- /dev/null +++ b/tests/system-test/1-insert/insert_timestamp.py @@ -0,0 +1,70 @@ +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from math import inf + + +class TDTestCase: + def init(self, conn, logSql, replicaVer=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + + #def prepare_data(self): + + + def run(self): + tdSql.execute("create database test_insert_timestamp;") + tdSql.execute("use test_insert_timestamp;") + tdSql.execute("create stable st(ts timestamp, c1 int) tags(id int);") + tdSql.execute("create table test_t using st tags(1);") + + tdSql.error("insert into test_t values(now + today(), 1 ); ") + tdSql.error("insert into test_t values(now - today(), 1 ); ") + tdSql.error("insert into test_t values(today() + now(), 1 ); ") + tdSql.error("insert into test_t values(today() - now(), 1 ); ") + tdSql.error("insert into test_t values(2h - now(), 1 ); ") + tdSql.error("insert into test_t values(2h - today(), 1 ); ") + tdSql.error("insert into test_t values(2h - 1h, 1 ); ") + tdSql.error("insert into test_t values(2h + 1h, 1 ); ") + tdSql.error("insert into test_t values('2023-11-28 00:00:00.000' + '2023-11-28 00:00:00.000', 1 ); ") + tdSql.error("insert into test_t values('2023-11-28 00:00:00.000' + 1701111600000, 1 ); ") + tdSql.error("insert into test_t values(1701111500000 + 1701111600000, 1 ); ") + tdSql.error("insert into test_insert_timestamp.test_t values(1701111600000 + 1h + 1s, 4); ") + + tdSql.execute("insert into test_insert_timestamp.test_t values(1701111600000 + 1h, 4); ") + tdSql.execute("insert into test_insert_timestamp.test_t values(2h + 1701111600000, 5); ") + tdSql.execute("insert into test_insert_timestamp.test_t values('2023-11-28 00:00:00.000' + 1h, 1); ") + tdSql.execute("insert into test_insert_timestamp.test_t values(3h + '2023-11-28 00:00:00.000', 3); ") + tdSql.execute("insert into test_insert_timestamp.test_t values(1701111600000 - 1h, 2); ") + tdSql.execute("insert into test_insert_timestamp.test_t values(1701122400000, 6); ") + tdSql.execute("insert into test_insert_timestamp.test_t values('2023-11-28 07:00:00.000', 7); ") + + tdSql.query(f'select ts, c1 from test_t order by ts;') + tdSql.checkRows(7) + tdSql.checkEqual(tdSql.queryResult[0][0], datetime.datetime(2023, 11, 28, 1, 0, 0) ) + tdSql.checkEqual(tdSql.queryResult[0][1], 1) + tdSql.checkEqual(tdSql.queryResult[1][0], datetime.datetime(2023, 11, 28, 2, 0, 0) ) + tdSql.checkEqual(tdSql.queryResult[1][1], 2) + tdSql.checkEqual(tdSql.queryResult[2][0], datetime.datetime(2023, 11, 28, 3, 0, 0) ) + tdSql.checkEqual(tdSql.queryResult[2][1], 3) + tdSql.checkEqual(tdSql.queryResult[3][0], datetime.datetime(2023, 11, 28, 4, 0, 0) ) + tdSql.checkEqual(tdSql.queryResult[3][1], 4) + tdSql.checkEqual(tdSql.queryResult[4][0], datetime.datetime(2023, 11, 28, 5, 0, 0) ) + tdSql.checkEqual(tdSql.queryResult[4][1], 5) + tdSql.checkEqual(tdSql.queryResult[5][0], datetime.datetime(2023, 11, 28, 6, 0, 0) ) + tdSql.checkEqual(tdSql.queryResult[5][1], 6) + tdSql.checkEqual(tdSql.queryResult[6][0], datetime.datetime(2023, 11, 28, 7, 0, 0) ) + tdSql.checkEqual(tdSql.queryResult[6][1], 7) + + tdSql.execute("drop table if exists test_t ;") + tdSql.execute("drop stable if exists st;") + tdSql.execute("drop database if exists test_insert_timestamp;") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/ts-4272.py b/tests/system-test/1-insert/ts-4272.py new file mode 100644 index 0000000000..bb81305eb3 --- /dev/null +++ b/tests/system-test/1-insert/ts-4272.py @@ -0,0 +1,205 @@ + +import csv +from datetime import datetime + +import taos +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + self.testcasePath = os.path.split(__file__)[0] + self.testcasefilename = os.path.split(__file__)[-1] + self.ts = 1700638570000 # 2023-11-22T07:36:10.000Z + self.db = 'db1' + self.tb1 = 'd001' + self.tb2 = 'd002' + self.stable0 = "meters" + self.stable1 = "stb_1" + self.stable2 = "stb_null" + self.tag1 = f'using {self.stable0}(groupId) tags(1)' + self.tag2 = f'using {self.stable0}(groupId) tags(2)' + self.file1 = f"{self.testcasePath}/b.csv" + self.file2 = f"{self.testcasePath}/c.csv" + + #os.system("rm -rf %s/b.csv" %self.testcasePath) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), logSql) + + def check_count(self, rows, records): + tdSql.execute(f"use {self.db};") + tdSql.query(f"select tbname,count(*) from {self.stable0} group by tbname order by tbname;") + tdSql.checkRows(rows) + for i in range(rows): + tdSql.checkData(i, 1, records[i]) + + def reset_tb(self): + # create database and tables + # os.system("taos -s 'drop database if exists d1;'") + # os.system("taos -s 'create database d1;use d1;create stable meters (ts timestamp, current float, voltage int, phase float) tags (location binary(64), groupId int);'") + # os.system(f"taos -s 'use d1;create table d2001 using meters(groupId) tags(5);'") + # res = os.system(f"taos -s 'use d1;create table d2002 using meters(groupId) tags(6);'") + # if (0 != res): + # tdLog.exit(f"create tb error") + + tdSql.execute(f"drop database if exists {self.db};") + tdSql.execute(f"create database {self.db};") + tdSql.execute(f"use {self.db};") + tdSql.execute(f"create stable {self.stable0} (ts timestamp, current float, voltage int, phase float) tags (location binary(64), groupId int);") + tdSql.execute(f"create table {self.tb1} {self.tag1};") + tdSql.execute(f"create table {self.tb2} {self.tag2};") + tdSql.execute(f"create stable {self.stable1} (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);") + tdSql.execute(f"create stable {self.stable2} (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);") + + def test(self, sql): + sql = f"use {self.db};" + sql + res = os.system(f'taos -s "{sql}"') + # if (0 != res): + # tdLog.exit(f"taos sql error") + + + def check(self): + # same table, auto create + create + sql = f"insert into {self.tb1} {self.tag1} file '{self.file1}' {self.tb1} {self.tag1} file '{self.file2}';" + self.test(sql) + + # same table, create + insert + sql = f"insert into {self.tb1} {self.tag1} file '{self.file1}' {self.tb1} file '{self.file2}';" + self.test(sql) + + # same table, insert + create + sql = f"insert into {self.tb1} file '{self.file1}' {self.tb1} {self.tag1} file '{self.file2}';" + self.test(sql) + + # same table, insert + insert + sql = f"insert into {self.tb1} file '{self.file1}' {self.tb1} file '{self.file2}';" + self.test(sql) + + # diff table auto create + create + sql = f"insert into {self.tb1} {self.tag1} file '{self.file1}' {self.tb2} {self.tag2} file '{self.file2}';" + self.test(sql) + + # diff table, create + insert + sql = f"insert into {self.tb1} {self.tag1} file '{self.file1}' {self.tb2} file '{self.file2}';" + self.test(sql) + + # diff table, insert + create + sql = f"insert into {self.tb1} file '{self.file1}' {self.tb2} {self.tag2} file '{self.file2}';" + self.test(sql) + + # diff table, insert + insert + sql = f"insert into {self.tb1} file '{self.file1}' {self.tb2} file '{self.file2}';" + self.test(sql) + + # bigNum = 1010000 + # self.check_count(5, [2100, 2100, bigNum, bigNum, bigNum]) + + result = os.popen("taos -s 'select count(*) from %s.%s'" %(self.db, self.tb1)) + res = result.read() + if (f"OK" in res): + tdLog.info(f"check count success") + + def make_csv(self, filepath, once, qtime, startts): + f = open(filepath, 'w') + with f: + writer = csv.writer(f) + for j in range(qtime): + ts = startts + j*once + rows = [] + for i in range(once): + rows.append([ts + i, 0.3 + (i%10)/100.0, 210 + i%10, 10.0 + (i%20)/20.0]) + writer.writerows(rows) + f.close() + print(datetime.now(), filepath, " ready!") + + def test_mix(self): + #forbid use both value and file in one insert + result = os.popen(f"insert into {self.tb1} file '{self.file2}' {self.tb2} values('2021-07-13 14:06:34.630', 10.2, 219, 0.32);") + res = result.read() + if (f"error" in res): + tdLog.info(f"forbid success") + + def test_bigcsv(self): + # prepare csv + print("start csv data prepare") + once = 10000 + qtime1 = 101 + qtime2 = 100 + rowNum1 = qtime1 * once + rowNum2 = qtime2 * once + self.make_csv(self.file1, once, qtime1, self.ts - 86400000) + self.make_csv(self.file2, once, qtime2, self.ts) + print("end csv data prepare") + + # auto create + insert + sql = f"insert into {self.tb1} {self.tag1} file '{self.file1}';" + self.test(sql) + + # only insert + sql = f"insert into {self.tb2} file '{self.file2}';" + self.test(sql) + print("end insert to table") + + #tdSql.execute(f"use d1;") + tdSql.query(f"select tbname,count(*) from {self.stable0} group by tbname order by tbname;") + tdSql.checkRows(2) + tdSql.checkData(0, 1, rowNum1) + tdSql.checkData(1, 1, rowNum2) + print("check insert file to table success") + + def make_stable_csv(self, filepath, once, qtime, startts, table_name): + f = open(filepath, 'w') + with f: + writer = csv.writer(f) + for j in range(qtime): + offset = j*once + ts = startts + offset + rows = [] + for i in range(once): + rows.append([table_name, ts + i, offset + i, 'NULL']) + writer.writerows(rows) + f.close() + print(datetime.now(), filepath, " ready!") + + def test_stable_csv(self): + # prepare csv + print("start stable_csv data prepare") + once = 10000 + qtime1 = 101 + qtime2 = 100 + # rowNum1 = qtime1 * once + # rowNum2 = qtime2 * once + child_1 = f"{self.stable1}_1" + child_2 = f"{self.stable2}_1" + self.make_stable_csv(self.file1, once, qtime1, self.ts - 86400000, child_1) + self.make_stable_csv(self.file2, once, qtime2, self.ts, child_2) + print("end stable_csv data prepare") + + # insert create child table of stable + sql = f"insert into {self.db}.{self.stable1}(tbname,ts,q_int,q_binary) file '{self.file1}' {self.db}.{self.stable2}(tbname,ts,q_int,q_binary) file '{self.file2}';" + self.test(sql) + print("end insert to stable") + + #tdSql.execute(f"insert into {self.db}.{child_1}(ts, q_int) values(now, 1);") + tdSql.query(f"select tbname,count(*) from {self.stable1} group by tbname order by tbname;") + tdSql.checkRows(0) + print("check stable success") + + def run(self): + tdSql.prepare() + self.reset_tb() + self.test_stable_csv() + self.test_bigcsv() + self.test_mix() + self.check() + tdSql.close() + + def stop(self): + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/insert_select.py b/tests/system-test/2-query/insert_select.py index e74cf7a8d1..cbdcf366d2 100644 --- a/tests/system-test/2-query/insert_select.py +++ b/tests/system-test/2-query/insert_select.py @@ -70,7 +70,116 @@ class TDTestCase: tdSql.error('''insert into %s.tb1 (c8, c9) values(now, 1);'''%(database)) - + def use_select_sort(self,database): + ts = 1604298064000 + + tdSql.execute('''drop database if exists %s ;''' %database) + tdSql.execute('''create database %s keep 36500 ;'''%(database)) + tdSql.execute('''use %s;'''%database) + + tdSql.execute('''create stable %s.st (ts timestamp, val int, vt timestamp) tags (location NCHAR(100));'''%(database)) + tdSql.execute('''create table %s.t1 using %s.st (location) tags ("0001");'''%(database,database)) + tdSql.execute('''create table %s.t2 using %s.st (location) tags ("0002");'''%(database,database)) + tdSql.execute('''create table %s.mt (ts timestamp, val int);'''%(database)) + + + tdSql.execute(f'''insert into %s.t1 values({ts}, 1, {ts}) ({ts}, 2, {ts});'''%(database)) + tdSql.query("select ts, val from %s.t1;"%database) + tdSql.checkData(0,1,2) + + ts += 1 + tdSql.execute(f'''insert into %s.t2 values({ts}, 1, {ts}) ({ts}, 5, {ts}) ({ts}, 2, {ts});'''%(database)) + tdSql.query("select ts, val from %s.t2;"%database) + tdSql.checkData(0,1,2) + + tdSql.execute('''delete from %s.t2;'''%(database)) + tdSql.execute('''delete from %s.t1;'''%(database)) + + ts -= 10 + tdSql.execute(f'''insert into %s.t1 values({ts}, 1, {ts}) %s.t2 values({ts}, 2, {ts});'''%(database,database)) + ts += 11 + tdSql.execute(f'''insert into %s.t1 values({ts}, 1, {ts}) %s.t2 values({ts}, 2, {ts});'''%(database,database)) + ts += 1 + tdSql.execute(f'''insert into %s.t1 values({ts}, 1, {ts}) %s.t2 values({ts}, 2, {ts});'''%(database,database)) + + tdSql.query("select count(*) from %s.st;"%database) + tdSql.checkData(0,0,6) + + tdSql.query('''select vt, val from %s.st order by vt, val desc;'''%(database)) + tdSql.checkData(0,1,2) + tdSql.checkData(1,1,1) + tdSql.checkData(2,1,2) + tdSql.checkData(3,1,1) + tdSql.checkData(4,1,2) + tdSql.checkData(5,1,1) + + tdSql.execute('''insert into %s.mt select vt, val from %s.st order by vt, val desc;'''%(database,database)) + tdSql.query("select count(*) from %s.mt;"%database) + tdSql.checkData(0,0,3) + + tdSql.query('''select ts, val from %s.mt order by ts asc;'''%(database)) + tdSql.checkData(0,1,1) + tdSql.checkData(1,1,1) + tdSql.checkData(2,1,1) + + tdSql.execute('''delete from %s.mt;'''%(database)) + tdSql.query('''select vt, val from %s.st order by vt, val asc;'''%(database)) + tdSql.checkData(0,1,1) + tdSql.checkData(1,1,2) + tdSql.checkData(2,1,1) + tdSql.checkData(3,1,2) + tdSql.checkData(4,1,1) + tdSql.checkData(5,1,2) + + tdSql.execute('''insert into %s.mt select vt, val from %s.st order by vt, val asc;'''%(database,database)) + tdSql.query("select count(*) from %s.mt;"%database) + tdSql.checkData(0,0,3) + + tdSql.query('''select ts, val from %s.mt order by ts asc;'''%(database)) + tdSql.checkData(0,1,2) + tdSql.checkData(1,1,2) + tdSql.checkData(2,1,2) + + tdSql.execute('''delete from %s.mt;'''%(database)) + tdSql.query('''select vt, val from %s.st order by ts, val asc;'''%(database)) + tdSql.checkData(0,1,1) + tdSql.checkData(1,1,2) + tdSql.checkData(2,1,1) + tdSql.checkData(3,1,2) + tdSql.checkData(4,1,1) + tdSql.checkData(5,1,2) + + tdSql.execute('''insert into %s.mt select vt, val from %s.st order by ts, val asc;'''%(database,database)) + tdSql.query("select count(*) from %s.mt;"%database) + tdSql.checkData(0,0,3) + + tdSql.query('''select ts, val from %s.mt order by ts asc;'''%(database)) + tdSql.checkData(0,1,2) + tdSql.checkData(1,1,2) + tdSql.checkData(2,1,2) + + tdSql.execute('''delete from %s.mt;'''%(database)) + ts += 1 + tdSql.execute(f'''insert into %s.t1 values({ts}, -1, {ts}) %s.t2 values({ts}, -2, {ts});'''%(database,database)) + tdSql.query('''select vt, val from %s.st order by val asc;'''%(database)) + tdSql.checkData(0,1,-2) + tdSql.checkData(1,1,-1) + tdSql.checkData(2,1,1) + tdSql.checkData(3,1,1) + tdSql.checkData(4,1,1) + tdSql.checkData(5,1,2) + tdSql.checkData(6,1,2) + tdSql.checkData(7,1,2) + + tdSql.execute('''insert into %s.mt select vt, val from %s.st order by val asc;'''%(database,database)) + tdSql.query("select count(*) from %s.mt;"%database) + tdSql.checkData(0,0,4) + + tdSql.query('''select ts, val from %s.mt order by ts asc;'''%(database)) + tdSql.checkData(0,1,2) + tdSql.checkData(1,1,2) + tdSql.checkData(2,1,2) + tdSql.checkData(3,1,-1) def run(self): @@ -88,6 +197,8 @@ class TDTestCase: self.users_bug_TD_20592("%s" %self.db) + self.use_select_sort("%s" %self.db) + #taos -f sql print("taos -f sql start!") taos_cmd1 = "taos -f %s/%s.sql" % (self.testcasePath,self.testcaseFilename) diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index ebd580efd4..8db92f38a2 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -394,9 +394,12 @@ class TDTestCase: tdSql.execute(f"create table if not exists {dbname}.jsons3(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") tdSql.execute(f"insert into {dbname}.jsons3_1 using {dbname}.jsons3 tags('{{\"tag1\":\"fff\",\"tag2\":5, \"tag3\":true}}') values(1591060618000, 3, false, 'json3', '你是3')") tdSql.execute(f"insert into {dbname}.jsons3_2 using {dbname}.jsons3 tags('{{\"tag1\":5,\"tag2\":\"beijing\"}}') values (1591060638000, 2, true, 'json3', 'sss')") + tdSql.execute(f"insert into {dbname}.jsons3_3 using {dbname}.jsons3 tags(NULL) values (1591060638000, 2, true, 'json3', 'sss')") tdSql.query(f"select 'sss',33,a.jtag->'tag3' from {dbname}.jsons2 a,jsons3 b where a.ts=b.ts and a.jtag->'tag1'=b.jtag->'tag1'") tdSql.checkData(0, 0, "sss") tdSql.checkData(0, 2, "true") + tdSql.query(f"show create table jsons3_3") + tdSql.checkNotEqual(tdSql.queryResult[0][1].find("TAGS (null)"), 0) res = tdSql.getColNameList(f"select 'sss',33,a.jtag->'tag3' from {dbname}.jsons2 a,jsons3 b where a.ts=b.ts and a.jtag->'tag1'=b.jtag->'tag1'") cname_list = [] diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 0f0936ebab..01795f6eef 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -1,3 +1,4 @@ +from sqlite3 import ProgrammingError import taos import sys import time @@ -319,10 +320,144 @@ class TDTestCase: tdSql.checkData(0, 0, '2018-11-25 19:30:00.000') tdSql.checkData(0, 1, '2018-11-25 19:30:01.000') + def test_cache_scan_with_drop_and_add_column(self): + tdSql.query("select last(c10) from meters") + tdSql.checkData(0, 0, '2018-11-25 19:30:01') + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c10; alter table test.meters add column c11 int"]) + p.check_returncode() + tdSql.query("select last(c10) from meters", queryTimes=1) + tdSql.checkData(0, 0, None) + tdSql.query('select last(*) from meters', queryTimes=1) + tdSql.checkData(0, 10, None) + tdSql.query('select last(c10), c10, ts from meters', queryTimes=1) + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + + def test_cache_scan_with_drop_and_add_column2(self): + tdSql.query("select last(c1) from meters") + tdSql.checkData(0, 0, '999') + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c12 int"]) + p.check_returncode() + tdSql.query("select last(c1) from meters", queryTimes=1) + tdSql.checkData(0, 0, None) + tdSql.query('select last(*) from meters', queryTimes=1) + print(str(tdSql.queryResult)) + tdSql.checkData(0, 1, None) + tdSql.query('select last(c1), c1, ts from meters', queryTimes=1) + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + + try: + tdSql.query('select ts, last(c1), c1, ts, c1 from meters', queryTimes=1) + except Exception as e: + if str(e).count('Invalid column name') == 1: + print('column has been dropped, the cache has been updated: %s' % (str(e))) + return + else: + raise + tdSql.checkRows(1) + tdSql.checkCols(5) + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(0, 3, None) + tdSql.checkData(0, 4, None) + + try: + tdSql.query('select last(c1), last(c2), last(c3) from meters', queryTimes=1) + except Exception as e: + if str(e).count('Invalid column name') == 1: + print('column has been dropped, the cache has been updated: %s' % (str(e))) + return + else: + raise + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, None) + + def test_cache_scan_with_drop_column(self): + tdSql.query('select last(*) from meters') + print(str(tdSql.queryResult)) + tdSql.checkCols(11) + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c9"]) + p.check_returncode() + tdSql.query('select last(*) from meters') + print(str(tdSql.queryResult)) + tdSql.checkCols(11) + tdSql.checkData(0, 9, None) + + def test_cache_scan_last_row_with_drop_column(self): + tdSql.query('select last_row(*) from meters') + print(str(tdSql.queryResult)) + tdSql.checkCols(11) + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c10; alter table test.meters add column c11 int"]) + p.check_returncode() + tdSql.query('select last_row(*) from meters') + print(str(tdSql.queryResult)) + tdSql.checkCols(11) + tdSql.checkData(0, 10, None) + + def test_cache_scan_last_row_with_drop_column2(self): + tdSql.query('select last_row(c1) from meters') + print(str(tdSql.queryResult)) + tdSql.checkCols(1) + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c11 int"]) + p.check_returncode() + tdSql.query('select last_row(c1) from meters', queryTimes=1) + print(str(tdSql.queryResult)) + tdSql.checkCols(1) + tdSql.checkData(0, 0, None) + + def test_cache_scan_last_row_with_partition_by(self): + tdSql.query('select last(c1) from meters partition by t1') + print(str(tdSql.queryResult)) + tdSql.checkCols(1) + tdSql.checkRows(5) + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c11 int"]) + p.check_returncode() + tdSql.query('select last_row(c1) from meters partition by t1', queryTimes=1) + print(str(tdSql.queryResult)) + tdSql.checkCols(1) + tdSql.checkRows(5) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + + def test_cache_scan_last_row_with_partition_by_tbname(self): + tdSql.query('select last(c1) from meters partition by tbname', queryTimes=1) + print(str(tdSql.queryResult)) + tdSql.checkCols(1) + tdSql.checkRows(10) + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c11 int"]) + p.check_returncode() + tdSql.query('select last_row(c1) from meters partition by tbname', queryTimes=1) + print(str(tdSql.queryResult)) + tdSql.checkCols(1) + tdSql.checkRows(10) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + + + def run(self): self.prepareTestEnv() #time.sleep(99999999) self.test_last_cache_scan() + #self.test_cache_scan_with_drop_and_add_column() + self.test_cache_scan_with_drop_and_add_column2() + #self.test_cache_scan_with_drop_column() + #self.test_cache_scan_last_row_with_drop_column() + #self.test_cache_scan_last_row_with_drop_column2() + #self.test_cache_scan_last_row_with_partition_by() + #self.test_cache_scan_last_row_with_partition_by_tbname() def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index 0369f45723..e28f3b1edd 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -80,12 +80,12 @@ class TDTestCase: tdSql.checkData(0, 1, 13.000000000) tdSql.checkData(0, 2, "web01") tdSql.checkData(0, 3, None) - tdSql.checkData(0, 4, "lga") + tdSql.checkData(0, 4, 1) tdSql.checkData(1, 1, 9.000000000) tdSql.checkData(1, 2, "web02") tdSql.checkData(3, 3, "t1") - tdSql.checkData(0, 4, "lga") + tdSql.checkData(2, 4, 4) tdSql.query(f"select * from {dbname}.macylr") tdSql.checkRows(2) diff --git a/tests/system-test/2-query/sml_TS-3724.py b/tests/system-test/2-query/sml_TS-3724.py index 410e266f10..b537ad9b9a 100644 --- a/tests/system-test/2-query/sml_TS-3724.py +++ b/tests/system-test/2-query/sml_TS-3724.py @@ -85,12 +85,12 @@ class TDTestCase: tdSql.checkData(0, 1, 13.000000000) tdSql.checkData(0, 2, "web01") tdSql.checkData(0, 3, None) - tdSql.checkData(0, 4, "lga") + tdSql.checkData(0, 4, 1) tdSql.checkData(1, 1, 9.000000000) tdSql.checkData(1, 2, "web02") tdSql.checkData(3, 3, "t1") - tdSql.checkData(0, 4, "lga") + tdSql.checkData(2, 4, 4) tdSql.query(f"select * from {dbname}.macylr") tdSql.checkRows(2) diff --git a/tests/system-test/2-query/state_window.py b/tests/system-test/2-query/state_window.py new file mode 100644 index 0000000000..a211cd9dbe --- /dev/null +++ b/tests/system-test/2-query/state_window.py @@ -0,0 +1,203 @@ +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +# from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 4 + self.ctbNum = 1 + self.rowsPerTbl = 10 + self.duraion = '1h' + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): + if dropFlag == 1: + tsql.execute("drop database if exists %s"%(dbName)) + + tsql.execute("create database if not exists %s vgroups %d replica %d duration %s"%(dbName, vgroups, replica, duration)) + tdLog.debug("complete to create database %s"%(dbName)) + return + + def create_stable(self,tsql, paraDict): + colString = tdCom.gen_column_type_str(colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"]) + tagString = tdCom.gen_tag_type_str(tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"]) + sqlString = f"create table if not exists %s.%s (%s) tags (%s)"%(paraDict["dbName"], paraDict["stbName"], colString, tagString) + tdLog.debug("%s"%(sqlString)) + tsql.execute(sqlString) + return + + def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1,ctbStartIdx=0): + for i in range(ctbNum): + sqlString = "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)" % \ + (dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,(i+ctbStartIdx) % 5,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx) + tsql.execute(sqlString) + + tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName)) + return + + def insert_data(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs,tsStep): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + for i in range(ctbNum): + rowsBatched = 0 + sql += " %s%d values "%(ctbPrefix,i) + for j in range(rowsPerTbl): + if (i < ctbNum/2): + sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, 1, j%10, j%10, j%10, j%10, j%10, j%10) + else: + sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10) + rowsBatched += 1 + if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + rowsBatched = 0 + if j < rowsPerTbl - 1: + sql = "insert into %s%d values " %(ctbPrefix,i) + else: + sql = "insert into " + if sql != pre_insert: + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'test', + 'dropFlag': 1, + 'vgroups': 2, + 'stbName': 'meters', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'FLOAT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'smallint', 'count':1},{'type': 'tinyint', 'count':1},{'type': 'bool', 'count':1},{'type': 'binary', 'len':10, 'count':1},{'type': 'nchar', 'len':10, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'nchar', 'len':20, 'count':1},{'type': 'binary', 'len':20, 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'smallint', 'count':1},{'type': 'DOUBLE', 'count':1}], + 'ctbPrefix': 't', + 'ctbStartIdx': 0, + 'ctbNum': 100, + 'rowsPerTbl': 10000, + 'batchNum': 3000, + 'startTs': 1537146000000, + 'tsStep': 600000} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdLog.info("create database") + self.create_database(tsql=tdSql, dbName=paraDict["dbName"], dropFlag=paraDict["dropFlag"], vgroups=paraDict["vgroups"], replica=self.replicaVar, duration=self.duraion) + + tdLog.info("create stb") + self.create_stable(tsql=tdSql, paraDict=paraDict) + + tdLog.info("create child tables") + self.create_ctable(tsql=tdSql, dbName=paraDict["dbName"], \ + stbName=paraDict["stbName"],ctbPrefix=paraDict["ctbPrefix"],\ + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict["ctbStartIdx"]) + self.insert_data(tsql=tdSql, dbName=paraDict["dbName"],\ + ctbPrefix=paraDict["ctbPrefix"],ctbNum=paraDict["ctbNum"],\ + rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],\ + startTs=paraDict["startTs"],tsStep=paraDict["tsStep"]) + return + + def prepare_original_data(self): + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 3,3,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("flush database test", queryTimes=1) + time.sleep(2) + + def test_crash_for_state_window1(self): + tdSql.execute("drop database if exists test") + self.prepareTestEnv() + tdSql.execute("alter local 'queryPolicy' '3'") + self.prepare_original_data() + tdSql.execute("insert into t0 values(now, 4,4,4,4,4,4,4,4,4)", queryTimes=1) + tdSql.execute("select bottom(c1, 1), c2 from t0 state_window(c2) order by ts", queryTimes=1) + + def test_crash_for_state_window2(self): + tdSql.execute("drop database if exists test") + self.prepareTestEnv() + tdSql.execute("alter local 'queryPolicy' '3'") + self.prepare_original_data() + tdSql.execute("insert into t0 values(now, 4,NULL,4,4,4,4,4,4,4)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 4,4,4,4,4,4,4,4,4)", queryTimes=1) + tdSql.execute("select bottom(c1, 1), c2 from t0 state_window(c2) order by ts", queryTimes=1) + + def test_crash_for_state_window3(self): + tdSql.execute("drop database if exists test") + self.prepareTestEnv() + tdSql.execute("alter local 'queryPolicy' '3'") + self.prepare_original_data() + tdSql.execute("insert into t0 values(now, 4,NULL,4,4,4,4,4,4,4)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 4,5,4,4,4,4,4,4,4)", queryTimes=1) + tdSql.execute("select bottom(c1, 1), c2 from t0 state_window(c2) order by ts", queryTimes=1) + + def test_crash_for_state_window4(self): + tdSql.execute("drop database if exists test") + self.prepareTestEnv() + tdSql.execute("alter local 'queryPolicy' '3'") + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 3,3,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 3,NULL,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 3,NULL,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("flush database test", queryTimes=1) + time.sleep(2) + tdSql.execute("insert into t0 values(now, 3,NULL,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("select bottom(c1, 1), c2 from t0 state_window(c2) order by ts", queryTimes=1) + + def test_crash_for_state_window5(self): + tdSql.execute("drop database if exists test") + self.prepareTestEnv() + tdSql.execute("alter local 'queryPolicy' '3'") + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 3,3,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 3,NULL,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 3,NULL,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("flush database test", queryTimes=1) + time.sleep(2) + tdSql.execute("insert into t0 values(now, 3,NULL,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 3,3,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("select bottom(c1, 1), c2 from t0 state_window(c2) order by ts", queryTimes=1) + + def run(self): + self.test_crash_for_state_window1() + self.test_crash_for_state_window2() + self.test_crash_for_state_window3() + self.test_crash_for_state_window4() + self.test_crash_for_state_window5() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/timezone.py b/tests/system-test/2-query/timezone.py index 316e776cb1..33bcb16595 100644 --- a/tests/system-test/2-query/timezone.py +++ b/tests/system-test/2-query/timezone.py @@ -112,10 +112,59 @@ class TDTestCase: self.data_check(timezone,self.stbname,'stable') for i in range(self.tbnum): self.data_check(timezone,f'{self.stbname}_{i}','child_table') + tdSql.execute(f'drop database {self.dbname}') + + def timezone_format_test(self): + tdSql.execute(f'create database {self.dbname}') + tdSql.execute(self.setsql.set_create_stable_sql(f'{self.dbname}.stb', {'ts':'timestamp','id':'int'}, {'status':'int'})) + + tdSql.execute(f"insert into {self.dbname}.d0 using {self.dbname}.stb tags (1) values ('2021-07-01 00:00:00.000',0);") + tdSql.query(f"select ts from {self.dbname}.d0;") + tdSql.checkData(0, 0, "2021-07-01 00:00:00.000") + + tdSql.execute(f"insert into {self.dbname}.d1 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000+07:50',1)") + tdSql.query(f"select ts from {self.dbname}.d1") + tdSql.checkData(0, 0, "2021-07-01 00:10:00.000") + + tdSql.execute(f"insert into {self.dbname}.d2 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000+12:00',1)") + tdSql.query(f"select ts from {self.dbname}.d2") + tdSql.checkData(0, 0, "2021-06-30 20:00:00.000") + + tdSql.execute(f"insert into {self.dbname}.d3 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-00:10',1)") + tdSql.query(f"select ts from {self.dbname}.d3") + tdSql.checkData(0, 0, "2021-07-01 08:10:00.000") + + tdSql.execute(f"insert into {self.dbname}.d4 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-12:00',1)") + tdSql.query(f"select ts from {self.dbname}.d4") + tdSql.checkData(0, 0, "2021-07-01 20:00:00.000") + + tdSql.execute(f"insert into {self.dbname}.d5 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-1200',1)") + tdSql.query(f"select ts from {self.dbname}.d5") + tdSql.checkData(0, 0, "2021-07-01 20:00:00.000") + + tdSql.execute(f"insert into {self.dbname}.d6 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-115',1)") + tdSql.query(f"select ts from {self.dbname}.d6") + tdSql.checkData(0, 0, "2021-07-01 19:05:00.000") + + tdSql.execute(f"insert into {self.dbname}.d7 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-1105',1)") + tdSql.query(f"select ts from {self.dbname}.d7") + tdSql.checkData(0, 0, "2021-07-01 19:05:00.000") + + tdSql.error(f"insert into {self.dbname}.d21 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000+12:10',1)") + tdSql.error(f"insert into {self.dbname}.d22 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-24:10',1)") + tdSql.error(f"insert into {self.dbname}.d23 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000+12:10',1)") + tdSql.error(f"insert into {self.dbname}.d24 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-24:10',1)") + tdSql.error(f"insert into {self.dbname}.d24 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-24100',1)") + tdSql.error(f"insert into {self.dbname}.d24 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-1210',1)") + + tdSql.execute(f'drop database {self.dbname}') + + def run(self): # sourcery skip: extract-duplicate-method timezone = self.get_system_timezone() self.timezone_check_ntb(timezone) self.timezone_check_stb(timezone) + self.timezone_format_test() def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeRoll.py b/tests/system-test/6-cluster/5dnode3mnodeRoll.py index 9d62eb3b4b..11a153c48f 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRoll.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRoll.py @@ -4,7 +4,7 @@ import taos import sys import time import os - +import platform from util.log import * from util.sql import * from util.cases import * @@ -96,7 +96,10 @@ class TDTestCase: packagePath = "/usr/local/src/" dataPath = cPath + "/../data/" - packageName = "TDengine-server-"+ BASEVERSION + "-Linux-x64.tar.gz" + if platform.system() == "Linux" and platform.machine() == "aarch64": + packageName = "TDengine-server-"+ BASEVERSION + "-Linux-arm64.tar.gz" + else: + packageName = "TDengine-server-"+ BASEVERSION + "-Linux-x64.tar.gz" packageTPath = packageName.split("-Linux-")[0] my_file = Path(f"{packagePath}/{packageName}") if not my_file.exists(): diff --git a/tests/system-test/7-tmq/raw_block_interface_test.py b/tests/system-test/7-tmq/raw_block_interface_test.py index 1e89de1cce..1c9798421d 100644 --- a/tests/system-test/7-tmq/raw_block_interface_test.py +++ b/tests/system-test/7-tmq/raw_block_interface_test.py @@ -36,7 +36,21 @@ class TDTestCase: buildPath = tdCom.getBuildPath() cmdStr = '%s/build/bin/write_raw_block_test'%(buildPath) tdLog.info(cmdStr) - os.system(cmdStr) + retCode = os.system(cmdStr) + # run program code from system return , 0 is success + runCode = retCode & 0xFF + # program retur code from main function + progCode = retCode >> 8 + + tdLog.info(f"{cmdStr} ret={retCode} runCode={runCode} progCode={progCode}") + + if runCode != 0: + tdLog.exit(f"run {cmdStr} failed, have system error.") + return + + if progCode != 0: + tdLog.exit(f"{cmdStr} found problem, return code = {progCode}.") + return self.checkData() diff --git a/tests/system-test/7-tmq/tmqDnodeRestart.py b/tests/system-test/7-tmq/tmqDnodeRestart.py index 74aba31726..0ac8482163 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart.py @@ -4,6 +4,7 @@ import sys import time import socket import os +import platform import threading from enum import Enum @@ -184,6 +185,9 @@ class TDTestCase: paraDict['vgroups'] = self.vgroups paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl + # ARM64：time cost is so long for stopping taosd, so add the pollDdelay to 120s + if platform.system() == "Linux" and platform.machine() == "aarch64": + paraDict['pollDelay'] = 300 tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-column.py b/tests/system-test/7-tmq/tmqVnodeSplit-column.py index 54a43465e7..1fe2b5809a 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-column.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-column.py @@ -23,7 +23,7 @@ class TDTestCase: def __init__(self): self.vgroups = 1 self.ctbNum = 10 - self.rowsPerTbl = 10000 + self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -49,7 +49,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -118,7 +118,7 @@ class TDTestCase: 'ctbPrefix': 'ctb1', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -188,7 +188,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0]: + if expectrowcnt / 2 > resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-db.py b/tests/system-test/7-tmq/tmqVnodeSplit-db.py index e4353d3268..f66acf4fcd 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-db.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-db.py @@ -23,7 +23,7 @@ class TDTestCase: def __init__(self): self.vgroups = 1 self.ctbNum = 10 - self.rowsPerTbl = 10000 + self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -49,7 +49,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -118,7 +118,7 @@ class TDTestCase: 'ctbPrefix': 'ctb1', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -189,7 +189,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0]: + if expectrowcnt / 2 > resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py index 8276ae638b..68fb07b813 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py @@ -25,7 +25,7 @@ class TDTestCase: def __init__(self): self.vgroups = 1 self.ctbNum = 10 - self.rowsPerTbl = 10000 + self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -51,7 +51,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -120,7 +120,7 @@ class TDTestCase: 'ctbPrefix': 'ctb1', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 120, @@ -189,7 +189,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0]: + if expectrowcnt / 2 > resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py index 0d247b2848..6140e8a544 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py @@ -25,7 +25,7 @@ class TDTestCase: def __init__(self): self.vgroups = 1 self.ctbNum = 10 - self.rowsPerTbl = 10000 + self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -51,7 +51,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -120,7 +120,7 @@ class TDTestCase: 'ctbPrefix': 'ctb1', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 120, @@ -189,7 +189,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0]: + if expectrowcnt / 2 > resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py index cda5a27919..18b80b7f8d 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py @@ -27,7 +27,7 @@ class TDTestCase: def __init__(self): self.vgroups = 1 self.ctbNum = 10 - self.rowsPerTbl = 10000 + self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -53,7 +53,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -122,7 +122,7 @@ class TDTestCase: 'ctbPrefix': 'ctb1', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 120, @@ -192,7 +192,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0]: + if expectrowcnt / 2 > resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb.py index 17a427567e..c203350322 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb.py @@ -25,7 +25,7 @@ class TDTestCase: def __init__(self): self.vgroups = 1 self.ctbNum = 10 - self.rowsPerTbl = 10000 + self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -51,7 +51,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -120,7 +120,7 @@ class TDTestCase: 'ctbPrefix': 'ctb1', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -190,7 +190,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0]: + if expectrowcnt / 2 > resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-db.py b/tests/system-test/7-tmq/tmqVnodeTransform-db.py index 005bca70d6..5c61908d96 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-db.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-db.py @@ -20,7 +20,7 @@ class TDTestCase: def __init__(self): self.vgroups = 1 self.ctbNum = 10 - self.rowsPerTbl = 10000 + self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -46,7 +46,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 30, @@ -138,7 +138,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 10, @@ -217,7 +217,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 10, diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py index ec1331ae59..64cdf6d153 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py @@ -20,7 +20,7 @@ class TDTestCase: def __init__(self): self.vgroups = 1 self.ctbNum = 10 - self.rowsPerTbl = 10000 + self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -46,7 +46,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -137,7 +137,7 @@ class TDTestCase: 'ctbPrefix': 'ctb1', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -207,7 +207,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0]: + if expectrowcnt / 2 > resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) diff --git a/tests/system-test/7-tmq/tmqVnodeTransform.py b/tests/system-test/7-tmq/tmqVnodeTransform.py index aab94bc7a2..3698297618 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform.py @@ -20,7 +20,7 @@ class TDTestCase: def __init__(self): self.vgroups = 1 self.ctbNum = 10 - self.rowsPerTbl = 10000 + self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -46,7 +46,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -137,7 +137,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -203,7 +203,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0]: + if expectrowcnt / 2 > resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) diff --git a/tests/system-test/8-stream/snode_restart_with_checkpoint.py b/tests/system-test/8-stream/snode_restart_with_checkpoint.py index 9567bbe439..d7bfd7b407 100644 --- a/tests/system-test/8-stream/snode_restart_with_checkpoint.py +++ b/tests/system-test/8-stream/snode_restart_with_checkpoint.py @@ -31,7 +31,7 @@ class TDTestCase: tdSql.query("use test") tdSql.query("create snode on dnode 4") tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s)") - tdLog.debug("========create stream useing snode and insert data ok========") + tdLog.debug("========create stream using snode and insert data ok========") time.sleep(60) tdDnodes = cluster.dnodes diff --git a/tests/system-test/8-stream/stream_basic.py b/tests/system-test/8-stream/stream_basic.py new file mode 100644 index 0000000000..7f4d1d5ee3 --- /dev/null +++ b/tests/system-test/8-stream/stream_basic.py @@ -0,0 +1,110 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +from util.autogen import * + +import random +import time +import traceback +import os +from os import path + + +class TDTestCase: + # init + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + + # autoGen + self.autoGen = AutoGen() + + def waitTranslation(self, waitSeconds): + # wait end + for i in range(waitSeconds): + sql ="show transactions;" + rows = tdSql.query(sql) + if rows == 0: + return True + tdLog.info(f"i={i} wait for translation finish ...") + time.sleep(1) + + return False + + def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + paths = [] + for root, dirs, files in os.walk(projPath): + if ((tool) in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + paths.append(os.path.join(root, tool)) + break + if (len(paths) == 0): + tdLog.exit("taosBenchmark not found!") + return + else: + tdLog.info("taosBenchmark found in %s" % paths[0]) + return paths[0] + + def taosBenchmark(self, param): + binPath = self.getPath() + cmd = f"{binPath} {param}" + tdLog.info(cmd) + os.system(cmd) + + # run + def run(self): + # gen data + random.seed(int(time.time())) + self.taosBenchmark(" -d db -t 2 -v 2 -n 1000000 -y") + # create stream + tdSql.execute("use db") + tdSql.execute("create stream stream1 fill_history 1 into sta as select count(*) as cnt from meters interval(10a);",show=True) + sql = "select count(*) from sta" + # loop wait max 60s to check count is ok + tdLog.info("loop wait result ...") + tdSql.checkDataLoop(0, 0, 99999, sql, loopCount=120, waitTime=0.5) + + # check all data is correct + sql = "select * from sta where cnt != 20;" + tdSql.query(sql) + tdSql.checkRows(0) + + # check ts interval is correct + sql = "select * from ( select diff(_wstart) as tsdif from sta ) where tsdif != 10;" + tdSql.query(sql) + tdSql.checkRows(0) + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/pytest.sh b/tests/system-test/pytest.sh index dfdfc32ed0..2c95516d10 100755 --- a/tests/system-test/pytest.sh +++ b/tests/system-test/pytest.sh @@ -79,7 +79,8 @@ else unset LD_PRELOAD #export LD_PRELOAD=libasan.so.5 - export LD_PRELOAD=$(gcc -print-file-name=libasan.so) + #export LD_PRELOAD=$(gcc -print-file-name=libasan.so) + export LD_PRELOAD="$(realpath "$(gcc -print-file-name=libasan.so)") $(realpath "$(gcc -print-file-name=libstdc++.so)")" echo "Preload AsanSo:" $? $* -a 2>$AsanFile @@ -104,4 +105,4 @@ else echo "Execute script failure" exit 1 fi -fi \ No newline at end of file +fi diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 115abdcd36..e8a5b04178 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1091,7 +1091,12 @@ void shellSourceFile(const char *file) { char *line = taosMemoryMalloc(TSDB_MAX_ALLOWED_SQL_LEN + 1); while ((read_len = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) != -1) { - if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue; + if ( cmd_len + read_len >= TSDB_MAX_ALLOWED_SQL_LEN) { + printf("read command line too long over 1M, ignore this line. cmd_len = %d read_len=%d \n", (int32_t)cmd_len, read_len); + cmd_len = 0; + memset(line, 0, TSDB_MAX_ALLOWED_SQL_LEN + 1); + continue; + } line[--read_len] = '\0'; if (read_len == 0 || shellIsCommentLine(line)) { // line starts with # diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 9153706d23..2c334eb67b 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -116,8 +116,7 @@ int smlProcess_json1_Test() { const char *sql[] = { "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":0,\"value\":18,\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":" - "\"lga\"}},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":" - "\"lga\"}}]"}; + "34}},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":4}}]"}; char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { @@ -142,8 +141,8 @@ int smlProcess_json1_Test() { const char *sql2[] = { - "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":\"web01\",\"dc\":\"lga\"}" - "},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":\"lga\"}" + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":\"web01\",\"dc\":1}" + "},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":4}" "}]", }; @@ -270,6 +269,98 @@ int smlProcess_json3_Test() { return code; } +int smlProcess_json_tag_not_same_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "[{\"metric\":\"jstable\",\"timestamp\":0,\"value\":18,\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":" + "\"lga\"}},{\"metric\":\"jstable\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":" + "\"lga\"}}]"}; + + char *sql1[1] = {0}; + for (int i = 0; i < 1; i++) { + sql1[i] = taosMemoryCalloc(1, 1024); + strncpy(sql1[i], sql[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + int code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } + taos_free_result(pRes); + + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql1[i]); + } + ASSERT(code == 0); + + + const char *sql2[] = { + "[{\"metric\":\"jstable\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":6,\"dc\":\"lga\"}}]", + }; + + char *sql3[1] = {0}; + for (int i = 0; i < 1; i++) { + sql3[i] = taosMemoryCalloc(1, 1024); + strncpy(sql3[i], sql2[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } + taos_free_result(pRes); + + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql3[i]); + } + + ASSERT(code != 0); + + const char *sql4[] = { + "[{\"metric\":\"jstable\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":6,\"dc\":\"lga\"}}," + "{\"metric\":\"jstable\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":false,\"dc\":\"lga\"}}]", + }; + char *sql5[1] = {0}; + for (int i = 0; i < 1; i++) { + sql5[i] = taosMemoryCalloc(1, 1024); + strncpy(sql5[i], sql4[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql5, sizeof(sql5) / sizeof(sql5[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } + taos_free_result(pRes); + + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql5[i]); + } + ASSERT(code != 0); + + taos_close(taos); + + return code; +} + int sml_TD15662_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -1729,6 +1820,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_19221_Test(); ASSERT(!ret); + ret = smlProcess_json_tag_not_same_Test(); + ASSERT(ret); ret = sml_ts3724_Test(); ASSERT(!ret); diff --git a/utils/test/c/write_raw_block_test.c b/utils/test/c/write_raw_block_test.c index ee2594af7a..162ecd229c 100644 --- a/utils/test/c/write_raw_block_test.c +++ b/utils/test/c/write_raw_block_test.c @@ -19,8 +19,8 @@ #include "taos.h" #include "types.h" -int buildStable(TAOS* pConn, TAOS_RES* pRes) { - pRes = taos_query(pConn, +int buildStable(TAOS* pConn) { + TAOS_RES* pRes = taos_query(pConn, "CREATE STABLE `meters` (`ts` TIMESTAMP, `current` INT, `voltage` INT, `phase` FLOAT) TAGS " "(`groupid` INT, `location` VARCHAR(16))"); if (taos_errno(pRes) != 0) { @@ -57,6 +57,34 @@ int buildStable(TAOS* pConn, TAOS_RES* pRes) { } taos_free_result(pRes); + pRes = taos_query(pConn, "create table ntba(ts timestamp, addr binary(32))"); + if (taos_errno(pRes) != 0) { + printf("failed to create ntba, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table ntbb(ts timestamp, addr binary(8))"); + if (taos_errno(pRes) != 0) { + printf("failed to create ntbb, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ntba values(now,'123456789abcdefg123456789')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert table ntba, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ntba values(now,'hello')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert table ntba, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + return 0; } @@ -65,40 +93,43 @@ int32_t init_env() { if (pConn == NULL) { return -1; } + int32_t ret = -1; TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_raw"); if (taos_errno(pRes) != 0) { printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); - return -1; + goto END; } taos_free_result(pRes); pRes = taos_query(pConn, "create database if not exists db_raw vgroups 2"); if (taos_errno(pRes) != 0) { printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); - return -1; + goto END; } taos_free_result(pRes); pRes = taos_query(pConn, "use db_raw"); if (taos_errno(pRes) != 0) { printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); - return -1; + goto END; } taos_free_result(pRes); - buildStable(pConn, pRes); + buildStable(pConn); pRes = taos_query(pConn, "select * from d0"); if (taos_errno(pRes) != 0) { printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); - return -1; + goto END; } void *data = NULL; int32_t numOfRows = 0; int error_code = taos_fetch_raw_block(pRes, &numOfRows, &data); - ASSERT(error_code == 0); - ASSERT(numOfRows == 1); + if(error_code !=0 ){ + printf("error fetch raw block, reason:%s\n", taos_errstr(pRes)); + goto END; + } taos_write_raw_block(pConn, numOfRows, data, "d1"); taos_free_result(pRes); @@ -106,23 +137,78 @@ int32_t init_env() { pRes = taos_query(pConn, "select ts,phase from d0"); if (taos_errno(pRes) != 0) { printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); - return -1; + goto END; } error_code = taos_fetch_raw_block(pRes, &numOfRows, &data); - ASSERT(error_code == 0); - ASSERT(numOfRows == 1); + if(error_code !=0 ){ + printf("error fetch raw block, reason:%s\n", taos_errstr(pRes)); + goto END; + } int numFields = taos_num_fields(pRes); TAOS_FIELD *fields = taos_fetch_fields(pRes); taos_write_raw_block_with_fields(pConn, numOfRows, data, "d2", fields, numFields); taos_free_result(pRes); - taos_close(pConn); - return 0; + // check error msg + pRes = taos_query(pConn, "select * from ntba"); + if (taos_errno(pRes) != 0) { + printf("error select * from ntba, reason:%s\n", taos_errstr(pRes)); + goto END; + } + + data = NULL; + numOfRows = 0; + error_code = taos_fetch_raw_block(pRes, &numOfRows, &data); + if(error_code !=0 ){ + printf("error fetch select * from ntba, reason:%s\n", taos_errstr(pRes)); + goto END; + } + error_code = taos_write_raw_block(pConn, numOfRows, data, "ntbb"); + if(error_code == 0) { + printf(" taos_write_raw_block to ntbb expect failed , but success!\n"); + goto END; + } + + // pass NULL return last error code describe + const char* err = taos_errstr(NULL); + printf("write_raw_block return code =0x%x err=%s\n", error_code, err); + if(strcmp(err, "success") == 0) { + printf("expect failed , but error string is success! err=%s\n", err); + goto END; + } + + // no exist table + error_code = taos_write_raw_block(pConn, numOfRows, data, "no-exist-table"); + if(error_code == 0) { + printf(" taos_write_raw_block to no-exist-table expect failed , but success!\n"); + goto END; + } + + err = taos_errstr(NULL); + printf("write_raw_block no exist table return code =0x%x err=%s\n", error_code, err); + if(strcmp(err, "success") == 0) { + printf("expect failed write no exist table, but error string is success! err=%s\n", err); + goto END; + } + + // success + ret = 0; + +END: + // free + if(pRes) taos_free_result(pRes); + if(pConn) taos_close(pConn); + return ret; } int main(int argc, char* argv[]) { - if (init_env() < 0) { - return -1; + printf("test write_raw_block...\n"); + int ret = init_env(); + if (ret < 0) { + printf("test write_raw_block failed.\n"); + return ret; } -} + printf("test write_raw_block ok.\n"); + return 0; +} \ No newline at end of file