diff --git a/docs/en/08-client-libraries/04-java.mdx b/docs/en/08-client-libraries/04-java.mdx
index 8aa4a8994e..7b9aa35e4a 100644
--- a/docs/en/08-client-libraries/04-java.mdx
+++ b/docs/en/08-client-libraries/04-java.mdx
@@ -378,7 +378,7 @@ The configuration parameters in properties are as follows.
- TSDBDriver.PROPERTY_KEY_DISABLE_SSL_CERT_VALIDATION: Whether to disable SSL certification validation. It only takes effect when using Websocket connections. true: enabled, false: disabled. The default is false.
-For JDBC native connections, you can specify other parameters, such as log level, SQL length, etc., by specifying URL and Properties. For more detailed configuration, please refer to [Client Configuration](../../reference/config/#configuration-file-on-client-side).
+For JDBC native connections, you can specify other parameters, such as log level, SQL length, etc., by specifying URL and Properties. For more detailed configuration, please refer to [Client Configuration](../../reference/config/).
### Priority of configuration parameters
diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md
index 2d6c1469b6..8bfdaeb9c8 100755
--- a/docs/en/12-taos-sql/06-select.md
+++ b/docs/en/12-taos-sql/06-select.md
@@ -444,7 +444,7 @@ FROM temp_ctable t1 LEFT ASOF JOIN temp_stable t2
ON t1.ts = t2.ts AND t1.deviceid = t2.deviceid;
```
-For more information about JOIN operations, please refer to the page [TDengine Join] (../join).
+For more information about JOIN operations, please refer to the page [TDengine Join](../join).
## Nested Query
@@ -459,7 +459,7 @@ SELECT ... FROM (SELECT ... FROM ...) ...;
:::info
- The result of a nested query is returned as a virtual table used by the outer query. It's recommended to give an alias to this table for the convenience of using it in the outer query.
-- Outer queries support directly referencing columns or pseudo-columns of inner queries in the form of column names or `column names`.
+- Outer queries support directly referencing columns or pseudo-columns of inner queries in the form of column names or \`column names\`.
- JOIN operation is allowed between tables/STables inside both inner and outer queries. Join operation can be performed on the result set of the inner query.
- The features that can be used in the inner query are the same as those that can be used in a non-nested query.
- `ORDER BY` inside the inner query is unnecessary and will slow down the query performance significantly. It is best to avoid the use of `ORDER BY` inside the inner query.
diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md
index afcebcb818..e130442279 100644
--- a/docs/en/12-taos-sql/10-function.md
+++ b/docs/en/12-taos-sql/10-function.md
@@ -1209,27 +1209,40 @@ ignore_negative: {
### DIFF
```sql
-DIFF(expr [, ignore_negative])
+DIFF(expr [, ignore_option])
-ignore_negative: {
+ignore_option: {
0
| 1
+ | 2
+ | 3
}
```
-**Description**: The different of each row with its previous row for a specific column. `ignore_negative` can be specified as 0 or 1, the default value is 1 if it's not specified. `1` means negative values are ignored. For tables with composite primary key, the data with the smallest primary key value is used to calculate the difference.
+**Description**: The difference of each row with its previous row for a specific column. `ignore_option` takes the value of 0|1|2|3, the default value is 0 if it's not specified.
+- `0` means that negative values (diff results) are not ignored and null values are not ignored
+- `1` means that negative values (diff results) are treated as null values
+- `2` means that negative values (diff results) are not ignored but null values are ignored
+- `3` means that negative values (diff results) are ignored and null values are ignored
+- For tables with composite primary key, the data with the smallest primary key value is used to calculate the difference.
-**Return value type**:Same as the data type of the column being operated upon
+**Return value type**: `bool`, `timestamp` and `integer` value type all return `int_64`, `float` type returns `double`; if the diff result overflows, it is returned as overflow.
-**Applicable data types**: Numeric
+**Applicable data types**: Numeric type, timestamp and bool type.
**Applicable table types**: standard tables and supertables
**More explanation**:
-- The number of result rows is the number of rows subtracted by one, no output for the first row
-- It can be used together with a selected column. For example: select \_rowts, DIFF() from.
-
+- diff is to calculate the difference of a specific column in current row and the **first valid data before the row**. The **first valid data before the row** refers to the most adjacent non-null value of same column with smaller timestamp.
+- The diff result of numeric type is the corresponding arithmatic difference; the timestamp is calculated based on the timestamp precision of the database; when calculating diff, `true` is treated as 1 and `false` is treated as 0
+- If the data of current row is NULL or can't find the **first valid data before the current row**, the diff result is NULL
+- When ignoring negative values (ignore_option is set to 1 or 3), if the diff result is negative, the result is set to null, and then filtered according to the null value filtering rule
+- When the diff result has an overflow, whether to ignore the negative value depends on the result of the logical operation is positive or negative. For example, the value of 9223372036854775800 - (-9223372036854775806) exceeds the range of BIGINT, and the diff result will display the overflow value -10, but it will not be ignored as a negative value
+- Single or multiple diffs can be used in a single statement, and for each diff you can specify same or different `ignore_option`. When there are multiple diffs in a single statement, when and only when all the diff results are NULL for a row and each diff's `ignore_option` is specified as ignoring NULL, the output of this row will be removed from the result set.
+- Can be used with the selected associated columns. For example: `select _rowts, DIFF()`.
+- When there is not composite primary key, if there are the same timestamps across different subtables, it will prompt "Duplicate timestamps not allowed"
+- When using with composite primary key, there may be same combination of timestamp and complete primary key across sub-tables, which row will be used depends on which row is found first, that means the result of running diff() multiple times may be different in such a case
### IRATE
diff --git a/docs/en/12-taos-sql/12-distinguished.md b/docs/en/12-taos-sql/12-distinguished.md
index 5dca92a35c..818b67db9b 100644
--- a/docs/en/12-taos-sql/12-distinguished.md
+++ b/docs/en/12-taos-sql/12-distinguished.md
@@ -80,7 +80,7 @@ These pseudocolumns occur after the aggregation clause.
`FILL` clause is used to specify how to fill when there is data missing in any window, including:
1. NONE: No fill (the default fill mode)
-2. VALUE: Fill with a fixed value, which should be specified together, for example `FILL(VALUE, 1.23)` Note: The value filled depends on the data type. For example, if you run FILL(VALUE 1.23) on an integer column, the value 1 is filled.
+2. VALUE: Fill with a fixed value, which should be specified together, for example `FILL(VALUE, 1.23)` Note: The value filled depends on the data type. For example, if you run FILL(VALUE 1.23) on an integer column, the value 1 is filled. If multiple columns in select list need to be filled, then in the fill clause there must be a fill value for each of these columns, for example, `SELECT _wstart, min(c1), max(c1) FROM ... FILL(VALUE, 0, 0)`.
3. PREV: Fill with the previous non-NULL value, `FILL(PREV)`
4. NULL: Fill with NULL, `FILL(NULL)`
5. LINEAR: Fill with the closest non-NULL value, `FILL(LINEAR)`
diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md
index a6759da858..9c6c57ba6a 100644
--- a/docs/en/12-taos-sql/14-stream.md
+++ b/docs/en/12-taos-sql/14-stream.md
@@ -30,7 +30,7 @@ subquery: SELECT [DISTINCT] select_list
from_clause
[WHERE condition]
[PARTITION BY tag_list]
- [window_clause]
+ window_clause
```
Session windows, state windows, and sliding windows are supported. When you configure a session or state window for a supertable, you must use PARTITION BY TBNAME. If the source table has a composite primary key, state windows, event windows, and count windows are not supported.
@@ -193,11 +193,32 @@ All [scalar functions](../function/#scalar-functions) are available in stream pr
- [unique](../function/#unique)
- [mode](../function/#mode)
-## Pause\Resume stream
+## Pause Resume stream
1.pause stream
+```sql
PAUSE STREAM [IF EXISTS] stream_name;
+```
If "IF EXISTS" is not specified and the stream does not exist, an error will be reported; If "IF EXISTS" is specified and the stream does not exist, success is returned; If the stream exists, paused all stream tasks.
2.resume stream
+```sql
RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name;
+```
If "IF EXISTS" is not specified and the stream does not exist, an error will be reported. If "IF EXISTS" is specified and the stream does not exist, success is returned; If the stream exists, all of the stream tasks will be resumed. If "IGNORE UntREATED" is specified, data written during the pause period of stream is ignored when resuming stream.
+
+## Stream State Backup
+The intermediate processing results of stream, a.k.a stream state, need to be persistent on the disk properly during stream processing. The stream state, consisting of multiple files on disk, may be transferred between different computing nodes during the stream processing, as a result of a leader/follower switch or physical computing node offline. You need to deploy the rsync on each physical node to enable the backup and restore processing work, since _ver_.3.3.2.1. To ensure it works correctly, please refer to the following instructions:
+1. add the option "snodeAddress" in the configure file
+2. add the option "checkpointBackupDir" in the configure file to set the backup data directory.
+3. create a _snode_ before creating a stream to ensure the backup service is activated. Otherwise, the checkpoint may not generated during the stream procedure.
+
+>snodeAddress 127.0.0.1:873
+>
+>checkpointBackupDir /home/user/stream/backup/checkpoint/
+
+## create snode
+The snode, stream node for short, on which the aggregate tasks can be deployed on, is a stateful computing node dedicated to the stream processing. An important feature is to backup and restore the stream state files. The snode needs to be created before creating stream tasks. Use the following SQL statement to create a snode in a TDengine cluster, and only one snode is allowed in a TDengine cluster for now.
+```sql
+CREATE SNODE ON DNODE id
+```
+is the ordinal number of a dnode, which can be acquired by using ```show dnodes``` statement.
diff --git a/docs/en/12-taos-sql/18-escape.md b/docs/en/12-taos-sql/18-escape.md
index 2d067b2ad9..a14591b0da 100644
--- a/docs/en/12-taos-sql/18-escape.md
+++ b/docs/en/12-taos-sql/18-escape.md
@@ -18,7 +18,7 @@ description: This document describes the usage of escape characters in TDengine.
## Restrictions
-1. If there are escape characters in identifiers (database name, table name, column name)
+1. If there are escape characters in identifiers (database name, table name, column name, alias Name)
- Identifier without ``: Error will be returned because identifier must be constituted of digits, ASCII characters or underscore and can't be started with digits
- Identifier quoted with ``: Original content is kept, no escaping
2. If there are escape characters in values
diff --git a/docs/en/20-third-party/01-grafana.mdx b/docs/en/20-third-party/01-grafana.mdx
index 007f1734b7..cd1a48f027 100644
--- a/docs/en/20-third-party/01-grafana.mdx
+++ b/docs/en/20-third-party/01-grafana.mdx
@@ -22,9 +22,7 @@ Record these values:
- TDengine REST API url: `http://tdengine.local:6041`.
- TDengine cluster authorization, with user + password.
-## Configuring Grafana
-
-### Install Grafana Plugin and Configure Data Source
+## Install Grafana Plugin and Configure Data Source
@@ -34,8 +32,6 @@ Under Grafana 8, plugin catalog allows you to [browse and manage plugins within
Installation may cost some minutes, you can **Create a TDengine data source** when installation finished.
Then you can add a TDengine data source by filling up the configuration options.
-
-
- Host: IP address of the server where the components of the TDengine cluster provide REST service and the port number of the TDengine REST service (6041), by default use `http://localhost:6041`.
- User: TDengine user name.
- Password: TDengine user password.
@@ -99,8 +95,6 @@ Now users can log in to the Grafana server (username/password: admin/admin) dire
Click `Add data source` to enter the Add data source page, and enter TDengine in the query box to add it.
Enter the datasource configuration page, and follow the default prompts to modify the corresponding configuration.
-
-
- Host: IP address of the server where the components of the TDengine cluster provide REST service and the port number of the TDengine REST service (6041), by default use `http://localhost:6041`.
- User: TDengine user name.
- Password: TDengine user password.
@@ -177,37 +171,112 @@ Open Grafana (http://localhost:3000), and you can add dashboard with TDengine no
-### Create Dashboard
+:::info
-Go back to the main interface to create a dashboard and click Add Query to enter the panel query page:
+In the following introduction, we take Grafana v11.0.0 as an example. Other versions may have different features, please refer to [Grafana's official website](https://grafana.com/docs/grafana/latest/).
+
+:::
+
+## Built-in Variables and Custom Variables
+The Variable feature in Grafana is very powerful. It can be used in queries, panel titles, labels, etc., to create more dynamic and interactive Dashboards, improving user experience and efficiency.
+
+The main functions and characteristics of variables include:
+
+- Dynamic data query: Variables can be used in query statements, allowing users to dynamically change query conditions by selecting different variable values, thus viewing different data views. This is very useful for scenarios that need to dynamically display data based on user input.
+
+- Improved reusability: By defining variables, the same configuration or query logic can be reused in multiple places without the need to rewrite the same code. This makes the maintenance and updating of Dashboards simpler and more efficient.
+
+- Flexible configuration options: Variables offer a variety of configuration options, such as predefined static value lists, dynamic value querying from data sources, regular expression filtering, etc., making the application of variables more flexible and powerful.
+
+
+Grafana provides both built-in variables and custom variables, which can be referenced in SQL writing. We can use `$variableName` to reference the variable, where `variableName` is the name of the variable. For detailed reference, please refer to [Variable reference](https://grafana.com/docs/grafana/latest/dashboards/variables/variable-syntax/).
+
+### Built-in Variables
+Grafana has built-in variables such as `from`, `to`, and `interval`, all derived from Grafana plugin panels. Their meanings are as follows:
+- `from` is the start time of the query range
+- `to` is the end time of the query range
+- `interval` represent time spans
+
+It is recommended to set the start and end times of the query range for each query, which can effectively reduce the amount of data scanned by the TDengine server during query execution. `interval` is the size of the window split, which in Grafana version 11, is calculated based on the time range and the number of return points.
+In addition to the above three common variables, Grafana also provides variables such as `__timezone`, `__org`, `__user`, etc. For details, please refer to [Built-in Variables](https://grafana.com/docs/grafana/latest/dashboards/variables/add-template-variables/#global-variables).
+
+### Custom Variables
+We can add custom variables in the Dashboard. The usage of custom variables is no different from that of built-in variables; they are referenced in SQL with `$variableName`.
+Custom variables support multiple types, such as `Query`, `Constant`, `Interval`, `Data source`, etc.
+Custom variables can reference other custom variables, for example, one variable represents a region, and another variable can reference the value of the region to query devices in that region.
+
+#### Adding Query Type Variables
+In the Dashboard configuration, select `Variables`, then click `New variable`:
+1. In the `Name` field, enter your variable name, here we set the variable name as `selected_groups`.
+2. In the `Select variable type` dropdown menu, select `Query`.
+Depending on the selected variable type, configure the corresponding options. For example, if you choose `Query`, you need to specify the data source and the query statement for obtaining variable values. Here, taking smart meters as an example, we set the query type, select the data source, and configure the SQL as `select distinct(groupid) from power.meters where groupid < 3 and ts > $from and ts < $to;`
+3. After clicking `Run Query` at the bottom, you can see the variable values generated based on your configuration in the `Preview of values` section.
+4. Other configurations are not detailed here. After completing the configuration, click the `Apply` button at the bottom of the page, then click `Save dashboard` in the top right corner to save.
+
+After completing the above steps, we have successfully added a new custom variable `$selected_groups` to the Dashboard. We can later reference this variable in the Dashboard's queries with `$selected_groups`.
+
+We can also add another custom variable to reference this `selected_groups` variable, for example, we add a query variable named `tbname_max_current`, with the SQL as `select tbname from power.meters where groupid = $selected_groups and ts > $from and ts < $to;`
+
+#### Adding Interval Type Variables
+We can customize the time window interval to better fit business needs.
+1. In the `Name` field, enter the variable name as `interval`.
+2. In the `Select variable type` dropdown menu, select `Interval`.
+3. In the `Interval options` enter `1s,2s,5s,10s,15s,30s,1m`.
+4. Other configurations are not detailed here. After completing the configuration, click the `Apply` button at the bottom of the page, then click `Save dashboard` in the top right corner to save.
+
+After completing the above steps, we have successfully added a new custom variable `$interval` to the Dashboard. We can later reference this variable in the Dashboard's queries with `$interval`.
+
+## TDengine Time Series Query Support
+On top of supporting standard SQL, TDengine also provides a series of special query syntaxes that meet the needs of time series business scenarios, bringing great convenience to the development of applications in time series scenarios.
+- `partition by` this clause can split data by certain dimensions and then perform a series of calculations within the split data space, which can replace `group by` in most cases.
+- `interval` this clause is used to generate time windows of the same time interval.
+- `fill` this clause is used to specify how to fill when there is data missing in any window.
+- `Window Pseudocolumns` If you need to output the time window information corresponding to the aggregation result in the results, you need to use window pseudocolumns in the SELECT clause: the start time of the time window (_wstart), the end time of the time window (_wend), etc.
+
+For a detailed introduction to these features, please refer to [Time-Series Extensions](../../taos-sql/distinguished/).
+
+
+## Create Dashboard
+
+Return to the main interface to create a Dashboard, click Add Query to enter the panel query page:
As shown above, select the `TDengine` data source in the `Query` and enter the corresponding SQL in the query box below for query. We will continue to use power meters as an example. In order to demonstrate the beautiful curves, **virtual data** is used here.
-- INPUT SQL: Enter the desired query (the results being two columns and multiple rows), such as `select _wstart as ts, avg(current) as current from power.meters where ts > $from and ts < $to interval($interval) fill(null)`. In this statement, `$from`, `$to`, and `$interval` are variables that Grafana replaces with the query time range and interval. In addition to the built-in variables, custom template variables are also supported.
-- ALIAS BY: This allows you to set the current query alias.
-- GENERATE SQL: Clicking this button will automatically replace the corresponding variables and generate the final executed statement.
-- Group by column(s): `group by` or `partition by` columns name split by comma. By setting `Group by column(s)`, it can show multi-dimension data if Sql is `group by` or `partition by`. Such as, it can show data by `groupid` if sql is `select _wstart as ts, groupid, avg(current) as current from power.meters where ts > $from and ts < $to partition by groupid interval($interval) fill(null)` and `Group by column(s)` is `groupid`.
-- Group By Format: format legend for `group by` or `partition by`. For example, in the above Input SQL, set `Group By Format` to `groupid-{{groupid}}`, and display the legend name as the formatted group name.
+## Time Series Data Display
+Suppose we want to query the average current size over a period of time, with the time window divided by $interval, and fill with null if data is missing in any time window interval.
+- INPUT SQL: Enter the statement to be queried (the result set of this SQL statement should be two columns and multiple rows), here enter: `select _wstart as ts, avg(current) as current from power.meters where groupid in ($selected_groups) and ts > $from and ts < $to interval($interval) fill(null)`, where from, to, and interval are built-in variables of the Grafana, selected_groups is a custom variable.
+- ALIAS BY: You can set the current query alias.
+- GENERATE SQL: Clicking this button will automatically replace the corresponding variables and generate the final execution statement.
+
+In the custom variables at the top, if the value of `selected_groups` is selected as 1, then the query for the average value change of all device currents in the `meters` supertable where `groupid` is 1 is as shown in the following figure:
+
+
:::note
-Since the REST connection because is stateless. Grafana plugin can use <db_name>.<table_name> in the SQL command to specify the database name.
+Since the REST interface is stateless, it is not possible to use the `use db` statement to switch databases. In the SQL statement in the Grafana plugin, you can use \.\ to specify the database.
:::
-Query the average current changes of all devices in the `meters` stable as shown in the following figure:
+## Time Series Data Group Display
+Suppose we want to query the average current value over a period of time, displayed by `groupid` grouping, we can modify the previous SQL to `select _wstart as ts, groupid, avg(current) as current from power.meters where ts > $from and ts < $to partition by groupid interval($interval) fill(null)`
-
+- Group by column(s): **Half-width** comma-separated `group by` or `partition by` column names. If it is a `group by` or `partition by` query statement, setting the `Group by` column can display multidimensional data. Here, set the Group by column name to `groupid`, which can display data grouped by `groupid`.
+- Group By Format: Legend formatting format for multidimensional data in Group by or Partition by scenarios. For example, the above INPUT SQL, setting `Group By Format` to `groupid-{{groupid}}`, the displayed legend name is the formatted group name.
-Query the average current value of all devices in the 'meters' stable and display it in groups according to the `groupid` as shown in the following figure:
+After completing the settings, the data is displayed grouped by `groupid` as shown in the following figure:
-
+
> For more information on how to use Grafana to create the appropriate monitoring interface and for more details on using Grafana, refer to the official Grafana [documentation](https://grafana.com/docs/).
-### Importing the Dashboard
+## Performance Suggestions
+- **Include time range in all queries**, in time series databases, if the time range is not included in the query, it will lead to table scanning and poor performance. A common SQL writing example is `select column_name from db.table where ts > $from and ts < $to;`
+- For queries of the latest status type, we generally recommend **enabling cache when creating the database** (`CACHEMODEL` set to last_row or both), a common SQL writing example is `select last(column_name) from db.table where ts > $from and ts < $to;`
+
+## Importing the Dashboard
You can install TDinsight dashboard in data source configuration page (like `http://localhost:3000/datasources/edit/1/dashboards`) as a monitoring visualization tool for TDengine cluster. Ensure that you use TDinsight for 3.x. Please note TDinsight for 3.x needs to configure and run taoskeeper correctly.
@@ -221,3 +290,137 @@ For more dashboards using TDengine data source, [search here in Grafana](https:/
- [15155](https://grafana.com/grafana/dashboards/15155): TDengine alert demo.
- [15167](https://grafana.com/grafana/dashboards/15167): TDinsight.
- [16388](https://grafana.com/grafana/dashboards/16388): Telegraf node metrics dashboard using TDengine data source.
+
+
+## Alert Configuration Introduction
+### Alert Configuration Steps
+The TDengine Grafana plugin supports alerts. To configure alerts, the following steps are required:
+1. Configure Contact Points: Set up notification channels, including DingDing, Email, Slack, WebHook, Prometheus Alertmanager, etc.
+2. Configure Notification Policies: Set up routing for which channel to send alerts to, as well as the timing and frequency of notifications.
+3. Configure "Alert rules": Set up detailed alert rules.
+ 3.1 Configure alert name.
+ 3.2 Configure query and alert trigger conditions.
+ 3.3 Configure evaluation behavior.
+ 3.4 Configure labels and notifications.
+ 3.5 Configure annotations.
+
+### Alert Configuration Web UI
+In Grafana 11, the alert Web UI has 6 tabs: "Alert rules", "Contact points", "Notification policies", "Silences", "Groups", and "Settings".
+- "Alert rules" displays and configures alert rules.
+- "Contact points" support notification channels such as DingDing, Email, Slack, WebHook, Prometheus Alertmanager, etc.
+- "Notification policies" sets up routing for which channel to send alerts to, as well as the timing and frequency of notifications.
+- "Silences" configures silent periods for alerts.
+- "Groups" displays grouped alerts after they are triggered.
+- "Admin" allows modifying alert configurations through JSON.
+
+## Configuring Email Contact Point
+### Modifying Grafana Server Configuration File
+Add SMTP/Emailing and Alerting modules to the Grafana service configuration file. For Linux systems, the configuration file is usually located at `/etc/grafana/grafana.ini`.
+Add the following content to the configuration file:
+
+```ini
+#################################### SMTP / Emailing ##########################
+[smtp]
+enabled = true
+host = smtp.qq.com:465 #Email service used
+user = receiver@foxmail.com
+password = *********** #Use mail authorization code
+skip_verify = true
+from_address = sender@foxmail.com
+```
+
+Then restart the Grafana service. For example, on a Linux system, execute `systemctl restart grafana-server.service`
+
+### Grafana Configuration for Email Contact Point
+
+Find "Home" -> "Alerting" -> "Contact points" on the Grafana page to create a new contact point
+"Name": Email Contact Point
+"Integration": Select the contact type, here choose Email, fill in the email receiving address, and save the contact point after completion
+
+
+
+## Configuring Feishu Contact Point
+
+### Feishu Robot Configuration
+1. "Feishu Workspace" -> "Get Apps" -> "Search for Feishu Robot Assistant" -> "Create Command"
+2. Choose Trigger: Grafana
+3. Choose Action: Send a message through the official robot, fill in the recipient and message content
+
+
+
+### Grafana Configuration for Feishu Contact Point
+
+Find "Home" -> "Alerting" -> "Contact points" on the Grafana page to create a new contact point
+"Name": Feishu Contact Point
+"Integration": Select the contact type, here choose Webhook, and fill in the URL (the Grafana trigger Webhook address in Feishu Robot Assistant), then save the contact point
+
+
+
+## Notification Policy
+After configuring the contact points, you can see there is a Default Policy
+
+
+
+Click on the "..." on the right -> "Edit", then edit the default notification policy, a configuration window pops up:
+
+
+
+Configure the parameters as shown in the screenshot above.
+
+## Configuring Alert Rules
+
+### Define Query and Alert Conditions
+
+Select "Edit" -> "Alert" -> "New alert rule" in the panel where you want to configure the alert.
+
+1. "Enter alert rule name": Here, enter `power meters alert` as an example for smart meters.
+2. "Define query and alert condition":
+ 2.1 Choose data source: `TDengine Datasource`
+ 2.2 Query statement:
+ ```sql
+ select _wstart as ts, groupid, avg(current) as current from power.meters where ts > $from and ts < $to partition by groupid interval($interval) fill(null)
+ ```
+ 2.3 Set "Expression": `Threshold is above 100`
+ 2.4 Click "Set as alert condition"
+ 2.5 "Preview": View the results of the set rules
+
+After completing the settings, you can see the image displayed below:
+
+
+
+### Configuring Expressions and Calculation Rules
+
+Grafana's "Expression" supports various operations and calculations on data, which are divided into:
+1. "Reduce": Aggregates the values of a time series within the selected time range into a single value
+ 1.1 "Function" is used to set the aggregation method, supporting Min, Max, Last, Mean, Sum, and Count.
+ 1.2 "Mode" supports the following three:
+ - "Strict": If no data is queried, the data will be assigned NaN.
+ - "Drop Non-numeric Value": Remove illegal data results.
+ - "Replace Non-numeric Value": If it is illegal data, replace it with a constant value.
+2. "Threshold": Checks whether the time series data meets the threshold judgment condition. Returns 0 when the condition is false, and 1 when true. Supports the following methods:
+ - Is above (x > y)
+ - Is below (x < y)
+ - Is within range (x > y1 AND x < y2)
+ - Is outside range (x < y1 AND x > y2)
+3. "Math": Performs mathematical operations on the data of the time series.
+4. "Resample": Changes the timestamps in each time series to have a consistent interval, so that mathematical operations can be performed between them.
+5. "Classic condition (legacy)": Multiple logical conditions can be configured to determine whether to trigger an alert.
+
+As shown in the screenshot above, here we set the alert to trigger when the maximum value exceeds 100.
+
+### Configuring Evaluation behavior
+
+
+
+Configure the parameters as shown in the screenshot above.
+
+### Configuring Labels and Notifications
+
+
+Configure the parameters as shown in the screenshot above.
+
+### Configuring annotations
+
+
+
+After setting "Summary" and "Description", you will receive an alert notification if the alert is triggered.
diff --git a/docs/en/20-third-party/11-kafka.md b/docs/en/20-third-party/11-kafka.md
index 344db06322..8abfd297b5 100644
--- a/docs/en/20-third-party/11-kafka.md
+++ b/docs/en/20-third-party/11-kafka.md
@@ -94,7 +94,7 @@ The output as bellow:
The role of the TDengine Sink Connector is to synchronize the data of the specified topic to TDengine. Users do not need to create databases and super tables in advance. The name of the target database can be specified manually (see the configuration parameter connection.database), or it can be generated according to specific rules (see the configuration parameter connection.database.prefix).
-TDengine Sink Connector internally uses TDengine [modeless write interface](../../client-libraries/cpp#modeless write-api) to write data to TDengine, currently supports data in three formats: [InfluxDB line protocol format](../../develop/insert-data/influxdb-line), [OpenTSDB Telnet protocol format](../../develop/insert-data/opentsdb-telnet), and [OpenTSDB JSON protocol format](../../develop/insert-data/opentsdb-json).
+TDengine Sink Connector internally uses TDengine [modeless write interface](../../client-libraries/cpp/#schemaless-writing-api) to write data to TDengine, currently supports data in three formats: [InfluxDB line protocol format](../../develop/insert-data/influxdb-line), [OpenTSDB Telnet protocol format](../../develop/insert-data/opentsdb-telnet), and [OpenTSDB JSON protocol format](../../develop/insert-data/opentsdb-json).
The following example synchronizes the data of the topic meters to the target database power. The data format is the InfluxDB Line protocol format.
diff --git a/docs/en/20-third-party/75-powerbi.md b/docs/en/20-third-party/75-powerbi.md
index 0f7a1de0ca..f6b0a786cb 100644
--- a/docs/en/20-third-party/75-powerbi.md
+++ b/docs/en/20-third-party/75-powerbi.md
@@ -25,7 +25,7 @@ description: Use PowerBI and TDengine to analyze time series data
[DSN]: Data Source Name, required field, such as "MyTDengine"
-Depending on your TDengine server version, download appropriate version of TDengine client package from TDengine website [Download Link](../../get-started/package/), or TDengine explorer if you are using a local TDengine cluster. Install the TDengine client package on same Windows machine where PowerBI is running.
+Depending on your TDengine server version, download appropriate version of TDengine client package from TDengine website [Download Link](https://docs.tdengine.com/get-started/package/), or TDengine explorer if you are using a local TDengine cluster. Install the TDengine client package on same Windows machine where PowerBI is running.
[URL]: taos://localhost:6041
diff --git a/docs/en/20-third-party/grafana/alert-annotations.webp b/docs/en/20-third-party/grafana/alert-annotations.webp
new file mode 100644
index 0000000000..b5001f3e8a
Binary files /dev/null and b/docs/en/20-third-party/grafana/alert-annotations.webp differ
diff --git a/docs/en/20-third-party/grafana/alert-email.webp b/docs/en/20-third-party/grafana/alert-email.webp
new file mode 100644
index 0000000000..53a91de0a8
Binary files /dev/null and b/docs/en/20-third-party/grafana/alert-email.webp differ
diff --git a/docs/en/20-third-party/grafana/alert-evaluation.webp b/docs/en/20-third-party/grafana/alert-evaluation.webp
new file mode 100644
index 0000000000..839510450a
Binary files /dev/null and b/docs/en/20-third-party/grafana/alert-evaluation.webp differ
diff --git a/docs/en/20-third-party/grafana/alert-feishu1.webp b/docs/en/20-third-party/grafana/alert-feishu1.webp
new file mode 100644
index 0000000000..d0a3cdc2cc
Binary files /dev/null and b/docs/en/20-third-party/grafana/alert-feishu1.webp differ
diff --git a/docs/en/20-third-party/grafana/alert-feishu2.webp b/docs/en/20-third-party/grafana/alert-feishu2.webp
new file mode 100644
index 0000000000..558af02d08
Binary files /dev/null and b/docs/en/20-third-party/grafana/alert-feishu2.webp differ
diff --git a/docs/en/20-third-party/grafana/alert-labels.webp b/docs/en/20-third-party/grafana/alert-labels.webp
new file mode 100644
index 0000000000..558fbdd790
Binary files /dev/null and b/docs/en/20-third-party/grafana/alert-labels.webp differ
diff --git a/docs/en/20-third-party/grafana/alert-notification1.webp b/docs/en/20-third-party/grafana/alert-notification1.webp
new file mode 100644
index 0000000000..65f3640d44
Binary files /dev/null and b/docs/en/20-third-party/grafana/alert-notification1.webp differ
diff --git a/docs/en/20-third-party/grafana/alert-notification2.webp b/docs/en/20-third-party/grafana/alert-notification2.webp
new file mode 100644
index 0000000000..c54f74c065
Binary files /dev/null and b/docs/en/20-third-party/grafana/alert-notification2.webp differ
diff --git a/docs/en/20-third-party/grafana/alert-rules1.webp b/docs/en/20-third-party/grafana/alert-rules1.webp
new file mode 100644
index 0000000000..5ea238b4b9
Binary files /dev/null and b/docs/en/20-third-party/grafana/alert-rules1.webp differ
diff --git a/docs/en/20-third-party/grafana/create_dashboard2.webp b/docs/en/20-third-party/grafana/create_dashboard2.webp
index 32a209872d..f9e70cdd31 100644
Binary files a/docs/en/20-third-party/grafana/create_dashboard2.webp and b/docs/en/20-third-party/grafana/create_dashboard2.webp differ
diff --git a/docs/en/20-third-party/grafana/create_dashboard3.webp b/docs/en/20-third-party/grafana/create_dashboard3.webp
index 9ab8e015b7..679e2b094b 100644
Binary files a/docs/en/20-third-party/grafana/create_dashboard3.webp and b/docs/en/20-third-party/grafana/create_dashboard3.webp differ
diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx
index 117ec5e5d1..3105dbdbb6 100644
--- a/docs/zh/08-connector/14-java.mdx
+++ b/docs/zh/08-connector/14-java.mdx
@@ -379,7 +379,7 @@ properties 中的配置参数如下：
- TSDBDriver.PROPERTY_KEY_DISABLE_SSL_CERT_VALIDATION: 关闭 SSL 证书验证 。仅在使用 Websocket 连接时生效。true: 启用，false: 不启用。默认为 false。
-此外对 JDBC 原生连接，通过指定 URL 和 Properties 还可以指定其他参数，比如日志级别、SQL 长度等。更多详细配置请参考[客户端配置](../../reference/config/#仅客户端适用)。
+此外对 JDBC 原生连接，通过指定 URL 和 Properties 还可以指定其他参数，比如日志级别、SQL 长度等。更多详细配置请参考[客户端配置](../../reference/config/)。
### 配置参数的优先级
diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md
index acf42a3b8f..f10c5ebb69 100755
--- a/docs/zh/12-taos-sql/06-select.md
+++ b/docs/zh/12-taos-sql/06-select.md
@@ -459,7 +459,7 @@ SELECT ... FROM (SELECT ... FROM ...) ...;
:::info
- 内层查询的返回结果将作为“虚拟表”供外层查询使用，此虚拟表建议起别名，以便于外层查询中方便引用。
-- 外层查询支持直接通过列名或`列名`的形式引用内层查询的列或伪列。
+- 外层查询支持直接通过列名或\`列名\`的形式引用内层查询的列或伪列。
- 在内层和外层查询中，都支持普通的表间/超级表间 JOIN。内层查询的计算结果也可以再参与数据子表的 JOIN 操作。
- 内层查询支持的功能特性与非嵌套的查询语句能力是一致的。
- 内层查询的 ORDER BY 子句一般没有意义，建议避免这样的写法以免无谓的资源消耗。
diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md
index 3e78ac0da0..ca6ba378ca 100644
--- a/docs/zh/12-taos-sql/10-function.md
+++ b/docs/zh/12-taos-sql/10-function.md
@@ -1200,27 +1200,40 @@ ignore_negative: {
### DIFF
```sql
-DIFF(expr [, ignore_negative])
+DIFF(expr [, ignore_option])
-ignore_negative: {
+ignore_option: {
0
| 1
+ | 2
+ | 3
}
```
-**功能说明**：统计表中某列的值与前一行对应值的差。 ignore_negative 取值为 0|1 , 可以不填，默认值为 0. 不忽略负值。ignore_negative 为 1 时表示忽略负数。对于你存在复合主键的表的查询，若时间戳相同的数据存在多条，则只有对应的复合主键最小的数据参与运算。
+**功能说明**：统计表中特定列与之前行的当前列有效值之差。 ignore_option 取值为 0|1|2|3 , 可以不填，默认值为 0.
+- `0` 表示不忽略(diff结果)负值不忽略 null 值
+- `1` 表示(diff结果)负值作为 null 值
+- `2` 表示不忽略(diff结果)负值但忽略 null 值
+- `3` 表示忽略(diff结果)负值且忽略 null 值
+- 对于存在复合主键的表的查询，若时间戳相同的数据存在多条，则只有对应的复合主键最小的数据参与运算。
-**返回数据类型**：同应用字段。
+**返回数据类型**：bool、时间戳及整型数值类型均返回 int_64，浮点类型返回 double, 若 diff 结果溢出则返回溢出后的值。
-**适用数据类型**：数值类型。
+**适用数据类型**：数值类型、时间戳和 bool 类型。
**适用于**：表和超级表。
**使用说明**:
-- 输出结果行数是范围内总行数减一，第一行没有结果输出。
-- 可以与选择相关联的列一起使用。 例如: select \_rowts, DIFF() from。
-
+- diff 是计算本行特定列与同列的前一个有效数据的差值，同列的前一个有效数据：指的是同一列中时间戳较小的最临近的非空值。
+- 数值类型 diff 结果为对应的算术差值；时间戳类型根据数据库的时间戳精度进行差值计算；bool 类型计算差值时 true 视为 1， false 视为 0
+- 如当前行数据为 null 或者没有找到同列前一个有效数据时，diff 结果为 null
+- 忽略负值时（ ignore_option 设置为 1 或 3 ），如果 diff 结果为负值，则结果设置为 null，然后根据 null 值过滤规则进行过滤
+- 当 diff 结果发生溢出时，结果是否是`应该忽略的负值`取决于逻辑运算结果是正数还是负数，例如 9223372036854775800 - (-9223372036854775806) 的值超出 BIGINT 的范围 ，diff 结果会显示溢出值 -10，但并不会被作为负值忽略
+- 单个语句中可以使用单个或者多个 diff，并且每个 diff 可以指定相同或不同的 ignore_option ，当单个语句中存在多个 diff 时当且仅当某行所有 diff 的结果都为 null ，并且 ignore_option 都设置为忽略 null 值，该行才从结果集中剔除
+- 可以选择与相关联的列一起使用。 例如: select _rowts, DIFF() from。
+- 当没有复合主键时，如果不同的子表有相同时间戳的数据，会提示 "Duplicate timestamps not allowed"
+- 当使用复合主键时，不同子表的时间戳和主键组合可能相同，使用哪一行取决于先找到哪一行，这意味着在这种情况下多次运行 diff() 的结果可能会不同。
### IRATE
diff --git a/docs/zh/12-taos-sql/12-distinguished.md b/docs/zh/12-taos-sql/12-distinguished.md
index b979d44a5e..bf24d0adac 100755
--- a/docs/zh/12-taos-sql/12-distinguished.md
+++ b/docs/zh/12-taos-sql/12-distinguished.md
@@ -76,7 +76,7 @@ window_clause: {
FILL 语句指定某一窗口区间数据缺失的情况下的填充模式。填充模式包括以下几种：
1. 不进行填充：NONE（默认填充模式）。
-2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。这里需要注意，最终填充的值受由相应列的类型决定，如 FILL(VALUE, 1.23)，相应列为 INT 类型，则填充值为 1。
+2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。这里需要注意，最终填充的值受由相应列的类型决定，如 FILL(VALUE, 1.23)，相应列为 INT 类型，则填充值为 1, 若查询列表中有多列需要FILL, 则需要给每一个FILL列指定VALUE, 如`SELECT _wstart, min(c1), max(c1) FROM ... FILL(VALUE, 0, 0)`。
3. PREV 填充：使用前一个非 NULL 值填充数据。例如：FILL(PREV)。
4. NULL 填充：使用 NULL 填充数据。例如：FILL(NULL)。
5. LINEAR 填充：根据前后距离最近的非 NULL 值做线性插值填充。例如：FILL(LINEAR)。
diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md
index 8d814d05a1..c0d14f0455 100644
--- a/docs/zh/12-taos-sql/14-stream.md
+++ b/docs/zh/12-taos-sql/14-stream.md
@@ -27,7 +27,7 @@ subquery: SELECT select_list
from_clause
[WHERE condition]
[PARTITION BY tag_list]
- [window_clause]
+ window_clause
```
支持会话窗口、状态窗口、滑动窗口、事件窗口和计数窗口，其中，状态窗口、事件窗口和计数窗口搭配超级表时必须与partition by tbname一起使用。对于数据源表是复合主键的流，不支持状态窗口、事件窗口、计数窗口的计算。
@@ -271,4 +271,23 @@ PAUSE STREAM [IF EXISTS] stream_name;
2.流计算恢复计算任务
RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name;
-没有指定IF EXISTS，如果该stream不存在，则报错，如果存在，则恢复流计算；指定了IF EXISTS，如果stream不存在，则返回成功；如果存在，则恢复流计算。如果指定IGNORE UNTREATED，则恢复流计算时，忽略流计算暂停期间写入的数据。
\ No newline at end of file
+没有指定IF EXISTS，如果该stream不存在，则报错，如果存在，则恢复流计算；指定了IF EXISTS，如果stream不存在，则返回成功；如果存在，则恢复流计算。如果指定IGNORE UNTREATED，则恢复流计算时，忽略流计算暂停期间写入的数据。
+
+## 状态数据备份与同步
+流计算的中间结果成为计算的状态数据，需要在流计算整个生命周期中进行持久化保存。为了确保流计算中间状态能够在集群环境下在不同的节点间可靠地同步和迁移，至3.3.2.1 版本开始，需要在运行环境中部署 rsync 软件，还需要增加以下的步骤：
+1. 在配置文件中配置 snode 的地址（IP+端口）和状态数据备份目录（该目录系 snode 所在的物理节点的目录）。
+2. 然后创建 snode。
+完成上述两个步骤以后才能创建流。
+如果没有创建 snode 并正确配置 snode 的地址，流计算过程中将无法生成检查点（checkpoint），并可能导致后续的计算结果产生错误。
+
+> snodeAddress 127.0.0.1:873
+>
+> checkpointBackupDir /home/user/stream/backup/checkpoint/
+
+
+## 创建 snode 的方式
+使用以下命令创建 snode（stream node）， snode 是流计算中有状态的计算节点，可用于部署聚合任务，同时负责备份不同的流计算任务生成的检查点数据。
+```sql
+CREATE SNODE ON DNODE [id]
+```
+其中的 id 是集群中的 dnode 的序号。请注意选择的dnode，流计算的中间状态将自动在其上进行备份。
diff --git a/docs/zh/12-taos-sql/18-escape.md b/docs/zh/12-taos-sql/18-escape.md
index 81e4179042..6bd20abf36 100644
--- a/docs/zh/12-taos-sql/18-escape.md
+++ b/docs/zh/12-taos-sql/18-escape.md
@@ -20,7 +20,7 @@ description: TDengine 中使用转义字符的详细规则
## 转义字符使用规则
-1. 标识符里有转义字符（数据库名、表名、列名）
+1. 标识符里有转义字符（数据库名、表名、列名、别名）
1. 普通标识符： 直接提示错误的标识符，因为标识符规定必须是数字、字母和下划线，并且不能以数字开头。
2. 反引号``标识符： 保持原样，不转义
2. 数据里有转义字符
diff --git a/docs/zh/20-third-party/01-grafana.mdx b/docs/zh/20-third-party/01-grafana.mdx
index 065ce9e57e..63a2a79d4f 100644
--- a/docs/zh/20-third-party/01-grafana.mdx
+++ b/docs/zh/20-third-party/01-grafana.mdx
@@ -13,7 +13,7 @@ TDengine 能够与开源数据可视化系统 [Grafana](https://www.grafana.com/
要让 Grafana 能正常添加 TDengine 数据源，需要以下几方面的准备工作。
-- Grafana 服务已经部署并正常运行。目前 TDengine 支持 Grafana 7.5 以上的版本。用户可以根据当前的操作系统，到 Grafana 官网下载安装包，并执行安装。下载地址如下：\。
+- Grafana 服务已经部署并正常运行。目前 TDengine 支持 Grafana 7.5 以上的版本。用户可以根据当前的操作系统，到 Grafana 官网下载安装包，并执行安装。下载地址如下：[https://grafana.com/grafana/download](https://grafana.com/grafana/download) 。
- TDengine 集群已经部署并正常运行。
- taosAdapter 已经安装并正常运行。具体细节请参考 [taosAdapter 的使用手册](../../reference/taosadapter)
@@ -22,26 +22,19 @@ TDengine 能够与开源数据可视化系统 [Grafana](https://www.grafana.com/
- TDengine 集群 REST API 地址，如：`http://tdengine.local:6041`。
- TDengine 集群认证信息，可使用用户名及密码。
-## 配置 Grafana
-
-### 安装 Grafana Plugin 并配置数据源
+## 安装 Grafana Plugin 并配置数据源
使用 Grafana 最新版本（8.5+），您可以在 Grafana 中[浏览和管理插件](https://grafana.com/docs/grafana/next/administration/plugin-management/#plugin-catalog)（对于 7.x 版本，请采用 **安装脚本** 或 **手动安装** 方式）。在 Grafana 管理界面中的 **Configurations > Plugins** 页面直接搜索 `TDengine` 并按照提示安装。
-安装完毕后，按照指示 **Create a TDengine data source** 添加数据源。
-输入 TDengine 相关配置，如下图所示：
-
-
-
-- Host： TDengine 集群中提供 REST 服务的 IP 地址与端口号，默认 \。
+安装完毕后，按照指示 **Create a TDengine data source** 添加数据源，输入 TDengine 相关配置：
+- Host： TDengine 集群中提供 REST 服务的 IP 地址与端口号，默认 `http://localhost:6041`
- User：TDengine 用户名。
- Password：TDengine 用户密码。
点击 `Save & Test` 进行测试，成功会提示：`TDengine Data source is working`。
-配置完毕，现在可以使用 TDengine 创建 Dashboard 了。
@@ -93,13 +86,11 @@ sudo unzip tdengine-datasource-$GF_VERSION.zip -d /var/lib/grafana/plugins/
GF_INSTALL_PLUGINS=tdengine-datasource
```
-之后，用户可以直接通过 \ 的网址，登录 Grafana 服务器（用户名/密码：admin/admin），通过左侧 `Configuration -> Data Sources` 可以添加数据源，
+之后，用户可以直接通过 http://localhost:3000 的网址，登录 Grafana 服务器（用户名/密码：admin/admin），通过左侧 `Configuration -> Data Sources` 可以添加数据源，
-点击 `Add data source` 可进入新增数据源页面，在查询框中输入 TDengine， 然后点击 `select` 选择添加后会进入数据源配置页面，按照默认提示修改相应配置即可：
+点击 `Add data source` 可进入新增数据源页面，在查询框中输入 TDengine， 然后点击 `select` 选择添加后会进入数据源配置页面，按照默认提示修改相应配置：
-
-
-- Host： TDengine 集群中提供 REST 服务的 IP 地址与端口号，默认 \。
+- Host： TDengine 集群中提供 REST 服务的 IP 地址与端口号，默认 `http://localhost:6041`
- User：TDengine 用户名。
- Password：TDengine 用户密码。
@@ -170,25 +161,92 @@ docker run -d \
3. 使用 docker-compose 命令启动 TDengine + Grafana ：`docker-compose up -d`。
-打开 Grafana \，现在可以添加 Dashboard 了。
+打开 Grafana [http://localhost:3000](http://localhost:3000)，现在可以添加 Dashboard 了。
-### 创建 Dashboard
-回到主界面创建 Dashboard，点击 Add Query 进入面板查询页面：
+:::info
+
+下文介绍中，都以 Grafana v11.0.0 版本为例，其他版本功能可能有差异，请参考 [Grafana 官网](https://grafana.com/docs/grafana/latest/)。
+
+:::
+
+## 内置变量和自定义变量
+Grafana 中的 Variable（变量）功能非常强大，可以在 Dashboard 的查询、面板标题、标签等地方使用，用来创建更加动态和交互式的 Dashbord，提高用户体验和效率。
+
+变量的主要作用和特点包括：
+
+- 动态数据查询：变量可以用于查询语句中，使得用户可以通过选择不同的变量值来动态更改查询条件，从而查看不同的数据视图。这对于需要根据用户输入动态展示数据的场景非常有用。
+
+- 提高可重用性：通过定义变量，可以在多个地方重用相同的配置或查询逻辑，而不需要重复编写相同的代码。这使得 Dashboard 的维护和更新变得更加简单高效。
+
+- 灵活的配置选项：变量提供了多种配置选项，如预定义的静态值列表、从数据源动态查询值、正则表达式过滤等，使得变量的应用更加灵活和强大。
+
+
+Grafana 提供了内置变量和自定义变量，它们都可以可以在编写 SQL 时引用，引用的方式是 `$variableName`，`variableName` 是变量的名字，其他引用方式请参考 [引用方式](https://grafana.com/docs/grafana/latest/dashboards/variables/variable-syntax/)。
+
+### 内置变量
+Grafana 内置了 `from`、`to` 和 `interval` 等变量，都取自于 Grafana 插件面板。其含义如下：
+- `from` 查询范围的起始时间
+- `to` 查询范围的结束时间
+- `interval` 窗口切分间隔
+
+对于每个查询都建议设置查询范围的起始时间和结束时间，可以有效的减少 TDengine 服务端执行查询扫描的数据量。`internal` 是窗口切分的大小，在 Grafana 11 版本中，其大小为时间间隔和返回点数计算而得。
+除了上述三个常用变量，Grafana 还提供了如 `__timezone`, `__org`, `__user` 等变量，详情请参考 [内置变量](https://grafana.com/docs/grafana/latest/dashboards/variables/add-template-variables/#global-variables)。
+
+### 自定义变量
+我们可以在 Dashbord 中增加自定义变量。自定义变量和内置变量的使用方式没有区别，都是在 SQL 中用 `$variableName` 进行引用。
+自定义变量支持多种类型，常见的类型包括 `Query`（查询）、`Constant`（常量）、`Interval`（间隔）、`Data source`（数据源）等。
+自定义变量可以引用其他自定义变量，比如一个变量表示区域，另一个变量可以引用区域的值，来查询这个区域的设备。
+#### 添加查询类型变量
+在 Dashbord 的配置中，选择 【Variables】，然后点击 【New variable】：
+1. 在 “Name“ 字段中，输入你的变量名，此处我们设置变量名为 `selected_groups`。
+2. 在 【Select variable type】下拉菜单中，选择 ”Query“（查询）。
+根据选择的变量类型，配置相应的选项。例如，如果选择了 “Query” 类型，你需要指定数据源和用于获取变量值的查询语句。此处我们还以智能电表为例，设置查询类型，选择数据源后，配置 SQL 为 `select distinct(groupid) from power.meters where groupid < 3 and ts > $from and ts < $to;`
+3. 点击底部的【Run Query】后，可以在 “Preview of values”（值预览）部分，查看到根据你的配置生成的变量值。
+4. 还有其他配置不再赘述，完成配置后，点击页面底部的【Apply】（应用）按钮，然后点击右上角的【Save dashboard】保存。
+
+完成以上步骤后，我们就成功在 Dashboard 中添加了一个新的自定义变量 `$selected_groups`。我们可以后面在 Dashboard 的查询中通过 `$selected_groups` 的方式引用这个变量。
+
+我们还可以再新增自定义变量来引用这个 `selected_groups` 变量，比如我们新增一个名为 `tbname_max_current` 的查询变量，其 SQL 为 `select tbname from power.meters where groupid = $selected_groups and ts > $from and ts < $to;`
+
+#### 添加间隔类型变量
+我们可以自定义时间窗口间隔，可以更加贴合业务需求。
+1. 在 “Name“ 字段中，输入变量名为 `interval`。
+2. 在 【Select variable type】下拉菜单中，选择 “Interval”（间隔）。
+3. 在 【Interval options】选项中输入 `1s,2s,5s,10s,15s,30s,1m`。
+4. 还有其他配置不再赘述，完成配置后，点击页面底部的【Apply】（应用）按钮，然后点击右上角的【Save dashboard】保存。
+
+完成以上步骤后，我们就成功在 Dashboard 中添加了一个新的自定义变量 `$interval`。我们可以后面在 Dashboard 的查询中通过 `$interval` 的方式引用这个变量。
+
+## TDengine 时间序列查询支持
+TDengine 在支持标准 SQL 的基础之上，还提供了一系列满足时序业务场景需求的特色查询语法，这些语法能够为时序场景的应用的开发带来极大的便利。
+- `partition by` 子句可以按一定的维度对数据进行切分，然后在切分出的数据空间内再进行一系列的计算。绝大多数情况可以替代 `group by`。
+- `interval` 子句用于产生相等时间周期的窗口
+- `fill` 语句指定某一窗口区间数据缺失的情况下的填充模式
+- `时间戳伪列` 如果需要在结果中输出聚合结果所对应的时间窗口信息，需要在 SELECT 子句中使用时间戳相关的伪列： 时间窗口起始时间 (_wstart), 时间窗口结束时间 (_wend) 等
+
+上述特性详细介绍可以参考 [特色查询](../../taos-sql/distinguished/)。
+
+## 创建 Dashboard
+
+回到主界面创建 Dashboard，点击【Add Query】进入面板查询页面：
-如上图所示，在 Query 中选中 `TDengine` 数据源，在下方查询框可输入相应 SQL 进行查询。 我们继续用智能电表来举例，为了展示曲线美观，此处**用了虚拟数据**。
-具体说明如下：
+如上图所示，在 ”Query“ 中选中 `TDengine` 数据源，在下方查询框可输入相应 SQL 进行查询。 我们继续用智能电表来举例，为了展示曲线美观，此处**用了虚拟数据**。
-- INPUT SQL：输入要查询的语句（该 SQL 语句的结果集应为两列多行），例如：`select _wstart as ts, avg(current) as current from power.meters where ts > $from and ts < $to interval($interval) fill(null)` ，其中，from、to 和 interval 为 TDengine 插件的内置变量，表示从 Grafana 插件面板获取的时间查询范围和窗口切分间隔。除了内置变量外，也支持使用自定义模板变量。
-- ALIAS BY：可设置当前查询别名。
-- GENERATE SQL： 点击该按钮会自动替换相应变量，并生成最终执行的语句。
-- Group by column(s)： **半角**逗号分隔的 `group by` 或 `partition by` 列名。如果是 `group by` or `partition by` 查询语句，设置 `Group by` 列，可以展示多维数据。例如：INPUT SQL 为 `select _wstart as ts, groupid, avg(current) as current from power.meters where ts > $from and ts < $to partition by groupid interval($interval) fill(null)`，设置 Group by 列名为 `groupid`，可以按 `groupid` 展示数据。
-- Group By Format： Group by 或 Partition by 场景下多维数据 legend 格式化格式。例如上述 INPUT SQL，将 `Group By Format` 设置为 `groupid-{{groupid}}`，展示的 legend 名字为格式化的分组名。
+## 时间序列数据展示
+假设我们想查询一段时间内的平均电流大小，时间窗口按 `$interval` 切分，若某一时间窗口区间数据缺失，填充 null。
+- “INPUT SQL“：输入要查询的语句（该 SQL 语句的结果集应为两列多行），此处输入：`select _wstart as ts, avg(current) as current from power.meters where groupid in ($selected_groups) and ts > $from and ts < $to interval($interval) fill(null)` ，其中，from、to 和 interval 为 Grafana 内置变量，selected_groups 为自定义变量。
+- “ALIAS BY“：可设置当前查询别名。
+- “GENERATE SQL“： 点击该按钮会自动替换相应变量，并生成最终执行的语句。
+
+在顶部的自定义变量中，若选择 `selected_groups` 的值为 1，则查询 `meters` 超级表中 `groupid` 为 1 的所有设备电流平均值变化如下图：
+
+
:::note
@@ -196,17 +254,24 @@ docker run -d \
:::
-查询 `meters` 超级表所有设备电流平均值变化如下图：
+## 时间序列数据分组展示
+假设我们想查询一段时间内的平均电流大小，按 `groupid` 分组展示，我们可以修改之前的 SQL 为 `select _wstart as ts, groupid, avg(current) as current from power.meters where ts > $from and ts < $to partition by groupid interval($interval) fill(null)`
-
+- “Group by column(s)“： **半角**逗号分隔的 `group by` 或 `partition by` 列名。如果是 `group by` 或 `partition by` 查询语句，设置 “Group by“ 列，可以展示多维数据。此处设置 “Group by“ 列名为 `groupid`，可以按 `groupid` 分组展示数据。
+- “Group By Format“： `Group by` 或 `Partition by` 场景下多维数据 legend 格式化格式。例如上述 INPUT SQL，将 “Group By Format“ 设置为 `groupid-{{groupid}}`，展示的 legend 名字为格式化的分组名。
-查询 `meters` 超级表所有设备电流平均值，并按照 `groupid` 分组展示如下图：
+完成设置后，按照 `groupid` 分组展示如下图：
> 关于如何使用 Grafana 创建相应的监测界面以及更多有关使用 Grafana 的信息，请参考 Grafana 官方的[文档](https://grafana.com/docs/)。
-### 导入 Dashboard
+## 性能建议
+- **所有查询加上时间范围**，在时序数据库中，如果不加查询的时间范围，会扫表导致性能低下。常见的 SQL 写法是 `select column_name from db.table where ts > $from and ts < $to;`
+- 对于最新状态类型的查询，我们一般建议在**创建数据库的时候打开缓存**（`CACHEMODEL` 设置为 last_row 或者 both），常见的 SQL 写法是 `select last(column_name) from db.table where ts > $from and ts < $to;`
+
+
+## 导入 Dashboard
在数据源配置页面，您可以为该数据源导入 TDinsight 面板，作为 TDengine 集群的监控可视化工具。如果 TDengine 服务端为 3.0 版本请选择 `TDinsight for 3.x` 导入。注意 TDinsight for 3.x 需要运行和配置 taoskeeper。
@@ -216,7 +281,148 @@ docker run -d \
使用 TDengine 作为数据源的其他面板，可以[在此搜索](https://grafana.com/grafana/dashboards/?dataSource=tdengine-datasource)。以下是一份不完全列表：
-- [15146](https://grafana.com/grafana/dashboards/15146): 监控多个 TDengine 集群
-- [15155](https://grafana.com/grafana/dashboards/15155): TDengine 告警示例
-- [15167](https://grafana.com/grafana/dashboards/15167): TDinsight
-- [16388](https://grafana.com/grafana/dashboards/16388): Telegraf 采集节点信息的数据展示
+- [15146](https://grafana.com/grafana/dashboards/15146)： 监控多个 TDengine 集群
+- [15155](https://grafana.com/grafana/dashboards/15155)： TDengine 告警示例
+- [15167](https://grafana.com/grafana/dashboards/15167)： TDinsight
+- [16388](https://grafana.com/grafana/dashboards/16388)： Telegraf 采集节点信息的数据展示
+
+## 告警配置简介
+### 告警配置流程
+TDengine Grafana 插件支持告警，如果要配置告警，需要以下几个步骤：
+1. 配置联络点（“Contact points“）：配置通知渠道，包括 DingDing、Email、Slack、WebHook、Prometheus Alertmanager 等
+2. 配置告警通知策略（“Notification policies“）：配置告警发送到哪个通道的路由，以及发送通知的时间和重复频率
+3. 配置 “Alert rules“：配置详细的告警规则
+ 3.1 配置告警名称
+ 3.2 配置查询及告警触发条件
+ 3.3 配置规则评估策略
+ 3.4 配置标签和告警通道
+ 3.5 配置通知文案
+
+### 告警配置界面
+在Grafana 11 告警界面一共有 6 个 Tab，分别是 “Alert rules“、“Contact points“、“Notification policies“、“Silences“、 “Groups“ 和 “Settings“。
+- “Alert rules“ 告警规则列表，用于展示和配置告警规则
+- “Contact points“ 通知渠道，包括 DingDing、Email、Slack、WebHook、Prometheus Alertmanager 等
+- “Notification policies“ 配置告警发送到哪个通道的路由，以及发送通知的时间和重复频率
+- “Silences“ 配置告警静默时间段
+- “Groups“ 告警组，配置的告警触发后会在这里分组显示
+- “Settings“ 提供通过 JSON 方式修改告警配置
+
+## 配置邮件联络点
+### Grafana Server 配置文件修改
+在 Grafana 服务的配置文件中添加 SMTP/Emailing 和 Alerting 模块，以 Linux 系统为例，其配置文件一般位于 `/etc/grafana/grafana.ini`
+在配置文件中增加下面内容：
+
+```ini
+#################################### SMTP / Emailing ##########################
+[smtp]
+enabled = true
+host = smtp.qq.com:465 #使用的邮箱
+user = receiver@foxmail.com
+password = *********** #使用mail授权码
+skip_verify = true
+from_address = sender@foxmail.com
+```
+
+然后重启 Grafana 服务即可， 以 Linux 系统为例，执行 `systemctl restart grafana-server.service`
+
+### Grafana 页面创建新联络点
+
+在 Grafana 页面找到 “Home“ -> “Alerting“ -> “Contact points“，创建新联络点
+”Name“： Email Contact Point
+“Integration“：选择联络类型，这里选择 Email，填写邮件接收地址，完成后保存联络点
+
+
+
+## 配置飞书联络点
+
+### 飞书机器人配置
+1. “飞书工作台“ -> “获取应用“ -> “搜索飞书机器人助手“ -> “新建指令“
+2. 选择触发器：Grafana
+3. 选择操作：通过官方机器人发送消息，填写发送对象和发送内容
+
+
+
+### Grafana 配置飞书联络点
+
+在 Grafana 页面找到 “Home“ -> “Alerting“ -> “Contact points“ 创建新联络点
+“Name“：Feishu Contact Point
+“Integration“：选择联络类型，这里选择 Webhook，并填写 URL (在飞书机器人助手的 Grafana 触发器 Webhook 地址)，完成后保存联络点
+
+
+
+## 通知策略
+配置好联络点后，可以看到已有一个Default Policy
+
+
+
+点击右侧 ”...“ -> ”Edit“，然后编辑默认通知策略，弹出配置窗口：
+
+
+
+然后配置下列参数：
+- “Group wait“： 发送首次告警之前的等待时间。
+- “Group interval“： 发送第一个告警后，为该组发送下一批新告警的等待时间。
+- “Repeat interval“： 成功发送告警后再次重复发送告警的等待时间。
+
+## 配置告警规则
+
+### 配置查询和告警触发条件
+
+在需要配置告警的面板中选择 “Edit“ -> “Alert“ -> “New alert rule“。
+
+1. “Enter alert rule name“ (输入告警规则名称)：此处以智能电表为例输入 `power meters alert`
+2. “Define query and alert condition“ (定义查询和告警触发条件)
+ 2.1 选择数据源：`TDengine Datasource`
+ 2.2 查询语句：
+ ```sql
+ select _wstart as ts, groupid, avg(current) as current from power.meters where ts > $from and ts < $to partition by groupid interval($interval) fill(null)
+ ```
+ 2.3 设置 ”Expression“（表达式）：`Threshold is above 100`
+ 2.4 点击【Set as alert condition】
+ 2.5 “Preview“：查看设置的规则的结果
+
+完成设置后可以看到下面图片展示：
+
+
+
+### 配置表达式和计算规则
+
+Grafana 的 “Expression“（表达式）支持对数据做各种操作和计算，其类型分为：
+1. “Reduce“：将所选时间范围内的时间序列值聚合为单个值
+ 1.1 “Function“ 用来设置聚合方法，支持 Min、Max、Last、Mean、Sum 和 Count。
+ 1.2 “Mode“ 支持下面三种：
+ - “Strict“：如果查询不到数据，数据会赋值为 NaN。
+ - “Drop Non-numeric Value“：去掉非法数据结果。
+ - “Replace Non-numeric Value“：如果是非法数据，使用固定值进行替换。
+2. “Threshold“：检查时间序列数据是否符合阈值判断条件。当条件为假时返回 0，为真则返回1。支持下列方式：
+ - Is above (x > y)
+ - Is below (x < y)
+ - Is within range (x > y1 AND x < y2)
+ - Is outside range (x < y1 AND x > y2)
+3. “Math“：对时间序列的数据进行数学运算。
+4. “Resample“：更改每个时间序列中的时间戳使其具有一致的时间间隔，以便在它们之间执行数学运算。
+5. “Classic condition (legacy)“： 可配置多个逻辑条件，判断是否触发告警。
+
+如上节截图显示，此处我们设置最大值超过 100 触发告警。
+
+### 配置评估策略
+
+
+
+完成下面配置：
+- “Folder“：设置告警规则所属目录。
+- “Evaluation group“：设置告警规则评估组。“Evaluation group“ 可以选择已有组或者新建组，新建组可以设置组名和评估时间间隔。
+- “Pending period“：在告警规则的阈值被触发后，异常值持续多长时间可以触发告警，合理设置可以避免误报。
+
+### 配置标签和告警通道
+
+
+完成下面配置：
+- “Labels“ 将标签添加到规则中，以便进行搜索、静默或路由到通知策略。
+- “Contact point“ 选择联络点，当告警发生时通过设置的联络点进行通知。
+
+### 配置通知文案
+
+
+
+设置 “Summary” 和 ”Description” 后，若告警触发，将会收到告警通知。
diff --git a/docs/zh/20-third-party/11-kafka.md b/docs/zh/20-third-party/11-kafka.md
index 2ee315697f..d42822a187 100644
--- a/docs/zh/20-third-party/11-kafka.md
+++ b/docs/zh/20-third-party/11-kafka.md
@@ -93,7 +93,7 @@ curl http://localhost:8083/connectors
TDengine Sink Connector 的作用是同步指定 topic 的数据到 TDengine。用户无需提前创建数据库和超级表。可手动指定目标数据库的名字（见配置参数 connection.database）， 也可按一定规则生成(见配置参数 connection.database.prefix)。
-TDengine Sink Connector 内部使用 TDengine [无模式写入接口](../../connector/cpp#无模式写入-api)写数据到 TDengine，目前支持三种格式的数据：[InfluxDB 行协议格式](../../develop/insert-data/influxdb-line)、 [OpenTSDB Telnet 协议格式](../../develop/insert-data/opentsdb-telnet) 和 [OpenTSDB JSON 协议格式](../../develop/insert-data/opentsdb-json)。
+TDengine Sink Connector 内部使用 TDengine [无模式写入接口](../../connector/cpp/#无模式schemaless写入-api)写数据到 TDengine，目前支持三种格式的数据：[InfluxDB 行协议格式](../../develop/insert-data/influxdb-line)、 [OpenTSDB Telnet 协议格式](../../develop/insert-data/opentsdb-telnet) 和 [OpenTSDB JSON 协议格式](../../develop/insert-data/opentsdb-json)。
下面的示例将主题 meters 的数据，同步到目标数据库 power。数据格式为 InfluxDB Line 协议格式。
diff --git a/docs/zh/20-third-party/alert-annotations.webp b/docs/zh/20-third-party/alert-annotations.webp
new file mode 100644
index 0000000000..b5001f3e8a
Binary files /dev/null and b/docs/zh/20-third-party/alert-annotations.webp differ
diff --git a/docs/zh/20-third-party/alert-email.webp b/docs/zh/20-third-party/alert-email.webp
new file mode 100644
index 0000000000..53a91de0a8
Binary files /dev/null and b/docs/zh/20-third-party/alert-email.webp differ
diff --git a/docs/zh/20-third-party/alert-evaluation.webp b/docs/zh/20-third-party/alert-evaluation.webp
new file mode 100644
index 0000000000..839510450a
Binary files /dev/null and b/docs/zh/20-third-party/alert-evaluation.webp differ
diff --git a/docs/zh/20-third-party/alert-feishu1.webp b/docs/zh/20-third-party/alert-feishu1.webp
new file mode 100644
index 0000000000..d0a3cdc2cc
Binary files /dev/null and b/docs/zh/20-third-party/alert-feishu1.webp differ
diff --git a/docs/zh/20-third-party/alert-feishu2.webp b/docs/zh/20-third-party/alert-feishu2.webp
new file mode 100644
index 0000000000..558af02d08
Binary files /dev/null and b/docs/zh/20-third-party/alert-feishu2.webp differ
diff --git a/docs/zh/20-third-party/alert-labels.webp b/docs/zh/20-third-party/alert-labels.webp
new file mode 100644
index 0000000000..558fbdd790
Binary files /dev/null and b/docs/zh/20-third-party/alert-labels.webp differ
diff --git a/docs/zh/20-third-party/alert-notification1.webp b/docs/zh/20-third-party/alert-notification1.webp
new file mode 100644
index 0000000000..65f3640d44
Binary files /dev/null and b/docs/zh/20-third-party/alert-notification1.webp differ
diff --git a/docs/zh/20-third-party/alert-notification2.webp b/docs/zh/20-third-party/alert-notification2.webp
new file mode 100644
index 0000000000..c54f74c065
Binary files /dev/null and b/docs/zh/20-third-party/alert-notification2.webp differ
diff --git a/docs/zh/20-third-party/alert-rules1.webp b/docs/zh/20-third-party/alert-rules1.webp
new file mode 100644
index 0000000000..5ea238b4b9
Binary files /dev/null and b/docs/zh/20-third-party/alert-rules1.webp differ
diff --git a/docs/zh/20-third-party/create_dashboard2.webp b/docs/zh/20-third-party/create_dashboard2.webp
index 32a209872d..f9e70cdd31 100644
Binary files a/docs/zh/20-third-party/create_dashboard2.webp and b/docs/zh/20-third-party/create_dashboard2.webp differ
diff --git a/docs/zh/20-third-party/create_dashboard3.webp b/docs/zh/20-third-party/create_dashboard3.webp
index 9ab8e015b7..679e2b094b 100644
Binary files a/docs/zh/20-third-party/create_dashboard3.webp and b/docs/zh/20-third-party/create_dashboard3.webp differ
diff --git a/include/common/tglobal.h b/include/common/tglobal.h
index 96b9617fc4..3fd3cc4ca9 100644
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@@ -134,7 +134,6 @@ extern uint16_t tsMonitorPort;
extern int32_t tsMonitorMaxLogs;
extern bool tsMonitorComp;
extern bool tsMonitorLogProtocol;
-extern int32_t tsMonitorIntervalForBasic;
extern bool tsMonitorForceV2;
// audit
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 4715290f72..29b16ba934 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -966,20 +966,20 @@ int32_t tSerializeSConnectReq(void* buf, int32_t bufLen, SConnectReq* pReq);
int32_t tDeserializeSConnectReq(void* buf, int32_t bufLen, SConnectReq* pReq);
typedef struct {
- int32_t acctId;
- int64_t clusterId;
- uint32_t connId;
- int32_t dnodeNum;
- int8_t superUser;
- int8_t sysInfo;
- int8_t connType;
- SEpSet epSet;
- int32_t svrTimestamp;
- int32_t passVer;
- int32_t authVer;
- char sVer[TSDB_VERSION_LEN];
- char sDetailVer[128];
- int64_t whiteListVer;
+ int32_t acctId;
+ int64_t clusterId;
+ uint32_t connId;
+ int32_t dnodeNum;
+ int8_t superUser;
+ int8_t sysInfo;
+ int8_t connType;
+ SEpSet epSet;
+ int32_t svrTimestamp;
+ int32_t passVer;
+ int32_t authVer;
+ char sVer[TSDB_VERSION_LEN];
+ char sDetailVer[128];
+ int64_t whiteListVer;
SMonitorParas monitorParas;
} SConnectRsp;
@@ -1639,15 +1639,15 @@ void tFreeSFuncInfo(SFuncInfo* pInfo);
void tFreeSRetrieveFuncRsp(SRetrieveFuncRsp* pRsp);
typedef struct {
- int32_t statusInterval;
- int64_t checkTime; // 1970-01-01 00:00:00.000
- char timezone[TD_TIMEZONE_LEN]; // tsTimezone
- char locale[TD_LOCALE_LEN]; // tsLocale
- char charset[TD_LOCALE_LEN]; // tsCharset
- int8_t ttlChangeOnWrite;
- int8_t enableWhiteList;
- int8_t encryptionKeyStat;
- uint32_t encryptionKeyChksum;
+ int32_t statusInterval;
+ int64_t checkTime; // 1970-01-01 00:00:00.000
+ char timezone[TD_TIMEZONE_LEN]; // tsTimezone
+ char locale[TD_LOCALE_LEN]; // tsLocale
+ char charset[TD_LOCALE_LEN]; // tsCharset
+ int8_t ttlChangeOnWrite;
+ int8_t enableWhiteList;
+ int8_t encryptionKeyStat;
+ uint32_t encryptionKeyChksum;
SMonitorParas monitorParas;
} SClusterCfg;
@@ -1746,9 +1746,9 @@ typedef enum {
} MONITOR_TYPE;
typedef struct {
- int32_t contLen;
- char* pCont;
- MONITOR_TYPE type;
+ int32_t contLen;
+ char* pCont;
+ MONITOR_TYPE type;
} SStatisReq;
int32_t tSerializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq);
@@ -3036,8 +3036,8 @@ typedef struct {
int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
} SVCreateTbBatchReq;
-int tEncodeSVCreateTbBatchReq(SEncoder* pCoder, const SVCreateTbBatchReq* pReq);
-int tDecodeSVCreateTbBatchReq(SDecoder* pCoder, SVCreateTbBatchReq* pReq);
+int tEncodeSVCreateTbBatchReq(SEncoder* pCoder, const SVCreateTbBatchReq* pReq);
+int tDecodeSVCreateTbBatchReq(SDecoder* pCoder, SVCreateTbBatchReq* pReq);
void tDeleteSVCreateTbBatchReq(SVCreateTbBatchReq* pReq);
typedef struct {
@@ -3276,10 +3276,10 @@ typedef struct {
} SClientHbRsp;
typedef struct {
- int64_t reqId;
- int64_t rspId;
- int32_t svrTimestamp;
- SArray* rsps; // SArray
+ int64_t reqId;
+ int64_t rspId;
+ int32_t svrTimestamp;
+ SArray* rsps; // SArray
SMonitorParas monitorParas;
} SClientHbBatchRsp;
@@ -3515,7 +3515,7 @@ typedef struct SVUpdateCheckpointInfoReq {
int64_t checkpointVer;
int64_t checkpointTs;
int32_t transId;
- int64_t hStreamId; // add encode/decode
+ int64_t hStreamId; // add encode/decode
int64_t hTaskId;
int8_t dropRelHTask;
} SVUpdateCheckpointInfoReq;
@@ -3994,7 +3994,7 @@ int32_t tDecodeSTaosxRsp(SDecoder* pDecoder, void* pRsp);
void tDeleteSTaosxRsp(void* pRsp);
typedef struct SMqBatchMetaRsp {
- SMqRspHead head; // not serialize
+ SMqRspHead head; // not serialize
STqOffsetVal rspOffset;
SArray* batchMetaLen;
SArray* batchMetaReq;
diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h
index c92649f1f7..3515df3127 100644
--- a/include/common/tmsgdef.h
+++ b/include/common/tmsgdef.h
@@ -250,7 +250,7 @@
TD_DEF_MSG_TYPE(TDMT_MND_DROP_TB_WITH_TSMA, "drop-tb-with-tsma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_UPDATE_CHKPT_EVT, "stream-update-chkpt-evt", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHKPT_REPORT, "stream-chkpt-report", NULL, NULL)
- TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHKPT_CONSEN, "stream-chkpt-consen", NULL, NULL)
+ TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CONSEN_TIMER, "stream-consen-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG)
@@ -341,6 +341,7 @@
TD_DEF_MSG_TYPE(TDMT_STREAM_CREATE, "stream-create", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_DROP, "stream-drop", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE_TRIGGER, "stream-retri-trigger", NULL, NULL)
+ TD_DEF_MSG_TYPE(TDMT_STREAM_CONSEN_CHKPT, "stream-consen-chkpt", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_STREAM_MSG)
TD_NEW_MSG_SEG(TDMT_MON_MSG) //5 << 8
diff --git a/include/common/ttime.h b/include/common/ttime.h
index d890b729d4..f50b5ee9d7 100644
--- a/include/common/ttime.h
+++ b/include/common/ttime.h
@@ -127,6 +127,9 @@ int32_t TEST_char2ts(const char* format, int64_t* ts, int32_t precision, const c
/// @return 0 success, other fail
int32_t offsetOfTimezone(char* tzStr, int64_t* offset);
+bool checkRecursiveTsmaInterval(int64_t baseInterval, int8_t baseUnit, int64_t interval, int8_t unit, int8_t precision,
+ bool checkEq);
+
#ifdef __cplusplus
}
#endif
diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h
index 566e8dbbd8..4d5e18520c 100644
--- a/include/dnode/vnode/tqCommon.h
+++ b/include/dnode/vnode/tqCommon.h
@@ -29,7 +29,8 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg)
int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
-int32_t tqStreamProcessConsensusChkptRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamProcessConsensusChkptRsp2(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sversion, char* msg, int32_t msgLen,
bool isLeader, bool restored);
@@ -37,12 +38,12 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader);
int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta);
int32_t tqStreamTasksGetTotalNum(SStreamMeta* pMeta);
-int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* msg);
int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg);
int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* pMsg, bool fromVnode);
-int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored, char* msg, int32_t msgLen);
+int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored, char* msg);
void tqSetRestoreVersionInfo(SStreamTask* pTask);
int32_t tqExpandStreamTask(SStreamTask* pTask);
diff --git a/include/libs/function/function.h b/include/libs/function/function.h
index 87bbe21133..924682f223 100644
--- a/include/libs/function/function.h
+++ b/include/libs/function/function.h
@@ -41,6 +41,7 @@ typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx);
typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock);
typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
typedef int32_t (*FExecCombine)(struct SqlFunctionCtx *pDestCtx, struct SqlFunctionCtx *pSourceCtx);
+typedef int32_t (*processFuncByRow)(SArray* pCtx); // array of SqlFunctionCtx
typedef struct SScalarFuncExecFuncs {
FExecGetEnv getEnv;
@@ -48,11 +49,12 @@ typedef struct SScalarFuncExecFuncs {
} SScalarFuncExecFuncs;
typedef struct SFuncExecFuncs {
- FExecGetEnv getEnv;
- FExecInit init;
- FExecProcess process;
- FExecFinalize finalize;
- FExecCombine combine;
+ FExecGetEnv getEnv;
+ FExecInit init;
+ FExecProcess process;
+ FExecFinalize finalize;
+ FExecCombine combine;
+ processFuncByRow processFuncByRow;
} SFuncExecFuncs;
#define MAX_INTERVAL_TIME_WINDOW 10000000 // maximum allowed time windows in final results
@@ -253,6 +255,7 @@ typedef struct SqlFunctionCtx {
bool hasPrimaryKey;
SFuncInputRowIter rowIter;
bool bInputFinished;
+ bool hasWindowOrGroup; // denote that the function is used with time window or group
} SqlFunctionCtx;
typedef struct tExprNode {
diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h
index e77635727b..86db6640c5 100644
--- a/include/libs/function/functionMgt.h
+++ b/include/libs/function/functionMgt.h
@@ -255,6 +255,7 @@ bool fmIsIgnoreNullFunc(int32_t funcId);
bool fmIsConstantResFunc(SFunctionNode* pFunc);
bool fmIsSkipScanCheckFunc(int32_t funcId);
bool fmIsPrimaryKeyFunc(int32_t funcId);
+bool fmIsProcessByRowFunc(int32_t funcId);
void getLastCacheDataType(SDataType* pType, int32_t pkBytes);
SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList);
diff --git a/include/libs/monitor/clientMonitor.h b/include/libs/monitor/clientMonitor.h
index 4c7ab6f65a..0085173ecd 100644
--- a/include/libs/monitor/clientMonitor.h
+++ b/include/libs/monitor/clientMonitor.h
@@ -38,6 +38,13 @@ typedef enum {
SLOW_LOG_READ_QUIT = 3,
} SLOW_LOG_QUEUE_TYPE;
+static char* queueTypeStr[] = {
+ "SLOW_LOG_WRITE",
+ "SLOW_LOG_READ_RUNNING",
+ "SLOW_LOG_READ_BEGINNIG",
+ "SLOW_LOG_READ_QUIT"
+};
+
#define SLOW_LOG_SEND_SIZE_MAX 1024*1024
typedef struct {
@@ -65,7 +72,7 @@ typedef struct {
} MonitorSlowLogData;
void monitorClose();
-void monitorInit();
+int32_t monitorInit();
void monitorClientSQLReqInit(int64_t clusterKey);
void monitorClientSlowQueryInit(int64_t clusterId);
diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h
index 9d7878ecf7..6007d52bb4 100644
--- a/include/libs/monitor/monitor.h
+++ b/include/libs/monitor/monitor.h
@@ -226,7 +226,6 @@ void monSetQmInfo(SMonQmInfo *pInfo);
void monSetSmInfo(SMonSmInfo *pInfo);
void monSetBmInfo(SMonBmInfo *pInfo);
void monGenAndSendReport();
-void monGenAndSendReportBasic();
void monSendContent(char *pCont, const char* uri);
void tFreeSMonMmInfo(SMonMmInfo *pInfo);
diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h
index f97240a167..f98ecee5c5 100644
--- a/include/libs/nodes/cmdnodes.h
+++ b/include/libs/nodes/cmdnodes.h
@@ -640,6 +640,7 @@ typedef struct SCreateTSMAStmt {
STSMAOptions* pOptions;
SNode* pPrevQuery;
SMCreateSmaReq* pReq;
+ uint8_t precision;
} SCreateTSMAStmt;
typedef struct SDropTSMAStmt {
diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h
index 457937835d..34b42fd9e1 100644
--- a/include/libs/nodes/querynodes.h
+++ b/include/libs/nodes/querynodes.h
@@ -415,6 +415,7 @@ typedef struct SSelectStmt {
int32_t returnRows; // EFuncReturnRows
ETimeLineMode timeLineCurMode;
ETimeLineMode timeLineResMode;
+ int32_t lastProcessByRowFuncId;
bool timeLineFromOrderBy;
bool isEmptyResult;
bool isSubquery;
diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h
index ad41b9a542..3ac357055e 100644
--- a/include/libs/parser/parser.h
+++ b/include/libs/parser/parser.h
@@ -89,6 +89,7 @@ typedef struct SParseContext {
bool isView;
bool isAudit;
bool nodeOffline;
+ bool isStmtBind;
const char* svrVer;
SArray* pTableMetaPos; // sql table pos => catalog data pos
SArray* pTableVgroupPos; // sql table pos => catalog data pos
diff --git a/include/libs/stream/streamMsg.h b/include/libs/stream/streamMsg.h
index b69032330d..34921daac3 100644
--- a/include/libs/stream/streamMsg.h
+++ b/include/libs/stream/streamMsg.h
@@ -216,6 +216,7 @@ typedef struct SRestoreCheckpointInfo {
int64_t startTs;
int64_t streamId;
int64_t checkpointId; // latest checkpoint id
+ int32_t transId; // transaction id of the update the consensus-checkpointId transaction
int32_t taskId;
int32_t nodeId;
} SRestoreCheckpointInfo;
@@ -223,16 +224,6 @@ typedef struct SRestoreCheckpointInfo {
int32_t tEncodeRestoreCheckpointInfo (SEncoder* pEncoder, const SRestoreCheckpointInfo* pReq);
int32_t tDecodeRestoreCheckpointInfo(SDecoder* pDecoder, SRestoreCheckpointInfo* pReq);
-typedef struct SRestoreCheckpointInfoRsp {
- int64_t streamId;
- int64_t checkpointId;
- int64_t startTs;
- int32_t taskId;
-} SRestoreCheckpointInfoRsp;
-
-int32_t tEncodeRestoreCheckpointInfoRsp(SEncoder* pCoder, const SRestoreCheckpointInfoRsp* pInfo);
-int32_t tDecodeRestoreCheckpointInfoRsp(SDecoder* pCoder, SRestoreCheckpointInfoRsp* pInfo);
-
typedef struct {
SMsgHead head;
int64_t streamId;
@@ -242,8 +233,7 @@ typedef struct {
typedef struct SCheckpointConsensusEntry {
SRestoreCheckpointInfo req;
- SRpcMsg rsp;
- int64_t ts;
+ int64_t ts;
} SCheckpointConsensusEntry;
#ifdef __cplusplus
diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index e98039d2fe..5ba0ce454c 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -272,9 +272,9 @@ typedef struct SCheckpointInfo {
int64_t checkpointTime; // latest checkpoint time
int64_t processedVer;
int64_t nextProcessVer; // current offset in WAL, not serialize it
-
+ int64_t msgVer;
+ int32_t consensusTransId;// consensus checkpoint id
SActiveCheckpointInfo* pActiveInfo;
- int64_t msgVer;
} SCheckpointInfo;
typedef struct SStreamStatus {
@@ -289,6 +289,8 @@ typedef struct SStreamStatus {
int32_t inScanHistorySentinel;
bool appendTranstateBlock; // has append the transfer state data block already
bool removeBackendFiles; // remove backend files on disk when free stream tasks
+ bool sendConsensusChkptId;
+ bool requireConsensusChkptId;
} SStreamStatus;
typedef struct SDataRange {
@@ -528,10 +530,11 @@ typedef int32_t (*__state_trans_user_fn)(SStreamTask*, void* param);
SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam,
SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5);
+void tFreeStreamTask(SStreamTask* pTask);
int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
-void tFreeStreamTask(SStreamTask* pTask);
int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, int64_t ver);
+void streamFreeTaskState(SStreamTask* pTask, ETaskStatus status);
int32_t tDecodeStreamTaskChkInfo(SDecoder* pDecoder, SCheckpointInfo* pChkpInfo);
int32_t tDecodeStreamTaskId(SDecoder* pDecoder, STaskId* pTaskId);
@@ -568,14 +571,16 @@ typedef struct {
} SStreamScanHistoryReq;
typedef struct STaskCkptInfo {
- int64_t latestId; // saved checkpoint id
- int64_t latestVer; // saved checkpoint ver
- int64_t latestTime; // latest checkpoint time
- int64_t latestSize; // latest checkpoint size
- int8_t remoteBackup; // latest checkpoint backup done
- int64_t activeId; // current active checkpoint id
- int32_t activeTransId; // checkpoint trans id
- int8_t failed; // denote if the checkpoint is failed or not
+ int64_t latestId; // saved checkpoint id
+ int64_t latestVer; // saved checkpoint ver
+ int64_t latestTime; // latest checkpoint time
+ int64_t latestSize; // latest checkpoint size
+ int8_t remoteBackup; // latest checkpoint backup done
+ int64_t activeId; // current active checkpoint id
+ int32_t activeTransId; // checkpoint trans id
+ int8_t failed; // denote if the checkpoint is failed or not
+ int8_t consensusChkptId; // required the consensus-checkpointId
+ int64_t consensusTs; //
} STaskCkptInfo;
typedef struct STaskStatusEntry {
@@ -586,8 +591,6 @@ typedef struct STaskStatusEntry {
int32_t nodeId;
SVersionRange verRange; // start/end version in WAL, only valid for source task
int64_t processedVer; // only valid for source task
- bool inputQChanging; // inputQ is changing or not
- int64_t inputQUnchangeCounter;
double inputQUsed; // in MiB
double inputRate;
double procsThroughput; // duration between one element put into input queue and being processed.
@@ -616,8 +619,8 @@ typedef struct SStreamTaskState {
typedef struct SCheckpointConsensusInfo {
SArray* pTaskList;
- int64_t checkpointId;
- int64_t genTs;
+ int32_t numOfTasks;
+ int64_t streamId;
} SCheckpointConsensusInfo;
int32_t streamSetupScheduleTrigger(SStreamTask* pTask);
@@ -800,6 +803,7 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r
void streamTaskSendRetrieveRsp(SStreamRetrieveReq* pReq, SRpcMsg* pRsp);
int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp);
+int32_t streamTaskSendPreparedCheckpointsourceRsp(SStreamTask* pTask);
#ifdef __cplusplus
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 7ac22ac40f..24f9d041fc 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -46,9 +46,11 @@ const char* terrstr();
char* taosGetErrMsgReturn();
char* taosGetErrMsg();
int32_t* taosGetErrno();
+int32_t* taosGetErrln();
int32_t taosGetErrSize();
#define terrno (*taosGetErrno())
#define terrMsg (taosGetErrMsg())
+#define terrln (*taosGetErrln())
#define SET_ERROR_MSG(MSG, ...) \
snprintf(terrMsg, ERR_MSG_LEN, MSG, ##__VA_ARGS__)
@@ -136,6 +138,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_TIMEOUT_ERROR TAOS_DEF_ERROR_CODE(0, 0x012C)
#define TSDB_CODE_MSG_ENCODE_ERROR TAOS_DEF_ERROR_CODE(0, 0x012D)
#define TSDB_CODE_NO_ENOUGH_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x012E)
+#define TSDB_CODE_THIRDPARTY_ERROR TAOS_DEF_ERROR_CODE(0, 0x012F)
#define TSDB_CODE_APP_IS_STARTING TAOS_DEF_ERROR_CODE(0, 0x0130)
#define TSDB_CODE_APP_IS_STOPPING TAOS_DEF_ERROR_CODE(0, 0x0131)
@@ -145,6 +148,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_IP_NOT_IN_WHITE_LIST TAOS_DEF_ERROR_CODE(0, 0x0134)
#define TSDB_CODE_FAILED_TO_CONNECT_S3 TAOS_DEF_ERROR_CODE(0, 0x0135)
#define TSDB_CODE_MSG_PREPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0136) // internal
+#define TSDB_CODE_OUT_OF_BUFFER TAOS_DEF_ERROR_CODE(0, 0x0137)
//client
#define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200)
@@ -831,6 +835,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_PAR_TBNAME_ERROR TAOS_DEF_ERROR_CODE(0, 0x267D)
#define TSDB_CODE_PAR_TBNAME_DUPLICATED TAOS_DEF_ERROR_CODE(0, 0x267E)
#define TSDB_CODE_PAR_TAG_NAME_DUPLICATED TAOS_DEF_ERROR_CODE(0, 0x267F)
+#define TSDB_CODE_PAR_NOT_ALLOWED_DIFFERENT_BY_ROW_FUNC TAOS_DEF_ERROR_CODE(0, 0x2680)
#define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF)
//planner
diff --git a/include/util/tcompression.h b/include/util/tcompression.h
index 683a8f2cee..cea648a3a6 100644
--- a/include/util/tcompression.h
+++ b/include/util/tcompression.h
@@ -198,15 +198,6 @@ int32_t tsCompressBigint2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int3
int32_t nBuf);
int32_t tsDecompressBigint2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg,
void *pBuf, int32_t nBuf);
-// for internal usage
-int32_t getWordLength(char type);
-
-int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, char *const output, const char type);
-int32_t tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output);
-int32_t tsDecompressFloatImplAvx2(const char *const input, const int32_t nelements, char *const output);
-int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output,
- bool bigEndian);
-int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, char *const output, bool bigEndian);
/*************************************************************************
* STREAM COMPRESSION
diff --git a/include/util/tutil.h b/include/util/tutil.h
index c049949590..d1a18dc3e8 100644
--- a/include/util/tutil.h
+++ b/include/util/tutil.h
@@ -117,6 +117,15 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
}
}
+#define TAOS_CHECK_ERRNO(CODE) \
+ do { \
+ terrno = (CODE); \
+ if (terrno != TSDB_CODE_SUCCESS) { \
+ terrln = __LINE__; \
+ goto _exit; \
+ } \
+ } while (0)
+
#define TSDB_CHECK_CODE(CODE, LINO, LABEL) \
do { \
if (TSDB_CODE_SUCCESS != (CODE)) { \
diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h
index 7a84215e12..7c342b85d4 100644
--- a/source/client/inc/clientInt.h
+++ b/source/client/inc/clientInt.h
@@ -283,6 +283,7 @@ typedef struct SRequestObj {
bool inRetry;
bool isSubReq;
bool inCallback;
+ bool isStmtBind; // is statement bind parameter
uint32_t prevCode; // previous error code: todo refactor, add update flag for catalog
uint32_t retry;
int64_t allocatorRefId;
@@ -454,6 +455,8 @@ enum {
void sqlReqLog(int64_t rid, bool killed, int32_t code, int8_t type);
+void tmqMgmtClose(void);
+
#ifdef __cplusplus
}
#endif
diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c
index 3a821768f8..ecfa1e3392 100644
--- a/source/client/src/clientEnv.c
+++ b/source/client/src/clientEnv.c
@@ -864,10 +864,15 @@ void taos_init_imp(void) {
initQueryModuleMsgHandle();
if (taosConvInit() != 0) {
+ tscInitRes = -1;
tscError("failed to init conv");
return;
}
-
+ if (monitorInit() != 0){
+ tscInitRes = -1;
+ tscError("failed to init monitor");
+ return;
+ }
rpcInit();
SCatalogCfg cfg = {.maxDBCacheNum = 100, .maxTblCacheNum = 100};
@@ -891,7 +896,6 @@ void taos_init_imp(void) {
taosThreadMutexInit(&appInfo.mutex, NULL);
tscCrashReportInit();
- monitorInit();
tscDebug("client is initialized successfully");
}
diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 7034778887..ffbc56415d 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -206,6 +206,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
(*pRequest)->sqlstr[sqlLen] = 0;
(*pRequest)->sqlLen = sqlLen;
(*pRequest)->validateOnly = validateSql;
+ (*pRequest)->isStmtBind = false;
((SSyncQueryParam*)(*pRequest)->body.interParam)->userParam = param;
@@ -266,7 +267,8 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC
.isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)),
.enableSysInfo = pTscObj->sysInfo,
.svrVer = pTscObj->sVer,
- .nodeOffline = (pTscObj->pAppInfo->onlineDnodes < pTscObj->pAppInfo->totalDnodes)};
+ .nodeOffline = (pTscObj->pAppInfo->onlineDnodes < pTscObj->pAppInfo->totalDnodes),
+ .isStmtBind = pRequest->isStmtBind};
cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog);
@@ -299,7 +301,7 @@ int32_t execLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
int8_t biMode = atomic_load_8(&pRequest->pTscObj->biMode);
int32_t code = qExecCommand(&pRequest->pTscObj->id, pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp, biMode);
if (TSDB_CODE_SUCCESS == code && NULL != pRsp) {
- code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false);
+ code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, pRequest->body.resInfo.convertUcs4);
}
return code;
@@ -338,7 +340,7 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
int32_t code = qExecCommand(&pRequest->pTscObj->id, pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp,
atomic_load_8(&pRequest->pTscObj->biMode));
if (TSDB_CODE_SUCCESS == code && NULL != pRsp) {
- code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false);
+ code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, pRequest->body.resInfo.convertUcs4);
}
SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index 56f89ffba6..f65edc103a 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -57,8 +57,6 @@ void taos_cleanup(void) {
}
monitorClose();
- taosHashCleanup(appInfo.pInstMap);
- taosHashCleanup(appInfo.pInstMapByClusterId);
tscStopCrashReport();
hbMgrCleanUp();
@@ -85,6 +83,8 @@ void taos_cleanup(void) {
taosConvDestroy();
+ tmqMgmtClose();
+
tscInfo("all local resources released");
taosCleanupCfg();
taosCloseLog();
diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c
index 479ea76fe3..ae2a57ba97 100644
--- a/source/client/src/clientMonitor.c
+++ b/source/client/src/clientMonitor.c
@@ -18,6 +18,7 @@ int32_t quitCnt = 0;
tsem2_t monitorSem;
STaosQueue* monitorQueue;
SHashObj* monitorSlowLogHash;
+char tmpSlowLogPath[PATH_MAX] = {0};
static int32_t getSlowLogTmpDir(char* tmpPath, int32_t size){
if (tsTempDir == NULL) {
@@ -453,6 +454,10 @@ static int64_t getFileSize(char* path){
}
static int32_t sendSlowLog(int64_t clusterId, char* data, TdFilePtr pFile, int64_t offset, SLOW_LOG_QUEUE_TYPE type, char* fileName, void* pTransporter, SEpSet *epSet){
+ if (data == NULL){
+ taosMemoryFree(fileName);
+ return -1;
+ }
MonitorSlowLogData* pParam = taosMemoryMalloc(sizeof(MonitorSlowLogData));
if(pParam == NULL){
taosMemoryFree(data);
@@ -468,18 +473,26 @@ static int32_t sendSlowLog(int64_t clusterId, char* data, TdFilePtr pFile, int64
return sendReport(pTransporter, epSet, data, MONITOR_TYPE_SLOW_LOG, pParam);
}
-static void monitorSendSlowLogAtBeginning(int64_t clusterId, char** fileName, TdFilePtr pFile, int64_t offset, void* pTransporter, SEpSet *epSet){
+static int32_t monitorReadSend(int64_t clusterId, TdFilePtr pFile, int64_t* offset, int64_t size, SLOW_LOG_QUEUE_TYPE type, char* fileName){
+ SAppInstInfo* pInst = getAppInstByClusterId(clusterId);
+ if(pInst == NULL){
+ tscError("failed to get app instance by clusterId:%" PRId64, clusterId);
+ return -1;
+ }
+ SEpSet ep = getEpSet_s(&pInst->mgmtEp);
+ char* data = readFile(pFile, offset, size);
+ return sendSlowLog(clusterId, data, (type == SLOW_LOG_READ_BEGINNIG ? pFile : NULL), *offset, type, fileName, pInst->pTransporter, &ep);
+}
+
+static void monitorSendSlowLogAtBeginning(int64_t clusterId, char** fileName, TdFilePtr pFile, int64_t offset){
int64_t size = getFileSize(*fileName);
if(size <= offset){
processFileInTheEnd(pFile, *fileName);
tscDebug("[monitor] monitorSendSlowLogAtBeginning delete file:%s", *fileName);
}else{
- char* data = readFile(pFile, &offset, size);
- if(data != NULL){
- sendSlowLog(clusterId, data, pFile, offset, SLOW_LOG_READ_BEGINNIG, *fileName, pTransporter, epSet);
- *fileName = NULL;
- }
- tscDebug("[monitor] monitorSendSlowLogAtBeginning send slow log file:%p, data:%s", pFile, data);
+ int32_t code = monitorReadSend(clusterId, pFile, &offset, size, SLOW_LOG_READ_BEGINNIG, *fileName);
+ tscDebug("[monitor] monitorSendSlowLogAtBeginning send slow log clusterId:%"PRId64",ret:%d", clusterId, code);
+ *fileName = NULL;
}
}
@@ -500,17 +513,8 @@ static void monitorSendSlowLogAtRunning(int64_t clusterId){
tscDebug("[monitor] monitorSendSlowLogAtRunning truncate file to 0 file:%p", pClient->pFile);
pClient->offset = 0;
}else{
- SAppInstInfo* pInst = getAppInstByClusterId(clusterId);
- if(pInst == NULL){
- tscError("failed to get app instance by clusterId:%" PRId64, clusterId);
- return;
- }
- SEpSet ep = getEpSet_s(&pInst->mgmtEp);
- char* data = readFile(pClient->pFile, &pClient->offset, size);
- if(data != NULL){
- sendSlowLog(clusterId, data, pClient->pFile, pClient->offset, SLOW_LOG_READ_RUNNING, NULL, pInst->pTransporter, &ep);
- }
- tscDebug("[monitor] monitorSendSlowLogAtRunning send slow log:%s", data);
+ int32_t code = monitorReadSend(clusterId, pClient->pFile, &pClient->offset, size, SLOW_LOG_READ_RUNNING, NULL);
+ tscDebug("[monitor] monitorSendSlowLogAtRunning send slow log clusterId:%"PRId64",ret:%d", clusterId, code);
}
}
@@ -532,16 +536,8 @@ static bool monitorSendSlowLogAtQuit(int64_t clusterId) {
return true;
}
}else{
- SAppInstInfo* pInst = getAppInstByClusterId(clusterId);
- if(pInst == NULL) {
- return true;
- }
- SEpSet ep = getEpSet_s(&pInst->mgmtEp);
- char* data = readFile(pClient->pFile, &pClient->offset, size);
- if(data != NULL){
- sendSlowLog(clusterId, data, pClient->pFile, pClient->offset, SLOW_LOG_READ_QUIT, NULL, pInst->pTransporter, &ep);
- }
- tscInfo("[monitor] monitorSendSlowLogAtQuit send slow log:%s", data);
+ int32_t code = monitorReadSend(clusterId, pClient->pFile, &pClient->offset, size, SLOW_LOG_READ_QUIT, NULL);
+ tscDebug("[monitor] monitorSendSlowLogAtQuit send slow log clusterId:%"PRId64",ret:%d", clusterId, code);
}
return false;
}
@@ -558,16 +554,11 @@ static void monitorSendAllSlowLogAtQuit(){
pClient->pFile = NULL;
}else if(pClient->offset == 0){
int64_t* clusterId = (int64_t*)taosHashGetKey(pIter, NULL);
- SAppInstInfo* pInst = getAppInstByClusterId(*clusterId);
- if(pInst == NULL) {
- continue;
- }
- SEpSet ep = getEpSet_s(&pInst->mgmtEp);
- char* data = readFile(pClient->pFile, &pClient->offset, size);
- if(data != NULL && sendSlowLog(*clusterId, data, NULL, pClient->offset, SLOW_LOG_READ_QUIT, NULL, pInst->pTransporter, &ep) == 0){
+ int32_t code = monitorReadSend(*clusterId, pClient->pFile, &pClient->offset, size, SLOW_LOG_READ_QUIT, NULL);
+ tscDebug("[monitor] monitorSendAllSlowLogAtQuit send slow log clusterId:%"PRId64",ret:%d", *clusterId, code);
+ if (code == 0){
quitCnt ++;
}
- tscInfo("[monitor] monitorSendAllSlowLogAtQuit send slow log :%s", data);
}
}
}
@@ -613,12 +604,8 @@ static void monitorSendAllSlowLog(){
}
continue;
}
- SEpSet ep = getEpSet_s(&pInst->mgmtEp);
- char* data = readFile(pClient->pFile, &pClient->offset, size);
- if(data != NULL){
- sendSlowLog(*clusterId, data, NULL, pClient->offset, SLOW_LOG_READ_RUNNING, NULL, pInst->pTransporter, &ep);
- }
- tscDebug("[monitor] monitorSendAllSlowLog send slow log :%s", data);
+ int32_t code = monitorReadSend(*clusterId, pClient->pFile, &pClient->offset, size, SLOW_LOG_READ_RUNNING, NULL);
+ tscDebug("[monitor] monitorSendAllSlowLog send slow log clusterId:%"PRId64",ret:%d", *clusterId, code);
}
}
}
@@ -631,7 +618,7 @@ static void monitorSendAllSlowLogFromTempDir(int64_t clusterId){
return;
}
char namePrefix[PATH_MAX] = {0};
- if (snprintf(namePrefix, sizeof(namePrefix), "%s%"PRIx64, TD_TMP_FILE_PREFIX, pInst->clusterId) < 0) {
+ if (snprintf(namePrefix, sizeof(namePrefix), "%s%"PRIx64, TD_TMP_FILE_PREFIX, clusterId) < 0) {
tscError("failed to generate slow log file name prefix");
return;
}
@@ -656,7 +643,7 @@ static void monitorSendAllSlowLogFromTempDir(int64_t clusterId){
if (strcmp(name, ".") == 0 ||
strcmp(name, "..") == 0 ||
strstr(name, namePrefix) == NULL) {
- tscInfo("skip file:%s, for cluster id:%"PRIx64, name, pInst->clusterId);
+ tscInfo("skip file:%s, for cluster id:%"PRIx64, name, clusterId);
continue;
}
@@ -672,9 +659,8 @@ static void monitorSendAllSlowLogFromTempDir(int64_t clusterId){
taosCloseFile(&pFile);
continue;
}
- SEpSet ep = getEpSet_s(&pInst->mgmtEp);
char *tmp = taosStrdup(filename);
- monitorSendSlowLogAtBeginning(pInst->clusterId, &tmp, pFile, 0, pInst->pTransporter, &ep);
+ monitorSendSlowLogAtBeginning(clusterId, &tmp, pFile, 0);
taosMemoryFree(tmp);
}
@@ -690,35 +676,13 @@ static void* monitorThreadFunc(void *param){
}
#endif
- char tmpPath[PATH_MAX] = {0};
- if (getSlowLogTmpDir(tmpPath, sizeof(tmpPath)) < 0){
- return NULL;
- }
-
- if (taosMulModeMkDir(tmpPath, 0777, true) != 0) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- printf("failed to create dir:%s since %s", tmpPath, terrstr());
- return NULL;
- }
-
- if (tsem2_init(&monitorSem, 0, 0) != 0) {
- tscError("sem init error since %s", terrstr());
- return NULL;
- }
-
- monitorQueue = taosOpenQueue();
- if(monitorQueue == NULL){
- tscError("open queue error since %s", terrstr());
- return NULL;
- }
-
if (-1 != atomic_val_compare_exchange_32(&slowLogFlag, -1, 0)) {
return NULL;
}
tscDebug("monitorThreadFunc start");
int64_t quitTime = 0;
while (1) {
- if (slowLogFlag > 0) {
+ if (atomic_load_32(&slowLogFlag) > 0 > 0) {
if(quitCnt == 0){
monitorSendAllSlowLogAtQuit();
if(quitCnt == 0){
@@ -738,16 +702,12 @@ static void* monitorThreadFunc(void *param){
if (slowLogData != NULL) {
if (slowLogData->type == SLOW_LOG_READ_BEGINNIG){
if(slowLogData->pFile != NULL){
- SAppInstInfo* pInst = getAppInstByClusterId(slowLogData->clusterId);
- if(pInst != NULL) {
- SEpSet ep = getEpSet_s(&pInst->mgmtEp);
- monitorSendSlowLogAtBeginning(slowLogData->clusterId, &(slowLogData->fileName), slowLogData->pFile, slowLogData->offset, pInst->pTransporter, &ep);
- }
+ monitorSendSlowLogAtBeginning(slowLogData->clusterId, &(slowLogData->fileName), slowLogData->pFile, slowLogData->offset);
}else{
monitorSendAllSlowLogFromTempDir(slowLogData->clusterId);
}
} else if(slowLogData->type == SLOW_LOG_WRITE){
- monitorWriteSlowLog2File(slowLogData, tmpPath);
+ monitorWriteSlowLog2File(slowLogData, tmpSlowLogPath);
} else if(slowLogData->type == SLOW_LOG_READ_RUNNING){
monitorSendSlowLogAtRunning(slowLogData->clusterId);
} else if(slowLogData->type == SLOW_LOG_READ_QUIT){
@@ -767,10 +727,7 @@ static void* monitorThreadFunc(void *param){
}
tsem2_timewait(&monitorSem, 100);
}
-
- taosCloseQueue(monitorQueue);
- tsem2_destroy(&monitorSem);
- slowLogFlag = -2;
+ atomic_store_32(&slowLogFlag, -2);
return NULL;
}
@@ -799,27 +756,59 @@ static void tscMonitorStop() {
}
}
-void monitorInit() {
+int32_t monitorInit() {
tscInfo("[monitor] tscMonitor init");
monitorCounterHash = (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
if (monitorCounterHash == NULL) {
tscError("failed to create monitorCounterHash");
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
}
taosHashSetFreeFp(monitorCounterHash, destroyMonitorClient);
monitorSlowLogHash = (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
if (monitorSlowLogHash == NULL) {
tscError("failed to create monitorSlowLogHash");
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
}
taosHashSetFreeFp(monitorSlowLogHash, destroySlowLogClient);
monitorTimer = taosTmrInit(0, 0, 0, "MONITOR");
if (monitorTimer == NULL) {
tscError("failed to create monitor timer");
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ if (getSlowLogTmpDir(tmpSlowLogPath, sizeof(tmpSlowLogPath)) < 0){
+ terrno = TSDB_CODE_TSC_INTERNAL_ERROR;
+ return -1;
+ }
+
+ if (taosMulModeMkDir(tmpSlowLogPath, 0777, true) != 0) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ tscError("failed to create dir:%s since %s", tmpSlowLogPath, terrstr());
+ return -1;
+ }
+
+ if (tsem2_init(&monitorSem, 0, 0) != 0) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ tscError("sem init error since %s", terrstr());
+ return -1;
+ }
+
+ monitorQueue = taosOpenQueue();
+ if(monitorQueue == NULL){
+ tscError("open queue error since %s", terrstr());
+ return -1;
}
taosInitRWLatch(&monitorLock);
- tscMonitortInit();
+ if (tscMonitortInit() != 0){
+ return -1;
+ }
+ return 0;
}
void monitorClose() {
@@ -834,20 +823,23 @@ void monitorClose() {
taosHashCleanup(monitorCounterHash);
taosHashCleanup(monitorSlowLogHash);
taosTmrCleanUp(monitorTimer);
+ taosCloseQueue(monitorQueue);
+ tsem2_destroy(&monitorSem);
taosWUnLockLatch(&monitorLock);
}
int32_t monitorPutData2MonitorQueue(MonitorSlowLogData data){
+ if (atomic_load_32(&slowLogFlag) == -2) {
+ tscError("[monitor] slow log thread is exiting");
+ return -1;
+ }
MonitorSlowLogData* slowLogData = taosAllocateQitem(sizeof(MonitorSlowLogData), DEF_QITEM, 0);
if (slowLogData == NULL) {
tscError("[monitor] failed to allocate slow log data");
return -1;
}
*slowLogData = data;
- tscDebug("[monitor] write slow log to queue, clusterId:%"PRIx64 " type:%d", slowLogData->clusterId, slowLogData->type);
- while (atomic_load_32(&slowLogFlag) == -1) {
- taosMsleep(5);
- }
+ tscDebug("[monitor] write slow log to queue, clusterId:%"PRIx64 " type:%s, data:%s", slowLogData->clusterId, queueTypeStr[slowLogData->type], slowLogData->data);
if (taosWriteQitem(monitorQueue, slowLogData) == 0){
tsem2_post(&monitorSem);
}else{
diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c
index e5baa7137e..d587deffc5 100644
--- a/source/client/src/clientMsgHandler.c
+++ b/source/client/src/clientMsgHandler.c
@@ -154,13 +154,14 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
if(taosHashGet(appInfo.pInstMapByClusterId, &connectRsp.clusterId, LONG_BYTES) == NULL){
if(taosHashPut(appInfo.pInstMapByClusterId, &connectRsp.clusterId, LONG_BYTES, &pTscObj->pAppInfo, POINTER_BYTES) != 0){
tscError("failed to put appInfo into appInfo.pInstMapByClusterId");
+ }else{
+ MonitorSlowLogData data = {0};
+ data.clusterId = pTscObj->pAppInfo->clusterId;
+ data.type = SLOW_LOG_READ_BEGINNIG;
+ monitorPutData2MonitorQueue(data);
+ monitorClientSlowQueryInit(connectRsp.clusterId);
+ monitorClientSQLReqInit(connectRsp.clusterId);
}
- MonitorSlowLogData data = {0};
- data.clusterId = pTscObj->pAppInfo->clusterId;
- data.type = SLOW_LOG_READ_BEGINNIG;
- monitorPutData2MonitorQueue(data);
- monitorClientSlowQueryInit(connectRsp.clusterId);
- monitorClientSQLReqInit(connectRsp.clusterId);
}
taosThreadMutexLock(&clientHbMgr.lock);
diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c
index e8b76d34c2..17b52521b8 100644
--- a/source/client/src/clientStmt.c
+++ b/source/client/src/clientStmt.c
@@ -72,6 +72,7 @@ static int32_t stmtCreateRequest(STscStmt* pStmt) {
}
if (TSDB_CODE_SUCCESS == code) {
pStmt->exec.pRequest->syncQuery = true;
+ pStmt->exec.pRequest->isStmtBind = true;
}
}
@@ -830,6 +831,7 @@ TAOS_STMT* stmtInit(STscObj* taos, int64_t reqid, TAOS_STMT_OPTIONS* pOptions) {
pStmt->bInfo.needParse = true;
pStmt->sql.status = STMT_INIT;
pStmt->reqid = reqid;
+ pStmt->errCode = TSDB_CODE_SUCCESS;
if (NULL != pOptions) {
memcpy(&pStmt->options, pOptions, sizeof(pStmt->options));
@@ -882,6 +884,10 @@ int stmtPrepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
STMT_DLOG_E("start to prepare");
+ if (pStmt->errCode != TSDB_CODE_SUCCESS) {
+ return pStmt->errCode;
+ }
+
if (pStmt->sql.status >= STMT_PREPARE) {
STMT_ERR_RET(stmtResetStmt(pStmt));
}
@@ -953,6 +959,10 @@ int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) {
STMT_DLOG("start to set tbName: %s", tbName);
+ if (pStmt->errCode != TSDB_CODE_SUCCESS) {
+ return pStmt->errCode;
+ }
+
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTBNAME));
int32_t insert = 0;
@@ -999,8 +1009,18 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) {
STMT_DLOG_E("start to set tbTags");
+ if (pStmt->errCode != TSDB_CODE_SUCCESS) {
+ return pStmt->errCode;
+ }
+
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS));
+ SBoundColInfo *tags_info = (SBoundColInfo*)pStmt->bInfo.boundTags;
+ if (tags_info->numOfBound <= 0 || tags_info->numOfCols <= 0) {
+ tscWarn("no tags bound in sql, will not bound tags");
+ return TSDB_CODE_SUCCESS;
+ }
+
if (pStmt->bInfo.inExecCache) {
return TSDB_CODE_SUCCESS;
}
@@ -1021,6 +1041,10 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) {
}
int stmtFetchTagFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields) {
+ if (pStmt->errCode != TSDB_CODE_SUCCESS) {
+ return pStmt->errCode;
+ }
+
if (STMT_TYPE_QUERY == pStmt->sql.type) {
tscError("invalid operation to get query tag fileds");
STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
@@ -1039,6 +1063,10 @@ int stmtFetchTagFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields
}
int stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields) {
+ if (pStmt->errCode != TSDB_CODE_SUCCESS) {
+ return pStmt->errCode;
+ }
+
if (STMT_TYPE_QUERY == pStmt->sql.type) {
tscError("invalid operation to get query column fileds");
STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
@@ -1150,8 +1178,13 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
STMT_DLOG("start to bind stmt data, colIdx: %d", colIdx);
+ if (pStmt->errCode != TSDB_CODE_SUCCESS) {
+ return pStmt->errCode;
+ }
+
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND));
+
if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 &&
STMT_TYPE_MULTI_INSERT != pStmt->sql.type) {
pStmt->bInfo.needParse = false;
@@ -1307,6 +1340,10 @@ int stmtAddBatch(TAOS_STMT* stmt) {
STMT_DLOG_E("start to add batch");
+ if (pStmt->errCode != TSDB_CODE_SUCCESS) {
+ return pStmt->errCode;
+ }
+
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_ADD_BATCH));
if (pStmt->sql.stbInterlaceMode) {
@@ -1471,6 +1508,10 @@ int stmtExec(TAOS_STMT* stmt) {
STMT_DLOG_E("start to exec");
+ if (pStmt->errCode != TSDB_CODE_SUCCESS) {
+ return pStmt->errCode;
+ }
+
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE));
if (STMT_TYPE_QUERY == pStmt->sql.type) {
@@ -1599,6 +1640,10 @@ int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
STMT_DLOG_E("start to get tag fields");
+ if (pStmt->errCode != TSDB_CODE_SUCCESS) {
+ return pStmt->errCode;
+ }
+
if (STMT_TYPE_QUERY == pStmt->sql.type) {
STMT_ERRI_JRET(TSDB_CODE_TSC_STMT_API_ERROR);
}
@@ -1637,6 +1682,10 @@ int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
STMT_DLOG_E("start to get col fields");
+ if (pStmt->errCode != TSDB_CODE_SUCCESS) {
+ return pStmt->errCode;
+ }
+
if (STMT_TYPE_QUERY == pStmt->sql.type) {
STMT_ERRI_JRET(TSDB_CODE_TSC_STMT_API_ERROR);
}
@@ -1674,6 +1723,10 @@ int stmtGetParamNum(TAOS_STMT* stmt, int* nums) {
STMT_DLOG_E("start to get param num");
+ if (pStmt->errCode != TSDB_CODE_SUCCESS) {
+ return pStmt->errCode;
+ }
+
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS));
if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 &&
@@ -1706,6 +1759,10 @@ int stmtGetParam(TAOS_STMT* stmt, int idx, int* type, int* bytes) {
STMT_DLOG_E("start to get param");
+ if (pStmt->errCode != TSDB_CODE_SUCCESS) {
+ return pStmt->errCode;
+ }
+
if (STMT_TYPE_QUERY == pStmt->sql.type) {
STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR);
}
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 21d1a528da..3c6ef00bf4 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -27,9 +27,9 @@
#define EMPTY_BLOCK_POLL_IDLE_DURATION 10
#define DEFAULT_AUTO_COMMIT_INTERVAL 5000
#define DEFAULT_HEARTBEAT_INTERVAL 3000
+#define DEFAULT_ASKEP_INTERVAL 1000
struct SMqMgmt {
- int8_t inited;
tmr_h timer;
int32_t rsetId;
};
@@ -737,35 +737,33 @@ static void generateTimedTask(int64_t refId, int32_t type) {
if (tmq == NULL) return;
int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0);
- if (pTaskType == NULL) return;
+ if (pTaskType != NULL){
+ *pTaskType = type;
+ if (taosWriteQitem(tmq->delayedTask, pTaskType) == 0){
+ tsem2_post(&tmq->rspSem);
+ }
+ }
- *pTaskType = type;
- taosWriteQitem(tmq->delayedTask, pTaskType);
- tsem2_post(&tmq->rspSem);
taosReleaseRef(tmqMgmt.rsetId, refId);
}
void tmqAssignAskEpTask(void* param, void* tmrId) {
- int64_t refId = *(int64_t*)param;
+ int64_t refId = (int64_t)param;
generateTimedTask(refId, TMQ_DELAYED_TASK__ASK_EP);
- taosMemoryFree(param);
}
void tmqReplayTask(void* param, void* tmrId) {
- int64_t refId = *(int64_t*)param;
+ int64_t refId = (int64_t)param;
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
- if (tmq == NULL) goto END;
+ if (tmq == NULL) return;
tsem2_post(&tmq->rspSem);
taosReleaseRef(tmqMgmt.rsetId, refId);
-END:
- taosMemoryFree(param);
}
void tmqAssignDelayedCommitTask(void* param, void* tmrId) {
- int64_t refId = *(int64_t*)param;
+ int64_t refId = (int64_t)param;
generateTimedTask(refId, TMQ_DELAYED_TASK__COMMIT);
- taosMemoryFree(param);
}
int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
@@ -802,11 +800,10 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
}
void tmqSendHbReq(void* param, void* tmrId) {
- int64_t refId = *(int64_t*)param;
+ int64_t refId = (int64_t)param;
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
if (tmq == NULL) {
- taosMemoryFree(param);
return;
}
@@ -880,7 +877,9 @@ void tmqSendHbReq(void* param, void* tmrId) {
OVER:
tDestroySMqHbReq(&req);
- taosTmrReset(tmqSendHbReq, DEFAULT_HEARTBEAT_INTERVAL, param, tmqMgmt.timer, &tmq->hbLiveTimer);
+ if(tmrId != NULL){
+ taosTmrReset(tmqSendHbReq, DEFAULT_HEARTBEAT_INTERVAL, param, tmqMgmt.timer, &tmq->hbLiveTimer);
+ }
taosReleaseRef(tmqMgmt.rsetId, refId);
}
@@ -908,21 +907,14 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) {
if (*pTaskType == TMQ_DELAYED_TASK__ASK_EP) {
askEp(pTmq, NULL, false, false);
- int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t));
- *pRefId = pTmq->refId;
-
tscDebug("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId);
- taosTmrReset(tmqAssignAskEpTask, 1000, pRefId, tmqMgmt.timer, &pTmq->epTimer);
+ taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer, &pTmq->epTimer);
} else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) {
tmq_commit_cb* pCallbackFn = pTmq->commitCb ? pTmq->commitCb : defaultCommitCbFn;
-
asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam);
- int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t));
- *pRefId = pTmq->refId;
-
tscDebug("consumer:0x%" PRIx64 " next commit to vnode(s) in %.2fs", pTmq->consumerId,
pTmq->autoCommitInterval / 1000.0);
- taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, pRefId, tmqMgmt.timer, &pTmq->commitTimer);
+ taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer, &pTmq->commitTimer);
} else {
tscError("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType);
}
@@ -1064,6 +1056,16 @@ void tmqFreeImpl(void* handle) {
taosArrayDestroyEx(tmq->clientTopics, freeClientVgImpl);
taos_close_internal(tmq->pTscObj);
+
+ if(tmq->commitTimer) {
+ taosTmrStopA(&tmq->commitTimer);
+ }
+ if(tmq->epTimer) {
+ taosTmrStopA(&tmq->epTimer);
+ }
+ if(tmq->hbLiveTimer) {
+ taosTmrStopA(&tmq->hbLiveTimer);
+ }
taosMemoryFree(tmq);
tscDebug("consumer:0x%" PRIx64 " closed", id);
@@ -1083,6 +1085,18 @@ static void tmqMgmtInit(void) {
}
}
+void tmqMgmtClose(void) {
+ if (tmqMgmt.timer) {
+ taosTmrCleanUp(tmqMgmt.timer);
+ tmqMgmt.timer = NULL;
+ }
+
+ if (tmqMgmt.rsetId >= 0) {
+ taosCloseRef(tmqMgmt.rsetId);
+ tmqMgmt.rsetId = -1;
+ }
+}
+
#define SET_ERROR_MSG_TMQ(MSG) \
if (errstr != NULL) snprintf(errstr, errstrLen, MSG);
@@ -1171,9 +1185,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
goto _failed;
}
- int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t));
- *pRefId = pTmq->refId;
- pTmq->hbLiveTimer = taosTmrStart(tmqSendHbReq, DEFAULT_HEARTBEAT_INTERVAL, pRefId, tmqMgmt.timer);
+ pTmq->hbLiveTimer = taosTmrStart(tmqSendHbReq, DEFAULT_HEARTBEAT_INTERVAL, (void*)pTmq->refId, tmqMgmt.timer);
char buf[TSDB_OFFSET_LEN] = {0};
STqOffsetVal offset = {.type = pTmq->resetOffsetCfg};
@@ -1301,18 +1313,9 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
}
// init ep timer
- if (tmq->epTimer == NULL) {
- int64_t* pRefId1 = taosMemoryMalloc(sizeof(int64_t));
- *pRefId1 = tmq->refId;
- tmq->epTimer = taosTmrStart(tmqAssignAskEpTask, 1000, pRefId1, tmqMgmt.timer);
- }
-
+ tmq->epTimer = taosTmrStart(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(tmq->refId), tmqMgmt.timer);
// init auto commit timer
- if (tmq->autoCommit && tmq->commitTimer == NULL) {
- int64_t* pRefId2 = taosMemoryMalloc(sizeof(int64_t));
- *pRefId2 = tmq->refId;
- tmq->commitTimer = taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, pRefId2, tmqMgmt.timer);
- }
+ tmq->commitTimer = taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, (void*)(tmq->refId), tmqMgmt.timer);
FAIL:
taosArrayDestroyP(req.topicNames, taosMemoryFree);
@@ -2015,9 +2018,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
pVg->blockReceiveTs = taosGetTimestampMs();
pVg->blockSleepForReplay = pRsp->rsp.sleepTime;
if (pVg->blockSleepForReplay > 0) {
- int64_t* pRefId1 = taosMemoryMalloc(sizeof(int64_t));
- *pRefId1 = tmq->refId;
- taosTmrStart(tmqReplayTask, pVg->blockSleepForReplay, pRefId1, tmqMgmt.timer);
+ taosTmrStart(tmqReplayTask, pVg->blockSleepForReplay, (void*)(tmq->refId), tmqMgmt.timer);
}
}
tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64
@@ -2274,7 +2275,7 @@ int32_t tmq_consumer_close(tmq_t* tmq) {
return code;
}
}
- taosSsleep(2); // sleep 2s for hb to send offset and rows to server
+ tmqSendHbReq((void*)(tmq->refId), NULL);
tmq_list_t* lst = tmq_list_new();
int32_t code = tmq_subscribe(tmq, lst);
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 3372c7b1cc..119438fbd6 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -111,7 +111,6 @@ uint16_t tsMonitorPort = 6043;
int32_t tsMonitorMaxLogs = 100;
bool tsMonitorComp = false;
bool tsMonitorLogProtocol = false;
-int32_t tsMonitorIntervalForBasic = 30;
bool tsMonitorForceV2 = true;
// audit
@@ -712,7 +711,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
- if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
@@ -1165,7 +1163,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsMonitorComp = cfgGetItem(pCfg, "monitorComp")->bval;
tsQueryRspPolicy = cfgGetItem(pCfg, "queryRspPolicy")->i32;
tsMonitorLogProtocol = cfgGetItem(pCfg, "monitorLogProtocol")->bval;
- tsMonitorIntervalForBasic = cfgGetItem(pCfg, "monitorIntervalForBasic")->i32;
tsMonitorForceV2 = cfgGetItem(pCfg, "monitorForceV2")->i32;
tsEnableAudit = cfgGetItem(pCfg, "audit")->bval;
@@ -1347,7 +1344,10 @@ int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char *
return -1;
}
- tstrncpy(tsDataDir, cfgGetItem(pCfg, "dataDir")->str, PATH_MAX);
+ if (taosSetTfsCfg(pCfg) != 0) {
+ cfgCleanup(pCfg);
+ return -1;
+ }
dDebugFlag = cfgGetItem(pCfg, "dDebugFlag")->i32;
cfgCleanup(pCfg);
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index 10719674f5..7e89753241 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -69,7 +69,7 @@
pReq->sql = NULL; \
} while (0)
-static int32_t tSerializeSMonitorParas(SEncoder *encoder, const SMonitorParas* pMonitorParas) {
+static int32_t tSerializeSMonitorParas(SEncoder *encoder, const SMonitorParas *pMonitorParas) {
if (tEncodeI8(encoder, pMonitorParas->tsEnableMonitor) < 0) return -1;
if (tEncodeI32(encoder, pMonitorParas->tsMonitorInterval) < 0) return -1;
if (tEncodeI32(encoder, pMonitorParas->tsSlowLogScope) < 0) return -1;
@@ -80,7 +80,7 @@ static int32_t tSerializeSMonitorParas(SEncoder *encoder, const SMonitorParas* p
return 0;
}
-static int32_t tDeserializeSMonitorParas(SDecoder *decoder, SMonitorParas* pMonitorParas){
+static int32_t tDeserializeSMonitorParas(SDecoder *decoder, SMonitorParas *pMonitorParas) {
if (tDecodeI8(decoder, (int8_t *)&pMonitorParas->tsEnableMonitor) < 0) return -1;
if (tDecodeI32(decoder, &pMonitorParas->tsMonitorInterval) < 0) return -1;
if (tDecodeI32(decoder, &pMonitorParas->tsSlowLogScope) < 0) return -1;
@@ -1577,7 +1577,7 @@ int32_t tDeserializeSStatisReq(void *buf, int32_t bufLen, SStatisReq *pReq) {
if (tDecodeCStrTo(&decoder, pReq->pCont) < 0) return -1;
}
if (!tDecodeIsEnd(&decoder)) {
- if (tDecodeI8(&decoder, (int8_t*)&pReq->type) < 0) return -1;
+ if (tDecodeI8(&decoder, (int8_t *)&pReq->type) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
@@ -5737,65 +5737,74 @@ _exit:
}
int32_t tSerializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeConfigReq *pReq) {
+ int32_t tlen;
SEncoder encoder = {0};
+
tEncoderInit(&encoder, buf, bufLen);
- if (tStartEncode(&encoder) < 0) return -1;
- if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1;
- if (tEncodeI32(&encoder, pReq->buffer) < 0) return -1;
- if (tEncodeI32(&encoder, pReq->pageSize) < 0) return -1;
- if (tEncodeI32(&encoder, pReq->pages) < 0) return -1;
- if (tEncodeI32(&encoder, pReq->cacheLastSize) < 0) return -1;
- if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1;
- if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1;
- if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1;
- if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1;
- if (tEncodeI32(&encoder, pReq->walFsyncPeriod) < 0) return -1;
- if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
- if (tEncodeI8(&encoder, pReq->strict) < 0) return -1;
- if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1;
+ TAOS_CHECK_ERRNO(tStartEncode(&encoder));
+ TAOS_CHECK_ERRNO(tEncodeI32(&encoder, pReq->vgVersion));
+ TAOS_CHECK_ERRNO(tEncodeI32(&encoder, pReq->buffer));
+ TAOS_CHECK_ERRNO(tEncodeI32(&encoder, pReq->pageSize));
+ TAOS_CHECK_ERRNO(tEncodeI32(&encoder, pReq->pages));
+ TAOS_CHECK_ERRNO(tEncodeI32(&encoder, pReq->cacheLastSize));
+ TAOS_CHECK_ERRNO(tEncodeI32(&encoder, pReq->daysPerFile));
+ TAOS_CHECK_ERRNO(tEncodeI32(&encoder, pReq->daysToKeep0));
+ TAOS_CHECK_ERRNO(tEncodeI32(&encoder, pReq->daysToKeep1));
+ TAOS_CHECK_ERRNO(tEncodeI32(&encoder, pReq->daysToKeep2));
+ TAOS_CHECK_ERRNO(tEncodeI32(&encoder, pReq->walFsyncPeriod));
+ TAOS_CHECK_ERRNO(tEncodeI8(&encoder, pReq->walLevel));
+ TAOS_CHECK_ERRNO(tEncodeI8(&encoder, pReq->strict));
+ TAOS_CHECK_ERRNO(tEncodeI8(&encoder, pReq->cacheLast));
for (int32_t i = 0; i < 7; ++i) {
- if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1;
+ TAOS_CHECK_ERRNO(tEncodeI64(&encoder, pReq->reserved[i]));
}
// 1st modification
- if (tEncodeI16(&encoder, pReq->sttTrigger) < 0) return -1;
- if (tEncodeI32(&encoder, pReq->minRows) < 0) return -1;
+ TAOS_CHECK_ERRNO(tEncodeI16(&encoder, pReq->sttTrigger));
+ TAOS_CHECK_ERRNO(tEncodeI32(&encoder, pReq->minRows));
// 2nd modification
- if (tEncodeI32(&encoder, pReq->walRetentionPeriod) < 0) return -1;
- if (tEncodeI32(&encoder, pReq->walRetentionSize) < 0) return -1;
- if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
+ TAOS_CHECK_ERRNO(tEncodeI32(&encoder, pReq->walRetentionPeriod));
+ TAOS_CHECK_ERRNO(tEncodeI32(&encoder, pReq->walRetentionSize));
+ TAOS_CHECK_ERRNO(tEncodeI32(&encoder, pReq->keepTimeOffset));
- if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1;
- if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
+ TAOS_CHECK_ERRNO(tEncodeI32(&encoder, pReq->s3KeepLocal));
+ TAOS_CHECK_ERRNO(tEncodeI8(&encoder, pReq->s3Compact));
tEndEncode(&encoder);
- int32_t tlen = encoder.pos;
+_exit:
+ if (terrno) {
+ uError("%s failed at line %d since %s", __func__, terrln, terrstr());
+ tlen = -1;
+ } else {
+ tlen = encoder.pos;
+ }
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeConfigReq *pReq) {
SDecoder decoder = {0};
+
tDecoderInit(&decoder, buf, bufLen);
- if (tStartDecode(&decoder) < 0) return -1;
- if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1;
- if (tDecodeI32(&decoder, &pReq->buffer) < 0) return -1;
- if (tDecodeI32(&decoder, &pReq->pageSize) < 0) return -1;
- if (tDecodeI32(&decoder, &pReq->pages) < 0) return -1;
- if (tDecodeI32(&decoder, &pReq->cacheLastSize) < 0) return -1;
- if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1;
- if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1;
- if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1;
- if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1;
- if (tDecodeI32(&decoder, &pReq->walFsyncPeriod) < 0) return -1;
- if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
- if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1;
- if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1;
+ TAOS_CHECK_ERRNO(tStartDecode(&decoder));
+ TAOS_CHECK_ERRNO(tDecodeI32(&decoder, &pReq->vgVersion));
+ TAOS_CHECK_ERRNO(tDecodeI32(&decoder, &pReq->buffer));
+ TAOS_CHECK_ERRNO(tDecodeI32(&decoder, &pReq->pageSize));
+ TAOS_CHECK_ERRNO(tDecodeI32(&decoder, &pReq->pages));
+ TAOS_CHECK_ERRNO(tDecodeI32(&decoder, &pReq->cacheLastSize));
+ TAOS_CHECK_ERRNO(tDecodeI32(&decoder, &pReq->daysPerFile));
+ TAOS_CHECK_ERRNO(tDecodeI32(&decoder, &pReq->daysToKeep0));
+ TAOS_CHECK_ERRNO(tDecodeI32(&decoder, &pReq->daysToKeep1));
+ TAOS_CHECK_ERRNO(tDecodeI32(&decoder, &pReq->daysToKeep2));
+ TAOS_CHECK_ERRNO(tDecodeI32(&decoder, &pReq->walFsyncPeriod));
+ TAOS_CHECK_ERRNO(tDecodeI8(&decoder, &pReq->walLevel));
+ TAOS_CHECK_ERRNO(tDecodeI8(&decoder, &pReq->strict));
+ TAOS_CHECK_ERRNO(tDecodeI8(&decoder, &pReq->cacheLast));
for (int32_t i = 0; i < 7; ++i) {
- if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1;
+ TAOS_CHECK_ERRNO(tDecodeI64(&decoder, &pReq->reserved[i]));
}
// 1st modification
@@ -5803,8 +5812,8 @@ int32_t tDeserializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeC
pReq->sttTrigger = -1;
pReq->minRows = -1;
} else {
- if (tDecodeI16(&decoder, &pReq->sttTrigger) < 0) return -1;
- if (tDecodeI32(&decoder, &pReq->minRows) < 0) return -1;
+ TAOS_CHECK_ERRNO(tDecodeI16(&decoder, &pReq->sttTrigger));
+ TAOS_CHECK_ERRNO(tDecodeI32(&decoder, &pReq->minRows));
}
// 2n modification
@@ -5812,24 +5821,29 @@ int32_t tDeserializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeC
pReq->walRetentionPeriod = -1;
pReq->walRetentionSize = -1;
} else {
- if (tDecodeI32(&decoder, &pReq->walRetentionPeriod) < 0) return -1;
- if (tDecodeI32(&decoder, &pReq->walRetentionSize) < 0) return -1;
+ TAOS_CHECK_ERRNO(tDecodeI32(&decoder, &pReq->walRetentionPeriod));
+ TAOS_CHECK_ERRNO(tDecodeI32(&decoder, &pReq->walRetentionSize));
}
pReq->keepTimeOffset = TSDB_DEFAULT_KEEP_TIME_OFFSET;
if (!tDecodeIsEnd(&decoder)) {
- if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
+ TAOS_CHECK_ERRNO(tDecodeI32(&decoder, &pReq->keepTimeOffset));
}
pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (!tDecodeIsEnd(&decoder)) {
- if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1;
- if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1;
+ TAOS_CHECK_ERRNO(tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0);
+ TAOS_CHECK_ERRNO(tDecodeI8(&decoder, &pReq->s3Compact) < 0);
}
tEndDecode(&decoder);
+
+_exit:
tDecoderClear(&decoder);
- return 0;
+ if (terrno) {
+ uError("%s failed at line %d since %s", __func__, terrln, terrstr());
+ }
+ return terrno;
}
int32_t tSerializeSAlterVnodeReplicaReq(void *buf, int32_t bufLen, SAlterVnodeReplicaReq *pReq) {
@@ -9296,7 +9310,7 @@ int32_t tDecodeSTqCheckInfo(SDecoder *pDecoder, STqCheckInfo *pInfo) {
}
void tDeleteSTqCheckInfo(STqCheckInfo *pInfo) { taosArrayDestroy(pInfo->colIdList); }
-int32_t tEncodeSMqRebVgReq(SEncoder* pCoder, const SMqRebVgReq* pReq) {
+int32_t tEncodeSMqRebVgReq(SEncoder *pCoder, const SMqRebVgReq *pReq) {
if (tStartEncode(pCoder) < 0) return -1;
if (tEncodeI64(pCoder, pReq->leftForVer) < 0) return -1;
if (tEncodeI32(pCoder, pReq->vgId) < 0) return -1;
@@ -9316,7 +9330,7 @@ int32_t tEncodeSMqRebVgReq(SEncoder* pCoder, const SMqRebVgReq* pReq) {
return 0;
}
-int32_t tDecodeSMqRebVgReq(SDecoder* pCoder, SMqRebVgReq* pReq) {
+int32_t tDecodeSMqRebVgReq(SDecoder *pCoder, SMqRebVgReq *pReq) {
if (tStartDecode(pCoder) < 0) return -1;
if (tDecodeI64(pCoder, &pReq->leftForVer) < 0) return -1;
@@ -9341,7 +9355,6 @@ int32_t tDecodeSMqRebVgReq(SDecoder* pCoder, SMqRebVgReq* pReq) {
return 0;
}
-
int32_t tEncodeDeleteRes(SEncoder *pCoder, const SDeleteRes *pRes) {
int32_t nUid = taosArrayGetSize(pRes->uidList);
diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c
index 52379f10d7..f7885caac0 100644
--- a/source/common/src/ttime.c
+++ b/source/common/src/ttime.c
@@ -1969,3 +1969,98 @@ int32_t TEST_char2ts(const char* format, int64_t* ts, int32_t precision, const c
taosArrayDestroy(formats);
return code;
}
+
+static int8_t UNIT_INDEX[26] = {/*a*/ 2, 0, -1, 6, -1, -1, -1,
+ /*h*/ 5, -1, -1, -1, -1, 4, 8,
+ /*o*/ -1, -1, -1, -1, 3, -1,
+ /*u*/ 1, -1, 7, -1, 9, -1};
+
+#define GET_UNIT_INDEX(idx) UNIT_INDEX[(idx) - 97]
+
+static int64_t UNIT_MATRIX[10][11] = { /* ns, us, ms, s, min, h, d, w, month, y*/
+ /*ns*/ { 1, 1000, 0},
+ /*us*/ {1000, 1, 1000, 0},
+ /*ms*/ { 0, 1000, 1, 1000, 0},
+ /*s*/ { 0, 0, 1000, 1, 60, 0},
+ /*min*/ { 0, 0, 0, 60, 1, 60, 0},
+ /*h*/ { 0, 0, 0, 0, 60, 1, 1, 0},
+ /*d*/ { 0, 0, 0, 0, 0, 24, 1, 7, 1, 0},
+ /*w*/ { 0, 0, 0, 0, 0, 0, 7, 1, -1, 0},
+ /*mon*/ { 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0},
+ /*y*/ { 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0}};
+
+static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx, bool checkEq) {
+ if (UNIT_MATRIX[baseIdx][idx] == -1) return false;
+ if (baseIdx == idx) {
+ if (interval < baseInterval) return false;
+ if (checkEq && interval == baseInterval) return false;
+ return interval % baseInterval == 0;
+ }
+ int8_t next = baseIdx + 1;
+ int64_t val = UNIT_MATRIX[baseIdx][next];
+ while (val != 0 && next <= idx) {
+ if (val == -1) {
+ next++;
+ val = UNIT_MATRIX[baseIdx][next];
+ continue;
+ }
+ if (val % baseInterval == 0 || baseInterval % val == 0) {
+ int8_t extra = baseInterval >= val ? 0 : 1;
+ bool needCheckEq = baseInterval >= val && !(baseIdx < next && val == 1);
+ if (!recursiveTsmaCheckRecursive(baseInterval / val + extra, next, interval, idx, needCheckEq && checkEq)) {
+ next++;
+ val = UNIT_MATRIX[baseIdx][next];
+ continue;
+ } else {
+ return true;
+ }
+ } else {
+ return false;
+ }
+ }
+ return false;
+}
+
+static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx, bool checkEq) {
+ if (UNIT_MATRIX[baseIdx][idx] == -1) return false;
+
+ if (baseIdx == idx) {
+ if (interval < baseInterval) return false;
+ if (checkEq && interval == baseInterval) return false;
+ return interval % baseInterval == 0;
+ }
+
+ int8_t next = baseIdx - 1;
+ int64_t val = UNIT_MATRIX[baseIdx][next];
+ while (val != 0 && next >= 0) {
+ return recursiveTsmaCheckRecursiveReverse(baseInterval * val, next, interval, idx, checkEq);
+ }
+ return false;
+}
+
+/*
+ * @breif check if tsma with param [interval], [unit] can create based on base tsma with baseInterval and baseUnit
+ * @param baseInterval, baseUnit, interval/unit of base tsma
+ * @param interval the tsma interval going to create. Not that if unit is not calander unit, then interval has already been
+ * translated to TICKS of [precision]
+ * @param unit the tsma unit going to create
+ * @param precision the precision of this db
+ * @param checkEq pass true if same interval is not acceptable, false if acceptable.
+ * @ret true the tsma can be created, else cannot
+ * */
+bool checkRecursiveTsmaInterval(int64_t baseInterval, int8_t baseUnit, int64_t interval, int8_t unit, int8_t precision, bool checkEq) {
+ bool baseIsCalendarDuration = IS_CALENDAR_TIME_DURATION(baseUnit);
+ if (!baseIsCalendarDuration) baseInterval = convertTimeFromPrecisionToUnit(baseInterval, precision, baseUnit);
+ bool isCalendarDuration = IS_CALENDAR_TIME_DURATION(unit);
+ if (!isCalendarDuration) interval = convertTimeFromPrecisionToUnit(interval, precision, unit);
+
+ bool needCheckEq = baseIsCalendarDuration == isCalendarDuration && checkEq;
+
+ int8_t baseIdx = GET_UNIT_INDEX(baseUnit), idx = GET_UNIT_INDEX(unit);
+ if (baseIdx <= idx) {
+ return recursiveTsmaCheckRecursive(baseInterval, baseIdx, interval, idx, needCheckEq);
+ } else {
+ return recursiveTsmaCheckRecursiveReverse(baseInterval, baseIdx, interval, idx, checkEq);
+ }
+ return true;
+}
diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c
index 65f695cb8b..1c5541de29 100644
--- a/source/dnode/mgmt/exe/dmMain.c
+++ b/source/dnode/mgmt/exe/dmMain.c
@@ -369,7 +369,7 @@ int mainWindows(int argc, char **argv) {
if(global.generateCode) {
bool toLogFile = false;
if(taosReadDataFolder(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs) != 0){
- encryptError("failed to generate encrypt code since taosd is running, please stop it first");
+ encryptError("failed to generate encrypt code since dataDir can not be set from cfg file");
return -1;
};
diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h
index 46f8dd06d4..be9ff56674 100644
--- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h
+++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h
@@ -43,7 +43,6 @@ typedef struct SDnodeMgmt {
GetMnodeLoadsFp getMnodeLoadsFp;
GetQnodeLoadsFp getQnodeLoadsFp;
int32_t statusSeq;
- SendMonitorReportFp sendMonitorReportFpBasic;
} SDnodeMgmt;
// dmHandle.c
diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c
index a651fbf060..b9dd45f1c0 100644
--- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c
+++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c
@@ -65,7 +65,6 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
pMgmt->processDropNodeFp = pInput->processDropNodeFp;
pMgmt->sendMonitorReportFp = pInput->sendMonitorReportFp;
pMgmt->sendAuditRecordsFp = pInput->sendAuditRecordFp;
- pMgmt->sendMonitorReportFpBasic = pInput->sendMonitorReportFpBasic;
pMgmt->getVnodeLoadsFp = pInput->getVnodeLoadsFp;
pMgmt->getVnodeLoadsLiteFp = pInput->getVnodeLoadsLiteFp;
pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp;
diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
index c48b614f96..eafa10aa32 100644
--- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
+++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
@@ -175,15 +175,6 @@ static void *dmMonitorThreadFp(void *param) {
taosMemoryTrim(0);
}
}
-
- if(tsMonitorForceV2){
- if (curTime < lastTimeForBasic) lastTimeForBasic = curTime;
- float intervalForBasic = (curTime - lastTimeForBasic) / 1000.0f;
- if (intervalForBasic >= tsMonitorIntervalForBasic) {
- (*pMgmt->sendMonitorReportFpBasic)();
- lastTimeForBasic = curTime;
- }
- }
}
return NULL;
diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
index 677e19d4c1..9b987b3237 100644
--- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
+++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
@@ -233,6 +233,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_STREAM_CONSEN_CHKPT_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_CREATE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_DROP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_CREATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
@@ -242,7 +243,6 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
- if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_CONSEN, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_KILL_COMPACT_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c
index 7a0189b7c1..5c2f54fd10 100644
--- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c
+++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c
@@ -76,6 +76,7 @@ SArray *smGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_STREAM_CONSEN_CHKPT, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
@@ -96,7 +97,6 @@ SArray *smGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
- if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_CONSEN_RSP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
code = 0;
_OVER:
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
index 001696aecc..fbe1925e3f 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@@ -972,10 +972,10 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
- if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_CONSEN_RSP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_STREAM_CONSEN_CHKPT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
index 45d1486912..8c1b33cb14 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
@@ -287,7 +287,8 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
return -1;
}
- SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen);
+ EQItype itype = APPLY_QUEUE == qtype ? DEF_QITEM : RPC_QITEM;
+ SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), itype, pRpc->contLen);
if (pMsg == NULL) {
rpcFreeCont(pRpc->pCont);
pRpc->pCont = NULL;
diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
index 90e44e5acc..bc6a4652e7 100644
--- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
+++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
@@ -128,7 +128,6 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
// dmMonitor.c
void dmSendMonitorReport();
void dmSendAuditRecords();
-void dmSendMonitorReportBasic();
void dmGetVnodeLoads(SMonVloadInfo *pInfo);
void dmGetVnodeLoadsLite(SMonVloadInfo *pInfo);
void dmGetMnodeLoads(SMonMloadInfo *pInfo);
diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c
index 54a118b666..4be1af30b5 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c
@@ -394,7 +394,6 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) {
.processDropNodeFp = dmProcessDropNodeReq,
.sendMonitorReportFp = dmSendMonitorReport,
.sendAuditRecordFp = auditSendRecordsInBatch,
- .sendMonitorReportFpBasic = dmSendMonitorReportBasic,
.getVnodeLoadsFp = dmGetVnodeLoads,
.getVnodeLoadsLiteFp = dmGetVnodeLoadsLite,
.getMnodeLoadsFp = dmGetMnodeLoads,
diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c
index 21e25f5535..d3197282b6 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c
@@ -123,16 +123,6 @@ void dmSendMonitorReport() {
monGenAndSendReport();
}
-void dmSendMonitorReportBasic() {
- if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return;
- dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort);
-
- SDnode *pDnode = dmInstance();
- dmGetDmMonitorInfoBasic(pDnode);
- dmGetMmMonitorInfo(pDnode);
- monGenAndSendReportBasic();
-}
-
//Todo: put this in seperate file in the future
void dmSendAuditRecords() {
auditSendRecordsInBatch();
diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
index 74bf1f964c..99d641ff3f 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
@@ -208,7 +208,9 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
}
pRpc->info.wrapper = pWrapper;
- pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen);
+
+ EQItype itype = IsReq(pRpc) ? RPC_QITEM : DEF_QITEM; // rsp msg is not restricted by tsRpcQueueMemoryUsed
+ pMsg = taosAllocateQitem(sizeof(SRpcMsg), itype, pRpc->contLen);
if (pMsg == NULL) goto _OVER;
memcpy(pMsg, pRpc, sizeof(SRpcMsg));
diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h
index aea3286d76..d316a82af2 100644
--- a/source/dnode/mgmt/node_util/inc/dmUtil.h
+++ b/source/dnode/mgmt/node_util/inc/dmUtil.h
@@ -155,7 +155,6 @@ typedef struct {
ProcessDropNodeFp processDropNodeFp;
SendMonitorReportFp sendMonitorReportFp;
SendAuditRecordsFp sendAuditRecordFp;
- SendMonitorReportFp sendMonitorReportFpBasic;
GetVnodeLoadsFp getVnodeLoadsFp;
GetVnodeLoadsFp getVnodeLoadsLiteFp;
GetMnodeLoadsFp getMnodeLoadsFp;
diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h
index b261f89057..0b6b6a9ef2 100644
--- a/source/dnode/mnode/impl/inc/mndStream.h
+++ b/source/dnode/mnode/impl/inc/mndStream.h
@@ -83,7 +83,7 @@ typedef struct SOrphanTask {
typedef struct {
SMsgHead head;
-} SMStreamReqCheckpointRsp, SMStreamUpdateChkptRsp;
+} SMStreamReqCheckpointRsp, SMStreamUpdateChkptRsp, SMStreamReqConsensChkptRsp;
typedef struct STaskChkptInfo {
int32_t nodeId;
@@ -133,8 +133,8 @@ int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream)
int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList);
int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq);
-int32_t mndSendConsensusCheckpointIdRsp(SArray* pList, int64_t checkpointId);
-
+int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, int32_t taskId, int64_t checkpointId,
+ int64_t ts);
void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo *pExecInfo);
SStreamTaskIter *createStreamTaskIter(SStreamObj *pStream);
@@ -146,14 +146,10 @@ void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInf
int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot);
void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
-SCheckpointConsensusInfo *mndGetConsensusInfo(SHashObj *pHash, int64_t streamId);
-void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpointInfo *pRestoreInfo, SRpcMsg *pMsg);
-int64_t mndGetConsensusCheckpointId(SCheckpointConsensusInfo *pInfo, SStreamObj *pStream);
-bool mndAllTaskSendCheckpointId(SCheckpointConsensusInfo *pInfo, int32_t numOfTasks, int32_t* pTotal);
+SCheckpointConsensusInfo *mndGetConsensusInfo(SHashObj *pHash, int64_t streamId, int32_t numOfTasks);
+void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpointInfo *pRestoreInfo);
void mndClearConsensusRspEntry(SCheckpointConsensusInfo *pInfo);
-int32_t doSendConsensusCheckpointRsp(SRestoreCheckpointInfo *pInfo, SRpcMsg *pMsg, int64_t checkpointId);
int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId);
-int32_t mndRegisterConsensusChkptId(SHashObj* pHash, int64_t streamId);
#ifdef __cplusplus
}
diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c
index 398ea5d589..723e3701a1 100644
--- a/source/dnode/mnode/impl/src/mndMain.c
+++ b/source/dnode/mnode/impl/src/mndMain.c
@@ -177,6 +177,15 @@ static void mndStreamCheckNode(SMnode *pMnode) {
}
}
+static void mndStreamConsensusChkpt(SMnode *pMnode) {
+ int32_t contLen = 0;
+ void *pReq = mndBuildTimerMsg(&contLen);
+ if (pReq != NULL) {
+ SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_CONSEN_TIMER, .pCont = pReq, .contLen = contLen};
+ tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
+ }
+}
+
static void mndPullupTelem(SMnode *pMnode) {
mTrace("pullup telem msg");
int32_t contLen = 0;
@@ -308,7 +317,6 @@ static int32_t minCronTime() {
min = TMIN(min, tsCompactPullupInterval);
min = TMIN(min, tsMqRebalanceInterval);
min = TMIN(min, tsStreamCheckpointInterval);
- min = TMIN(min, 6); // checkpointRemain
min = TMIN(min, tsStreamNodeCheckInterval);
min = TMIN(min, tsArbHeartBeatIntervalSec);
min = TMIN(min, tsArbCheckSyncIntervalSec);
@@ -353,6 +361,10 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) {
mndStreamCheckNode(pMnode);
}
+ if (sec % 5 == 0) {
+ mndStreamConsensusChkpt(pMnode);
+ }
+
if (sec % tsTelemInterval == (TMIN(60, (tsTelemInterval - 1)))) {
mndPullupTelem(pMnode);
}
diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c
index 5c8bb22c22..946df84a0f 100644
--- a/source/dnode/mnode/impl/src/mndSma.c
+++ b/source/dnode/mnode/impl/src/mndSma.c
@@ -2004,8 +2004,13 @@ static int32_t mndRetrieveTSMA(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo
// interval
char interval[64 + VARSTR_HEADER_SIZE] = {0};
- int32_t len = snprintf(interval + VARSTR_HEADER_SIZE, 64, "%" PRId64 "%c", pSma->interval,
- getPrecisionUnit(pSrcDb->cfg.precision));
+ int32_t len = 0;
+ if (!IS_CALENDAR_TIME_DURATION(pSma->intervalUnit)) {
+ len = snprintf(interval + VARSTR_HEADER_SIZE, 64, "%" PRId64 "%c", pSma->interval,
+ getPrecisionUnit(pSrcDb->cfg.precision));
+ } else {
+ len = snprintf(interval + VARSTR_HEADER_SIZE, 64, "%" PRId64 "%c", pSma->interval, pSma->intervalUnit);
+ }
varDataSetLen(interval, len);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, interval, false);
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index a137c10ed5..415d1ff9f0 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -59,7 +59,9 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg);
static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList);
static int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq);
static int32_t mndProcessCheckpointReport(SRpcMsg *pReq);
-static int32_t mndProcessConsensusCheckpointId(SRpcMsg *pReq);
+//static int32_t mndProcessConsensusCheckpointId(SRpcMsg *pMsg);
+static int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg);
+static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId, int32_t code);
static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList);
@@ -106,6 +108,7 @@ int32_t mndInitStream(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_VND_STREAM_TASK_UPDATE_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_VND_STREAM_TASK_RESET_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_UPDATE_CHKPT_RSP, mndTransProcessRsp);
+ mndSetMsgHandle(pMnode, TDMT_STREAM_CONSEN_CHKPT_RSP, mndTransProcessRsp);
// for msgs inside mnode
// TODO change the name
@@ -118,11 +121,11 @@ int32_t mndInitStream(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamCheckpoint);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_REQ_CHKPT, mndProcessStreamReqCheckpoint);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHKPT_REPORT, mndProcessCheckpointReport);
- mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHKPT_CONSEN, mndProcessConsensusCheckpointId);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_UPDATE_CHKPT_EVT, mndScanCheckpointReportInfo);
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_REPORT_CHECKPOINT, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_HEARTBEAT, mndProcessStreamHb);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_NODECHANGE_CHECK, mndProcessNodeCheckReq);
+ mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CONSEN_TIMER, mndProcessConsensusInTmr);
mndSetMsgHandle(pMnode, TDMT_MND_PAUSE_STREAM, mndProcessPauseStreamReq);
mndSetMsgHandle(pMnode, TDMT_MND_RESUME_STREAM, mndProcessResumeStreamReq);
@@ -803,7 +806,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
taosThreadMutexLock(&execInfo.lock);
mDebug("stream stream:%s start to register tasks into task nodeList and set initial checkpointId", createReq.name);
saveTaskAndNodeInfoIntoBuf(&streamObj, &execInfo);
- mndRegisterConsensusChkptId(execInfo.pStreamConsensus, streamObj.uid);
+// mndRegisterConsensusChkptId(execInfo.pStreamConsensus, streamObj.uid);
taosThreadMutexUnlock(&execInfo.lock);
// execute creation
@@ -1134,6 +1137,7 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s, checkpoint not issued", pEntry->id.streamId,
(int32_t)pEntry->id.taskId, pEntry->nodeId, streamTaskGetStatusStr(pEntry->status));
ready = false;
+ break;
}
if (pEntry->hTaskId != 0) {
@@ -1153,6 +1157,27 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
return ready ? 0 : -1;
}
+int64_t getStreamTaskLastReadyState(SArray *pTaskList, int64_t streamId) {
+ int64_t ts = -1;
+ int32_t taskId = -1;
+
+ for (int32_t i = 0; i < taosArrayGetSize(pTaskList); ++i) {
+ STaskId *p = taosArrayGet(pTaskList, i);
+ STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p));
+ if (pEntry == NULL || pEntry->id.streamId != streamId) {
+ continue;
+ }
+
+ if (pEntry->status == TASK_STATUS__READY && ts < pEntry->startTime) {
+ ts = pEntry->startTime;
+ taskId = pEntry->id.taskId;
+ }
+ }
+
+ mDebug("stream:0x%" PRIx64 " last ready ts:%" PRId64 " s-task:0x%x", streamId, ts, taskId);
+ return ts;
+}
+
typedef struct {
int64_t streamId;
int64_t duration;
@@ -1191,6 +1216,15 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
continue;
}
+ taosThreadMutexLock(&execInfo.lock);
+ int64_t startTs = getStreamTaskLastReadyState(execInfo.pTaskList, pStream->uid);
+ if (startTs != -1 && (now - startTs) < tsStreamCheckpointInterval * 1000) {
+ taosThreadMutexUnlock(&execInfo.lock);
+ sdbRelease(pSdb, pStream);
+ continue;
+ }
+ taosThreadMutexUnlock(&execInfo.lock);
+
SCheckpointInterval in = {.streamId = pStream->uid, .duration = duration};
taosArrayPush(pList, &in);
@@ -1237,9 +1271,6 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
code = mndProcessStreamCheckpointTrans(pMnode, p, checkpointId, 1, true);
sdbRelease(pSdb, p);
- // clear the consensus checkpoint info
- mndClearConsensusCheckpointId(execInfo.pStreamConsensus, p->uid);
-
if (code != -1) {
started += 1;
@@ -2309,7 +2340,7 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
taosThreadMutexUnlock(&execInfo.lock);
if (numOfNodes == 0) {
- mDebug("end to do stream task node change checking, no vgroup exists, do nothing");
+ mDebug("end to do stream task(s) node change checking, no stream tasks exist, do nothing");
execInfo.ts = ts;
atomic_store_32(&mndNodeCheckSentinel, 0);
return 0;
@@ -2612,113 +2643,238 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) {
taosThreadMutexUnlock(&execInfo.lock);
- {
- SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamUpdateChkptRsp)};
- rsp.pCont = rpcMallocCont(rsp.contLen);
- SMsgHead *pHead = rsp.pCont;
- pHead->vgId = htonl(req.nodeId);
-
- tmsgSendRsp(&rsp);
- pReq->info.handle = NULL; // disable auto rsp
- }
-
+ doSendQuickRsp(&pReq->info, sizeof(SMStreamUpdateChkptRsp), req.nodeId, TSDB_CODE_SUCCESS);
return 0;
}
-static int32_t mndProcessConsensusCheckpointId(SRpcMsg *pReq) {
- SMnode *pMnode = pReq->info.node;
- SDecoder decoder = {0};
+static int64_t getConsensusId(int64_t streamId, int32_t numOfTasks, int32_t* pExistedTasks, bool *pAllSame) {
+ int32_t num = 0;
+ int64_t chkId = INT64_MAX;
+ *pExistedTasks = 0;
+ *pAllSame = true;
- SRestoreCheckpointInfo req = {0};
- tDecoderInit(&decoder, pReq->pCont, pReq->contLen);
+ for(int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
+ STaskId* p = taosArrayGet(execInfo.pTaskList, i);
+ if (p->streamId != streamId) {
+ continue;
+ }
- if (tDecodeRestoreCheckpointInfo(&decoder, &req)) {
- tDecoderClear(&decoder);
- terrno = TSDB_CODE_INVALID_MSG;
- mError("invalid task consensus-checkpoint msg received");
+ num += 1;
+ STaskStatusEntry* pe = taosHashGet(execInfo.pTaskMap, p, sizeof(*p));
+ if (chkId > pe->checkpointInfo.latestId) {
+ if (chkId != INT64_MAX) {
+ *pAllSame = false;
+ }
+ chkId = pe->checkpointInfo.latestId;
+ }
+ }
+
+ *pExistedTasks = num;
+ if (num < numOfTasks) { // not all task send info to mnode through hbMsg, no valid checkpoint Id
return -1;
}
- tDecoderClear(&decoder);
- mDebug("receive stream task consensus-checkpoint msg, vgId:%d, s-task:0x%" PRIx64 "-0x%x, checkpointId:%" PRId64,
- req.nodeId, req.streamId, req.taskId, req.checkpointId);
+ return chkId;
+}
+
+static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId, int32_t code) {
+ SRpcMsg rsp = {.code = code, .info = *pInfo, .contLen = msgSize};
+ rsp.pCont = rpcMallocCont(rsp.contLen);
+ SMsgHead *pHead = rsp.pCont;
+ pHead->vgId = htonl(vgId);
+
+ tmsgSendRsp(&rsp);
+ pInfo->handle = NULL; // disable auto rsp
+}
+
+//static int32_t mndProcessConsensusCheckpointId(SRpcMsg *pMsg) {
+// SMnode *pMnode = pMsg->info.node;
+// SDecoder decoder = {0};
+//
+// SRestoreCheckpointInfo req = {0};
+// tDecoderInit(&decoder, pMsg->pCont, pMsg->contLen);
+//
+// if (tDecodeRestoreCheckpointInfo(&decoder, &req)) {
+// tDecoderClear(&decoder);
+// terrno = TSDB_CODE_INVALID_MSG;
+// mError("invalid task consensus-checkpoint msg received");
+// return -1;
+// }
+// tDecoderClear(&decoder);
+//
+// mDebug("receive stream task consensus-checkpoint msg, vgId:%d, s-task:0x%" PRIx64 "-0x%x, checkpointId:%" PRId64,
+// req.nodeId, req.streamId, req.taskId, req.checkpointId);
+//
+// // register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans.
+// taosThreadMutexLock(&execInfo.lock);
+//
+// // mnode handle the create stream transaction too slow may cause this problem
+// SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId);
+// if (pStream == NULL) {
+// mWarn("failed to find the stream:0x%" PRIx64 ", not handle consensus-checkpointId", req.streamId);
+//
+// // not in meta-store yet, try to acquire the task in exec buffer
+// // the checkpoint req arrives too soon before the completion of the create stream trans.
+// STaskId id = {.streamId = req.streamId, .taskId = req.taskId};
+// void *p = taosHashGet(execInfo.pTaskMap, &id, sizeof(id));
+// if (p == NULL) {
+// mError("failed to find the stream:0x%" PRIx64 " in buf, not handle consensus-checkpointId", req.streamId);
+// terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
+// taosThreadMutexUnlock(&execInfo.lock);
+//
+// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno);
+// return -1;
+// } else {
+// mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet",
+// req.streamId, req.taskId);
+// // todo wait for stream is created
+// }
+// }
+//
+// mInfo("vgId:%d stream:0x%" PRIx64 " %s meta-stored checkpointId:%" PRId64, req.nodeId, req.streamId, pStream->name,
+// pStream->checkpointId);
+//
+// int32_t numOfTasks = (pStream == NULL) ? 0 : mndGetNumOfStreamTasks(pStream);
+// if ((pStream != NULL) && (pStream->checkpointId == 0)) { // not generated checkpoint yet, return 0 directly
+// taosThreadMutexUnlock(&execInfo.lock);
+// mndCreateSetConsensusChkptIdTrans(pMnode, pStream, req.taskId, 0, req.startTs);
+//
+// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno);
+// return TSDB_CODE_SUCCESS;
+// }
+//
+// int32_t num = 0;
+// int64_t chkId = getConsensusId(req.streamId, numOfTasks, &num);
+//
+// // some tasks not send hbMsg to mnode yet, wait for 5s.
+// if (chkId == -1) {
+// mDebug("not all(%d/%d) task(s) send hbMsg yet, wait for a while and check again, s-task:0x%x", req.taskId, num,
+// numOfTasks);
+// SCheckpointConsensusInfo *pInfo = mndGetConsensusInfo(execInfo.pStreamConsensus, req.streamId, numOfTasks);
+// mndAddConsensusTasks(pInfo, &req);
+//
+// taosThreadMutexUnlock(&execInfo.lock);
+// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno);
+// return 0;
+// }
+//
+// if (chkId == req.checkpointId) {
+// mDebug("vgId:%d stream:0x%" PRIx64 " %s consensus-checkpointId is:%" PRId64 ", meta-stored checkpointId:%" PRId64,
+// req.nodeId, req.streamId, pStream->name, chkId, pStream->checkpointId);
+// mndCreateSetConsensusChkptIdTrans(pMnode, pStream, req.taskId, chkId, req.startTs);
+//
+// taosThreadMutexUnlock(&execInfo.lock);
+// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno);
+// return 0;
+// }
+//
+// // wait for 5s and check again
+// SCheckpointConsensusInfo *pInfo = mndGetConsensusInfo(execInfo.pStreamConsensus, req.streamId, numOfTasks);
+// mndAddConsensusTasks(pInfo, &req);
+//
+// if (pStream != NULL) {
+// mndReleaseStream(pMnode, pStream);
+// }
+//
+// taosThreadMutexUnlock(&execInfo.lock);
+// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno);
+// return 0;
+//}
+
+int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) {
+ SMnode *pMnode = pMsg->info.node;
+ int64_t now = taosGetTimestampMs();
+ SArray *pStreamList = taosArrayInit(4, sizeof(int64_t));
+
+ mDebug("start to process consensus-checkpointId in tmr");
+
+ bool allReady = true;
+ SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady);
+ taosArrayDestroy(pNodeSnapshot);
+ if (!allReady) {
+ mWarn("not all vnodes are ready, end to process the consensus-checkpointId in tmr process");
+ taosArrayDestroy(pStreamList);
+ return 0;
+ }
- // register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans.
taosThreadMutexLock(&execInfo.lock);
- SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId);
- if (pStream == NULL) {
- mWarn("failed to find the stream:0x%" PRIx64 ", not handle checkpoint-report, try to acquire in buf", req.streamId);
+ void *pIter = NULL;
+ while ((pIter = taosHashIterate(execInfo.pStreamConsensus, pIter)) != NULL) {
+ SCheckpointConsensusInfo *pInfo = (SCheckpointConsensusInfo *)pIter;
- // not in meta-store yet, try to acquire the task in exec buffer
- // the checkpoint req arrives too soon before the completion of the create stream trans.
- STaskId id = {.streamId = req.streamId, .taskId = req.taskId};
- void *p = taosHashGet(execInfo.pTaskMap, &id, sizeof(id));
- if (p == NULL) {
- mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint-report", req.streamId);
- terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
- taosThreadMutexUnlock(&execInfo.lock);
- return -1;
- } else {
- mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet",
- req.streamId, req.taskId);
+ int64_t streamId = -1;
+ int32_t num = taosArrayGetSize(pInfo->pTaskList);
+ SArray *pList = taosArrayInit(4, sizeof(int32_t));
+
+ SStreamObj *pStream = mndGetStreamObj(pMnode, pInfo->streamId);
+ if (pStream == NULL) { // stream has been dropped already
+ mDebug("stream:0x%" PRIx64 " dropped already, continue", pInfo->streamId);
+ taosArrayDestroy(pList);
+ continue;
}
- }
- int32_t numOfTasks = (pStream == NULL) ? 0 : mndGetNumOfStreamTasks(pStream);
+ for (int32_t j = 0; j < num; ++j) {
+ SCheckpointConsensusEntry *pe = taosArrayGet(pInfo->pTaskList, j);
+ streamId = pe->req.streamId;
- SCheckpointConsensusInfo *pInfo = mndGetConsensusInfo(execInfo.pStreamConsensus, req.streamId);
+ int32_t existed = 0;
+ bool allSame = true;
+ int64_t chkId = getConsensusId(pe->req.streamId, pInfo->numOfTasks, &existed, &allSame);
+ if (chkId == -1) {
+ mDebug("not all(%d/%d) task(s) send hbMsg yet, wait for a while and check again, s-task:0x%x", existed,
+ pInfo->numOfTasks, pe->req.taskId);
+ break;
+ }
- int64_t ckId = mndGetConsensusCheckpointId(pInfo, pStream);
- if (ckId != -1) { // consensus checkpoint id already exist
- SRpcMsg rsp = {0};
- rsp.code = 0;
- rsp.info = pReq->info;
- rsp.contLen = sizeof(SRestoreCheckpointInfoRsp) + sizeof(SMsgHead);
- rsp.pCont = rpcMallocCont(rsp.contLen);
+ if (((now - pe->ts) >= 10 * 1000) || allSame) {
+ mDebug("s-task:0x%x sendTs:%" PRId64 " wait %.2fs and all tasks have same checkpointId", pe->req.taskId,
+ pe->req.startTs, (now - pe->ts) / 1000.0);
+ ASSERT(chkId <= pe->req.checkpointId);
+ mndCreateSetConsensusChkptIdTrans(pMnode, pStream, pe->req.taskId, chkId, pe->req.startTs);
- SMsgHead *pHead = rsp.pCont;
- pHead->vgId = htonl(req.nodeId);
-
- mDebug("stream:0x%" PRIx64 " consensus-checkpointId:%" PRId64 " exists, return directly", req.streamId, ckId);
- doSendConsensusCheckpointRsp(&req, &rsp, ckId);
-
- taosThreadMutexUnlock(&execInfo.lock);
- pReq->info.handle = NULL; // disable auto rsp
-
- return TSDB_CODE_SUCCESS;
- }
-
- mndAddConsensusTasks(pInfo, &req, pReq);
-
- int32_t total = 0;
- if (mndAllTaskSendCheckpointId(pInfo, numOfTasks, &total)) { // all tasks has send the reqs
- // start transaction to set the checkpoint id
- int64_t checkpointId = mndGetConsensusCheckpointId(pInfo, pStream);
- mInfo("stream:0x%" PRIx64 " %s all %d tasks send latest checkpointId, the consensus-checkpointId is:%" PRId64
- " will be issued soon",
- req.streamId, pStream->name, numOfTasks, checkpointId);
-
- // start the checkpoint consensus trans
- int32_t code = mndSendConsensusCheckpointIdRsp(pInfo->pTaskList, checkpointId);
- if (code == TSDB_CODE_SUCCESS) {
- mndClearConsensusRspEntry(pInfo);
- mDebug("clear all waiting for rsp entry for stream:0x%" PRIx64, req.streamId);
- } else {
- mDebug("stream:0x%" PRIx64 " not start send consensus-checkpointId msg, due to not all task ready", req.streamId);
+ taosArrayPush(pList, &pe->req.taskId);
+ streamId = pe->req.streamId;
+ } else {
+ mDebug("s-task:0x%x sendTs:%" PRId64 " wait %.2fs already, wait for next round to check", pe->req.taskId,
+ pe->req.startTs, (now - pe->ts) / 1000.0);
+ }
}
- } else {
- mDebug("stream:0x%" PRIx64 " %d/%d tasks send consensus-checkpointId info", req.streamId, total, numOfTasks);
- }
- if (pStream != NULL) {
mndReleaseStream(pMnode, pStream);
+
+ if (taosArrayGetSize(pList) > 0) {
+ for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
+ int32_t *taskId = taosArrayGet(pList, i);
+ for (int32_t k = 0; k < taosArrayGetSize(pInfo->pTaskList); ++k) {
+ SCheckpointConsensusEntry *pe = taosArrayGet(pInfo->pTaskList, k);
+ if (pe->req.taskId == *taskId) {
+ taosArrayRemove(pInfo->pTaskList, k);
+ break;
+ }
+ }
+ }
+ }
+
+ taosArrayDestroy(pList);
+
+ if (taosArrayGetSize(pInfo->pTaskList) == 0) {
+ mndClearConsensusRspEntry(pInfo);
+ ASSERT(streamId != -1);
+ taosArrayPush(pStreamList, &streamId);
+ }
+ }
+
+ for (int32_t i = 0; i < taosArrayGetSize(pStreamList); ++i) {
+ int64_t *pStreamId = (int64_t *)taosArrayGet(pStreamList, i);
+ mndClearConsensusCheckpointId(execInfo.pStreamConsensus, *pStreamId);
}
taosThreadMutexUnlock(&execInfo.lock);
- pReq->info.handle = NULL; // disable auto rsp
- return 0;
+ taosArrayDestroy(pStreamList);
+ mDebug("end to process consensus-checkpointId in tmr");
+ return TSDB_CODE_SUCCESS;
}
static int32_t mndProcessCreateStreamReqFromMNode(SRpcMsg *pReq) {
diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c
index c7f97b4a62..1452ac77d2 100644
--- a/source/dnode/mnode/impl/src/mndStreamHb.c
+++ b/source/dnode/mnode/impl/src/mndStreamHb.c
@@ -246,7 +246,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
}
tDecoderClear(&decoder);
- mTrace("receive stream-meta hb from vgId:%d, active numOfTasks:%d, msgId:%d", req.vgId, req.numOfTasks, req.msgId);
+ mDebug("receive stream-meta hb from vgId:%d, active numOfTasks:%d, msgId:%d", req.vgId, req.numOfTasks, req.msgId);
pFailedChkpt = taosArrayInit(4, sizeof(SFailedCheckpointInfo));
pOrphanTasks = taosArrayInit(4, sizeof(SOrphanTask));
@@ -284,6 +284,23 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
continue;
}
+ STaskCkptInfo *pChkInfo = &p->checkpointInfo;
+ if (pChkInfo->consensusChkptId != 0) {
+ SRestoreCheckpointInfo cp = {
+ .streamId = p->id.streamId,
+ .taskId = p->id.taskId,
+ .checkpointId = p->checkpointInfo.latestId,
+ .startTs = pChkInfo->consensusTs,
+ };
+
+ SStreamObj *pStream = mndGetStreamObj(pMnode, p->id.streamId);
+ int32_t numOfTasks = mndGetNumOfStreamTasks(pStream);
+
+ SCheckpointConsensusInfo *pInfo = mndGetConsensusInfo(execInfo.pStreamConsensus, p->id.streamId, numOfTasks);
+ mndAddConsensusTasks(pInfo, &cp);
+ mndReleaseStream(pMnode, pStream);
+ }
+
if (pTaskEntry->stage != p->stage && pTaskEntry->stage != -1) {
updateStageInfo(pTaskEntry, p->stage);
if (pTaskEntry->nodeId == SNODE_HANDLE) {
@@ -292,7 +309,6 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
} else {
streamTaskStatusCopy(pTaskEntry, p);
- STaskCkptInfo *pChkInfo = &p->checkpointInfo;
if ((pChkInfo->activeId != 0) && pChkInfo->failed) {
mError("stream task:0x%" PRIx64 " checkpointId:%" PRIx64 " transId:%d failed, kill it", p->id.taskId,
pChkInfo->activeId, pChkInfo->activeTransId);
diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c
index 8fb5bc8a99..c4adbd0fc3 100644
--- a/source/dnode/mnode/impl/src/mndStreamUtil.c
+++ b/source/dnode/mnode/impl/src/mndStreamUtil.c
@@ -84,9 +84,10 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) {
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
SVgObj *pVgroup = NULL;
+ int32_t replica = -1; // do the replica check
*allReady = true;
- SArray *pVgroupListSnapshot = taosArrayInit(4, sizeof(SNodeEntry));
+ SArray *pVgroupList = taosArrayInit(4, sizeof(SNodeEntry));
while (1) {
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
@@ -97,6 +98,17 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) {
SNodeEntry entry = {.nodeId = pVgroup->vgId, .hbTimestamp = pVgroup->updateTime};
entry.epset = mndGetVgroupEpset(pMnode, pVgroup);
+ if (replica == -1) {
+ replica = pVgroup->replica;
+ } else {
+ if (replica != pVgroup->replica) {
+ mInfo("vgId:%d replica:%d inconsistent with other vgroups replica:%d, not ready for stream operations",
+ pVgroup->vgId, pVgroup->replica, replica);
+ *allReady = false;
+ break;
+ }
+ }
+
// if not all ready till now, no need to check the remaining vgroups.
if (*allReady) {
for (int32_t i = 0; i < pVgroup->replica; ++i) {
@@ -107,8 +119,10 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) {
}
ESyncState state = pVgroup->vnodeGid[i].syncState;
- if (state == TAOS_SYNC_STATE_OFFLINE || state == TAOS_SYNC_STATE_ERROR) {
- mInfo("vgId:%d offline/err, not ready for checkpoint or other operations", pVgroup->vgId);
+ if (state == TAOS_SYNC_STATE_OFFLINE || state == TAOS_SYNC_STATE_ERROR || state == TAOS_SYNC_STATE_LEARNER ||
+ state == TAOS_SYNC_STATE_CANDIDATE) {
+ mInfo("vgId:%d state:%d , not ready for checkpoint or other operations, not check other vgroups",
+ pVgroup->vgId, state);
*allReady = false;
break;
}
@@ -119,7 +133,7 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) {
epsetToStr(&entry.epset, buf, tListLen(buf));
mDebug("take node snapshot, nodeId:%d %s", entry.nodeId, buf);
- taosArrayPush(pVgroupListSnapshot, &entry);
+ taosArrayPush(pVgroupList, &entry);
sdbRelease(pSdb, pVgroup);
}
@@ -138,11 +152,11 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) {
epsetToStr(&entry.epset, buf, tListLen(buf));
mDebug("take snode snapshot, nodeId:%d %s", entry.nodeId, buf);
- taosArrayPush(pVgroupListSnapshot, &entry);
+ taosArrayPush(pVgroupList, &entry);
sdbRelease(pSdb, pObj);
}
- return pVgroupListSnapshot;
+ return pVgroupList;
}
SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId) {
@@ -637,6 +651,7 @@ void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo* pExecInfo) {
void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
taosThreadMutexLock(&pExecNode->lock);
+ // 1. remove task entries
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
@@ -646,8 +661,11 @@ void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
}
ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList));
- taosThreadMutexUnlock(&pExecNode->lock);
+ // 2. remove stream entry in consensus hash table
+ mndClearConsensusCheckpointId(execInfo.pStreamConsensus, pStream->uid);
+
+ taosThreadMutexUnlock(&pExecNode->lock);
destroyStreamTaskIter(pIter);
}
@@ -821,50 +839,113 @@ int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) {
return TSDB_CODE_SUCCESS;
}
-int32_t doSendConsensusCheckpointRsp(SRestoreCheckpointInfo* pInfo, SRpcMsg* pMsg, int64_t checkpointId) {
+static int32_t mndStreamSetChkptIdAction(SMnode *pMnode, STrans *pTrans, SStreamTask* pTask, int64_t checkpointId, int64_t ts) {
+ SRestoreCheckpointInfo req = {
+ .taskId = pTask->id.taskId,
+ .streamId = pTask->id.streamId,
+ .checkpointId = checkpointId,
+ .startTs = ts,
+ .nodeId = pTask->info.nodeId,
+ .transId = pTrans->id,
+ };
+
int32_t code = 0;
int32_t blen;
-
- SRestoreCheckpointInfoRsp req = {
- .streamId = pInfo->streamId, .taskId = pInfo->taskId, .checkpointId = checkpointId, .startTs = pInfo->startTs};
-
- tEncodeSize(tEncodeRestoreCheckpointInfoRsp, &req, blen, code);
+ tEncodeSize(tEncodeRestoreCheckpointInfo, &req, blen, code);
if (code < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
int32_t tlen = sizeof(SMsgHead) + blen;
- void *abuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
+
+ void *pBuf = taosMemoryMalloc(tlen);
+ if (pBuf == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ void *abuf = POINTER_SHIFT(pBuf, sizeof(SMsgHead));
SEncoder encoder;
tEncoderInit(&encoder, abuf, tlen);
- tEncodeRestoreCheckpointInfoRsp(&encoder, &req);
+ tEncodeRestoreCheckpointInfo(&encoder, &req);
- SMsgHead *pMsgHead = (SMsgHead *)pMsg->pCont;
+ SMsgHead *pMsgHead = (SMsgHead *)pBuf;
pMsgHead->contLen = htonl(tlen);
- pMsgHead->vgId = htonl(pInfo->nodeId);
+ pMsgHead->vgId = htonl(pTask->info.nodeId);
+
tEncoderClear(&encoder);
- tmsgSendRsp(pMsg);
+ SEpSet epset = {0};
+ bool hasEpset = false;
+ code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
+ if (code != TSDB_CODE_SUCCESS || !hasEpset) {
+ taosMemoryFree(pBuf);
+ return code;
+ }
+
+ code = setTransAction(pTrans, pBuf, tlen, TDMT_STREAM_CONSEN_CHKPT, &epset, 0, TSDB_CODE_VND_INVALID_VGROUP_ID);
+ if (code != TSDB_CODE_SUCCESS) {
+ taosMemoryFree(pBuf);
+ }
+
return code;
}
-int32_t mndSendConsensusCheckpointIdRsp(SArray* pInfoList, int64_t checkpointId) {
- for(int32_t i = 0; i < taosArrayGetSize(pInfoList); ++i) {
- SCheckpointConsensusEntry* pInfo = taosArrayGet(pInfoList, i);
- doSendConsensusCheckpointRsp(&pInfo->req, &pInfo->rsp, checkpointId);
+int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, int32_t taskId, int64_t checkpointId,
+ int64_t ts) {
+ char msg[128] = {0};
+ snprintf(msg, tListLen(msg), "set consen-chkpt-id for task:0x%x", taskId);
+
+ STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_CONSEN_NAME, msg);
+ if (pTrans == NULL) {
+ return terrno;
}
- return 0;
+
+ STaskId id = {.streamId = pStream->uid, .taskId = taskId};
+ SStreamTask *pTask = mndGetStreamTask(&id, pStream);
+ ASSERT(pTask);
+
+ /*int32_t code = */ mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_CONSEN_NAME, pStream->uid);
+ int32_t code = mndStreamSetChkptIdAction(pMnode, pTrans, pTask, checkpointId, ts);
+ if (code != 0) {
+ sdbRelease(pMnode->pSdb, pStream);
+ mndTransDrop(pTrans);
+ return code;
+ }
+
+ code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY);
+ if (code != TSDB_CODE_SUCCESS) {
+ sdbRelease(pMnode->pSdb, pStream);
+ mndTransDrop(pTrans);
+ return -1;
+ }
+
+ if (mndTransPrepare(pMnode, pTrans) != 0) {
+ mError("trans:%d, failed to prepare set consensus-chkptId trans since %s", pTrans->id, terrstr());
+ sdbRelease(pMnode->pSdb, pStream);
+ mndTransDrop(pTrans);
+ return -1;
+ }
+
+ sdbRelease(pMnode->pSdb, pStream);
+ mndTransDrop(pTrans);
+
+ return TSDB_CODE_ACTION_IN_PROGRESS;
}
-SCheckpointConsensusInfo* mndGetConsensusInfo(SHashObj* pHash, int64_t streamId) {
+SCheckpointConsensusInfo* mndGetConsensusInfo(SHashObj* pHash, int64_t streamId, int32_t numOfTasks) {
void* pInfo = taosHashGet(pHash, &streamId, sizeof(streamId));
if (pInfo != NULL) {
return (SCheckpointConsensusInfo*)pInfo;
}
SCheckpointConsensusInfo p = {
- .genTs = -1, .checkpointId = -1, .pTaskList = taosArrayInit(4, sizeof(SCheckpointConsensusEntry))};
+ .pTaskList = taosArrayInit(4, sizeof(SCheckpointConsensusEntry)),
+ .numOfTasks = numOfTasks,
+ .streamId = streamId,
+ };
+
taosHashPut(pHash, &streamId, sizeof(streamId), &p, sizeof(p));
void* pChkptInfo = (SCheckpointConsensusInfo*)taosHashGet(pHash, &streamId, sizeof(streamId));
@@ -873,87 +954,27 @@ SCheckpointConsensusInfo* mndGetConsensusInfo(SHashObj* pHash, int64_t streamId)
// no matter existed or not, add the request into info list anyway, since we need to send rsp mannually
// discard the msg may lead to the lost of connections.
-void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpointInfo *pRestoreInfo, SRpcMsg *pMsg) {
- SCheckpointConsensusEntry info = {0};
+void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpointInfo *pRestoreInfo) {
+ SCheckpointConsensusEntry info = {.ts = taosGetTimestampMs()};
memcpy(&info.req, pRestoreInfo, sizeof(info.req));
- info.rsp.code = 0;
- info.rsp.info = pMsg->info;
- info.rsp.contLen = sizeof(SRestoreCheckpointInfoRsp) + sizeof(SMsgHead);
- info.rsp.pCont = rpcMallocCont(info.rsp.contLen);
-
- SMsgHead *pHead = info.rsp.pCont;
- pHead->vgId = htonl(pRestoreInfo->nodeId);
+ for (int32_t i = 0; i < taosArrayGetSize(pInfo->pTaskList); ++i) {
+ SCheckpointConsensusEntry *p = taosArrayGet(pInfo->pTaskList, i);
+ if (p->req.taskId == info.req.taskId) {
+ mDebug("s-task:0x%x already in consensus-checkpointId list for stream:0x%" PRIx64 ", update ts %" PRId64
+ "->%" PRId64 " total existed:%d",
+ pRestoreInfo->taskId, pRestoreInfo->streamId, p->req.startTs, info.req.startTs,
+ (int32_t)taosArrayGetSize(pInfo->pTaskList));
+ p->req.startTs = info.req.startTs;
+ return;
+ }
+ }
taosArrayPush(pInfo->pTaskList, &info);
-}
-
-static int32_t entryComparFn(const void* p1, const void* p2) {
- const SCheckpointConsensusEntry* pe1 = p1;
- const SCheckpointConsensusEntry* pe2 = p2;
-
- if (pe1->req.taskId == pe2->req.taskId) {
- return 0;
- }
-
- return pe1->req.taskId < pe2->req.taskId? -1:1;
-}
-
-bool mndAllTaskSendCheckpointId(SCheckpointConsensusInfo* pInfo, int32_t numOfTasks, int32_t* pTotal) {
- int32_t numOfExisted = taosArrayGetSize(pInfo->pTaskList);
- if (numOfExisted < numOfTasks) {
- if (pTotal != NULL) {
- *pTotal = numOfExisted;
- }
- return false;
- }
-
- taosArraySort(pInfo->pTaskList, entryComparFn);
-
- int32_t num = 1;
- int32_t taskId = ((SCheckpointConsensusEntry*)taosArrayGet(pInfo->pTaskList, 0))->req.taskId;
- for(int32_t i = 1; i < taosArrayGetSize(pInfo->pTaskList); ++i) {
- SCheckpointConsensusEntry* pe = taosArrayGet(pInfo->pTaskList, i);
- if (pe->req.taskId != taskId) {
- num += 1;
- taskId = pe->req.taskId;
- }
- }
-
- if (pTotal != NULL) {
- *pTotal = num;
- }
-
- ASSERT(num <= numOfTasks);
- return num == numOfTasks;
-}
-
-int64_t mndGetConsensusCheckpointId(SCheckpointConsensusInfo* pInfo, SStreamObj* pStream) {
- if (pInfo->genTs > 0) { // there is no checkpoint ever generated if the checkpointId is 0.
- mDebug("existed consensus-checkpointId:%" PRId64 " for stream:0x%" PRIx64 " %s exist, and return",
- pInfo->checkpointId, pStream->uid, pStream->name);
- return pInfo->checkpointId;
- }
-
- int32_t numOfTasks = mndGetNumOfStreamTasks(pStream);
- if (!mndAllTaskSendCheckpointId(pInfo, numOfTasks, NULL)) {
- return -1;
- }
-
- int64_t checkpointId = INT64_MAX;
-
- for (int32_t i = 0; i < taosArrayGetSize(pInfo->pTaskList); ++i) {
- SCheckpointConsensusEntry *pEntry = taosArrayGet(pInfo->pTaskList, i);
- if (pEntry->req.checkpointId < checkpointId) {
- checkpointId = pEntry->req.checkpointId;
- mTrace("stream:0x%" PRIx64 " %s task:0x%x vgId:%d latest checkpointId:%" PRId64, pStream->uid, pStream->name,
- pEntry->req.taskId, pEntry->req.nodeId, pEntry->req.checkpointId);
- }
- }
-
- pInfo->checkpointId = checkpointId;
- pInfo->genTs = taosGetTimestampMs();
- return checkpointId;
+ int32_t num = taosArrayGetSize(pInfo->pTaskList);
+ mDebug("s-task:0x%x checkpointId:%" PRId64 " added into consensus-checkpointId list, stream:0x%" PRIx64
+ " waiting tasks:%d",
+ pRestoreInfo->taskId, pRestoreInfo->checkpointId, pRestoreInfo->streamId, num);
}
void mndClearConsensusRspEntry(SCheckpointConsensusInfo* pInfo) {
@@ -968,15 +989,15 @@ int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId) {
return TSDB_CODE_SUCCESS;
}
-int32_t mndRegisterConsensusChkptId(SHashObj* pHash, int64_t streamId) {
- void* pInfo = taosHashGet(pHash, &streamId, sizeof(streamId));
- ASSERT(pInfo == NULL);
-
- SCheckpointConsensusInfo p = {.genTs = taosGetTimestampMs(), .checkpointId = 0, .pTaskList = NULL};
- taosHashPut(pHash, &streamId, sizeof(streamId), &p, sizeof(p));
-
- SCheckpointConsensusInfo* pChkptInfo = (SCheckpointConsensusInfo*)taosHashGet(pHash, &streamId, sizeof(streamId));
- ASSERT(pChkptInfo->genTs > 0 && pChkptInfo->checkpointId == 0);
- mDebug("s-task:0x%" PRIx64 " set the initial consensus-checkpointId:0", streamId);
- return TSDB_CODE_SUCCESS;
-}
\ No newline at end of file
+//int32_t mndRegisterConsensusChkptId(SHashObj* pHash, int64_t streamId) {
+// void* pInfo = taosHashGet(pHash, &streamId, sizeof(streamId));
+// ASSERT(pInfo == NULL);
+//
+// SCheckpointConsensusInfo p = {.genTs = taosGetTimestampMs(), .checkpointId = 0, .pTaskList = NULL};
+// taosHashPut(pHash, &streamId, sizeof(streamId), &p, sizeof(p));
+//
+// SCheckpointConsensusInfo* pChkptInfo = (SCheckpointConsensusInfo*)taosHashGet(pHash, &streamId, sizeof(streamId));
+// ASSERT(pChkptInfo->genTs > 0 && pChkptInfo->checkpointId == 0);
+// mDebug("s-task:0x%" PRIx64 " set the initial consensus-checkpointId:0", streamId);
+// return TSDB_CODE_SUCCESS;
+//}
\ No newline at end of file
diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c
index 4669fdfb38..58dab20859 100644
--- a/source/dnode/mnode/impl/src/mndTrans.c
+++ b/source/dnode/mnode/impl/src/mndTrans.c
@@ -1241,6 +1241,7 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray)
}
}
+// execute at bottom half
static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) {
if (pAction->rawWritten) return 0;
if (topHalf) {
@@ -1267,6 +1268,7 @@ static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransActi
return code;
}
+// execute at top half
static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) {
if (pAction->msgSent) return 0;
if (mndCannotExecuteTransAction(pMnode, topHalf)) {
@@ -1644,6 +1646,11 @@ static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans, boo
pTrans->stage = TRN_STAGE_FINISH; // TRN_STAGE_PRE_FINISH is not necessary
mInfo("trans:%d, stage from commitAction to finished", pTrans->id);
continueExec = true;
+ } else if (code == TSDB_CODE_MND_TRANS_CTX_SWITCH && topHalf) {
+ pTrans->code = 0;
+ pTrans->stage = TRN_STAGE_COMMIT;
+ mInfo("trans:%d, back to commit stage", pTrans->id);
+ continueExec = true;
} else {
pTrans->code = terrno;
pTrans->failedTimes++;
@@ -1783,11 +1790,13 @@ void mndTransExecuteImp(SMnode *pMnode, STrans *pTrans, bool topHalf) {
mndTransSendRpcRsp(pMnode, pTrans);
}
+// start trans, pullup, receive rsp, kill
void mndTransExecute(SMnode *pMnode, STrans *pTrans) {
bool topHalf = true;
return mndTransExecuteImp(pMnode, pTrans, topHalf);
}
+// update trans
void mndTransRefresh(SMnode *pMnode, STrans *pTrans) {
bool topHalf = false;
return mndTransExecuteImp(pMnode, pTrans, topHalf);
diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c
index 9686fd3789..cfa24b2430 100644
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@@ -43,14 +43,14 @@ int32_t sndBuildStreamTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProce
char *p = streamTaskGetStatus(pTask)->name;
if (pTask->info.fillHistory) {
- sndInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
+ sndInfo("vgId:%d build stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64
" child id:%d, level:%d, status:%s fill-history:%d, related stream task:0x%x trigger:%" PRId64 " ms",
SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory,
(int32_t)pTask->streamTaskId.taskId, pTask->info.delaySchedParam);
} else {
- sndInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
+ sndInfo("vgId:%d build stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64
" child id:%d, level:%d, status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64 " ms",
SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
@@ -149,15 +149,15 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) {
case TDMT_VND_STREAM_TASK_UPDATE:
return tqStreamTaskProcessUpdateReq(pSnode->pMeta, &pSnode->msgCb, pMsg, true);
case TDMT_VND_STREAM_TASK_RESET:
- return tqStreamTaskProcessTaskResetReq(pSnode->pMeta, pMsg);
+ return tqStreamTaskProcessTaskResetReq(pSnode->pMeta, pMsg->pCont);
case TDMT_STREAM_TASK_PAUSE:
return tqStreamTaskProcessTaskPauseReq(pSnode->pMeta, pMsg->pCont);
case TDMT_STREAM_TASK_RESUME:
return tqStreamTaskProcessTaskResumeReq(pSnode->pMeta, pMsg->info.conn.applyIndex, pMsg->pCont, false);
case TDMT_STREAM_TASK_UPDATE_CHKPT:
- return tqStreamTaskProcessUpdateCheckpointReq(pSnode->pMeta, true, pMsg->pCont, pMsg->contLen);
- case TDMT_MND_STREAM_CHKPT_CONSEN_RSP:
- return tqStreamProcessConsensusChkptRsp(pSnode->pMeta, pMsg);
+ return tqStreamTaskProcessUpdateCheckpointReq(pSnode->pMeta, true, pMsg->pCont);
+ case TDMT_STREAM_CONSEN_CHKPT:
+ return tqStreamTaskProcessConsenChkptIdReq(pSnode->pMeta, pMsg);
default:
ASSERT(0);
}
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 1bec226489..4a47e08730 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -298,6 +298,7 @@ int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg);
int32_t tqStreamProgressRetrieveReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskUpdateCheckpointReq(STQ* pTq, char* msg, int32_t msgLen);
+int32_t tqProcessTaskConsenChkptIdReq(STQ* pTq, SRpcMsg* pMsg);
// sma
int32_t smaInit();
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 0a64b9c165..ac57a003c5 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -1016,7 +1016,11 @@ int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) {
}
int32_t tqProcessTaskUpdateCheckpointReq(STQ* pTq, char* msg, int32_t msgLen) {
- return tqStreamTaskProcessUpdateCheckpointReq(pTq->pStreamMeta, pTq->pVnode->restored, msg, msgLen);
+ return tqStreamTaskProcessUpdateCheckpointReq(pTq->pStreamMeta, pTq->pVnode->restored, msg);
+}
+
+int32_t tqProcessTaskConsenChkptIdReq(STQ* pTq, SRpcMsg* pMsg) {
+ return tqStreamTaskProcessConsenChkptIdReq(pTq->pStreamMeta, pMsg);
}
int32_t tqProcessTaskPauseReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) {
@@ -1239,7 +1243,7 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {
}
int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) {
- return tqStreamTaskProcessTaskResetReq(pTq->pStreamMeta, pMsg);
+ return tqStreamTaskProcessTaskResetReq(pTq->pStreamMeta, pMsg->pCont);
}
int32_t tqProcessTaskRetrieveTriggerReq(STQ* pTq, SRpcMsg* pMsg) {
@@ -1277,5 +1281,5 @@ int32_t tqProcessTaskChkptReportRsp(STQ* pTq, SRpcMsg* pMsg) {
}
int32_t tqProcessTaskConsensusChkptRsp(STQ* pTq, SRpcMsg* pMsg) {
- return tqStreamProcessConsensusChkptRsp(pTq->pStreamMeta, pMsg);
+ return tqStreamProcessConsensusChkptRsp2(pTq->pStreamMeta, pMsg);
}
diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c
index 290266d94a..c79fc66a06 100644
--- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c
+++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c
@@ -51,10 +51,9 @@ int32_t streamStateSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamS
SStreamSnapReader* pSnapReader = NULL;
- if (streamSnapReaderOpen(meta, sver, chkpId, meta->path, &pSnapReader) == 0) {
+ if ((code = streamSnapReaderOpen(meta, sver, chkpId, meta->path, &pSnapReader)) == 0) {
pReader->complete = 1;
} else {
- code = -1;
taosMemoryFree(pReader);
goto _err;
}
@@ -75,7 +74,7 @@ _err:
int32_t streamStateSnapReaderClose(SStreamStateReader* pReader) {
int32_t code = 0;
tqDebug("vgId:%d, vnode %s snapshot reader closed", TD_VID(pReader->pTq->pVnode), STREAM_STATE_TRANSFER);
- streamSnapReaderClose(pReader->pReaderImpl);
+ code = streamSnapReaderClose(pReader->pReaderImpl);
taosMemoryFree(pReader);
return code;
}
@@ -138,32 +137,36 @@ int32_t streamStateSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamS
pWriter->sver = sver;
pWriter->ever = ever;
- taosMkDir(pTq->pStreamMeta->path);
-
- SStreamSnapWriter* pSnapWriter = NULL;
- if (streamSnapWriterOpen(pTq, sver, ever, pTq->pStreamMeta->path, &pSnapWriter) < 0) {
+ if (taosMkDir(pTq->pStreamMeta->path) != 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ tqError("vgId:%d, vnode %s snapshot writer failed to create directory %s since %s", TD_VID(pTq->pVnode),
+ STREAM_STATE_TRANSFER, pTq->pStreamMeta->path, tstrerror(code));
goto _err;
}
- tqDebug("vgId:%d, vnode %s snapshot writer opened, path:%s", TD_VID(pTq->pVnode), STREAM_STATE_TRANSFER, pTq->pStreamMeta->path);
+ SStreamSnapWriter* pSnapWriter = NULL;
+ if ((code = streamSnapWriterOpen(pTq, sver, ever, pTq->pStreamMeta->path, &pSnapWriter)) < 0) {
+ goto _err;
+ }
+
+ tqDebug("vgId:%d, vnode %s snapshot writer opened, path:%s", TD_VID(pTq->pVnode), STREAM_STATE_TRANSFER,
+ pTq->pStreamMeta->path);
pWriter->pWriterImpl = pSnapWriter;
*ppWriter = pWriter;
- return code;
+ return 0;
+
_err:
tqError("vgId:%d, vnode %s snapshot writer failed to open since %s", TD_VID(pTq->pVnode), STREAM_STATE_TRANSFER,
tstrerror(code));
taosMemoryFree(pWriter);
*ppWriter = NULL;
- return -1;
+ return code;
}
int32_t streamStateSnapWriterClose(SStreamStateWriter* pWriter, int8_t rollback) {
- int32_t code = 0;
tqDebug("vgId:%d, vnode %s snapshot writer closed", TD_VID(pWriter->pTq->pVnode), STREAM_STATE_TRANSFER);
- code = streamSnapWriterClose(pWriter->pWriterImpl, rollback);
-
- return code;
+ return streamSnapWriterClose(pWriter->pWriterImpl, rollback);
}
int32_t streamStateSnapWrite(SStreamStateWriter* pWriter, uint8_t* pData, uint32_t nData) {
diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c
index 668e178d2d..1f3c049211 100644
--- a/source/dnode/vnode/src/tqCommon/tqCommon.c
+++ b/source/dnode/vnode/src/tqCommon/tqCommon.c
@@ -228,6 +228,9 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
}
updated = streamTaskUpdateEpsetInfo(pTask, req.pNodeList);
+
+ // send the checkpoint-source-rsp for source task to end the checkpoint trans in mnode
+ streamTaskSendPreparedCheckpointsourceRsp(pTask);
streamTaskResetStatus(pTask);
streamTaskStopMonitorCheckRsp(&pTask->taskCheckInfo, pTask->id.idStr);
@@ -264,7 +267,6 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
tqDebug("s-task:%s vgId:%d not save task since not update epset actually, stop task", idstr, vgId);
}
- // stop
streamTaskStop(pTask);
if (ppHTask != NULL) {
streamTaskStop(*ppHTask);
@@ -279,7 +281,10 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
int32_t updateTasks = taosHashGetSize(pMeta->updateInfo.pTasks);
- pMeta->startInfo.tasksWillRestart = 1;
+ if (restored) {
+ tqDebug("vgId:%d s-task:0x%x update epset transId:%d, set the restart flag", vgId, req.taskId, req.transId);
+ pMeta->startInfo.tasksWillRestart = 1;
+ }
if (updateTasks < numOfTasks) {
tqDebug("vgId:%d closed tasks:%d, unclosed:%d, all tasks will be started when nodeEp update completed", vgId,
@@ -292,8 +297,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
streamMetaClearUpdateTaskList(pMeta);
if (!restored) {
- tqDebug("vgId:%d vnode restore not completed, not start the tasks, clear the start after nodeUpdate flag", vgId);
- pMeta->startInfo.tasksWillRestart = 0;
+ tqDebug("vgId:%d vnode restore not completed, not start all tasks", vgId);
} else {
tqDebug("vgId:%d all %d task(s) nodeEp updated and closed, transId:%d", vgId, numOfTasks, req.transId);
#if 0
@@ -666,7 +670,7 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
return 0;
}
-int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored, char* msg, int32_t msgLen) {
+int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored, char* msg) {
SVUpdateCheckpointInfoReq* pReq = (SVUpdateCheckpointInfoReq*)msg;
int32_t vgId = pMeta->vgId;
@@ -738,6 +742,7 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
streamMetaStartAllTasks(pMeta);
} else {
streamMetaResetStartInfo(&pMeta->startInfo, pMeta->vgId);
+ pMeta->startInfo.restartCount = 0;
streamMetaWUnLock(pMeta);
tqInfo("vgId:%d, follower node not start stream tasks or stream is disabled", vgId);
}
@@ -854,8 +859,8 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) {
return TSDB_CODE_SUCCESS;
}
-int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
- SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg->pCont;
+int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
+ SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg;
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId);
if (pTask == NULL) {
@@ -867,7 +872,6 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr);
taosThreadMutexLock(&pTask->lock);
-
streamTaskClearCheckInfo(pTask, true);
// clear flag set during do checkpoint, and open inputQ for all upstream tasks
@@ -882,9 +886,10 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
streamTaskSetStatusReady(pTask);
} else if (pState->state == TASK_STATUS__UNINIT) {
- tqDebug("s-task:%s start task by checking downstream tasks", pTask->id.idStr);
- ASSERT(pTask->status.downstreamReady == 0);
- tqStreamTaskRestoreCheckpoint(pMeta, pTask->id.streamId, pTask->id.taskId);
+// tqDebug("s-task:%s start task by checking downstream tasks", pTask->id.idStr);
+// ASSERT(pTask->status.downstreamReady == 0);
+// tqStreamTaskRestoreCheckpoint(pMeta, pTask->id.streamId, pTask->id.taskId);
+ tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState->name);
} else {
tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState->name);
}
@@ -1111,6 +1116,8 @@ int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { ret
int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); }
+int32_t tqStreamProcessConsensusChkptRsp2(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); }
+
int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SMStreamCheckpointReadyRspMsg* pRsp = pMsg->pCont;
@@ -1126,22 +1133,21 @@ int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
return TSDB_CODE_SUCCESS;
}
-int32_t tqStreamProcessConsensusChkptRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
+int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
int32_t vgId = pMeta->vgId;
+ int32_t code = 0;
+
char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
int32_t len = pMsg->contLen - sizeof(SMsgHead);
- SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS};
int64_t now = taosGetTimestampMs();
- SRestoreCheckpointInfoRsp req = {0};
+ SRestoreCheckpointInfo req = {0};
SDecoder decoder;
tDecoderInit(&decoder, (uint8_t*)msg, len);
- rsp.info.handle = NULL;
- if (tDecodeRestoreCheckpointInfoRsp(&decoder, &req) < 0) {
- // rsp.code = TSDB_CODE_MSG_DECODE_ERROR; // disable it temporarily
- tqError("vgId:%d failed to decode restore task checkpointId, code:%s", vgId, tstrerror(rsp.code));
+ if (tDecodeRestoreCheckpointInfo(&decoder, &req) < 0) {
+ tqError("vgId:%d failed to decode set consensus checkpointId req, code:%s", vgId, tstrerror(code));
tDecoderClear(&decoder);
return TSDB_CODE_SUCCESS;
}
@@ -1150,7 +1156,7 @@ int32_t tqStreamProcessConsensusChkptRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.taskId);
if (pTask == NULL) {
- tqError("vgId:%d process restore checkpointId req, failed to acquire task:0x%x, it may have been dropped already",
+ tqError("vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, req.taskId);
streamMetaAddFailedTask(pMeta, req.streamId, req.taskId);
return TSDB_CODE_SUCCESS;
@@ -1172,16 +1178,25 @@ int32_t tqStreamProcessConsensusChkptRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
taosThreadMutexLock(&pTask->lock);
ASSERT(pTask->chkInfo.checkpointId >= req.checkpointId);
+ if (pTask->chkInfo.consensusTransId >= req.transId) {
+ tqDebug("s-task:%s vgId:%d latest consensus transId:%d, expired consensus trans:%d, discard",
+ pTask->id.idStr, vgId, pTask->chkInfo.consensusTransId, req.transId);
+ taosThreadMutexUnlock(&pTask->lock);
+ streamMetaReleaseTask(pMeta, pTask);
+ return TSDB_CODE_SUCCESS;
+ }
+
if (pTask->chkInfo.checkpointId != req.checkpointId) {
- tqDebug("s-task:%s vgId:%d update the checkpoint from %" PRId64 " to %" PRId64, pTask->id.idStr, vgId,
- pTask->chkInfo.checkpointId, req.checkpointId);
+ tqDebug("s-task:%s vgId:%d update the checkpoint from %" PRId64 " to %" PRId64" transId:%d", pTask->id.idStr, vgId,
+ pTask->chkInfo.checkpointId, req.checkpointId, req.transId);
pTask->chkInfo.checkpointId = req.checkpointId;
tqSetRestoreVersionInfo(pTask);
} else {
- tqDebug("s-task:%s vgId:%d consensus-checkpointId:%" PRId64 " equals to current checkpointId, no need to update",
- pTask->id.idStr, vgId, req.checkpointId);
+ tqDebug("s-task:%s vgId:%d consensus-checkpointId:%" PRId64 " equals to current id, transId:%d not update",
+ pTask->id.idStr, vgId, req.checkpointId, req.transId);
}
+ pTask->chkInfo.consensusTransId = req.transId;
taosThreadMutexUnlock(&pTask->lock);
if (pMeta->role == NODE_ROLE_LEADER) {
diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c
index bee5a1d990..4597a44845 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@@ -1072,7 +1072,7 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray
rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch;
for (int i = 0; i < num_keys; ++i) {
SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[i];
- SLastUpdateCtx *updCtx = (SLastUpdateCtx *)taosArrayGet(updCtxArray, i);
+ SLastUpdateCtx *updCtx = (SLastUpdateCtx *)taosArrayGet(updCtxArray, idxKey->idx);
SRowKey *pRowKey = &updCtx->tsdbRowKey.key;
SColVal *pColVal = &updCtx->colVal;
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 9006d36e65..fb09a46e75 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -630,6 +630,11 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg
goto _err;
}
} break;
+ case TDMT_STREAM_CONSEN_CHKPT: {
+ if (pVnode->restored) {
+ tqProcessTaskConsenChkptIdReq(pVnode->pTq, pMsg);
+ }
+ } break;
case TDMT_STREAM_TASK_PAUSE: {
if (pVnode->restored && vnodeIsLeader(pVnode) &&
tqProcessTaskPauseReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) {
@@ -647,11 +652,6 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg
tqProcessTaskResetReq(pVnode->pTq, pMsg);
}
} break;
- case TDMT_MND_STREAM_CHKPT_CONSEN_RSP: {
- if (pVnode->restored) {
- tqProcessTaskConsensusChkptRsp(pVnode->pTq, pMsg);
- }
- } break;
case TDMT_VND_ALTER_CONFIRM:
needCommit = pVnode->config.hashChange;
if (vnodeProcessAlterConfirmReq(pVnode, ver, pReq, len, pRsp) < 0) {
@@ -738,7 +738,7 @@ int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
return qWorkerPreprocessQueryMsg(pVnode->pQuery, pMsg, TDMT_SCH_QUERY == pMsg->msgType);
}
-int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo* pInfo) {
+int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
vTrace("message in vnode query queue is processing");
if ((pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_VND_TMQ_CONSUME ||
pMsg->msgType == TDMT_VND_TMQ_CONSUME_PUSH) &&
@@ -1978,6 +1978,9 @@ _exit:
return code;
}
+extern int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb);
+extern int32_t tsdbEnableBgTask(STsdb *pTsdb);
+
static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
bool walChanged = false;
bool tsdbChanged = false;
@@ -2075,7 +2078,14 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pRe
}
if (req.sttTrigger != -1 && req.sttTrigger != pVnode->config.sttTrigger) {
- pVnode->config.sttTrigger = req.sttTrigger;
+ if (req.sttTrigger > 1 && pVnode->config.sttTrigger > 1) {
+ pVnode->config.sttTrigger = req.sttTrigger;
+ } else {
+ vnodeAWait(&pVnode->commitTask);
+ tsdbDisableAndCancelAllBgTask(pVnode->pTsdb);
+ pVnode->config.sttTrigger = req.sttTrigger;
+ tsdbEnableBgTask(pVnode->pTsdb);
+ }
}
if (req.minRows != -1 && req.minRows != pVnode->config.tsdbCfg.minRows) {
diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h
index 4b3875cc23..de0c0aa841 100644
--- a/source/libs/executor/inc/executorInt.h
+++ b/source/libs/executor/inc/executorInt.h
@@ -128,6 +128,7 @@ typedef struct SExprSupp {
SqlFunctionCtx* pCtx;
int32_t* rowEntryInfoOffset; // offset value for each row result cell info
SFilterInfo* pFilterInfo;
+ bool hasWindowOrGroup;
} SExprSupp;
typedef enum {
diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c
index a7b4532bd0..5ff0601b7a 100644
--- a/source/libs/executor/src/aggregateoperator.c
+++ b/source/libs/executor/src/aggregateoperator.c
@@ -77,6 +77,8 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
goto _error;
}
+ pOperator->exprSupp.hasWindowOrGroup = false;
+
SSDataBlock* pResBlock = createDataBlockFromDescNode(pAggNode->node.pOutputDataBlockDesc);
initBasicInfo(&pInfo->binfo, pResBlock);
@@ -519,6 +521,7 @@ int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo
}
for (int32_t i = 0; i < numOfCols; ++i) {
+ pSup->pCtx[i].hasWindowOrGroup = pSup->hasWindowOrGroup;
if (pState) {
pSup->pCtx[i].saveHandle.pBuf = NULL;
pSup->pCtx[i].saveHandle.pState = pState;
diff --git a/source/libs/executor/src/countwindowoperator.c b/source/libs/executor/src/countwindowoperator.c
index 47b24421c8..3a6c85532a 100644
--- a/source/libs/executor/src/countwindowoperator.c
+++ b/source/libs/executor/src/countwindowoperator.c
@@ -207,6 +207,8 @@ SOperatorInfo* createCountwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo
goto _error;
}
+ pOperator->exprSupp.hasWindowOrGroup = true;
+
int32_t code = TSDB_CODE_SUCCESS;
SCountWinodwPhysiNode* pCountWindowNode = (SCountWinodwPhysiNode*)physiNode;
diff --git a/source/libs/executor/src/eventwindowoperator.c b/source/libs/executor/src/eventwindowoperator.c
index d73274f85e..b898ea576a 100644
--- a/source/libs/executor/src/eventwindowoperator.c
+++ b/source/libs/executor/src/eventwindowoperator.c
@@ -66,6 +66,8 @@ SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo
goto _error;
}
+ pOperator->exprSupp.hasWindowOrGroup = true;
+
SEventWinodwPhysiNode* pEventWindowNode = (SEventWinodwPhysiNode*)physiNode;
int32_t tsSlotId = ((SColumnNode*)pEventWindowNode->window.pTspk)->slotId;
diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index 142fa6d861..36c918ef49 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -1716,6 +1716,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
pCtx->param = pFunct->pParam;
pCtx->saveHandle.currentPage = -1;
pCtx->pStore = pStore;
+ pCtx->hasWindowOrGroup = false;
}
for (int32_t i = 1; i < numOfOutput; ++i) {
@@ -2187,7 +2188,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle*
for (int i = 0; i < numOfTables; i++) {
STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i);
- info->groupId = info->uid;
+ info->groupId = groupByTbname ? info->uid : 0;
taosHashPut(pTableListInfo->remainGroups, &(info->groupId), sizeof(info->groupId), &(info->uid),
sizeof(info->uid));
diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c
index 3de8468bb8..fad2b263b2 100644
--- a/source/libs/executor/src/executorInt.c
+++ b/source/libs/executor/src/executorInt.c
@@ -909,12 +909,21 @@ void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock) {
initResultRowInfo(&pInfo->resultRowInfo);
}
-static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
+static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, SExprInfo* pExpr, int32_t numOfOutput) {
if (pCtx == NULL) {
return NULL;
}
for (int32_t i = 0; i < numOfOutput; ++i) {
+ if (pExpr != NULL) {
+ SExprInfo* pExprInfo = &pExpr[i];
+ for (int32_t j = 0; j < pExprInfo->base.numOfParams; ++j) {
+ if (pExprInfo->base.pParam[j].type == FUNC_PARAM_TYPE_VALUE) {
+ taosMemoryFree(pCtx[i].input.pData[j]);
+ taosMemoryFree(pCtx[i].input.pColumnDataAgg[j]);
+ }
+ }
+ }
for (int32_t j = 0; j < pCtx[i].numOfParams; ++j) {
taosVariantDestroy(&pCtx[i].param[j].param);
}
@@ -947,7 +956,7 @@ int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr, S
}
void cleanupExprSupp(SExprSupp* pSupp) {
- destroySqlFunctionCtx(pSupp->pCtx, pSupp->numOfExprs);
+ destroySqlFunctionCtx(pSupp->pCtx, pSupp->pExprInfo, pSupp->numOfExprs);
if (pSupp->pExprInfo != NULL) {
destroyExprInfo(pSupp->pExprInfo, pSupp->numOfExprs);
taosMemoryFreeClear(pSupp->pExprInfo);
diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c
index 3a124197e0..f0fddc28fc 100644
--- a/source/libs/executor/src/groupoperator.c
+++ b/source/libs/executor/src/groupoperator.c
@@ -502,6 +502,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode*
goto _error;
}
+ pOperator->exprSupp.hasWindowOrGroup = true;
+
SSDataBlock* pResBlock = createDataBlockFromDescNode(pAggNode->node.pOutputDataBlockDesc);
initBasicInfo(&pInfo->binfo, pResBlock);
diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c
index 19828d5146..6167497c21 100644
--- a/source/libs/executor/src/projectoperator.c
+++ b/source/libs/executor/src/projectoperator.c
@@ -99,6 +99,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys
goto _error;
}
+ pOperator->exprSupp.hasWindowOrGroup = false;
pOperator->pTaskInfo = pTaskInfo;
int32_t numOfCols = 0;
@@ -409,6 +410,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
pOperator->pTaskInfo = pTaskInfo;
SExprSupp* pSup = &pOperator->exprSupp;
+ pSup->hasWindowOrGroup = false;
SIndefRowsFuncPhysiNode* pPhyNode = (SIndefRowsFuncPhysiNode*)pNode;
@@ -724,9 +726,12 @@ static void setPseudoOutputColInfo(SSDataBlock* pResult, SqlFunctionCtx* pCtx, S
int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx,
int32_t numOfOutput, SArray* pPseudoList) {
+ int32_t code = TSDB_CODE_SUCCESS;
setPseudoOutputColInfo(pResult, pCtx, pPseudoList);
pResult->info.dataLoad = 1;
+ SArray* processByRowFunctionCtx = NULL;
+
if (pSrcBlock == NULL) {
for (int32_t k = 0; k < numOfOutput; ++k) {
int32_t outputSlotId = pExpr[k].base.resSchema.slotId;
@@ -743,7 +748,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
}
pResult->info.rows = 1;
- return TSDB_CODE_SUCCESS;
+ goto _exit;
}
if (pResult != pSrcBlock) {
@@ -816,10 +821,10 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
SColumnInfoData idata = {.info = pResColData->info, .hasNull = true};
SScalarParam dest = {.columnData = &idata};
- int32_t code = scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest);
+ code = scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest);
if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(pBlockList);
- return code;
+ goto _exit;
}
int32_t startOffset = createNewColModel ? 0 : pResult->info.rows;
@@ -852,11 +857,21 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
pfCtx->pDstBlock = pResult;
}
- int32_t code = pfCtx->fpSet.process(pfCtx);
+ code = pfCtx->fpSet.process(pfCtx);
if (code != TSDB_CODE_SUCCESS) {
- return code;
+ goto _exit;
}
numOfRows = pResInfo->numOfRes;
+ if (fmIsProcessByRowFunc(pfCtx->functionId)) {
+ if (NULL == processByRowFunctionCtx) {
+ processByRowFunctionCtx = taosArrayInit(1, sizeof(SqlFunctionCtx*));
+ if (!processByRowFunctionCtx) {
+ code = terrno;
+ goto _exit;
+ }
+ }
+ taosArrayPush(processByRowFunctionCtx, &pfCtx);
+ }
} else if (fmIsAggFunc(pfCtx->functionId)) {
// selective value output should be set during corresponding function execution
if (fmIsSelectValueFunc(pfCtx->functionId)) {
@@ -886,10 +901,10 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
SColumnInfoData idata = {.info = pResColData->info, .hasNull = true};
SScalarParam dest = {.columnData = &idata};
- int32_t code = scalarCalculate((SNode*)pExpr[k].pExpr->_function.pFunctNode, pBlockList, &dest);
+ code = scalarCalculate((SNode*)pExpr[k].pExpr->_function.pFunctNode, pBlockList, &dest);
if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(pBlockList);
- return code;
+ goto _exit;
}
int32_t startOffset = createNewColModel ? 0 : pResult->info.rows;
@@ -905,9 +920,21 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
}
}
+ if (processByRowFunctionCtx && taosArrayGetSize(processByRowFunctionCtx) > 0){
+ SqlFunctionCtx** pfCtx = taosArrayGet(processByRowFunctionCtx, 0);
+ code = (*pfCtx)->fpSet.processFuncByRow(processByRowFunctionCtx);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _exit;
+ }
+ numOfRows = (*pfCtx)->resultInfo->numOfRes;
+ }
if (!createNewColModel) {
pResult->info.rows += numOfRows;
}
-
- return TSDB_CODE_SUCCESS;
+_exit:
+ if(processByRowFunctionCtx) {
+ taosArrayDestroy(processByRowFunctionCtx);
+ processByRowFunctionCtx = NULL;
+ }
+ return code;
}
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index ed70b31845..c7a7615164 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -2198,6 +2198,17 @@ static void rebuildDeleteBlockData(SSDataBlock* pBlock, STimeWindow* pWindow, co
taosMemoryFree(p);
}
+static int32_t colIdComparFn(const void* param1, const void * param2) {
+ int32_t p1 = *(int32_t*) param1;
+ int32_t p2 = *(int32_t*) param2;
+
+ if (p1 == p2) {
+ return 0;
+ } else {
+ return (p1 < p2)? -1:1;
+ }
+}
+
static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock, STimeWindow* pTimeWindow, bool filter) {
SDataBlockInfo* pBlockInfo = &pInfo->pRes->info;
SOperatorInfo* pOperator = pInfo->pStreamScanOp;
@@ -2214,6 +2225,8 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
pBlockInfo->id.groupId = tableListGetTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid);
+ SArray* pColList = taosArrayInit(4, sizeof(int32_t));
+
// todo extract method
for (int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) {
SColMatchItem* pColMatchInfo = taosArrayGet(pInfo->matchInfo.pList, i);
@@ -2228,6 +2241,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->dstSlotId);
colDataAssign(pDst, pResCol, pBlock->info.rows, &pInfo->pRes->info);
colExists = true;
+ taosArrayPush(pColList, &pColMatchInfo->dstSlotId);
break;
}
}
@@ -2236,6 +2250,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
if (!colExists) {
SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->dstSlotId);
colDataSetNNULL(pDst, 0, pBlockInfo->rows);
+ taosArrayPush(pColList, &pColMatchInfo->dstSlotId);
}
}
@@ -2251,8 +2266,35 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
// reset the error code.
terrno = 0;
+
+ for(int32_t i = 0; i < pInfo->numOfPseudoExpr; ++i) {
+ taosArrayPush(pColList, &pInfo->pPseudoExpr[i].base.resSchema.slotId);
+ }
}
+ taosArraySort(pColList, colIdComparFn);
+
+ int32_t i = 0, j = 0;
+ while(i < taosArrayGetSize(pColList)) {
+ int32_t slot1 = *(int32_t*)taosArrayGet(pColList, i);
+ if (slot1 > j) {
+ SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, j);
+ colDataSetNNULL(pDst, 0, pBlockInfo->rows);
+ j += 1;
+ } else {
+ i += 1;
+ j += 1;
+ }
+ }
+
+ while(j < taosArrayGetSize(pInfo->pRes->pDataBlock)) {
+ SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, j);
+ colDataSetNNULL(pDst, 0, pBlockInfo->rows);
+ j += 1;
+ }
+
+ taosArrayDestroy(pColList);
+
if (filter) {
doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
}
diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c
index 78d14578a6..2bbda6e84d 100644
--- a/source/libs/executor/src/streamtimewindowoperator.c
+++ b/source/libs/executor/src/streamtimewindowoperator.c
@@ -1240,8 +1240,15 @@ void copyIntervalDeleteKey(SSHashObj* pMap, SArray* pWins) {
static SSDataBlock* buildIntervalResult(SOperatorInfo* pOperator) {
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
- SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
- uint16_t opType = pOperator->operatorType;
+
+ SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ uint16_t opType = pOperator->operatorType;
+
+ // check if query task is closed or not
+ if (isTaskKilled(pTaskInfo)) {
+ return NULL;
+ }
+
if (IS_FINAL_INTERVAL_OP(pOperator)) {
doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
if (pInfo->pPullDataRes->info.rows != 0) {
@@ -1558,6 +1565,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
goto _error;
}
+ pOperator->exprSupp.hasWindowOrGroup = true;
pOperator->pTaskInfo = pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
@@ -3133,6 +3141,7 @@ _error:
static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) {
tSimpleHashClear(pInfo->streamAggSup.pResultRows);
pInfo->streamAggSup.stateStore.streamStateSessionClear(pInfo->streamAggSup.pState);
+ pInfo->clearState = false;
}
void deleteSessionWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate,
@@ -3182,7 +3191,6 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
// semi session operator clear disk buffer
clearStreamSessionOperator(pInfo);
setStreamOperatorCompleted(pOperator);
- pInfo->clearState = false;
return NULL;
}
}
@@ -4221,6 +4229,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->ignoreExpiredDataSaved = false;
SExprSupp* pSup = &pOperator->exprSupp;
+ pSup->hasWindowOrGroup = true;
+
initBasicInfo(&pInfo->binfo, pResBlock);
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index 8b151edb27..bad15d1834 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -1213,6 +1213,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
initBasicInfo(&pInfo->binfo, pResBlock);
SExprSupp* pSup = &pOperator->exprSupp;
+ pSup->hasWindowOrGroup = true;
+
pInfo->primaryTsIndex = ((SColumnNode*)pPhyNode->window.pTspk)->slotId;
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
@@ -1461,6 +1463,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi
goto _error;
}
+ pOperator->exprSupp.hasWindowOrGroup = true;
int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
SColumnNode* pColNode = (SColumnNode*)(pStateNode->pStateKey);
@@ -1558,6 +1561,8 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW
goto _error;
}
+ pOperator->exprSupp.hasWindowOrGroup = true;
+
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
initResultSizeInfo(&pOperator->resultInfo, 4096);
@@ -1845,6 +1850,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream,
SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo;
SExprSupp* pSup = &pOperator->exprSupp;
+ pSup->hasWindowOrGroup = true;
int32_t code = filterInitFromNode((SNode*)pNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
if (code != TSDB_CODE_SUCCESS) {
@@ -2147,6 +2153,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge
pIntervalInfo->binfo.outputTsOrder = pIntervalPhyNode->window.node.outputTsOrder;
SExprSupp* pExprSupp = &pOperator->exprSupp;
+ pExprSupp->hasWindowOrGroup = true;
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
initResultSizeInfo(&pOperator->resultInfo, 4096);
diff --git a/source/libs/function/inc/builtins.h b/source/libs/function/inc/builtins.h
index 8c07a9d530..343f5b8367 100644
--- a/source/libs/function/inc/builtins.h
+++ b/source/libs/function/inc/builtins.h
@@ -50,6 +50,7 @@ typedef struct SBuiltinFuncDefinition {
const char* pStateFunc;
FCreateMergeFuncParameters createMergeParaFuc;
FEstimateReturnRows estimateReturnRowsFunc;
+ processFuncByRow processFuncByRow;
} SBuiltinFuncDefinition;
extern const SBuiltinFuncDefinition funcMgtBuiltins[];
diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h
index ecc64fcd00..7615584f8c 100644
--- a/source/libs/function/inc/builtinsimpl.h
+++ b/source/libs/function/inc/builtinsimpl.h
@@ -133,6 +133,7 @@ int32_t getApercentileMaxSize();
bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo);
int32_t diffFunction(SqlFunctionCtx* pCtx);
+int32_t diffFunctionByRow(SArray* pCtx);
bool getDerivativeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool derivativeFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo);
diff --git a/source/libs/function/inc/functionMgtInt.h b/source/libs/function/inc/functionMgtInt.h
index a3f97af5d9..2c5c7725d5 100644
--- a/source/libs/function/inc/functionMgtInt.h
+++ b/source/libs/function/inc/functionMgtInt.h
@@ -57,6 +57,7 @@ extern "C" {
#define FUNC_MGT_PRIMARY_KEY_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(28)
#define FUNC_MGT_TSMA_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(29)
#define FUNC_MGT_COUNT_LIKE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(30) // funcs that should also return 0 when no rows found
+#define FUNC_MGT_PROCESS_BY_ROW FUNC_MGT_FUNC_CLASSIFICATION_MASK(31)
#define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0)
diff --git a/source/libs/function/inc/tpercentile.h b/source/libs/function/inc/tpercentile.h
index 5351594b2f..34815a34ad 100644
--- a/source/libs/function/inc/tpercentile.h
+++ b/source/libs/function/inc/tpercentile.h
@@ -67,7 +67,7 @@ typedef struct tMemBucket {
SHashObj *groupPagesMap; // disk page map for different groups;
} tMemBucket;
-tMemBucket *tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval);
+tMemBucket *tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval, bool hasWindowOrGroup);
void tMemBucketDestroy(tMemBucket *pBucket);
diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index 26bdf7e51e..234dfb4bd9 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -1965,9 +1965,9 @@ static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
}
SValueNode* pValue = (SValueNode*)pParamNode1;
- if (pValue->datum.i != 0 && pValue->datum.i != 1) {
+ if (pValue->datum.i < 0 || pValue->datum.i > 3) {
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
- "Second parameter of DIFF function should be only 0 or 1");
+ "Second parameter of DIFF function should be a number between 0 and 3.");
}
pValue->notReserved = true;
@@ -1977,7 +1977,7 @@ static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (IS_SIGNED_NUMERIC_TYPE(colType) || IS_TIMESTAMP_TYPE(colType) || TSDB_DATA_TYPE_BOOL == colType) {
resType = TSDB_DATA_TYPE_BIGINT;
} else if (IS_UNSIGNED_NUMERIC_TYPE(colType)) {
- resType = TSDB_DATA_TYPE_UBIGINT;
+ resType = TSDB_DATA_TYPE_BIGINT;
} else {
resType = TSDB_DATA_TYPE_DOUBLE;
}
@@ -1989,7 +1989,7 @@ static EFuncReturnRows diffEstReturnRows(SFunctionNode* pFunc) {
if (1 == LIST_LENGTH(pFunc->pParameterList)) {
return FUNC_RETURN_ROWS_N_MINUS_1;
}
- return 1 == ((SValueNode*)nodesListGetNode(pFunc->pParameterList, 1))->datum.i ? FUNC_RETURN_ROWS_INDEFINITE
+ return 1 < ((SValueNode*)nodesListGetNode(pFunc->pParameterList, 1))->datum.i ? FUNC_RETURN_ROWS_INDEFINITE
: FUNC_RETURN_ROWS_N_MINUS_1;
}
@@ -3206,7 +3206,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "diff",
.type = FUNCTION_TYPE_DIFF,
- .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
+ .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_PROCESS_BY_ROW |
FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_CUMULATIVE_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
.translateFunc = translateDiff,
.getEnvFunc = getDiffFuncEnv,
@@ -3215,6 +3215,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.sprocessFunc = diffScalarFunction,
.finalizeFunc = functionFinalize,
.estimateReturnRowsFunc = diffEstReturnRows,
+ .processFuncByRow = diffFunctionByRow,
},
{
.name = "statecount",
diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index 1aa92479b9..d5117e0eec 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -110,10 +110,9 @@ typedef enum {
} EAPerctAlgoType;
typedef struct SDiffInfo {
- bool hasPrev;
- bool includeNull;
- bool ignoreNegative; // replace the ignore with case when
- bool firstOutput;
+ bool hasPrev;
+ bool isFirstRow;
+ int8_t ignoreOption; // replace the ignore with case when
union {
int64_t i64;
double d64;
@@ -122,6 +121,12 @@ typedef struct SDiffInfo {
int64_t prevTs;
} SDiffInfo;
+bool ignoreNegative(int8_t ignoreOption){
+ return (ignoreOption & 0x1) == 0x1;
+}
+bool ignoreNull(int8_t ignoreOption){
+ return (ignoreOption & 0x2) == 0x2;
+}
typedef struct SSpreadInfo {
double result;
bool hasResult;
@@ -1897,7 +1902,7 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) {
pResInfo->complete = true;
return TSDB_CODE_SUCCESS;
} else {
- pInfo->pMemBucket = tMemBucketCreate(pCol->info.bytes, type, pInfo->minval, pInfo->maxval);
+ pInfo->pMemBucket = tMemBucketCreate(pCol->info.bytes, type, pInfo->minval, pInfo->maxval, pCtx->hasWindowOrGroup);
}
}
@@ -3100,15 +3105,14 @@ bool diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) {
SDiffInfo* pDiffInfo = GET_ROWCELL_INTERBUF(pResInfo);
pDiffInfo->hasPrev = false;
+ pDiffInfo->isFirstRow = true;
pDiffInfo->prev.i64 = 0;
pDiffInfo->prevTs = -1;
if (pCtx->numOfParams > 1) {
- pDiffInfo->ignoreNegative = pCtx->param[1].param.i; // TODO set correct param
+ pDiffInfo->ignoreOption = pCtx->param[1].param.i; // TODO set correct param
} else {
- pDiffInfo->ignoreNegative = false;
+ pDiffInfo->ignoreOption = 0;
}
- pDiffInfo->includeNull = true;
- pDiffInfo->firstOutput = false;
return true;
}
@@ -3144,91 +3148,153 @@ static int32_t doSetPrevVal(SDiffInfo* pDiffInfo, int32_t type, const char* pv,
return TSDB_CODE_FUNC_FUNTION_PARA_TYPE;
}
pDiffInfo->prevTs = ts;
-
+ pDiffInfo->hasPrev = true;
return TSDB_CODE_SUCCESS;
}
+static int32_t diffIsNegtive(SDiffInfo* pDiffInfo, int32_t type, const char* pv) {
+ switch (type) {
+ case TSDB_DATA_TYPE_UINT: {
+ int64_t v = *(uint32_t*)pv;
+ return v < pDiffInfo->prev.i64;
+ }
+ case TSDB_DATA_TYPE_INT: {
+ int64_t v = *(int32_t*)pv;
+ return v < pDiffInfo->prev.i64;
+ }
+ case TSDB_DATA_TYPE_BOOL: {
+ int64_t v = *(bool*)pv;
+ return v < pDiffInfo->prev.i64;
+ }
+ case TSDB_DATA_TYPE_UTINYINT: {
+ int64_t v = *(uint8_t*)pv;
+ return v < pDiffInfo->prev.i64;
+ }
+ case TSDB_DATA_TYPE_TINYINT: {
+ int64_t v = *(int8_t*)pv;
+ return v < pDiffInfo->prev.i64;
+ }
+ case TSDB_DATA_TYPE_USMALLINT: {
+ int64_t v = *(uint16_t*)pv;
+ return v < pDiffInfo->prev.i64;
+ }
+ case TSDB_DATA_TYPE_SMALLINT: {
+ int64_t v = *(int16_t*)pv;
+ return v < pDiffInfo->prev.i64;
+ }
+ case TSDB_DATA_TYPE_UBIGINT:{
+ uint64_t v = *(uint64_t*)pv;
+ return v < (uint64_t)pDiffInfo->prev.i64;
+ }
+ case TSDB_DATA_TYPE_TIMESTAMP:
+ case TSDB_DATA_TYPE_BIGINT: {
+ int64_t v = *(int64_t*)pv;
+ return v < pDiffInfo->prev.i64;
+ }
+ case TSDB_DATA_TYPE_FLOAT: {
+ float v = *(float*)pv;
+ return v < pDiffInfo->prev.d64;
+ }
+ case TSDB_DATA_TYPE_DOUBLE: {
+ double v = *(double*)pv;
+ return v < pDiffInfo->prev.d64;
+ }
+ default:
+ return false;
+ }
+
+ return false;
+}
+
+static void tryToSetInt64(SDiffInfo* pDiffInfo, int32_t type, SColumnInfoData* pOutput, int64_t v, int32_t pos) {
+ bool isNegative = v < pDiffInfo->prev.i64;
+ if(type == TSDB_DATA_TYPE_UBIGINT){
+ isNegative = (uint64_t)v < (uint64_t)pDiffInfo->prev.i64;
+ }
+ int64_t delta = v - pDiffInfo->prev.i64;
+ if (isNegative && ignoreNegative(pDiffInfo->ignoreOption)) {
+ colDataSetNull_f_s(pOutput, pos);
+ pOutput->hasNull = true;
+ } else {
+ colDataSetInt64(pOutput, pos, &delta);
+ }
+ pDiffInfo->prev.i64 = v;
+}
+
+static void tryToSetDouble(SDiffInfo* pDiffInfo, SColumnInfoData* pOutput, double v, int32_t pos) {
+ double delta = v - pDiffInfo->prev.d64;
+ if (delta < 0 && ignoreNegative(pDiffInfo->ignoreOption)) {
+ colDataSetNull_f_s(pOutput, pos);
+ } else {
+ colDataSetDouble(pOutput, pos, &delta);
+ }
+ pDiffInfo->prev.d64 = v;
+}
+
static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, SColumnInfoData* pOutput, int32_t pos,
int64_t ts) {
+ if (!pDiffInfo->hasPrev) {
+ colDataSetNull_f_s(pOutput, pos);
+ return doSetPrevVal(pDiffInfo, type, pv, ts);
+ }
pDiffInfo->prevTs = ts;
switch (type) {
- case TSDB_DATA_TYPE_UINT:
+ case TSDB_DATA_TYPE_UINT: {
+ int64_t v = *(uint32_t*)pv;
+ tryToSetInt64(pDiffInfo, type, pOutput, v, pos);
+ break;
+ }
case TSDB_DATA_TYPE_INT: {
- int32_t v = *(int32_t*)pv;
- int64_t delta = v - pDiffInfo->prev.i64; // direct previous may be null
- if (delta < 0 && pDiffInfo->ignoreNegative) {
- colDataSetNull_f_s(pOutput, pos);
- } else {
- colDataSetInt64(pOutput, pos, &delta);
- }
- pDiffInfo->prev.i64 = v;
-
+ int64_t v = *(int32_t*)pv;
+ tryToSetInt64(pDiffInfo, type, pOutput, v, pos);
+ break;
+ }
+ case TSDB_DATA_TYPE_BOOL: {
+ int64_t v = *(bool*)pv;
+ tryToSetInt64(pDiffInfo, type, pOutput, v, pos);
+ break;
+ }
+ case TSDB_DATA_TYPE_UTINYINT: {
+ int64_t v = *(uint8_t*)pv;
+ tryToSetInt64(pDiffInfo, type, pOutput, v, pos);
break;
}
- case TSDB_DATA_TYPE_BOOL:
- case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_TINYINT: {
- int8_t v = *(int8_t*)pv;
- int64_t delta = v - pDiffInfo->prev.i64; // direct previous may be null
- if (delta < 0 && pDiffInfo->ignoreNegative) {
- colDataSetNull_f_s(pOutput, pos);
- } else {
- colDataSetInt64(pOutput, pos, &delta);
- }
- pDiffInfo->prev.i64 = v;
+ int64_t v = *(int8_t*)pv;
+ tryToSetInt64(pDiffInfo, type, pOutput, v, pos);
+ break;
+ }
+ case TSDB_DATA_TYPE_USMALLINT:{
+ int64_t v = *(uint16_t*)pv;
+ tryToSetInt64(pDiffInfo, type, pOutput, v, pos);
break;
}
- case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_SMALLINT: {
- int16_t v = *(int16_t*)pv;
- int64_t delta = v - pDiffInfo->prev.i64; // direct previous may be null
- if (delta < 0 && pDiffInfo->ignoreNegative) {
- colDataSetNull_f_s(pOutput, pos);
- } else {
- colDataSetInt64(pOutput, pos, &delta);
- }
- pDiffInfo->prev.i64 = v;
+ int64_t v = *(int16_t*)pv;
+ tryToSetInt64(pDiffInfo, type, pOutput, v, pos);
break;
}
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_UBIGINT:
case TSDB_DATA_TYPE_BIGINT: {
int64_t v = *(int64_t*)pv;
- int64_t delta = v - pDiffInfo->prev.i64; // direct previous may be null
- if (delta < 0 && pDiffInfo->ignoreNegative) {
- colDataSetNull_f_s(pOutput, pos);
- } else {
- colDataSetInt64(pOutput, pos, &delta);
- }
- pDiffInfo->prev.i64 = v;
+ tryToSetInt64(pDiffInfo, type, pOutput, v, pos);
break;
}
case TSDB_DATA_TYPE_FLOAT: {
- float v = *(float*)pv;
- double delta = v - pDiffInfo->prev.d64; // direct previous may be null
- if ((delta < 0 && pDiffInfo->ignoreNegative) || isinf(delta) || isnan(delta)) { // check for overflow
- colDataSetNull_f_s(pOutput, pos);
- } else {
- colDataSetDouble(pOutput, pos, &delta);
- }
- pDiffInfo->prev.d64 = v;
+ double v = *(float*)pv;
+ tryToSetDouble(pDiffInfo, pOutput, v, pos);
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
double v = *(double*)pv;
- double delta = v - pDiffInfo->prev.d64; // direct previous may be null
- if ((delta < 0 && pDiffInfo->ignoreNegative) || isinf(delta) || isnan(delta)) { // check for overflow
- colDataSetNull_f_s(pOutput, pos);
- } else {
- colDataSetDouble(pOutput, pos, &delta);
- }
- pDiffInfo->prev.d64 = v;
+ tryToSetDouble(pDiffInfo, pOutput, v, pos);
break;
}
default:
return TSDB_CODE_FUNC_FUNTION_PARA_TYPE;
}
-
+ pDiffInfo->hasPrev = true;
return TSDB_CODE_SUCCESS;
}
@@ -3271,71 +3337,155 @@ bool funcInputGetNextRowIndex(SInputColumnInfoData* pInput, int32_t from, bool f
}
}
-int32_t diffFunction(SqlFunctionCtx* pCtx) {
+int32_t diffResultIsNull(SqlFunctionCtx* pCtx, SFuncInputRow* pRow){
+ SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
+ SDiffInfo* pDiffInfo = GET_ROWCELL_INTERBUF(pResInfo);
+
+ if (pRow->isDataNull || !pDiffInfo->hasPrev ) {
+ return true;
+ } else if (ignoreNegative(pDiffInfo->ignoreOption)){
+ return diffIsNegtive(pDiffInfo, pCtx->input.pData[0]->info.type, pRow->pData);
+ }
+ return false;
+}
+
+bool isFirstRow(SqlFunctionCtx* pCtx, SFuncInputRow* pRow) {
+ SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
+ SDiffInfo* pDiffInfo = GET_ROWCELL_INTERBUF(pResInfo);
+ return pDiffInfo->isFirstRow;
+}
+
+int32_t trySetPreVal(SqlFunctionCtx* pCtx, SFuncInputRow* pRow) {
+ SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
+ SDiffInfo* pDiffInfo = GET_ROWCELL_INTERBUF(pResInfo);
+ pDiffInfo->isFirstRow = false;
+ if (pRow->isDataNull) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ SInputColumnInfoData* pInput = &pCtx->input;
+ SColumnInfoData* pInputCol = pInput->pData[0];
+ int8_t inputType = pInputCol->info.type;
+
+ char* pv = pRow->pData;
+ return doSetPrevVal(pDiffInfo, inputType, pv, pRow->ts);
+}
+
+int32_t setDoDiffResult(SqlFunctionCtx* pCtx, SFuncInputRow* pRow, int32_t pos) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SDiffInfo* pDiffInfo = GET_ROWCELL_INTERBUF(pResInfo);
SInputColumnInfoData* pInput = &pCtx->input;
- SColumnInfoData* pInputCol = pInput->pData[0];
- int8_t inputType = pInputCol->info.type;
+ SColumnInfoData* pInputCol = pInput->pData[0];
+ int8_t inputType = pInputCol->info.type;
+ SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput;
- TSKEY* tsList = (int64_t*)pInput->pPTS->pData;
+ if (pRow->isDataNull) {
+ colDataSetNull_f_s(pOutput, pos);
+ pOutput->hasNull = true;
- int32_t numOfElems = 0;
- int32_t startOffset = pCtx->offset;
-
- SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput;
-
- funcInputUpdate(pCtx);
-
- SFuncInputRow row = {0};
- while (funcInputGetNextRow(pCtx, &row)) {
- int32_t pos = startOffset + numOfElems;
-
- if (row.isDataNull) {
- if (pDiffInfo->includeNull) {
- colDataSetNull_f_s(pOutput, pos);
-
- // handle selectivity
- if (pCtx->subsidiaries.num > 0) {
- appendSelectivityCols(pCtx, row.block, row.rowIndex, pos);
- }
-
- numOfElems += 1;
- }
- continue;
+ // handle selectivity
+ if (pCtx->subsidiaries.num > 0) {
+ appendSelectivityCols(pCtx, pRow->block, pRow->rowIndex, pos);
}
-
- char* pv = row.pData;
-
- if (pDiffInfo->hasPrev) {
- if (row.ts == pDiffInfo->prevTs) {
- return TSDB_CODE_FUNC_DUP_TIMESTAMP;
- }
- int32_t code = doHandleDiff(pDiffInfo, inputType, pv, pOutput, pos, row.ts);
- if (code != TSDB_CODE_SUCCESS) {
- return code;
- }
- // handle selectivity
- if (pCtx->subsidiaries.num > 0) {
- appendSelectivityCols(pCtx, row.block, row.rowIndex, pos);
- }
-
- numOfElems++;
- } else {
- int32_t code = doSetPrevVal(pDiffInfo, inputType, pv, row.ts);
- if (code != TSDB_CODE_SUCCESS) {
- return code;
- }
- }
-
- pDiffInfo->hasPrev = true;
+ return TSDB_CODE_SUCCESS;
+ }
+
+ char* pv = pRow->pData;
+
+ if (pRow->ts == pDiffInfo->prevTs) {
+ return TSDB_CODE_FUNC_DUP_TIMESTAMP;
+ }
+ int32_t code = doHandleDiff(pDiffInfo, inputType, pv, pOutput, pos, pRow->ts);
+ if (code != TSDB_CODE_SUCCESS) {
+ return code;
+ }
+ // handle selectivity
+ if (pCtx->subsidiaries.num > 0) {
+ appendSelectivityCols(pCtx, pRow->block, pRow->rowIndex, pos);
}
- pResInfo->numOfRes = numOfElems;
return TSDB_CODE_SUCCESS;
}
+int32_t diffFunction(SqlFunctionCtx* pCtx) {
+ return TSDB_CODE_SUCCESS;
+}
+
+int32_t diffFunctionByRow(SArray* pCtxArray) {
+ int32_t code = TSDB_CODE_SUCCESS;
+ int diffColNum = pCtxArray->size;
+ if(diffColNum == 0) {
+ return TSDB_CODE_SUCCESS;
+ }
+ int32_t numOfElems = 0;
+
+ SArray* pRows = taosArrayInit_s(sizeof(SFuncInputRow), diffColNum);
+
+ bool keepNull = false;
+ for (int i = 0; i < diffColNum; ++i) {
+ SqlFunctionCtx* pCtx = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, i);
+ funcInputUpdate(pCtx);
+ SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
+ SDiffInfo* pDiffInfo = GET_ROWCELL_INTERBUF(pResInfo);
+ if (!ignoreNull(pDiffInfo->ignoreOption)) {
+ keepNull = true;
+ }
+ }
+
+ SqlFunctionCtx* pCtx0 = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, 0);
+ SFuncInputRow* pRow0 = (SFuncInputRow*)taosArrayGet(pRows, 0);
+ int32_t startOffset = pCtx0->offset;
+ while (funcInputGetNextRow(pCtx0, pRow0)) {
+ bool hasNotNullValue = !diffResultIsNull(pCtx0, pRow0);
+ for (int i = 1; i < diffColNum; ++i) {
+ SqlFunctionCtx* pCtx = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, i);
+ SFuncInputRow* pRow = (SFuncInputRow*)taosArrayGet(pRows, i);
+ if(!funcInputGetNextRow(pCtx, pRow)) {
+ // rows are not equal
+ code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
+ goto _exit;
+ }
+ if (!diffResultIsNull(pCtx, pRow)) {
+ hasNotNullValue = true;
+ }
+ }
+ int32_t pos = startOffset + numOfElems;
+
+ bool newRow = false;
+ for (int i = 0; i < diffColNum; ++i) {
+ SqlFunctionCtx* pCtx = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, i);
+ SFuncInputRow* pRow = (SFuncInputRow*)taosArrayGet(pRows, i);
+ if ((keepNull || hasNotNullValue) && !isFirstRow(pCtx, pRow)){
+ code = setDoDiffResult(pCtx, pRow, pos);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _exit;
+ }
+ newRow = true;
+ } else {
+ code = trySetPreVal(pCtx, pRow);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _exit;
+ }
+ }
+ }
+ if (newRow) ++numOfElems;
+ }
+
+ for (int i = 0; i < diffColNum; ++i) {
+ SqlFunctionCtx* pCtx = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, i);
+ SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
+ pResInfo->numOfRes = numOfElems;
+ }
+
+_exit:
+ if (pRows) {
+ taosArrayDestroy(pRows);
+ pRows = NULL;
+ }
+ return code;
+}
+
int32_t getTopBotInfoSize(int64_t numOfItems) { return sizeof(STopBotRes) + numOfItems * sizeof(STopBotResItem); }
bool getTopBotFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c
index 1e12595b28..b99e67697c 100644
--- a/source/libs/function/src/functionMgt.c
+++ b/source/libs/function/src/functionMgt.c
@@ -141,6 +141,7 @@ int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet) {
pFpSet->process = funcMgtBuiltins[funcId].processFunc;
pFpSet->finalize = funcMgtBuiltins[funcId].finalizeFunc;
pFpSet->combine = funcMgtBuiltins[funcId].combineFunc;
+ pFpSet->processFuncByRow = funcMgtBuiltins[funcId].processFuncByRow;
return TSDB_CODE_SUCCESS;
}
@@ -274,6 +275,8 @@ bool fmIsBlockDistFunc(int32_t funcId) {
return FUNCTION_TYPE_BLOCK_DIST == funcMgtBuiltins[funcId].type;
}
+bool fmIsProcessByRowFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_PROCESS_BY_ROW); }
+
bool fmIsIgnoreNullFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_IGNORE_NULL_FUNC); }
void fmFuncMgtDestroy() {
diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c
index 776a7fb95a..a068186992 100644
--- a/source/libs/function/src/tpercentile.c
+++ b/source/libs/function/src/tpercentile.c
@@ -238,14 +238,20 @@ static void resetSlotInfo(tMemBucket *pBucket) {
}
}
-tMemBucket *tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval) {
+tMemBucket *tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval, bool hasWindowOrGroup) {
tMemBucket *pBucket = (tMemBucket *)taosMemoryCalloc(1, sizeof(tMemBucket));
if (pBucket == NULL) {
return NULL;
}
- pBucket->numOfSlots = DEFAULT_NUM_OF_SLOT;
- pBucket->bufPageSize = 16384 * 4; // 16k per page
+ if (hasWindowOrGroup) {
+ // With window or group by, we need to shrink page size and reduce page num to save memory.
+ pBucket->numOfSlots = DEFAULT_NUM_OF_SLOT / 8 ; // 128 bucket
+ pBucket->bufPageSize = 4096; // 4k per page
+ } else {
+ pBucket->numOfSlots = DEFAULT_NUM_OF_SLOT;
+ pBucket->bufPageSize = 16384 * 4; // 16k per page
+ }
pBucket->type = dataType;
pBucket->bytes = nElemSize;
diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c
index 21c196872c..3389780916 100644
--- a/source/libs/monitor/src/monMain.c
+++ b/source/libs/monitor/src/monMain.c
@@ -568,6 +568,25 @@ void monSendReport(SMonInfo *pMonitor){
}
}
+void monSendReportBasic(SMonInfo *pMonitor) {
+ char *pCont = tjsonToString(pMonitor->pJson);
+ if (tsMonitorLogProtocol) {
+ if (pCont != NULL) {
+ uInfoL("report cont basic:\n%s", pCont);
+ } else {
+ uInfo("report cont basic is null");
+ }
+ }
+ if (pCont != NULL) {
+ EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT;
+ if (taosSendHttpReport(tsMonitor.cfg.server, tsMonFwBasicUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) !=
+ 0) {
+ uError("failed to send monitor msg");
+ }
+ taosMemoryFree(pCont);
+ }
+}
+
void monGenAndSendReport() {
SMonInfo *pMonitor = monCreateMonitorInfo();
if (pMonitor == NULL) return;
@@ -595,38 +614,11 @@ void monGenAndSendReport() {
monGenVnodeRoleTable(pMonitor);
monSendPromReport();
- }
-
- monCleanupMonitorInfo(pMonitor);
-}
-
-void monSendReportBasic(SMonInfo *pMonitor){
- char *pCont = tjsonToString(pMonitor->pJson);
- if(tsMonitorLogProtocol){
- if(pCont != NULL){
- uInfoL("report cont basic:\n%s", pCont);
+ if (pMonitor->mmInfo.cluster.first_ep_dnode_id != 0) {
+ monGenBasicJsonBasic(pMonitor);
+ monGenClusterJsonBasic(pMonitor);
+ monSendReportBasic(pMonitor);
}
- else{
- uInfo("report cont basic is null");
- }
- }
- if (pCont != NULL) {
- EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT;
- if (taosSendHttpReport(tsMonitor.cfg.server, tsMonFwBasicUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) {
- uError("failed to send monitor msg");
- }
- taosMemoryFree(pCont);
- }
-}
-
-void monGenAndSendReportBasic() {
- SMonInfo *pMonitor = monCreateMonitorInfo();
-
- monGenBasicJsonBasic(pMonitor);
- monGenClusterJsonBasic(pMonitor);
-
- if (pMonitor->mmInfo.cluster.first_ep_dnode_id != 0) {
- monSendReportBasic(pMonitor);
}
monCleanupMonitorInfo(pMonitor);
diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c
index 0289469221..313d9449d2 100644
--- a/source/libs/parser/src/parInsertSql.c
+++ b/source/libs/parser/src/parInsertSql.c
@@ -30,6 +30,7 @@ typedef struct SInsertParseContext {
bool forceUpdate;
bool needTableTagVal;
bool needRequest; // whether or not request server
+ bool isStmtBind; // whether is stmt bind
} SInsertParseContext;
typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param);
@@ -1978,7 +1979,6 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt
static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataCxt* pTableCxt, bool* pGotRow,
SToken* pToken) {
SBoundColInfo* pCols = &pTableCxt->boundColsInfo;
- bool isParseBindParam = false;
SSchema* pSchemas = getTableColumnSchema(pTableCxt->pMeta);
int32_t code = TSDB_CODE_SUCCESS;
@@ -1996,7 +1996,7 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataC
SColVal* pVal = taosArrayGet(pTableCxt->pValues, pCols->pColIndex[i]);
if (pToken->type == TK_NK_QUESTION) {
- isParseBindParam = true;
+ pCxt->isStmtBind = true;
if (NULL == pCxt->pComCxt->pStmtCb) {
code = buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", pToken->z);
break;
@@ -2007,8 +2007,8 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataC
break;
}
- if (isParseBindParam) {
- code = buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and values");
+ if (pCxt->isStmtBind) {
+ code = buildInvalidOperationMsg(&pCxt->msg, "stmt bind param does not support normal value in sql");
break;
}
@@ -2025,7 +2025,7 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataC
}
}
- if (TSDB_CODE_SUCCESS == code && !isParseBindParam) {
+ if (TSDB_CODE_SUCCESS == code && !pCxt->isStmtBind) {
SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1);
code = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow);
if (TSDB_CODE_SUCCESS == code) {
@@ -2035,7 +2035,7 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataC
}
}
- if (TSDB_CODE_SUCCESS == code && !isParseBindParam) {
+ if (TSDB_CODE_SUCCESS == code && !pCxt->isStmtBind) {
*pGotRow = true;
}
@@ -2410,6 +2410,7 @@ static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModif
}
if (TK_NK_QUESTION == pTbName->type) {
+ pCxt->isStmtBind = true;
if (NULL == pCxt->pComCxt->pStmtCb) {
return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", pTbName->z);
}
@@ -2443,6 +2444,13 @@ static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModif
pTbName->n = strlen(tbName);
}
+ if (pCxt->isStmtBind) {
+ if (TK_NK_ID == pTbName->type || (tbNameAfterDbName != NULL && *(tbNameAfterDbName + 1) != '?')) {
+ // In SQL statements, the table name has already been specified.
+ parserWarn("0x%" PRIx64 " table name is specified in sql, ignore the table name in bind param", pCxt->pComCxt->requestId);
+ }
+ }
+
*pHasData = true;
return TSDB_CODE_SUCCESS;
}
@@ -2936,7 +2944,8 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal
.missCache = false,
.usingDuplicateTable = false,
.needRequest = true,
- .forceUpdate = (NULL != pCatalogReq ? pCatalogReq->forceUpdate : false)};
+ .forceUpdate = (NULL != pCatalogReq ? pCatalogReq->forceUpdate : false),
+ .isStmtBind = pCxt->isStmtBind};
int32_t code = initInsertQuery(&context, pCatalogReq, pMetaData, pQuery);
if (TSDB_CODE_SUCCESS == code) {
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index f6072be962..4628e2be60 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -883,6 +883,8 @@ static uint8_t getPrecisionFromCurrStmt(SNode* pCurrStmt, uint8_t defaultVal) {
if (isDeleteStmt(pCurrStmt)) {
return ((SDeleteStmt*)pCurrStmt)->precision;
}
+ if (pCurrStmt && nodeType(pCurrStmt) == QUERY_NODE_CREATE_TSMA_STMT)
+ return ((SCreateTSMAStmt*)pCurrStmt)->precision;
return defaultVal;
}
@@ -2194,11 +2196,17 @@ static int32_t translateIndefiniteRowsFunc(STranslateContext* pCxt, SFunctionNod
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC);
}
SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt;
- if (pSelect->hasAggFuncs || pSelect->hasMultiRowsFunc ||
- (pSelect->hasIndefiniteRowsFunc &&
- (FUNC_RETURN_ROWS_INDEFINITE == pSelect->returnRows || pSelect->returnRows != fmGetFuncReturnRows(pFunc)))) {
+ if (pSelect->hasAggFuncs || pSelect->hasMultiRowsFunc) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC);
}
+ if (pSelect->hasIndefiniteRowsFunc &&
+ (FUNC_RETURN_ROWS_INDEFINITE == pSelect->returnRows || pSelect->returnRows != fmGetFuncReturnRows(pFunc)) &&
+ (pSelect->lastProcessByRowFuncId == -1 || !fmIsProcessByRowFunc(pFunc->funcId))) {
+ return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC);
+ }
+ if (pSelect->lastProcessByRowFuncId != -1 && pSelect->lastProcessByRowFuncId != pFunc->funcId) {
+ return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_DIFFERENT_BY_ROW_FUNC);
+ }
if (NULL != pSelect->pWindow || NULL != pSelect->pGroupByList) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC,
"%s function is not supported in window query or group query", pFunc->functionName);
@@ -2462,6 +2470,9 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) {
} else if (fmIsInterpFunc(pFunc->funcId)) {
pSelect->returnRows = fmGetFuncReturnRows(pFunc);
}
+ if (fmIsProcessByRowFunc(pFunc->funcId)) {
+ pSelect->lastProcessByRowFuncId = pFunc->funcId;
+ }
pSelect->hasMultiRowsFunc = pSelect->hasMultiRowsFunc ? true : fmIsMultiRowsFunc(pFunc->funcId);
if (fmIsSelectFunc(pFunc->funcId)) {
@@ -3398,6 +3409,7 @@ static int32_t checkIsEmptyResult(STranslateContext* pCxt, SSelectStmt* pSelect)
static int32_t resetSelectFuncNumWithoutDup(SSelectStmt* pSelect) {
if (pSelect->selectFuncNum <= 1) return TSDB_CODE_SUCCESS;
pSelect->selectFuncNum = 0;
+ pSelect->lastProcessByRowFuncId = -1;
SNodeList* pNodeList = nodesMakeList();
int32_t code = nodesCollectSelectFuncs(pSelect, SQL_CLAUSE_FROM, NULL, fmIsSelectFunc, pNodeList);
if (TSDB_CODE_SUCCESS != code) {
@@ -11132,22 +11144,41 @@ static int32_t rewriteTSMAFuncs(STranslateContext* pCxt, SCreateTSMAStmt* pStmt,
static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStmt, SMCreateSmaReq* pReq,
SName* useTbName) {
- SName name;
+ SName name;
+ SDbCfgInfo pDbInfo = {0};
+ int32_t code = TSDB_CODE_SUCCESS;
tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tsmaName, &name), pReq->name);
memset(&name, 0, sizeof(SName));
toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, useTbName);
tNameExtractFullName(useTbName, pReq->stb);
pReq->igExists = pStmt->ignoreExists;
+
+ code = getDBCfg(pCxt, pStmt->dbName, &pDbInfo);
+ if (code != TSDB_CODE_SUCCESS) {
+ return code;
+ }
+ pStmt->precision = pDbInfo.precision;
+ code = translateValue(pCxt, (SValueNode*)pStmt->pOptions->pInterval);
+ if (code == DEAL_RES_ERROR) {
+ return code;
+ }
pReq->interval = ((SValueNode*)pStmt->pOptions->pInterval)->datum.i;
- pReq->intervalUnit = TIME_UNIT_MILLISECOND;
+ pReq->intervalUnit = ((SValueNode*)pStmt->pOptions->pInterval)->unit;
#define TSMA_MIN_INTERVAL_MS 1000 * 60 // 1m
-#define TSMA_MAX_INTERVAL_MS (60 * 60 * 1000) // 1h
- if (pReq->interval > TSMA_MAX_INTERVAL_MS || pReq->interval < TSMA_MIN_INTERVAL_MS) {
- return TSDB_CODE_TSMA_INVALID_INTERVAL;
- }
+#define TSMA_MAX_INTERVAL_MS (60UL * 60UL * 1000UL * 24UL * 365UL) // 1y
- int32_t code = TSDB_CODE_SUCCESS;
+ if (!IS_CALENDAR_TIME_DURATION(pReq->intervalUnit)) {
+ int64_t factor = TSDB_TICK_PER_SECOND(pDbInfo.precision) / TSDB_TICK_PER_SECOND(TSDB_TIME_PRECISION_MILLI);
+ if (pReq->interval > TSMA_MAX_INTERVAL_MS * factor || pReq->interval < TSMA_MIN_INTERVAL_MS * factor) {
+ return TSDB_CODE_TSMA_INVALID_INTERVAL;
+ }
+ } else {
+ if (pReq->intervalUnit == TIME_UNIT_MONTH && (pReq->interval < 1 || pReq->interval > 12))
+ return TSDB_CODE_TSMA_INVALID_INTERVAL;
+ if (pReq->intervalUnit == TIME_UNIT_YEAR && (pReq->interval != 1))
+ return TSDB_CODE_TSMA_INVALID_INTERVAL;
+ }
STableMeta* pTableMeta = NULL;
STableTSMAInfo* pRecursiveTsma = NULL;
@@ -11160,7 +11191,8 @@ static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStm
pReq->recursiveTsma = true;
tNameExtractFullName(useTbName, pReq->baseTsmaName);
SValueNode* pInterval = (SValueNode*)pStmt->pOptions->pInterval;
- if (pRecursiveTsma->interval < pInterval->datum.i && pInterval->datum.i % pRecursiveTsma->interval == 0) {
+ if (checkRecursiveTsmaInterval(pRecursiveTsma->interval, pRecursiveTsma->unit, pInterval->datum.i,
+ pInterval->unit, pDbInfo.precision, true)) {
} else {
code = TSDB_CODE_TSMA_INVALID_PARA;
}
@@ -11231,7 +11263,8 @@ static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStm
}
static int32_t translateCreateTSMA(STranslateContext* pCxt, SCreateTSMAStmt* pStmt) {
- int32_t code = doTranslateValue(pCxt, (SValueNode*)pStmt->pOptions->pInterval);
+ pCxt->pCurrStmt = (SNode*)pStmt;
+ int32_t code = TSDB_CODE_SUCCESS;
SName useTbName = {0};
if (code == TSDB_CODE_SUCCESS) {
diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c
index 416faafe35..d67c7d306f 100644
--- a/source/libs/parser/src/parUtil.c
+++ b/source/libs/parser/src/parUtil.c
@@ -221,6 +221,8 @@ static char* getSyntaxErrFormat(int32_t errCode) {
return "Table name:%s duplicated";
case TSDB_CODE_PAR_TAG_NAME_DUPLICATED:
return "Tag name:%s duplicated";
+ case TSDB_CODE_PAR_NOT_ALLOWED_DIFFERENT_BY_ROW_FUNC:
+ return "Some functions cannot appear in the select list at the same time";
default:
return "Unknown error";
}
@@ -772,6 +774,7 @@ SNode* createSelectStmtImpl(bool isDistinct, SNodeList* pProjectionList, SNode*
select->onlyHasKeepOrderFunc = true;
select->timeRange = TSWINDOW_INITIALIZER;
select->pHint = pHint;
+ select->lastProcessByRowFuncId = -1;
return (SNode*)select;
}
diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c
index d0763a914e..05ee27f6de 100644
--- a/source/libs/planner/src/planOptimizer.c
+++ b/source/libs/planner/src/planOptimizer.c
@@ -6018,6 +6018,7 @@ typedef struct STSMAOptUsefulTsma {
SArray* pTsmaScanCols; // SArray index of tsmaFuncs array
char targetTbName[TSDB_TABLE_NAME_LEN]; // the scanning table name, used only when pTsma is not NULL
uint64_t targetTbUid; // the scanning table uid, used only when pTsma is not NULL
+ int8_t precision;
} STSMAOptUsefulTsma;
typedef struct STSMAOptCtx {
@@ -6085,12 +6086,19 @@ static void clearTSMAOptCtx(STSMAOptCtx* pTsmaOptCtx) {
taosMemoryFreeClear(pTsmaOptCtx->queryInterval);
}
-static bool tsmaOptCheckValidInterval(int64_t tsmaInterval, int8_t tsmaIntevalUnit, const STSMAOptCtx* pTsmaOptCtx) {
+static bool tsmaOptCheckValidInterval(int64_t tsmaInterval, int8_t unit, const STSMAOptCtx* pTsmaOptCtx) {
if (!pTsmaOptCtx->queryInterval) return true;
- bool validInterval = pTsmaOptCtx->queryInterval->interval % tsmaInterval == 0;
- bool validSliding = pTsmaOptCtx->queryInterval->sliding % tsmaInterval == 0;
- bool validOffset = pTsmaOptCtx->queryInterval->offset % tsmaInterval == 0;
+ bool validInterval = checkRecursiveTsmaInterval(tsmaInterval, unit, pTsmaOptCtx->queryInterval->interval,
+ pTsmaOptCtx->queryInterval->intervalUnit,
+ pTsmaOptCtx->queryInterval->precision, false);
+ bool validSliding =
+ checkRecursiveTsmaInterval(tsmaInterval, unit, pTsmaOptCtx->queryInterval->sliding,
+ pTsmaOptCtx->queryInterval->slidingUnit, pTsmaOptCtx->queryInterval->precision, false);
+ bool validOffset =
+ pTsmaOptCtx->queryInterval->offset == 0 ||
+ checkRecursiveTsmaInterval(tsmaInterval, unit, pTsmaOptCtx->queryInterval->offset,
+ pTsmaOptCtx->queryInterval->offsetUnit, pTsmaOptCtx->queryInterval->precision, false);
return validInterval && validSliding && validOffset;
}
@@ -6171,7 +6179,8 @@ static bool tsmaOptCheckTags(STSMAOptCtx* pCtx, const STableTSMAInfo* pTsma) {
}
static int32_t tsmaOptFilterTsmas(STSMAOptCtx* pTsmaOptCtx) {
- STSMAOptUsefulTsma usefulTsma = {.pTsma = NULL, .scanRange = {.skey = TSKEY_MIN, .ekey = TSKEY_MAX}};
+ STSMAOptUsefulTsma usefulTsma = {
+ .pTsma = NULL, .scanRange = {.skey = TSKEY_MIN, .ekey = TSKEY_MAX}, .precision = pTsmaOptCtx->precision};
SArray* pTsmaScanCols = NULL;
for (int32_t i = 0; i < pTsmaOptCtx->pTsmas->size; ++i) {
@@ -6208,31 +6217,26 @@ static int32_t tsmaOptFilterTsmas(STSMAOptCtx* pTsmaOptCtx) {
}
static int32_t tsmaInfoCompWithIntervalDesc(const void* pLeft, const void* pRight) {
+ const int64_t factors[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
const STSMAOptUsefulTsma *p = pLeft, *q = pRight;
int64_t pInterval = p->pTsma->interval, qInterval = q->pTsma->interval;
- int32_t code = getDuration(pInterval, p->pTsma->unit, &pInterval, TSDB_TIME_PRECISION_MILLI);
- ASSERT(code == TSDB_CODE_SUCCESS);
- code = getDuration(qInterval, q->pTsma->unit, &qInterval, TSDB_TIME_PRECISION_MILLI);
- ASSERT(code == TSDB_CODE_SUCCESS);
+ int8_t pUnit = p->pTsma->unit, qUnit = q->pTsma->unit;
+ if (TIME_UNIT_MONTH == pUnit) {
+ pInterval = pInterval * 31 * (NANOSECOND_PER_DAY / factors[p->precision]);
+ } else if (TIME_UNIT_YEAR == pUnit){
+ pInterval = pInterval * 365 * (NANOSECOND_PER_DAY / factors[p->precision]);
+ }
+ if (TIME_UNIT_MONTH == qUnit) {
+ qInterval = qInterval * 31 * (NANOSECOND_PER_DAY / factors[q->precision]);
+ } else if (TIME_UNIT_YEAR == qUnit){
+ qInterval = qInterval * 365 * (NANOSECOND_PER_DAY / factors[q->precision]);
+ }
+
if (pInterval > qInterval) return -1;
if (pInterval < qInterval) return 1;
return 0;
}
-static const STSMAOptUsefulTsma* tsmaOptFindUsefulTsma(const SArray* pUsefulTsmas, int32_t startIdx,
- int64_t alignInterval, int64_t alignInterval2,
- int8_t precision) {
- int64_t tsmaInterval;
- for (int32_t i = startIdx; i < pUsefulTsmas->size; ++i) {
- const STSMAOptUsefulTsma* pUsefulTsma = taosArrayGet(pUsefulTsmas, i);
- getDuration(pUsefulTsma->pTsma->interval, pUsefulTsma->pTsma->unit, &tsmaInterval, precision);
- if (alignInterval % tsmaInterval == 0 && alignInterval2 % tsmaInterval == 0) {
- return pUsefulTsma;
- }
- }
- return NULL;
-}
-
static void tsmaOptInitIntervalFromTsma(SInterval* pInterval, const STableTSMAInfo* pTsma, int8_t precision) {
pInterval->interval = pTsma->interval;
pInterval->intervalUnit = pTsma->unit;
@@ -6243,14 +6247,28 @@ static void tsmaOptInitIntervalFromTsma(SInterval* pInterval, const STableTSMAIn
pInterval->precision = precision;
}
+static const STSMAOptUsefulTsma* tsmaOptFindUsefulTsma(const SArray* pUsefulTsmas, int32_t startIdx,
+ int64_t startAlignInterval, int64_t endAlignInterval,
+ int8_t precision) {
+ SInterval tsmaInterval;
+ for (int32_t i = startIdx; i < pUsefulTsmas->size; ++i) {
+ const STSMAOptUsefulTsma* pUsefulTsma = taosArrayGet(pUsefulTsmas, i);
+ tsmaOptInitIntervalFromTsma(&tsmaInterval, pUsefulTsma->pTsma, precision);
+ if (taosTimeTruncate(startAlignInterval, &tsmaInterval) == startAlignInterval &&
+ taosTimeTruncate(endAlignInterval, &tsmaInterval) == endAlignInterval) {
+ return pUsefulTsma;
+ }
+ }
+ return NULL;
+}
+
static void tsmaOptSplitWindows(STSMAOptCtx* pTsmaOptCtx, const STimeWindow* pScanRange) {
bool needTailWindow = false;
bool isSkeyAlignedWithTsma = true, isEkeyAlignedWithTsma = true;
int64_t winSkey = TSKEY_MIN, winEkey = TSKEY_MAX;
int64_t startOfSkeyFirstWin = pScanRange->skey, endOfSkeyFirstWin;
int64_t startOfEkeyFirstWin = pScanRange->ekey, endOfEkeyFirstWin;
- int64_t tsmaInterval;
- SInterval interval;
+ SInterval interval, tsmaInterval;
STimeWindow scanRange = *pScanRange;
const SInterval* pInterval = pTsmaOptCtx->queryInterval;
const STSMAOptUsefulTsma* pUsefulTsma = taosArrayGet(pTsmaOptCtx->pUsefulTsmas, 0);
@@ -6263,14 +6281,14 @@ static void tsmaOptSplitWindows(STSMAOptCtx* pTsmaOptCtx, const STimeWindow* pSc
pInterval = &interval;
}
- tsmaInterval = pTsma->interval;
+ tsmaOptInitIntervalFromTsma(&tsmaInterval, pTsma, pTsmaOptCtx->precision);
// check for head windows
if (pScanRange->skey != TSKEY_MIN) {
startOfSkeyFirstWin = taosTimeTruncate(pScanRange->skey, pInterval);
endOfSkeyFirstWin =
taosTimeAdd(startOfSkeyFirstWin, pInterval->interval, pInterval->intervalUnit, pTsmaOptCtx->precision);
- isSkeyAlignedWithTsma = ((pScanRange->skey - startOfSkeyFirstWin) % tsmaInterval == 0);
+ isSkeyAlignedWithTsma = taosTimeTruncate(pScanRange->skey, &tsmaInterval) == pScanRange->skey;
} else {
endOfSkeyFirstWin = TSKEY_MIN;
}
@@ -6280,7 +6298,7 @@ static void tsmaOptSplitWindows(STSMAOptCtx* pTsmaOptCtx, const STimeWindow* pSc
startOfEkeyFirstWin = taosTimeTruncate(pScanRange->ekey, pInterval);
endOfEkeyFirstWin =
taosTimeAdd(startOfEkeyFirstWin, pInterval->interval, pInterval->intervalUnit, pTsmaOptCtx->precision);
- isEkeyAlignedWithTsma = ((pScanRange->ekey + 1 - startOfEkeyFirstWin) % tsmaInterval == 0);
+ isEkeyAlignedWithTsma = taosTimeTruncate(pScanRange->ekey + 1, &tsmaInterval) == (pScanRange->ekey + 1);
if (startOfEkeyFirstWin > startOfSkeyFirstWin) {
needTailWindow = true;
}
@@ -6292,8 +6310,7 @@ static void tsmaOptSplitWindows(STSMAOptCtx* pTsmaOptCtx, const STimeWindow* pSc
scanRange.ekey,
taosTimeAdd(startOfSkeyFirstWin, pInterval->interval * 1, pInterval->intervalUnit, pTsmaOptCtx->precision) - 1);
const STSMAOptUsefulTsma* pTsmaFound =
- tsmaOptFindUsefulTsma(pTsmaOptCtx->pUsefulTsmas, 1, scanRange.skey - startOfSkeyFirstWin,
- (scanRange.ekey + 1 - startOfSkeyFirstWin), pTsmaOptCtx->precision);
+ tsmaOptFindUsefulTsma(pTsmaOptCtx->pUsefulTsmas, 1, scanRange.skey, scanRange.ekey + 1, pTsmaOptCtx->precision);
STSMAOptUsefulTsma usefulTsma = {.pTsma = pTsmaFound ? pTsmaFound->pTsma : NULL,
.scanRange = scanRange,
.pTsmaScanCols = pTsmaFound ? pTsmaFound->pTsmaScanCols : NULL};
diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c
index 795c21c234..fc92be8214 100644
--- a/source/libs/scheduler/src/scheduler.c
+++ b/source/libs/scheduler/src/scheduler.c
@@ -201,6 +201,7 @@ void schedulerDestroy(void) {
}
SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock);
+ taosTmrCleanUp(schMgmt.timer);
qWorkerDestroy(&schMgmt.queryMgmt);
schMgmt.queryMgmt = NULL;
}
diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp
index 6d2215264e..78e876f82c 100644
--- a/source/libs/scheduler/test/schedulerTests.cpp
+++ b/source/libs/scheduler/test/schedulerTests.cpp
@@ -36,9 +36,9 @@
#include "tdatablock.h"
#include "tdef.h"
#include "tglobal.h"
+#include "tmisce.h"
#include "trpc.h"
#include "tvariant.h"
-#include "tmisce.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
@@ -54,7 +54,8 @@
namespace {
-extern "C" int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDataBuf *pMsg, int32_t rspCode);
+extern "C" int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDataBuf *pMsg,
+ int32_t rspCode);
extern "C" int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t rspCode);
int64_t insertJobRefId = 0;
@@ -74,8 +75,9 @@ int32_t schtStartFetch = 0;
void schtInitLogFile() {
const char *defaultLogFileNamePrefix = "taoslog";
const int32_t maxLogFileNum = 10;
-
+ rpcInit();
tsAsyncLog = 0;
+ rpcInit();
qDebugFlag = 159;
strcpy(tsLogDir, TD_LOG_DIR_PATH);
@@ -84,12 +86,10 @@ void schtInitLogFile() {
}
}
-void schtQueryCb(SExecResult *pResult, void *param, int32_t code) {
- *(int32_t *)param = 1;
-}
+void schtQueryCb(SExecResult *pResult, void *param, int32_t code) { *(int32_t *)param = 1; }
-int32_t schtBuildQueryRspMsg(uint32_t *msize, void** rspMsg) {
- SQueryTableRsp rsp = {0};
+int32_t schtBuildQueryRspMsg(uint32_t *msize, void **rspMsg) {
+ SQueryTableRsp rsp = {0};
rsp.code = 0;
rsp.affectedRows = 0;
rsp.tbVerInfo = NULL;
@@ -99,7 +99,7 @@ int32_t schtBuildQueryRspMsg(uint32_t *msize, void** rspMsg) {
qError("tSerializeSQueryTableRsp failed");
return TSDB_CODE_OUT_OF_MEMORY;
}
-
+
void *pRsp = taosMemoryCalloc(msgSize, 1);
if (NULL == pRsp) {
qError("rpcMallocCont %d failed", msgSize);
@@ -117,9 +117,8 @@ int32_t schtBuildQueryRspMsg(uint32_t *msize, void** rspMsg) {
return TSDB_CODE_SUCCESS;
}
-
-int32_t schtBuildFetchRspMsg(uint32_t *msize, void** rspMsg) {
- SRetrieveTableRsp* rsp = (SRetrieveTableRsp*)taosMemoryCalloc(sizeof(SRetrieveTableRsp), 1);
+int32_t schtBuildFetchRspMsg(uint32_t *msize, void **rspMsg) {
+ SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(sizeof(SRetrieveTableRsp), 1);
rsp->completed = 1;
rsp->numOfRows = 10;
rsp->compLen = 0;
@@ -130,14 +129,14 @@ int32_t schtBuildFetchRspMsg(uint32_t *msize, void** rspMsg) {
return TSDB_CODE_SUCCESS;
}
-int32_t schtBuildSubmitRspMsg(uint32_t *msize, void** rspMsg) {
+int32_t schtBuildSubmitRspMsg(uint32_t *msize, void **rspMsg) {
SSubmitRsp2 submitRsp = {0};
- int32_t msgSize = 0, ret = 0;
- SEncoder ec = {0};
-
+ int32_t msgSize = 0, ret = 0;
+ SEncoder ec = {0};
+
tEncodeSize(tEncodeSSubmitRsp2, &submitRsp, msgSize, ret);
- void* msg = taosMemoryCalloc(1, msgSize);
- tEncoderInit(&ec, (uint8_t*)msg, msgSize);
+ void *msg = taosMemoryCalloc(1, msgSize);
+ tEncoderInit(&ec, (uint8_t *)msg, msgSize);
tEncodeSSubmitRsp2(&ec, &submitRsp);
tEncoderClear(&ec);
@@ -147,7 +146,6 @@ int32_t schtBuildSubmitRspMsg(uint32_t *msize, void** rspMsg) {
return TSDB_CODE_SUCCESS;
}
-
void schtBuildQueryDag(SQueryPlan *dag) {
uint64_t qId = schtQueryId;
@@ -157,8 +155,8 @@ void schtBuildQueryDag(SQueryPlan *dag) {
SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
- SSubplan *scanPlan = (SSubplan*)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
- SSubplan *mergePlan = (SSubplan*)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
+ SSubplan *scanPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
+ SSubplan *mergePlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
scanPlan->id.queryId = qId;
scanPlan->id.groupId = 0x0000000000000002;
@@ -210,7 +208,7 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) {
SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
- SSubplan *mergePlan = (SSubplan*)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
+ SSubplan *mergePlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
merge->pNodeList = nodesMakeList();
scan->pNodeList = nodesMakeList();
@@ -218,7 +216,7 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) {
mergePlan->pChildren = nodesMakeList();
for (int32_t i = 0; i < scanPlanNum; ++i) {
- SSubplan *scanPlan = (SSubplan*)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
+ SSubplan *scanPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
scanPlan->id.queryId = qId;
scanPlan->id.groupId = 0x0000000000000002;
scanPlan->id.subplanId = 0x0000000000000003 + i;
@@ -272,7 +270,7 @@ void schtBuildInsertDag(SQueryPlan *dag) {
SNodeListNode *inserta = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
inserta->pNodeList = nodesMakeList();
- SSubplan *insertPlan = (SSubplan*)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
+ SSubplan *insertPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
insertPlan->id.queryId = qId;
insertPlan->id.groupId = 0x0000000000000003;
@@ -287,14 +285,14 @@ void schtBuildInsertDag(SQueryPlan *dag) {
insertPlan->pChildren = NULL;
insertPlan->pParents = NULL;
insertPlan->pNode = NULL;
- insertPlan->pDataSink = (SDataSinkNode*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_INSERT);
- ((SDataInserterNode*)insertPlan->pDataSink)->size = 1;
- ((SDataInserterNode*)insertPlan->pDataSink)->pData = taosMemoryCalloc(1, 1);
+ insertPlan->pDataSink = (SDataSinkNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_INSERT);
+ ((SDataInserterNode *)insertPlan->pDataSink)->size = 1;
+ ((SDataInserterNode *)insertPlan->pDataSink)->pData = taosMemoryCalloc(1, 1);
insertPlan->msgType = TDMT_VND_SUBMIT;
nodesListAppend(inserta->pNodeList, (SNode *)insertPlan);
- insertPlan = (SSubplan*)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
+ insertPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
insertPlan->id.queryId = qId;
insertPlan->id.groupId = 0x0000000000000003;
@@ -309,9 +307,9 @@ void schtBuildInsertDag(SQueryPlan *dag) {
insertPlan->pChildren = NULL;
insertPlan->pParents = NULL;
insertPlan->pNode = NULL;
- insertPlan->pDataSink = (SDataSinkNode*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_INSERT);
- ((SDataInserterNode*)insertPlan->pDataSink)->size = 1;
- ((SDataInserterNode*)insertPlan->pDataSink)->pData = taosMemoryCalloc(1, 1);
+ insertPlan->pDataSink = (SDataSinkNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_INSERT);
+ ((SDataInserterNode *)insertPlan->pDataSink)->size = 1;
+ ((SDataInserterNode *)insertPlan->pDataSink)->pData = taosMemoryCalloc(1, 1);
insertPlan->msgType = TDMT_VND_SUBMIT;
nodesListAppend(inserta->pNodeList, (SNode *)insertPlan);
@@ -389,7 +387,8 @@ void schtSetRpcSendRequest() {
}
}
-int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet *epSet, int64_t *pTransporterId, SMsgSendInfo *pInfo, bool persistHandle, void* rpcCtx) {
+int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet *epSet, int64_t *pTransporterId, SMsgSendInfo *pInfo,
+ bool persistHandle, void *rpcCtx) {
if (pInfo) {
taosMemoryFreeClear(pInfo->param);
taosMemoryFreeClear(pInfo->msgInfo.pData);
@@ -439,11 +438,11 @@ void *schtSendRsp(void *param) {
SSchTask *task = *(SSchTask **)pIter;
SDataBuf msg = {0};
- void* rmsg = NULL;
+ void *rmsg = NULL;
schtBuildSubmitRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_VND_SUBMIT_RSP;
msg.pData = rmsg;
-
+
schHandleResponseMsg(pJob, task, task->execId, &msg, 0);
pIter = taosHashIterate(pJob->execTasks, pIter);
@@ -452,7 +451,7 @@ void *schtSendRsp(void *param) {
schReleaseJob(job);
schtJobDone = true;
-
+
return NULL;
}
@@ -462,13 +461,13 @@ void *schtCreateFetchRspThread(void *param) {
taosSsleep(1);
- int32_t code = 0;
+ int32_t code = 0;
SDataBuf msg = {0};
- void* rmsg = NULL;
+ void *rmsg = NULL;
schtBuildFetchRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_MERGE_FETCH_RSP;
msg.pData = rmsg;
-
+
code = schHandleResponseMsg(pJob, pJob->fetchTask, pJob->fetchTask->execId, &msg, 0);
schReleaseJob(job);
@@ -529,7 +528,7 @@ void *schtRunJobThread(void *aa) {
char *dbname = "1.db1";
char *tablename = "table1";
SVgroupInfo vgInfo = {0};
- SQueryPlan* dag = (SQueryPlan*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
+ SQueryPlan *dag = (SQueryPlan *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
schtInitLogFile();
@@ -601,7 +600,7 @@ void *schtRunJobThread(void *aa) {
param->taskId = task->taskId;
SDataBuf msg = {0};
- void* rmsg = NULL;
+ void *rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
@@ -622,7 +621,7 @@ void *schtRunJobThread(void *aa) {
param->taskId = task->taskId - 1;
SDataBuf msg = {0};
- void* rmsg = NULL;
+ void *rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
@@ -686,7 +685,6 @@ void *schtFreeJobThread(void *aa) {
return NULL;
}
-
} // namespace
TEST(queryTest, normalCase) {
@@ -696,7 +694,7 @@ TEST(queryTest, normalCase) {
char *tablename = "table1";
SVgroupInfo vgInfo = {0};
int64_t job = 0;
- SQueryPlan* dag = (SQueryPlan*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
+ SQueryPlan *dag = (SQueryPlan *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
SArray *qnodeList = taosArrayInit(1, sizeof(SQueryNodeLoad));
@@ -737,13 +735,13 @@ TEST(queryTest, normalCase) {
SSchTask *task = *(SSchTask **)pIter;
SDataBuf msg = {0};
- void* rmsg = NULL;
+ void *rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
-
+
code = schHandleResponseMsg(pJob, task, task->execId, &msg, 0);
-
+
ASSERT_EQ(code, 0);
pIter = taosHashIterate(pJob->execTasks, pIter);
}
@@ -753,13 +751,13 @@ TEST(queryTest, normalCase) {
SSchTask *task = *(SSchTask **)pIter;
if (JOB_TASK_STATUS_EXEC == task->status) {
SDataBuf msg = {0};
- void* rmsg = NULL;
+ void *rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
-
+
code = schHandleResponseMsg(pJob, task, task->execId, &msg, 0);
-
+
ASSERT_EQ(code, 0);
}
@@ -793,7 +791,7 @@ TEST(queryTest, normalCase) {
taosMemoryFreeClear(data);
schReleaseJob(job);
-
+
schedulerDestroy();
schedulerFreeJob(&job, 0);
@@ -808,7 +806,7 @@ TEST(queryTest, readyFirstCase) {
char *tablename = "table1";
SVgroupInfo vgInfo = {0};
int64_t job = 0;
- SQueryPlan* dag = (SQueryPlan*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
+ SQueryPlan *dag = (SQueryPlan *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
SArray *qnodeList = taosArrayInit(1, sizeof(SQueryNodeLoad));
@@ -816,7 +814,7 @@ TEST(queryTest, readyFirstCase) {
load.addr.epSet.numOfEps = 1;
strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep");
load.addr.epSet.eps[0].port = 6031;
- taosArrayPush(qnodeList, &load);
+ taosArrayPush(qnodeList, &load);
int32_t code = schedulerInit();
ASSERT_EQ(code, 0);
@@ -848,11 +846,11 @@ TEST(queryTest, readyFirstCase) {
SSchTask *task = *(SSchTask **)pIter;
SDataBuf msg = {0};
- void* rmsg = NULL;
+ void *rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
-
+
code = schHandleResponseMsg(pJob, task, task->execId, &msg, 0);
ASSERT_EQ(code, 0);
@@ -865,13 +863,13 @@ TEST(queryTest, readyFirstCase) {
if (JOB_TASK_STATUS_EXEC == task->status) {
SDataBuf msg = {0};
- void* rmsg = NULL;
+ void *rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
-
+
code = schHandleResponseMsg(pJob, task, task->execId, &msg, 0);
-
+
ASSERT_EQ(code, 0);
}
@@ -919,7 +917,7 @@ TEST(queryTest, flowCtrlCase) {
char *tablename = "table1";
SVgroupInfo vgInfo = {0};
int64_t job = 0;
- SQueryPlan* dag = (SQueryPlan*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
+ SQueryPlan *dag = (SQueryPlan *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
schtInitLogFile();
@@ -933,7 +931,6 @@ TEST(queryTest, flowCtrlCase) {
load.addr.epSet.eps[0].port = 6031;
taosArrayPush(qnodeList, &load);
-
int32_t code = schedulerInit();
ASSERT_EQ(code, 0);
@@ -968,13 +965,13 @@ TEST(queryTest, flowCtrlCase) {
if (JOB_TASK_STATUS_EXEC == task->status && 0 != task->lastMsgType) {
SDataBuf msg = {0};
- void* rmsg = NULL;
+ void *rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
-
+
code = schHandleResponseMsg(pJob, task, task->execId, &msg, 0);
-
+
ASSERT_EQ(code, 0);
}
@@ -1005,7 +1002,7 @@ TEST(queryTest, flowCtrlCase) {
schedulerFreeJob(&job, 0);
- taosThreadJoin(thread1, NULL);
+ taosThreadJoin(thread1, NULL);
}
TEST(insertTest, normalCase) {
@@ -1014,7 +1011,7 @@ TEST(insertTest, normalCase) {
char *dbname = "1.db1";
char *tablename = "table1";
SVgroupInfo vgInfo = {0};
- SQueryPlan* dag = (SQueryPlan*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
+ SQueryPlan *dag = (SQueryPlan *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
uint64_t numOfRows = 0;
SArray *qnodeList = taosArrayInit(1, sizeof(SQueryNodeLoad));
@@ -1067,7 +1064,7 @@ TEST(insertTest, normalCase) {
schedulerDestroy();
- taosThreadJoin(thread1, NULL);
+ taosThreadJoin(thread1, NULL);
}
TEST(multiThread, forceFree) {
@@ -1076,7 +1073,7 @@ TEST(multiThread, forceFree) {
TdThread thread1, thread2, thread3;
taosThreadCreate(&(thread1), &thattr, schtRunJobThread, NULL);
-// taosThreadCreate(&(thread2), &thattr, schtFreeJobThread, NULL);
+ // taosThreadCreate(&(thread2), &thattr, schtFreeJobThread, NULL);
taosThreadCreate(&(thread3), &thattr, schtFetchRspThread, NULL);
while (true) {
@@ -1089,7 +1086,7 @@ TEST(multiThread, forceFree) {
}
schtTestStop = true;
- //taosSsleep(3);
+ // taosSsleep(3);
}
TEST(otherTest, otherCase) {
@@ -1097,12 +1094,13 @@ TEST(otherTest, otherCase) {
schReleaseJob(0);
schFreeRpcCtx(NULL);
- ASSERT_EQ(schDumpEpSet(NULL), (char*)NULL);
+ ASSERT_EQ(schDumpEpSet(NULL), (char *)NULL);
ASSERT_EQ(strcmp(schGetOpStr(SCH_OP_NULL), "NULL"), 0);
ASSERT_EQ(strcmp(schGetOpStr((SCH_OP_TYPE)100), "UNKNOWN"), 0);
}
int main(int argc, char **argv) {
+ schtInitLogFile();
taosSeedRand(taosGetTimestampSec());
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h
index e0933f4fac..5cae2086a5 100644
--- a/source/libs/stream/inc/streamBackendRocksdb.h
+++ b/source/libs/stream/inc/streamBackendRocksdb.h
@@ -131,20 +131,21 @@ typedef struct {
TdThreadRwlock rwLock;
} SBkdMgt;
-bool streamBackendDataIsExist(const char* path, int64_t chkpId, int32_t vgId);
+#define META_ON_S3_FORMATE "%s_%" PRId64 "\n%s_%" PRId64 "\n%s_%" PRId64 ""
+
+bool streamBackendDataIsExist(const char* path, int64_t chkpId);
void* streamBackendInit(const char* path, int64_t chkpId, int32_t vgId);
void streamBackendCleanup(void* arg);
void streamBackendHandleCleanup(void* arg);
int32_t streamBackendLoadCheckpointInfo(void* pMeta);
-int32_t streamBackendDoCheckpoint(void* pMeta, int64_t checkpointId);
+int32_t streamBackendDoCheckpoint(void* pMeta, int64_t checkpointId, int64_t processver);
SListNode* streamBackendAddCompare(void* backend, void* arg);
void streamBackendDelCompare(void* backend, void* arg);
int32_t streamStateCvtDataFormat(char* path, char* key, void* cfInst);
-STaskDbWrapper* taskDbOpen(const char* path, const char* key, int64_t chkptId);
+STaskDbWrapper* taskDbOpen(const char* path, const char* key, int64_t chkptId, int64_t* processVer);
void taskDbDestroy(void* pBackend, bool flush);
void taskDbDestroy2(void* pBackend);
-int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId);
void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId);
@@ -250,7 +251,7 @@ int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId);
int32_t taskDbBuildSnap(void* arg, SArray* pSnap);
int32_t taskDbDestroySnap(void* arg, SArray* pSnapInfo);
-int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId);
+int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId, int64_t processId);
SBkdMgt* bkdMgtCreate(char* path);
int32_t bkdMgtAddChkp(SBkdMgt* bm, char* task, char* path);
@@ -260,6 +261,7 @@ void bkdMgtDestroy(SBkdMgt* bm);
int32_t taskDbGenChkpUploadData(void* arg, void* bkdMgt, int64_t chkpId, int8_t type, char** path, SArray* list,
const char* id);
+int32_t remoteChkpGetDelFile(char* path, SArray* toDel);
void* taskAcquireDb(int64_t refId);
void taskReleaseDb(int64_t refId);
diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index 3e0e19bdbd..8a34fdc30b 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -150,6 +150,9 @@ static rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const cha
void taskDbRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp);
void taskDbUnRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp);
+int32_t chkpAddExtraInfo(char* pChkpIdDir, int64_t chkpId, int64_t processId);
+int32_t chkpLoadExtraInfo(char* pChkpIdDir, int64_t* chkpId, int64_t* processId);
+
#define GEN_COLUMN_FAMILY_NAME(name, idstr, SUFFIX) sprintf(name, "%s_%s", idstr, (SUFFIX));
int32_t copyFiles(const char* src, const char* dst);
uint32_t nextPow2(uint32_t x);
@@ -194,28 +197,50 @@ int32_t getCfIdx(const char* cfName) {
return idx;
}
-bool isValidCheckpoint(const char* dir) { return true; }
+bool isValidCheckpoint(const char* dir) {
+ // not implement yet
+ return true;
+}
+/*
+ *copy pChkpIdDir's file to state dir
+ */
int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) {
// impl later
int32_t code = 0;
+ int32_t cap = strlen(path) + 64;
+ int32_t nBytes = 0;
+
+ char* state = taosMemoryCalloc(1, cap);
+ if (state == NULL) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
+ nBytes = snprintf(state, cap, "%s%s%s", path, TD_DIRSEP, "state");
+ if (nBytes <= 0 || nBytes >= cap) {
+ taosMemoryFree(state);
+ return TSDB_CODE_OUT_OF_RANGE;
+ }
- /*param@1: checkpointId dir
- param@2: state
- copy pChkpIdDir's file to state dir
- opt to set hard link to previous file
- */
- char* state = taosMemoryCalloc(1, strlen(path) + 32);
- sprintf(state, "%s%s%s", path, TD_DIRSEP, "state");
if (chkpId != 0) {
- char* chkp = taosMemoryCalloc(1, strlen(path) + 64);
- sprintf(chkp, "%s%s%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, "checkpoints", TD_DIRSEP, chkpId);
+ char* chkp = taosMemoryCalloc(1, cap);
+ if (chkp == NULL) {
+ taosMemoryFree(state);
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
+ nBytes = snprintf(chkp, cap, "%s%s%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, "checkpoints", TD_DIRSEP, chkpId);
+ if (nBytes <= 0 || nBytes >= cap) {
+ taosMemoryFree(state);
+ taosMemoryFree(chkp);
+ return TSDB_CODE_OUT_OF_RANGE;
+ }
+
if (taosIsDir(chkp) && isValidCheckpoint(chkp)) {
cleanDir(state, "");
code = backendCopyFiles(chkp, state);
- stInfo("copy snap file from %s to %s", chkp, state);
if (code != 0) {
- stError("failed to restart stream backend from %s, reason: %s", chkp, tstrerror(TAOS_SYSTEM_ERROR(errno)));
+ stError("failed to restart stream backend from %s, reason: %s", chkp, tstrerror(TAOS_SYSTEM_ERROR(code)));
} else {
stInfo("start to restart stream backend at checkpoint path: %s", chkp);
}
@@ -223,101 +248,175 @@ int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) {
} else {
stError("failed to start stream backend at %s, reason: %s, restart from default state dir:%s", chkp,
tstrerror(TAOS_SYSTEM_ERROR(errno)), state);
- taosMkDir(state);
+ code = taosMkDir(state);
+ if (code != 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ }
}
taosMemoryFree(chkp);
}
*dst = state;
- return 0;
+ return code;
}
-int32_t remoteChkp_readMetaData(char* path, SArray* list) {
- char* metaPath = taosMemoryCalloc(1, strlen(path));
- sprintf(metaPath, "%s%s%s", path, TD_DIRSEP, "META");
+typedef struct {
+ char pCurrName[24];
+ int64_t currChkptId;
- TdFilePtr pFile = taosOpenFile(path, TD_FILE_READ);
- if (pFile == NULL) {
- return -1;
+ char pManifestName[24];
+ int64_t manifestChkptId;
+
+ char processName[24];
+ int64_t processId;
+} SSChkpMetaOnS3;
+
+int32_t remoteChkp_readMetaData(char* path, SSChkpMetaOnS3** pMeta) {
+ int32_t code = 0;
+ int32_t cap = strlen(path) + 32;
+ TdFilePtr pFile = NULL;
+
+ char* metaPath = taosMemoryCalloc(1, cap);
+ if (metaPath == NULL) {
+ return TSDB_CODE_OUT_OF_MEMORY;
}
- char buf[128] = {0};
- if (taosReadFile(pFile, buf, sizeof(buf)) <= 0) {
+ int32_t n = snprintf(metaPath, cap, "%s%s%s", path, TD_DIRSEP, "META");
+ if (n <= 0 || n >= cap) {
taosMemoryFree(metaPath);
- taosCloseFile(&pFile);
- return -1;
+ return TSDB_CODE_OUT_OF_MEMORY;
}
- int32_t len = strnlen(buf, tListLen(buf));
- for (int i = 0; i < len; i++) {
- if (buf[i] == '\n') {
- char* item = taosMemoryCalloc(1, i + 1);
- memcpy(item, buf, i);
- taosArrayPush(list, &item);
-
- item = taosMemoryCalloc(1, len - i);
- memcpy(item, buf + i + 1, len - i - 1);
- taosArrayPush(list, &item);
- }
+ pFile = taosOpenFile(path, TD_FILE_READ);
+ if (pFile == NULL) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _EXIT;
}
+ char buf[256] = {0};
+ if (taosReadFile(pFile, buf, sizeof(buf)) <= 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _EXIT;
+ }
+
+ SSChkpMetaOnS3* p = taosMemoryCalloc(1, sizeof(SSChkpMetaOnS3));
+ if (p == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _EXIT;
+ }
+ n = sscanf(buf, META_ON_S3_FORMATE, p->pCurrName, &p->currChkptId, p->pManifestName, &p->manifestChkptId,
+ p->processName, &p->processId);
+ if (n != 6) {
+ code = TSDB_CODE_INVALID_MSG;
+ taosMemoryFree(p);
+ goto _EXIT;
+ }
+
+ if (p->currChkptId != p->manifestChkptId) {
+ code = TSDB_CODE_INVALID_MSG;
+ taosMemoryFree(p);
+ goto _EXIT;
+ }
+ *pMeta = p;
+ code = 0;
+_EXIT:
taosCloseFile(&pFile);
taosMemoryFree(metaPath);
- return 0;
+ return code;
}
-int32_t remoteChkp_validMetaFile(char* name, char* prename, int64_t chkpId) {
- int8_t valid = 0;
- for (int i = 0; i < strlen(name); i++) {
- if (name[i] == '_') {
- memcpy(prename, name, i);
- if (taosStr2int64(name + i + 1) != chkpId) {
- break;
- } else {
- valid = 1;
- }
- }
+
+int32_t remoteChkp_validAndCvtMeta(char* path, SSChkpMetaOnS3* pMeta, int64_t chkpId) {
+ int32_t code = 0;
+ int32_t nBytes = 0;
+
+ int32_t cap = strlen(path) + 64;
+ char* src = taosMemoryCalloc(1, cap);
+ char* dst = taosMemoryCalloc(1, cap);
+ if (src == NULL || dst == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _EXIT;
}
- return valid;
-}
-int32_t remoteChkp_validAndCvtMeta(char* path, SArray* list, int64_t chkpId) {
- int32_t complete = 1;
- int32_t len = strlen(path) + 32;
- char* src = taosMemoryCalloc(1, len);
- char* dst = taosMemoryCalloc(1, len);
- int8_t count = 0;
- for (int i = 0; i < taosArrayGetSize(list); i++) {
- char* p = taosArrayGetP(list, i);
- sprintf(src, "%s%s%s", path, TD_DIRSEP, p);
+ if (pMeta->currChkptId != chkpId || pMeta->manifestChkptId != chkpId) {
+ code = TSDB_CODE_INVALID_CFG;
+ goto _EXIT;
+ }
+ // rename current_chkp/mainfest to current
+ for (int i = 0; i < 2; i++) {
+ char* key = (i == 0 ? pMeta->pCurrName : pMeta->pManifestName);
+ if (strlen(key) <= 0) {
+ code = TSDB_CODE_INVALID_PARA;
+ goto _EXIT;
+ }
+
+ nBytes = snprintf(src, cap, "%s%s%s_%" PRId64 "", path, TD_DIRSEP, key, pMeta->currChkptId);
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ goto _EXIT;
+ }
- // check file exist
if (taosStatFile(src, NULL, NULL, NULL) != 0) {
- complete = 0;
- break;
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _EXIT;
}
- // check file name
- char temp[64] = {0};
- if (remoteChkp_validMetaFile(p, temp, chkpId)) {
- count++;
+ nBytes = snprintf(dst, cap, "%s%s%s", path, TD_DIRSEP, key);
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ goto _EXIT;
}
- // rename file
- sprintf(dst, "%s%s%s", path, TD_DIRSEP, temp);
- taosRenameFile(src, dst);
+ if (taosRenameFile(src, dst) != 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _EXIT;
+ }
- memset(src, 0, len);
- memset(dst, 0, len);
- }
- if (count != taosArrayGetSize(list)) {
- complete = 0;
+ memset(src, 0, cap);
+ memset(dst, 0, cap);
}
+ code = 0;
+// rename manifest_chkp to manifest
+_EXIT:
taosMemoryFree(src);
taosMemoryFree(dst);
+ return code;
+}
+int32_t remoteChkpGetDelFile(char* path, SArray* toDel) {
+ int32_t code = 0;
+ int32_t nBytes = 0;
- return complete == 1 ? 0 : -1;
+ SSChkpMetaOnS3* pMeta = NULL;
+ code = remoteChkp_readMetaData(path, &pMeta);
+ if (code != 0) {
+ return code;
+ }
+
+ for (int i = 0; i < 2; i++) {
+ char* key = (i == 0 ? pMeta->pCurrName : pMeta->pManifestName);
+
+ int32_t cap = strlen(key) + 32;
+ char* p = taosMemoryCalloc(1, cap);
+ if (p == NULL) {
+ taosMemoryFree(pMeta);
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
+ nBytes = snprintf(p, cap, "%s_%" PRId64 "", key, pMeta->currChkptId);
+ if (nBytes <= 0 || nBytes >= cap) {
+ taosMemoryFree(pMeta);
+ taosMemoryFree(p);
+ return TSDB_CODE_OUT_OF_RANGE;
+ }
+ if (taosArrayPush(toDel, &p) == NULL) {
+ taosMemoryFree(pMeta);
+ taosMemoryFree(p);
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+ }
+
+ return 0;
}
void cleanDir(const char* pPath, const char* id) {
@@ -330,73 +429,126 @@ void cleanDir(const char* pPath, const char* id) {
}
}
-void validateDir(const char* pPath) {
+int32_t createDirIfNotExist(const char* pPath) {
if (!taosIsDir(pPath)) {
- taosMulMkDir(pPath);
+ return taosMulMkDir(pPath);
+ } else {
+ return 0;
}
}
-int32_t rebuildFromRemoteChkp_rsync(const char* key, char* chkptPath, int64_t checkpointId, char* defaultPath) {
+int32_t rebuildFromRemoteChkp_rsync(const char* key, char* checkpointPath, int64_t checkpointId, char* defaultPath) {
int32_t code = 0;
- if (taosIsDir(chkptPath)) {
- taosRemoveDir(chkptPath);
- stDebug("remove local checkpoint data dir:%s succ", chkptPath);
+ if (taosIsDir(checkpointPath)) {
+ taosRemoveDir(checkpointPath);
+ stDebug("remove local checkpoint data dir:%s succ", checkpointPath);
}
cleanDir(defaultPath, key);
stDebug("clear local default dir before downloading checkpoint data:%s succ", defaultPath);
- code = streamTaskDownloadCheckpointData(key, chkptPath);
+ code = streamTaskDownloadCheckpointData(key, checkpointPath);
if (code != 0) {
stError("failed to download checkpoint data:%s", key);
return code;
}
stDebug("download remote checkpoint data for checkpointId:%" PRId64 ", %s", checkpointId, key);
- return backendCopyFiles(chkptPath, defaultPath);
+ return backendCopyFiles(checkpointPath, defaultPath);
+}
+
+int32_t rebuildDataFromS3(char* chkpPath, int64_t chkpId) {
+ SSChkpMetaOnS3* pMeta = NULL;
+
+ int32_t code = remoteChkp_readMetaData(chkpPath, &pMeta);
+ if (code != 0) {
+ return code;
+ }
+
+ if (pMeta->currChkptId != chkpId || pMeta->manifestChkptId != chkpId) {
+ taosMemoryFree(pMeta);
+ return TSDB_CODE_INVALID_PARA;
+ }
+
+ code = remoteChkp_validAndCvtMeta(chkpPath, pMeta, chkpId);
+ if (code != 0) {
+ taosMemoryFree(pMeta);
+ return code;
+ }
+ taosMemoryFree(pMeta);
+
+ return chkpAddExtraInfo(chkpPath, chkpId, pMeta->processId);
}
int32_t rebuildFromRemoteChkp_s3(const char* key, char* chkpPath, int64_t chkpId, char* defaultPath) {
+ int8_t rename = 0;
int32_t code = streamTaskDownloadCheckpointData(key, chkpPath);
if (code != 0) {
return code;
}
- int32_t len = strlen(defaultPath) + 32;
- char* tmp = taosMemoryCalloc(1, len);
- sprintf(tmp, "%s%s", defaultPath, "_tmp");
- if (taosIsDir(tmp)) taosRemoveDir(tmp);
- if (taosIsDir(defaultPath)) taosRenameFile(defaultPath, tmp);
+ int32_t cap = strlen(defaultPath) + 32;
- SArray* list = taosArrayInit(2, sizeof(void*));
- code = remoteChkp_readMetaData(chkpPath, list);
- if (code == 0) {
- code = remoteChkp_validAndCvtMeta(chkpPath, list, chkpId);
- }
- taosArrayDestroyP(list, taosMemoryFree);
-
- if (code == 0) {
- taosMkDir(defaultPath);
- code = backendCopyFiles(chkpPath, defaultPath);
+ char* defaultTmp = taosMemoryCalloc(1, cap);
+ if (defaultTmp == NULL) {
+ return TSDB_CODE_OUT_OF_MEMORY;
}
- if (code != 0) {
- if (taosIsDir(defaultPath)) taosRemoveDir(defaultPath);
- if (taosIsDir(tmp)) taosRenameFile(tmp, defaultPath);
+ int32_t nBytes = snprintf(defaultPath, cap, "%s%s", defaultPath, "_tmp");
+ if (nBytes <= 0 || nBytes >= cap) {
+ taosMemoryFree(defaultPath);
+ return TSDB_CODE_OUT_OF_RANGE;
+ }
+
+ if (taosIsDir(defaultTmp)) taosRemoveDir(defaultTmp);
+ if (taosIsDir(defaultPath)) {
+ code = taosRenameFile(defaultPath, defaultTmp);
+ if (code != 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _EXIT;
+ } else {
+ rename = 1;
+ }
} else {
- taosRemoveDir(tmp);
+ code = taosMkDir(defaultPath);
+ if (code != 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _EXIT;
+ }
}
- taosMemoryFree(tmp);
+ code = rebuildDataFromS3(chkpPath, chkpId);
+ if (code != 0) {
+ goto _EXIT;
+ }
+
+ code = backendCopyFiles(chkpPath, defaultPath);
+ if (code != 0) {
+ goto _EXIT;
+ }
+ code = 0;
+
+_EXIT:
+ if (code != 0) {
+ if (rename) {
+ taosRenameFile(defaultTmp, defaultPath);
+ }
+ }
+
+ if (taosIsDir(defaultPath)) {
+ taosRemoveDir(defaultPath);
+ }
+
+ taosMemoryFree(defaultTmp);
return code;
}
-int32_t rebuildFromRemoteCheckpoint(const char* key, char* chkptPath, int64_t checkpointId, char* defaultPath) {
+int32_t rebuildFromRemoteCheckpoint(const char* key, char* checkpointPath, int64_t checkpointId, char* defaultPath) {
ECHECKPOINT_BACKUP_TYPE type = streamGetCheckpointBackupType();
if (type == DATA_UPLOAD_S3) {
- return rebuildFromRemoteChkp_s3(key, chkptPath, checkpointId, defaultPath);
+ return rebuildFromRemoteChkp_s3(key, checkpointPath, checkpointId, defaultPath);
} else if (type == DATA_UPLOAD_RSYNC) {
- return rebuildFromRemoteChkp_rsync(key, chkptPath, checkpointId, defaultPath);
+ return rebuildFromRemoteChkp_rsync(key, checkpointPath, checkpointId, defaultPath);
} else {
stError("%s no remote backup checkpoint data for:%" PRId64, key, checkpointId);
}
@@ -423,46 +575,70 @@ int32_t backendFileCopyFilesImpl(const char* src, const char* dst) {
const char* current = "CURRENT";
size_t currLen = strlen(current);
+ const char* info = "info";
+ size_t infoLen = strlen(info);
+
int32_t code = 0;
int32_t sLen = strlen(src);
int32_t dLen = strlen(dst);
- char* srcName = taosMemoryCalloc(1, sLen + 64);
- char* dstName = taosMemoryCalloc(1, dLen + 64);
+ int32_t cap = TMAX(sLen, dLen) + 64;
+ int32_t nBytes = 0;
+
+ char* srcName = taosMemoryCalloc(1, cap);
+ char* dstName = taosMemoryCalloc(1, cap);
+ if (srcName == NULL || dstName == NULL) {
+ taosMemoryFree(srcName);
+ taosMemoryFree(dstName);
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ return code;
+ }
// copy file to dst
TdDirPtr pDir = taosOpenDir(src);
if (pDir == NULL) {
- taosMemoryFree(srcName);
- taosMemoryFree(dstName);
code = TAOS_SYSTEM_ERROR(errno);
-
- errno = 0;
- return code;
+ goto _ERROR;
}
errno = 0;
TdDirEntryPtr de = NULL;
-
while ((de = taosReadDir(pDir)) != NULL) {
char* name = taosGetDirEntryName(de);
if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) {
continue;
}
- sprintf(srcName, "%s%s%s", src, TD_DIRSEP, name);
- sprintf(dstName, "%s%s%s", dst, TD_DIRSEP, name);
+ nBytes = snprintf(srcName, cap, "%s%s%s", src, TD_DIRSEP, name);
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ goto _ERROR;
+ }
+
+ nBytes = snprintf(dstName, cap, "%s%s%s", dst, TD_DIRSEP, name);
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ goto _ERROR;
+ }
if (strncmp(name, current, strlen(name) <= currLen ? strlen(name) : currLen) == 0) {
code = copyFiles_create(srcName, dstName, 0);
if (code != 0) {
- code = TAOS_SYSTEM_ERROR(code);
+ code = TAOS_SYSTEM_ERROR(errno);
stError("failed to copy file, detail: %s to %s reason:%s", srcName, dstName, tstrerror(code));
goto _ERROR;
}
+ } else if (strncmp(name, info, strlen(name) <= infoLen ? strlen(name) : infoLen) == 0) {
+ code = copyFiles_create(srcName, dstName, 0);
+ if (code != 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ stError("failed to copy file, detail: %s to %s reason:%s", srcName, dstName, tstrerror(code));
+ goto _ERROR;
+ }
+
} else {
code = copyFiles_hardlink(srcName, dstName, 0);
if (code != 0) {
- code = TAOS_SYSTEM_ERROR(code);
+ code = TAOS_SYSTEM_ERROR(errno);
stError("failed to hard link file, detail:%s to %s, reason:%s", srcName, dstName, tstrerror(code));
goto _ERROR;
} else {
@@ -470,28 +646,26 @@ int32_t backendFileCopyFilesImpl(const char* src, const char* dst) {
}
}
- memset(srcName, 0, sLen + 64);
- memset(dstName, 0, dLen + 64);
+ memset(srcName, 0, cap);
+ memset(dstName, 0, cap);
}
taosMemoryFreeClear(srcName);
taosMemoryFreeClear(dstName);
taosCloseDir(&pDir);
- errno = 0;
return code;
_ERROR:
taosMemoryFreeClear(srcName);
taosMemoryFreeClear(dstName);
taosCloseDir(&pDir);
- errno = 0;
return code;
}
int32_t backendCopyFiles(const char* src, const char* dst) { return backendFileCopyFilesImpl(src, dst); }
static int32_t rebuildFromLocalCheckpoint(const char* pTaskIdStr, const char* checkpointPath, int64_t checkpointId,
- const char* defaultPath) {
+ const char* defaultPath, int64_t* processVer) {
int32_t code = 0;
cleanDir(defaultPath, pTaskIdStr);
@@ -502,7 +676,7 @@ static int32_t rebuildFromLocalCheckpoint(const char* pTaskIdStr, const char* ch
if (code != TSDB_CODE_SUCCESS) {
cleanDir(defaultPath, pTaskIdStr);
stError("%s failed to start stream backend from local %s, reason:%s, try download checkpoint from remote",
- pTaskIdStr, checkpointPath, tstrerror(TAOS_SYSTEM_ERROR(errno)));
+ pTaskIdStr, checkpointPath, tstrerror(TAOS_SYSTEM_ERROR(code)));
code = TSDB_CODE_SUCCESS;
} else {
stInfo("%s copy checkpoint data from:%s to:%s succ, try to start stream backend", pTaskIdStr, checkpointPath,
@@ -516,41 +690,81 @@ static int32_t rebuildFromLocalCheckpoint(const char* pTaskIdStr, const char* ch
return code;
}
-int32_t restoreCheckpointData(const char* path, const char* key, int64_t chkptId, char** dbPrefixPath, char** dbPath) {
+int32_t restoreCheckpointData(const char* path, const char* key, int64_t chkptId, char** dbPrefixPath, char** dbPath,
+ int64_t* processVer) {
int32_t code = 0;
- char* prefixPath = taosMemoryCalloc(1, strlen(path) + 128);
- sprintf(prefixPath, "%s%s%s", path, TD_DIRSEP, key);
+ char* prefixPath = NULL;
+ char* defaultPath = NULL;
+ char* checkpointPath = NULL;
+ char* checkpointRoot = NULL;
- validateDir(prefixPath);
+ int32_t cap = strlen(path) + 128;
+ int32_t nBytes;
- char* defaultPath = taosMemoryCalloc(1, strlen(path) + 256);
- sprintf(defaultPath, "%s%s%s", prefixPath, TD_DIRSEP, "state");
+ // alloc buf
+ prefixPath = taosMemoryCalloc(1, cap);
+ defaultPath = taosMemoryCalloc(1, cap);
+ checkpointPath = taosMemoryCalloc(1, cap);
+ checkpointRoot = taosMemoryCalloc(1, cap);
+ if (prefixPath == NULL || defaultPath == NULL || checkpointPath == NULL || checkpointRoot == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _EXIT;
+ }
- validateDir(defaultPath);
- int32_t pathLen = strlen(path) + 256;
+ nBytes = snprintf(prefixPath, cap, "%s%s%s", path, TD_DIRSEP, key);
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ goto _EXIT;
+ }
- char* checkpointRoot = taosMemoryCalloc(1, pathLen);
- sprintf(checkpointRoot, "%s%s%s", prefixPath, TD_DIRSEP, "checkpoints");
+ code = createDirIfNotExist(prefixPath);
+ if (code != 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _EXIT;
+ }
- validateDir(checkpointRoot);
- taosMemoryFree(checkpointRoot);
+ nBytes = snprintf(defaultPath, cap, "%s%s%s", prefixPath, TD_DIRSEP, "state");
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ goto _EXIT;
+ }
+
+ code = createDirIfNotExist(defaultPath);
+ if (code != 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _EXIT;
+ }
+
+ nBytes = snprintf(checkpointRoot, cap, "%s%s%s", prefixPath, TD_DIRSEP, "checkpoints");
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ goto _EXIT;
+ }
+
+ code = createDirIfNotExist(checkpointRoot);
+ if (code != 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ goto _EXIT;
+ }
stDebug("%s check local backend dir:%s, checkpointId:%" PRId64 " succ", key, defaultPath, chkptId);
-
- char* chkptPath = taosMemoryCalloc(1, pathLen);
if (chkptId > 0) {
- snprintf(chkptPath, pathLen, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint",
- chkptId);
+ nBytes = snprintf(checkpointPath, cap, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP,
+ "checkpoint", chkptId);
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ goto _EXIT;
+ }
- code = rebuildFromLocalCheckpoint(key, chkptPath, chkptId, defaultPath);
+ code = rebuildFromLocalCheckpoint(key, checkpointPath, chkptId, defaultPath, processVer);
if (code != 0) {
- code = rebuildFromRemoteCheckpoint(key, chkptPath, chkptId, defaultPath);
+ code = rebuildFromRemoteCheckpoint(key, checkpointPath, chkptId, defaultPath);
}
if (code != 0) {
- stError("failed to start stream backend at %s, reason: %s, restart from default defaultPath:%s", chkptPath,
- tstrerror(code), defaultPath);
+ stError("failed to start stream backend at %s, restart from default defaultPath:%s, reason:%s", checkpointPath,
+ defaultPath, tstrerror(code));
code = 0; // reset the error code
}
} else { // no valid checkpoint id
@@ -559,21 +773,40 @@ int32_t restoreCheckpointData(const char* path, const char* key, int64_t chkptId
cleanDir(defaultPath, key);
}
- taosMemoryFree(chkptPath);
-
*dbPath = defaultPath;
*dbPrefixPath = prefixPath;
+ defaultPath = NULL;
+ prefixPath = NULL;
+ code = 0;
+
+_EXIT:
+ taosMemoryFree(defaultPath);
+ taosMemoryFree(prefixPath);
+ taosMemoryFree(checkpointPath);
+ taosMemoryFree(checkpointRoot);
return code;
}
+bool streamBackendDataIsExist(const char* path, int64_t chkpId) {
+ bool exist = true;
+ int32_t cap = strlen(path) + 32;
-bool streamBackendDataIsExist(const char* path, int64_t chkpId, int32_t vgId) {
- bool exist = true;
- char* state = taosMemoryCalloc(1, strlen(path) + 32);
- sprintf(state, "%s%s%s", path, TD_DIRSEP, "state");
- if (!taosDirExist(state)) {
- exist = false;
+ char* state = taosMemoryCalloc(1, cap);
+ if (state == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return false;
}
+
+ int16_t nBytes = snprintf(state, cap, "%s%s%s", path, TD_DIRSEP, "state");
+ if (nBytes <= 0 || nBytes >= cap) {
+ terrno = TSDB_CODE_OUT_OF_RANGE;
+ exist = false;
+ } else {
+ if (!taosDirExist(state)) {
+ exist = false;
+ }
+ }
+
taosMemoryFree(state);
return exist;
}
@@ -1074,12 +1307,14 @@ int32_t chkpDoDbCheckpoint(rocksdb_t* db, char* path) {
if (cp == NULL || err != NULL) {
stError("failed to do checkpoint at:%s, reason:%s", path, err);
taosMemoryFreeClear(err);
+ code = TSDB_CODE_THIRDPARTY_ERROR;
goto _ERROR;
}
rocksdb_checkpoint_create(cp, path, UINT64_MAX, &err);
if (err != NULL) {
stError("failed to do checkpoint at:%s, reason:%s", path, err);
taosMemoryFreeClear(err);
+ code = TSDB_CODE_THIRDPARTY_ERROR;
} else {
code = 0;
}
@@ -1093,13 +1328,17 @@ int32_t chkpPreFlushDb(rocksdb_t* db, rocksdb_column_family_handle_t** cf, int32
char* err = NULL;
rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create();
+ if (flushOpt == NULL) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
rocksdb_flushoptions_set_wait(flushOpt, 1);
rocksdb_flush_cfs(db, flushOpt, cf, nCf, &err);
if (err != NULL) {
stError("failed to flush db before streamBackend clean up, reason:%s", err);
taosMemoryFree(err);
- code = -1;
+ code = TSDB_CODE_THIRDPARTY_ERROR;
}
rocksdb_flushoptions_destroy(flushOpt);
return code;
@@ -1107,31 +1346,51 @@ int32_t chkpPreFlushDb(rocksdb_t* db, rocksdb_column_family_handle_t** cf, int32
int32_t chkpPreBuildDir(char* path, int64_t chkpId, char** chkpDir, char** chkpIdDir) {
int32_t code = 0;
- char* pChkpDir = taosMemoryCalloc(1, 256);
- char* pChkpIdDir = taosMemoryCalloc(1, 256);
+ int32_t cap = strlen(path) + 256;
+ int32_t nBytes = 0;
- sprintf(pChkpDir, "%s%s%s", path, TD_DIRSEP, "checkpoints");
- code = taosMulModeMkDir(pChkpDir, 0755, true);
- if (code != 0) {
- stError("failed to prepare checkpoint dir, path:%s, reason:%s", path, tstrerror(code));
- taosMemoryFree(pChkpDir);
- taosMemoryFree(pChkpIdDir);
- code = -1;
- return code;
+ char* pChkpDir = taosMemoryCalloc(1, cap);
+ char* pChkpIdDir = taosMemoryCalloc(1, cap);
+ if (pChkpDir == NULL || pChkpIdDir == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _EXIT;
+ }
+
+ nBytes = snprintf(pChkpDir, cap, "%s%s%s", path, TD_DIRSEP, "checkpoints");
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ goto _EXIT;
+ }
+
+ nBytes = snprintf(pChkpIdDir, cap, "%s%s%s%" PRId64, pChkpDir, TD_DIRSEP, "checkpoint", chkpId);
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ goto _EXIT;
+ }
+
+ code = taosMulModeMkDir(pChkpDir, 0755, true);
+ if (code != 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ stError("failed to prepare checkpoint dir, path:%s, reason:%s", path, tstrerror(code));
+ goto _EXIT;
}
- sprintf(pChkpIdDir, "%s%s%s%" PRId64, pChkpDir, TD_DIRSEP, "checkpoint", chkpId);
if (taosIsDir(pChkpIdDir)) {
stInfo("stream rm exist checkpoint%s", pChkpIdDir);
taosRemoveDir(pChkpIdDir);
}
+
*chkpDir = pChkpDir;
*chkpIdDir = pChkpIdDir;
-
return 0;
+_EXIT:
+ taosMemoryFree(pChkpDir);
+ taosMemoryFree(pChkpIdDir);
+ return code;
}
int32_t taskDbBuildSnap(void* arg, SArray* pSnap) {
+ // vnode task->db
SStreamMeta* pMeta = arg;
taosThreadMutexLock(&pMeta->backendMutex);
@@ -1140,27 +1399,42 @@ int32_t taskDbBuildSnap(void* arg, SArray* pSnap) {
while (pIter) {
STaskDbWrapper* pTaskDb = *(STaskDbWrapper**)pIter;
- taskDbAddRef(pTaskDb);
- int64_t chkpId = pTaskDb->chkpId;
- taskDbRefChkp(pTaskDb, chkpId);
- code = taskDbDoCheckpoint(pTaskDb, chkpId);
- if (code != 0) {
- taskDbUnRefChkp(pTaskDb, chkpId);
+ void* p = taskDbAddRef(pTaskDb);
+ if (p == NULL) {
+ terrno = 0;
+ pIter = taosHashIterate(pMeta->pTaskDbUnique, pIter);
+ continue;
}
- taskDbRemoveRef(pTaskDb);
+ // add chkpId to in-use-ckpkIdSet
+ taskDbRefChkp(pTaskDb, pTaskDb->chkpId);
+
+ code = taskDbDoCheckpoint(pTaskDb, pTaskDb->chkpId, ((SStreamTask*)pTaskDb->pTask)->chkInfo.processedVer);
+ if (code != 0) {
+ // remove chkpId from in-use-ckpkIdSet
+ taskDbUnRefChkp(pTaskDb, pTaskDb->chkpId);
+ taskDbRemoveRef(pTaskDb);
+ break;
+ }
SStreamTask* pTask = pTaskDb->pTask;
SStreamTaskSnap snap = {.streamId = pTask->id.streamId,
.taskId = pTask->id.taskId,
.chkpId = pTaskDb->chkpId,
.dbPrefixPath = taosStrdup(pTaskDb->path)};
+ if (snap.dbPrefixPath == NULL) {
+ // remove chkpid from chkp-in-use set
+ taskDbUnRefChkp(pTaskDb, pTaskDb->chkpId);
+ taskDbRemoveRef(pTaskDb);
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ break;
+ }
taosArrayPush(pSnap, &snap);
+
pIter = taosHashIterate(pMeta->pTaskDbUnique, pIter);
}
taosThreadMutexUnlock(&pMeta->backendMutex);
-
return code;
}
int32_t taskDbDestroySnap(void* arg, SArray* pSnapInfo) {
@@ -1232,20 +1506,131 @@ int64_t taskGetDBRef(void* arg) {
return pDb->refId;
}
-int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId) {
+int32_t chkpLoadExtraInfo(char* pChkpIdDir, int64_t* chkpId, int64_t* processId) {
+ TdFilePtr pFile = NULL;
+ int32_t code = 0;
+
+ char buf[256] = {0};
+ int32_t nBytes = 0;
+
+ int32_t len = strlen(pChkpIdDir);
+ if (len == 0) {
+ code = TSDB_CODE_INVALID_PARA;
+ stError("failed to load extra info, dir:%s, reason:%s", pChkpIdDir, tstrerror(code));
+ return code;
+ }
+
+ int32_t cap = len + 64;
+ char* pDst = taosMemoryCalloc(1, cap);
+ if (pDst == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ stError("failed to alloc memory to load extra info, dir:%s", pChkpIdDir);
+ goto _EXIT;
+ }
+
+ nBytes = snprintf(pDst, cap, "%s%sinfo", pChkpIdDir, TD_DIRSEP);
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ stError("failed to build dst to load extra info, dir:%s", pChkpIdDir);
+ goto _EXIT;
+ }
+
+ pFile = taosOpenFile(pDst, TD_FILE_READ);
+ if (pFile == NULL) {
+ // compatible with previous version
+ *processId = -1;
+ code = 0;
+ stError("failed to open file to load extra info, file:%s, reason:%s", pDst, tstrerror(TAOS_SYSTEM_ERROR(errno)));
+ goto _EXIT;
+ }
+
+ if (taosReadFile(pFile, buf, sizeof(buf)) <= 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ stError("failed to read file to load extra info, file:%s, reason:%s", pDst, tstrerror(code));
+ goto _EXIT;
+ }
+
+ if (sscanf(buf, "%" PRId64 " %" PRId64 "", chkpId, processId) < 2) {
+ code = TSDB_CODE_INVALID_PARA;
+ stError("failed to read file content to load extra info, file:%s, reason:%s", pDst, tstrerror(code));
+ goto _EXIT;
+ }
+ code = 0;
+_EXIT:
+ taosMemoryFree(pDst);
+ taosCloseFile(&pFile);
+ return code;
+}
+int32_t chkpAddExtraInfo(char* pChkpIdDir, int64_t chkpId, int64_t processId) {
+ int32_t code = 0;
+
+ TdFilePtr pFile = NULL;
+
+ char buf[256] = {0};
+ int32_t nBytes = 0;
+
+ int32_t len = strlen(pChkpIdDir);
+ if (len == 0) {
+ code = TSDB_CODE_INVALID_PARA;
+ stError("failed to add extra info, dir:%s, reason:%s", pChkpIdDir, tstrerror(code));
+ return code;
+ }
+
+ int32_t cap = len + 64;
+ char* pDst = taosMemoryCalloc(1, cap);
+ if (pDst == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ stError("failed to alloc memory to add extra info, dir:%s", pChkpIdDir);
+ goto _EXIT;
+ }
+
+ nBytes = snprintf(pDst, cap, "%s%sinfo", pChkpIdDir, TD_DIRSEP);
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ stError("failed to build dst to add extra info, dir:%s, reason:%s", pChkpIdDir, tstrerror(code));
+ goto _EXIT;
+ }
+
+ pFile = taosOpenFile(pDst, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+ if (pFile == NULL) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ stError("failed to open file to add extra info, file:%s, reason:%s", pDst, tstrerror(code));
+ goto _EXIT;
+ }
+
+ nBytes = snprintf(buf, sizeof(buf), "%" PRId64 " %" PRId64 "", chkpId, processId);
+ if (nBytes <= 0 || nBytes >= sizeof(buf)) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ stError("failed to build content to add extra info, dir:%s,reason:%s", pChkpIdDir, tstrerror(code));
+ goto _EXIT;
+ }
+
+ if (nBytes != taosWriteFile(pFile, buf, nBytes)) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ stError("failed to write file to add extra info, file:%s, reason:%s", pDst, tstrerror(code));
+ goto _EXIT;
+ }
+ code = 0;
+
+_EXIT:
+ taosCloseFile(&pFile);
+ taosMemoryFree(pDst);
+ return code;
+}
+int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId, int64_t processId) {
STaskDbWrapper* pTaskDb = arg;
int64_t st = taosGetTimestampMs();
- int32_t code = -1;
+ int32_t code = 0;
int64_t refId = pTaskDb->refId;
if (taosAcquireRef(taskDbWrapperId, refId) == NULL) {
- return -1;
+ code = terrno;
+ return code;
}
char* pChkpDir = NULL;
char* pChkpIdDir = NULL;
- if (chkpPreBuildDir(pTaskDb->path, chkpId, &pChkpDir, &pChkpIdDir) != 0) {
- code = -1;
+ if ((code = chkpPreBuildDir(pTaskDb->path, chkpId, &pChkpDir, &pChkpIdDir)) < 0) {
goto _EXIT;
}
// Get all cf and acquire cfWrappter
@@ -1256,32 +1641,58 @@ int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId) {
int64_t written = atomic_load_64(&pTaskDb->dataWritten);
+ // flush db
if (written > 0) {
stDebug("stream backend:%p start to flush db at:%s, data written:%" PRId64 "", pTaskDb, pChkpIdDir, written);
code = chkpPreFlushDb(pTaskDb->db, ppCf, nCf);
+ if (code != 0) goto _EXIT;
} else {
stDebug("stream backend:%p not need flush db at:%s, data written:%" PRId64 "", pTaskDb, pChkpIdDir, written);
}
+
+ // do checkpoint
if ((code = chkpDoDbCheckpoint(pTaskDb->db, pChkpIdDir)) != 0) {
stError("stream backend:%p failed to do checkpoint at:%s", pTaskDb, pChkpIdDir);
+ goto _EXIT;
} else {
stDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pTaskDb, pChkpIdDir,
taosGetTimestampMs() - st);
}
+ // add extra info to checkpoint
+ if ((code = chkpAddExtraInfo(pChkpIdDir, chkpId, processId)) != 0) {
+ stError("stream backend:%p failed to add extra info to checkpoint at:%s", pTaskDb, pChkpIdDir);
+ goto _EXIT;
+ }
+
+ // delete ttl checkpoint
code = chkpMayDelObsolete(pTaskDb, chkpId, pChkpDir);
+ if (code < 0) {
+ goto _EXIT;
+ }
+
atomic_store_64(&pTaskDb->dataWritten, 0);
pTaskDb->chkpId = chkpId;
_EXIT:
- taosMemoryFree(pChkpDir);
+
+ // clear checkpoint dir if failed
+ if (code != 0 && pChkpDir != NULL) {
+ if (taosDirExist(pChkpIdDir)) {
+ taosRemoveDir(pChkpIdDir);
+ }
+ }
taosMemoryFree(pChkpIdDir);
+ taosMemoryFree(pChkpDir);
+
taosReleaseRef(taskDbWrapperId, refId);
taosMemoryFree(ppCf);
return code;
}
-int32_t streamBackendDoCheckpoint(void* arg, int64_t chkpId) { return taskDbDoCheckpoint(arg, chkpId); }
+int32_t streamBackendDoCheckpoint(void* arg, int64_t chkpId, int64_t processVer) {
+ return taskDbDoCheckpoint(arg, chkpId, processVer);
+}
SListNode* streamBackendAddCompare(void* backend, void* arg) {
SBackendWrapper* pHandle = (SBackendWrapper*)backend;
@@ -1659,7 +2070,9 @@ int32_t valueEncode(void* value, int32_t vlen, int64_t ttl, char** dest) {
len += taosEncodeFixedI32((void**)&buf, key.len);
len += taosEncodeFixedI32((void**)&buf, key.rawLen);
len += taosEncodeFixedI8((void**)&buf, key.compress);
- len += taosEncodeBinary((void**)&buf, (char*)value, key.len);
+ if (value != NULL && key.len != 0) {
+ len += taosEncodeBinary((void**)&buf, (char*)value, key.len);
+ }
*dest = p;
} else {
char* buf = *dest;
@@ -1667,7 +2080,9 @@ int32_t valueEncode(void* value, int32_t vlen, int64_t ttl, char** dest) {
len += taosEncodeFixedI32((void**)&buf, key.len);
len += taosEncodeFixedI32((void**)&buf, key.rawLen);
len += taosEncodeFixedI8((void**)&buf, key.compress);
- len += taosEncodeBinary((void**)&buf, (char*)value, key.len);
+ if (value != NULL && key.len != 0) {
+ len += taosEncodeBinary((void**)&buf, (char*)value, key.len);
+ }
}
taosMemoryFree(dst);
@@ -2005,12 +2420,16 @@ void taskDbDestroyChkpOpt(STaskDbWrapper* pTaskDb) {
int32_t taskDbBuildFullPath(char* path, char* key, char** dbFullPath, char** stateFullPath) {
int32_t code = 0;
+ char* statePath = taosMemoryCalloc(1, strlen(path) + 128);
+ if (statePath == NULL) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
- char* statePath = taosMemoryCalloc(1, strlen(path) + 128);
sprintf(statePath, "%s%s%s", path, TD_DIRSEP, key);
if (!taosDirExist(statePath)) {
code = taosMulMkDir(statePath);
if (code != 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
stError("failed to create dir: %s, reason:%s", statePath, tstrerror(code));
taosMemoryFree(statePath);
return code;
@@ -2018,10 +2437,16 @@ int32_t taskDbBuildFullPath(char* path, char* key, char** dbFullPath, char** sta
}
char* dbPath = taosMemoryCalloc(1, strlen(statePath) + 128);
+ if (dbPath == NULL) {
+ taosMemoryFree(statePath);
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
sprintf(dbPath, "%s%s%s", statePath, TD_DIRSEP, "state");
if (!taosDirExist(dbPath)) {
code = taosMulMkDir(dbPath);
if (code != 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
stError("failed to create dir: %s, reason:%s", dbPath, tstrerror(code));
taosMemoryFree(statePath);
taosMemoryFree(dbPath);
@@ -2099,15 +2524,32 @@ _EXIT:
return NULL;
}
-STaskDbWrapper* taskDbOpen(const char* path, const char* key, int64_t chkptId) {
+STaskDbWrapper* taskDbOpen(const char* path, const char* key, int64_t chkptId, int64_t* processVer) {
char* statePath = NULL;
char* dbPath = NULL;
-
- if (restoreCheckpointData(path, key, chkptId, &statePath, &dbPath) != 0) {
+ int code = 0;
+ terrno = 0;
+ if ((code = restoreCheckpointData(path, key, chkptId, &statePath, &dbPath, processVer)) < 0) {
+ terrno = code;
+ stError("failed to restore checkpoint data, path:%s, key:%s, checkpointId: %" PRId64 "reason:%s", path, key,
+ chkptId, tstrerror(terrno));
return NULL;
}
STaskDbWrapper* pTaskDb = taskDbOpenImpl(key, statePath, dbPath);
+ if (pTaskDb != NULL) {
+ int64_t chkpId = -1, ver = -1;
+ if ((code = chkpLoadExtraInfo(dbPath, &chkpId, &ver) == 0)) {
+ *processVer = ver;
+ } else {
+ terrno = code;
+ stError("failed to load extra info, path:%s, key:%s, checkpointId: %" PRId64 "reason:%s", path, key, chkptId,
+ tstrerror(terrno));
+ taskDbDestroy(pTaskDb, false);
+ return NULL;
+ }
+ }
+
taosMemoryFree(dbPath);
taosMemoryFree(statePath);
return pTaskDb;
@@ -2194,15 +2636,31 @@ void taskDbDestroy(void* pDb, bool flush) {
void taskDbDestroy2(void* pDb) { taskDbDestroy(pDb, true); }
int32_t taskDbGenChkpUploadData__rsync(STaskDbWrapper* pDb, int64_t chkpId, char** path) {
- int32_t code = -1;
+ int32_t code = 0;
int64_t refId = pDb->refId;
+ int32_t nBytes = 0;
if (taosAcquireRef(taskDbWrapperId, refId) == NULL) {
- return -1;
+ code = terrno;
+ return code;
+ }
+
+ int32_t cap = strlen(pDb->path) + 128;
+
+ char* buf = taosMemoryCalloc(1, cap);
+ if (buf == NULL) {
+ taosReleaseRef(taskDbWrapperId, refId);
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
+ nBytes =
+ snprintf(buf, cap, "%s%s%s%s%s%" PRId64 "", pDb->path, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", chkpId);
+ if (nBytes <= 0 || nBytes >= cap) {
+ taosMemoryFree(buf);
+ taosReleaseRef(taskDbWrapperId, refId);
+ return TSDB_CODE_OUT_OF_RANGE;
}
- char* buf = taosMemoryCalloc(1, strlen(pDb->path) + 128);
- sprintf(buf, "%s%s%s%s%s%" PRId64 "", pDb->path, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", chkpId);
if (taosIsDir(buf)) {
code = 0;
*path = buf;
@@ -2217,15 +2675,28 @@ int32_t taskDbGenChkpUploadData__rsync(STaskDbWrapper* pDb, int64_t chkpId, char
int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64_t chkpId, char** path, SArray* list,
const char* idStr) {
int32_t code = 0;
+ int32_t cap = strlen(pDb->path) + 32;
SBkdMgt* p = (SBkdMgt*)bkdChkpMgt;
- char* temp = taosMemoryCalloc(1, strlen(pDb->path) + 32);
- sprintf(temp, "%s%s%s%" PRId64, pDb->path, TD_DIRSEP, "tmp", chkpId);
+ char* temp = taosMemoryCalloc(1, cap);
+ if (temp == NULL) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
+ int32_t nBytes = snprintf(temp, cap, "%s%s%s%" PRId64, pDb->path, TD_DIRSEP, "tmp", chkpId);
+ if (nBytes <= 0 || nBytes >= cap) {
+ taosMemoryFree(temp);
+ return TSDB_CODE_OUT_OF_RANGE;
+ }
if (taosDirExist(temp)) {
cleanDir(temp, idStr);
} else {
- taosMkDir(temp);
+ code = taosMkDir(temp);
+ if (code != 0) {
+ taosMemoryFree(temp);
+ return TAOS_SYSTEM_ERROR(errno);
+ }
}
code = bkdMgtGetDelta(p, pDb->idstr, chkpId, list, temp);
@@ -2321,7 +2792,8 @@ int32_t streamStateCvtDataFormat(char* path, char* key, void* pCfInst) {
int32_t code = 0;
- STaskDbWrapper* pTaskDb = taskDbOpen(path, key, 0);
+ int64_t processVer = -1;
+ STaskDbWrapper* pTaskDb = taskDbOpen(path, key, 0, &processVer);
RocksdbCfInst* pSrcBackend = pCfInst;
for (int i = 0; i < nCf; i++) {
@@ -3962,6 +4434,9 @@ int32_t compareHashTableImpl(SHashObj* p1, SHashObj* p2, SArray* diff) {
char* name = taosHashGetKey(pIter, &len);
if (!isBkdDataMeta(name, len) && !taosHashGet(p1, name, len)) {
char* fname = taosMemoryCalloc(1, len + 1);
+ if (fname == NULL) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
strncpy(fname, name, len);
taosArrayPush(diff, &fname);
}
@@ -3973,7 +4448,9 @@ int32_t compareHashTable(SHashObj* p1, SHashObj* p2, SArray* add, SArray* del) {
int32_t code = 0;
code = compareHashTableImpl(p1, p2, add);
- code = compareHashTableImpl(p2, p1, del);
+ if (code != 0) {
+ code = compareHashTableImpl(p2, p1, del);
+ }
return code;
}
@@ -4014,26 +4491,29 @@ void strArrayDebugInfo(SArray* pArr, char** buf) {
*buf = p;
}
void dbChkpDebugInfo(SDbChkp* pDb) {
- // stTrace("chkp get file list: curr");
- char* p[4] = {NULL};
+ if (stDebugFlag & DEBUG_INFO) {
+ char* p[4] = {NULL};
- hashTableToDebug(pDb->pSstTbl[pDb->idx], &p[0]);
- stTrace("chkp previous file: [%s]", p[0]);
+ hashTableToDebug(pDb->pSstTbl[pDb->idx], &p[0]);
+ stTrace("chkp previous file: [%s]", p[0]);
- hashTableToDebug(pDb->pSstTbl[1 - pDb->idx], &p[1]);
- stTrace("chkp curr file: [%s]", p[1]);
+ hashTableToDebug(pDb->pSstTbl[1 - pDb->idx], &p[1]);
+ stTrace("chkp curr file: [%s]", p[1]);
- strArrayDebugInfo(pDb->pAdd, &p[2]);
- stTrace("chkp newly addded file: [%s]", p[2]);
+ strArrayDebugInfo(pDb->pAdd, &p[2]);
+ stTrace("chkp newly addded file: [%s]", p[2]);
- strArrayDebugInfo(pDb->pDel, &p[3]);
- stTrace("chkp newly deleted file: [%s]", p[3]);
+ strArrayDebugInfo(pDb->pDel, &p[3]);
+ stTrace("chkp newly deleted file: [%s]", p[3]);
- for (int i = 0; i < 4; i++) {
- taosMemoryFree(p[i]);
+ for (int i = 0; i < 4; i++) {
+ taosMemoryFree(p[i]);
+ }
}
}
int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) {
+ int32_t code = 0;
+ int32_t nBytes;
taosThreadRwlockWrlock(&p->rwLock);
p->preCkptId = p->curChkpId;
@@ -4048,13 +4528,24 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) {
int32_t sstLen = strlen(pSST);
memset(p->buf, 0, p->len);
- sprintf(p->buf, "%s%s%s%scheckpoint%" PRId64 "", p->path, TD_DIRSEP, "checkpoints", TD_DIRSEP, chkpId);
+
+ nBytes =
+ snprintf(p->buf, p->len, "%s%s%s%scheckpoint%" PRId64 "", p->path, TD_DIRSEP, "checkpoints", TD_DIRSEP, chkpId);
+ if (nBytes <= 0 || nBytes >= p->len) {
+ taosThreadRwlockUnlock(&p->rwLock);
+ return TSDB_CODE_OUT_OF_RANGE;
+ }
taosArrayClearP(p->pAdd, taosMemoryFree);
taosArrayClearP(p->pDel, taosMemoryFree);
taosHashClear(p->pSstTbl[1 - p->idx]);
- TdDirPtr pDir = taosOpenDir(p->buf);
+ TdDirPtr pDir = taosOpenDir(p->buf);
+ if (pDir == NULL) {
+ taosThreadRwlockUnlock(&p->rwLock);
+ return TAOS_SYSTEM_ERROR(errno);
+ }
+
TdDirEntryPtr de = NULL;
int8_t dummy = 0;
while ((de = taosReadDir(pDir)) != NULL) {
@@ -4062,23 +4553,36 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) {
if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue;
if (strlen(name) == currLen && strcmp(name, pCurrent) == 0) {
taosMemoryFreeClear(p->pCurrent);
+
p->pCurrent = taosStrdup(name);
- // taosHashPut(p->pSstTbl[1 - p->idx], name, strlen(name), &dummy, sizeof(dummy));
+ if (p->pCurrent == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ break;
+ }
continue;
}
if (strlen(name) >= maniLen && strncmp(name, pManifest, maniLen) == 0) {
taosMemoryFreeClear(p->pManifest);
p->pManifest = taosStrdup(name);
- // taosHashPut(p->pSstTbl[1 - p->idx], name, strlen(name), &dummy, sizeof(dummy));
+ if (p->pManifest == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ break;
+ }
continue;
}
if (strlen(name) >= sstLen && strncmp(name + strlen(name) - 4, pSST, sstLen) == 0) {
- taosHashPut(p->pSstTbl[1 - p->idx], name, strlen(name), &dummy, sizeof(dummy));
+ if (taosHashPut(p->pSstTbl[1 - p->idx], name, strlen(name), &dummy, sizeof(dummy)) != 0) {
+ break;
+ }
continue;
}
}
taosCloseDir(&pDir);
+ if (code != 0) {
+ taosThreadRwlockUnlock(&p->rwLock);
+ return code;
+ }
if (p->init == 0) {
void* pIter = taosHashIterate(p->pSstTbl[1 - p->idx], NULL);
@@ -4087,6 +4591,11 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) {
char* name = taosHashGetKey(pIter, &len);
if (name != NULL && !isBkdDataMeta(name, len)) {
char* fname = taosMemoryCalloc(1, len + 1);
+ if (fname == NULL) {
+ taosThreadRwlockUnlock(&p->rwLock);
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
strncpy(fname, name, len);
taosArrayPush(p->pAdd, &fname);
}
@@ -4122,34 +4631,78 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) {
taosThreadRwlockUnlock(&p->rwLock);
- return 0;
+ return code;
}
+void dbChkpDestroy(SDbChkp* pChkp);
+
SDbChkp* dbChkpCreate(char* path, int64_t initChkpId) {
SDbChkp* p = taosMemoryCalloc(1, sizeof(SDbChkp));
+ if (p == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ goto _EXIT;
+ }
+
p->curChkpId = initChkpId;
p->preCkptId = -1;
p->pSST = taosArrayInit(64, sizeof(void*));
+ if (p->pSST == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ dbChkpDestroy(p);
+ return NULL;
+ }
+
p->path = path;
p->len = strlen(path) + 128;
p->buf = taosMemoryCalloc(1, p->len);
+ if (p->buf == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ goto _EXIT;
+ }
p->idx = 0;
p->pSstTbl[0] = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
+ if (p->pSstTbl[0] == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ goto _EXIT;
+ }
+
p->pSstTbl[1] = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
+ if (p->pSstTbl[1] == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ goto _EXIT;
+ }
p->pAdd = taosArrayInit(64, sizeof(void*));
+ if (p->pAdd == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ goto _EXIT;
+ }
+
p->pDel = taosArrayInit(64, sizeof(void*));
+ if (p->pDel == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ goto _EXIT;
+ }
+
p->update = 0;
taosThreadRwlockInit(&p->rwLock, NULL);
SArray* list = NULL;
int32_t code = dbChkpGetDelta(p, initChkpId, list);
+ if (code != 0) {
+ goto _EXIT;
+ }
return p;
+_EXIT:
+ dbChkpDestroy(p);
+ return NULL;
}
void dbChkpDestroy(SDbChkp* pChkp) {
+ if (pChkp == NULL) return;
+
taosMemoryFree(pChkp->buf);
taosMemoryFree(pChkp->path);
@@ -4171,35 +4724,71 @@ int32_t dbChkpInit(SDbChkp* p) {
}
#endif
int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) {
+ static char* chkpMeta = "META";
+ int32_t code = 0;
+
taosThreadRwlockRdlock(&p->rwLock);
- int32_t code = -1;
- int32_t len = p->len + 128;
- char* srcBuf = taosMemoryCalloc(1, len);
- char* dstBuf = taosMemoryCalloc(1, len);
+ int32_t cap = p->len + 128;
- char* srcDir = taosMemoryCalloc(1, len);
- char* dstDir = taosMemoryCalloc(1, len);
+ char* buffer = taosMemoryCalloc(4, cap);
+ if (buffer == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _ERROR;
+ }
- sprintf(srcDir, "%s%s%s%s%s%" PRId64 "", p->path, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", p->curChkpId);
- sprintf(dstDir, "%s", dname);
+ char* srcBuf = buffer;
+ char* dstBuf = &srcBuf[cap];
+ char* srcDir = &dstBuf[cap];
+ char* dstDir = &srcDir[cap];
+
+ int nBytes = snprintf(srcDir, cap, "%s%s%s%s%s%" PRId64 "", p->path, TD_DIRSEP, "checkpoints", TD_DIRSEP,
+ "checkpoint", p->curChkpId);
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ goto _ERROR;
+ }
+
+ nBytes = snprintf(dstDir, cap, "%s", dname);
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ goto _ERROR;
+ }
if (!taosDirExist(srcDir)) {
stError("failed to dump srcDir %s, reason: not exist such dir", srcDir);
+ code = TSDB_CODE_INVALID_PARA;
+ goto _ERROR;
+ }
+ int64_t chkpId = 0, processId = -1;
+ code = chkpLoadExtraInfo(srcDir, &chkpId, &processId);
+ if (code < 0) {
+ stError("failed to load extra info from %s, reason:%s", srcDir, code != 0 ? "unkown" : tstrerror(code));
+
goto _ERROR;
}
// add file to $name dir
for (int i = 0; i < taosArrayGetSize(p->pAdd); i++) {
- memset(srcBuf, 0, len);
- memset(dstBuf, 0, len);
+ memset(srcBuf, 0, cap);
+ memset(dstBuf, 0, cap);
char* filename = taosArrayGetP(p->pAdd, i);
- sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, filename);
- sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, filename);
+ nBytes = snprintf(srcBuf, cap, "%s%s%s", srcDir, TD_DIRSEP, filename);
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ goto _ERROR;
+ }
+
+ nBytes = snprintf(dstBuf, cap, "%s%s%s", dstDir, TD_DIRSEP, filename);
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ goto _ERROR;
+ }
if (taosCopyFile(srcBuf, dstBuf) < 0) {
- stError("failed to copy file from %s to %s", srcBuf, dstBuf);
+ code = TAOS_SYSTEM_ERROR(errno);
+ stError("failed to copy file from %s to %s, reason:%s", srcBuf, dstBuf, tstrerror(code));
goto _ERROR;
}
}
@@ -4207,44 +4796,84 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) {
for (int i = 0; i < taosArrayGetSize(p->pDel); i++) {
char* filename = taosArrayGetP(p->pDel, i);
char* p = taosStrdup(filename);
+ if (p == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _ERROR;
+ }
taosArrayPush(list, &p);
}
// copy current file to dst dir
- memset(srcBuf, 0, len);
- memset(dstBuf, 0, len);
- sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, p->pCurrent);
- sprintf(dstBuf, "%s%s%s_%" PRId64 "", dstDir, TD_DIRSEP, p->pCurrent, p->curChkpId);
+ memset(srcBuf, 0, cap);
+ memset(dstBuf, 0, cap);
+
+ nBytes = snprintf(srcBuf, cap, "%s%s%s", srcDir, TD_DIRSEP, p->pCurrent);
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ goto _ERROR;
+ }
+
+ nBytes = snprintf(dstBuf, cap, "%s%s%s_%" PRId64 "", dstDir, TD_DIRSEP, p->pCurrent, p->curChkpId);
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ goto _ERROR;
+ }
+
if (taosCopyFile(srcBuf, dstBuf) < 0) {
- stError("failed to copy file from %s to %s", srcBuf, dstBuf);
+ code = TAOS_SYSTEM_ERROR(errno);
+ stError("failed to copy file from %s to %s, reason:%s", srcBuf, dstBuf, tstrerror(code));
goto _ERROR;
}
// copy manifest file to dst dir
- memset(srcBuf, 0, len);
- memset(dstBuf, 0, len);
- sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, p->pManifest);
- sprintf(dstBuf, "%s%s%s_%" PRId64 "", dstDir, TD_DIRSEP, p->pManifest, p->curChkpId);
- if (taosCopyFile(srcBuf, dstBuf) < 0) {
- stError("failed to copy file from %s to %s", srcBuf, dstBuf);
+ memset(srcBuf, 0, cap);
+ memset(dstBuf, 0, cap);
+
+ nBytes = snprintf(srcBuf, cap, "%s%s%s", srcDir, TD_DIRSEP, p->pManifest);
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
goto _ERROR;
}
- static char* chkpMeta = "META";
- memset(dstBuf, 0, len);
- sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, chkpMeta);
- tstrncpy(dstDir, dstBuf, strlen(dstBuf) + 1);
+ nBytes = snprintf(dstBuf, cap, "%s%s%s_%" PRId64 "", dstDir, TD_DIRSEP, p->pManifest, p->curChkpId);
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ goto _ERROR;
+ }
+
+ if (taosCopyFile(srcBuf, dstBuf) < 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ stError("failed to copy file from %s to %s, reason:%s", srcBuf, dstBuf, tstrerror(code));
+ goto _ERROR;
+ }
+ memset(dstBuf, 0, cap);
+ nBytes = snprintf(dstDir, cap, "%s%s%s", dstDir, TD_DIRSEP, chkpMeta);
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ goto _ERROR;
+ }
TdFilePtr pFile = taosOpenFile(dstDir, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
- stError("chkp failed to create meta file: %s", dstDir);
+ code = TAOS_SYSTEM_ERROR(errno);
+ stError("chkp failed to create meta file: %s, reason:%s", dstDir, tstrerror(code));
goto _ERROR;
}
- char content[128] = {0};
- snprintf(content, sizeof(content), "%s_%" PRId64 "\n%s_%" PRId64 "", p->pCurrent, p->curChkpId, p->pManifest,
- p->curChkpId);
- if (taosWriteFile(pFile, content, strlen(content)) <= 0) {
- stError("chkp failed to write meta file: %s", dstDir);
+
+ char content[256] = {0};
+ nBytes = snprintf(content, sizeof(content), META_ON_S3_FORMATE, p->pCurrent, p->curChkpId, p->pManifest, p->curChkpId,
+ "processVer", processId);
+ if (nBytes <= 0 || nBytes >= sizeof(content)) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ stError("chkp failed to format meta file: %s, reason: invalid msg", dstDir);
+ taosCloseFile(&pFile);
+ goto _ERROR;
+ }
+
+ nBytes = taosWriteFile(pFile, content, strlen(content));
+ if (nBytes != strlen(content)) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ stError("chkp failed to write meta file: %s,reason:%s", dstDir, tstrerror(code));
taosCloseFile(&pFile);
goto _ERROR;
}
@@ -4256,18 +4885,39 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) {
code = 0;
_ERROR:
+ taosMemoryFree(buffer);
taosThreadRwlockUnlock(&p->rwLock);
- taosMemoryFree(srcBuf);
- taosMemoryFree(dstBuf);
- taosMemoryFree(srcDir);
- taosMemoryFree(dstDir);
return code;
}
+
SBkdMgt* bkdMgtCreate(char* path) {
+ terrno = 0;
SBkdMgt* p = taosMemoryCalloc(1, sizeof(SBkdMgt));
+ if (p == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return NULL;
+ }
+
p->pDbChkpTbl = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+ if (p->pDbChkpTbl == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ bkdMgtDestroy(p);
+ return NULL;
+ }
+
p->path = taosStrdup(path);
- taosThreadRwlockInit(&p->rwLock, NULL);
+ if (p->path == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ bkdMgtDestroy(p);
+ return NULL;
+ }
+
+ if (taosThreadRwlockInit(&p->rwLock, NULL) != 0) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ bkdMgtDestroy(p);
+ return NULL;
+ }
+
return p;
}
@@ -4289,28 +4939,52 @@ void bkdMgtDestroy(SBkdMgt* bm) {
}
int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list, char* dname) {
int32_t code = 0;
-
taosThreadRwlockWrlock(&bm->rwLock);
SDbChkp** ppChkp = taosHashGet(bm->pDbChkpTbl, taskId, strlen(taskId));
SDbChkp* pChkp = ppChkp != NULL ? *ppChkp : NULL;
if (pChkp == NULL) {
- char* path = taosMemoryCalloc(1, strlen(bm->path) + 64);
- sprintf(path, "%s%s%s", bm->path, TD_DIRSEP, taskId);
+ int32_t cap = strlen(bm->path) + 64;
+ char* path = taosMemoryCalloc(1, cap);
+ if (path == NULL) {
+ taosThreadRwlockUnlock(&bm->rwLock);
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
+ int32_t nBytes = snprintf(path, cap, "%s%s%s", bm->path, TD_DIRSEP, taskId);
+ if (nBytes <= 0 || nBytes >= cap) {
+ taosMemoryFree(path);
+ taosThreadRwlockUnlock(&bm->rwLock);
+ code = TSDB_CODE_OUT_OF_RANGE;
+ return code;
+ }
SDbChkp* p = dbChkpCreate(path, chkpId);
- taosHashPut(bm->pDbChkpTbl, taskId, strlen(taskId), &p, sizeof(void*));
+ if (p == NULL) {
+ taosMemoryFree(path);
+ taosThreadRwlockUnlock(&bm->rwLock);
+ code = terrno;
+ return code;
+ }
+
+ if (taosHashPut(bm->pDbChkpTbl, taskId, strlen(taskId), &p, sizeof(void*)) != 0) {
+ dbChkpDestroy(p);
+ taosThreadRwlockUnlock(&bm->rwLock);
+ code = terrno;
+ return code;
+ }
pChkp = p;
-
code = dbChkpDumpTo(pChkp, dname, list);
taosThreadRwlockUnlock(&bm->rwLock);
return code;
+ } else {
+ code = dbChkpGetDelta(pChkp, chkpId, NULL);
+ if (code == 0) {
+ code = dbChkpDumpTo(pChkp, dname, list);
+ }
}
- code = dbChkpGetDelta(pChkp, chkpId, NULL);
- code = dbChkpDumpTo(pChkp, dname, list);
-
taosThreadRwlockUnlock(&bm->rwLock);
return code;
}
diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c
index 8778e3314a..226a06be7e 100644
--- a/source/libs/stream/src/streamCheckStatus.c
+++ b/source/libs/stream/src/streamCheckStatus.c
@@ -666,13 +666,18 @@ void rspMonitorFn(void* param, void* tmrId) {
stDebug("s-task:%s status:%s vgId:%d quit from monitor check-rsp tmr, ref:%d", id, pStat->name, vgId, ref);
streamTaskCompleteCheckRsp(pInfo, true, id);
- addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId);
+
+ // not record the failed of the current task if try to close current vnode
+ // otherwise, the put of message operation may incur invalid read of message queue.
+ if (!pMeta->closeFlag) {
+ addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId);
+ }
streamMetaReleaseTask(pMeta, pTask);
return;
}
- if (state == TASK_STATUS__DROPPING || state == TASK_STATUS__READY || state == TASK_STATUS__PAUSE) {
+ if (state == TASK_STATUS__DROPPING || state == TASK_STATUS__READY) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s status:%s vgId:%d quit from monitor check-rsp tmr, ref:%d", id, pStat->name, vgId, ref);
@@ -698,31 +703,31 @@ void rspMonitorFn(void* param, void* tmrId) {
if (pStat->state == TASK_STATUS__UNINIT) {
getCheckRspStatus(pInfo, timeoutDuration, &numOfReady, &numOfFault, &numOfNotRsp, pTimeoutList, pNotReadyList, id);
+
+ numOfNotReady = (int32_t)taosArrayGetSize(pNotReadyList);
+ numOfTimeout = (int32_t)taosArrayGetSize(pTimeoutList);
+
+ // fault tasks detected, not try anymore
+ ASSERT((numOfReady + numOfFault + numOfNotReady + numOfTimeout + numOfNotRsp) == total);
+ if (numOfFault > 0) {
+ int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
+ stDebug(
+ "s-task:%s status:%s vgId:%d all rsp. quit from monitor rsp tmr, since vnode-transfer/leader-change/restart "
+ "detected, total:%d, notRsp:%d, notReady:%d, fault:%d, timeout:%d, ready:%d ref:%d",
+ id, pStat->name, vgId, total, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref);
+
+ streamTaskCompleteCheckRsp(pInfo, false, id);
+ taosThreadMutexUnlock(&pInfo->checkInfoLock);
+ streamMetaReleaseTask(pMeta, pTask);
+
+ taosArrayDestroy(pNotReadyList);
+ taosArrayDestroy(pTimeoutList);
+ return;
+ }
} else { // unexpected status
stError("s-task:%s unexpected task status:%s during waiting for check rsp", id, pStat->name);
}
- numOfNotReady = (int32_t)taosArrayGetSize(pNotReadyList);
- numOfTimeout = (int32_t)taosArrayGetSize(pTimeoutList);
-
- // fault tasks detected, not try anymore
- ASSERT((numOfReady + numOfFault + numOfNotReady + numOfTimeout + numOfNotRsp) == total);
- if (numOfFault > 0) {
- int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
- stDebug(
- "s-task:%s status:%s vgId:%d all rsp. quit from monitor rsp tmr, since vnode-transfer/leader-change/restart "
- "detected, total:%d, notRsp:%d, notReady:%d, fault:%d, timeout:%d, ready:%d ref:%d",
- id, pStat->name, vgId, total, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref);
-
- streamTaskCompleteCheckRsp(pInfo, false, id);
- taosThreadMutexUnlock(&pInfo->checkInfoLock);
- streamMetaReleaseTask(pMeta, pTask);
-
- taosArrayDestroy(pNotReadyList);
- taosArrayDestroy(pTimeoutList);
- return;
- }
-
// checking of downstream tasks has been stopped by other threads
if (pInfo->stopCheckProcess == 1) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index bc973f17d7..a3a7035905 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -57,6 +57,13 @@ SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpoint
pBlock->info.childId = pTask->info.selfChildId;
pChkpoint->blocks = taosArrayInit(4, sizeof(SSDataBlock)); // pBlock;
+ if (pChkpoint->blocks == NULL) {
+ taosMemoryFree(pBlock);
+ taosFreeQitem(pChkpoint);
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return NULL;
+ }
+
taosArrayPush(pChkpoint->blocks, pBlock);
taosMemoryFree(pBlock);
@@ -112,7 +119,12 @@ int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTri
int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId, int32_t downstreamNodeId,
SRpcHandleInfo* pRpcInfo, int32_t code) {
int32_t size = sizeof(SMsgHead) + sizeof(SCheckpointTriggerRsp);
- void* pBuf = rpcMallocCont(size);
+
+ void* pBuf = rpcMallocCont(size);
+ if (pBuf == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return terrno;
+ }
SCheckpointTriggerRsp* pRsp = POINTER_SHIFT(pBuf, sizeof(SMsgHead));
@@ -133,6 +145,7 @@ int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId
SRpcMsg rspMsg = {.code = 0, .pCont = pBuf, .contLen = size, .info = *pRpcInfo};
tmsgSendRsp(&rspMsg);
+
return 0;
}
@@ -405,12 +418,14 @@ int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstream
void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) {
pTask->chkInfo.startTs = 0; // clear the recorded start time
-
- streamTaskClearActiveInfo(pTask->chkInfo.pActiveInfo);
streamTaskOpenAllUpstreamInput(pTask); // open inputQ for all upstream tasks
+
+ taosThreadMutexLock(&pTask->chkInfo.pActiveInfo->lock);
+ streamTaskClearActiveInfo(pTask->chkInfo.pActiveInfo);
if (clearChkpReadyMsg) {
streamClearChkptReadyMsg(pTask->chkInfo.pActiveInfo);
}
+ taosThreadMutexUnlock(&pTask->chkInfo.pActiveInfo->lock);
}
int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SVUpdateCheckpointInfoReq* pReq) {
@@ -447,14 +462,6 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
SStreamTaskState* pStatus = streamTaskGetStatus(pTask);
- if (restored && (pStatus->state != TASK_STATUS__CK) && (pMeta->role == NODE_ROLE_LEADER)) {
- stDebug("s-task:0x%x vgId:%d restored:%d status:%s not update checkpoint-info, checkpointId:%" PRId64 "->%" PRId64
- " failed",
- pReq->taskId, vgId, restored, pStatus->name, pInfo->checkpointId, pReq->checkpointId);
- taosThreadMutexUnlock(&pTask->lock);
- return TSDB_CODE_STREAM_TASK_IVLD_STATUS;
- }
-
if (!restored) { // during restore procedure, do update checkpoint-info
stDebug("s-task:%s vgId:%d status:%s update the checkpoint-info during restore, checkpointId:%" PRId64 "->%" PRId64
" checkpointVer:%" PRId64 "->%" PRId64 " checkpointTs:%" PRId64 "->%" PRId64,
@@ -528,65 +535,57 @@ void streamTaskSetFailedCheckpointId(SStreamTask* pTask) {
}
static int32_t getCheckpointDataMeta(const char* id, const char* path, SArray* list) {
- char buf[128] = {0};
+ int32_t code = 0;
+ int32_t cap = strlen(path) + 64;
- char* file = taosMemoryCalloc(1, strlen(path) + 32);
- sprintf(file, "%s%s%s", path, TD_DIRSEP, "META_TMP");
+ char* filePath = taosMemoryCalloc(1, cap);
+ if (filePath == NULL) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
- int32_t code = downloadCheckpointDataByName(id, "META", file);
+ int32_t nBytes = snprintf(filePath, cap, "%s%s%s", path, TD_DIRSEP, "META_TMP");
+ if (nBytes <= 0 || nBytes >= cap) {
+ taosMemoryFree(filePath);
+ return TSDB_CODE_OUT_OF_RANGE;
+ }
+
+ code = downloadCheckpointDataByName(id, "META", filePath);
if (code != 0) {
- stDebug("%s chkp failed to download meta file:%s", id, file);
- taosMemoryFree(file);
+ stError("%s chkp failed to download meta file:%s", id, filePath);
+ taosMemoryFree(filePath);
return code;
}
- TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
- if (pFile == NULL) {
- stError("%s failed to open meta file:%s for checkpoint", id, file);
- code = -1;
- return code;
+ code = remoteChkpGetDelFile(filePath, list);
+ if (code != 0) {
+ stError("%s chkp failed to get to del:%s", id, filePath);
+ taosMemoryFree(filePath);
}
-
- if (taosReadFile(pFile, buf, sizeof(buf)) <= 0) {
- stError("%s failed to read meta file:%s for checkpoint", id, file);
- code = -1;
- } else {
- int32_t len = strnlen(buf, tListLen(buf));
- for (int i = 0; i < len; i++) {
- if (buf[i] == '\n') {
- char* item = taosMemoryCalloc(1, i + 1);
- memcpy(item, buf, i);
- taosArrayPush(list, &item);
-
- item = taosMemoryCalloc(1, len - i);
- memcpy(item, buf + i + 1, len - i - 1);
- taosArrayPush(list, &item);
- }
- }
- }
-
- taosCloseFile(&pFile);
- taosRemoveFile(file);
- taosMemoryFree(file);
- return code;
+ return 0;
}
int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t dbRefId, ECHECKPOINT_BACKUP_TYPE type) {
- char* path = NULL;
- int32_t code = 0;
- SArray* toDelFiles = taosArrayInit(4, POINTER_BYTES);
- int64_t now = taosGetTimestampMs();
+ int32_t code = 0;
+ char* path = NULL;
+
SStreamMeta* pMeta = pTask->pMeta;
const char* idStr = pTask->id.idStr;
+ int64_t now = taosGetTimestampMs();
+
+ SArray* toDelFiles = taosArrayInit(4, POINTER_BYTES);
+ if (toDelFiles == NULL) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
if ((code = taskDbGenChkpUploadData(pTask->pBackend, pMeta->bkdChkptMgt, checkpointId, type, &path, toDelFiles,
pTask->id.idStr)) != 0) {
- stError("s-task:%s failed to gen upload checkpoint:%" PRId64, idStr, checkpointId);
+ stError("s-task:%s failed to gen upload checkpoint:%" PRId64 ", reason:%s", idStr, checkpointId, tstrerror(code));
}
if (type == DATA_UPLOAD_S3) {
if (code == TSDB_CODE_SUCCESS && (code = getCheckpointDataMeta(idStr, path, toDelFiles)) != 0) {
- stError("s-task:%s failed to get checkpointData for checkpointId:%" PRId64 " meta", idStr, checkpointId);
+ stError("s-task:%s failed to get checkpointData for checkpointId:%" PRId64 ", reason:%s", idStr, checkpointId,
+ tstrerror(code));
}
}
@@ -595,7 +594,8 @@ int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t d
if (code == TSDB_CODE_SUCCESS) {
stDebug("s-task:%s upload checkpointId:%" PRId64 " to remote succ", idStr, checkpointId);
} else {
- stError("s-task:%s failed to upload checkpointId:%" PRId64 " data:%s", idStr, checkpointId, path);
+ stError("s-task:%s failed to upload checkpointId:%" PRId64 " path:%s,reason:%s", idStr, checkpointId, path,
+ tstrerror(code));
}
}
@@ -668,7 +668,8 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) {
if (pTask->info.taskLevel != TASK_LEVEL__SINK) {
stDebug("s-task:%s level:%d start gen checkpoint, checkpointId:%" PRId64, id, pTask->info.taskLevel, ckId);
- code = streamBackendDoCheckpoint(pTask->pBackend, ckId);
+ int64_t ver = pTask->chkInfo.processedVer;
+ code = streamBackendDoCheckpoint(pTask->pBackend, ckId, ver);
if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s gen checkpoint:%" PRId64 " failed, code:%s", id, ckId, tstrerror(terrno));
}
@@ -776,6 +777,11 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
SArray* pList = pTask->upstreamInfo.pList;
ASSERT(pTask->info.taskLevel > TASK_LEVEL__SOURCE);
SArray* pNotSendList = taosArrayInit(4, sizeof(SStreamUpstreamEpInfo));
+ if (pNotSendList == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ stDebug("s-task:%s start to triggerMonitor, reason:%s", id, tstrerror(terrno));
+ return;
+ }
for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pList, i);
@@ -982,52 +988,77 @@ void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId) {
}
static int32_t uploadCheckpointToS3(const char* id, const char* path) {
+ int32_t code = 0;
+ int32_t nBytes = 0;
+
+ if (s3Init() != 0) {
+ return TSDB_CODE_THIRDPARTY_ERROR;
+ }
+
TdDirPtr pDir = taosOpenDir(path);
- if (pDir == NULL) return -1;
+ if (pDir == NULL) {
+ return TAOS_SYSTEM_ERROR(errno);
+ }
TdDirEntryPtr de = NULL;
- s3Init();
while ((de = taosReadDir(pDir)) != NULL) {
char* name = taosGetDirEntryName(de);
if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0 || taosDirEntryIsDir(de)) continue;
char filename[PATH_MAX] = {0};
if (path[strlen(path) - 1] == TD_DIRSEP_CHAR) {
- snprintf(filename, sizeof(filename), "%s%s", path, name);
+ nBytes = snprintf(filename, sizeof(filename), "%s%s", path, name);
+ if (nBytes <= 0 || nBytes >= sizeof(filename)) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ break;
+ }
} else {
- snprintf(filename, sizeof(filename), "%s%s%s", path, TD_DIRSEP, name);
+ nBytes = snprintf(filename, sizeof(filename), "%s%s%s", path, TD_DIRSEP, name);
+ if (nBytes <= 0 || nBytes >= sizeof(filename)) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ break;
+ }
}
char object[PATH_MAX] = {0};
- snprintf(object, sizeof(object), "%s%s%s", id, TD_DIRSEP, name);
-
- if (s3PutObjectFromFile2(filename, object, 0) != 0) {
- taosCloseDir(&pDir);
- return -1;
+ nBytes = snprintf(object, sizeof(object), "%s%s%s", id, TD_DIRSEP, name);
+ if (nBytes <= 0 || nBytes >= sizeof(object)) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ break;
}
- stDebug("[s3] upload checkpoint:%s", filename);
- // break;
- }
+ code = s3PutObjectFromFile2(filename, object, 0);
+ if (code != 0) {
+ stError("[s3] failed to upload checkpoint:%s, reason:%s", filename, tstrerror(code));
+ } else {
+ stDebug("[s3] upload checkpoint:%s", filename);
+ }
+ }
taosCloseDir(&pDir);
- return 0;
+ return code;
}
int32_t downloadCheckpointByNameS3(const char* id, const char* fname, const char* dstName) {
- int32_t code = 0;
- char* buf = taosMemoryCalloc(1, strlen(id) + strlen(dstName) + 4);
+ int32_t nBytes;
+ int32_t cap = strlen(id) + strlen(dstName) + 16;
+
+ char* buf = taosMemoryCalloc(1, cap);
if (buf == NULL) {
- code = terrno = TSDB_CODE_OUT_OF_MEMORY;
- return code;
+ return TSDB_CODE_OUT_OF_MEMORY;
}
- sprintf(buf, "%s/%s", id, fname);
- if (s3GetObjectToFile(buf, dstName) != 0) {
- code = errno;
+ nBytes = snprintf(buf, cap, "%s/%s", id, fname);
+ if (nBytes <= 0 || nBytes >= cap) {
+ taosMemoryFree(buf);
+ return TSDB_CODE_OUT_OF_RANGE;
+ }
+ int32_t code = s3GetObjectToFile(buf, dstName);
+ if (code != 0) {
+ taosMemoryFree(buf);
+ return TAOS_SYSTEM_ERROR(errno);
}
-
taosMemoryFree(buf);
- return code;
+ return 0;
}
ECHECKPOINT_BACKUP_TYPE streamGetCheckpointBackupType() {
@@ -1041,13 +1072,17 @@ ECHECKPOINT_BACKUP_TYPE streamGetCheckpointBackupType() {
}
int32_t streamTaskUploadCheckpoint(const char* id, const char* path) {
+ int32_t code = 0;
if (id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX) {
stError("invalid parameters in upload checkpoint, %s", id);
- return -1;
+ return TSDB_CODE_INVALID_CFG;
}
if (strlen(tsSnodeAddress) != 0) {
- return uploadByRsync(id, path);
+ code = uploadByRsync(id, path);
+ if (code != 0) {
+ return TAOS_SYSTEM_ERROR(errno);
+ }
} else if (tsS3StreamEnabled) {
return uploadCheckpointToS3(id, path);
}
@@ -1059,7 +1094,7 @@ int32_t streamTaskUploadCheckpoint(const char* id, const char* path) {
int32_t downloadCheckpointDataByName(const char* id, const char* fname, const char* dstName) {
if (id == NULL || fname == NULL || strlen(id) == 0 || strlen(fname) == 0 || strlen(fname) >= PATH_MAX) {
stError("down load checkpoint data parameters invalid");
- return -1;
+ return TSDB_CODE_INVALID_PARA;
}
if (strlen(tsSnodeAddress) != 0) {
@@ -1089,7 +1124,7 @@ int32_t streamTaskDownloadCheckpointData(const char* id, char* path) {
int32_t deleteCheckpoint(const char* id) {
if (id == NULL || strlen(id) == 0) {
stError("deleteCheckpoint parameters invalid");
- return -1;
+ return TSDB_CODE_INVALID_PARA;
}
if (strlen(tsSnodeAddress) != 0) {
return deleteRsync(id);
@@ -1101,11 +1136,18 @@ int32_t deleteCheckpoint(const char* id) {
int32_t deleteCheckpointFile(const char* id, const char* name) {
char object[128] = {0};
- snprintf(object, sizeof(object), "%s/%s", id, name);
- char* tmp = object;
- s3DeleteObjects((const char**)&tmp, 1);
- return 0;
+ int32_t nBytes = snprintf(object, sizeof(object), "%s/%s", id, name);
+ if (nBytes <= 0 || nBytes >= sizeof(object)) {
+ return TSDB_CODE_OUT_OF_RANGE;
+ }
+
+ char* tmp = object;
+ int32_t code = s3DeleteObjects((const char**)&tmp, 1);
+ if (code != 0) {
+ return TSDB_CODE_THIRDPARTY_ERROR;
+ }
+ return code;
}
int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask) {
@@ -1115,8 +1157,20 @@ int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask) {
const char* id = pTask->id.idStr;
SCheckpointInfo* pInfo = &pTask->chkInfo;
- ASSERT(pTask->pBackend == NULL);
+ taosThreadMutexLock(&pTask->lock);
+ if (pTask->status.sendConsensusChkptId == true) {
+ stDebug("s-task:%s already start to consensus-checkpointId, not start again before it completed", id);
+ taosThreadMutexUnlock(&pTask->lock);
+ return TSDB_CODE_SUCCESS;
+ } else {
+ pTask->status.sendConsensusChkptId = true;
+ }
+ taosThreadMutexUnlock(&pTask->lock);
+
+ ASSERT(pTask->pBackend == NULL);
+ pTask->status.requireConsensusChkptId = true;
+#if 0
SRestoreCheckpointInfo req = {
.streamId = pTask->id.streamId,
.taskId = pTask->id.taskId,
@@ -1128,14 +1182,14 @@ int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask) {
tEncodeSize(tEncodeRestoreCheckpointInfo, &req, tlen, code);
if (code < 0) {
stError("s-task:%s vgId:%d encode stream task latest-checkpoint-id failed, code:%s", id, vgId, tstrerror(code));
- return -1;
+ return TSDB_CODE_INVALID_MSG;
}
void* buf = rpcMallocCont(tlen);
if (buf == NULL) {
stError("s-task:%s vgId:%d encode stream task latest-checkpoint-id msg failed, code:%s", id, vgId,
tstrerror(TSDB_CODE_OUT_OF_MEMORY));
- return -1;
+ return TSDB_CODE_OUT_OF_MEMORY;
}
SEncoder encoder;
@@ -1148,10 +1202,27 @@ int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask) {
tEncoderClear(&encoder);
SRpcMsg msg = {0};
- initRpcMsg(&msg, TDMT_MND_STREAM_CHKPT_CONSEN, buf, tlen);
+ initRpcMsg(&msg, TDMT_MND_STREAM_REQ_CONSEN_CHKPT, buf, tlen);
stDebug("s-task:%s vgId:%d send latest checkpointId:%" PRId64 " to mnode to get the consensus checkpointId", id, vgId,
pInfo->checkpointId);
tmsgSendReq(&pTask->info.mnodeEpset, &msg);
+#endif
return 0;
+}
+
+int32_t streamTaskSendPreparedCheckpointsourceRsp(SStreamTask* pTask) {
+ int32_t code = 0;
+ if (pTask->info.taskLevel != TASK_LEVEL__SOURCE) {
+ return code;
+ }
+
+ taosThreadMutexLock(&pTask->lock);
+ SStreamTaskState* p = streamTaskGetStatus(pTask);
+ if (p->state == TASK_STATUS__CK) {
+ code = streamTaskSendCheckpointSourceRsp(pTask);
+ }
+ taosThreadMutexUnlock(&pTask->lock);
+
+ return code;
}
\ No newline at end of file
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index 83e73e8c88..617adaa016 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -813,9 +813,20 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
taosThreadMutexLock(&pActiveInfo->lock);
SArray* pList = pActiveInfo->pReadyMsgList;
+ int32_t num = taosArrayGetSize(pList);
+
+ // active checkpoint info is cleared for now
+ if ((pActiveInfo->activeId == 0) && (pActiveInfo->transId == 0) && (num == 0) && (pTask->chkInfo.startTs == 0)) {
+ taosThreadMutexUnlock(&pActiveInfo->lock);
+ int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
+ stWarn("s-task:%s vgId:%d active checkpoint may be cleared, quit from readyMsg send tmr, ref:%d", id, vgId, ref);
+
+ streamMetaReleaseTask(pTask->pMeta, pTask);
+ return;
+ }
+
SArray* pNotRspList = taosArrayInit(4, sizeof(int32_t));
- int32_t num = taosArrayGetSize(pList);
ASSERT(taosArrayGetSize(pTask->upstreamInfo.pList) == num);
for (int32_t i = 0; i < num; ++i) {
diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c
index d6411e25f2..16cb23de10 100644
--- a/source/libs/stream/src/streamHb.c
+++ b/source/libs/stream/src/streamHb.c
@@ -168,7 +168,9 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) {
continue;
}
+ taosThreadMutexLock(&(*pTask)->lock);
STaskStatusEntry entry = streamTaskGetStatusEntry(*pTask);
+ taosThreadMutexUnlock(&(*pTask)->lock);
entry.inputRate = entry.inputQUsed * 100.0 / (2 * STREAM_TASK_QUEUE_CAPACITY_IN_SIZE);
if ((*pTask)->info.taskLevel == TASK_LEVEL__SINK) {
@@ -192,6 +194,12 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) {
}
}
+ if ((*pTask)->status.requireConsensusChkptId) {
+ entry.checkpointInfo.consensusChkptId = 1;
+ (*pTask)->status.requireConsensusChkptId = false;
+ stDebug("s-task:%s vgId:%d set the require consensus-checkpointId in hbMsg", (*pTask)->id.idStr, pMeta->vgId);
+ }
+
if ((*pTask)->exec.pWalReader != NULL) {
entry.processedVer = walReaderGetCurrentVer((*pTask)->exec.pWalReader) - 1;
if (entry.processedVer < 0) {
@@ -324,7 +332,7 @@ int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp) {
stDebug("vgId:%d process hbMsg rsp, msgId:%d rsp confirmed", pMeta->vgId, pRsp->msgId);
SMetaHbInfo* pInfo = pMeta->pHbInfo;
- streamMetaRLock(pMeta);
+ streamMetaWLock(pMeta);
// current waiting rsp recved
if (pRsp->msgId == pInfo->hbCount) {
@@ -337,6 +345,6 @@ int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp) {
stWarn("vgId:%d recv expired hb rsp, msgId:%d, discarded", pMeta->vgId, pRsp->msgId);
}
- streamMetaRUnLock(pMeta);
+ streamMetaWUnLock(pMeta);
return TSDB_CODE_SUCCESS;
}
\ No newline at end of file
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 7b94f642e2..d0b1f6ca93 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -182,9 +182,10 @@ int32_t streamMetaCheckBackendCompatible(SStreamMeta* pMeta) {
int32_t streamMetaCvtDbFormat(SStreamMeta* pMeta) {
int32_t code = 0;
int64_t chkpId = streamMetaGetLatestCheckpointId(pMeta);
-
- bool exist = streamBackendDataIsExist(pMeta->path, chkpId, pMeta->vgId);
+ terrno = 0;
+ bool exist = streamBackendDataIsExist(pMeta->path, chkpId);
if (exist == false) {
+ code = terrno;
return code;
}
@@ -252,8 +253,9 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key)
}
STaskDbWrapper* pBackend = NULL;
+ int64_t processVer = -1;
while (1) {
- pBackend = taskDbOpen(pMeta->path, key, chkpId);
+ pBackend = taskDbOpen(pMeta->path, key, chkpId, &processVer);
if (pBackend != NULL) {
break;
}
@@ -271,6 +273,8 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key)
pBackend->pTask = pTask;
pBackend->pMeta = pMeta;
+ if (processVer != -1) pTask->chkInfo.processedVer = processVer;
+
taosHashPut(pMeta->pTaskDbUnique, key, strlen(key), &pBackend, sizeof(void*));
taosThreadMutexUnlock(&pMeta->backendMutex);
@@ -308,7 +312,8 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTas
}
if (streamMetaMayCvtDbFormat(pMeta) < 0) {
- stError("vgId:%d convert sub info format failed, open stream meta failed", pMeta->vgId);
+ stError("vgId:%d convert sub info format failed, open stream meta failed, reason: %s", pMeta->vgId,
+ tstrerror(terrno));
goto _err;
}
@@ -393,6 +398,9 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTas
pMeta->qHandle = taosInitScheduler(32, 1, "stream-chkp", NULL);
pMeta->bkdChkptMgt = bkdMgtCreate(tpath);
+ if (pMeta->bkdChkptMgt == NULL) {
+ goto _err;
+ }
taosThreadMutexInit(&pMeta->backendMutex, NULL);
return pMeta;
@@ -408,9 +416,10 @@ _err:
if (pMeta->updateInfo.pTasks) taosHashCleanup(pMeta->updateInfo.pTasks);
if (pMeta->startInfo.pReadyTaskSet) taosHashCleanup(pMeta->startInfo.pReadyTaskSet);
if (pMeta->startInfo.pFailedTaskSet) taosHashCleanup(pMeta->startInfo.pFailedTaskSet);
+ if (pMeta->bkdChkptMgt) bkdMgtDestroy(pMeta->bkdChkptMgt);
taosMemoryFree(pMeta);
- stError("failed to open stream meta");
+ stError("failed to open stream meta, reason:%s", tstrerror(terrno));
return NULL;
}
@@ -900,7 +909,7 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) {
if (p == NULL) {
code = pMeta->buildTaskFn(pMeta->ahandle, pTask, pTask->chkInfo.checkpointVer + 1);
if (code < 0) {
- stError("failed to load s-task:0x%"PRIx64", code:%s, continue", id.taskId, tstrerror(terrno));
+ stError("failed to load s-task:0x%" PRIx64 ", code:%s, continue", id.taskId, tstrerror(terrno));
tFreeStreamTask(pTask);
continue;
}
@@ -985,7 +994,7 @@ void streamMetaNotifyClose(SStreamMeta* pMeta) {
streamMetaGetHbSendInfo(pMeta->pHbInfo, &startTs, &sendCount);
stInfo("vgId:%d notify all stream tasks that current vnode is closing. isLeader:%d startHb:%" PRId64 ", totalHb:%d",
- vgId, (pMeta->role == NODE_ROLE_LEADER), startTs, sendCount);
+ vgId, (pMeta->role == NODE_ROLE_LEADER), startTs, sendCount);
// wait for the stream meta hb function stopping
streamMetaWaitForHbTmrQuit(pMeta);
@@ -1031,10 +1040,11 @@ void streamMetaResetStartInfo(STaskStartInfo* pStartInfo, int32_t vgId) {
taosHashClear(pStartInfo->pFailedTaskSet);
pStartInfo->tasksWillRestart = 0;
pStartInfo->readyTs = 0;
+ pStartInfo->elapsedTime = 0;
// reset the sentinel flag value to be 0
pStartInfo->startAllTasks = 0;
- stDebug("vgId:%d clear all start-all-task info", vgId);
+ stDebug("vgId:%d clear start-all-task info", vgId);
}
void streamMetaRLock(SStreamMeta* pMeta) {
@@ -1170,7 +1180,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) {
int64_t now = taosGetTimestampMs();
int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList);
- stInfo("vgId:%d start to consensus checkpointId for all %d task(s), start ts:%"PRId64, vgId, numOfTasks, now);
+ stInfo("vgId:%d start to consensus checkpointId for all %d task(s), start ts:%" PRId64, vgId, numOfTasks, now);
if (numOfTasks == 0) {
stInfo("vgId:%d no tasks exist, quit from consensus checkpointId", pMeta->vgId);
@@ -1198,7 +1208,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) {
continue;
}
- if ((pTask->pBackend == NULL) && (pTask->info.fillHistory == 1 || HAS_RELATED_FILLHISTORY_TASK(pTask))) {
+ if ((pTask->pBackend == NULL) && ((pTask->info.fillHistory == 1) || HAS_RELATED_FILLHISTORY_TASK(pTask))) {
code = pMeta->expandTaskFn(pTask);
if (code != TSDB_CODE_SUCCESS) {
stError("s-task:0x%x vgId:%d failed to expand stream backend", pTaskId->taskId, vgId);
@@ -1344,11 +1354,18 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas
}
ASSERT(pTask->status.downstreamReady == 0);
+
+ // avoid initialization and destroy running concurrently.
+ taosThreadMutexLock(&pTask->lock);
if (pTask->pBackend == NULL) {
code = pMeta->expandTaskFn(pTask);
+ taosThreadMutexUnlock(&pTask->lock);
+
if (code != TSDB_CODE_SUCCESS) {
streamMetaAddFailedTaskSelf(pTask, pInfo->readyTs);
}
+ } else {
+ taosThreadMutexUnlock(&pTask->lock);
}
if (code == TSDB_CODE_SUCCESS) {
@@ -1392,17 +1409,24 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3
streamMetaWLock(pMeta);
- if (pStartInfo->startAllTasks != 1) {
- int64_t el = endTs - startTs;
- stDebug("vgId:%d not start all task(s), not record status, s-task:0x%x launch succ:%d elapsed time:%" PRId64 "ms",
- pMeta->vgId, taskId, ready, el);
+ SStreamTask** p = taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
+ if (p == NULL) { // task does not exists in current vnode, not record the complete info
+ stError("vgId:%d s-task:0x%x not exists discard the check downstream info", pMeta->vgId, taskId);
streamMetaWUnLock(pMeta);
return 0;
}
- void* p = taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
- if (p == NULL) { // task does not exists in current vnode, not record the complete info
- stError("vgId:%d s-task:0x%x not exists discard the check downstream info", pMeta->vgId, taskId);
+ // clear the send consensus-checkpointId flag
+ taosThreadMutexLock(&(*p)->lock);
+ (*p)->status.sendConsensusChkptId = false;
+ taosThreadMutexUnlock(&(*p)->lock);
+
+ if (pStartInfo->startAllTasks != 1) {
+ int64_t el = endTs - startTs;
+ stDebug(
+ "vgId:%d not in start all task(s) process, not record launch result status, s-task:0x%x launch succ:%d elapsed "
+ "time:%" PRId64 "ms",
+ pMeta->vgId, taskId, ready, el);
streamMetaWUnLock(pMeta);
return 0;
}
diff --git a/source/libs/stream/src/streamMsg.c b/source/libs/stream/src/streamMsg.c
index e0435156e2..40582b5144 100644
--- a/source/libs/stream/src/streamMsg.c
+++ b/source/libs/stream/src/streamMsg.c
@@ -349,6 +349,8 @@ int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pReq) {
if (tEncodeI64(pEncoder, ps->checkpointInfo.latestTime) < 0) return -1;
if (tEncodeI64(pEncoder, ps->checkpointInfo.latestSize) < 0) return -1;
if (tEncodeI8(pEncoder, ps->checkpointInfo.remoteBackup) < 0) return -1;
+ if (tEncodeI8(pEncoder, ps->checkpointInfo.consensusChkptId) < 0) return -1;
+ if (tEncodeI64(pEncoder, ps->checkpointInfo.consensusTs) < 0) return -1;
if (tEncodeI64(pEncoder, ps->startTime) < 0) return -1;
if (tEncodeI64(pEncoder, ps->startCheckpointId) < 0) return -1;
if (tEncodeI64(pEncoder, ps->startCheckpointVer) < 0) return -1;
@@ -403,6 +405,8 @@ int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) {
if (tDecodeI64(pDecoder, &entry.checkpointInfo.latestTime) < 0) return -1;
if (tDecodeI64(pDecoder, &entry.checkpointInfo.latestSize) < 0) return -1;
if (tDecodeI8(pDecoder, &entry.checkpointInfo.remoteBackup) < 0) return -1;
+ if (tDecodeI8(pDecoder, &entry.checkpointInfo.consensusChkptId) < 0) return -1;
+ if (tDecodeI64(pDecoder, &entry.checkpointInfo.consensusTs) < 0) return -1;
if (tDecodeI64(pDecoder, &entry.startTime) < 0) return -1;
if (tDecodeI64(pDecoder, &entry.startCheckpointId) < 0) return -1;
if (tDecodeI64(pDecoder, &entry.startCheckpointVer) < 0) return -1;
@@ -634,6 +638,7 @@ int32_t tEncodeRestoreCheckpointInfo (SEncoder* pEncoder, const SRestoreCheckpoi
if (tEncodeI64(pEncoder, pReq->startTs) < 0) return -1;
if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
if (tEncodeI64(pEncoder, pReq->checkpointId) < 0) return -1;
+ if (tEncodeI32(pEncoder, pReq->transId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->nodeId) < 0) return -1;
tEndEncode(pEncoder);
@@ -645,28 +650,9 @@ int32_t tDecodeRestoreCheckpointInfo(SDecoder* pDecoder, SRestoreCheckpointInfo*
if (tDecodeI64(pDecoder, &pReq->startTs) < 0) return -1;
if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
if (tDecodeI64(pDecoder, &pReq->checkpointId) < 0) return -1;
+ if (tDecodeI32(pDecoder, &pReq->transId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->nodeId) < 0) return -1;
tEndDecode(pDecoder);
return 0;
}
-
-int32_t tEncodeRestoreCheckpointInfoRsp(SEncoder* pCoder, const SRestoreCheckpointInfoRsp* pInfo) {
- if (tStartEncode(pCoder) < 0) return -1;
- if (tEncodeI64(pCoder, pInfo->startTs) < 0) return -1;
- if (tEncodeI64(pCoder, pInfo->streamId) < 0) return -1;
- if (tEncodeI32(pCoder, pInfo->taskId) < 0) return -1;
- if (tEncodeI64(pCoder, pInfo->checkpointId) < 0) return -1;
- tEndEncode(pCoder);
- return 0;
-}
-
-int32_t tDecodeRestoreCheckpointInfoRsp(SDecoder* pCoder, SRestoreCheckpointInfoRsp* pInfo) {
- if (tStartDecode(pCoder) < 0) return -1;
- if (tDecodeI64(pCoder, &pInfo->startTs) < 0) return -1;
- if (tDecodeI64(pCoder, &pInfo->streamId) < 0) return -1;
- if (tDecodeI32(pCoder, &pInfo->taskId) < 0) return -1;
- if (tDecodeI64(pCoder, &pInfo->checkpointId) < 0) return -1;
- tEndDecode(pCoder);
- return 0;
-}
\ No newline at end of file
diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c
index feea9f048e..37210434d2 100644
--- a/source/libs/stream/src/streamSessionState.c
+++ b/source/libs/stream/src/streamSessionState.c
@@ -240,7 +240,6 @@ _end:
int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen) {
SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState);
- memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey));
pNewPos->needFree = true;
pNewPos->beFlushed = true;
void* pBuff = NULL;
@@ -248,6 +247,7 @@ int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, v
if (code != TSDB_CODE_SUCCESS) {
return code;
}
+ memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey));
memcpy(pNewPos->pRowBuff, pBuff, *pVLen);
taosMemoryFreeClear(pBuff);
(*pVal) = pNewPos;
diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c
index 0871ff5eb7..02e4ed8d8b 100644
--- a/source/libs/stream/src/streamSnapshot.c
+++ b/source/libs/stream/src/streamSnapshot.c
@@ -24,6 +24,7 @@ enum SBackendFileType {
ROCKSDB_SST_TYPE = 3,
ROCKSDB_CURRENT_TYPE = 4,
ROCKSDB_CHECKPOINT_META_TYPE = 5,
+ ROCKSDB_CHECKPOINT_SELFCHECK_TYPE = 6,
};
typedef struct SBackendFileItem {
@@ -49,6 +50,7 @@ typedef struct SBackendSnapFiles2 {
char* pOptions;
SArray* pSst;
char* pCheckpointMeta;
+ char* pCheckpointSelfcheck;
char* path;
int64_t checkpointId;
@@ -111,6 +113,7 @@ const char* ROCKSDB_MAINFEST = "MANIFEST";
const char* ROCKSDB_SST = "sst";
const char* ROCKSDB_CURRENT = "CURRENT";
const char* ROCKSDB_CHECKPOINT_META = "CHECKPOINT";
+const char* ROCKSDB_CHECKPOINT_SELF_CHECK = "info";
static int64_t kBlockSize = 64 * 1024;
int32_t streamSnapHandleInit(SStreamSnapHandle* handle, char* path, void* pMeta);
@@ -127,6 +130,7 @@ int32_t streamGetFileSize(char* path, char* name, int64_t* sz) {
int32_t ret = 0;
char* fullname = taosMemoryCalloc(1, strlen(path) + 32);
+
sprintf(fullname, "%s%s%s", path, TD_DIRSEP, name);
ret = taosStatFile(fullname, sz, NULL, NULL);
@@ -148,8 +152,20 @@ int32_t streamDestroyTaskDbSnapInfo(void* arg, SArray* snap) { return taskDbDest
void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) {
if (qDebugFlag & DEBUG_DEBUG) {
- char* buf = taosMemoryCalloc(1, 512);
- sprintf(buf + strlen(buf), "[");
+ int16_t cap = 512;
+
+ char* buf = taosMemoryCalloc(1, cap);
+ if (buf == NULL) {
+ stError("%s failed to alloc memory, reason:%s", STREAM_STATE_TRANSFER, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
+ return;
+ }
+
+ int32_t nBytes = snprintf(buf + strlen(buf), cap, "[");
+ if (nBytes <= 0 || nBytes >= cap) {
+ taosMemoryFree(buf);
+ stError("%s failed to write buf, reason:%s", STREAM_STATE_TRANSFER, tstrerror(TSDB_CODE_OUT_OF_RANGE));
+ return;
+ }
if (pSnapFile->pCurrent) sprintf(buf, "current: %s,", pSnapFile->pCurrent);
if (pSnapFile->pMainfest) sprintf(buf + strlen(buf), "MANIFEST: %s,", pSnapFile->pMainfest);
@@ -157,10 +173,10 @@ void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) {
if (pSnapFile->pSst) {
for (int32_t i = 0; i < taosArrayGetSize(pSnapFile->pSst); i++) {
char* name = taosArrayGetP(pSnapFile->pSst, i);
- sprintf(buf + strlen(buf), "%s,", name);
+ if (strlen(buf) + strlen(name) < cap) sprintf(buf + strlen(buf), "%s,", name);
}
}
- sprintf(buf + strlen(buf) - 1, "]");
+ if ((strlen(buf)) < cap) sprintf(buf + strlen(buf) - 1, "]");
stInfo("%s %" PRId64 "-%" PRId64 " get file list: %s", STREAM_STATE_TRANSFER, pSnapFile->snapInfo.streamId,
pSnapFile->snapInfo.taskId, buf);
@@ -199,16 +215,25 @@ int32_t snapFileGenMeta(SBackendSnapFile2* pSnapFile) {
// meta
item.name = pSnapFile->pCheckpointMeta;
item.type = ROCKSDB_CHECKPOINT_META_TYPE;
+ if (streamGetFileSize(pSnapFile->path, item.name, &item.size) == 0) {
+ taosArrayPush(pSnapFile->pFileList, &item);
+ }
+
+ item.name = pSnapFile->pCheckpointSelfcheck;
+ item.type = ROCKSDB_CHECKPOINT_SELFCHECK_TYPE;
+
if (streamGetFileSize(pSnapFile->path, item.name, &item.size) == 0) {
taosArrayPush(pSnapFile->pFileList, &item);
}
return 0;
}
int32_t snapFileReadMeta(SBackendSnapFile2* pSnapFile) {
+ int32_t code = 0;
TdDirPtr pDir = taosOpenDir(pSnapFile->path);
if (NULL == pDir) {
- stError("%s failed to open %s", STREAM_STATE_TRANSFER, pSnapFile->path);
- return -1;
+ code = TAOS_SYSTEM_ERROR(errno);
+ stError("%s failed to open %s, reason:%s", STREAM_STATE_TRANSFER, pSnapFile->path, tstrerror(code));
+ return code;
}
TdDirEntryPtr pDirEntry;
@@ -216,43 +241,88 @@ int32_t snapFileReadMeta(SBackendSnapFile2* pSnapFile) {
char* name = taosGetDirEntryName(pDirEntry);
if (strlen(name) >= strlen(ROCKSDB_CURRENT) && 0 == strncmp(name, ROCKSDB_CURRENT, strlen(ROCKSDB_CURRENT))) {
pSnapFile->pCurrent = taosStrdup(name);
+ if (pSnapFile->pCurrent == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ break;
+ }
continue;
}
if (strlen(name) >= strlen(ROCKSDB_MAINFEST) && 0 == strncmp(name, ROCKSDB_MAINFEST, strlen(ROCKSDB_MAINFEST))) {
pSnapFile->pMainfest = taosStrdup(name);
+ if (pSnapFile->pMainfest == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ break;
+ }
continue;
}
if (strlen(name) >= strlen(ROCKSDB_OPTIONS) && 0 == strncmp(name, ROCKSDB_OPTIONS, strlen(ROCKSDB_OPTIONS))) {
pSnapFile->pOptions = taosStrdup(name);
+ if (pSnapFile->pOptions == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ break;
+ }
continue;
}
if (strlen(name) >= strlen(ROCKSDB_CHECKPOINT_META) &&
0 == strncmp(name, ROCKSDB_CHECKPOINT_META, strlen(ROCKSDB_CHECKPOINT_META))) {
pSnapFile->pCheckpointMeta = taosStrdup(name);
+ if (pSnapFile->pCheckpointMeta == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ break;
+ }
+ continue;
+ }
+ if (strlen(name) >= strlen(ROCKSDB_CHECKPOINT_SELF_CHECK) &&
+ 0 == strncmp(name, ROCKSDB_CHECKPOINT_SELF_CHECK, strlen(ROCKSDB_CHECKPOINT_SELF_CHECK))) {
+ pSnapFile->pCheckpointSelfcheck = taosStrdup(name);
+ if (pSnapFile->pCheckpointSelfcheck == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ break;
+ }
continue;
}
if (strlen(name) >= strlen(ROCKSDB_SST) &&
0 == strncmp(name + strlen(name) - strlen(ROCKSDB_SST), ROCKSDB_SST, strlen(ROCKSDB_SST))) {
char* sst = taosStrdup(name);
+ if (sst == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ break;
+ }
taosArrayPush(pSnapFile->pSst, &sst);
}
}
taosCloseDir(&pDir);
- return 0;
+ return code;
}
int32_t streamBackendSnapInitFile(char* metaPath, SStreamTaskSnap* pSnap, SBackendSnapFile2* pSnapFile) {
- int32_t code = -1;
+ int32_t code = 0;
+ int32_t nBytes = 0;
+ int32_t cap = strlen(pSnap->dbPrefixPath) + 256;
+
+ char* path = taosMemoryCalloc(1, cap);
+ if (path == NULL) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
+ nBytes = snprintf(path, cap, "%s%s%s%s%s%" PRId64 "", pSnap->dbPrefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP,
+ "checkpoint", pSnap->chkpId);
+ if (nBytes <= 0 || nBytes >= cap) {
+ code = TSDB_CODE_OUT_OF_RANGE;
+ goto _ERROR;
+ }
- char* path = taosMemoryCalloc(1, strlen(pSnap->dbPrefixPath) + 256);
- // char idstr[64] = {0};
- sprintf(path, "%s%s%s%s%s%" PRId64 "", pSnap->dbPrefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint",
- pSnap->chkpId);
if (!taosIsDir(path)) {
+ code = TSDB_CODE_INVALID_MSG;
goto _ERROR;
}
pSnapFile->pSst = taosArrayInit(16, sizeof(void*));
pSnapFile->pFileList = taosArrayInit(64, sizeof(SBackendFileItem));
+ if (pSnapFile->pSst == NULL || pSnapFile->pFileList == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _ERROR;
+ }
+
pSnapFile->path = path;
pSnapFile->snapInfo = *pSnap;
if ((code = snapFileReadMeta(pSnapFile)) != 0) {
@@ -264,7 +334,6 @@ int32_t streamBackendSnapInitFile(char* metaPath, SStreamTaskSnap* pSnap, SBacke
snapFileDebugInfo(pSnapFile);
path = NULL;
- code = 0;
_ERROR:
taosMemoryFree(path);
@@ -276,6 +345,7 @@ void snapFileDestroy(SBackendSnapFile2* pSnap) {
taosMemoryFree(pSnap->pMainfest);
taosMemoryFree(pSnap->pOptions);
taosMemoryFree(pSnap->path);
+ taosMemoryFree(pSnap->pCheckpointSelfcheck);
for (int32_t i = 0; i < taosArrayGetSize(pSnap->pSst); i++) {
char* sst = taosArrayGetP(pSnap->pSst, i);
taosMemoryFree(sst);
@@ -295,14 +365,25 @@ void snapFileDestroy(SBackendSnapFile2* pSnap) {
}
int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, void* pMeta) {
// impl later
+ int32_t code = 0;
SArray* pSnapInfoSet = taosArrayInit(4, sizeof(SStreamTaskSnap));
- int32_t code = streamCreateTaskDbSnapInfo(pMeta, path, pSnapInfoSet);
+ if (pSnapInfoSet == NULL) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
+ code = streamCreateTaskDbSnapInfo(pMeta, path, pSnapInfoSet);
if (code != 0) {
+ stError("failed to do task db snap info, reason:%s", tstrerror(code));
taosArrayDestroy(pSnapInfoSet);
- return -1;
+ return code;
}
SArray* pDbSnapSet = taosArrayInit(8, sizeof(SBackendSnapFile2));
+ if (pDbSnapSet == NULL) {
+ taosArrayDestroy(pSnapInfoSet);
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ return code;
+ }
for (int32_t i = 0; i < taosArrayGetSize(pSnapInfoSet); i++) {
SStreamTaskSnap* pSnap = taosArrayGet(pSnapInfoSet, i);
@@ -318,6 +399,10 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, void* pMeta
pHandle->currIdx = 0;
pHandle->pMeta = pMeta;
return 0;
+
+_err:
+ streamSnapHandleDestroy(pHandle);
+ return code;
}
void streamSnapHandleDestroy(SStreamSnapHandle* handle) {
@@ -348,9 +433,10 @@ int32_t streamSnapReaderOpen(void* pMeta, int64_t sver, int64_t chkpId, char* pa
return TSDB_CODE_OUT_OF_MEMORY;
}
- if (streamSnapHandleInit(&pReader->handle, (char*)path, pMeta) < 0) {
+ int32_t code = streamSnapHandleInit(&pReader->handle, (char*)path, pMeta);
+ if (code != 0) {
taosMemoryFree(pReader);
- return -1;
+ return code;
}
*ppReader = pReader;
@@ -410,10 +496,10 @@ _NEXT:
int64_t nread = taosPReadFile(pSnapFile->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pSnapFile->offset);
if (nread == -1) {
taosMemoryFree(buf);
- code = TAOS_SYSTEM_ERROR(terrno);
+ code = TAOS_SYSTEM_ERROR(errno);
stError("%s snap failed to read snap, file name:%s, type:%d,reason:%s", STREAM_STATE_TRANSFER, item->name,
item->type, tstrerror(code));
- return -1;
+ return code;
} else if (nread > 0 && nread <= kBlockSize) {
// left bytes less than kBlockSize
stDebug("%s read file %s, current offset:%" PRId64 ",size:% " PRId64 ", file no.%d", STREAM_STATE_TRANSFER,
@@ -473,6 +559,7 @@ _NEXT:
// SMetaSnapWriter ========================================
int32_t streamSnapWriterOpen(void* pMeta, int64_t sver, int64_t ever, char* path, SStreamSnapWriter** ppWriter) {
// impl later
+ int32_t code = 0;
SStreamSnapWriter* pWriter = taosMemoryCalloc(1, sizeof(SStreamSnapWriter));
if (pWriter == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
@@ -480,11 +567,27 @@ int32_t streamSnapWriterOpen(void* pMeta, int64_t sver, int64_t ever, char* path
SStreamSnapHandle* pHandle = &pWriter->handle;
pHandle->currIdx = 0;
+
pHandle->metaPath = taosStrdup(path);
+ if (pHandle->metaPath == NULL) {
+ taosMemoryFree(pWriter);
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ return code;
+ }
+
pHandle->pDbSnapSet = taosArrayInit(8, sizeof(SBackendSnapFile2));
+ if (pHandle->pDbSnapSet == NULL) {
+ streamSnapWriterClose(pWriter, 0);
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ return code;
+ }
SBackendSnapFile2 snapFile = {0};
- taosArrayPush(pHandle->pDbSnapSet, &snapFile);
+ if (taosArrayPush(pHandle->pDbSnapSet, &snapFile) == NULL) {
+ streamSnapWriterClose(pWriter, 0);
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ return code;
+ }
*ppWriter = pWriter;
return 0;
@@ -506,7 +609,7 @@ int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t
if (pSnapFile->fd == 0) {
pSnapFile->fd = streamOpenFile(pSnapFile->path, pItem->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pSnapFile->fd == NULL) {
- code = TAOS_SYSTEM_ERROR(terrno);
+ code = TAOS_SYSTEM_ERROR(errno);
stError("%s failed to open file name:%s%s%s, reason:%s", STREAM_STATE_TRANSFER, pHandle->metaPath, TD_DIRSEP,
pHdr->name, tstrerror(code));
}
@@ -514,7 +617,7 @@ int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t
if (strlen(pHdr->name) == strlen(pItem->name) && strcmp(pHdr->name, pItem->name) == 0) {
int64_t bytes = taosPWriteFile(pSnapFile->fd, pHdr->data, pHdr->size, pSnapFile->offset);
if (bytes != pHdr->size) {
- code = TAOS_SYSTEM_ERROR(terrno);
+ code = TAOS_SYSTEM_ERROR(errno);
stError("%s failed to write snap, file name:%s, reason:%s", STREAM_STATE_TRANSFER, pHdr->name, tstrerror(code));
return code;
} else {
@@ -535,12 +638,16 @@ int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t
SBackendFileItem* pItem = taosArrayGet(pSnapFile->pFileList, pSnapFile->currFileIdx);
pSnapFile->fd = streamOpenFile(pSnapFile->path, pItem->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pSnapFile->fd == NULL) {
- code = TAOS_SYSTEM_ERROR(terrno);
+ code = TAOS_SYSTEM_ERROR(errno);
stError("%s failed to open file name:%s%s%s, reason:%s", STREAM_STATE_TRANSFER, pSnapFile->path, TD_DIRSEP,
pHdr->name, tstrerror(code));
}
- taosPWriteFile(pSnapFile->fd, pHdr->data, pHdr->size, pSnapFile->offset);
+ if (taosPWriteFile(pSnapFile->fd, pHdr->data, pHdr->size, pSnapFile->offset) != pHdr->size) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ stError("%s failed to write snap, file name:%s, reason:%s", STREAM_STATE_TRANSFER, pHdr->name, tstrerror(code));
+ return code;
+ }
stInfo("succ to write data %s", pItem->name);
pSnapFile->offset += pHdr->size;
}
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index 1decfe198a..9bcad87264 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -268,13 +268,7 @@ void tFreeStreamTask(SStreamTask* pTask) {
}
streamTaskCleanupCheckInfo(&pTask->taskCheckInfo);
-
- if (pTask->pState) {
- stDebug("s-task:0x%x start to free task state", taskId);
- streamStateClose(pTask->pState, status1 == TASK_STATUS__DROPPING);
- taskDbRemoveRef(pTask->pBackend);
- pTask->pBackend = NULL;
- }
+ streamFreeTaskState(pTask, status1);
if (pTask->pNameMap) {
tSimpleHashCleanup(pTask->pNameMap);
@@ -311,6 +305,16 @@ void tFreeStreamTask(SStreamTask* pTask) {
stDebug("s-task:0x%x free task completed", taskId);
}
+void streamFreeTaskState(SStreamTask* pTask, ETaskStatus status) {
+ if (pTask->pState != NULL) {
+ stDebug("s-task:0x%x start to free task state", pTask->id.taskId);
+ streamStateClose(pTask->pState, status == TASK_STATUS__DROPPING);
+ taskDbRemoveRef(pTask->pBackend);
+ pTask->pBackend = NULL;
+ pTask->pState = NULL;
+ }
+}
+
static void setInitialVersionInfo(SStreamTask* pTask, int64_t ver) {
SCheckpointInfo* pChkInfo = &pTask->chkInfo;
SDataRange* pRange = &pTask->dataRange;
@@ -848,6 +852,8 @@ STaskStatusEntry streamTaskGetStatusEntry(SStreamTask* pTask) {
.checkpointInfo.latestTime = pTask->chkInfo.checkpointTime,
.checkpointInfo.latestSize = 0,
.checkpointInfo.remoteBackup = 0,
+ .checkpointInfo.consensusChkptId = 0,
+ .checkpointInfo.consensusTs = taosGetTimestampMs(),
.hTaskId = pTask->hTaskInfo.id.taskId,
.procsTotal = SIZE_IN_MiB(pExecInfo->inputDataSize),
.outputTotal = SIZE_IN_MiB(pExecInfo->outputDataSize),
diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c
index 85d3e0068a..f2bd99cdaf 100644
--- a/source/libs/stream/src/streamTaskSm.c
+++ b/source/libs/stream/src/streamTaskSm.c
@@ -79,6 +79,12 @@ static int32_t attachWaitedEvent(SStreamTask* pTask, SFutureHandleEventInfo* pEv
return 0;
}
+static int32_t stopTaskSuccFn(SStreamTask* pTask) {
+ SStreamTaskSM* pSM = pTask->status.pSM;
+ streamFreeTaskState(pTask, pSM->current.state);
+ return TSDB_CODE_SUCCESS;
+}
+
int32_t streamTaskInitStatus(SStreamTask* pTask) {
pTask->execInfo.checkTs = taosGetTimestampMs();
stDebug("s-task:%s start init, and check downstream tasks, set the init ts:%" PRId64, pTask->id.idStr,
@@ -634,21 +640,21 @@ void doInitStateTransferTable(void) {
// resume is completed by restore status of state-machine
// stop related event
- trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL);
+ trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL);
taosArrayPush(streamTaskSMTrans, &trans);
- trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL);
+ trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL);
taosArrayPush(streamTaskSMTrans, &trans);
- trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL);
+ trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL);
taosArrayPush(streamTaskSMTrans, &trans);
- trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL);
+ trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL);
taosArrayPush(streamTaskSMTrans, &trans);
- trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL);
+ trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL);
taosArrayPush(streamTaskSMTrans, &trans);
- trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL);
+ trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL);
taosArrayPush(streamTaskSMTrans, &trans);
- trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL);
+ trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL);
taosArrayPush(streamTaskSMTrans, &trans);
- trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL);
+ trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL);
taosArrayPush(streamTaskSMTrans, &trans);
// dropping related event
diff --git a/source/libs/stream/test/backendTest.cpp b/source/libs/stream/test/backendTest.cpp
index 2fb257fe4e..38d48a2a32 100644
--- a/source/libs/stream/test/backendTest.cpp
+++ b/source/libs/stream/test/backendTest.cpp
@@ -29,7 +29,7 @@ class BackendEnv : public ::testing::Test {
void *backendCreate() {
const char *streamPath = "/tmp";
- void * p = NULL;
+ void *p = NULL;
// char *absPath = NULL;
// // SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(streamPath, -1, 2);
@@ -52,7 +52,7 @@ SStreamState *stateCreate(const char *path) {
}
void *backendOpen() {
streamMetaInit();
- const char * path = "/tmp/backend";
+ const char *path = "/tmp/backend";
SStreamState *p = stateCreate(path);
ASSERT(p != NULL);
@@ -79,7 +79,7 @@ void *backendOpen() {
const char *val = "value data";
int32_t len = 0;
- char * newVal = NULL;
+ char *newVal = NULL;
streamStateGet_rocksdb(p, &key, (void **)&newVal, &len);
ASSERT(len == strlen(val));
}
@@ -100,7 +100,7 @@ void *backendOpen() {
const char *val = "value data";
int32_t len = 0;
- char * newVal = NULL;
+ char *newVal = NULL;
int32_t code = streamStateGet_rocksdb(p, &key, (void **)&newVal, &len);
ASSERT(code != 0);
}
@@ -130,7 +130,7 @@ void *backendOpen() {
winkey.groupId = 0;
winkey.ts = tsArray[0];
- char * val = NULL;
+ char *val = NULL;
int32_t len = 0;
pCurr = streamStateSeekKeyNext_rocksdb(p, &winkey);
@@ -157,7 +157,7 @@ void *backendOpen() {
key.ts = tsArray[i];
key.exprIdx = i;
- char * val = NULL;
+ char *val = NULL;
int32_t len = 0;
streamStateFuncGet_rocksdb(p, &key, (void **)&val, &len);
ASSERT(len == strlen("Value"));
@@ -168,7 +168,7 @@ void *backendOpen() {
key.ts = tsArray[i];
key.exprIdx = i;
- char * val = NULL;
+ char *val = NULL;
int32_t len = 0;
streamStateFuncDel_rocksdb(p, &key);
}
@@ -213,7 +213,7 @@ void *backendOpen() {
{
SSessionKey key;
memset(&key, 0, sizeof(key));
- char * val = NULL;
+ char *val = NULL;
int32_t vlen = 0;
code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen);
ASSERT(code == 0);
@@ -260,7 +260,7 @@ void *backendOpen() {
SWinKey key = {0}; // {.groupId = (uint64_t)(i), .ts = tsArray[i]};
key.groupId = (uint64_t)(i);
key.ts = tsArray[i];
- char * val = NULL;
+ char *val = NULL;
int32_t vlen = 0;
ASSERT(streamStateFillGet_rocksdb(p, &key, (void **)&val, &vlen) == 0);
taosMemoryFreeClear(val);
@@ -272,7 +272,7 @@ void *backendOpen() {
SStreamStateCur *pCurr = streamStateFillGetCur_rocksdb(p, &key);
ASSERT(pCurr != NULL);
- char * val = NULL;
+ char *val = NULL;
int32_t vlen = 0;
ASSERT(0 == streamStateFillGetKVByCur_rocksdb(pCurr, &key, (const void **)&val, &vlen));
ASSERT(vlen == strlen("Value"));
@@ -296,7 +296,7 @@ void *backendOpen() {
SWinKey key = {0}; // {.groupId = (uint64_t)(i), .ts = tsArray[i]};
key.groupId = (uint64_t)(i);
key.ts = tsArray[i];
- char * val = NULL;
+ char *val = NULL;
int32_t vlen = 0;
ASSERT(streamStateFillDel_rocksdb(p, &key) == 0);
taosMemoryFreeClear(val);
@@ -338,7 +338,7 @@ void *backendOpen() {
char key[128] = {0};
sprintf(key, "tbname_%d", i);
- char * val = NULL;
+ char *val = NULL;
int32_t len = 0;
code = streamDefaultGet_rocksdb(p, key, (void **)&val, &len);
ASSERT(code == 0);
@@ -354,7 +354,7 @@ TEST_F(BackendEnv, checkOpen) {
SStreamState *p = (SStreamState *)backendOpen();
int64_t tsStart = taosGetTimestampMs();
{
- void * pBatch = streamStateCreateBatch();
+ void *pBatch = streamStateCreateBatch();
int32_t size = 0;
for (int i = 0; i < size; i++) {
char key[128] = {0};
@@ -368,7 +368,7 @@ TEST_F(BackendEnv, checkOpen) {
streamStateDestroyBatch(pBatch);
}
{
- void * pBatch = streamStateCreateBatch();
+ void *pBatch = streamStateCreateBatch();
int32_t size = 0;
char valBuf[256] = {0};
for (int i = 0; i < size; i++) {
@@ -383,9 +383,9 @@ TEST_F(BackendEnv, checkOpen) {
streamStateDestroyBatch(pBatch);
}
// do checkpoint 2
- taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 2);
+ taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 2, 0);
{
- void * pBatch = streamStateCreateBatch();
+ void *pBatch = streamStateCreateBatch();
int32_t size = 0;
char valBuf[256] = {0};
for (int i = 0; i < size; i++) {
@@ -400,17 +400,17 @@ TEST_F(BackendEnv, checkOpen) {
streamStateDestroyBatch(pBatch);
}
- taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 3);
+ taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 3, 0);
const char *path = "/tmp/backend/stream";
const char *dump = "/tmp/backend/stream/dump";
// taosMkDir(dump);
taosMulMkDir(dump);
SBkdMgt *mgt = bkdMgtCreate((char *)path);
- SArray * result = taosArrayInit(4, sizeof(void *));
+ SArray *result = taosArrayInit(4, sizeof(void *));
bkdMgtGetDelta(mgt, p->pTdbState->idstr, 3, result, (char *)dump);
- taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 4);
+ taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 4, 0);
taosArrayClear(result);
bkdMgtGetDelta(mgt, p->pTdbState->idstr, 4, result, (char *)dump);
diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index ffcb1fbdb5..9818394a2a 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -832,6 +832,9 @@ static int32_t allocConnRef(SCliConn* conn, bool update) {
taosInitRWLatch(&exh->latch);
exh->refId = transAddExHandle(transGetRefMgt(), exh);
+ SExHandle* self = transAcquireExHandle(transGetRefMgt(), exh->refId);
+ ASSERT(exh == self);
+
QUEUE_INIT(&exh->q);
taosInitRWLatch(&exh->latch);
@@ -2829,10 +2832,11 @@ int transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) {
int64_t transAllocHandle() {
SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle));
- QUEUE_INIT(&exh->q);
- taosInitRWLatch(&exh->latch);
exh->refId = transAddExHandle(transGetRefMgt(), exh);
+ SExHandle* self = transAcquireExHandle(transGetRefMgt(), exh->refId);
+ ASSERT(exh == self);
+
QUEUE_INIT(&exh->q);
taosInitRWLatch(&exh->latch);
tDebug("pre alloc refId %" PRId64 "", exh->refId);
diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c
index 50eb8413c0..9bd60be2b6 100644
--- a/source/os/src/osSysinfo.c
+++ b/source/os/src/osSysinfo.c
@@ -29,6 +29,12 @@ typedef struct {
uint64_t nice;
uint64_t system;
uint64_t idle;
+ uint64_t wa;
+ uint64_t hi;
+ uint64_t si;
+ uint64_t st;
+ uint64_t guest;
+ uint64_t guest_nice;
} SysCpuInfo;
typedef struct {
@@ -173,8 +179,11 @@ static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) {
}
char cpu[10] = {0};
- sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system,
- &cpuInfo->idle);
+ sscanf(line,
+ "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64
+ " %" PRIu64,
+ cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle, &cpuInfo->wa, &cpuInfo->hi,
+ &cpuInfo->si, &cpuInfo->st, &cpuInfo->guest, &cpuInfo->guest_nice);
taosCloseFile(&pFile);
#endif
@@ -576,7 +585,8 @@ void taosGetCpuUsage(double *cpu_system, double *cpu_engine) {
SysCpuInfo sysCpu = {0};
ProcCpuInfo procCpu = {0};
if (taosGetSysCpuInfo(&sysCpu) == 0 && taosGetProcCpuInfo(&procCpu) == 0) {
- curSysUsed = sysCpu.user + sysCpu.nice + sysCpu.system;
+ curSysUsed = sysCpu.user + sysCpu.nice + sysCpu.system + sysCpu.wa + sysCpu.hi + sysCpu.si + sysCpu.st +
+ sysCpu.guest + sysCpu.guest_nice;
curSysTotal = curSysUsed + sysCpu.idle;
curProcTotal = procCpu.utime + procCpu.stime + procCpu.cutime + procCpu.cstime;
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index e9fa58e6e3..c7fd6f65c5 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -22,10 +22,12 @@
#define TAOS_ERROR_C
static threadlocal int32_t tsErrno;
+static threadlocal int32_t tsErrln;
static threadlocal char tsErrMsgDetail[ERR_MSG_LEN] = {0};
static threadlocal char tsErrMsgReturn[ERR_MSG_LEN] = {0};
int32_t* taosGetErrno() { return &tsErrno; }
+int32_t* taosGetErrln() { return &tsErrln; }
char* taosGetErrMsg() { return tsErrMsgDetail; }
char* taosGetErrMsgReturn() { return tsErrMsgReturn; }
@@ -96,6 +98,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_NOT_FOUND, "Not found")
TAOS_DEFINE_ERROR(TSDB_CODE_NO_DISKSPACE, "Out of disk space")
TAOS_DEFINE_ERROR(TSDB_CODE_TIMEOUT_ERROR, "Operation timeout")
TAOS_DEFINE_ERROR(TSDB_CODE_NO_ENOUGH_DISKSPACE, "No enough disk space")
+TAOS_DEFINE_ERROR(TSDB_CODE_THIRDPARTY_ERROR, "third party error, please check the log")
TAOS_DEFINE_ERROR(TSDB_CODE_APP_IS_STARTING, "Database is starting up")
TAOS_DEFINE_ERROR(TSDB_CODE_APP_IS_STOPPING, "Database is closing down")
@@ -104,6 +107,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CFG_VALUE, "Invalid configuration
TAOS_DEFINE_ERROR(TSDB_CODE_IP_NOT_IN_WHITE_LIST, "Not allowed to connect")
TAOS_DEFINE_ERROR(TSDB_CODE_FAILED_TO_CONNECT_S3, "Failed to connect to s3 server")
TAOS_DEFINE_ERROR(TSDB_CODE_MSG_PREPROCESSED, "Message has been processed in preprocess")
+TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_BUFFER, "Out of buffer")
//client
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION, "Invalid operation")
@@ -678,6 +682,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_PRIMARY_KEY_IS_NONE, "Primary key column
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TBNAME_ERROR, "Pseudo tag tbname not set")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TBNAME_DUPLICATED, "Table name duplicated")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TAG_NAME_DUPLICATED, "Tag name duplicated")
+TAOS_DEFINE_ERROR(TSDB_CODE_PAR_NOT_ALLOWED_DIFFERENT_BY_ROW_FUNC, "Some functions cannot appear in the select list at the same time")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTERNAL_ERROR, "Parser internal error")
//planner
diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c
index 7a4eb09b99..45a8a462fb 100644
--- a/source/util/src/tqueue.c
+++ b/source/util/src/tqueue.c
@@ -162,7 +162,7 @@ void *taosAllocateQitem(int32_t size, EQItype itype, int64_t dataSize) {
int64_t alloced = atomic_add_fetch_64(&tsRpcQueueMemoryUsed, size + dataSize);
if (alloced > tsRpcQueueMemoryAllowed) {
uError("failed to alloc qitem, size:%" PRId64 " alloc:%" PRId64 " allowed:%" PRId64, size + dataSize, alloced,
- tsRpcQueueMemoryUsed);
+ tsRpcQueueMemoryAllowed);
atomic_sub_fetch_64(&tsRpcQueueMemoryUsed, size + dataSize);
taosMemoryFree(pNode);
terrno = TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE;
@@ -494,6 +494,8 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo *
qall->start = queue->head;
qall->numOfItems = queue->numOfItems;
qall->memOfItems = queue->memOfItems;
+ qall->unAccessedNumOfItems = queue->numOfItems;
+ qall->unAccessMemOfItems = queue->memOfItems;
code = qall->numOfItems;
qinfo->ahandle = queue->ahandle;
diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c
index bc5c46351a..4a8a0823b7 100644
--- a/source/util/src/tworker.c
+++ b/source/util/src/tworker.c
@@ -577,7 +577,7 @@ static int64_t atomicCompareExchangeRunning(int64_t* ptr, int32_t* expectedVal,
}
}
-static int64_t atomicComapreAndExchangeActiveAndRunning(int64_t *ptr, int32_t *expectedActive, int32_t newActive,
+static int64_t atomicCompareExchangeActiveAndRunning(int64_t *ptr, int32_t *expectedActive, int32_t newActive,
int32_t *expectedRunning, int32_t newRunning) {
int64_t oldVal64 = *expectedActive, newVal64 = newActive;
oldVal64 <<= 32;
@@ -683,7 +683,7 @@ static bool tQueryAutoQWorkerTryDecActive(void* p, int32_t minActive) {
int64_t val64 = pPool->activeRunningN;
int32_t active = GET_ACTIVE_N(val64), running = GET_RUNNING_N(val64);
while (active > minActive) {
- if (atomicComapreAndExchangeActiveAndRunning(&pPool->activeRunningN, &active, active - 1, &running, running - 1))
+ if (atomicCompareExchangeActiveAndRunning(&pPool->activeRunningN, &active, active - 1, &running, running - 1))
return true;
}
atomicFetchSubRunning(&pPool->activeRunningN, 1);
@@ -691,13 +691,18 @@ static bool tQueryAutoQWorkerTryDecActive(void* p, int32_t minActive) {
}
static int32_t tQueryAutoQWorkerWaitingCheck(SQueryAutoQWorkerPool* pPool) {
- int32_t running = GET_RUNNING_N(pPool->activeRunningN);
- while (running < pPool->num) {
- if (atomicCompareExchangeRunning(&pPool->activeRunningN, &running, running + 1)) {
- return TSDB_CODE_SUCCESS;
+ while (1) {
+ int64_t val64 = pPool->activeRunningN;
+ int32_t running = GET_RUNNING_N(val64), active = GET_ACTIVE_N(val64);
+ while (running < pPool->num) {
+ if (atomicCompareExchangeActiveAndRunning(&pPool->activeRunningN, &active, active, &running, running + 1)) {
+ return TSDB_CODE_SUCCESS;
+ }
+ }
+ if (atomicCompareExchangeActive(&pPool->activeRunningN, &active, active - 1)) {
+ break;
}
}
- atomicFetchSubActive(&pPool->activeRunningN, 1);
// to wait for process
taosThreadMutexLock(&pPool->waitingBeforeProcessMsgLock);
atomic_fetch_add_32(&pPool->waitingBeforeProcessMsgN, 1);
@@ -976,7 +981,7 @@ static int32_t tQueryAutoQWorkerRecoverFromBlocking(void *p) {
int64_t val64 = pPool->activeRunningN;
int32_t running = GET_RUNNING_N(val64), active = GET_ACTIVE_N(val64);
while (running < pPool->num) {
- if (atomicComapreAndExchangeActiveAndRunning(&pPool->activeRunningN, &active, active + 1, &running, running + 1)) {
+ if (atomicCompareExchangeActiveAndRunning(&pPool->activeRunningN, &active, active + 1, &running, running + 1)) {
return TSDB_CODE_SUCCESS;
}
}
diff --git a/source/util/test/utilTests.cpp b/source/util/test/utilTests.cpp
index 6e32167f88..cb78743018 100644
--- a/source/util/test/utilTests.cpp
+++ b/source/util/test/utilTests.cpp
@@ -2,6 +2,7 @@
#include
#include
#include
+#include "ttime.h"
#include "tarray.h"
#include "tcompare.h"
@@ -382,3 +383,94 @@ TEST(utilTest, intToHextStr) {
ASSERT_STREQ(buf, destBuf);
}
}
+static int64_t getIntervalValWithPrecision(int64_t interval, int8_t unit, int8_t precision) {
+ if (IS_CALENDAR_TIME_DURATION(unit)) {
+ return interval;
+ }
+ if(0 != getDuration(interval, unit, &interval, precision)) {
+ assert(0);
+ }
+ return interval;
+}
+
+static bool tsmaIntervalCheck(int64_t baseInterval, int8_t baseUnit, int64_t interval, int8_t unit, int8_t precision) {
+ auto ret = checkRecursiveTsmaInterval(getIntervalValWithPrecision(baseInterval, baseUnit, precision), baseUnit,
+ getIntervalValWithPrecision(interval, unit, precision), unit, precision, true);
+ using namespace std;
+ cout << interval << unit << " on " << baseInterval << baseUnit << ": " << ret << endl;
+ return ret;
+}
+
+TEST(tsma, reverse_unit) {
+ ASSERT_TRUE(tsmaIntervalCheck(1, 'm', 120, 's', TSDB_TIME_PRECISION_MILLI));
+ ASSERT_TRUE(tsmaIntervalCheck(1, 'h', 120, 'm', TSDB_TIME_PRECISION_MILLI));
+ ASSERT_TRUE(tsmaIntervalCheck(20, 's', 2 * 20 * 1000, 'a', TSDB_TIME_PRECISION_MILLI));
+ ASSERT_TRUE(tsmaIntervalCheck(20, 's', 2 * 20 * 1000 * 1000, 'u', TSDB_TIME_PRECISION_MILLI));
+ ASSERT_TRUE(tsmaIntervalCheck(20, 's', 2UL * 20UL * 1000UL * 1000UL * 1000UL, 'b', TSDB_TIME_PRECISION_MILLI));
+
+ ASSERT_FALSE(tsmaIntervalCheck(1, 'h', 60, 'm', TSDB_TIME_PRECISION_MILLI));
+ ASSERT_FALSE(tsmaIntervalCheck(1, 'h', 6, 'm', TSDB_TIME_PRECISION_MILLI));
+
+ ASSERT_FALSE(tsmaIntervalCheck(2, 'h', 120, 'm', TSDB_TIME_PRECISION_MILLI));
+ ASSERT_TRUE(tsmaIntervalCheck(2, 'h', 240, 'm', TSDB_TIME_PRECISION_MILLI));
+ ASSERT_FALSE(tsmaIntervalCheck(1, 'd', 240, 'm', TSDB_TIME_PRECISION_MILLI));
+
+ ASSERT_FALSE(tsmaIntervalCheck(1, 'd', 1440, 'm', TSDB_TIME_PRECISION_MILLI));
+ ASSERT_TRUE(tsmaIntervalCheck(1, 'd', 2880, 'm', TSDB_TIME_PRECISION_MILLI));
+
+ ASSERT_FALSE(tsmaIntervalCheck(1, 'y', 365, 'd', TSDB_TIME_PRECISION_MILLI));
+ ASSERT_FALSE(tsmaIntervalCheck(1, 'n', 30, 'd', TSDB_TIME_PRECISION_MILLI));
+
+ ASSERT_TRUE(tsmaIntervalCheck(1, 'y', 24, 'n', TSDB_TIME_PRECISION_MILLI));
+
+ ASSERT_FALSE(tsmaIntervalCheck(55, 's', 55, 'm', TSDB_TIME_PRECISION_MILLI));
+ ASSERT_TRUE(tsmaIntervalCheck(10, 's', 1, 'm', TSDB_TIME_PRECISION_MICRO));
+ ASSERT_TRUE(tsmaIntervalCheck(10, 's', 2, 'm', TSDB_TIME_PRECISION_MICRO));
+ ASSERT_TRUE(tsmaIntervalCheck(10, 's', 20, 'm', TSDB_TIME_PRECISION_MICRO));
+ ASSERT_TRUE(tsmaIntervalCheck(10, 's', 50, 'm', TSDB_TIME_PRECISION_MICRO));
+
+ ASSERT_TRUE(tsmaIntervalCheck(120, 's', 30, 'm', TSDB_TIME_PRECISION_MICRO));
+ ASSERT_TRUE(tsmaIntervalCheck(360, 's', 30, 'm', TSDB_TIME_PRECISION_MICRO));
+ ASSERT_TRUE(tsmaIntervalCheck(600, 's', 30, 'm', TSDB_TIME_PRECISION_MICRO));
+ ASSERT_FALSE(tsmaIntervalCheck(600, 's', 15, 'm', TSDB_TIME_PRECISION_MICRO));
+
+ ASSERT_TRUE(tsmaIntervalCheck(10, 's', 1, 'h', TSDB_TIME_PRECISION_MICRO));
+ ASSERT_TRUE(tsmaIntervalCheck(15, 's', 1, 'h', TSDB_TIME_PRECISION_MICRO));
+ ASSERT_FALSE(tsmaIntervalCheck(7*60, 's', 1, 'h', TSDB_TIME_PRECISION_MICRO));
+ ASSERT_TRUE(tsmaIntervalCheck(10, 's', 1, 'd', TSDB_TIME_PRECISION_MICRO));
+ ASSERT_TRUE(tsmaIntervalCheck(10, 's', 1, 'w', TSDB_TIME_PRECISION_MICRO));
+ ASSERT_TRUE(tsmaIntervalCheck(1, 'd', 1, 'w', TSDB_TIME_PRECISION_MICRO));
+ ASSERT_TRUE(tsmaIntervalCheck(10, 's', 1, 'n', TSDB_TIME_PRECISION_MICRO));
+ ASSERT_TRUE(tsmaIntervalCheck(10, 's', 1, 'y', TSDB_TIME_PRECISION_MICRO));
+
+ ASSERT_TRUE(tsmaIntervalCheck(1, 'd', 1, 'w', TSDB_TIME_PRECISION_MICRO));
+ ASSERT_TRUE(tsmaIntervalCheck(1, 'd', 1, 'n', TSDB_TIME_PRECISION_MICRO));
+ ASSERT_TRUE(tsmaIntervalCheck(1, 'd', 2, 'n', TSDB_TIME_PRECISION_MICRO));
+ ASSERT_FALSE(tsmaIntervalCheck(2, 'd', 2, 'n', TSDB_TIME_PRECISION_MICRO));
+ ASSERT_FALSE(tsmaIntervalCheck(2, 'd', 2, 'y', TSDB_TIME_PRECISION_MICRO));
+ ASSERT_FALSE(tsmaIntervalCheck(2, 'd', 1, 'y', TSDB_TIME_PRECISION_MICRO));
+
+ ASSERT_FALSE(tsmaIntervalCheck(1, 'w', 1, 'n', TSDB_TIME_PRECISION_NANO));
+ ASSERT_FALSE(tsmaIntervalCheck(4, 'w', 1, 'n', TSDB_TIME_PRECISION_NANO));
+ ASSERT_FALSE(tsmaIntervalCheck(1, 'w', 1, 'y', TSDB_TIME_PRECISION_NANO));
+
+ ASSERT_TRUE(tsmaIntervalCheck(1, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
+ ASSERT_TRUE(tsmaIntervalCheck(2, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
+ ASSERT_TRUE(tsmaIntervalCheck(3, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
+ ASSERT_TRUE(tsmaIntervalCheck(4, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
+ ASSERT_FALSE(tsmaIntervalCheck(5, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
+ ASSERT_TRUE(tsmaIntervalCheck(6, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
+ ASSERT_FALSE(tsmaIntervalCheck(7, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
+ ASSERT_FALSE(tsmaIntervalCheck(8, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
+ ASSERT_FALSE(tsmaIntervalCheck(9, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
+ ASSERT_FALSE(tsmaIntervalCheck(10, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
+ ASSERT_FALSE(tsmaIntervalCheck(11, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
+
+ ASSERT_FALSE(tsmaIntervalCheck(1, 'w', 1, 'w', TSDB_TIME_PRECISION_NANO));
+ ASSERT_FALSE(tsmaIntervalCheck(120, 's', 2, 'm', TSDB_TIME_PRECISION_NANO));
+
+ ASSERT_FALSE(tsmaIntervalCheck(2, 'n', 2, 'n', TSDB_TIME_PRECISION_NANO));
+ ASSERT_FALSE(tsmaIntervalCheck(2, 'y', 2, 'y', TSDB_TIME_PRECISION_NANO));
+ ASSERT_FALSE(tsmaIntervalCheck(12, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
+ ASSERT_TRUE(tsmaIntervalCheck(3, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
+}
diff --git a/tests/army/query/queryBugs.py b/tests/army/query/queryBugs.py
index ca28ff549c..7583382290 100644
--- a/tests/army/query/queryBugs.py
+++ b/tests/army/query/queryBugs.py
@@ -28,8 +28,7 @@ from frame import *
class TDTestCase(TBase):
-
- # fix
+ # fix
def FIX_TD_30686(self):
tdLog.info("check bug TD_30686 ...\n")
sqls = [
@@ -49,6 +48,32 @@ class TDTestCase(TBase):
]
tdSql.checkDataMem(sql, results)
+ def FIX_TS_5105(self):
+ tdLog.info("check bug TS_5105 ...\n")
+ ts1 = "2024-07-03 10:00:00.000"
+ ts2 = "2024-07-03 13:00:00.000"
+ sqls = [
+ "drop database if exists ts_5105",
+ "create database ts_5105 cachemodel 'both';",
+ "use ts_5105;",
+ "CREATE STABLE meters (ts timestamp, current float) TAGS (location binary(64), groupId int);",
+ "CREATE TABLE d1001 USING meters TAGS ('California.B', 2);",
+ "CREATE TABLE d1002 USING meters TAGS ('California.S', 3);",
+ f"INSERT INTO d1001 VALUES ('{ts1}', 10);",
+ f"INSERT INTO d1002 VALUES ('{ts2}', 13);",
+ ]
+ tdSql.executes(sqls)
+
+ sql = "select last(ts), last_row(ts) from meters;"
+
+ # 执行多次，有些时候last_row(ts)会返回错误的值，详见TS-5105
+ for i in range(1, 10):
+ tdLog.debug(f"{i}th execute sql: {sql}")
+ tdSql.query(sql)
+ tdSql.checkRows(1)
+ tdSql.checkData(0, 0, ts2)
+ tdSql.checkData(0, 1, ts2)
+
# run
def run(self):
tdLog.debug(f"start to excute {__file__}")
@@ -57,11 +82,10 @@ class TDTestCase(TBase):
self.FIX_TD_30686()
# TS BUGS
-
+ self.FIX_TS_5105()
tdLog.success(f"{__file__} successfully executed")
-
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/army/query/query_last_row_repeatly.py b/tests/army/query/query_last_row_repeatly.py
deleted file mode 100644
index 3cca032176..0000000000
--- a/tests/army/query/query_last_row_repeatly.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from frame.log import *
-from frame.cases import *
-from frame.sql import *
-from frame.caseBase import *
-from frame import *
-
-
-class TDTestCase(TBase):
- def init(self, conn, logSql, replicaVar=1):
- self.replicaVar = int(replicaVar)
- tdLog.debug("start to execute %s" % __file__)
- tdSql.init(conn.cursor(), logSql)
-
- def run(self):
- sqls = [
- "drop database if exists ts_5101",
- "create database ts_5101 cachemodel 'both';",
- "use ts_5101;",
- "CREATE STABLE meters (ts timestamp, current float) TAGS (location binary(64), groupId int);",
- "CREATE TABLE d1001 USING meters TAGS ('California.B', 2);",
- "CREATE TABLE d1002 USING meters TAGS ('California.S', 3);",
- "INSERT INTO d1001 VALUES ('2024-07-03 10:00:00.000', 10);",
- "INSERT INTO d1002 VALUES ('2024-07-03 13:00:00.000', 13);",
- ]
- tdSql.executes(sqls)
-
- # 执行多次，有些时候last_row(ts)会返回错误的值，详见TS-5105
- for i in range(1, 10):
- sql = "select last(ts), last_row(ts) from meters;"
- tdLog.debug(f"{i}th execute sql: {sql}")
- tdSql.query(sql)
- tdSql.checkRows(1)
- tdSql.checkData(0, 0, "2024-07-03 13:00:00.000")
- tdSql.checkData(0, 1, "2024-07-03 13:00:00.000")
-
- def stop(self):
- tdSql.close()
- tdLog.success("%s successfully executed" % __file__)
-
-
-tdCases.addWindows(__file__, TDTestCase())
-tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index aff5bedaf8..4338187791 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -20,7 +20,6 @@
,,y,army,./pytest.sh python3 ./test.py -f insert/test_column_tag_boundary.py
,,y,army,./pytest.sh python3 ./test.py -f query/fill/fill_desc.py -N 3 -L 3 -D 2
,,y,army,./pytest.sh python3 ./test.py -f query/fill/fill_null.py
-,,y,army,./pytest.sh python3 ./test.py -f query/query_last_row_repeatly.py
,,y,army,./pytest.sh python3 ./test.py -f cluster/incSnapshot.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f query/query_basic.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f query/accuracy/test_query_accuracy.py
@@ -132,6 +131,11 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma.py -Q 4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py
diff --git a/tests/script/tsim/stream/distributeMultiLevelInterval0.sim b/tests/script/tsim/stream/distributeMultiLevelInterval0.sim
index 784ab7f4a5..3ac1fbdd2b 100644
--- a/tests/script/tsim/stream/distributeMultiLevelInterval0.sim
+++ b/tests/script/tsim/stream/distributeMultiLevelInterval0.sim
@@ -21,8 +21,37 @@ sql create table ts3 using st tags(3,2,2);
sql create table ts4 using st tags(4,2,2);
sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt1 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4 from st interval(10s);
+print ====check task status start
+
+$loop_count = 0
+
+loopCheck:
+
sleep 1000
+$loop_count = $loop_count + 1
+if $loop_count == 30 then
+ return -1
+endi
+
+print 1 select * from information_schema.ins_stream_tasks;
+sql select * from information_schema.ins_stream_tasks;
+
+if $rows == 0 then
+ print rows=$rows
+ goto loopCheck
+endi
+
+print 1 select * from information_schema.ins_stream_tasks where status != "ready";
+sql select * from information_schema.ins_stream_tasks where status != "ready";
+
+if $rows != 0 then
+ print rows=$rows
+ goto loopCheck
+endi
+
+print ====check task status end
+
sql insert into ts1 values(1648791213000,1,1,3,4.1);
sql insert into ts1 values(1648791223000,2,2,3,1.1);
sql insert into ts1 values(1648791233000,3,3,3,2.1);
@@ -123,6 +152,8 @@ if $data31 != 2 then
goto loop1
endi
+sleep 5000
+
sql delete from ts2 where ts = 1648791243000 ;
$loop_count = 0
diff --git a/tests/script/tsim/stream/distributeSession0.sim b/tests/script/tsim/stream/distributeSession0.sim
index 7eb8c725c8..9a112c2cdf 100644
--- a/tests/script/tsim/stream/distributeSession0.sim
+++ b/tests/script/tsim/stream/distributeSession0.sim
@@ -41,8 +41,37 @@ sql create table ts1 using st tags(1,1,1);
sql create table ts2 using st tags(2,2,2);
sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamtST as select _wstart, count(*) c1, sum(a) c2 , max(b) c3 from st session(ts, 10s) ;
+print ====check task status start
+
+$loop_count = 0
+
+loopCheck:
+
sleep 1000
+$loop_count = $loop_count + 1
+if $loop_count == 30 then
+ return -1
+endi
+
+print 1 select * from information_schema.ins_stream_tasks;
+sql select * from information_schema.ins_stream_tasks;
+
+if $rows == 0 then
+ print rows=$rows
+ goto loopCheck
+endi
+
+print 1 select * from information_schema.ins_stream_tasks where status != "ready";
+sql select * from information_schema.ins_stream_tasks where status != "ready";
+
+if $rows != 0 then
+ print rows=$rows
+ goto loopCheck
+endi
+
+print ====check task status end
+
sql insert into ts1 values(1648791211000,1,1,1) (1648791211005,1,1,1);
sql insert into ts2 values(1648791221004,1,2,3) (1648791221008,2,2,3);
sql insert into ts1 values(1648791211005,1,1,1);
@@ -79,4 +108,73 @@ if $data03 != 7 then
return -1
endi
+print ===== step3
+
+sql create database test1 vgroups 4;
+sql use test1;
+sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
+sql create table ts1 using st tags(1,1,1);
+sql create table ts2 using st tags(2,2,2);
+sql create stream stream_t2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamtST2 as select _wstart, count(*) c1, sum(a) c2 , max(b) c3 from st partition by a session(ts, 10s) ;
+
+print ====check task status start
+
+$loop_count = 0
+
+loopCheck1:
+
+sleep 1000
+
+$loop_count = $loop_count + 1
+if $loop_count == 30 then
+ return -1
+endi
+
+print 1 select * from information_schema.ins_stream_tasks;
+sql select * from information_schema.ins_stream_tasks;
+
+if $rows == 0 then
+ print rows=$rows
+ goto loopCheck1
+endi
+
+print 1 select * from information_schema.ins_stream_tasks where status != "ready";
+sql select * from information_schema.ins_stream_tasks where status != "ready";
+
+if $rows != 0 then
+ print rows=$rows
+ goto loopCheck1
+endi
+
+print ====check task status end
+
+sql insert into ts1 values(1648791201000,1,1,1) (1648791210000,1,1,1);
+sql insert into ts1 values(1648791211000,2,1,1) (1648791212000,2,1,1);
+sql insert into ts2 values(1648791211000,3,1,1) (1648791212000,3,1,1);
+
+sql delete from st where ts = 1648791211000;
+
+$loop_count = 0
+loop2:
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+sleep 1000
+print 2 select * from streamtST2;
+sql select * from streamtST2;
+
+print $data00 $data01 $data02 $data03
+print $data10 $data11 $data12 $data13
+print $data20 $data21 $data22 $data23
+print $data30 $data31 $data32 $data33
+print $data40 $data41 $data42 $data43
+
+if $rows != 3 then
+ print =====rows=$rows
+ goto loop2
+endi
+
system sh/stop_dnodes.sh
diff --git a/tests/system-test/0-others/backquote_check.py b/tests/system-test/0-others/backquote_check.py
index d5b5d2ef1a..b35701f1df 100644
--- a/tests/system-test/0-others/backquote_check.py
+++ b/tests/system-test/0-others/backquote_check.py
@@ -97,13 +97,45 @@ class TDTestCase:
tdSql.checkEqual(tdSql.queryResult[0][1], 123)
tdSql.checkEqual(tdSql.queryResult[0][2], 4)
- tdSql.execute(f'drop database {self.dbname}')
+ # tdSql.execute(f'drop database {self.dbname}')
+
+ def view_name_check(self):
+ """Cover the view name with backquote"""
+ tdSql.execute(f'create view `v1` as select * from (select `ts`, `c0` from ntb1) `t1`;')
+ tdSql.query('select * from `v1`;')
+ tdSql.checkRows(1)
+ tdSql.execute(f'drop view `v1`;')
+
+ def query_check(self):
+ """Cover the table name, column name with backquote in query statement"""
+ tdSql.query(f'select `t1`.`ts`, `t1`.`c0` + 2 as `c1` from `{self.ntbname1}` `t1` union select `t2`.`ts`, `t2`.`c0` from `{self.ntbname2}` `t2`')
+ tdSql.checkRows(2)
+
+ tdSql.query(f'select `t1`.`ts`, `t1`.`c0` + `t2`.`c0` as `c0`, `t1`.`c1` * `t2`.`c1` as `c1` from `{self.ntbname1}` `t1` join `{self.ntbname2}` `t2` on timetruncate(`t1`.`ts`, 1s) = timetruncate(`t2`.`ts`, 1s);')
+ tdSql.checkRows(1)
+ tdSql.checkEqual(tdSql.queryResult[0][1], 3)
+
+ tdSql.query(f'select `t1`.`ts`, `t1`.`c1`, `t1`.`c2` from (select `ts`, `c0` + 1 as `c1`, `c1` + 2 as `c2` from `{self.ntbname1}`) `t1`;')
+ tdSql.checkEqual(tdSql.queryResult[0][1], 2)
+ tdSql.checkEqual(tdSql.queryResult[0][2], 3)
+
+ tdSql.query(f'select `t`.`ts`, cast(`t`.`v1` as int) + `t`.`c0` as `v` from (select `ts`, "12" as `v1`, `c0`, `c1` from `ntb1`) `t`;')
+ tdSql.checkRows(1)
+ tdSql.checkEqual(tdSql.queryResult[0][1], 13)
+
+ tdSql.query(f'select count(`t1`.`ts`) from (select `t`.`ts` from `{self.ntbname1}` `t`) `t1`;')
+ tdSql.checkRows(1)
+
def run(self):
self.topic_name_check()
self.db_name_check()
self.stream_name_check()
self.table_name_check()
+ self.view_name_check()
+ self.query_check()
+
def stop(self):
+ tdSql.execute(f'drop database {self.dbname}')
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
diff --git a/tests/system-test/1-insert/stmt_error.py b/tests/system-test/1-insert/stmt_error.py
index c6d747c317..0bfbedb9a1 100644
--- a/tests/system-test/1-insert/stmt_error.py
+++ b/tests/system-test/1-insert/stmt_error.py
@@ -24,7 +24,7 @@ class TDTestCase:
case1 : [TD-11899] : this is an test case for check stmt error use .
'''
return
-
+
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__)
@@ -49,7 +49,7 @@ class TDTestCase:
ff float, dd double, bb binary(65059), nn nchar(100), tt timestamp)",
)
conn.load_table_info("log")
-
+
stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
params = new_bind_params(16)
@@ -123,7 +123,7 @@ class TDTestCase:
ff float, dd double, bb binary(100), nn nchar(100), tt timestamp , error_data int )",
)
conn.load_table_info("log")
-
+
stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,1000)")
params = new_bind_params(16)
@@ -195,20 +195,74 @@ class TDTestCase:
except Exception as err:
conn.close()
raise err
-
+
+ def test_stmt_nornmal_value_error(self, conn):
+ # type: (TaosConnection) -> None
+ dbname = "pytest_taos_stmt_error"
+ try:
+ conn.execute("drop database if exists %s" % dbname)
+ conn.execute("create database if not exists %s" % dbname)
+ conn.select_db(dbname)
+
+ conn.execute(
+ "create table if not exists log(ts timestamp, bo bool, nil tinyint, ti tinyint, si smallint, ii int,\
+ bi bigint, tu tinyint unsigned, su smallint unsigned, iu int unsigned, bu bigint unsigned, \
+ ff float, dd double, bb binary(100), nn nchar(100), tt timestamp , error_data int )",
+ )
+ conn.load_table_info("log")
+
+
+ stmt = conn.statement("insert into log values(NOW(),?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
+ params = new_bind_params(16)
+ params[0].timestamp(1626861392589, PrecisionEnum.Milliseconds)
+ params[1].bool(True)
+ params[2].tinyint(None)
+ params[3].tinyint(2)
+ params[4].smallint(3)
+ params[5].int(4)
+ params[6].bigint(5)
+ params[7].tinyint_unsigned(6)
+ params[8].smallint_unsigned(7)
+ params[9].int_unsigned(8)
+ params[10].bigint_unsigned(9)
+ params[11].float(10.1)
+ params[12].double(10.11)
+ params[13].binary("hello")
+ params[14].nchar("stmt")
+ params[15].timestamp(1626861392589, PrecisionEnum.Milliseconds)
+
+ stmt.bind_param(params)
+ stmt.execute()
+
+ conn.close()
+
+ except Exception as err:
+ conn.execute("drop database if exists %s" % dbname)
+ conn.close()
+ raise err
+
def run(self):
-
+
self.test_stmt_insert(self.conn())
try:
self.test_stmt_insert_error(self.conn())
except Exception as error :
-
- if str(error)=='[0x0200]: no mix usage for ? and values':
+
+ if str(error)=='[0x0200]: stmt bind param does not support normal value in sql':
tdLog.info('=========stmt error occured for bind part column ==============')
else:
tdLog.exit("expect error(%s) not occured" % str(error))
- try:
+ try:
+ self.test_stmt_nornmal_value_error(self.conn())
+ except Exception as error :
+
+ if str(error)=='[0x0200]: stmt bind param does not support normal value in sql':
+ tdLog.info('=========stmt error occured for bind part column ==============')
+ else:
+ tdLog.exit("expect error(%s) not occured" % str(error))
+
+ try:
self.test_stmt_insert_error_null_timestamp(self.conn())
tdLog.exit("expect error not occured - 1")
except Exception as error :
diff --git a/tests/system-test/2-query/diff.py b/tests/system-test/2-query/diff.py
index 15f73344d3..fd7ef00680 100644
--- a/tests/system-test/2-query/diff.py
+++ b/tests/system-test/2-query/diff.py
@@ -45,12 +45,357 @@ class TDTestCase:
else:
tdSql.checkData(i, j, 1)
+ def ignoreTest(self):
+ dbname = "db"
+
+ ts1 = 1694912400000
+ tdSql.execute(f'''create table {dbname}.stb30749(ts timestamp, col1 tinyint, col2 smallint) tags(loc nchar(20))''')
+ tdSql.execute(f"create table {dbname}.stb30749_1 using {dbname}.stb30749 tags('shanghai')")
+
+ tdSql.execute(f"insert into {dbname}.stb30749_1 values(%d, null, 1)" % (ts1 + 1))
+ tdSql.execute(f"insert into {dbname}.stb30749_1 values(%d, 3, null)" % (ts1 + 2))
+ tdSql.execute(f"insert into {dbname}.stb30749_1 values(%d, 4, 3)" % (ts1 + 3))
+ tdSql.execute(f"insert into {dbname}.stb30749_1 values(%d, 1, 1)" % (ts1 + 4))
+ tdSql.execute(f"insert into {dbname}.stb30749_1 values(%d, 2, null)" % (ts1 + 5))
+ tdSql.execute(f"insert into {dbname}.stb30749_1 values(%d, null, null)" % (ts1 + 6))
+
+ tdSql.query(f"select ts, diff(col1) from {dbname}.stb30749_1")
+ tdSql.checkRows(5)
+ tdSql.checkData(0, 0, '2023-09-17 09:00:00.002')
+ tdSql.checkData(0, 1, None)
+ tdSql.checkData(1, 0, '2023-09-17 09:00:00.003')
+ tdSql.checkData(1, 1, 1)
+ tdSql.checkData(2, 0, '2023-09-17 09:00:00.004')
+ tdSql.checkData(2, 1, -3)
+ tdSql.checkData(3, 0, '2023-09-17 09:00:00.005')
+ tdSql.checkData(3, 1, 1)
+ tdSql.checkData(4, 0, '2023-09-17 09:00:00.006')
+ tdSql.checkData(4, 1, None)
+
+ tdSql.query(f"select ts, diff(col1, 1) from {dbname}.stb30749_1")
+ tdSql.checkRows(5)
+ tdSql.checkData(0, 0, '2023-09-17 09:00:00.002')
+ tdSql.checkData(0, 1, None)
+ tdSql.checkData(1, 0, '2023-09-17 09:00:00.003')
+ tdSql.checkData(1, 1, 1)
+ tdSql.checkData(2, 0, '2023-09-17 09:00:00.004')
+ tdSql.checkData(2, 1, None)
+ tdSql.checkData(3, 0, '2023-09-17 09:00:00.005')
+ tdSql.checkData(3, 1, 1)
+ tdSql.checkData(4, 0, '2023-09-17 09:00:00.006')
+ tdSql.checkData(4, 1, None)
+
+ tdSql.query(f"select ts, diff(col1, 2) from {dbname}.stb30749_1")
+ tdSql.checkRows(3)
+ tdSql.checkData(0, 0, '2023-09-17 09:00:00.003')
+ tdSql.checkData(0, 1, 1)
+ tdSql.checkData(1, 0, '2023-09-17 09:00:00.004')
+ tdSql.checkData(1, 1, -3)
+ tdSql.checkData(2, 0, '2023-09-17 09:00:00.005')
+ tdSql.checkData(2, 1, 1)
+
+ tdSql.query(f"select ts, diff(col1, 3) from {dbname}.stb30749_1")
+ tdSql.checkRows(2)
+ tdSql.checkData(0, 0, '2023-09-17 09:00:00.003')
+ tdSql.checkData(0, 1, 1)
+ tdSql.checkData(1, 0, '2023-09-17 09:00:00.005')
+ tdSql.checkData(1, 1, 1)
+
+ tdSql.query(f"select ts, diff(col1, 3), diff(col2, 0) from {dbname}.stb30749_1")
+ tdSql.checkRows(5)
+ tdSql.checkData(0, 0, '2023-09-17 09:00:00.002')
+ tdSql.checkData(1, 2, 2)
+ tdSql.checkData(2, 1, None)
+ tdSql.checkData(2, 2, -2)
+
+ tdSql.query(f"select ts, diff(col1, 3), diff(col2, 1) from {dbname}.stb30749_1")
+ tdSql.checkRows(5)
+ tdSql.checkData(0, 0, '2023-09-17 09:00:00.002')
+ tdSql.checkData(1, 2, 2)
+ tdSql.checkData(2, 1, None)
+ tdSql.checkData(2, 2, None)
+
+ tdSql.query(f"select ts, diff(col1, 2), diff(col2, 2) from {dbname}.stb30749_1")
+ tdSql.checkRows(3)
+ tdSql.checkData(0, 0, '2023-09-17 09:00:00.003')
+ tdSql.checkData(1, 0, '2023-09-17 09:00:00.004')
+ tdSql.checkData(2, 0, '2023-09-17 09:00:00.005')
+ tdSql.checkData(0, 1, 1)
+ tdSql.checkData(1, 1, -3)
+ tdSql.checkData(2, 1, 1)
+ tdSql.checkData(0, 2, 2)
+ tdSql.checkData(1, 2, -2)
+ tdSql.checkData(2, 2, None)
+
+ tdSql.query(f"select ts, diff(col1, 3), diff(col2, 2) from {dbname}.stb30749_1")
+ tdSql.checkRows(3)
+ tdSql.checkData(0, 0, '2023-09-17 09:00:00.003')
+ tdSql.checkData(1, 0, '2023-09-17 09:00:00.004')
+ tdSql.checkData(2, 0, '2023-09-17 09:00:00.005')
+ tdSql.checkData(0, 1, 1)
+ tdSql.checkData(1, 1, None)
+ tdSql.checkData(2, 1, 1)
+ tdSql.checkData(0, 2, 2)
+ tdSql.checkData(1, 2, -2)
+ tdSql.checkData(2, 2, None)
+
+ tdSql.query(f"select ts, diff(col1, 3), diff(col2, 3) from {dbname}.stb30749_1")
+ tdSql.checkRows(2)
+ tdSql.checkData(0, 0, '2023-09-17 09:00:00.003')
+ tdSql.checkData(1, 0, '2023-09-17 09:00:00.005')
+ tdSql.checkData(0, 1, 1)
+ tdSql.checkData(1, 1, 1)
+ tdSql.checkData(0, 2, 2)
+ tdSql.checkData(1, 2, None)
+
+ tdSql.execute(f"create table {dbname}.stb30749_2 using {dbname}.stb30749 tags('shanghai')")
+
+ tdSql.execute(f"insert into {dbname}.stb30749_2 values(%d, null, 1)" % (ts1 - 1))
+ tdSql.execute(f"insert into {dbname}.stb30749_2 values(%d, 4, 3)" % (ts1 + 0))
+ tdSql.execute(f"insert into {dbname}.stb30749_2 values(%d, null, null)" % (ts1 + 10))
+
+ tdSql.query(f"select ts, diff(col1), diff(col2, 1) from {dbname}.stb30749")
+ tdSql.checkRows(8)
+ tdSql.checkData(2, 0, '2023-09-17 09:00:00.002')
+ tdSql.checkData(3, 0, '2023-09-17 09:00:00.003')
+ tdSql.checkData(2, 1, -1)
+ tdSql.checkData(2, 2, None)
+ tdSql.checkData(3, 1, 1)
+ tdSql.checkData(3, 2, 2)
+
+ tdSql.query(f"select ts, diff(col1), diff(col2) from {dbname}.stb30749")
+ tdSql.checkRows(8)
+ tdSql.checkData(2, 0, '2023-09-17 09:00:00.002')
+ tdSql.checkData(3, 0, '2023-09-17 09:00:00.003')
+ tdSql.checkData(2, 1, -1)
+ tdSql.checkData(2, 2, None)
+ tdSql.checkData(3, 1, 1)
+ tdSql.checkData(3, 2, 2)
+
+ tdSql.query(f"select ts, diff(col1), diff(col2, 3) from {dbname}.stb30749")
+ tdSql.checkRows(8)
+ tdSql.checkData(2, 0, '2023-09-17 09:00:00.002')
+ tdSql.checkData(3, 0, '2023-09-17 09:00:00.003')
+ tdSql.checkData(2, 1, -1)
+ tdSql.checkData(2, 2, None)
+ tdSql.checkData(3, 1, 1)
+ tdSql.checkData(3, 2, 2)
+
+ tdSql.query(f"select ts, diff(col1, 1), diff(col2, 2) from {dbname}.stb30749")
+ tdSql.checkRows(8)
+ tdSql.checkData(2, 0, '2023-09-17 09:00:00.002')
+ tdSql.checkData(3, 0, '2023-09-17 09:00:00.003')
+ tdSql.checkData(2, 1, None)
+ tdSql.checkData(2, 2, None)
+ tdSql.checkData(3, 1, 1)
+ tdSql.checkData(3, 2, 2)
+
+ tdSql.query(f"select ts, diff(col1, 1), diff(col2, 3) from {dbname}.stb30749")
+ tdSql.checkRows(8)
+ tdSql.checkData(2, 0, '2023-09-17 09:00:00.002')
+ tdSql.checkData(3, 0, '2023-09-17 09:00:00.003')
+ tdSql.checkData(2, 1, None)
+ tdSql.checkData(2, 2, None)
+ tdSql.checkData(3, 1, 1)
+ tdSql.checkData(3, 2, 2)
+
+ tdSql.query(f"select ts, diff(col1, 2), diff(col2, 2) from {dbname}.stb30749")
+ tdSql.checkRows(6)
+ tdSql.checkData(2, 0, '2023-09-17 09:00:00.002')
+ tdSql.checkData(3, 0, '2023-09-17 09:00:00.003')
+ tdSql.checkData(2, 1, -1)
+ tdSql.checkData(2, 2, None)
+ tdSql.checkData(3, 1, 1)
+ tdSql.checkData(3, 2, 2)
+
+ tdSql.query(f"select ts, diff(col1, 3), diff(col2, 2) from {dbname}.stb30749")
+ tdSql.checkRows(5)
+ tdSql.checkData(2, 0, '2023-09-17 09:00:00.003')
+ tdSql.checkData(3, 0, '2023-09-17 09:00:00.004')
+ tdSql.checkData(2, 1, 1)
+ tdSql.checkData(2, 2, 2)
+ tdSql.checkData(3, 1, None)
+ tdSql.checkData(3, 2, -2)
+
+ tdSql.query(f"select ts, diff(col1, 2), diff(col2, 3) from {dbname}.stb30749")
+ tdSql.checkRows(5)
+ tdSql.checkData(2, 0, '2023-09-17 09:00:00.003')
+ tdSql.checkData(3, 0, '2023-09-17 09:00:00.004')
+ tdSql.checkData(2, 1, 1)
+ tdSql.checkData(2, 2, 2)
+ tdSql.checkData(3, 1, -3)
+ tdSql.checkData(3, 2, None)
+
+ tdSql.query(f"select ts, diff(col1, 3), diff(col2, 3) from {dbname}.stb30749")
+ tdSql.checkRows(3)
+ tdSql.checkData(1, 0, '2023-09-17 09:00:00.003')
+ tdSql.checkData(2, 0, '2023-09-17 09:00:00.005')
+ tdSql.checkData(1, 1, 1)
+ tdSql.checkData(1, 2, 2)
+ tdSql.checkData(2, 1, 1)
+ tdSql.checkData(2, 2, None)
+
+ tdSql.query(f"select ts, diff(col1), diff(col2) from {dbname}.stb30749 partition by tbname")
+ tdSql.checkRows(7)
+ tdSql.checkData(0, 0, '2023-09-17 09:00:00.002')
+ tdSql.checkData(1, 0, '2023-09-17 09:00:00.003')
+ tdSql.checkData(0, 1, None)
+ tdSql.checkData(0, 2, None)
+ tdSql.checkData(1, 1, 1)
+ tdSql.checkData(1, 2, 2)
+
+ tdSql.query(f"select ts, diff(col1, 3), diff(col2, 2) from {dbname}.stb30749 partition by tbname")
+ tdSql.checkRows(4)
+ tdSql.checkData(3, 0, '2023-09-17 09:00:00.000')
+ tdSql.checkData(3, 1, None)
+ tdSql.checkData(3, 2, 2)
+
+ tdSql.execute(f"insert into {dbname}.stb30749_2 values(%d, null, 1)" % (ts1 + 1))
+ tdSql.error(f"select ts, diff(col1, 3), diff(col2, 2) from {dbname}.stb30749")
+
+ def withPkTest(self):
+ dbname = "db"
+
+ ts1 = 1694912400000
+ tdSql.execute(f'''create table {dbname}.stb5(ts timestamp, col1 int PRIMARY KEY, col2 smallint) tags(loc nchar(20))''')
+ tdSql.execute(f"create table {dbname}.stb5_1 using {dbname}.stb5 tags('shanghai')")
+
+ tdSql.execute(f"insert into {dbname}.stb5_1 values(%d, 2, 1)" % (ts1 + 1))
+ tdSql.execute(f"insert into {dbname}.stb5_1 values(%d, 3, null)" % (ts1 + 2))
+ tdSql.execute(f"insert into {dbname}.stb5_1 values(%d, 4, 3)" % (ts1 + 3))
+
+ tdSql.execute(f"create table {dbname}.stb5_2 using {dbname}.stb5 tags('shanghai')")
+
+ tdSql.execute(f"insert into {dbname}.stb5_2 values(%d, 5, 4)" % (ts1 + 1))
+ tdSql.query(f"select ts, diff(col1, 3), diff(col2, 2) from {dbname}.stb5")
+ tdSql.checkRows(2)
+
+ tdSql.execute(f"insert into {dbname}.stb5_2 values(%d, 3, 3)" % (ts1 + 2))
+ tdSql.query(f"select ts, diff(col1, 3), diff(col2, 2) from {dbname}.stb5")
+ tdSql.checkRows(2)
+
+
+ def intOverflowTest(self):
+ dbname = "db"
+
+ ts1 = 1694912400000
+ tdSql.execute(f'''create table {dbname}.stb6(ts timestamp, c1 int, c2 smallint, c3 int unsigned, c4 BIGINT, c5 BIGINT unsigned) tags(loc nchar(20))''')
+ tdSql.execute(f"create table {dbname}.stb6_1 using {dbname}.stb6 tags('shanghai')")
+
+ tdSql.execute(f"insert into {dbname}.stb6_1 values(%d, -2147483648, -32768, 0, 9223372036854775806, 9223372036854775806)" % (ts1 + 1))
+ tdSql.execute(f"insert into {dbname}.stb6_1 values(%d, 2147483647, 32767, 4294967295, 0, 0)" % (ts1 + 2))
+ tdSql.execute(f"insert into {dbname}.stb6_1 values(%d, -10, -10, 0, -9223372036854775806, 16223372036854775806)" % (ts1 + 3))
+
+ tdSql.query(f"select ts, diff(c1), diff(c2), diff(c3), diff(c4), diff(c5) from {dbname}.stb6_1")
+ tdSql.checkRows(2)
+ tdSql.checkData(0, 0, '2023-09-17 09:00:00.002')
+ tdSql.checkData(0, 1, 4294967295)
+ tdSql.checkData(0, 2, 65535)
+ tdSql.checkData(0, 3, 4294967295)
+ tdSql.checkData(0, 4, -9223372036854775806)
+ tdSql.checkData(0, 5, -9223372036854775806)
+ tdSql.checkData(1, 0, '2023-09-17 09:00:00.003')
+ tdSql.checkData(1, 1, -2147483657)
+ tdSql.checkData(1, 2, -32777)
+ tdSql.checkData(1, 3, -4294967295)
+ tdSql.checkData(1, 4, -9223372036854775806)
+
+ tdSql.query(f"select ts, diff(c1, 1), diff(c2) from {dbname}.stb6_1")
+ tdSql.checkRows(2)
+ tdSql.checkData(0, 1, 4294967295)
+ tdSql.checkData(0, 2, 65535)
+ tdSql.checkData(1, 1, None)
+ tdSql.checkData(1, 2, -32777)
+
+ tdSql.query(f"select ts, diff(c1, 1), diff(c2, 1) from {dbname}.stb6_1")
+ tdSql.checkRows(2)
+ tdSql.checkData(0, 1, 4294967295)
+ tdSql.checkData(0, 2, 65535)
+ tdSql.checkData(1, 1, None)
+ tdSql.checkData(1, 2, None)
+
+ tdSql.query(f"select ts, diff(c1, 2), diff(c2, 3) from {dbname}.stb6_1")
+ tdSql.checkRows(2)
+ tdSql.checkData(0, 1, 4294967295)
+ tdSql.checkData(0, 2, 65535)
+ tdSql.checkData(1, 1, -2147483657)
+ tdSql.checkData(1, 2, None)
+
+ tdSql.query(f"select ts, diff(c1, 3), diff(c2, 3) from {dbname}.stb6_1")
+ tdSql.checkRows(1)
+ tdSql.checkData(0, 1, 4294967295)
+ tdSql.checkData(0, 2, 65535)
+
+ tdSql.execute(f"insert into {dbname}.stb6_1 values(%d, -10, -10, 0, 9223372036854775800, 0)" % (ts1 + 4))
+ tdSql.execute(f"insert into {dbname}.stb6_1 values(%d, -10, -10, 0, 9223372036854775800, 16223372036854775806)" % (ts1 + 5))
+
+ tdSql.query(f"select ts, diff(c4, 0) from {dbname}.stb6_1")
+ tdSql.checkRows(4)
+
+ tdSql.query(f"select ts, diff(c4, 1) from {dbname}.stb6_1")
+ tdSql.checkRows(4)
+ tdSql.checkData(2, 1, -10)
+
+ tdSql.query(f"select ts, diff(c4, 2) from {dbname}.stb6_1")
+ tdSql.checkRows(4)
+
+ tdSql.query(f"select ts, diff(c4, 3) from {dbname}.stb6_1")
+ tdSql.checkRows(2)
+ tdSql.checkData(0, 1, -10)
+ tdSql.checkData(1, 1, 0)
+
+ tdSql.query(f"select ts, diff(c5, 0) from {dbname}.stb6_1")
+ tdSql.checkRows(4)
+
+ tdSql.query(f"select ts, diff(c5, 1) from {dbname}.stb6_1")
+ tdSql.checkRows(4)
+ tdSql.checkData(0, 1, None)
+ tdSql.checkData(1, 0, '2023-09-17 09:00:00.003')
+ tdSql.checkData(2, 1, None)
+ tdSql.checkData(3, 0, '2023-09-17 09:00:00.005')
+
+ tdSql.query(f"select ts, diff(c5, 2) from {dbname}.stb6_1")
+ tdSql.checkRows(4)
+
+ tdSql.query(f"select ts, diff(c5, 3) from {dbname}.stb6_1")
+ tdSql.checkRows(2)
+ tdSql.checkData(0, 0, '2023-09-17 09:00:00.003')
+ tdSql.checkData(1, 0, '2023-09-17 09:00:00.005')
+
+ def doubleOverflowTest(self):
+ dbname = "db"
+
+ ts1 = 1694912400000
+ tdSql.execute(f'''create table {dbname}.stb7(ts timestamp, c1 float, c2 double) tags(loc nchar(20))''')
+ tdSql.execute(f"create table {dbname}.stb7_1 using {dbname}.stb7 tags('shanghai')")
+
+ tdSql.execute(f"insert into {dbname}.stb7_1 values(%d, 334567777777777777777343434343333333733, 334567777777777777777343434343333333733)" % (ts1 + 1))
+ tdSql.execute(f"insert into {dbname}.stb7_1 values(%d, -334567777777777777777343434343333333733, -334567777777777777777343434343333333733)" % (ts1 + 2))
+ tdSql.execute(f"insert into {dbname}.stb7_1 values(%d, 334567777777777777777343434343333333733, 334567777777777777777343434343333333733)" % (ts1 + 3))
+
+ tdSql.query(f"select ts, diff(c1), diff(c2) from {dbname}.stb7_1")
+ tdSql.checkRows(2)
+
+ tdSql.query(f"select ts, diff(c1, 1), diff(c2, 1) from {dbname}.stb7_1")
+ tdSql.checkRows(2)
+ tdSql.checkData(0, 1, None)
+ tdSql.checkData(0, 2, None)
+
+ tdSql.query(f"select ts, diff(c1, 3), diff(c2, 3) from {dbname}.stb7_1")
+ tdSql.checkRows(1)
+ tdSql.checkData(0, 0, '2023-09-17 09:00:00.003')
+
def run(self):
tdSql.prepare()
dbname = "db"
# full type test
self.full_datatype_test()
+
+ self.ignoreTest()
+ self.withPkTest()
+ self.intOverflowTest()
+ self.doubleOverflowTest()
tdSql.execute(
f"create table {dbname}.ntb(ts timestamp,c1 int,c2 double,c3 float)")
@@ -219,9 +564,18 @@ class TDTestCase:
tdSql.error(f"select diff(col1,1.23) from {dbname}.stb_1")
tdSql.error(f"select diff(col1,-1) from {dbname}.stb_1")
tdSql.query(f"select ts,diff(col1),ts from {dbname}.stb_1")
- tdSql.error(f"select diff(col1, 1),diff(col2) from {dbname}.stb_1")
- tdSql.error(f"select diff(col1, 1),diff(col2, 0) from {dbname}.stb_1")
- tdSql.error(f"select diff(col1, 1),diff(col2, 1) from {dbname}.stb_1")
+ tdSql.error(f"select diff(col1, -1) from {dbname}.stb_1")
+ tdSql.error(f"select diff(col1, 4) from {dbname}.stb_1")
+ tdSql.error(f"select diff(col1, 1),diff(col2, 4) from {dbname}.stb_1")
+
+ tdSql.query(f"select diff(col1, 1),diff(col2) from {dbname}.stb_1")
+ tdSql.checkRows(self.rowNum)
+
+ tdSql.query(f"select diff(col1, 1),diff(col2, 0) from {dbname}.stb_1")
+ tdSql.checkRows(self.rowNum)
+
+ tdSql.query(f"select diff(col1, 1),diff(col2, 1) from {dbname}.stb_1")
+ tdSql.checkRows(self.rowNum)
tdSql.query(f"select diff(ts) from {dbname}.stb_1")
tdSql.checkRows(10)
diff --git a/tests/system-test/2-query/max_partition.py b/tests/system-test/2-query/max_partition.py
index 7e43597948..c4635dcf50 100644
--- a/tests/system-test/2-query/max_partition.py
+++ b/tests/system-test/2-query/max_partition.py
@@ -172,7 +172,7 @@ class TDTestCase:
tdSql.checkRows(90)
tdSql.query(f"select c1 , diff(c1 , 0) from {dbname}.stb partition by c1")
- tdSql.checkRows(140)
+ tdSql.checkRows(139)
tdSql.query(f"select c1 , csum(c1) from {dbname}.stb partition by c1")
tdSql.checkRows(100)
diff --git a/tests/system-test/2-query/tsma.py b/tests/system-test/2-query/tsma.py
index d7dc1d24f3..f2900634e8 100644
--- a/tests/system-test/2-query/tsma.py
+++ b/tests/system-test/2-query/tsma.py
@@ -687,10 +687,10 @@ class TDTestCase:
tdLog.debug("insert data ............ [OK]")
return
- def init_data(self, ctb_num: int = 10, rows_per_ctb: int = 10000, start_ts: int = 1537146000000, ts_step: int = 500):
+ def init_data(self, db: str = 'test', ctb_num: int = 10, rows_per_ctb: int = 10000, start_ts: int = 1537146000000, ts_step: int = 500):
tdLog.printNoPrefix(
"======== prepare test env include database, stable, ctables, and insert data: ")
- paraDict = {'dbName': 'test',
+ paraDict = {'dbName': db,
'dropFlag': 1,
'vgroups': 2,
'stbName': 'meters',
@@ -707,8 +707,8 @@ class TDTestCase:
'tsStep': ts_step}
paraDict['vgroups'] = self.vgroups
- paraDict['ctbNum'] = self.ctbNum
- paraDict['rowsPerTbl'] = self.rowsPerTbl
+ paraDict['ctbNum'] = ctb_num
+ paraDict['rowsPerTbl'] = rows_per_ctb
tdLog.info("create database")
self.create_database(tsql=tdSql, dbName=paraDict["dbName"], dropFlag=paraDict["dropFlag"],
@@ -972,16 +972,16 @@ class TDTestCase:
sql = 'select avg(c2), "recursive test.tsma4" from test.meters'
ctx = TSMAQCBuilder().with_sql(sql).should_query_with_tsma(
'tsma4', UsedTsma.TS_MIN, UsedTsma.TS_MAX).get_qc()
- #time.sleep(999999)
self.tsma_tester.check_sql(sql, ctx)
self.check(self.test_query_tsma_all(select_func_list))
self.create_recursive_tsma(
- 'tsma4', 'tsma6', 'test', '1h', 'meters', tsma_func_list)
+ 'tsma4', 'tsma6', 'test', '5h', 'meters', tsma_func_list)
ctx = TSMAQCBuilder().with_sql(sql).should_query_with_tsma(
'tsma6', UsedTsma.TS_MIN, UsedTsma.TS_MAX).get_qc()
self.tsma_tester.check_sql(sql, ctx)
self.check(self.test_query_tsma_all(select_func_list))
+ #time.sleep(999999)
tdSql.error('drop tsma test.tsma3', -2147482491)
tdSql.error('drop tsma test.tsma4', -2147482491)
@@ -1217,7 +1217,6 @@ class TDTestCase:
self.test_create_tsma()
self.test_drop_tsma()
self.test_tb_ddl_with_created_tsma()
-
def run(self):
self.init_data()
@@ -1358,18 +1357,19 @@ class TDTestCase:
'create table nsdb.meters(ts timestamp, c1 int, c2 int, c3 varchar(255)) tags(t1 int, t2 int)', queryTimes=1)
self.create_tsma('tsma1', 'nsdb', 'meters', ['avg(c1)', 'avg(c2)'], '5m')
# Invalid tsma interval, 1ms ~ 1h is allowed
- tdSql.error(
- 'create tsma tsma2 on nsdb.meters function(avg(c1), avg(c2)) interval(2h)', -2147471097)
- tdSql.error(
- 'create tsma tsma2 on nsdb.meters function(avg(c1), avg(c2)) interval(3601s)', -2147471097)
- tdSql.error(
- 'create tsma tsma2 on nsdb.meters function(avg(c1), avg(c2)) interval(3600001a)', -2147471097)
- tdSql.error(
- 'create tsma tsma2 on nsdb.meters function(avg(c1), avg(c2)) interval(3600001000u)', -2147471097)
- tdSql.error(
- 'create tsma tsma2 on nsdb.meters function(avg(c1), avg(c2)) interval(999999b)', -2147471097)
- tdSql.error(
- 'create tsma tsma2 on nsdb.meters function(avg(c1), avg(c2)) interval(999u)', -2147471097)
+ def _():
+ tdSql.error(
+ 'create tsma tsma2 on nsdb.meters function(avg(c1), avg(c2)) interval(2h)', -2147471097)
+ tdSql.error(
+ 'create tsma tsma2 on nsdb.meters function(avg(c1), avg(c2)) interval(3601s)', -2147471097)
+ tdSql.error(
+ 'create tsma tsma2 on nsdb.meters function(avg(c1), avg(c2)) interval(3600001a)', -2147471097)
+ tdSql.error(
+ 'create tsma tsma2 on nsdb.meters function(avg(c1), avg(c2)) interval(3600001000u)', -2147471097)
+ tdSql.error(
+ 'create tsma tsma2 on nsdb.meters function(avg(c1), avg(c2)) interval(999999b)', -2147471097)
+ tdSql.error(
+ 'create tsma tsma2 on nsdb.meters function(avg(c1), avg(c2)) interval(999u)', -2147471097)
# invalid tsma func param
tdSql.error(
'create tsma tsma2 on nsdb.meters function(avg(c1, c2), avg(c2)) interval(10m)', -2147471096)
@@ -1446,8 +1446,6 @@ class TDTestCase:
['avg(c1)', 'avg(c2)'], 'nsdb', 'meters', '10m', 'tsma1')
tdSql.execute('drop tsma nsdb.tsma1', queryTimes=1)
- tdSql.error(
- 'create tsma tsma1 on test.meters function(avg(c1), avg(c2)) interval(2h)', -2147471097)
self.wait_query('show transactions', 0, 10, lambda row: row[3] != 'stream-chkpt-u')
tdSql.execute('drop database nsdb')
@@ -1611,7 +1609,6 @@ class TDTestCase:
# def test_split_dnode(self):
-
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
diff --git a/tests/system-test/2-query/tsma2.py b/tests/system-test/2-query/tsma2.py
new file mode 100644
index 0000000000..ded30412a5
--- /dev/null
+++ b/tests/system-test/2-query/tsma2.py
@@ -0,0 +1,909 @@
+from random import randrange
+import time
+import threading
+import secrets
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+# from tmqCommon import *
+
+ROUND = 100
+
+ignore_some_tests: int = 1
+
+class TSMA:
+ def __init__(self):
+ self.tsma_name = ''
+ self.db_name = ''
+ self.original_table_name = ''
+ self.funcs = []
+ self.cols = []
+ self.interval: str = ''
+
+
+class UsedTsma:
+ TS_MIN = '-9223372036854775808'
+ TS_MAX = '9223372036854775806'
+ TSMA_RES_STB_POSTFIX = '_tsma_res_stb_'
+
+ def __init__(self) -> None:
+ self.name = '' # tsma name or table name
+ self.time_range_start: float = float(UsedTsma.TS_MIN)
+ self.time_range_end: float = float(UsedTsma.TS_MAX)
+ self.is_tsma_ = False
+
+ def __eq__(self, __value: object) -> bool:
+ if isinstance(__value, self.__class__):
+ return self.name == __value.name \
+ and self.time_range_start == __value.time_range_start \
+ and self.time_range_end == __value.time_range_end \
+ and self.is_tsma_ == __value.is_tsma_
+ else:
+ return False
+
+ def __ne__(self, __value: object) -> bool:
+ return not self.__eq__(__value)
+
+ def __str__(self) -> str:
+ return "%s: from %s to %s is_tsma: %d" % (self.name, self.time_range_start, self.time_range_end, self.is_tsma_)
+
+ def __repr__(self) -> str:
+ return self.__str__()
+
+ def setIsTsma(self):
+ self.is_tsma_ = self.name.endswith(self.TSMA_RES_STB_POSTFIX)
+ if not self.is_tsma_:
+ self.is_tsma_ = len(self.name) == 32 # for tsma output child table
+
+class TSMAQueryContext:
+ def __init__(self) -> None:
+ self.sql = ''
+ self.used_tsmas: List[UsedTsma] = []
+ self.ignore_tsma_check_ = False
+ self.ignore_res_order_ = False
+
+ def __eq__(self, __value) -> bool:
+ if isinstance(__value, self.__class__):
+ if self.ignore_tsma_check_ or __value.ignore_tsma_check_:
+ return True
+ if len(self.used_tsmas) != len(__value.used_tsmas):
+ return False
+ for used_tsma1, used_tsma2 in zip(self.used_tsmas, __value.used_tsmas):
+ if not used_tsma1 == used_tsma2:
+ return False
+ return True
+ else:
+ return False
+
+ def __ne__(self, __value: object) -> bool:
+ return self.__eq__(__value)
+
+ def __str__(self) -> str:
+ return str(self.used_tsmas)
+
+ def has_tsma(self) -> bool:
+ for tsma in self.used_tsmas:
+ if tsma.is_tsma_:
+ return True
+ return False
+
+
+class TSMAQCBuilder:
+ def __init__(self) -> None:
+ self.qc_: TSMAQueryContext = TSMAQueryContext()
+
+ def get_qc(self) -> TSMAQueryContext:
+ return self.qc_
+
+ def with_sql(self, sql: str):
+ self.qc_.sql = sql
+ return self
+
+ def to_timestamp(self, ts: str) -> float:
+ if ts == UsedTsma.TS_MAX or ts == UsedTsma.TS_MIN:
+ return float(ts)
+ tdSql.query(
+ "select to_timestamp('%s', 'yyyy-mm-dd hh24-mi-ss.ms')" % (ts))
+ res = tdSql.queryResult[0][0]
+ return res.timestamp() * 1000
+
+ def md5(self, buf: str) -> str:
+ tdSql.query(f'select md5("{buf}")')
+ res = tdSql.queryResult[0][0]
+ return res
+
+ def should_query_with_table(self, tb_name: str, ts_begin: str = UsedTsma.TS_MIN, ts_end: str = UsedTsma.TS_MAX) -> 'TSMAQCBuilder':
+ used_tsma: UsedTsma = UsedTsma()
+ used_tsma.name = tb_name
+ used_tsma.time_range_start = self.to_timestamp(ts_begin)
+ used_tsma.time_range_end = self.to_timestamp(ts_end)
+ used_tsma.is_tsma_ = False
+ self.qc_.used_tsmas.append(used_tsma)
+ return self
+
+ def should_query_with_tsma_ctb(self, db_name: str, tsma_name: str, ctb_name: str, ts_begin: str = UsedTsma.TS_MIN, ts_end: str = UsedTsma.TS_MAX) -> 'TSMAQCBuilder':
+ used_tsma: UsedTsma = UsedTsma()
+ name = f'1.{db_name}.{tsma_name}_{ctb_name}'
+ used_tsma.name = self.md5(name)
+ used_tsma.time_range_start = self.to_timestamp(ts_begin)
+ used_tsma.time_range_end = self.to_timestamp(ts_end)
+ used_tsma.is_tsma_ = True
+ self.qc_.used_tsmas.append(used_tsma)
+ return self
+
+ def ignore_query_table(self):
+ self.qc_.ignore_tsma_check_ = True
+ return self
+
+ def ignore_res_order(self, ignore: bool):
+ self.qc_.ignore_res_order_ = ignore
+ return self
+
+ def should_query_with_tsma(self, tsma_name: str, ts_begin: str = UsedTsma.TS_MIN, ts_end: str = UsedTsma.TS_MAX, child_tb: bool = False) -> 'TSMAQCBuilder':
+ used_tsma: UsedTsma = UsedTsma()
+ if child_tb:
+ used_tsma.name = tsma_name
+ else:
+ used_tsma.name = tsma_name + UsedTsma.TSMA_RES_STB_POSTFIX
+ used_tsma.time_range_start = self.to_timestamp(ts_begin)
+ used_tsma.time_range_end = self.to_timestamp(ts_end)
+ used_tsma.is_tsma_ = True
+ self.qc_.used_tsmas.append(used_tsma)
+ return self
+
+
+class TSMATester:
+ def __init__(self, tdSql: TDSql) -> None:
+ self.tsmas = []
+ self.tdSql: TDSql = tdSql
+
+ def explain_sql(self, sql: str):
+ tdSql.execute("alter local 'querySmaOptimize' '1'")
+ sql = "explain verbose true " + sql
+ tdSql.query(sql, queryTimes=1)
+ res = self.tdSql.queryResult
+ if self.tdSql.queryResult is None:
+ raise
+ return res
+
+ def get_tsma_query_ctx(self, sql: str):
+ explain_res = self.explain_sql(sql)
+ query_ctx: TSMAQueryContext = TSMAQueryContext()
+ query_ctx.sql = sql
+ query_ctx.used_tsmas = []
+ used_tsma: UsedTsma = UsedTsma()
+ for row in explain_res:
+ row = str(row)
+ if len(used_tsma.name) == 0:
+ idx = row.find("Table Scan on ")
+ if idx >= 0:
+ words = row[idx:].split(' ')
+ used_tsma.name = words[3]
+ used_tsma.setIsTsma()
+ else:
+ idx = row.find('Time Range:')
+ if idx >= 0:
+ row = row[idx:].split('[')[1]
+ row = row.split(']')[0]
+ words = row.split(',')
+ used_tsma.time_range_start = float(words[0].strip())
+ used_tsma.time_range_end = float(words[1].strip())
+ query_ctx.used_tsmas.append(used_tsma)
+ used_tsma = UsedTsma()
+
+ deduplicated_tsmas: list[UsedTsma] = []
+ if len(query_ctx.used_tsmas) > 0:
+ deduplicated_tsmas.append(query_ctx.used_tsmas[0])
+ for tsma in query_ctx.used_tsmas:
+ if tsma == deduplicated_tsmas[-1]:
+ continue
+ else:
+ deduplicated_tsmas.append(tsma)
+ query_ctx.used_tsmas = deduplicated_tsmas
+
+ return query_ctx
+
+ def check_explain(self, sql: str, expect: TSMAQueryContext) -> TSMAQueryContext:
+ query_ctx = self.get_tsma_query_ctx(sql)
+ if not query_ctx == expect:
+ tdLog.exit('check explain failed for sql: %s \nexpect: %s \nactual: %s' % (
+ sql, str(expect), str(query_ctx)))
+ elif expect.has_tsma():
+ tdLog.debug('check explain succeed for sql: %s \ntsma: %s' %
+ (sql, str(expect.used_tsmas)))
+ has_tsma = False
+ for tsma in query_ctx.used_tsmas:
+ has_tsma = has_tsma or tsma.is_tsma_
+ if not has_tsma and len(query_ctx.used_tsmas) > 1:
+ tdLog.exit(
+ f'explain err for sql: {sql}, has multi non tsmas, {query_ctx.used_tsmas}')
+ return query_ctx
+
+ def check_result(self, sql: str, skip_order: bool = False):
+ tdSql.execute("alter local 'querySmaOptimize' '1'")
+ tsma_res = tdSql.getResult(sql)
+
+ tdSql.execute("alter local 'querySmaOptimize' '0'")
+ no_tsma_res = tdSql.getResult(sql)
+
+ if no_tsma_res is None or tsma_res is None:
+ if no_tsma_res != tsma_res:
+ tdLog.exit("comparing tsma res for: %s got different rows of result: without tsma: %s, with tsma: %s" % (
+ sql, str(no_tsma_res), str(tsma_res)))
+ else:
+ return
+
+ if len(no_tsma_res) != len(tsma_res):
+ tdLog.exit("comparing tsma res for: %s got different rows of result: \nwithout tsma: %s\nwith tsma: %s" % (
+ sql, str(no_tsma_res), str(tsma_res)))
+ if skip_order:
+ try:
+ no_tsma_res.sort(
+ key=lambda x: [v is None for v in x] + list(x))
+ tsma_res.sort(key=lambda x: [v is None for v in x] + list(x))
+ except Exception as e:
+ tdLog.exit("comparing tsma res for: %s got different data: \nno tsma res: %s \n tsma res: %s err: %s" % (
+ sql, str(no_tsma_res), str(tsma_res), str(e)))
+
+ for row_no_tsma, row_tsma in zip(no_tsma_res, tsma_res):
+ if row_no_tsma != row_tsma:
+ tdLog.exit("comparing tsma res for: %s got different row data: no tsma row: %s, tsma row: %s \nno tsma res: %s \n tsma res: %s" % (
+ sql, str(row_no_tsma), str(row_tsma), str(no_tsma_res), str(tsma_res)))
+ tdLog.info('result check succeed for sql: %s. \n tsma-res: %s. \nno_tsma-res: %s' %
+ (sql, str(tsma_res), str(no_tsma_res)))
+
+ def check_sql(self, sql: str, expect: TSMAQueryContext):
+ tdLog.debug(f"start to check sql: {sql}")
+ actual_ctx = self.check_explain(sql, expect=expect)
+ tdLog.debug(f"ctx: {actual_ctx}")
+ if actual_ctx.has_tsma():
+ self.check_result(sql, expect.ignore_res_order_)
+
+ def check_sqls(self, sqls, expects):
+ for sql, query_ctx in zip(sqls, expects):
+ self.check_sql(sql, query_ctx)
+
+
+class TSMATesterSQLGeneratorOptions:
+ def __init__(self) -> None:
+ self.ts_min: int = 1537146000000 - 1000 * 60 * 60
+ self.ts_max: int = 1537150999000 + 1000 * 60 * 60
+ self.times: int = 100
+ self.pk_col: str = 'ts'
+ self.column_prefix: str = 'c'
+ self.column_num: int = 9 # c1 - c10
+ self.tags_prefix: str = 't'
+ self.tag_num: int = 6 # t1 - t6
+ self.str_tag_idx: List = [2, 3]
+ self.child_table_name_prefix: str = 't'
+ self.child_table_num: int = 10 # t0 - t9
+ self.interval: bool = False
+ # 70% generating a partition by, 30% no partition by, same as group by
+ self.partition_by: bool = False
+ self.group_by: bool = False
+ # generating no ts range condition is also possible
+ self.where_ts_range: bool = False
+ self.where_tbname_func: bool = False
+ self.where_tag_func: bool = False
+ self.where_col_func: bool = False
+ self.slimit_max = 10
+ self.limit_max = 10
+ self.norm_tb = False
+
+
+class TSMATesterSQLGeneratorRes:
+ def __init__(self):
+ self.has_where_ts_range: bool = False
+ self.has_interval: bool = False
+ self.partition_by: bool = False
+ self.group_by: bool = False
+ self.has_slimit: bool = False
+ self.has_limit: bool = False
+ self.has_user_order_by: bool = False
+
+ def can_ignore_res_order(self):
+ return not (self.has_limit and self.has_slimit)
+
+
+class TSMATestSQLGenerator:
+ def __init__(self, opts: TSMATesterSQLGeneratorOptions = TSMATesterSQLGeneratorOptions()):
+ self.db_name_: str = ''
+ self.tb_name_: str = ''
+ self.ts_scan_range_: List[float] = [
+ float(UsedTsma.TS_MIN), float(UsedTsma.TS_MAX)]
+ self.agg_funcs_: List[str] = []
+ self.tsmas_: List[TSMA] = [] # currently created tsmas
+ self.opts_: TSMATesterSQLGeneratorOptions = opts
+ self.res_: TSMATesterSQLGeneratorRes = TSMATesterSQLGeneratorRes()
+
+ self.select_list_: List[str] = []
+ self.where_list_: List[str] = []
+ self.group_or_partition_by_list: List[str] = []
+ self.interval: str = ''
+
+ def get_depth_one_str_funcs(self, name: str) -> List[str]:
+ concat1 = f'CONCAT({name}, "_concat")'
+ concat2 = f'CONCAT({name}, {name})'
+ concat3 = f'CONCAT({name}, {name}, {name})'
+ start = random.randint(1, 3)
+ len = random.randint(0, 3)
+ substr = f'SUBSTR({name}, {start}, {len})'
+ lower = f'LOWER({name})'
+ ltrim = f'LTRIM({name})'
+ return [concat1, concat2, concat3, substr, substr, lower, lower, ltrim, name]
+
+ def generate_depthed_str_func(self, name: str, depth: int) -> str:
+ if depth == 1:
+ return random.choice(self.get_depth_one_str_funcs(name))
+ name = self.generate_depthed_str_func(name, depth - 1)
+ return random.choice(self.get_depth_one_str_funcs(name))
+
+ def generate_str_func(self, column_name: str, depth: int = 0) -> str:
+ if depth == 0:
+ depth = random.randint(1, 3)
+
+ ret = self.generate_depthed_str_func(column_name, depth)
+ tdLog.debug(f'generating str func: {ret}')
+ return ret
+
+ def get_random_type(self, funcs):
+ rand: int = randrange(1, len(funcs))
+ return funcs[rand-1]()
+
+ def generate_select_list(self, user_select_list: str, partition_by_list: str):
+ res = user_select_list
+ if self.res_.has_interval and random.random() < 0.8:
+ res = res + ',_wstart, _wend'
+ if self.res_.partition_by or self.res_.group_by and random.random() < 0.8:
+ res = res + f',{partition_by_list}'
+ return res
+
+ def generate_order_by(self, user_order_by: str, partition_by_list: str):
+ auto_order_by = 'ORDER BY'
+ has_limit = self.res_.has_limit or self.res_.has_slimit
+ if has_limit and (self.res_.group_by or self.res_.partition_by):
+ auto_order_by = f'{auto_order_by} {partition_by_list},'
+ if has_limit and self.res_.has_interval:
+ auto_order_by = f'{auto_order_by} _wstart, _wend,'
+ if len(user_order_by) > 0:
+ self.res_.has_user_order_by = True
+ auto_order_by = f'{auto_order_by} {user_order_by},'
+ if auto_order_by == 'ORDER BY':
+ return ''
+ else:
+ return auto_order_by[:-1]
+
+ def generate_one(self, select_list: str, possible_tbs: List, order_by_list: str, interval_list: List[str] = []) -> str:
+ tb = random.choice(possible_tbs)
+ where = self.generate_where()
+ interval = self.generate_interval(interval_list)
+ (partition_by, partition_by_list) = self.generate_partition_by()
+ limit = self.generate_limit()
+ auto_select_list = self.generate_select_list(
+ select_list, partition_by_list)
+ order_by = self.generate_order_by(order_by_list, partition_by_list)
+ sql = f"SELECT {auto_select_list} FROM {tb} {where} {partition_by} {partition_by_list} {interval} {order_by} {limit}"
+ tdLog.debug(sql)
+ return sql
+
+ def can_ignore_res_order(self):
+ return self.res_.can_ignore_res_order()
+
+ def generate_where(self) -> str:
+ v = random.random()
+ where = ''
+ if not self.opts_.norm_tb:
+ if v < 0.2:
+ where = f'{self.generate_tbname_where()}'
+ elif v < 0.5:
+ where = f'{self.generate_tag_where()}'
+ elif v < 0.7:
+ op = random.choice(['AND', 'OR'])
+ where = f'{self.generate_tbname_where()} {op} {self.generate_tag_where()}'
+ ts_where = self.generate_ts_where_range()
+ if len(ts_where) > 0 or len(where) > 0:
+ op = ''
+ if len(where) > 0 and len(ts_where) > 0:
+ op = random.choice(['AND', 'AND', 'AND', 'AND', 'OR'])
+ return f'WHERE {ts_where} {op} {where}'
+ return ''
+
+ def generate_str_equal_operator(self, column_name: str, opts: List) -> str:
+ opt = random.choice(opts)
+ return f'{column_name} = "{opt}"'
+
+ # TODO support it
+ def generate_str_in_operator(self, column_name: str, opts: List) -> str:
+ opt = random.choice(opts)
+ IN = f'"{",".join(opts)}"'
+ return f'{column_name} in ({IN})'
+
+ def generate_str_like_operator(self, column_name: str, opts: List) -> str:
+ opt = random.choice(opts)
+ return f'{column_name} like "{opt}"'
+
+ def generate_tbname_where(self) -> str:
+ tbs = []
+ for idx in range(1, self.opts_.tag_num + 1):
+ tbs.append(f'{self.opts_.child_table_name_prefix}{idx}')
+
+ if random.random() < 0.5:
+ return self.generate_str_equal_operator('tbname', tbs)
+ else:
+ return self.generate_str_like_operator('tbname', ['t%', '%2'])
+
+ def generate_tag_where(self) -> str:
+ idx = random.randrange(1, self.opts_.tag_num + 1)
+ if random.random() < 0.5 and idx in self.opts_.str_tag_idx:
+ if random.random() < 0.5:
+ return self.generate_str_equal_operator(f'{self.opts_.tags_prefix}{idx}', [f'tb{random.randint(1,100)}'])
+ else:
+ return self.generate_str_like_operator(f'{self.opts_.tags_prefix}{idx}', ['%1', 'tb%', 'tb1%', '%1%'])
+ else:
+ operator = random.choice(['>', '>=', '<', '<=', '=', '!='])
+ val = random.randint(1, 100)
+ return f'{self.opts_.tags_prefix}{idx} {operator} {val}'
+
+ def generate_timestamp(self, min: float = -1, max: float = 0) -> int:
+ milliseconds_aligned: float = random.randint(int(min), int(max))
+ seconds_aligned = int(milliseconds_aligned / 1000) * 1000
+ if seconds_aligned < min:
+ seconds_aligned = int(min)
+ minutes_aligned = int(milliseconds_aligned / 1000 / 60) * 1000 * 60
+ if minutes_aligned < min:
+ minutes_aligned = int(min)
+ hour_aligned = int(milliseconds_aligned / 1000 /
+ 60 / 60) * 1000 * 60 * 60
+ if hour_aligned < min:
+ hour_aligned = int(min)
+
+ return random.choice([milliseconds_aligned, seconds_aligned, seconds_aligned, minutes_aligned, minutes_aligned, hour_aligned, hour_aligned])
+
+ def generate_ts_where_range(self):
+ if not self.opts_.where_ts_range:
+ return ''
+ left_operators = ['>', '>=', '']
+ right_operators = ['<', '<=', '']
+ left_operator = left_operators[random.randrange(0, 3)]
+ right_operator = right_operators[random.randrange(0, 3)]
+ a = ''
+ left_value = None
+ if left_operator:
+ left_value = self.generate_timestamp(
+ self.opts_.ts_min, self.opts_.ts_max)
+ a += f'{self.opts_.pk_col} {left_operator} {left_value}'
+ if right_operator:
+ if left_value:
+ start = left_value
+ else:
+ start = self.opts_.ts_min
+ right_value = self.generate_timestamp(start, self.opts_.ts_max)
+ if left_operator:
+ a += ' AND '
+ a += f'{self.opts_.pk_col} {right_operator} {right_value}'
+ # tdLog.debug(f'{self.opts_.pk_col} range with: {a}')
+ if len(a) > 0:
+ self.res_.has_where_ts_range = True
+ return a
+
+ def generate_limit(self) -> str:
+ ret = ''
+ can_have_slimit = self.res_.partition_by or self.res_.group_by
+ if can_have_slimit:
+ if random.random() < 0.4:
+ ret = f'SLIMIT {random.randint(0, self.opts_.slimit_max)}'
+ self.res_.has_slimit = True
+ if random.random() < 0.4:
+ self.res_.has_limit = True
+ ret = ret + f' LIMIT {random.randint(0, self.opts_.limit_max)}'
+ return ret
+
+ ## if offset is True, offset cannot be the same as interval
+ def generate_random_offset_sliding(self, interval: str, offset: bool = False) -> str:
+ unit = interval[-1]
+ hasUnit = unit.isalpha()
+ if not hasUnit:
+ start = 1
+ if offset:
+ start = 2
+ ret: int = int(int(interval) / random.randint(start, 5))
+ return str(ret)
+ return ''
+
+ # add sliding offset
+ def generate_interval(self, intervals: List[str]) -> str:
+ if not self.opts_.interval:
+ return ''
+ if random.random() < 0.4: # no interval
+ return ''
+ value = random.choice(intervals)
+ self.res_.has_interval = True
+ has_offset = False
+ offset = ''
+ has_sliding = False
+ sliding = ''
+ num: int = int(value[:-1])
+ unit = value[-1]
+ if has_offset and num > 1:
+ offset = f', {self.generate_random_offset_sliding(value, True)}'
+ if has_sliding:
+ sliding = f'sliding({self.generate_random_offset_sliding(value)})'
+ return f'INTERVAL({value} {offset}) {sliding}'
+
+ def generate_tag_list(self):
+ used_tag_num = random.randrange(1, self.opts_.tag_num + 1)
+ ret = ''
+ for _ in range(used_tag_num):
+ tag_idx = random.randint(1, self.opts_.tag_num)
+ tag_name = self.opts_.tags_prefix + f'{tag_idx}'
+ if random.random() < 0.5 and tag_idx in self.opts_.str_tag_idx:
+ tag_func = self.generate_str_func(tag_name, 2)
+ else:
+ tag_func = tag_name
+ ret = ret + f'{tag_func},'
+ return ret[:-1]
+
+ def generate_tbname_tag_list(self):
+ tag_num = random.randrange(1, self.opts_.tag_num)
+ ret = ''
+ tbname_idx = random.randint(0, tag_num + 1)
+ for i in range(tag_num + 1):
+ if i == tbname_idx:
+ ret = ret + 'tbname,'
+ else:
+ tag_idx = random.randint(1, self.opts_.tag_num)
+ ret = ret + self.opts_.tags_prefix + f'{tag_idx},'
+ return ret[:-1]
+
+ def generate_partition_by(self):
+ if not self.opts_.partition_by and not self.opts_.group_by:
+ return ('', '')
+ # no partition or group
+ if random.random() < 0.3:
+ return ('', '')
+ ret = ''
+ rand = random.random()
+ if rand < 0.4:
+ if random.random() < 0.5:
+ ret = self.generate_str_func('tbname', 3)
+ else:
+ ret = 'tbname'
+ elif rand < 0.8:
+ ret = self.generate_tag_list()
+ else:
+ # tbname and tag
+ ret = self.generate_tbname_tag_list()
+ # tdLog.debug(f'partition by: {ret}')
+ if self.res_.has_interval or random.random() < 0.5:
+ self.res_.partition_by = True
+ return (str('PARTITION BY'), f'{ret}')
+ else:
+ self.res_.group_by = True
+ return (str('GROUP BY'), f'{ret}')
+
+ def generate_where_tbname(self) -> str:
+ return self.generate_str_func('tbname')
+
+ def generate_where_tag(self) -> str:
+ # tag_idx = random.randint(1, self.opts_.tag_num)
+ # tag = self.opts_.tags_prefix + str(tag_idx)
+ return self.generate_str_func('t3')
+
+ def generate_where_conditions(self) -> str:
+
+ pass
+
+ # generate func in tsmas(select list)
+ def _generate_agg_func_for_select(self) -> str:
+ pass
+
+ # order by, limit, having, subquery...
+
+
+class TDTestCase:
+ updatecfgDict = {'asynclog': 0, 'ttlUnit': 1, 'ttlPushInterval': 5, 'ratioOfVnodeStreamThrea': 4, 'maxTsmaNum': 3}
+
+ def __init__(self):
+ self.vgroups = 4
+ self.ctbNum = 10
+ self.rowsPerTbl = 10000
+ self.duraion = '1h'
+
+ def init(self, conn, logSql, replicaVar=1):
+ self.replicaVar = int(replicaVar)
+ tdLog.debug(f"start to excute {__file__}")
+ tdSql.init(conn.cursor(), False)
+ self.tsma_tester: TSMATester = TSMATester(tdSql)
+ self.tsma_sql_generator: TSMATestSQLGenerator = TSMATestSQLGenerator()
+
+ def create_database(self, tsql, dbName, dropFlag=1, vgroups=2, replica=1, duration: str = '1d'):
+ if dropFlag == 1:
+ tsql.execute("drop database if exists %s" % (dbName))
+
+ tsql.execute("create database if not exists %s vgroups %d replica %d duration %s" % (
+ dbName, vgroups, replica, duration))
+ tdLog.debug("complete to create database %s" % (dbName))
+ return
+
+ def create_stable(self, tsql, paraDict):
+ colString = tdCom.gen_column_type_str(
+ colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"])
+ tagString = tdCom.gen_tag_type_str(
+ tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"])
+ sqlString = f"create table if not exists %s.%s (%s) tags (%s)" % (
+ paraDict["dbName"], paraDict["stbName"], colString, tagString)
+ tdLog.debug("%s" % (sqlString))
+ tsql.execute(sqlString)
+ return
+
+ def create_ctable(self, tsql=None, dbName='dbx', stbName='stb', ctbPrefix='ctb', ctbNum=1, ctbStartIdx=0):
+ for i in range(ctbNum):
+ sqlString = "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)" % (dbName, ctbPrefix, i+ctbStartIdx, dbName, stbName, (i+ctbStartIdx) % 5, i+ctbStartIdx + random.randint(
+ 1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100))
+ tsql.execute(sqlString)
+
+ tdLog.debug("complete to create %d child tables by %s.%s" %
+ (ctbNum, dbName, stbName))
+ return
+
+ def init_normal_tb(self, tsql, db_name: str, tb_name: str, rows: int, start_ts: int, ts_step: int):
+ sql = 'CREATE TABLE %s.%s (ts timestamp, c1 INT, c2 INT, c3 INT, c4 double, c5 VARCHAR(255))' % (
+ db_name, tb_name)
+ tsql.execute(sql)
+ sql = 'INSERT INTO %s.%s values' % (db_name, tb_name)
+ for j in range(rows):
+ sql += f'(%d, %d,%d,%d,{random.random()},"varchar_%d"),' % (start_ts + j * ts_step + randrange(500), j %
+ 10 + randrange(200), j % 10, j % 10, j % 10 + randrange(100))
+ tsql.execute(sql)
+
+ def insert_data(self, tsql, dbName, ctbPrefix, ctbNum, rowsPerTbl, batchNum, startTs, tsStep):
+ tdLog.debug("start to insert data ............")
+ tsql.execute("use %s" % dbName)
+ pre_insert = "insert into "
+ sql = pre_insert
+
+ for i in range(ctbNum):
+ rowsBatched = 0
+ sql += " %s.%s%d values " % (dbName, ctbPrefix, i)
+ for j in range(rowsPerTbl):
+ if (i < ctbNum/2):
+ sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d') " % (startTs + j*tsStep + randrange(
+ 500), j % 10 + randrange(100), j % 10 + randrange(200), j % 10, j % 10, j % 10, j % 10, j % 10, j % 10)
+ else:
+ sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d') " % (
+ startTs + j*tsStep + randrange(500), j % 10, j % 10, j % 10, j % 10, j % 10, j % 10)
+ rowsBatched += 1
+ if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)):
+ tsql.execute(sql)
+ rowsBatched = 0
+ if j < rowsPerTbl - 1:
+ sql = "insert into %s.%s%d values " % (dbName, ctbPrefix, i)
+ else:
+ sql = "insert into "
+ if sql != pre_insert:
+ tsql.execute(sql)
+ tdLog.debug("insert data ............ [OK]")
+ return
+
+ def init_data(self, db: str = 'test', ctb_num: int = 10, rows_per_ctb: int = 10000, start_ts: int = 1537146000000, ts_step: int = 500):
+ tdLog.printNoPrefix(
+ "======== prepare test env include database, stable, ctables, and insert data: ")
+ paraDict = {'dbName': db,
+ 'dropFlag': 1,
+ 'vgroups': 2,
+ 'stbName': 'meters',
+ 'colPrefix': 'c',
+ 'tagPrefix': 't',
+ 'colSchema': [{'type': 'INT', 'count': 1}, {'type': 'BIGINT', 'count': 1}, {'type': 'FLOAT', 'count': 1}, {'type': 'DOUBLE', 'count': 1}, {'type': 'smallint', 'count': 1}, {'type': 'tinyint', 'count': 1}, {'type': 'bool', 'count': 1}, {'type': 'binary', 'len': 10, 'count': 1}, {'type': 'nchar', 'len': 10, 'count': 1}],
+ 'tagSchema': [{'type': 'INT', 'count': 1}, {'type': 'nchar', 'len': 20, 'count': 1}, {'type': 'binary', 'len': 20, 'count': 1}, {'type': 'BIGINT', 'count': 1}, {'type': 'smallint', 'count': 1}, {'type': 'DOUBLE', 'count': 1}],
+ 'ctbPrefix': 't',
+ 'ctbStartIdx': 0,
+ 'ctbNum': ctb_num,
+ 'rowsPerTbl': rows_per_ctb,
+ 'batchNum': 3000,
+ 'startTs': start_ts,
+ 'tsStep': ts_step}
+
+ paraDict['vgroups'] = self.vgroups
+ paraDict['ctbNum'] = ctb_num
+ paraDict['rowsPerTbl'] = rows_per_ctb
+
+ tdLog.info("create database")
+ self.create_database(tsql=tdSql, dbName=paraDict["dbName"], dropFlag=paraDict["dropFlag"],
+ vgroups=paraDict["vgroups"], replica=self.replicaVar, duration=self.duraion)
+
+ tdLog.info("create stb")
+ self.create_stable(tsql=tdSql, paraDict=paraDict)
+
+ tdLog.info("create child tables")
+ self.create_ctable(tsql=tdSql, dbName=paraDict["dbName"],
+ stbName=paraDict["stbName"], ctbPrefix=paraDict["ctbPrefix"],
+ ctbNum=paraDict["ctbNum"], ctbStartIdx=paraDict["ctbStartIdx"])
+ self.insert_data(tsql=tdSql, dbName=paraDict["dbName"],
+ ctbPrefix=paraDict["ctbPrefix"], ctbNum=paraDict["ctbNum"],
+ rowsPerTbl=paraDict["rowsPerTbl"], batchNum=paraDict["batchNum"],
+ startTs=paraDict["startTs"], tsStep=paraDict["tsStep"])
+ self.init_normal_tb(tdSql, paraDict['dbName'], 'norm_tb',
+ paraDict['rowsPerTbl'], paraDict['startTs'], paraDict['tsStep'])
+
+ def wait_for_tsma_calculation(self, func_list: list, db: str, tb: str, interval: str, tsma_name: str, timeout_seconds: int =600):
+ start_time = time.time()
+ while True:
+ current_time = time.time()
+ if current_time - start_time > timeout_seconds:
+ error_message = f"Timeout occurred while waiting for TSMA calculation to complete."
+ tdLog.exit(error_message)
+ sql = 'select %s from %s.%s interval(%s)' % (
+ ', '.join(func_list), db, tb, interval)
+ tdLog.debug(
+ f'waiting for tsma {db}.{tsma_name} to be useful with sql {sql}')
+ ctx: TSMAQueryContext = self.tsma_tester.get_tsma_query_ctx(sql)
+ if ctx.has_tsma():
+ if ctx.used_tsmas[0].name == tsma_name + UsedTsma.TSMA_RES_STB_POSTFIX:
+ break
+ elif len(ctx.used_tsmas[0].name) == 32:
+ name = f'1.{db}.{tsma_name}_{tb}'
+ if ctx.used_tsmas[0].name == TSMAQCBuilder().md5(name):
+ break
+ else:
+ time.sleep(1)
+ else:
+ time.sleep(1)
+ else:
+ time.sleep(1)
+ time.sleep(1)
+
+ def create_tsma(self, tsma_name: str, db: str, tb: str, func_list: list, interval: str, check_tsma_calculation : str=True):
+ tdSql.execute('use %s' % db)
+ sql = "CREATE TSMA %s ON %s.%s FUNCTION(%s) INTERVAL(%s)" % (
+ tsma_name, db, tb, ','.join(func_list), interval)
+ tdSql.execute(sql, queryTimes=1)
+ if check_tsma_calculation == True:
+ self.wait_for_tsma_calculation(func_list, db, tb, interval, tsma_name)
+
+ def create_error_tsma(self, tsma_name: str, db: str, tb: str, func_list: list, interval: str, expectedErrno: int):
+ tdSql.execute('use %s' % db)
+ sql = "CREATE TSMA %s ON %s.%s FUNCTION(%s) INTERVAL(%s)" % (
+ tsma_name, db, tb, ','.join(func_list), interval)
+ tdSql.error(sql, expectedErrno)
+
+ def create_recursive_tsma(self, base_tsma_name: str, new_tsma_name: str, db: str, interval: str, tb_name: str, func_list: List[str] = ['avg(c1)']):
+ tdSql.execute('use %s' % db, queryTimes=1)
+ sql = 'CREATE RECURSIVE TSMA %s ON %s.%s INTERVAL(%s)' % (
+ new_tsma_name, db, base_tsma_name, interval)
+ tdSql.execute(sql, queryTimes=1)
+ self.wait_for_tsma_calculation(
+ func_list, db, tb_name, interval, new_tsma_name)
+
+ def drop_tsma(self, tsma_name: str, db: str):
+ sql = 'DROP TSMA %s.%s' % (db, tsma_name)
+ tdSql.execute(sql, queryTimes=1)
+
+ def check_explain_res_has_row(self, plan_str_expect: str, explain_output):
+ plan_found = False
+ for row in explain_output:
+ if str(row).find(plan_str_expect) >= 0:
+ tdLog.debug("plan: [%s] found in: [%s]" %
+ (plan_str_expect, str(row)))
+ plan_found = True
+ break
+ if not plan_found:
+ tdLog.exit("plan: %s not found in res: [%s]" % (
+ plan_str_expect, str(explain_output)))
+
+ def check(self, ctxs: List):
+ for ctx in ctxs:
+ self.tsma_tester.check_sql(ctx.sql, ctx)
+
+ def run(self):
+ self.test_bigger_tsma_interval()
+
+ def test_create_recursive_tsma_interval(self, db: str, tb: str, func, interval: str, recursive_interval: str, succ: bool, code: int):
+ self.create_tsma('tsma1', db, tb, func, interval)
+ sql = f'CREATE RECURSIVE TSMA tsma2 ON {db}.tsma1 INTERVAL({recursive_interval})'
+ if not succ:
+ tdSql.error(sql, code)
+ else:
+ self.create_recursive_tsma('tsma1', 'tsma2', db, recursive_interval, tb, func)
+ self.drop_tsma('tsma2', db)
+ self.drop_tsma('tsma1', db)
+
+ def test_bigger_tsma_interval_query(self, func_list: List):
+ ## 3 tsmas, 12h, 1n, 1y
+ ctxs = []
+ interval_list = ['2h', '8h', '1d', '1n', '3n', '1w', '1y', '2y']
+ opts: TSMATesterSQLGeneratorOptions = TSMATesterSQLGeneratorOptions()
+ opts.interval = True
+ opts.where_ts_range = True
+ for _ in range(1, ROUND):
+ opts.partition_by = True
+ opts.group_by = True
+ opts.norm_tb = False
+ sql_generator = TSMATestSQLGenerator(opts)
+ sql = sql_generator.generate_one(
+ ','.join(func_list), ['db.meters', 'db.meters', 'db.t1', 'db.t9'], '', interval_list)
+ ctxs.append(TSMAQCBuilder().with_sql(sql).ignore_query_table(
+ ).ignore_res_order(sql_generator.can_ignore_res_order()).get_qc())
+ return ctxs
+
+ def test_bigger_tsma_interval(self):
+ db = 'db'
+ tb = 'meters'
+ func = ['max(c1)', 'min(c1)', 'min(c2)', 'max(c2)', 'avg(c1)', 'count(ts)']
+ self.init_data(db,10, 10000, 1500000000000, 11000000)
+ examples = [
+ ('10m', '1h', True), ('10m','1d',True), ('1m', '120s', True), ('1h','1d',True),
+ ('12h', '1y', False), ('1h', '1n', True), ('1h', '1y', True),
+ ('12n', '1y', False), ('2d','1n',False), ('55m', '55h', False), ('7m','7d',False),
+ ]
+ tdSql.execute('use db')
+ for (i, ri, ret) in examples:
+ self.test_create_recursive_tsma_interval(db, tb, func, i, ri, ret, -2147471099)
+
+ self.create_tsma('tsma1', db, tb, func, '1h')
+ self.create_recursive_tsma('tsma1', 'tsma2', db, '1n', tb, func)
+ self.create_recursive_tsma('tsma2', 'tsma3', db, '1y', tb, func)
+ self.check(self.test_bigger_tsma_interval_query(func))
+
+ ctxs = []
+ ctxs.append(TSMAQCBuilder().with_sql('SELECT max(c1) FROM db.meters').should_query_with_tsma('tsma3').get_qc())
+ ctxs.append(TSMAQCBuilder()
+ .with_sql('SELECT max(c1) FROM db.meters WHERE ts > "2024-09-03 18:40:00.324"')
+ .should_query_with_table('meters', '2024-09-03 18:40:00.325', '2024-12-31 23:59:59.999')
+ .should_query_with_tsma('tsma3', '2025-01-01 00:00:00.000', UsedTsma.TS_MAX)
+ .get_qc())
+
+ ctxs.append(TSMAQCBuilder()
+ .with_sql('SELECT max(c1) FROM db.meters WHERE ts >= "2024-09-03 18:00:00.000"')
+ .should_query_with_tsma('tsma1', '2024-09-03 18:00:00.000', '2024-12-31 23:59:59.999')
+ .should_query_with_tsma('tsma3', '2025-01-01 00:00:00.000', UsedTsma.TS_MAX)
+ .get_qc())
+
+ ctxs.append(TSMAQCBuilder()
+ .with_sql('SELECT max(c1) FROM db.meters WHERE ts >= "2024-09-01 00:00:00.000"')
+ .should_query_with_tsma('tsma2', '2024-09-01 00:00:00.000', '2024-12-31 23:59:59.999')
+ .should_query_with_tsma('tsma3', '2025-01-01 00:00:00.000', UsedTsma.TS_MAX)
+ .get_qc())
+
+ ctxs.append(TSMAQCBuilder()
+ .with_sql("SELECT max(c1) FROM db.meters INTERVAL(12n)")
+ .should_query_with_tsma('tsma3')
+ .get_qc())
+
+ ctxs.append(TSMAQCBuilder()
+ .with_sql("SELECT max(c1) FROM db.meters INTERVAL(13n)")
+ .should_query_with_tsma('tsma2')
+ .get_qc())
+
+ ctxs.append(TSMAQCBuilder()
+ .with_sql("SELECT max(c1),min(c1),min(c2),max(c2),avg(c1),count(ts) FROM db.t9 WHERE ts > '2018-09-17 08:16:00'")
+ .should_query_with_table('t9', '2018-09-17 08:16:00.001', '2018-12-31 23:59:59:999')
+ .should_query_with_tsma_ctb('db', 'tsma3', 't9', '2019-01-01')
+ .get_qc())
+
+ ctxs.append(TSMAQCBuilder()
+ .with_sql("SELECT max(c1), _wstart FROM db.meters WHERE ts >= '2024-09-03 18:40:00.324' INTERVAL(1d)")
+ .should_query_with_table('meters', '2024-09-03 18:40:00.324', '2024-09-03 23:59:59:999')
+ .should_query_with_tsma('tsma1', '2024-09-04 00:00:00.000')
+ .get_qc())
+
+ ctxs.append(TSMAQCBuilder()
+ .with_sql("SELECT max(c1), _wstart FROM db.meters WHERE ts >= '2024-09-03 18:40:00.324' INTERVAL(1n)")
+ .should_query_with_table('meters', '2024-09-03 18:40:00.324', '2024-09-30 23:59:59:999')
+ .should_query_with_tsma('tsma2', '2024-10-01 00:00:00.000')
+ .get_qc())
+
+ self.check(ctxs)
+ tdSql.execute('drop database db')
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success(f"{__file__} successfully executed")
+
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/system-test/7-tmq/tmqParamsTest.py b/tests/system-test/7-tmq/tmqParamsTest.py
index b25f23ef11..a323dff19e 100644
--- a/tests/system-test/7-tmq/tmqParamsTest.py
+++ b/tests/system-test/7-tmq/tmqParamsTest.py
@@ -134,6 +134,8 @@ class TDTestCase:
if offset_value != "earliest" and offset_value != "":
if offset_value == "latest":
offset_value_list = list(map(lambda x: (x[-2].replace("wal:", "").replace("earliest", "0").replace("latest", "0").replace(offset_value, "0")), subscription_info))
+ if None in offset_value_list:
+ continue
offset_value_list1 = list(map(lambda x: int(x.split("/")[0]), offset_value_list))
offset_value_list2 = list(map(lambda x: int(x.split("/")[1]), offset_value_list))
tdSql.checkEqual(offset_value_list1 == offset_value_list2, True)
@@ -142,6 +144,8 @@ class TDTestCase:
tdSql.checkEqual(sum(rows_value_list), expected_res)
elif offset_value == "none":
offset_value_list = list(map(lambda x: x[-2], subscription_info))
+ if None in offset_value_list:
+ continue
offset_value_list1 = list(map(lambda x: (x.split("/")[0]), offset_value_list))
tdSql.checkEqual(offset_value_list1, ['none']*len(subscription_info))
rows_value_list = list(map(lambda x: x[-1], subscription_info))
@@ -155,6 +159,8 @@ class TDTestCase:
# tdSql.checkEqual(sum(rows_value_list), expected_res)
else:
offset_value_list = list(map(lambda x: x[-2], subscription_info))
+ if None in offset_value_list:
+ continue
offset_value_list1 = list(map(lambda x: (x.split("/")[0]), offset_value_list))
tdSql.checkEqual(offset_value_list1, [None]*len(subscription_info))
rows_value_list = list(map(lambda x: x[-1], subscription_info))
@@ -162,6 +168,8 @@ class TDTestCase:
else:
if offset_value != "none":
offset_value_list = list(map(lambda x: (x[-2].replace("wal:", "").replace("earliest", "0").replace("latest", "0").replace(offset_value, "0")), subscription_info))
+ if None in offset_value_list:
+ continue
offset_value_list1 = list(map(lambda x: int(x.split("/")[0]), offset_value_list))
offset_value_list2 = list(map(lambda x: int(x.split("/")[1]), offset_value_list))
tdSql.checkEqual(offset_value_list1 <= offset_value_list2, True)
@@ -170,6 +178,8 @@ class TDTestCase:
tdSql.checkEqual(sum(rows_value_list), expected_res)
else:
offset_value_list = list(map(lambda x: x[-2], subscription_info))
+ if None in offset_value_list:
+ continue
offset_value_list1 = list(map(lambda x: (x.split("/")[0]), offset_value_list))
tdSql.checkEqual(offset_value_list1, ['none']*len(subscription_info))
rows_value_list = list(map(lambda x: x[-1], subscription_info))
diff --git a/tests/taosc_test/taoscTest.cpp b/tests/taosc_test/taoscTest.cpp
index 3f49b11b70..1b051f555e 100644
--- a/tests/taosc_test/taoscTest.cpp
+++ b/tests/taosc_test/taoscTest.cpp
@@ -30,7 +30,7 @@
#include "taos.h"
class taoscTest : public ::testing::Test {
- protected:
+protected:
static void SetUpTestCase() {
printf("start test setup.\n");
TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
@@ -188,7 +188,7 @@ TEST_F(taoscTest, taos_query_test) {
void queryCallback2(void* param, void* res, int32_t code) {
ASSERT_TRUE(code == 0);
ASSERT_TRUE(param == pUserParam);
- // After using taos_query_a to query, using taos_fetch_row in the callback will cause blocking.
+ // After using taos_query_a to query, using taos_fetch_row in the callback will cause blocking.
// Reason: schProcessOnCbBegin SCH_LOCK_TASK(pTask)
TAOS_ROW row;
row = taos_fetch_row(res);
@@ -254,7 +254,7 @@ TEST_F(taoscTest, taos_query_a_fetch_row) {
printf("taos_query_a_fetch_row taos_fetch_row start...\n");
while ((row = taos_fetch_row(*pres))) {
- getRecordCounts++;
+ getRecordCounts++;
}
printf("taos_query_a_fetch_row taos_fetch_row end. %p record count:%d.\n", *pres, getRecordCounts);
taos_free_result(*pres);
@@ -264,4 +264,3 @@ TEST_F(taoscTest, taos_query_a_fetch_row) {
printf("taos_query_a_fetch_row test finished.\n");
}
-