Merge branch '3.0' of https://github.com/taosdata/TDengine into fix/TS-4718
This commit is contained in:
yihaoDeng
commit
edabe85952
|
|
@ -25,7 +25,7 @@ create_definition:
|
|||
col_name column_definition
|
||||
|
||||
column_definition:
|
||||
type_name [comment 'string_value']
|
||||
type_name [comment 'string_value'] [PRIMARY KEY]
|
||||
|
||||
table_options:
|
||||
table_option ...
|
||||
|
|
@ -41,11 +41,12 @@ table_option: {
|
|||
**More explanations**
|
||||
|
||||
1. The first column of a table MUST be of type TIMESTAMP. It is automatically set as the primary key.
|
||||
2. The maximum length of the table name is 192 bytes.
|
||||
3. The maximum length of each row is 48k(64k since version 3.0.5.0) bytes, please note that the extra 2 bytes used by each BINARY/NCHAR/GEOMETRY column are also counted.
|
||||
4. The name of the subtable can only consist of characters from the English alphabet, digits and underscore. Table names can't start with a digit. Table names are case insensitive.
|
||||
5. The maximum length in bytes must be specified when using BINARY/NCHAR/GEOMETRY types.
|
||||
6. Escape character "\`" can be used to avoid the conflict between table names and reserved keywords, above rules will be bypassed when using escape character on table names, but the upper limit for the name length is still valid. The table names specified using escape character are case sensitive.
|
||||
2. In addition to the timestamp primary key column, an additional primary key column can be specified using the `PRIMARY KEY` keyword. The second column specified as the primary key must be of type integer or string (varchar).
|
||||
3. The maximum length of the table name is 192 bytes.
|
||||
4. The maximum length of each row is 48k(64k since version 3.0.5.0) bytes, please note that the extra 2 bytes used by each BINARY/NCHAR/GEOMETRY column are also counted.
|
||||
5. The name of the subtable can only consist of characters from the English alphabet, digits and underscore. Table names can't start with a digit. Table names are case insensitive.
|
||||
6. The maximum length in bytes must be specified when using BINARY/NCHAR/GEOMETRY types.
|
||||
7. Escape character "\`" can be used to avoid the conflict between table names and reserved keywords, above rules will be bypassed when using escape character on table names, but the upper limit for the name length is still valid. The table names specified using escape character are case sensitive.
|
||||
For example \`aBc\` and \`abc\` are different table names but `abc` and `aBc` are same table names because they are both converted to `abc` internally.
|
||||
Only ASCII visible characters can be used with escape character.
|
||||
|
||||
|
|
@ -107,6 +108,7 @@ You can perform the following modifications on existing tables:
|
|||
2. DROP COLUMN: deletes a column from the supertable.
|
||||
3. MODIFY COLUMN: changes the length of the data type specified for the column. Note that you can only specify a length greater than the current length.
|
||||
4. RENAME COLUMN: renames a specified column in the table.
|
||||
5. The primary key column of a table cannot be modified or added or deleted using ADD/DROP COLUMN.
|
||||
|
||||
### Add a Column
|
||||
|
||||
|
|
|
|||
|
|
@ -147,6 +147,7 @@ Modifications to the table schema of a supertable take effect on all subtables w
|
|||
- DROP TAG: deletes a tag from the supertable. When you delete a tag from a supertable, it is automatically deleted from all subtables within the supertable.
|
||||
- MODIFY TAG: modifies the definition of a tag in the supertable. You can use this keyword to change the length of a BINARY or NCHAR tag column. Note that you can only specify a length greater than the current length.
|
||||
- RENAME TAG: renames a specified tag in the supertable. When you rename a tag in a supertable, it is automatically renamed in all subtables within the supertable.
|
||||
- Like odinary tables, the primary key of a supertable cannot be modified or added or deleted using ADD/DROP COLUMN.
|
||||
|
||||
### Add a Column
|
||||
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ INSERT INTO
|
|||
```
|
||||
|
||||
6. However, an INSERT statement that writes data to multiple subtables can succeed for some tables and fail for others. This situation is caused because vnodes perform write operations independently of each other. One vnode failing to write data does not affect the ability of other vnodes to write successfully.
|
||||
7. The primary key column value must be specified and cannot be NULL.
|
||||
|
||||
**Normal Syntax**
|
||||
1. The USING clause automatically creates the specified subtable if it does not exist. If it's unknown whether the table already exists, the table can be created automatically while inserting using the SQL statement below. To use this functionality, a STable must be used as template and tag values must be provided. Any tags that you do not specify will be assigned a null value.
|
||||
|
|
|
|||
|
|
@ -503,38 +503,38 @@ TO_CHAR(ts, format_str_literal)
|
|||
|
||||
**Supported Formats**
|
||||
|
||||
| **Format** | **Comment**| **example** |
|
||||
| --- | --- | --- |
|
||||
|AM,am,PM,pm| Meridiem indicator(without periods) | 07:00:00am|
|
||||
|A.M.,a.m.,P.M.,p.m.| Meridiem indicator(with periods)| 07:00:00a.m.|
|
||||
|YYYY,yyyy|year, 4 or more digits| 2023-10-10|
|
||||
|YYY,yyy| year, last 3 digits| 023-10-10|
|
||||
|YY,yy| year, last 2 digits| 23-10-10|
|
||||
|Y,y| year, last digit| 3-10-10|
|
||||
|MONTH|full uppercase of month| 2023-JANUARY-01|
|
||||
|Month|full capitalized month| 2023-January-01|
|
||||
|month|full lowercase of month| 2023-january-01|
|
||||
|MON| abbreviated uppercase of month(3 char)| JAN, SEP|
|
||||
|Mon| abbreviated capitalized month| Jan, Sep|
|
||||
|mon|abbreviated lowercase of month| jan, sep|
|
||||
|MM,mm|month number 01-12|2023-01-01|
|
||||
|DD,dd|month day, 01-31||
|
||||
|DAY|full uppercase of week day|MONDAY|
|
||||
|Day|full capitalized week day|Monday|
|
||||
|day|full lowercase of week day|monday|
|
||||
|DY|abbreviated uppercase of week day|MON|
|
||||
|Dy|abbreviated capitalized week day|Mon|
|
||||
|dy|abbreviated lowercase of week day|mon|
|
||||
|DDD|year day, 001-366||
|
||||
|D,d|week day number, 1-7, Sunday(1) to Saturday(7)||
|
||||
|HH24,hh24|hour of day, 00-23|2023-01-30 23:59:59|
|
||||
|hh12,HH12, hh, HH| hour of day, 01-12|2023-01-30 12:59:59PM|
|
||||
|MI,mi|minute, 00-59||
|
||||
|SS,ss|second, 00-59||
|
||||
|MS,ms|milli second, 000-999||
|
||||
|US,us|micro second, 000000-999999||
|
||||
|NS,ns|nano second, 000000000-999999999||
|
||||
|TZH,tzh|time zone hour|2023-01-30 11:59:59PM +08|
|
||||
| **Format** | **Comment** | **example** |
|
||||
| ------------------- | ---------------------------------------------- | ------------------------- |
|
||||
| AM,am,PM,pm | Meridiem indicator(without periods) | 07:00:00am |
|
||||
| A.M.,a.m.,P.M.,p.m. | Meridiem indicator(with periods) | 07:00:00a.m. |
|
||||
| YYYY,yyyy | year, 4 or more digits | 2023-10-10 |
|
||||
| YYY,yyy | year, last 3 digits | 023-10-10 |
|
||||
| YY,yy | year, last 2 digits | 23-10-10 |
|
||||
| Y,y | year, last digit | 3-10-10 |
|
||||
| MONTH | full uppercase of month | 2023-JANUARY-01 |
|
||||
| Month | full capitalized month | 2023-January-01 |
|
||||
| month | full lowercase of month | 2023-january-01 |
|
||||
| MON | abbreviated uppercase of month(3 char) | JAN, SEP |
|
||||
| Mon | abbreviated capitalized month | Jan, Sep |
|
||||
| mon | abbreviated lowercase of month | jan, sep |
|
||||
| MM,mm | month number 01-12 | 2023-01-01 |
|
||||
| DD,dd | month day, 01-31 | |
|
||||
| DAY | full uppercase of week day | MONDAY |
|
||||
| Day | full capitalized week day | Monday |
|
||||
| day | full lowercase of week day | monday |
|
||||
| DY | abbreviated uppercase of week day | MON |
|
||||
| Dy | abbreviated capitalized week day | Mon |
|
||||
| dy | abbreviated lowercase of week day | mon |
|
||||
| DDD | year day, 001-366 | |
|
||||
| D,d | week day number, 1-7, Sunday(1) to Saturday(7) | |
|
||||
| HH24,hh24 | hour of day, 00-23 | 2023-01-30 23:59:59 |
|
||||
| hh12,HH12, hh, HH | hour of day, 01-12 | 2023-01-30 12:59:59PM |
|
||||
| MI,mi | minute, 00-59 | |
|
||||
| SS,ss | second, 00-59 | |
|
||||
| MS,ms | milli second, 000-999 | |
|
||||
| US,us | micro second, 000000-999999 | |
|
||||
| NS,ns | nano second, 000000000-999999999 | |
|
||||
| TZH,tzh | time zone hour | 2023-01-30 11:59:59PM +08 |
|
||||
|
||||
**More explanations**:
|
||||
- The output format of `Month`, `Day` are left aligined, like`2023-OCTOBER -01`, `2023-SEPTEMBER-01`, `September` is the longest, no paddings. Week days are slimilar.
|
||||
|
|
@ -955,6 +955,7 @@ FIRST(expr)
|
|||
- FIRST(\*) can be used to get the first non-null value of all columns; When querying a super table and multiResultFunctionStarReturnTags is set to 0 (default), FIRST(\*) only returns columns of super table; When set to 1, returns columns and tags of the super table.
|
||||
- NULL will be returned if all the values of the specified column are all NULL
|
||||
- A result will NOT be returned if all the columns in the result set are all NULL
|
||||
- For a table with composite primary key, the data with the smallest primary key value is returned.
|
||||
|
||||
### INTERP
|
||||
|
||||
|
|
@ -988,6 +989,7 @@ ignore_null_values: {
|
|||
- `INTERP` can be applied to supertable by interpolating primary key sorted data of all its childtables. It can also be used with `partition by tbname` when applied to supertable to generate interpolation on each single timeline.
|
||||
- Pseudocolumn `_irowts` can be used along with `INTERP` to return the timestamps associated with interpolation points(support after version 3.0.2.0).
|
||||
- Pseudocolumn `_isfilled` can be used along with `INTERP` to indicate whether the results are original records or data points generated by interpolation algorithm(support after version 3.0.3.0).
|
||||
- For a table with composite primary key, onley the data with the smallest primary key value is used to generate interpolation.
|
||||
|
||||
**Example**
|
||||
|
||||
|
|
@ -1017,6 +1019,7 @@ LAST(expr)
|
|||
- LAST(\*) can be used to get the last non-NULL value of all columns; When querying a super table and multiResultFunctionStarReturnTags is set to 0 (default), LAST(\*) only returns columns of super table; When set to 1, returns columns and tags of the super table.
|
||||
- If the values of a column in the result set are all NULL, NULL is returned for that column; if all columns in the result are all NULL, no result will be returned.
|
||||
- When it's used on a STable, if there are multiple values with the timestamp in the result set, one of them will be returned randomly and it's not guaranteed that the same value is returned if the same query is run multiple times.
|
||||
- For a table with composite primary key, the data with the largest primary key value is returned.
|
||||
|
||||
|
||||
### LAST_ROW
|
||||
|
|
@ -1038,6 +1041,7 @@ LAST_ROW(expr)
|
|||
- LAST_ROW(\*) can be used to get the last value of all columns; When querying a super table and multiResultFunctionStarReturnTags is set to 0 (default), LAST_ROW(\*) only returns columns of super table; When set to 1, returns columns and tags of the super table.
|
||||
- When it's used on a STable, if there are multiple values with the timestamp in the result set, one of them will be returned randomly and it's not guaranteed that the same value is returned if the same query is run multiple times.
|
||||
- Can't be used with `INTERVAL`.
|
||||
- Like `LAST`, the data with the largest primary key value is returned for a table with composite primary key.
|
||||
|
||||
### MAX
|
||||
|
||||
|
|
@ -1144,7 +1148,7 @@ TOP(expr, k)
|
|||
UNIQUE(expr)
|
||||
```
|
||||
|
||||
**Description**: The values that occur the first time in the specified column. The effect is similar to `distinct` keyword.
|
||||
**Description**: The values that occur the first time in the specified column. The effect is similar to `distinct` keyword. For a table with composite primary key, only the data with the smallest primary key value is returned.
|
||||
|
||||
**Return value type**:Same as the data type of the column being operated upon
|
||||
|
||||
|
|
@ -1190,7 +1194,7 @@ ignore_negative: {
|
|||
}
|
||||
```
|
||||
|
||||
**Description**: The derivative of a specific column. The time rage can be specified by parameter `time_interval`, the minimum allowed time range is 1 second (1s); the value of `ignore_negative` can be 0 or 1, 1 means negative values are ignored.
|
||||
**Description**: The derivative of a specific column. The time rage can be specified by parameter `time_interval`, the minimum allowed time range is 1 second (1s); the value of `ignore_negative` can be 0 or 1, 1 means negative values are ignored. For tables with composite primary key, the data with the smallest primary key value is used to calculate the derivative.
|
||||
|
||||
**Return value type**: DOUBLE
|
||||
|
||||
|
|
@ -1213,7 +1217,7 @@ ignore_negative: {
|
|||
}
|
||||
```
|
||||
|
||||
**Description**: The different of each row with its previous row for a specific column. `ignore_negative` can be specified as 0 or 1, the default value is 1 if it's not specified. `1` means negative values are ignored.
|
||||
**Description**: The different of each row with its previous row for a specific column. `ignore_negative` can be specified as 0 or 1, the default value is 1 if it's not specified. `1` means negative values are ignored. For tables with composite primary key, the data with the smallest primary key value is used to calculate the difference.
|
||||
|
||||
**Return value type**:Same as the data type of the column being operated upon
|
||||
|
||||
|
|
@ -1233,7 +1237,7 @@ ignore_negative: {
|
|||
IRATE(expr)
|
||||
```
|
||||
|
||||
**Description**: instantaneous rate on a specific column. The last two samples in the specified time range are used to calculate instantaneous rate. If the last sample value is smaller, then only the last sample value is used instead of the difference between the last two sample values.
|
||||
**Description**: instantaneous rate on a specific column. The last two samples in the specified time range are used to calculate instantaneous rate. If the last sample value is smaller, then only the last sample value is used instead of the difference between the last two sample values. For tables with composite primary key, the data with the smallest primary key value is used to calculate the rate.
|
||||
|
||||
**Return value type**: DOUBLE
|
||||
|
||||
|
|
@ -1323,7 +1327,7 @@ STATEDURATION(expr, oper, val, unit)
|
|||
TWA(expr)
|
||||
```
|
||||
|
||||
**Description**: Time weighted average on a specific column within a time range
|
||||
**Description**: Time weighted average on a specific column within a time range. For tables with composite primary key, the data with the smallest primary key value is used to calculate the average.
|
||||
|
||||
**Return value type**: DOUBLE
|
||||
|
||||
|
|
|
|||
|
|
@ -11,13 +11,14 @@ Because stream processing is built in to TDengine, you are no longer reliant on
|
|||
## Create a Stream
|
||||
|
||||
```sql
|
||||
CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name SUBTABLE(expression) AS subquery
|
||||
CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name[(field1_name, field2_name [PRIMARY KEY], ...)] [TAGS (create_definition [, create_definition] ...)] SUBTABLE(expression) AS subquery
|
||||
stream_options: {
|
||||
TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time]
|
||||
WATERMARK time
|
||||
IGNORE EXPIRED [0|1]
|
||||
DELETE_MARK time
|
||||
FILL_HISTORY [0|1]
|
||||
IGNORE UPDATE [0|1]
|
||||
}
|
||||
|
||||
```
|
||||
|
|
@ -32,7 +33,7 @@ subquery: SELECT [DISTINCT] select_list
|
|||
[window_clause]
|
||||
```
|
||||
|
||||
Session windows, state windows, and sliding windows are supported. When you configure a session or state window for a supertable, you must use PARTITION BY TBNAME.
|
||||
Session windows, state windows, and sliding windows are supported. When you configure a session or state window for a supertable, you must use PARTITION BY TBNAME. If the source table has a composite primary key, state windows, event windows, and count windows are not supported.
|
||||
|
||||
Subtable Clause defines the naming rules of auto-created subtable, you can see more details in below part: Partitions of Stream.
|
||||
|
||||
|
|
|
|||
|
|
@ -191,11 +191,11 @@ Left/Right ASOF Join are supported between super tables, normal tables, child ta
|
|||
|
||||
| **Operator** | **Meaning for Left ASOF Join** |
|
||||
| :-------------: | ------------------------ |
|
||||
| > | Match rows in the right table whose primary key timestamp is less than and the most closed to the left table's primary key timestamp |
|
||||
| >= | Match rows in the right table whose primary key timestamp is less than or equal to and the most closed to the left table's primary key timestamp |
|
||||
| > | Match rows in the right table whose primary key timestamp is less than and the most closed to the left table's primary key timestamp |
|
||||
| >= | Match rows in the right table whose primary key timestamp is less than or equal to and the most closed to the left table's primary key timestamp |
|
||||
| = | Match rows in the right table whose primary key timestamp is equal to the left table's primary key timestamp |
|
||||
| < | Match rows in the right table whose the primary key timestamp is greater than and the most closed to the left table's primary key timestamp |
|
||||
| <= | Match rows in the right table whose primary key timestamp is greater than or equal to and the most closed to the left table's primary key timestamp |
|
||||
| < | Match rows in the right table whose the primary key timestamp is greater than and the most closed to the left table's primary key timestamp |
|
||||
| <= | Match rows in the right table whose primary key timestamp is greater than or equal to and the most closed to the left table's primary key timestamp |
|
||||
|
||||
For Right ASOF Join, the above operators have the opposite meaning.
|
||||
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ where:
|
|||
- `tag_set` will be used as tags, with format like `<tag_key>=<tag_value>,<tag_key>=<tag_value>` Enter a space between `tag_set` and `field_set`.
|
||||
- `field_set`will be used as data columns, with format like `<field_key>=<field_value>,<field_key>=<field_value>` Enter a space between `field_set` and `timestamp`.
|
||||
- `timestamp` is the primary key timestamp corresponding to this row of data
|
||||
- schemaless writing does not support writing data to tables with a second primary key column.
|
||||
|
||||
All data in tag_set is automatically converted to the NCHAR data type and does not require double quotes (").
|
||||
|
||||
|
|
@ -38,24 +39,24 @@ In the schemaless writing data line protocol, each data item in the field_set ne
|
|||
- If there are double quotes on both sides and a B/b prefix, it means VARBINARY type. Hexadecimal start with \x or string can be used in double quotes. For example `B"\x98f46e"` `B"hello"`.
|
||||
- Spaces, equals sign (=), comma (,), double quote ("), and backslash (\\) need to be escaped with a backslash (\\) in front. (All refer to the ASCII character). The rules are as follows:
|
||||
|
||||
| **Serial number** | **Element** | **Escape characters** |
|
||||
| -------- | ----------- | ----------------------------- |
|
||||
| 1 | Measurement | Comma, Space |
|
||||
| 2 | Tag key | Comma, Equals Sign, Space |
|
||||
| 3 | Tag value | Comma, Equals Sign, Space |
|
||||
| 4 | Field key | Comma, Equals Sign, Space |
|
||||
| 5 | Field value | Double quote, Backslash |
|
||||
| **Serial number** | **Element** | **Escape characters** |
|
||||
| ----------------- | ----------- | ------------------------- |
|
||||
| 1 | Measurement | Comma, Space |
|
||||
| 2 | Tag key | Comma, Equals Sign, Space |
|
||||
| 3 | Tag value | Comma, Equals Sign, Space |
|
||||
| 4 | Field key | Comma, Equals Sign, Space |
|
||||
| 5 | Field value | Double quote, Backslash |
|
||||
|
||||
With two contiguous backslashes, the first is interpreted as an escape character. Examples of backslash escape rules are as follows:
|
||||
|
||||
| **Serial number** | **Backslashes** | **Interpreted as** |
|
||||
| -------- | ----------- | ----------------------------- |
|
||||
| 1 | \ | \ |
|
||||
| 2 | \\\\ | \ |
|
||||
| 3 | \\\\\\ | \\\\ |
|
||||
| 4 | \\\\\\\\ | \\\\ |
|
||||
| 5 | \\\\\\\\\\ | \\\\\\ |
|
||||
| 6 | \\\\\\\\\\\\ | \\\\\\ |
|
||||
| **Serial number** | **Backslashes** | **Interpreted as** |
|
||||
| ----------------- | --------------- | ------------------ |
|
||||
| 1 | \ | \ |
|
||||
| 2 | \\\\ | \ |
|
||||
| 3 | \\\\\\ | \\\\ |
|
||||
| 4 | \\\\\\\\ | \\\\ |
|
||||
| 5 | \\\\\\\\\\ | \\\\\\ |
|
||||
| 6 | \\\\\\\\\\\\ | \\\\\\ |
|
||||
|
||||
- Numeric types will be distinguished from data types by the suffix.
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,92 @@
|
|||
|
||||
---
|
||||
|
||||
title: Configurable Column Compression
|
||||
description: Configurable column storage compression method
|
||||
---
|
||||
|
||||
# Configurable Storage Compression
|
||||
|
||||
Since TDengine 3.3.0.0, more advanced compression feature is introduced, you can specify compression or not, the compression method and compression level for each column.
|
||||
|
||||
## Compression Terminology Definition
|
||||
|
||||
### Compression Level Definition
|
||||
|
||||
- Level 1 Compression: Encoding the data, which is essentially a form of compression
|
||||
- Level 2 Compression: Compressing data blocks.
|
||||
|
||||
### Compression Algorithm Level
|
||||
|
||||
In this article, it specifically refers to the level within the secondary compression algorithm, such as zstd, at least 8 levels can be selected, each level has different performance, essentially it is a tradeoff between compression ratio, compression speed, and decompression speed. To avoid the difficulty of choice, it is simplified and defined as the following three levels:
|
||||
|
||||
- high: The highest compression ratio, the worst compression speed and decompression speed.
|
||||
- low: The best compression speed and decompression speed, the lowest compression ratio.
|
||||
- medium: Balancing compression ratio, compression speed, and decompression speed.
|
||||
|
||||
### Compression Algorithm List
|
||||
|
||||
- Encoding algorithm list (Level 1 compression): simple8b, bit-packing, delta-i, delta-d, disabled
|
||||
|
||||
- Compression algorithm list (Level 2 compression): lz4, zlib, zstd, tsz, xz, disabled
|
||||
|
||||
- Default compression algorithm list and applicable range for each data type
|
||||
|
||||
| Data Type | Optional Encoding Algorithm | Default Encoding Algorithm | Optional Compression Algorithm|Default Compression Algorithm| Default Compression Level|
|
||||
| :-----------:|:----------:|:-------:|:-------:|:----------:|:----:|
|
||||
tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium|
|
||||
| bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium|
|
||||
|float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|tsz| medium|
|
||||
|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| medium|
|
||||
|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| medium|
|
||||
|
||||
Note: For floating point types, if configured as tsz, its precision is determined by the global configuration of taosd. If configured as tsz, but the lossy compression flag is not configured, lz4 is used for compression by default.
|
||||
|
||||
## SQL
|
||||
|
||||
### Create Table with Compression
|
||||
|
||||
```sql
|
||||
CREATE [dbname.]tabname (colName colType [ENCODE 'encode_type'] [COMPRESS 'compress_type' [LEVEL 'level'], [, other cerate_definition]...])
|
||||
```
|
||||
|
||||
**Parameter Description**
|
||||
|
||||
- tabname: Super table or ordinary table name
|
||||
- encode_type: Level 1 compression, specific parameters see the above list
|
||||
- compress_type: Level 2 compression, specific parameters see the above list
|
||||
- level: Specifically refers to the level of secondary compression, the default value is medium, supports abbreviation as 'h'/'l'/'m'
|
||||
|
||||
**Function Description**
|
||||
|
||||
- Specify the compression method for the column when creating a table
|
||||
|
||||
### Change Compression Method
|
||||
|
||||
```sql
|
||||
ALTER TABLE [db_name.]tabName MODIFY COLUMN colName [ENCODE 'ecode_type'] [COMPRESS 'compress_type'] [LEVEL "high"]
|
||||
```
|
||||
|
||||
**Parameter Description**
|
||||
|
||||
- tabName: Table name, can be a super table or an ordinary table
|
||||
- colName: The column to change the compression algorithm, can only be a normal column
|
||||
|
||||
**Function Description**
|
||||
|
||||
- Change the compression method of the column
|
||||
|
||||
### View Compression Dethod
|
||||
|
||||
```sql
|
||||
DESCRIBE [dbname.]tabName
|
||||
```
|
||||
|
||||
**Function Description**
|
||||
|
||||
- Display basic information of the column, including type and compression method
|
||||
|
||||
## Compatibility
|
||||
|
||||
- Fully compatible with existing data
|
||||
- Can't be rolled back once you upgrade to 3.3.0.0
|
||||
|
|
@ -23,7 +23,7 @@ create_subtable_clause: {
|
|||
}
|
||||
|
||||
create_definition:
|
||||
col_name column_type
|
||||
col_name column_type [PRIMARY KEY]
|
||||
|
||||
table_options:
|
||||
table_option ...
|
||||
|
|
@ -38,12 +38,13 @@ table_option: {
|
|||
|
||||
**使用说明**
|
||||
|
||||
1. 表的第一个字段必须是 TIMESTAMP，并且系统自动将其设为主键；
|
||||
2. 表名最大长度为 192；
|
||||
3. 表的每行长度不能超过 48KB（从 3.0.5.0 版本开始为 64KB）;（注意：每个 BINARY/NCHAR/GEOMETRY 类型的列还会额外占用 2 个字节的存储位置）
|
||||
4. 子表名只能由字母、数字和下划线组成，且不能以数字开头，不区分大小写
|
||||
5. 使用数据类型 BINARY/NCHAR/GEOMETRY，需指定其最长的字节数，如 BINARY(20)，表示 20 字节；
|
||||
6. 为了兼容支持更多形式的表名，TDengine 引入新的转义符 "\`"，可以让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查。但是同样具有长度限制要求。使用转义字符以后，不再对转义字符中的内容进行大小写统一。
|
||||
1. 表的第一个字段必须是 TIMESTAMP，并且系统自动将其设为主键。
|
||||
2. 除时间戳主键列之外，还可以通过 PRIAMRY KEY 关键字指定第二列为额外的主键列。被指定为主键列的第二列必须为整型或字符串类型（varchar）。
|
||||
3. 表名最大长度为 192。
|
||||
4. 表的每行长度不能超过 48KB（从 3.0.5.0 版本开始为 64KB）;（注意：每个 BINARY/NCHAR/GEOMETRY 类型的列还会额外占用 2 个字节的存储位置）。
|
||||
5. 子表名只能由字母、数字和下划线组成，且不能以数字开头，不区分大小写。
|
||||
6. 使用数据类型 BINARY/NCHAR/GEOMETRY，需指定其最长的字节数，如 BINARY(20)，表示 20 字节。
|
||||
7. 为了兼容支持更多形式的表名，TDengine 引入新的转义符 "\`"，可以让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查。但是同样具有长度限制要求。使用转义字符以后，不再对转义字符中的内容进行大小写统一，
|
||||
例如：\`aBc\` 和 \`abc\` 是不同的表名，但是 abc 和 aBc 是相同的表名。
|
||||
|
||||
**参数说明**
|
||||
|
|
@ -106,6 +107,7 @@ alter_table_option: {
|
|||
2. DROP COLUMN：删除列。
|
||||
3. MODIFY COLUMN：修改列定义，如果数据列的类型是可变长类型，那么可以使用此指令修改其宽度，只能改大，不能改小。
|
||||
4. RENAME COLUMN：修改列名称。
|
||||
5. 普通表的主键列不能被修改，也不能通过 ADD/DROP COLUMN 来添加/删除主键列。
|
||||
|
||||
### 增加列
|
||||
|
||||
|
|
|
|||
|
|
@ -148,6 +148,7 @@ alter_table_option: {
|
|||
- DROP TAG：删除超级表的一个标签。从超级表删除某个标签后，该超级表下的所有子表也会自动删除该标签。
|
||||
- MODIFY TAG：修改超级表的一个标签的列宽度。标签的类型只能是 nchar 和 binary，使用此指令可以修改其宽度，只能改大，不能改小。
|
||||
- RENAME TAG：修改超级表的一个标签的名称。从超级表修改某个标签名后，该超级表下的所有子表也会自动更新该标签名。
|
||||
- 与普通表一样，超级表的主键列不允许被修改，也不允许通过 ADD/DROP COLUMN 来添加或删除主键列。
|
||||
|
||||
### 增加列
|
||||
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ INSERT INTO
|
|||
INSERT INTO d1001 USING meters TAGS('Beijing.Chaoyang', 2) VALUES('a');
|
||||
```
|
||||
6. 对于向多个子表插入数据的情况，依然会有部分数据写入失败，部分数据写入成功的情况。这是因为多个子表可能分布在不同的 VNODE 上，客户端将 INSERT 语句完整解析后，将数据发往各个涉及的 VNODE 上，每个 VNODE 独立进行写入操作。如果某个 VNODE 因为某些原因（比如网络问题或磁盘故障）导致写入失败，并不会影响其他 VNODE 节点的写入。
|
||||
7. 主键列值必须指定且不能为 NULL。
|
||||
|
||||
**正常语法说明**
|
||||
|
||||
|
|
|
|||
|
|
@ -503,38 +503,38 @@ TO_CHAR(ts, format_str_literal)
|
|||
|
||||
**支持的格式**
|
||||
|
||||
| **格式** | **说明**| **例子** |
|
||||
| --- | --- | --- |
|
||||
|AM,am,PM,pm| 无点分隔的上午下午 | 07:00:00am|
|
||||
|A.M.,a.m.,P.M.,p.m.| 有点分隔的上午下午| 07:00:00a.m.|
|
||||
|YYYY,yyyy|年, 4个及以上数字| 2023-10-10|
|
||||
|YYY,yyy| 年, 最后3位数字| 023-10-10|
|
||||
|YY,yy| 年, 最后2位数字| 23-10-10|
|
||||
|Y,y|年, 最后一位数字| 3-10-10|
|
||||
|MONTH|月, 全大写| 2023-JANUARY-01|
|
||||
|Month|月, 首字母大写| 2023-January-01|
|
||||
|month|月, 全小写| 2023-january-01|
|
||||
|MON| 月, 缩写, 全大写(三个字符)| JAN, SEP|
|
||||
|Mon| 月, 缩写, 首字母大写| Jan, Sep|
|
||||
|mon|月, 缩写, 全小写| jan, sep|
|
||||
|MM,mm|月, 数字 01-12|2023-01-01|
|
||||
|DD,dd|月日, 01-31||
|
||||
|DAY|周日, 全大写|MONDAY|
|
||||
|Day|周日, 首字符大写|Monday|
|
||||
|day|周日, 全小写|monday|
|
||||
|DY|周日, 缩写, 全大写|MON|
|
||||
|Dy|周日, 缩写, 首字符大写|Mon|
|
||||
|dy|周日, 缩写, 全小写|mon|
|
||||
|DDD|年日, 001-366||
|
||||
|D,d|周日, 数字, 1-7, Sunday(1) to Saturday(7)||
|
||||
|HH24,hh24|小时, 00-23|2023-01-30 23:59:59|
|
||||
|hh12,HH12, hh, HH| 小时, 01-12|2023-01-30 12:59:59PM|
|
||||
|MI,mi|分钟, 00-59||
|
||||
|SS,ss|秒, 00-59||
|
||||
|MS,ms|毫秒, 000-999||
|
||||
|US,us|微秒, 000000-999999||
|
||||
|NS,ns|纳秒, 000000000-999999999||
|
||||
|TZH,tzh|时区小时|2023-01-30 11:59:59PM +08|
|
||||
| **格式** | **说明** | **例子** |
|
||||
| ------------------- | ----------------------------------------- | ------------------------- |
|
||||
| AM,am,PM,pm | 无点分隔的上午下午 | 07:00:00am |
|
||||
| A.M.,a.m.,P.M.,p.m. | 有点分隔的上午下午 | 07:00:00a.m. |
|
||||
| YYYY,yyyy | 年, 4个及以上数字 | 2023-10-10 |
|
||||
| YYY,yyy | 年, 最后3位数字 | 023-10-10 |
|
||||
| YY,yy | 年, 最后2位数字 | 23-10-10 |
|
||||
| Y,y | 年, 最后一位数字 | 3-10-10 |
|
||||
| MONTH | 月, 全大写 | 2023-JANUARY-01 |
|
||||
| Month | 月, 首字母大写 | 2023-January-01 |
|
||||
| month | 月, 全小写 | 2023-january-01 |
|
||||
| MON | 月, 缩写, 全大写(三个字符) | JAN, SEP |
|
||||
| Mon | 月, 缩写, 首字母大写 | Jan, Sep |
|
||||
| mon | 月, 缩写, 全小写 | jan, sep |
|
||||
| MM,mm | 月, 数字 01-12 | 2023-01-01 |
|
||||
| DD,dd | 月日, 01-31 | |
|
||||
| DAY | 周日, 全大写 | MONDAY |
|
||||
| Day | 周日, 首字符大写 | Monday |
|
||||
| day | 周日, 全小写 | monday |
|
||||
| DY | 周日, 缩写, 全大写 | MON |
|
||||
| Dy | 周日, 缩写, 首字符大写 | Mon |
|
||||
| dy | 周日, 缩写, 全小写 | mon |
|
||||
| DDD | 年日, 001-366 | |
|
||||
| D,d | 周日, 数字, 1-7, Sunday(1) to Saturday(7) | |
|
||||
| HH24,hh24 | 小时, 00-23 | 2023-01-30 23:59:59 |
|
||||
| hh12,HH12, hh, HH | 小时, 01-12 | 2023-01-30 12:59:59PM |
|
||||
| MI,mi | 分钟, 00-59 | |
|
||||
| SS,ss | 秒, 00-59 | |
|
||||
| MS,ms | 毫秒, 000-999 | |
|
||||
| US,us | 微秒, 000000-999999 | |
|
||||
| NS,ns | 纳秒, 000000000-999999999 | |
|
||||
| TZH,tzh | 时区小时 | 2023-01-30 11:59:59PM +08 |
|
||||
|
||||
**使用说明**:
|
||||
- `Month`, `Day`等的输出格式是左对齐的, 右侧添加空格, 如`2023-OCTOBER -01`, `2023-SEPTEMBER-01`, 9月是月份中英文字母数最长的, 因此9月没有空格. 星期类似.
|
||||
|
|
@ -957,6 +957,7 @@ FIRST(expr)
|
|||
- 如果要返回各个列的首个（时间戳最小）非 NULL 值，可以使用 FIRST(\*)；查询超级表，且multiResultFunctionStarReturnTags设置为 0 (默认值) 时，FIRST(\*)只返回超级表的普通列；设置为 1 时，返回超级表的普通列和标签列。
|
||||
- 如果结果集中的某列全部为 NULL 值，则该列的返回结果也是 NULL；
|
||||
- 如果结果集中所有列全部为 NULL 值，则不返回结果。
|
||||
- 对于存在复合主键的表的查询，若最小时间戳的数据有多条，则只有对应的复合主键最小的数据被返回。
|
||||
|
||||
### INTERP
|
||||
|
||||
|
|
@ -989,6 +990,7 @@ ignore_null_values: {
|
|||
- INTERP 作用于超级表时, 会将该超级表下的所有子表数据按照主键列排序后进行插值计算，也可以搭配 PARTITION BY tbname 使用，将结果强制规约到单个时间线。
|
||||
- INTERP 可以与伪列 _irowts 一起使用，返回插值点所对应的时间戳(3.0.2.0版本以后支持)。
|
||||
- INTERP 可以与伪列 _isfilled 一起使用，显示返回结果是否为原始记录或插值算法产生的数据(3.0.3.0版本以后支持)。
|
||||
- INTERP 对于带复合主键的表的查询，若存在相同时间戳的数据，则只有对应的复合主键最小的数据参与运算。
|
||||
|
||||
### LAST
|
||||
|
||||
|
|
@ -1009,6 +1011,7 @@ LAST(expr)
|
|||
- 如果要返回各个列的最后（时间戳最大）一个非 NULL 值，可以使用 LAST(\*)；查询超级表，且multiResultFunctionStarReturnTags设置为 0 (默认值) 时，LAST(\*)只返回超级表的普通列；设置为 1 时，返回超级表的普通列和标签列。
|
||||
- 如果结果集中的某列全部为 NULL 值，则该列的返回结果也是 NULL；如果结果集中所有列全部为 NULL 值，则不返回结果。
|
||||
- 在用于超级表时，时间戳完全一样且同为最大的数据行可能有多个，那么会从中随机返回一条，而并不保证多次运行所挑选的数据行必然一致。
|
||||
- 对于存在复合主键的表的查询，若最大时间戳的数据有多条，则只有对应的复合主键最大的数据被返回。
|
||||
|
||||
|
||||
### LAST_ROW
|
||||
|
|
@ -1029,6 +1032,7 @@ LAST_ROW(expr)
|
|||
- 如果要返回各个列的最后一条记录（时间戳最大），可以使用 LAST_ROW(\*)；查询超级表，且multiResultFunctionStarReturnTags设置为 0 (默认值) 时，LAST_ROW(\*)只返回超级表的普通列；设置为 1 时，返回超级表的普通列和标签列。
|
||||
- 在用于超级表时，时间戳完全一样且同为最大的数据行可能有多个，那么会从中随机返回一条，而并不保证多次运行所挑选的数据行必然一致。
|
||||
- 不能与 INTERVAL 一起使用。
|
||||
- 与 LAST 函数一样，对于存在复合主键的表的查询，若最大时间戳的数据有多条，则只有对应的复合主键最大的数据被返回。
|
||||
|
||||
### MAX
|
||||
|
||||
|
|
@ -1135,7 +1139,7 @@ TOP(expr, k)
|
|||
UNIQUE(expr)
|
||||
```
|
||||
|
||||
**功能说明**：返回该列数据首次出现的值。该函数功能与 distinct 相似。
|
||||
**功能说明**：返回该列数据首次出现的值。该函数功能与 distinct 相似。对于存在复合主键的表的查询，若最小时间戳的数据有多条，则只有对应的复合主键最小的数据被返回。
|
||||
|
||||
**返回数据类型**：同应用的字段。
|
||||
|
||||
|
|
@ -1181,7 +1185,7 @@ ignore_negative: {
|
|||
}
|
||||
```
|
||||
|
||||
**功能说明**：统计表中某列数值的单位变化率。其中单位时间区间的长度可以通过 time_interval 参数指定，最小可以是 1 秒（1s）；ignore_negative 参数的值可以是 0 或 1，为 1 时表示忽略负值。
|
||||
**功能说明**：统计表中某列数值的单位变化率。其中单位时间区间的长度可以通过 time_interval 参数指定，最小可以是 1 秒（1s）；ignore_negative 参数的值可以是 0 或 1，为 1 时表示忽略负值。对于存在复合主键的表的查询，若时间戳相同的数据存在多条，则只有对应的复合主键最小的数据参与运算。
|
||||
|
||||
**返回数据类型**：DOUBLE。
|
||||
|
||||
|
|
@ -1204,7 +1208,7 @@ ignore_negative: {
|
|||
}
|
||||
```
|
||||
|
||||
**功能说明**：统计表中某列的值与前一行对应值的差。 ignore_negative 取值为 0|1 , 可以不填，默认值为 0. 不忽略负值。ignore_negative 为 1 时表示忽略负数。
|
||||
**功能说明**：统计表中某列的值与前一行对应值的差。 ignore_negative 取值为 0|1 , 可以不填，默认值为 0. 不忽略负值。ignore_negative 为 1 时表示忽略负数。对于你存在复合主键的表的查询，若时间戳相同的数据存在多条，则只有对应的复合主键最小的数据参与运算。
|
||||
|
||||
**返回数据类型**：同应用字段。
|
||||
|
||||
|
|
@ -1224,7 +1228,7 @@ ignore_negative: {
|
|||
IRATE(expr)
|
||||
```
|
||||
|
||||
**功能说明**：计算瞬时增长率。使用时间区间中最后两个样本数据来计算瞬时增长速率；如果这两个值呈递减关系，那么只取最后一个数用于计算，而不是使用二者差值。
|
||||
**功能说明**：计算瞬时增长率。使用时间区间中最后两个样本数据来计算瞬时增长速率；如果这两个值呈递减关系，那么只取最后一个数用于计算，而不是使用二者差值。对于存在复合主键的表的查询，若时间戳相同的数据存在多条，则只有对应的复合主键最小的数据参与运算。
|
||||
|
||||
**返回数据类型**：DOUBLE。
|
||||
|
||||
|
|
@ -1314,7 +1318,7 @@ STATEDURATION(expr, oper, val, unit)
|
|||
TWA(expr)
|
||||
```
|
||||
|
||||
**功能说明**：时间加权平均函数。统计表中某列在一段时间内的时间加权平均。
|
||||
**功能说明**：时间加权平均函数。统计表中某列在一段时间内的时间加权平均。对于存在复合主键的表的查询，若时间戳相同的数据存在多条，则只有对应的复合主键最小的数据参与运算。
|
||||
|
||||
**返回数据类型**：DOUBLE。
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ description: 流式计算的相关 SQL 的详细语法
|
|||
## 创建流式计算
|
||||
|
||||
```sql
|
||||
CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name[(field1_name, ...)] [TAGS (create_definition [, create_definition] ...)] SUBTABLE(expression) AS subquery
|
||||
CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name[(field1_name, field2_name [PRIMARY KEY], ...)] [TAGS (create_definition [, create_definition] ...)] SUBTABLE(expression) AS subquery
|
||||
stream_options: {
|
||||
TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time]
|
||||
WATERMARK time
|
||||
|
|
@ -30,9 +30,9 @@ subquery: SELECT select_list
|
|||
[window_clause]
|
||||
```
|
||||
|
||||
支持会话窗口、状态窗口、滑动窗口、事件窗口和计数窗口，其中，状态窗口、事件窗口和计数窗口搭配超级表时必须与partition by tbname一起使用
|
||||
支持会话窗口、状态窗口、滑动窗口、事件窗口和计数窗口，其中，状态窗口、事件窗口和计数窗口搭配超级表时必须与partition by tbname一起使用。对于数据源表是复合主键的流，不支持状态窗口、事件窗口、计数窗口的计算。
|
||||
|
||||
stb_name 是保存计算结果的超级表的表名，如果该超级表不存在，会自动创建；如果已存在，则检查列的schema信息。详见 写入已存在的超级表
|
||||
stb_name 是保存计算结果的超级表的表名，如果该超级表不存在，会自动创建；如果已存在，则检查列的schema信息。详见 写入已存在的超级表。
|
||||
|
||||
TAGS 子句定义了流计算中创建TAG的规则，可以为每个partition对应的子表生成自定义的TAG值，详见 自定义TAG
|
||||
```sql
|
||||
|
|
|
|||
|
|
@ -185,11 +185,11 @@ SELECT ... FROM table_name1 LEFT|RIGHT ASOF JOIN table_name2 [ON ...] [JLIMIT jl
|
|||
|
||||
| **运算符** | **Left ASOF 时含义** |
|
||||
| :-------------: | ------------------------ |
|
||||
| > | 匹配右表中主键时间戳小于左表主键时间戳且时间戳最接近的数据行 |
|
||||
| >= | 匹配右表中主键时间戳小于等于左表主键时间戳且时间戳最接近的数据行 |
|
||||
| > | 匹配右表中主键时间戳小于左表主键时间戳且时间戳最接近的数据行 |
|
||||
| >= | 匹配右表中主键时间戳小于等于左表主键时间戳且时间戳最接近的数据行 |
|
||||
| = | 匹配右表中主键时间戳等于左表主键时间戳的行 |
|
||||
| < | 匹配右表中主键时间戳大于左表主键时间戳且时间戳最接近的数据行 |
|
||||
| <= | 匹配右表中主键时间戳大于等于左表主键时间戳且时间戳最接近的数据行 |
|
||||
| < | 匹配右表中主键时间戳大于左表主键时间戳且时间戳最接近的数据行 |
|
||||
| <= | 匹配右表中主键时间戳大于等于左表主键时间戳且时间戳最接近的数据行 |
|
||||
|
||||
对于 Right ASOF 来说，上述运算符含义正好相反。
|
||||
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ measurement,tag_set field_set timestamp
|
|||
- tag_set 将作为标签数据，其格式形如 `<tag_key>=<tag_value>,<tag_key>=<tag_value>`，也即可以使用英文逗号来分隔多个标签数据。它与 field_set 之间使用一个半角空格来分隔。
|
||||
- field_set 将作为普通列数据，其格式形如 `<field_key>=<field_value>,<field_key>=<field_value>`，同样是使用英文逗号来分隔多个普通列的数据。它与 timestamp 之间使用一个半角空格来分隔。
|
||||
- timestamp 即本行数据对应的主键时间戳。
|
||||
- 无模式写入不支持含第二主键列的表的数据写入。
|
||||
|
||||
tag_set 中的所有的数据自动转化为 nchar 数据类型，并不需要使用双引号（")。
|
||||
|
||||
|
|
|
|||
|
|
@ -201,7 +201,7 @@ TDengine 采用数据驱动的方式让缓存中的数据写入硬盘进行持
|
|||
|
||||
除此之外，TDengine 也提供了数据分级存储的功能，将不同时间段的数据存储在挂载的不同介质上的目录里，从而实现不同“热度”的数据存储在不同的存储介质上，充分利用存储，节约成本。比如，最新采集的数据需要经常访问，对硬盘的读取性能要求高，那么用户可以配置将这些数据存储在 SSD 盘上。超过一定期限的数据，查询需求量没有那么高，那么可以存储在相对便宜的 HDD 盘上。
|
||||
|
||||
多级存储支持 3 级，每级最多可配置 16 个挂载点。
|
||||
多级存储支持 3 级，每级最多可配置 128 个挂载点。
|
||||
|
||||
TDengine 多级存储配置方式如下（在配置文件/etc/taos/taos.cfg 中）：
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,91 @@
|
|||
---
|
||||
title: 可配置压缩算法
|
||||
description: 可配置压缩算法
|
||||
---
|
||||
|
||||
# 可配置存储压缩
|
||||
|
||||
从 TDengine 3.3.0.0 版本开始，TDengine 提供了更高级的压缩功能，用户可以在建表时针对每一列配置是否进行压缩、以及使用的压缩算法和压缩级别。
|
||||
|
||||
## 压缩术语定义
|
||||
|
||||
### 压缩等级
|
||||
|
||||
- 一级压缩：对数据进行编码，本质也是一种压缩
|
||||
- 二级压缩：在编码的基础上对数据块进行压缩
|
||||
|
||||
### 压缩级别
|
||||
|
||||
在本文中特指二级压缩算法内部的级别，比如zstd，至少8个level可选，每个level 下都有不同表现，本质是压缩率、压缩速度、解压速度之间的 tradeoff，为了避免选择困难，特简化定义为如下三种级别：
|
||||
|
||||
- high：压缩率最高，压缩速度和解压速度相对最差。
|
||||
- low：压缩速度和解压速度最好，压缩率相对最低。
|
||||
- medium：兼顾压缩率、压缩速度和解压速度。
|
||||
|
||||
### 压缩算法列表
|
||||
|
||||
- 编码算法列表（一级压缩):simple8b, bit-packing,delta-i, delta-d, disabled
|
||||
|
||||
- 压缩算法列表(二级压缩): lz4、zlib、zstd、tsz、xz、disabled
|
||||
|
||||
- 各个数据类型的默认压缩算法列表和适用范围
|
||||
|
||||
| 数据类型 | 可选编码算法 | 编码算法默认值 | 可选压缩算法|可选压缩算法| 压缩等级默认值|
|
||||
| :-----------:|:----------:|:-------:|:-------:|:----------:|:----:|
|
||||
tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium|
|
||||
| bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium|
|
||||
|float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|tsz| medium|
|
||||
|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| medium|
|
||||
|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| medium|
|
||||
|
||||
注意: 针对浮点类型，如果配置为tsz, 其精度由taosd的全局配置决定，如果配置为tsz, 但是没有配置有损压缩标志, 则使用lz4进行压缩
|
||||
|
||||
## SQL 语法
|
||||
|
||||
### 建表时指定压缩
|
||||
|
||||
```sql
|
||||
CREATE [dbname.]tabname (colName colType [ENCODE 'encode_type'] [COMPRESS 'compress_type' [LEVEL 'level'], [, other cerate_definition]...])
|
||||
```
|
||||
|
||||
**参数说明**
|
||||
|
||||
- tabname：超级表或者普通表名称
|
||||
- encode_type: 一级压缩，具体参数见上面列表
|
||||
- compress_type: 二级压缩,具体参数见上面列表
|
||||
- level: 特指二级压缩的级别，默认值为medium, 支持简写为 'h'/'l'/'m'
|
||||
|
||||
**功能说明**
|
||||
|
||||
- 创建表的时候指定列的压缩方式
|
||||
|
||||
### 更改列的压缩方式
|
||||
|
||||
```sql
|
||||
ALTER TABLE [db_name.]tabName MODIFY COLUMN colName [ENCODE 'ecode_type'] [COMPRESS 'compress_type'] [LEVEL "high"]
|
||||
|
||||
```
|
||||
|
||||
**参数说明**
|
||||
|
||||
- tabName: 表名，可以为超级表、普通表
|
||||
- colName: 待更改压缩算法的列, 只能为普通列
|
||||
|
||||
**功能说明**
|
||||
|
||||
- 更改列的压缩方式
|
||||
|
||||
### 查看列的压缩方式
|
||||
|
||||
```sql
|
||||
DESCRIBE [dbname.]tabName
|
||||
```
|
||||
|
||||
**功能说明**
|
||||
|
||||
- 显示列的基本信息，包括类型、压缩方式
|
||||
|
||||
## 兼容性
|
||||
|
||||
- 完全兼容已经存在的数据
|
||||
- 从更低版本升级到 3.3.0.0 后不能回退
|
||||
|
|
@ -15,7 +15,7 @@
|
|||
|
||||
#ifndef _TD_COMMON_DEF_H_
|
||||
#define _TD_COMMON_DEF_H_
|
||||
// #include "taosdef.h"
|
||||
|
||||
#include "tarray.h"
|
||||
#include "tmsg.h"
|
||||
#include "tvariant.h"
|
||||
|
|
@ -412,6 +412,7 @@ typedef struct STUidTagInfo {
|
|||
#define UD_TAG_COLUMN_INDEX 2
|
||||
|
||||
int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime);
|
||||
int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol);
|
||||
|
||||
#define TSMA_RES_STB_POSTFIX "_tsma_res_stb_"
|
||||
#define MD5_OUTPUT_LEN 32
|
||||
|
|
|
|||
|
|
@ -260,7 +260,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
|
|||
bool tsc);
|
||||
void taosCleanupCfg();
|
||||
|
||||
int32_t taosCfgDynamicOptions(SConfig *pCfg, char *name, bool forServer);
|
||||
int32_t taosCfgDynamicOptions(SConfig *pCfg, const char *name, bool forServer);
|
||||
|
||||
struct SConfig *taosGetCfg();
|
||||
|
||||
|
|
|
|||
|
|
@ -424,7 +424,7 @@ typedef struct STaskOutputInfo {
|
|||
};
|
||||
int8_t type;
|
||||
STokenBucket* pTokenBucket;
|
||||
SArray* pDownstreamUpdateList;
|
||||
SArray* pNodeEpsetUpdateList;
|
||||
} STaskOutputInfo;
|
||||
|
||||
typedef struct SUpstreamInfo {
|
||||
|
|
@ -435,6 +435,7 @@ typedef struct SUpstreamInfo {
|
|||
typedef struct SDownstreamStatusInfo {
|
||||
int64_t reqId;
|
||||
int32_t taskId;
|
||||
int32_t vgId;
|
||||
int64_t rspTs;
|
||||
int32_t status;
|
||||
} SDownstreamStatusInfo;
|
||||
|
|
@ -445,6 +446,8 @@ typedef struct STaskCheckInfo {
|
|||
int32_t notReadyTasks;
|
||||
int32_t inCheckProcess;
|
||||
int32_t stopCheckProcess;
|
||||
int32_t notReadyRetryCount;
|
||||
int32_t timeoutRetryCount;
|
||||
tmr_h checkRspTmr;
|
||||
TdThreadMutex checkInfoLock;
|
||||
} STaskCheckInfo;
|
||||
|
|
@ -484,7 +487,7 @@ struct SStreamTask {
|
|||
SSHashObj* pNameMap;
|
||||
void* pBackend;
|
||||
int8_t subtableWithoutMd5;
|
||||
char reserve[255];
|
||||
char reserve[256];
|
||||
};
|
||||
|
||||
typedef int32_t (*startComplete_fn_t)(struct SStreamMeta*);
|
||||
|
|
@ -845,12 +848,9 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, void* pTask, char* key);
|
|||
bool streamTaskIsSinkTask(const SStreamTask* pTask);
|
||||
int32_t streamTaskSendCheckpointReq(SStreamTask* pTask);
|
||||
|
||||
int32_t streamTaskAddReqInfo(STaskCheckInfo* pInfo, int64_t reqId, int32_t taskId, const char* id);
|
||||
int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t status, int64_t rspTs, int64_t reqId,
|
||||
int32_t* pNotReady, const char* id);
|
||||
void streamTaskCleanCheckInfo(STaskCheckInfo* pInfo);
|
||||
int32_t streamTaskStartMonitorCheckRsp(SStreamTask* pTask);
|
||||
int32_t streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id);
|
||||
void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo);
|
||||
|
||||
void streamTaskStatusInit(STaskStatusEntry* pEntry, const SStreamTask* pTask);
|
||||
void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc);
|
||||
|
|
|
|||
|
|
@ -98,34 +98,35 @@ typedef struct {
|
|||
const char *value;
|
||||
} SConfigPair;
|
||||
|
||||
typedef struct SConfig {
|
||||
ECfgSrcType stype;
|
||||
SArray *array;
|
||||
} SConfig;
|
||||
|
||||
SConfig *cfgInit();
|
||||
int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const void *sourceStr);
|
||||
int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs); // SConfigPair
|
||||
void cfgCleanup(SConfig *pCfg);
|
||||
typedef struct SConfig SConfig;
|
||||
typedef struct SConfigIter SConfigIter;
|
||||
|
||||
SConfig *cfgInit();
|
||||
int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const void *sourceStr);
|
||||
int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs); // SConfigPair
|
||||
void cfgCleanup(SConfig *pCfg);
|
||||
int32_t cfgGetSize(SConfig *pCfg);
|
||||
SConfigItem *cfgGetItem(SConfig *pCfg, const char *name);
|
||||
int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype);
|
||||
SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName);
|
||||
int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype, bool lock);
|
||||
int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer);
|
||||
|
||||
int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer);
|
||||
SConfigIter *cfgCreateIter(SConfig *pConf);
|
||||
SConfigItem *cfgNextIter(SConfigIter *pIter);
|
||||
void cfgDestroyIter(SConfigIter *pIter);
|
||||
void cfgLock(SConfig *pCfg);
|
||||
void cfgUnLock(SConfig *pCfg);
|
||||
|
||||
// clang-format off
|
||||
int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope, int8_t dynScope);
|
||||
int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope,
|
||||
int8_t dynScope);
|
||||
int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope,
|
||||
int8_t dynScope);
|
||||
int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope,
|
||||
int8_t dynScope);
|
||||
int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, int8_t dynScope);
|
||||
int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, int8_t dynScope);
|
||||
int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope, int8_t dynScope);
|
||||
int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope);
|
||||
int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope);
|
||||
int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope);
|
||||
int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope);
|
||||
int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope);
|
||||
// clang-format on
|
||||
|
||||
const char *cfgStypeStr(ECfgSrcType type);
|
||||
const char *cfgDtypeStr(ECfgDataType type);
|
||||
|
|
|
|||
|
|
@ -188,8 +188,8 @@ typedef enum ELogicConditionType {
|
|||
LOGIC_COND_TYPE_NOT,
|
||||
} ELogicConditionType;
|
||||
|
||||
#define ENCRYPTED_LEN(len) (len/16) * 16 + (len%16?1:0) * 16
|
||||
#define ENCRYPT_KEY_LEN 16
|
||||
#define ENCRYPTED_LEN(len) (len / 16) * 16 + (len % 16 ? 1 : 0) * 16
|
||||
#define ENCRYPT_KEY_LEN 16
|
||||
#define ENCRYPT_KEY_LEN_MIN 8
|
||||
|
||||
#define TSDB_INT32_ID_LEN 11
|
||||
|
|
@ -525,7 +525,7 @@ typedef enum ELogicConditionType {
|
|||
#define TSDB_ARB_DUMMY_TIME 4765104000000 // 2121-01-01 00:00:00.000, :P
|
||||
|
||||
#define TFS_MAX_TIERS 3
|
||||
#define TFS_MAX_DISKS_PER_TIER 16
|
||||
#define TFS_MAX_DISKS_PER_TIER 128
|
||||
#define TFS_MAX_DISKS (TFS_MAX_TIERS * TFS_MAX_DISKS_PER_TIER)
|
||||
#define TFS_MIN_LEVEL 0
|
||||
#define TFS_MAX_LEVEL (TFS_MAX_TIERS - 1)
|
||||
|
|
@ -535,7 +535,7 @@ typedef enum ELogicConditionType {
|
|||
|
||||
enum { TRANS_STAT_INIT = 0, TRANS_STAT_EXECUTING, TRANS_STAT_EXECUTED, TRANS_STAT_ROLLBACKING, TRANS_STAT_ROLLBACKED };
|
||||
enum { TRANS_OPER_INIT = 0, TRANS_OPER_EXECUTE, TRANS_OPER_ROLLBACK };
|
||||
enum { ENCRYPT_KEY_STAT_UNKNOWN = 0, ENCRYPT_KEY_STAT_UNSET, ENCRYPT_KEY_STAT_SET, ENCRYPT_KEY_STAT_LOADED};
|
||||
enum { ENCRYPT_KEY_STAT_UNKNOWN = 0, ENCRYPT_KEY_STAT_UNSET, ENCRYPT_KEY_STAT_SET, ENCRYPT_KEY_STAT_LOADED };
|
||||
|
||||
typedef struct {
|
||||
char dir[TSDB_FILENAME_LEN];
|
||||
|
|
|
|||
|
|
@ -57,9 +57,9 @@ else
|
|||
arch=$cpuType
|
||||
fi
|
||||
|
||||
echo "${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r ${arch} -e taoskeeper"
|
||||
echo "${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r ${arch} -e taoskeeper -t ver-${tdengine_ver}"
|
||||
echo "$top_dir=${top_dir}"
|
||||
taoskeeper_binary=`${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r $arch -e taoskeeper`
|
||||
taoskeeper_binary=`${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r $arch -e taoskeeper -t ver-${tdengine_ver}`
|
||||
echo "taoskeeper_binary: ${taoskeeper_binary}"
|
||||
|
||||
# copy config files
|
||||
|
|
@ -76,6 +76,13 @@ if [ -f "${compile_dir}/test/cfg/taosadapter.service" ]; then
|
|||
cp ${compile_dir}/test/cfg/taosadapter.service ${pkg_dir}${install_home_path}/cfg || :
|
||||
fi
|
||||
|
||||
if [ -f "%{_compiledir}/../../../explorer/target/taos-explorer.service" ]; then
|
||||
cp %{_compiledir}/../../../explorer/target/taos-explorer.service ${pkg_dir}${install_home_path}/cfg || :
|
||||
fi
|
||||
if [ -f "%{_compiledir}/../../../explorer/server/example/explorer.toml" ]; then
|
||||
cp %{_compiledir}/../../../explorer/server/example/explorer.toml ${pkg_dir}${install_home_path}/cfg || :
|
||||
fi
|
||||
|
||||
cp ${taoskeeper_binary} ${pkg_dir}${install_home_path}/bin
|
||||
#cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_path}/init.d
|
||||
cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script
|
||||
|
|
@ -93,6 +100,10 @@ if [ -f "${compile_dir}/build/bin/taosadapter" ]; then
|
|||
cp ${compile_dir}/build/bin/taosadapter ${pkg_dir}${install_home_path}/bin ||:
|
||||
fi
|
||||
|
||||
if [ -f "${compile_dir}/../../../explorer/target/release/taos-explorer" ]; then
|
||||
cp ${compile_dir}/../../../explorer/target/release/taos-explorer ${pkg_dir}${install_home_path}/bin ||:
|
||||
fi
|
||||
|
||||
cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin
|
||||
cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver
|
||||
[ -f ${compile_dir}/build/lib/${wslibfile} ] && cp ${compile_dir}/build/lib/${wslibfile} ${pkg_dir}${install_home_path}/driver ||:
|
||||
|
|
|
|||
|
|
@ -72,6 +72,14 @@ if [ -f %{_compiledir}/../build-taoskeeper/taoskeeper.service ]; then
|
|||
cp %{_compiledir}/../build-taoskeeper/taoskeeper.service %{buildroot}%{homepath}/cfg ||:
|
||||
fi
|
||||
|
||||
if [ -f %{_compiledir}/../../../explorer/target/taos-explorer.service ]; then
|
||||
cp %{_compiledir}/../../../explorer/target/taos-explorer.service %{buildroot}%{homepath}/cfg ||:
|
||||
fi
|
||||
|
||||
if [ -f %{_compiledir}/../../../explorer/server/example/explorer.toml ]; then
|
||||
cp %{_compiledir}/../../../explorer/server/example/explorer.toml %{buildroot}%{homepath}/cfg ||:
|
||||
fi
|
||||
|
||||
#cp %{_compiledir}/../packaging/rpm/taosd %{buildroot}%{homepath}/init.d
|
||||
cp %{_compiledir}/../packaging/tools/post.sh %{buildroot}%{homepath}/script
|
||||
cp %{_compiledir}/../packaging/tools/preun.sh %{buildroot}%{homepath}/script
|
||||
|
|
@ -84,6 +92,10 @@ cp %{_compiledir}/build/bin/udfd %{buildroot}%{homepath}/bin
|
|||
cp %{_compiledir}/build/bin/taosBenchmark %{buildroot}%{homepath}/bin
|
||||
cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin
|
||||
|
||||
if [ -f %{_compiledir}/../../../explorer/target/release/taos-explorer ]; then
|
||||
cp %{_compiledir}/../../../explorer/target/release/taos-explorer %{buildroot}%{homepath}/bin
|
||||
fi
|
||||
|
||||
if [ -f %{_compiledir}/../build-taoskeeper/taoskeeper ]; then
|
||||
cp %{_compiledir}/../build-taoskeeper/taoskeeper %{buildroot}%{homepath}/bin
|
||||
fi
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -231,12 +231,8 @@ fi
|
|||
|
||||
if [ "$verMode" == "cluster" ]; then
|
||||
sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh
|
||||
sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" remove_temp.sh
|
||||
sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" remove_temp.sh
|
||||
sed -i "s/configFile2=\"taos.cfg\"/configFile2=\"${clientName2}.cfg\"/g" remove_temp.sh
|
||||
sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" remove_temp.sh
|
||||
cusDomain=`echo "${cusEmail2}" | sed 's/^[^@]*@//'`
|
||||
sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusDomain}\"/g" remove_temp.sh
|
||||
sed -i "s/PREFIX=\"taos\"/PREFIX=\"${serverName2}\"/g" remove_temp.sh
|
||||
sed -i "s/productName=\"TDengine\"/productName=\"${productName2}\"/g" remove_temp.sh
|
||||
mv remove_temp.sh ${install_dir}/bin/remove.sh
|
||||
fi
|
||||
if [ "$verMode" == "cloud" ]; then
|
||||
|
|
@ -262,12 +258,10 @@ cp ${install_files} ${install_dir}
|
|||
cp ${install_dir}/install.sh install_temp.sh
|
||||
if [ "$verMode" == "cluster" ]; then
|
||||
sed -i 's/verMode=edge/verMode=cluster/g' install_temp.sh
|
||||
sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" install_temp.sh
|
||||
sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" install_temp.sh
|
||||
sed -i "s/configFile2=\"taos.cfg\"/configFile2=\"${clientName2}.cfg\"/g" install_temp.sh
|
||||
sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" install_temp.sh
|
||||
sed -i "s/PREFIX=\"taos\"/PREFIX=\"${serverName2}\"/g" install_temp.sh
|
||||
sed -i "s/productName=\"TDengine\"/productName=\"${productName2}\"/g" install_temp.sh
|
||||
cusDomain=`echo "${cusEmail2}" | sed 's/^[^@]*@//'`
|
||||
sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusDomain}\"/g" install_temp.sh
|
||||
sed -i "s/emailName=\"taosdata.com\"/emailName=\"${cusDomain}\"/g" install_temp.sh
|
||||
mv install_temp.sh ${install_dir}/install.sh
|
||||
fi
|
||||
if [ "$verMode" == "cloud" ]; then
|
||||
|
|
@ -367,8 +361,7 @@ if [ "$verMode" == "cluster" ]; then
|
|||
|
||||
# copy taosx
|
||||
if [ -d ${top_dir}/../enterprise/src/plugins/taosx/release/taosx ]; then
|
||||
cp -r ${top_dir}/../enterprise/src/plugins/taosx/release/taosx ${install_dir}
|
||||
cp ${top_dir}/../enterprise/packaging/install_taosx.sh ${install_dir}/taosx
|
||||
cp -r ${top_dir}/../enterprise/src/plugins/taosx/release/taosx ${install_dir}
|
||||
cp ${top_dir}/../enterprise/src/plugins/taosx/packaging/uninstall.sh ${install_dir}/taosx
|
||||
sed -i 's/target=\"\"/target=\"taosx\"/g' ${install_dir}/taosx/uninstall.sh
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -770,6 +770,32 @@ int taos_init() {
|
|||
return tscInitRes;
|
||||
}
|
||||
|
||||
const char* getCfgName(TSDB_OPTION option) {
|
||||
const char* name = NULL;
|
||||
|
||||
switch (option) {
|
||||
case TSDB_OPTION_SHELL_ACTIVITY_TIMER:
|
||||
name = "shellActivityTimer";
|
||||
break;
|
||||
case TSDB_OPTION_LOCALE:
|
||||
name = "locale";
|
||||
break;
|
||||
case TSDB_OPTION_CHARSET:
|
||||
name = "charset";
|
||||
break;
|
||||
case TSDB_OPTION_TIMEZONE:
|
||||
name = "timezone";
|
||||
break;
|
||||
case TSDB_OPTION_USE_ADAPTER:
|
||||
name = "useAdapter";
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return name;
|
||||
}
|
||||
|
||||
int taos_options_imp(TSDB_OPTION option, const char *str) {
|
||||
if (option == TSDB_OPTION_CONFIGDIR) {
|
||||
#ifndef WINDOWS
|
||||
|
|
@ -799,39 +825,26 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
|
|||
|
||||
SConfig *pCfg = taosGetCfg();
|
||||
SConfigItem *pItem = NULL;
|
||||
const char *name = getCfgName(option);
|
||||
|
||||
switch (option) {
|
||||
case TSDB_OPTION_SHELL_ACTIVITY_TIMER:
|
||||
pItem = cfgGetItem(pCfg, "shellActivityTimer");
|
||||
break;
|
||||
case TSDB_OPTION_LOCALE:
|
||||
pItem = cfgGetItem(pCfg, "locale");
|
||||
break;
|
||||
case TSDB_OPTION_CHARSET:
|
||||
pItem = cfgGetItem(pCfg, "charset");
|
||||
break;
|
||||
case TSDB_OPTION_TIMEZONE:
|
||||
pItem = cfgGetItem(pCfg, "timezone");
|
||||
break;
|
||||
case TSDB_OPTION_USE_ADAPTER:
|
||||
pItem = cfgGetItem(pCfg, "useAdapter");
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
if (name == NULL) {
|
||||
tscError("Invalid option %d", option);
|
||||
return -1;
|
||||
}
|
||||
|
||||
pItem = cfgGetItem(pCfg, name);
|
||||
if (pItem == NULL) {
|
||||
tscError("Invalid option %d", option);
|
||||
return -1;
|
||||
}
|
||||
|
||||
int code = cfgSetItem(pCfg, pItem->name, str, CFG_STYPE_TAOS_OPTIONS);
|
||||
int code = cfgSetItem(pCfg, name, str, CFG_STYPE_TAOS_OPTIONS, true);
|
||||
if (code != 0) {
|
||||
tscError("failed to set cfg:%s to %s since %s", pItem->name, str, terrstr());
|
||||
tscError("failed to set cfg:%s to %s since %s", name, str, terrstr());
|
||||
} else {
|
||||
tscInfo("set cfg:%s to %s", pItem->name, str);
|
||||
tscInfo("set cfg:%s to %s", name, str);
|
||||
if (TSDB_OPTION_SHELL_ACTIVITY_TIMER == option || TSDB_OPTION_USE_ADAPTER == option) {
|
||||
code = taosCfgDynamicOptions(pCfg, pItem->name, false);
|
||||
code = taosCfgDynamicOptions(pCfg, name, false);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -115,6 +115,30 @@ static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sch
|
|||
return string;
|
||||
}
|
||||
|
||||
static int32_t setCompressOption(cJSON* json, uint32_t para) {
|
||||
uint8_t encode = COMPRESS_L1_TYPE_U32(para);
|
||||
if (encode != 0) {
|
||||
const char* encodeStr = columnEncodeStr(encode);
|
||||
cJSON* encodeJson = cJSON_CreateString(encodeStr);
|
||||
cJSON_AddItemToObject(json, "encode", encodeJson);
|
||||
return 0;
|
||||
}
|
||||
uint8_t compress = COMPRESS_L2_TYPE_U32(para);
|
||||
if (compress != 0) {
|
||||
const char* compressStr = columnCompressStr(compress);
|
||||
cJSON* compressJson = cJSON_CreateString(compressStr);
|
||||
cJSON_AddItemToObject(json, "compress", compressJson);
|
||||
return 0;
|
||||
}
|
||||
uint8_t level = COMPRESS_L2_TYPE_LEVEL_U32(para);
|
||||
if (level != 0) {
|
||||
const char* levelStr = columnLevelStr(level);
|
||||
cJSON* levelJson = cJSON_CreateString(levelStr);
|
||||
cJSON_AddItemToObject(json, "level", levelJson);
|
||||
return 0;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) {
|
||||
SMAlterStbReq req = {0};
|
||||
cJSON* json = NULL;
|
||||
|
|
@ -199,6 +223,13 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) {
|
|||
cJSON_AddItemToObject(json, "colNewName", colNewName);
|
||||
break;
|
||||
}
|
||||
case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS: {
|
||||
TAOS_FIELD* field = taosArrayGet(req.pFields, 0);
|
||||
cJSON* colName = cJSON_CreateString(field->name);
|
||||
cJSON_AddItemToObject(json, "colName", colName);
|
||||
setCompressOption(json, field->bytes);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
@ -568,6 +599,12 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) {
|
|||
cJSON_AddItemToObject(json, "colValueNull", isNullCJson);
|
||||
break;
|
||||
}
|
||||
case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS: {
|
||||
cJSON* colName = cJSON_CreateString(vAlterTbReq.colName);
|
||||
cJSON_AddItemToObject(json, "colName", colName);
|
||||
setCompressOption(json, vAlterTbReq.compress);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -832,7 +832,7 @@ static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) {
|
|||
return result;
|
||||
}
|
||||
|
||||
static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols,
|
||||
static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, SArray *checkDumplicateCols,
|
||||
ESchemaAction *action, bool isTag) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
for (int j = 0; j < taosArrayGetSize(cols); ++j) {
|
||||
|
|
@ -843,6 +843,13 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH
|
|||
return code;
|
||||
}
|
||||
}
|
||||
|
||||
for (int j = 0; j < taosArrayGetSize(checkDumplicateCols); ++j) {
|
||||
SSmlKv *kv = (SSmlKv *)taosArrayGet(checkDumplicateCols, j);
|
||||
if(taosHashGet(schemaHash, kv->key, kv->keyLen) != NULL){
|
||||
return TSDB_CODE_PAR_DUPLICATED_COLUMN;
|
||||
}
|
||||
}
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
|
@ -1106,7 +1113,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
|
|||
}
|
||||
|
||||
ESchemaAction action = SCHEMA_ACTION_NULL;
|
||||
code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->tags, &action, true);
|
||||
code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->tags, sTableData->cols, &action, true);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto end;
|
||||
}
|
||||
|
|
@ -1181,7 +1188,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
|
|||
taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES);
|
||||
}
|
||||
action = SCHEMA_ACTION_NULL;
|
||||
code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, &action, false);
|
||||
code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, sTableData->tags, &action, false);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto end;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -827,10 +827,18 @@ TEST(clientCase, projection_query_tables) {
|
|||
// }
|
||||
// taos_free_result(pRes);
|
||||
|
||||
TAOS_RES* pRes = taos_query(pConn, "use test");
|
||||
TAOS_RES* pRes = taos_query(pConn, "alter local 'fqdn 127.0.0.1'");
|
||||
if (taos_errno(pRes) != 0) {
|
||||
printf("failed to exec query, %s\n", taos_errstr(pRes));
|
||||
}
|
||||
|
||||
taos_free_result(pRes);
|
||||
|
||||
pRes = taos_query(pConn, "create table st2 (ts timestamp, k int primary key, j varchar(1000)) tags(a int)");
|
||||
pRes = taos_query(pConn, "select last(ts), ts from cache_1.t1");
|
||||
// pRes = taos_query(pConn, "select last(ts), ts from cache_1.no_pk_t1");
|
||||
if (taos_errno(pRes) != 0) {
|
||||
printf("failed to exec query, %s\n", taos_errstr(pRes));
|
||||
}
|
||||
taos_free_result(pRes);
|
||||
|
||||
// pRes = taos_query(pConn, "create stream stream_1 trigger at_once fill_history 1 ignore expired 0 into str_res1 as select _wstart as ts, count(*) from stable_1 interval(10s);");
|
||||
|
|
|
|||
|
|
@ -191,8 +191,8 @@ static const SSysDbTableSchema streamTaskSchema[] = {
|
|||
{.name = "start_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
|
||||
{.name = "start_ver", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
|
||||
{.name = "checkpoint_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
|
||||
{.name = "checkpoint_id", .bytes = 25, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
|
||||
{.name = "checkpoint_version", .bytes = 25, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
|
||||
{.name = "checkpoint_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
|
||||
{.name = "checkpoint_version", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
|
||||
{.name = "ds_err_info", .bytes = 25, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
|
||||
{.name = "history_task_id", .bytes = 16 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
|
||||
{.name = "history_task_status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
|
||||
|
|
|
|||
|
|
@ -1032,7 +1032,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) {
|
|||
sndDebugFlag = cfgGetItem(pCfg, "sndDebugFlag")->i32;
|
||||
}
|
||||
|
||||
static int32_t taosSetSlowLogScope(char *pScope) {
|
||||
static int32_t taosSetSlowLogScope(const char *pScope) {
|
||||
if (NULL == pScope || 0 == strlen(pScope)) {
|
||||
tsSlowLogScope = SLOW_LOG_TYPE_ALL;
|
||||
return 0;
|
||||
|
|
@ -1081,13 +1081,13 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
|
|||
SEp firstEp = {0};
|
||||
taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp);
|
||||
snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port);
|
||||
cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype);
|
||||
cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype, true);
|
||||
|
||||
SConfigItem *pSecondpItem = cfgGetItem(pCfg, "secondEp");
|
||||
SEp secondEp = {0};
|
||||
taosGetFqdnPortFromEp(strlen(pSecondpItem->str) == 0 ? defaultFirstEp : pSecondpItem->str, &secondEp);
|
||||
snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port);
|
||||
cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype);
|
||||
cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype, true);
|
||||
|
||||
tstrncpy(tsTempDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX);
|
||||
taosExpandDir(tsTempDir, tsTempDir, PATH_MAX);
|
||||
|
|
@ -1149,9 +1149,10 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
|
|||
|
||||
static void taosSetSystemCfg(SConfig *pCfg) {
|
||||
SConfigItem *pItem = cfgGetItem(pCfg, "timezone");
|
||||
|
||||
osSetTimezone(pItem->str);
|
||||
uDebug("timezone format changed from %s to %s", pItem->str, tsTimezoneStr);
|
||||
cfgSetItem(pCfg, "timezone", tsTimezoneStr, pItem->stype);
|
||||
cfgSetItem(pCfg, "timezone", tsTimezoneStr, pItem->stype, true);
|
||||
|
||||
const char *locale = cfgGetItem(pCfg, "locale")->str;
|
||||
const char *charset = cfgGetItem(pCfg, "charset")->str;
|
||||
|
|
@ -1505,7 +1506,7 @@ static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize,
|
|||
return terrno == TSDB_CODE_SUCCESS ? 0 : -1;
|
||||
}
|
||||
|
||||
static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
|
||||
static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) {
|
||||
terrno = TSDB_CODE_SUCCESS;
|
||||
|
||||
if (strcasecmp(name, "resetlog") == 0) {
|
||||
|
|
@ -1515,15 +1516,20 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
cfgLock(pCfg);
|
||||
|
||||
SConfigItem *pItem = cfgGetItem(pCfg, name);
|
||||
if (!pItem || (pItem->dynScope & CFG_DYN_SERVER) == 0) {
|
||||
uError("failed to config:%s, not support", name);
|
||||
terrno = TSDB_CODE_INVALID_CFG;
|
||||
|
||||
cfgUnLock(pCfg);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (strncasecmp(name, "debugFlag", 9) == 0) {
|
||||
taosSetAllDebugFlag(pCfg, pItem->i32);
|
||||
cfgUnLock(pCfg);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -1580,16 +1586,21 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
|
|||
}
|
||||
}
|
||||
|
||||
cfgUnLock(pCfg);
|
||||
return terrno == TSDB_CODE_SUCCESS ? 0 : -1;
|
||||
}
|
||||
|
||||
static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
|
||||
static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) {
|
||||
terrno = TSDB_CODE_SUCCESS;
|
||||
|
||||
cfgLock(pCfg);
|
||||
|
||||
SConfigItem *pItem = cfgGetItem(pCfg, name);
|
||||
if (!pItem || (pItem->dynScope & CFG_DYN_CLIENT) == 0) {
|
||||
if ((pItem == NULL) || (pItem->dynScope & CFG_DYN_CLIENT) == 0) {
|
||||
uError("failed to config:%s, not support", name);
|
||||
terrno = TSDB_CODE_INVALID_CFG;
|
||||
|
||||
cfgUnLock(pCfg);
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -1598,6 +1609,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
|
|||
int32_t len = strlen(name);
|
||||
char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0};
|
||||
strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len));
|
||||
|
||||
switch (lowcaseName[0]) {
|
||||
case 'd': {
|
||||
if (strcasecmp("debugFlag", name) == 0) {
|
||||
|
|
@ -1628,7 +1640,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
|
|||
SEp firstEp = {0};
|
||||
taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp);
|
||||
snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port);
|
||||
cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype);
|
||||
|
||||
cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype, false);
|
||||
uInfo("localEp set to '%s', tsFirst set to '%s'", tsLocalEp, tsFirst);
|
||||
matched = true;
|
||||
} else if (strcasecmp("firstEp", name) == 0) {
|
||||
|
|
@ -1643,7 +1656,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
|
|||
SEp firstEp = {0};
|
||||
taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp);
|
||||
snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port);
|
||||
cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype);
|
||||
|
||||
cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype, false);
|
||||
uInfo("localEp set to '%s', tsFirst set to '%s'", tsLocalEp, tsFirst);
|
||||
matched = true;
|
||||
}
|
||||
|
|
@ -1690,7 +1704,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
|
|||
SEp secondEp = {0};
|
||||
taosGetFqdnPortFromEp(strlen(pItem->str) == 0 ? tsFirst : pItem->str, &secondEp);
|
||||
snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port);
|
||||
cfgSetItem(pCfg, "secondEp", tsSecond, pItem->stype);
|
||||
cfgSetItem(pCfg, "secondEp", tsSecond, pItem->stype, false);
|
||||
uInfo("%s set to %s", name, tsSecond);
|
||||
matched = true;
|
||||
} else if (strcasecmp("smlChildTableName", name) == 0) {
|
||||
|
|
@ -1721,11 +1735,13 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
|
|||
SEp firstEp = {0};
|
||||
taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp);
|
||||
snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port);
|
||||
cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype);
|
||||
|
||||
cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype, false);
|
||||
uInfo("localEp set to '%s', tsFirst set to '%s'", tsLocalEp, tsFirst);
|
||||
matched = true;
|
||||
} else if (strcasecmp("slowLogScope", name) == 0) {
|
||||
if (taosSetSlowLogScope(pItem->str)) {
|
||||
cfgUnLock(pCfg);
|
||||
return -1;
|
||||
}
|
||||
uInfo("%s set to %s", name, pItem->str);
|
||||
|
|
@ -1737,7 +1753,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
|
|||
if (strcasecmp("timezone", name) == 0) {
|
||||
osSetTimezone(pItem->str);
|
||||
uInfo("%s set from %s to %s", name, tsTimezoneStr, pItem->str);
|
||||
cfgSetItem(pCfg, "timezone", tsTimezoneStr, pItem->stype);
|
||||
|
||||
cfgSetItem(pCfg, "timezone", tsTimezoneStr, pItem->stype, false);
|
||||
matched = true;
|
||||
} else if (strcasecmp("tempDir", name) == 0) {
|
||||
uInfo("%s set from %s to %s", name, tsTempDir, pItem->str);
|
||||
|
|
@ -1745,6 +1762,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
|
|||
taosExpandDir(tsTempDir, tsTempDir, PATH_MAX);
|
||||
if (taosMulMkDir(tsTempDir) != 0) {
|
||||
uError("failed to create tempDir:%s since %s", tsTempDir, terrstr());
|
||||
cfgUnLock(pCfg);
|
||||
return -1;
|
||||
}
|
||||
matched = true;
|
||||
|
|
@ -1800,12 +1818,16 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
|
|||
}
|
||||
|
||||
_out:
|
||||
cfgUnLock(pCfg);
|
||||
return terrno == TSDB_CODE_SUCCESS ? 0 : -1;
|
||||
}
|
||||
|
||||
int32_t taosCfgDynamicOptions(SConfig *pCfg, char *name, bool forServer) {
|
||||
if (forServer) return taosCfgDynamicOptionsForServer(pCfg, name);
|
||||
return taosCfgDynamicOptionsForClient(pCfg, name);
|
||||
int32_t taosCfgDynamicOptions(SConfig *pCfg, const char *name, bool forServer) {
|
||||
if (forServer) {
|
||||
return taosCfgDynamicOptionsForServer(pCfg, name);
|
||||
} else {
|
||||
return taosCfgDynamicOptionsForClient(pCfg, name);
|
||||
}
|
||||
}
|
||||
|
||||
void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal) {
|
||||
|
|
@ -1873,7 +1895,9 @@ static void taosSetAllDebugFlag(SConfig *pCfg, int32_t flag) {
|
|||
taosArrayClear(noNeedToSetVars); // reset array
|
||||
|
||||
uInfo("all debug flag are set to %d", flag);
|
||||
if (terrno == TSDB_CODE_CFG_NOT_FOUND) terrno = TSDB_CODE_SUCCESS; // ignore not exist
|
||||
if (terrno == TSDB_CODE_CFG_NOT_FOUND) {
|
||||
terrno = TSDB_CODE_SUCCESS; // ignore not exist
|
||||
}
|
||||
}
|
||||
|
||||
int8_t taosGranted(int8_t type) {
|
||||
|
|
|
|||
|
|
@ -17,6 +17,8 @@
|
|||
#include "tmisce.h"
|
||||
#include "tglobal.h"
|
||||
#include "tjson.h"
|
||||
#include "tdatablock.h"
|
||||
|
||||
int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) {
|
||||
pEp->port = 0;
|
||||
memset(pEp->fqdn, 0, TSDB_FQDN_LEN);
|
||||
|
|
@ -97,10 +99,10 @@ void epsetSort(SEpSet* pDst) {
|
|||
SEp* s = &pDst->eps[j + 1];
|
||||
int cmp = strncmp(f->fqdn, s->fqdn, sizeof(f->fqdn));
|
||||
if (cmp > 0 || (cmp == 0 && f->port > s->port)) {
|
||||
SEp ep = {0};
|
||||
epAssign(&ep, f);
|
||||
SEp ep1 = {0};
|
||||
epAssign(&ep1, f);
|
||||
epAssign(f, s);
|
||||
epAssign(s, &ep);
|
||||
epAssign(s, &ep1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -216,3 +218,43 @@ int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t
|
|||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) {
|
||||
SConfig* pConf = taosGetCfg();
|
||||
int32_t numOfRows = 0;
|
||||
int32_t col = startCol;
|
||||
SConfigItem* pItem = NULL;
|
||||
|
||||
blockDataEnsureCapacity(pBlock, cfgGetSize(pConf));
|
||||
SConfigIter* pIter = cfgCreateIter(pConf);
|
||||
|
||||
while ((pItem = cfgNextIter(pIter)) != NULL) {
|
||||
col = startCol;
|
||||
|
||||
// GRANT_CFG_SKIP;
|
||||
char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE);
|
||||
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, col++);
|
||||
colDataSetVal(pColInfo, numOfRows, name, false);
|
||||
|
||||
char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
int32_t valueLen = 0;
|
||||
cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen);
|
||||
varDataSetLen(value, valueLen);
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, col++);
|
||||
colDataSetVal(pColInfo, numOfRows, value, false);
|
||||
|
||||
char scope[TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
cfgDumpItemScope(pItem, &scope[VARSTR_HEADER_SIZE], TSDB_CONFIG_SCOPE_LEN, &valueLen);
|
||||
varDataSetLen(scope, valueLen);
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, col++);
|
||||
colDataSetVal(pColInfo, numOfRows, scope, false);
|
||||
|
||||
numOfRows++;
|
||||
}
|
||||
|
||||
pBlock->info.rows = numOfRows;
|
||||
|
||||
cfgDestroyIter(pIter);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@
|
|||
static int32_t tDecodeSVAlterTbReqCommon(SDecoder *pDecoder, SVAlterTbReq *pReq);
|
||||
static int32_t tDecodeSBatchDeleteReqCommon(SDecoder *pDecoder, SBatchDeleteReq *pReq);
|
||||
static int32_t tEncodeTableTSMAInfoRsp(SEncoder *pEncoder, const STableTSMAInfoRsp *pRsp);
|
||||
static int32_t tDecodeTableTSMAInfoRsp(SDecoder* pDecoder, STableTSMAInfoRsp* pRsp);
|
||||
static int32_t tDecodeTableTSMAInfoRsp(SDecoder *pDecoder, STableTSMAInfoRsp *pRsp);
|
||||
|
||||
int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) {
|
||||
if (pMsg == NULL) {
|
||||
|
|
@ -895,8 +895,8 @@ int32_t tSerializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq
|
|||
if (tEncodeI64(&encoder, pReq->normSourceTbUid) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, taosArrayGetSize(pReq->pVgroupVerList)) < 0) return -1;
|
||||
|
||||
for(int32_t i = 0; i < taosArrayGetSize(pReq->pVgroupVerList); ++i) {
|
||||
SVgroupVer* p = taosArrayGet(pReq->pVgroupVerList, i);
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pReq->pVgroupVerList); ++i) {
|
||||
SVgroupVer *p = taosArrayGet(pReq->pVgroupVerList, i);
|
||||
if (tEncodeI32(&encoder, p->vgId) < 0) return -1;
|
||||
if (tEncodeI64(&encoder, p->ver) < 0) return -1;
|
||||
}
|
||||
|
|
@ -8000,7 +8000,7 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea
|
|||
}
|
||||
}
|
||||
if (!tDecodeIsEnd(&decoder)) {
|
||||
if (tDecodeI64(&decoder, &pReq->smaId)< 0) return -1;
|
||||
if (tDecodeI64(&decoder, &pReq->smaId) < 0) return -1;
|
||||
}
|
||||
|
||||
tEndDecode(&decoder);
|
||||
|
|
@ -8445,8 +8445,8 @@ static int32_t tDecodeSVDropTbRsp(SDecoder *pCoder, SVDropTbRsp *pReq) {
|
|||
}
|
||||
|
||||
int32_t tEncodeSVDropTbBatchReq(SEncoder *pCoder, const SVDropTbBatchReq *pReq) {
|
||||
int32_t nReqs = taosArrayGetSize(pReq->pArray);
|
||||
SVDropTbReq *pDropTbReq;
|
||||
int32_t nReqs = taosArrayGetSize(pReq->pArray);
|
||||
SVDropTbReq *pDropTbReq;
|
||||
|
||||
if (tStartEncode(pCoder) < 0) return -1;
|
||||
|
||||
|
|
@ -8709,6 +8709,7 @@ int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) {
|
|||
}
|
||||
break;
|
||||
case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS:
|
||||
if (tEncodeCStr(pEncoder, pReq->colName) < 0) return -1;
|
||||
if (tEncodeU32(pEncoder, pReq->compress) < 0) return -1;
|
||||
break;
|
||||
default:
|
||||
|
|
@ -8763,6 +8764,7 @@ static int32_t tDecodeSVAlterTbReqCommon(SDecoder *pDecoder, SVAlterTbReq *pReq)
|
|||
}
|
||||
break;
|
||||
case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS:
|
||||
if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1;
|
||||
if (tDecodeU32(pDecoder, &pReq->compress) < 0) return -1;
|
||||
break;
|
||||
default:
|
||||
|
|
@ -9200,7 +9202,7 @@ int32_t tEncodeMqDataRspCommon(SEncoder *pEncoder, const SMqDataRspCommon *pRsp)
|
|||
|
||||
int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const void *pRsp) {
|
||||
if (tEncodeMqDataRspCommon(pEncoder, pRsp) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, ((SMqDataRsp*)pRsp)->sleepTime) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, ((SMqDataRsp *)pRsp)->sleepTime) < 0) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -9253,7 +9255,7 @@ int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRspCommon *pRsp) {
|
|||
int32_t tDecodeMqDataRsp(SDecoder *pDecoder, void *pRsp) {
|
||||
if (tDecodeMqDataRspCommon(pDecoder, pRsp) < 0) return -1;
|
||||
if (!tDecodeIsEnd(pDecoder)) {
|
||||
if (tDecodeI64(pDecoder, &((SMqDataRsp*)pRsp)->sleepTime) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &((SMqDataRsp *)pRsp)->sleepTime) < 0) return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
|
@ -9272,9 +9274,7 @@ static void tDeleteMqDataRspCommon(void *rsp) {
|
|||
tOffsetDestroy(&pRsp->rspOffset);
|
||||
}
|
||||
|
||||
void tDeleteMqDataRsp(void *rsp) {
|
||||
tDeleteMqDataRspCommon(rsp);
|
||||
}
|
||||
void tDeleteMqDataRsp(void *rsp) { tDeleteMqDataRspCommon(rsp); }
|
||||
|
||||
int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const void *rsp) {
|
||||
if (tEncodeMqDataRspCommon(pEncoder, rsp) < 0) return -1;
|
||||
|
|
@ -9300,7 +9300,7 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, void *rsp) {
|
|||
pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t));
|
||||
pRsp->createTableReq = taosArrayInit(pRsp->createTableNum, sizeof(void *));
|
||||
for (int32_t i = 0; i < pRsp->createTableNum; i++) {
|
||||
void * pCreate = NULL;
|
||||
void *pCreate = NULL;
|
||||
uint64_t len = 0;
|
||||
if (tDecodeBinaryAlloc(pDecoder, &pCreate, &len) < 0) return -1;
|
||||
int32_t l = (int32_t)len;
|
||||
|
|
@ -10114,7 +10114,7 @@ void setFieldWithOptions(SFieldWithOptions *fieldWithOptions, SField *field) {
|
|||
fieldWithOptions->type = field->type;
|
||||
strncpy(fieldWithOptions->name, field->name, TSDB_COL_NAME_LEN);
|
||||
}
|
||||
int32_t tSerializeTableTSMAInfoReq(void* buf, int32_t bufLen, const STableTSMAInfoReq* pReq) {
|
||||
int32_t tSerializeTableTSMAInfoReq(void *buf, int32_t bufLen, const STableTSMAInfoReq *pReq) {
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, buf, bufLen);
|
||||
|
||||
|
|
@ -10129,13 +10129,13 @@ int32_t tSerializeTableTSMAInfoReq(void* buf, int32_t bufLen, const STableTSMAIn
|
|||
return tlen;
|
||||
}
|
||||
|
||||
int32_t tDeserializeTableTSMAInfoReq(void* buf, int32_t bufLen, STableTSMAInfoReq* pReq) {
|
||||
int32_t tDeserializeTableTSMAInfoReq(void *buf, int32_t bufLen, STableTSMAInfoReq *pReq) {
|
||||
SDecoder decoder = {0};
|
||||
tDecoderInit(&decoder, buf, bufLen);
|
||||
|
||||
if (tStartDecode(&decoder) < 0) return -1;
|
||||
if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, (uint8_t*)&pReq->fetchingWithTsmaName) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, (uint8_t *)&pReq->fetchingWithTsmaName) < 0) return -1;
|
||||
|
||||
tEndDecode(&decoder);
|
||||
|
||||
|
|
@ -10143,7 +10143,7 @@ int32_t tDeserializeTableTSMAInfoReq(void* buf, int32_t bufLen, STableTSMAInfoRe
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int32_t tEncodeTableTSMAInfo(SEncoder* pEncoder, const STableTSMAInfo* pTsmaInfo) {
|
||||
static int32_t tEncodeTableTSMAInfo(SEncoder *pEncoder, const STableTSMAInfo *pTsmaInfo) {
|
||||
if (tEncodeCStr(pEncoder, pTsmaInfo->name) < 0) return -1;
|
||||
if (tEncodeU64(pEncoder, pTsmaInfo->tsmaId) < 0) return -1;
|
||||
if (tEncodeCStr(pEncoder, pTsmaInfo->tb) < 0) return -1;
|
||||
|
|
@ -10160,7 +10160,7 @@ static int32_t tEncodeTableTSMAInfo(SEncoder* pEncoder, const STableTSMAInfo* pT
|
|||
int32_t size = pTsmaInfo->pFuncs ? pTsmaInfo->pFuncs->size : 0;
|
||||
if (tEncodeI32(pEncoder, size) < 0) return -1;
|
||||
for (int32_t i = 0; i < size; ++i) {
|
||||
STableTSMAFuncInfo* pFuncInfo = taosArrayGet(pTsmaInfo->pFuncs, i);
|
||||
STableTSMAFuncInfo *pFuncInfo = taosArrayGet(pTsmaInfo->pFuncs, i);
|
||||
if (tEncodeI32(pEncoder, pFuncInfo->funcId) < 0) return -1;
|
||||
if (tEncodeI16(pEncoder, pFuncInfo->colId) < 0) return -1;
|
||||
}
|
||||
|
|
@ -10168,13 +10168,13 @@ static int32_t tEncodeTableTSMAInfo(SEncoder* pEncoder, const STableTSMAInfo* pT
|
|||
size = pTsmaInfo->pTags ? pTsmaInfo->pTags->size : 0;
|
||||
if (tEncodeI32(pEncoder, size) < 0) return -1;
|
||||
for (int32_t i = 0; i < size; ++i) {
|
||||
const SSchema* pSchema = taosArrayGet(pTsmaInfo->pTags, i);
|
||||
const SSchema *pSchema = taosArrayGet(pTsmaInfo->pTags, i);
|
||||
if (tEncodeSSchema(pEncoder, pSchema) < 0) return -1;
|
||||
}
|
||||
size = pTsmaInfo->pUsedCols ? pTsmaInfo->pUsedCols->size : 0;
|
||||
if (tEncodeI32(pEncoder, size) < 0) return -1;
|
||||
for (int32_t i = 0; i < size; ++i) {
|
||||
const SSchema* pSchema = taosArrayGet(pTsmaInfo->pUsedCols, i);
|
||||
const SSchema *pSchema = taosArrayGet(pTsmaInfo->pUsedCols, i);
|
||||
if (tEncodeSSchema(pEncoder, pSchema) < 0) return -1;
|
||||
}
|
||||
|
||||
|
|
@ -10187,7 +10187,7 @@ static int32_t tEncodeTableTSMAInfo(SEncoder* pEncoder, const STableTSMAInfo* pT
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int32_t tDecodeTableTSMAInfo(SDecoder* pDecoder, STableTSMAInfo* pTsmaInfo) {
|
||||
static int32_t tDecodeTableTSMAInfo(SDecoder *pDecoder, STableTSMAInfo *pTsmaInfo) {
|
||||
if (tDecodeCStrTo(pDecoder, pTsmaInfo->name) < 0) return -1;
|
||||
if (tDecodeU64(pDecoder, &pTsmaInfo->tsmaId) < 0) return -1;
|
||||
if (tDecodeCStrTo(pDecoder, pTsmaInfo->tb) < 0) return -1;
|
||||
|
|
@ -10219,7 +10219,7 @@ static int32_t tDecodeTableTSMAInfo(SDecoder* pDecoder, STableTSMAInfo* pTsmaInf
|
|||
if (!pTsmaInfo->pTags) return -1;
|
||||
for (int32_t i = 0; i < size; ++i) {
|
||||
SSchema schema = {0};
|
||||
if(tDecodeSSchema(pDecoder, &schema) < 0) return -1;
|
||||
if (tDecodeSSchema(pDecoder, &schema) < 0) return -1;
|
||||
taosArrayPush(pTsmaInfo->pTags, &schema);
|
||||
}
|
||||
}
|
||||
|
|
@ -10239,7 +10239,7 @@ static int32_t tDecodeTableTSMAInfo(SDecoder* pDecoder, STableTSMAInfo* pTsmaInf
|
|||
if (tDecodeI64(pDecoder, &pTsmaInfo->reqTs) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pTsmaInfo->rspTs) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pTsmaInfo->delayDuration) < 0) return -1;
|
||||
if (tDecodeI8(pDecoder, (int8_t*)&pTsmaInfo->fillHistoryFinished) < 0) return -1;
|
||||
if (tDecodeI8(pDecoder, (int8_t *)&pTsmaInfo->fillHistoryFinished) < 0) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -10247,13 +10247,13 @@ static int32_t tEncodeTableTSMAInfoRsp(SEncoder *pEncoder, const STableTSMAInfoR
|
|||
int32_t size = pRsp->pTsmas ? pRsp->pTsmas->size : 0;
|
||||
if (tEncodeI32(pEncoder, size) < 0) return -1;
|
||||
for (int32_t i = 0; i < size; ++i) {
|
||||
STableTSMAInfo* pInfo = taosArrayGetP(pRsp->pTsmas, i);
|
||||
STableTSMAInfo *pInfo = taosArrayGetP(pRsp->pTsmas, i);
|
||||
if (tEncodeTableTSMAInfo(pEncoder, pInfo) < 0) return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t tDecodeTableTSMAInfoRsp(SDecoder* pDecoder, STableTSMAInfoRsp* pRsp) {
|
||||
static int32_t tDecodeTableTSMAInfoRsp(SDecoder *pDecoder, STableTSMAInfoRsp *pRsp) {
|
||||
int32_t size = 0;
|
||||
if (tDecodeI32(pDecoder, &size) < 0) return -1;
|
||||
if (size <= 0) return 0;
|
||||
|
|
@ -10268,7 +10268,7 @@ static int32_t tDecodeTableTSMAInfoRsp(SDecoder* pDecoder, STableTSMAInfoRsp* pR
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t tSerializeTableTSMAInfoRsp(void* buf, int32_t bufLen, const STableTSMAInfoRsp* pRsp) {
|
||||
int32_t tSerializeTableTSMAInfoRsp(void *buf, int32_t bufLen, const STableTSMAInfoRsp *pRsp) {
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, buf, bufLen);
|
||||
|
||||
|
|
@ -10282,7 +10282,7 @@ int32_t tSerializeTableTSMAInfoRsp(void* buf, int32_t bufLen, const STableTSMAIn
|
|||
return tlen;
|
||||
}
|
||||
|
||||
int32_t tDeserializeTableTSMAInfoRsp(void* buf, int32_t bufLen, STableTSMAInfoRsp* pRsp) {
|
||||
int32_t tDeserializeTableTSMAInfoRsp(void *buf, int32_t bufLen, STableTSMAInfoRsp *pRsp) {
|
||||
SDecoder decoder = {0};
|
||||
tDecoderInit(&decoder, buf, bufLen);
|
||||
|
||||
|
|
@ -10295,7 +10295,7 @@ int32_t tDeserializeTableTSMAInfoRsp(void* buf, int32_t bufLen, STableTSMAInfoRs
|
|||
return 0;
|
||||
}
|
||||
|
||||
void tFreeTableTSMAInfo(void* p) {
|
||||
void tFreeTableTSMAInfo(void *p) {
|
||||
STableTSMAInfo *pTsmaInfo = p;
|
||||
if (pTsmaInfo) {
|
||||
taosArrayDestroy(pTsmaInfo->pFuncs);
|
||||
|
|
@ -10305,20 +10305,20 @@ void tFreeTableTSMAInfo(void* p) {
|
|||
}
|
||||
}
|
||||
|
||||
void tFreeAndClearTableTSMAInfo(void* p) {
|
||||
STableTSMAInfo* pTsmaInfo = (STableTSMAInfo*)p;
|
||||
void tFreeAndClearTableTSMAInfo(void *p) {
|
||||
STableTSMAInfo *pTsmaInfo = (STableTSMAInfo *)p;
|
||||
if (pTsmaInfo) {
|
||||
tFreeTableTSMAInfo(pTsmaInfo);
|
||||
taosMemoryFree(pTsmaInfo);
|
||||
}
|
||||
}
|
||||
|
||||
int32_t tCloneTbTSMAInfo(STableTSMAInfo* pInfo, STableTSMAInfo** pRes) {
|
||||
int32_t tCloneTbTSMAInfo(STableTSMAInfo *pInfo, STableTSMAInfo **pRes) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
if (NULL == pInfo) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
STableTSMAInfo* pRet = taosMemoryCalloc(1, sizeof(STableTSMAInfo));
|
||||
STableTSMAInfo *pRet = taosMemoryCalloc(1, sizeof(STableTSMAInfo));
|
||||
if (!pRet) return TSDB_CODE_OUT_OF_MEMORY;
|
||||
|
||||
*pRet = *pInfo;
|
||||
|
|
@ -10357,7 +10357,7 @@ static int32_t tEncodeStreamProgressReq(SEncoder *pEncoder, const SStreamProgres
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t tSerializeStreamProgressReq(void* buf, int32_t bufLen, const SStreamProgressReq* pReq) {
|
||||
int32_t tSerializeStreamProgressReq(void *buf, int32_t bufLen, const SStreamProgressReq *pReq) {
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, buf, bufLen);
|
||||
|
||||
|
|
@ -10371,7 +10371,7 @@ int32_t tSerializeStreamProgressReq(void* buf, int32_t bufLen, const SStreamProg
|
|||
return tlen;
|
||||
}
|
||||
|
||||
static int32_t tDecodeStreamProgressReq(SDecoder* pDecoder, SStreamProgressReq* pReq) {
|
||||
static int32_t tDecodeStreamProgressReq(SDecoder *pDecoder, SStreamProgressReq *pReq) {
|
||||
if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->vgId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->fetchIdx) < 0) return -1;
|
||||
|
|
@ -10379,7 +10379,7 @@ static int32_t tDecodeStreamProgressReq(SDecoder* pDecoder, SStreamProgressReq*
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t tDeserializeStreamProgressReq(void* buf, int32_t bufLen, SStreamProgressReq* pReq) {
|
||||
int32_t tDeserializeStreamProgressReq(void *buf, int32_t bufLen, SStreamProgressReq *pReq) {
|
||||
SDecoder decoder = {0};
|
||||
tDecoderInit(&decoder, (char *)buf, bufLen);
|
||||
|
||||
|
|
@ -10392,7 +10392,7 @@ int32_t tDeserializeStreamProgressReq(void* buf, int32_t bufLen, SStreamProgress
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int32_t tEncodeStreamProgressRsp(SEncoder* pEncoder, const SStreamProgressRsp* pRsp) {
|
||||
static int32_t tEncodeStreamProgressRsp(SEncoder *pEncoder, const SStreamProgressRsp *pRsp) {
|
||||
if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->vgId) < 0) return -1;
|
||||
if (tEncodeI8(pEncoder, pRsp->fillHisFinished) < 0) return -1;
|
||||
|
|
@ -10402,7 +10402,7 @@ static int32_t tEncodeStreamProgressRsp(SEncoder* pEncoder, const SStreamProgres
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t tSerializeStreamProgressRsp(void* buf, int32_t bufLen, const SStreamProgressRsp* pRsp) {
|
||||
int32_t tSerializeStreamProgressRsp(void *buf, int32_t bufLen, const SStreamProgressRsp *pRsp) {
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, buf, bufLen);
|
||||
|
||||
|
|
@ -10416,17 +10416,17 @@ int32_t tSerializeStreamProgressRsp(void* buf, int32_t bufLen, const SStreamProg
|
|||
return tlen;
|
||||
}
|
||||
|
||||
static int32_t tDecodeStreamProgressRsp(SDecoder* pDecoder, SStreamProgressRsp* pRsp) {
|
||||
static int32_t tDecodeStreamProgressRsp(SDecoder *pDecoder, SStreamProgressRsp *pRsp) {
|
||||
if (tDecodeI64(pDecoder, &pRsp->streamId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->vgId) < 0) return -1;
|
||||
if (tDecodeI8(pDecoder, (int8_t*)&pRsp->fillHisFinished) < 0) return -1;
|
||||
if (tDecodeI8(pDecoder, (int8_t *)&pRsp->fillHisFinished) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pRsp->progressDelay) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->fetchIdx) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->subFetchIdx) < 0) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tDeserializeSStreamProgressRsp(void* buf, int32_t bufLen, SStreamProgressRsp* pRsp) {
|
||||
int32_t tDeserializeSStreamProgressRsp(void *buf, int32_t bufLen, SStreamProgressRsp *pRsp) {
|
||||
SDecoder decoder = {0};
|
||||
tDecoderInit(&decoder, buf, bufLen);
|
||||
|
||||
|
|
@ -10440,22 +10440,22 @@ int32_t tDeserializeSStreamProgressRsp(void* buf, int32_t bufLen, SStreamProgres
|
|||
}
|
||||
|
||||
int32_t tEncodeSMDropTbReqOnSingleVg(SEncoder *pEncoder, const SMDropTbReqsOnSingleVg *pReq) {
|
||||
const SVgroupInfo* pVgInfo = &pReq->vgInfo;
|
||||
const SVgroupInfo *pVgInfo = &pReq->vgInfo;
|
||||
if (tEncodeI32(pEncoder, pVgInfo->vgId) < 0) return -1;
|
||||
if (tEncodeU32(pEncoder, pVgInfo->hashBegin) < 0) return -1;
|
||||
if (tEncodeU32(pEncoder, pVgInfo->hashEnd) < 0) return -1;
|
||||
if (tEncodeSEpSet(pEncoder, &pVgInfo->epSet) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pVgInfo->numOfTable) < 0) return -1;
|
||||
int32_t size = pReq->pTbs ? pReq->pTbs->size: 0;
|
||||
int32_t size = pReq->pTbs ? pReq->pTbs->size : 0;
|
||||
if (tEncodeI32(pEncoder, size) < 0) return -1;
|
||||
for (int32_t i = 0; i < size; ++i) {
|
||||
const SVDropTbReq* pInfo = taosArrayGet(pReq->pTbs, i);
|
||||
const SVDropTbReq *pInfo = taosArrayGet(pReq->pTbs, i);
|
||||
if (tEncodeSVDropTbReq(pEncoder, pInfo) < 0) return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tDecodeSMDropTbReqOnSingleVg(SDecoder* pDecoder, SMDropTbReqsOnSingleVg* pReq) {
|
||||
int32_t tDecodeSMDropTbReqOnSingleVg(SDecoder *pDecoder, SMDropTbReqsOnSingleVg *pReq) {
|
||||
if (tDecodeI32(pDecoder, &pReq->vgInfo.vgId) < 0) return -1;
|
||||
if (tDecodeU32(pDecoder, &pReq->vgInfo.hashBegin) < 0) return -1;
|
||||
if (tDecodeU32(pDecoder, &pReq->vgInfo.hashEnd) < 0) return -1;
|
||||
|
|
@ -10477,18 +10477,18 @@ int32_t tDecodeSMDropTbReqOnSingleVg(SDecoder* pDecoder, SMDropTbReqsOnSingleVg*
|
|||
}
|
||||
|
||||
void tFreeSMDropTbReqOnSingleVg(void *p) {
|
||||
SMDropTbReqsOnSingleVg* pReq = p;
|
||||
SMDropTbReqsOnSingleVg *pReq = p;
|
||||
taosArrayDestroy(pReq->pTbs);
|
||||
}
|
||||
|
||||
int32_t tSerializeSMDropTbsReq(void* buf, int32_t bufLen, const SMDropTbsReq* pReq){
|
||||
int32_t tSerializeSMDropTbsReq(void *buf, int32_t bufLen, const SMDropTbsReq *pReq) {
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, buf, bufLen);
|
||||
tStartEncode(&encoder);
|
||||
int32_t size = pReq->pVgReqs ? pReq->pVgReqs->size : 0;
|
||||
if (tEncodeI32(&encoder, size) < 0) return -1;
|
||||
for (int32_t i = 0; i < size; ++i) {
|
||||
SMDropTbReqsOnSingleVg* pVgReq = taosArrayGet(pReq->pVgReqs, i);
|
||||
SMDropTbReqsOnSingleVg *pVgReq = taosArrayGet(pReq->pVgReqs, i);
|
||||
if (tEncodeSMDropTbReqOnSingleVg(&encoder, pVgReq) < 0) return -1;
|
||||
}
|
||||
tEndEncode(&encoder);
|
||||
|
|
@ -10497,7 +10497,7 @@ int32_t tSerializeSMDropTbsReq(void* buf, int32_t bufLen, const SMDropTbsReq* pR
|
|||
return tlen;
|
||||
}
|
||||
|
||||
int32_t tDeserializeSMDropTbsReq(void* buf, int32_t bufLen, SMDropTbsReq* pReq) {
|
||||
int32_t tDeserializeSMDropTbsReq(void *buf, int32_t bufLen, SMDropTbsReq *pReq) {
|
||||
SDecoder decoder = {0};
|
||||
tDecoderInit(&decoder, buf, bufLen);
|
||||
tStartDecode(&decoder);
|
||||
|
|
@ -10518,12 +10518,12 @@ int32_t tDeserializeSMDropTbsReq(void* buf, int32_t bufLen, SMDropTbsReq* pReq)
|
|||
return 0;
|
||||
}
|
||||
|
||||
void tFreeSMDropTbsReq(void* p) {
|
||||
SMDropTbsReq* pReq = p;
|
||||
void tFreeSMDropTbsReq(void *p) {
|
||||
SMDropTbsReq *pReq = p;
|
||||
taosArrayDestroyEx(pReq->pVgReqs, tFreeSMDropTbReqOnSingleVg);
|
||||
}
|
||||
|
||||
int32_t tEncodeVFetchTtlExpiredTbsRsp(SEncoder* pCoder, const SVFetchTtlExpiredTbsRsp* pRsp) {
|
||||
int32_t tEncodeVFetchTtlExpiredTbsRsp(SEncoder *pCoder, const SVFetchTtlExpiredTbsRsp *pRsp) {
|
||||
if (tEncodeI32(pCoder, pRsp->vgId) < 0) return -1;
|
||||
int32_t size = pRsp->pExpiredTbs ? pRsp->pExpiredTbs->size : 0;
|
||||
if (tEncodeI32(pCoder, size) < 0) return -1;
|
||||
|
|
@ -10533,7 +10533,7 @@ int32_t tEncodeVFetchTtlExpiredTbsRsp(SEncoder* pCoder, const SVFetchTtlExpiredT
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t tDecodeVFetchTtlExpiredTbsRsp(SDecoder* pCoder, SVFetchTtlExpiredTbsRsp* pRsp) {
|
||||
int32_t tDecodeVFetchTtlExpiredTbsRsp(SDecoder *pCoder, SVFetchTtlExpiredTbsRsp *pRsp) {
|
||||
if (tDecodeI32(pCoder, &pRsp->vgId) < 0) return -1;
|
||||
int32_t size = 0;
|
||||
if (tDecodeI32(pCoder, &size) < 0) return -1;
|
||||
|
|
@ -10549,7 +10549,7 @@ int32_t tDecodeVFetchTtlExpiredTbsRsp(SDecoder* pCoder, SVFetchTtlExpiredTbsRsp*
|
|||
return 0;
|
||||
}
|
||||
|
||||
void tFreeFetchTtlExpiredTbsRsp(void* p) {
|
||||
SVFetchTtlExpiredTbsRsp* pRsp = p;
|
||||
void tFreeFetchTtlExpiredTbsRsp(void *p) {
|
||||
SVFetchTtlExpiredTbsRsp *pRsp = p;
|
||||
taosArrayDestroy(pRsp->pExpiredTbs);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -219,7 +219,7 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
|||
dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value);
|
||||
|
||||
SConfig *pCfg = taosGetCfg();
|
||||
cfgSetItem(pCfg, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_CMD);
|
||||
cfgSetItem(pCfg, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_CMD, true);
|
||||
taosCfgDynamicOptions(pCfg, cfgReq.config, true);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -254,7 +254,6 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) {
|
|||
pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_OK;
|
||||
pStatus->details[0] = 0;
|
||||
|
||||
SServerStatusRsp statusRsp = {0};
|
||||
SMonMloadInfo minfo = {0};
|
||||
(*pMgmt->getMnodeLoadsFp)(&minfo);
|
||||
if (minfo.isMnode &&
|
||||
|
|
@ -333,39 +332,10 @@ SSDataBlock *dmBuildVariablesBlock(void) {
|
|||
}
|
||||
|
||||
int32_t dmAppendVariablesToBlock(SSDataBlock *pBlock, int32_t dnodeId) {
|
||||
int32_t numOfCfg = taosArrayGetSize(tsCfg->array);
|
||||
int32_t numOfRows = 0;
|
||||
blockDataEnsureCapacity(pBlock, numOfCfg);
|
||||
/*int32_t code = */dumpConfToDataBlock(pBlock, 1);
|
||||
|
||||
for (int32_t i = 0, c = 0; i < numOfCfg; ++i, c = 0) {
|
||||
SConfigItem *pItem = taosArrayGet(tsCfg->array, i);
|
||||
// GRANT_CFG_SKIP;
|
||||
|
||||
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
|
||||
colDataSetVal(pColInfo, i, (const char *)&dnodeId, false);
|
||||
|
||||
char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE);
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
|
||||
colDataSetVal(pColInfo, i, name, false);
|
||||
|
||||
char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
int32_t valueLen = 0;
|
||||
cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen);
|
||||
varDataSetLen(value, valueLen);
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
|
||||
colDataSetVal(pColInfo, i, value, false);
|
||||
|
||||
char scope[TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
cfgDumpItemScope(pItem, &scope[VARSTR_HEADER_SIZE], TSDB_CONFIG_SCOPE_LEN, &valueLen);
|
||||
varDataSetLen(scope, valueLen);
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
|
||||
colDataSetVal(pColInfo, i, scope, false);
|
||||
|
||||
numOfRows++;
|
||||
}
|
||||
|
||||
pBlock->info.rows = numOfRows;
|
||||
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, 0);
|
||||
colDataSetNItems(pColInfo, 0, (const char *)&dnodeId, pBlock->info.rows, false);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@
|
|||
#define _DEFAULT_SOURCE
|
||||
#include "mmInt.h"
|
||||
|
||||
#define PROCESS_THRESHOLD (2000 * 1000)
|
||||
|
||||
static inline int32_t mmAcquire(SMnodeMgmt *pMgmt) {
|
||||
int32_t code = 0;
|
||||
taosThreadRwlockRdlock(&pMgmt->lock);
|
||||
|
|
@ -53,6 +55,14 @@ static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) {
|
|||
|
||||
int32_t code = mndProcessRpcMsg(pMsg);
|
||||
|
||||
if (pInfo->timestamp != 0) {
|
||||
int64_t cost = taosGetTimestampUs() - pInfo->timestamp;
|
||||
if (cost > PROCESS_THRESHOLD) {
|
||||
dGWarn("worker:%d,message has been processed for too long, type:%s, cost: %" PRId64 "s", pInfo->threadNum,
|
||||
TMSG_INFO(pMsg->msgType), cost / (1000 * 1000));
|
||||
}
|
||||
}
|
||||
|
||||
if (IsReq(pMsg) && pMsg->info.handle != NULL && code != TSDB_CODE_ACTION_IN_PROGRESS) {
|
||||
if (code != 0 && terrno != 0) code = terrno;
|
||||
mmSendRsp(pMsg, code);
|
||||
|
|
|
|||
|
|
@ -27,6 +27,8 @@
|
|||
#define ARBGROUP_VER_NUMBER 1
|
||||
#define ARBGROUP_RESERVE_SIZE 64
|
||||
|
||||
static SHashObj *arbUpdateHash = NULL;
|
||||
|
||||
static int32_t mndArbGroupActionInsert(SSdb *pSdb, SArbGroup *pGroup);
|
||||
static int32_t mndArbGroupActionUpdate(SSdb *pSdb, SArbGroup *pOld, SArbGroup *pNew);
|
||||
static int32_t mndArbGroupActionDelete(SSdb *pSdb, SArbGroup *pGroup);
|
||||
|
|
@ -74,10 +76,14 @@ int32_t mndInitArbGroup(SMnode *pMnode) {
|
|||
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_ARBGROUP, mndRetrieveArbGroups);
|
||||
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_ARBGROUP, mndCancelGetNextArbGroup);
|
||||
|
||||
arbUpdateHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK);
|
||||
|
||||
return sdbSetTable(pMnode->pSdb, table);
|
||||
}
|
||||
|
||||
void mndCleanupArbGroup(SMnode *pMnode) {}
|
||||
void mndCleanupArbGroup(SMnode *pMnode) {
|
||||
taosHashCleanup(arbUpdateHash);
|
||||
}
|
||||
|
||||
SArbGroup *mndAcquireArbGroup(SMnode *pMnode, int32_t vgId) {
|
||||
SArbGroup *pGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &vgId);
|
||||
|
|
@ -221,8 +227,7 @@ static int32_t mndArbGroupActionUpdate(SSdb *pSdb, SArbGroup *pOld, SArbGroup *p
|
|||
mInfo("arbgroup:%d, skip to perform update action, old row:%p new row:%p, old version:%" PRId64
|
||||
" new version:%" PRId64,
|
||||
pOld->vgId, pOld, pNew, pOld->version, pNew->version);
|
||||
taosThreadMutexUnlock(&pOld->mutex);
|
||||
return 0;
|
||||
goto _OVER;
|
||||
}
|
||||
|
||||
for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
|
||||
|
|
@ -232,7 +237,11 @@ static int32_t mndArbGroupActionUpdate(SSdb *pSdb, SArbGroup *pOld, SArbGroup *p
|
|||
pOld->assignedLeader.dnodeId = pNew->assignedLeader.dnodeId;
|
||||
memcpy(pOld->assignedLeader.token, pNew->assignedLeader.token, TSDB_ARB_TOKEN_SIZE);
|
||||
pOld->version++;
|
||||
|
||||
_OVER:
|
||||
taosThreadMutexUnlock(&pOld->mutex);
|
||||
|
||||
taosHashRemove(arbUpdateHash, &pOld->vgId, sizeof(int32_t));
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -645,6 +654,11 @@ static void *mndBuildArbUpdateGroupReq(int32_t *pContLen, SArbGroup *pNewGroup)
|
|||
}
|
||||
|
||||
static int32_t mndPullupArbUpdateGroup(SMnode *pMnode, SArbGroup *pNewGroup) {
|
||||
if (taosHashGet(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId)) != NULL) {
|
||||
mInfo("vgId:%d, arb skip to pullup arb-update-group request, since it is in process", pNewGroup->vgId);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t contLen = 0;
|
||||
void *pHead = mndBuildArbUpdateGroupReq(&contLen, pNewGroup);
|
||||
if (!pHead) {
|
||||
|
|
@ -653,7 +667,11 @@ static int32_t mndPullupArbUpdateGroup(SMnode *pMnode, SArbGroup *pNewGroup) {
|
|||
}
|
||||
SRpcMsg rpcMsg = {.msgType = TDMT_MND_ARB_UPDATE_GROUP, .pCont = pHead, .contLen = contLen, .info.noResp = true};
|
||||
|
||||
return tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
|
||||
int32_t ret = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
|
||||
if (ret == 0) {
|
||||
taosHashPut(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId), NULL, 0);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int32_t mndProcessArbUpdateGroupReq(SRpcMsg *pReq) {
|
||||
|
|
@ -930,8 +948,12 @@ static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) {
|
|||
|
||||
SVArbCheckSyncRsp syncRsp = {0};
|
||||
if (tDeserializeSVArbCheckSyncRsp(pRsp->pCont, pRsp->contLen, &syncRsp) != 0) {
|
||||
terrno = TSDB_CODE_INVALID_MSG;
|
||||
mInfo("arb sync check failed, since:%s", tstrerror(pRsp->code));
|
||||
if (pRsp->code == TSDB_CODE_MND_ARB_TOKEN_MISMATCH) {
|
||||
terrno = TSDB_CODE_SUCCESS;
|
||||
return 0;
|
||||
}
|
||||
terrno = TSDB_CODE_INVALID_MSG;
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -22,10 +22,10 @@
|
|||
#include "mndPrivilege.h"
|
||||
#include "mndQnode.h"
|
||||
#include "mndShow.h"
|
||||
#include "mndSma.h"
|
||||
#include "mndStb.h"
|
||||
#include "mndUser.h"
|
||||
#include "mndView.h"
|
||||
#include "mndSma.h"
|
||||
#include "tglobal.h"
|
||||
#include "tversion.h"
|
||||
|
||||
|
|
@ -57,6 +57,13 @@ typedef struct {
|
|||
int64_t lastAccessTimeMs;
|
||||
} SAppObj;
|
||||
|
||||
typedef struct {
|
||||
int32_t totalDnodes;
|
||||
int32_t onlineDnodes;
|
||||
SEpSet epSet;
|
||||
SArray *pQnodeList;
|
||||
} SConnPreparedObj;
|
||||
|
||||
static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType, uint32_t ip, uint16_t port,
|
||||
int32_t pid, const char *app, int64_t startTime);
|
||||
static void mndFreeConn(SConnObj *pConn);
|
||||
|
|
@ -460,7 +467,7 @@ static int32_t mndGetOnlineDnodeNum(SMnode *pMnode, int32_t *num) {
|
|||
}
|
||||
|
||||
static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHbReq *pHbReq,
|
||||
SClientHbBatchRsp *pBatchRsp) {
|
||||
SClientHbBatchRsp *pBatchRsp, SConnPreparedObj *pObj) {
|
||||
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
|
||||
SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL};
|
||||
SRpcConnInfo connInfo = pMsg->info.conn;
|
||||
|
|
@ -503,11 +510,11 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
|
|||
}
|
||||
|
||||
rspBasic->connId = pConn->id;
|
||||
rspBasic->totalDnodes = mndGetDnodeSize(pMnode);
|
||||
mndGetOnlineDnodeNum(pMnode, &rspBasic->onlineDnodes);
|
||||
mndGetMnodeEpSet(pMnode, &rspBasic->epSet);
|
||||
|
||||
mndCreateQnodeList(pMnode, &rspBasic->pQnodeList, -1);
|
||||
rspBasic->connId = pConn->id;
|
||||
rspBasic->totalDnodes = pObj->totalDnodes;
|
||||
rspBasic->onlineDnodes = pObj->onlineDnodes;
|
||||
rspBasic->epSet = pObj->epSet;
|
||||
rspBasic->pQnodeList = taosArrayDup(pObj->pQnodeList, NULL);
|
||||
|
||||
mndReleaseConn(pMnode, pConn, true);
|
||||
|
||||
|
|
@ -608,7 +615,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
|
|||
}
|
||||
#endif
|
||||
case HEARTBEAT_KEY_TSMA: {
|
||||
void * rspMsg = NULL;
|
||||
void *rspMsg = NULL;
|
||||
int32_t rspLen = 0;
|
||||
mndValidateTSMAInfo(pMnode, kv->value, kv->valueLen / sizeof(STSMAVersion), &rspMsg, &rspLen);
|
||||
if (rspMsg && rspLen > 0) {
|
||||
|
|
@ -641,6 +648,12 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
SConnPreparedObj obj = {0};
|
||||
obj.totalDnodes = mndGetDnodeSize(pMnode);
|
||||
mndGetOnlineDnodeNum(pMnode, &obj.onlineDnodes);
|
||||
mndGetMnodeEpSet(pMnode, &obj.epSet);
|
||||
mndCreateQnodeList(pMnode, &obj.pQnodeList, -1);
|
||||
|
||||
SClientHbBatchRsp batchRsp = {0};
|
||||
batchRsp.svrTimestamp = taosGetTimestampSec();
|
||||
batchRsp.rsps = taosArrayInit(0, sizeof(SClientHbRsp));
|
||||
|
|
@ -649,7 +662,7 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) {
|
|||
for (int i = 0; i < sz; i++) {
|
||||
SClientHbReq *pHbReq = taosArrayGet(batchReq.reqs, i);
|
||||
if (pHbReq->connKey.connType == CONN_TYPE__QUERY) {
|
||||
mndProcessQueryHeartBeat(pMnode, pReq, pHbReq, &batchRsp);
|
||||
mndProcessQueryHeartBeat(pMnode, pReq, pHbReq, &batchRsp, &obj);
|
||||
} else if (pHbReq->connKey.connType == CONN_TYPE__TMQ) {
|
||||
SClientHbRsp *pRsp = mndMqHbBuildRsp(pMnode, pHbReq);
|
||||
if (pRsp != NULL) {
|
||||
|
|
@ -668,6 +681,8 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) {
|
|||
pReq->info.rspLen = tlen;
|
||||
pReq->info.rsp = buf;
|
||||
|
||||
taosArrayDestroy(obj.pQnodeList);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1457,8 +1457,8 @@ static void mndCreateTSMABuildCreateStreamReq(SCreateTSMACxt *pCxt) {
|
|||
pCxt->pCreateStreamReq->igUpdate = 0;
|
||||
pCxt->pCreateStreamReq->lastTs = pCxt->pCreateSmaReq->lastTs;
|
||||
pCxt->pCreateStreamReq->smaId = pCxt->pSma->uid;
|
||||
pCxt->pCreateStreamReq->ast = strdup(pCxt->pCreateSmaReq->ast);
|
||||
pCxt->pCreateStreamReq->sql = strdup(pCxt->pCreateSmaReq->sql);
|
||||
pCxt->pCreateStreamReq->ast = taosStrdup(pCxt->pCreateSmaReq->ast);
|
||||
pCxt->pCreateStreamReq->sql = taosStrdup(pCxt->pCreateSmaReq->sql);
|
||||
|
||||
// construct tags
|
||||
pCxt->pCreateStreamReq->pTags = taosArrayInit(pCxt->pCreateStreamReq->numOfTags, sizeof(SField));
|
||||
|
|
@ -1494,7 +1494,7 @@ static void mndCreateTSMABuildCreateStreamReq(SCreateTSMACxt *pCxt) {
|
|||
static void mndCreateTSMABuildDropStreamReq(SCreateTSMACxt* pCxt) {
|
||||
tstrncpy(pCxt->pDropStreamReq->name, pCxt->streamName, TSDB_STREAM_FNAME_LEN);
|
||||
pCxt->pDropStreamReq->igNotExists = false;
|
||||
pCxt->pDropStreamReq->sql = strdup(pCxt->pDropSmaReq->name);
|
||||
pCxt->pDropStreamReq->sql = taosStrdup(pCxt->pDropSmaReq->name);
|
||||
pCxt->pDropStreamReq->sqlLen = strlen(pCxt->pDropStreamReq->sql);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -847,7 +847,7 @@ int64_t mndStreamGenChkptId(SMnode *pMnode, bool lock) {
|
|||
if (pIter == NULL) break;
|
||||
|
||||
maxChkptId = TMAX(maxChkptId, pStream->checkpointId);
|
||||
mDebug("stream:%p, %s id:%" PRIx64 "checkpoint %" PRId64 "", pStream, pStream->name, pStream->uid,
|
||||
mDebug("stream:%p, %s id:0x%" PRIx64 " checkpoint %" PRId64 "", pStream, pStream->name, pStream->uid,
|
||||
pStream->checkpointId);
|
||||
sdbRelease(pSdb, pStream);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1156,14 +1156,24 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
|
|||
|
||||
// check if the checkpoint msg already sent or not.
|
||||
if (status == TASK_STATUS__CK) {
|
||||
tqWarn("s-task:%s recv checkpoint-source msg again checkpointId:%" PRId64
|
||||
" transId:%d already received, ignore this msg and continue process checkpoint",
|
||||
tqWarn("s-task:%s repeatly recv checkpoint-source msg checkpointId:%" PRId64
|
||||
" transId:%d already handled, ignore msg and continue process checkpoint",
|
||||
pTask->id.idStr, pTask->chkInfo.checkpointingId, req.transId);
|
||||
|
||||
taosThreadMutexUnlock(&pTask->lock);
|
||||
streamMetaReleaseTask(pMeta, pTask);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
} else { // checkpoint already finished, and not in checkpoint status
|
||||
if (req.checkpointId <= pTask->chkInfo.checkpointId) {
|
||||
tqWarn("s-task:%s repeatly recv checkpoint-source msg checkpointId:%" PRId64
|
||||
" transId:%d already handled, ignore and discard", pTask->id.idStr, req.checkpointId, req.transId);
|
||||
|
||||
taosThreadMutexUnlock(&pTask->lock);
|
||||
streamMetaReleaseTask(pMeta, pTask);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
streamProcessCheckpointSourceReq(pTask, &req);
|
||||
|
|
|
|||
|
|
@ -485,6 +485,7 @@ SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, in
|
|||
|
||||
int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId, const char* id) {
|
||||
if (tSimpleHashGetSize(pSinkTableMap) > MAX_CACHE_TABLE_INFO_NUM) {
|
||||
taosMemoryFreeClear(pTableSinkInfo); // too many items, failed to cache it
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -195,13 +195,22 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
|
|||
const char* idstr = pTask->id.idStr;
|
||||
|
||||
if (pMeta->updateInfo.transId != req.transId) {
|
||||
ASSERT(req.transId > pMeta->updateInfo.transId);
|
||||
tqInfo("s-task:%s vgId:%d receive new trans to update nodeEp msg from mnode, transId:%d, prev transId:%d", idstr,
|
||||
vgId, req.transId, pMeta->updateInfo.transId);
|
||||
if (req.transId < pMeta->updateInfo.transId) {
|
||||
tqError("s-task:%s vgId:%d disorder update nodeEp msg recv, discarded, newest transId:%d, recv:%d", idstr, vgId,
|
||||
pMeta->updateInfo.transId, req.transId);
|
||||
rsp.code = TSDB_CODE_SUCCESS;
|
||||
streamMetaWUnLock(pMeta);
|
||||
|
||||
// info needs to be kept till the new trans to update the nodeEp arrived.
|
||||
taosHashClear(pMeta->updateInfo.pTasks);
|
||||
pMeta->updateInfo.transId = req.transId;
|
||||
taosArrayDestroy(req.pNodeList);
|
||||
return rsp.code;
|
||||
} else {
|
||||
tqInfo("s-task:%s vgId:%d receive new trans to update nodeEp msg from mnode, transId:%d, prev transId:%d", idstr,
|
||||
vgId, req.transId, pMeta->updateInfo.transId);
|
||||
|
||||
// info needs to be kept till the new trans to update the nodeEp arrived.
|
||||
taosHashClear(pMeta->updateInfo.pTasks);
|
||||
pMeta->updateInfo.transId = req.transId;
|
||||
}
|
||||
} else {
|
||||
tqDebug("s-task:%s vgId:%d recv trans to update nodeEp from mnode, transId:%d", idstr, vgId, req.transId);
|
||||
}
|
||||
|
|
@ -211,7 +220,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
|
|||
|
||||
void* pReqTask = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry));
|
||||
if (pReqTask != NULL) {
|
||||
tqDebug("s-task:%s (vgId:%d) already update in trans:%d, discard the nodeEp update msg", idstr, vgId, req.transId);
|
||||
tqDebug("s-task:%s (vgId:%d) already update in transId:%d, discard the nodeEp update msg", idstr, vgId, req.transId);
|
||||
rsp.code = TSDB_CODE_SUCCESS;
|
||||
streamMetaWUnLock(pMeta);
|
||||
taosArrayDestroy(req.pNodeList);
|
||||
|
|
@ -235,7 +244,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
|
|||
} else {
|
||||
tqDebug("s-task:%s fill-history task update nodeEp along with stream task", (*ppHTask)->id.idStr);
|
||||
bool updateEpSet = streamTaskUpdateEpsetInfo(*ppHTask, req.pNodeList);
|
||||
if (!updated) {
|
||||
if (updateEpSet) {
|
||||
updated = updateEpSet;
|
||||
}
|
||||
|
||||
|
|
@ -245,14 +254,15 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
|
|||
}
|
||||
|
||||
if (updated) {
|
||||
tqDebug("s-task:%s vgId:%d save task after update epset", idstr, vgId);
|
||||
tqDebug("s-task:%s vgId:%d save task after update epset, and stop task", idstr, vgId);
|
||||
streamMetaSaveTask(pMeta, pTask);
|
||||
if (ppHTask != NULL) {
|
||||
streamMetaSaveTask(pMeta, *ppHTask);
|
||||
}
|
||||
} else {
|
||||
tqDebug("s-task:%s vgId:%d not save task since not update epset actually, stop task", idstr, vgId);
|
||||
}
|
||||
|
||||
tqDebug("s-task:%s vgId:%d start to stop task after save task", idstr, vgId);
|
||||
streamTaskStop(pTask);
|
||||
|
||||
// keep the already updated info
|
||||
|
|
|
|||
|
|
@ -64,13 +64,15 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
|
|||
col_id_t colId = -1;
|
||||
|
||||
SArray* funcTypeBlockArray = taosArrayInit(pReader->numOfCols, sizeof(int32_t));
|
||||
|
||||
for (int32_t i = 0; i < pReader->numOfCols; ++i) {
|
||||
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]);
|
||||
int32_t funcType = FUNCTION_TYPE_CACHE_LAST;
|
||||
|
||||
if (pReader->pFuncTypeList != NULL && taosArrayGetSize(pReader->pFuncTypeList) > i) {
|
||||
funcType = *(int32_t*)taosArrayGet(pReader->pFuncTypeList, i);
|
||||
taosArrayInsert(funcTypeBlockArray, dstSlotIds[i], taosArrayGet(pReader->pFuncTypeList, i));
|
||||
}
|
||||
taosArrayInsert(funcTypeBlockArray, dstSlotIds[i], taosArrayGet(pReader->pFuncTypeList, i));
|
||||
|
||||
if (slotIds[i] == -1) {
|
||||
if (FUNCTION_TYPE_CACHE_LAST_ROW == funcType) {
|
||||
|
|
@ -367,11 +369,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
|
|||
goto _end;
|
||||
}
|
||||
|
||||
int32_t pkBufLen = 0;
|
||||
if (pr->rowKey.numOfPKs > 0) {
|
||||
pkBufLen = pr->pkColumn.bytes;
|
||||
}
|
||||
|
||||
int32_t pkBufLen = (pr->rowKey.numOfPKs > 0)? pr->pkColumn.bytes:0;
|
||||
for (int32_t j = 0; j < pr->numOfCols; ++j) {
|
||||
int32_t bytes = (slotIds[j] == -1) ? 1 : pr->pSchema->columns[slotIds[j]].bytes;
|
||||
|
||||
|
|
|
|||
|
|
@ -897,6 +897,7 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) {
|
|||
|
||||
// commit
|
||||
code = commit_edit(fs);
|
||||
ASSERT(code == 0);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
// schedule merge
|
||||
|
|
@ -973,11 +974,11 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) {
|
|||
|
||||
_exit:
|
||||
if (code) {
|
||||
TSDB_ERROR_LOG(TD_VID(fs->tsdb->pVnode), lino, code);
|
||||
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(fs->tsdb->pVnode), __func__, lino, tstrerror(code));
|
||||
} else {
|
||||
tsdbDebug("vgId:%d %s done, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype);
|
||||
tsem_post(&fs->canEdit);
|
||||
tsdbInfo("vgId:%d %s done, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype);
|
||||
}
|
||||
tsem_post(&fs->canEdit);
|
||||
return code;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -580,9 +580,9 @@ int32_t tsdbMerge(void *arg) {
|
|||
}
|
||||
*/
|
||||
// do merge
|
||||
tsdbDebug("vgId:%d merge begin, fid:%d", TD_VID(tsdb->pVnode), merger->fid);
|
||||
tsdbInfo("vgId:%d merge begin, fid:%d", TD_VID(tsdb->pVnode), merger->fid);
|
||||
code = tsdbDoMerge(merger);
|
||||
tsdbDebug("vgId:%d merge done, fid:%d", TD_VID(tsdb->pVnode), mergeArg->fid);
|
||||
tsdbInfo("vgId:%d merge done, fid:%d", TD_VID(tsdb->pVnode), mergeArg->fid);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
_exit:
|
||||
|
|
|
|||
|
|
@ -1054,14 +1054,30 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo
|
|||
}
|
||||
}
|
||||
|
||||
static void blockInfoToRecord(SBrinRecord* record, SFileDataBlockInfo* pBlockInfo) {
|
||||
static void blockInfoToRecord(SBrinRecord* record, SFileDataBlockInfo* pBlockInfo, SBlockLoadSuppInfo* pSupp) {
|
||||
record->uid = pBlockInfo->uid;
|
||||
record->firstKey = (STsdbRowKey){
|
||||
.key = {.ts = pBlockInfo->firstKey, .numOfPKs = 0},
|
||||
};
|
||||
record->lastKey = (STsdbRowKey){
|
||||
.key = {.ts = pBlockInfo->lastKey, .numOfPKs = 0},
|
||||
};
|
||||
record->firstKey = (STsdbRowKey){.key = {.ts = pBlockInfo->firstKey, .numOfPKs = pSupp->numOfPks}};
|
||||
record->lastKey = (STsdbRowKey){.key = {.ts = pBlockInfo->lastKey, .numOfPKs = pSupp->numOfPks}};
|
||||
|
||||
if (pSupp->numOfPks > 0) {
|
||||
SValue* pFirst = &record->firstKey.key.pks[0];
|
||||
SValue* pLast = &record->lastKey.key.pks[0];
|
||||
|
||||
pFirst->type = pSupp->pk.type;
|
||||
pLast->type = pSupp->pk.type;
|
||||
|
||||
if (IS_VAR_DATA_TYPE(pFirst->type)) {
|
||||
pFirst->pData = (uint8_t*) varDataVal(pBlockInfo->firstPk.pData);
|
||||
pFirst->nData = varDataLen(pBlockInfo->firstPk.pData);
|
||||
|
||||
pLast->pData = (uint8_t*) varDataVal(pBlockInfo->lastPk.pData);
|
||||
pLast->nData = varDataLen(pBlockInfo->lastPk.pData);
|
||||
} else {
|
||||
pFirst->val = pBlockInfo->firstPk.val;
|
||||
pLast->val = pBlockInfo->lastPk.val;
|
||||
}
|
||||
}
|
||||
|
||||
record->minVer = pBlockInfo->minVer;
|
||||
record->maxVer = pBlockInfo->maxVer;
|
||||
record->blockOffset = pBlockInfo->blockOffset;
|
||||
|
|
@ -1091,7 +1107,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro
|
|||
int32_t step = asc ? 1 : -1;
|
||||
|
||||
SBrinRecord tmp;
|
||||
blockInfoToRecord(&tmp, pBlockInfo);
|
||||
blockInfoToRecord(&tmp, pBlockInfo, pSupInfo);
|
||||
SBrinRecord* pRecord = &tmp;
|
||||
|
||||
// no data exists, return directly.
|
||||
|
|
@ -1290,7 +1306,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI
|
|||
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
|
||||
|
||||
SBrinRecord tmp;
|
||||
blockInfoToRecord(&tmp, pBlockInfo);
|
||||
blockInfoToRecord(&tmp, pBlockInfo, pSup);
|
||||
SBrinRecord* pRecord = &tmp;
|
||||
code = tsdbDataFileReadBlockDataByColumn(pReader->pFileReader, pRecord, pBlockData, pSchema, &pSup->colId[1],
|
||||
pSup->numOfCols - 1);
|
||||
|
|
@ -1325,9 +1341,9 @@ static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* p
|
|||
(pVerRange->maxVer < pBlock->maxVer && pVerRange->maxVer >= pBlock->minVer);
|
||||
}
|
||||
|
||||
static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pBlockInfo,
|
||||
STableBlockScanInfo* pScanInfo, int32_t* nextIndex, int32_t order,
|
||||
SBrinRecord* pRecord) {
|
||||
static bool getNeighborBlockOfTable(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pBlockInfo,
|
||||
STableBlockScanInfo* pScanInfo, int32_t* nextIndex, int32_t order,
|
||||
SBrinRecord* pRecord, SBlockLoadSuppInfo* pSupInfo) {
|
||||
bool asc = ASCENDING_TRAVERSE(order);
|
||||
int32_t step = asc ? 1 : -1;
|
||||
|
||||
|
|
@ -1341,7 +1357,7 @@ static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlo
|
|||
|
||||
STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx + step);
|
||||
SFileDataBlockInfo* p = taosArrayGet(pBlockIter->blockList, pTableDataBlockIdx->globalIndex);
|
||||
blockInfoToRecord(pRecord, p);
|
||||
blockInfoToRecord(pRecord, p, pSupInfo);
|
||||
|
||||
*nextIndex = pBlockInfo->tbBlockIdx + step;
|
||||
return true;
|
||||
|
|
@ -1391,12 +1407,40 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte
|
|||
}
|
||||
|
||||
// todo: this attribute could be acquired during extractin the global ordered block list.
|
||||
static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* pRec, int32_t order) {
|
||||
static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* pRec, int32_t order, int32_t pkType, int32_t numOfPk) {
|
||||
// it is the last block in current file, no chance to overlap with neighbor blocks.
|
||||
if (ASCENDING_TRAVERSE(order)) {
|
||||
return pBlock->lastKey == pRec->firstKey.key.ts;
|
||||
if (pBlock->lastKey == pRec->firstKey.key.ts) {
|
||||
if (numOfPk > 0) {
|
||||
SValue v1 = {.type = pkType};
|
||||
if (IS_VAR_DATA_TYPE(pkType)) {
|
||||
v1.pData = (uint8_t*)varDataVal(pBlock->lastPk.pData), v1.nData = varDataLen(pBlock->lastPk.pData);
|
||||
} else {
|
||||
v1.val = pBlock->lastPk.val;
|
||||
}
|
||||
return (tValueCompare(&v1, &pRec->firstKey.key.pks[0]) == 0);
|
||||
} else { // no pk
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
return pBlock->firstKey == pRec->lastKey.key.ts;
|
||||
if (pBlock->firstKey == pRec->lastKey.key.ts) {
|
||||
if (numOfPk > 0) {
|
||||
SValue v1 = {.type = pkType};
|
||||
if (IS_VAR_DATA_TYPE(pkType)) {
|
||||
v1.pData = (uint8_t*)varDataVal(pBlock->firstPk.pData), v1.nData = varDataLen(pBlock->firstPk.pData);
|
||||
} else {
|
||||
v1.val = pBlock->firstPk.val;
|
||||
}
|
||||
return (tValueCompare(&v1, &pRec->lastKey.key.pks[0]) == 0);
|
||||
} else { // no pk
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1430,23 +1474,26 @@ static bool keyOverlapFileBlock(TSDBKEY key, SFileDataBlockInfo* pBlock, SVersio
|
|||
|
||||
static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pBlockInfo,
|
||||
STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, STsdbReader* pReader) {
|
||||
SBrinRecord rec = {0};
|
||||
int32_t neighborIndex = 0;
|
||||
SBrinRecord rec = {0};
|
||||
int32_t neighborIndex = 0;
|
||||
int32_t order = pReader->info.order;
|
||||
SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
|
||||
|
||||
bool hasNeighbor = getNeighborBlockOfSameTable(&pReader->status.blockIter, pBlockInfo, pScanInfo, &neighborIndex,
|
||||
pReader->info.order, &rec);
|
||||
bool hasNeighbor =
|
||||
getNeighborBlockOfTable(&pReader->status.blockIter, pBlockInfo, pScanInfo, &neighborIndex, order, &rec, pSupInfo);
|
||||
|
||||
// overlap with neighbor
|
||||
if (hasNeighbor) {
|
||||
pInfo->overlapWithNeighborBlock = overlapWithNeighborBlock2(pBlockInfo, &rec, pReader->info.order);
|
||||
pInfo->overlapWithNeighborBlock =
|
||||
overlapWithNeighborBlock2(pBlockInfo, &rec, order, pSupInfo->pk.type, pSupInfo->numOfPks);
|
||||
}
|
||||
|
||||
SBrinRecord pRecord;
|
||||
blockInfoToRecord(&pRecord, pBlockInfo);
|
||||
blockInfoToRecord(&pRecord, pBlockInfo, pSupInfo);
|
||||
|
||||
// has duplicated ts of different version in this block
|
||||
pInfo->hasDupTs = (pBlockInfo->numRow > pBlockInfo->count) || (pBlockInfo->count <= 0);
|
||||
pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, &pRecord, pReader->info.order);
|
||||
pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, &pRecord, order);
|
||||
|
||||
// todo handle the primary key overlap case
|
||||
ASSERT(pScanInfo->sttKeyInfo.status != STT_FILE_READER_UNINIT);
|
||||
|
|
@ -2380,28 +2427,30 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI
|
|||
|
||||
static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pBlockScanInfo,
|
||||
STsdbReader* pReader, bool* loadNeighbor) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1;
|
||||
int32_t nextIndex = -1;
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t order = pReader->info.order;
|
||||
SDataBlockIter* pIter = &pReader->status.blockIter;
|
||||
SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
|
||||
int32_t step = ASCENDING_TRAVERSE(order) ? 1 : -1;
|
||||
int32_t nextIndex = -1;
|
||||
SBrinRecord rec = {0};
|
||||
|
||||
*loadNeighbor = false;
|
||||
|
||||
SBrinRecord rec = {0};
|
||||
bool hasNeighbor = getNeighborBlockOfSameTable(&pReader->status.blockIter, pBlockInfo, pBlockScanInfo, &nextIndex,
|
||||
pReader->info.order, &rec);
|
||||
bool hasNeighbor = getNeighborBlockOfTable(pIter, pBlockInfo, pBlockScanInfo, &nextIndex, order, &rec, pSupInfo);
|
||||
if (!hasNeighbor) { // do nothing
|
||||
return code;
|
||||
}
|
||||
|
||||
if (overlapWithNeighborBlock2(pBlockInfo, &rec, pReader->info.order)) { // load next block
|
||||
// load next block
|
||||
if (overlapWithNeighborBlock2(pBlockInfo, &rec, order, pReader->suppInfo.pk.type, pReader->suppInfo.numOfPks)) {
|
||||
SReaderStatus* pStatus = &pReader->status;
|
||||
SDataBlockIter* pBlockIter = &pStatus->blockIter;
|
||||
|
||||
// 1. find the next neighbor block in the scan block list
|
||||
STableDataBlockIdx* tableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, nextIndex);
|
||||
int32_t neighborIndex = tableDataBlockIdx->globalIndex;
|
||||
|
||||
// 2. remove it from the scan block list
|
||||
int32_t neighborIndex = tableDataBlockIdx->globalIndex;
|
||||
setFileBlockActiveInBlockIter(pReader, pBlockIter, neighborIndex, step);
|
||||
|
||||
// 3. load the neighbor block, and set it to be the currently accessed file data block
|
||||
|
|
@ -4836,11 +4885,11 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock,
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter);
|
||||
SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter);
|
||||
SBlockLoadSuppInfo* pSup = &pReader->suppInfo;
|
||||
|
||||
SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock;
|
||||
if (pResBlock->info.id.uid != pFBlock->uid) {
|
||||
if (pResBlock->info.id.uid != pBlockInfo->uid) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
|
@ -4848,10 +4897,10 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock,
|
|||
TARRAY2_CLEAR(&pSup->colAggArray, 0);
|
||||
|
||||
SBrinRecord pRecord;
|
||||
blockInfoToRecord(&pRecord, pFBlock);
|
||||
blockInfoToRecord(&pRecord, pBlockInfo, pSup);
|
||||
code = tsdbDataFileReadBlockSma(pReader->pFileReader, &pRecord, &pSup->colAggArray);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64 ", code:%s, %s", 0, pFBlock->uid, tstrerror(code),
|
||||
tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64 ", code:%s, %s", 0, pBlockInfo->uid, tstrerror(code),
|
||||
pReader->idStr);
|
||||
return code;
|
||||
}
|
||||
|
|
@ -4880,7 +4929,7 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock,
|
|||
}
|
||||
|
||||
// do fill all null column value SMA info
|
||||
doFillNullColSMA(pSup, pFBlock->numRow, numOfCols, pTsAgg);
|
||||
doFillNullColSMA(pSup, pBlockInfo->numRow, numOfCols, pTsAgg);
|
||||
|
||||
size_t size = pSup->colAggArray.size;
|
||||
|
||||
|
|
@ -4906,7 +4955,7 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock,
|
|||
// double elapsedTime = (taosGetTimestampUs() - st) / 1000.0;
|
||||
pReader->cost.smaLoadTime += 0; // elapsedTime;
|
||||
|
||||
tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pFBlock->uid, pReader->idStr);
|
||||
tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pBlockInfo->uid, pReader->idStr);
|
||||
return code;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -926,7 +926,7 @@ static int32_t execAlterLocal(SAlterLocalStmt* pStmt) {
|
|||
return terrno;
|
||||
}
|
||||
|
||||
if (cfgSetItem(tsCfg, pStmt->config, pStmt->value, CFG_STYPE_ALTER_CMD)) {
|
||||
if (cfgSetItem(tsCfg, pStmt->config, pStmt->value, CFG_STYPE_ALTER_CMD, true)) {
|
||||
return terrno;
|
||||
}
|
||||
|
||||
|
|
@ -967,46 +967,11 @@ static int32_t buildLocalVariablesResultDataBlock(SSDataBlock** pOutput) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t setLocalVariablesResultIntoDataBlock(SSDataBlock* pBlock) {
|
||||
int32_t numOfCfg = taosArrayGetSize(tsCfg->array);
|
||||
int32_t numOfRows = 0;
|
||||
blockDataEnsureCapacity(pBlock, numOfCfg);
|
||||
|
||||
for (int32_t i = 0, c = 0; i < numOfCfg; ++i, c = 0) {
|
||||
SConfigItem* pItem = taosArrayGet(tsCfg->array, i);
|
||||
// GRANT_CFG_SKIP;
|
||||
|
||||
char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE);
|
||||
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
|
||||
colDataSetVal(pColInfo, i, name, false);
|
||||
|
||||
char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
int32_t valueLen = 0;
|
||||
cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen);
|
||||
varDataSetLen(value, valueLen);
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
|
||||
colDataSetVal(pColInfo, i, value, false);
|
||||
|
||||
char scope[TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
cfgDumpItemScope(pItem, &scope[VARSTR_HEADER_SIZE], TSDB_CONFIG_SCOPE_LEN, &valueLen);
|
||||
varDataSetLen(scope, valueLen);
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
|
||||
colDataSetVal(pColInfo, i, scope, false);
|
||||
|
||||
numOfRows++;
|
||||
}
|
||||
|
||||
pBlock->info.rows = numOfRows;
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t execShowLocalVariables(SRetrieveTableRsp** pRsp) {
|
||||
SSDataBlock* pBlock = NULL;
|
||||
int32_t code = buildLocalVariablesResultDataBlock(&pBlock);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = setLocalVariablesResultIntoDataBlock(pBlock);
|
||||
code = dumpConfToDataBlock(pBlock, 0);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = buildRetrieveTableRsp(pBlock, SHOW_LOCAL_VARIABLES_RESULT_COLS, pRsp);
|
||||
|
|
|
|||
|
|
@ -278,7 +278,7 @@ int32_t doAggregateImpl(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx) {
|
|||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
for (int32_t k = 0; k < pOperator->exprSupp.numOfExprs; ++k) {
|
||||
if (functionNeedToExecute(&pCtx[k])) {
|
||||
// todo add a dummy funtion to avoid process check
|
||||
// todo add a dummy function to avoid process check
|
||||
if (pCtx[k].fpSet.process == NULL) {
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -221,6 +221,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
|
|||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
STableListInfo* pTableList = pInfo->pTableList;
|
||||
SStoreCacheReader* pReaderFn = &pInfo->readHandle.api.cacheFn;
|
||||
SSDataBlock* pBufRes = pInfo->pBufferedRes;
|
||||
|
||||
uint64_t suid = tableListGetSuid(pTableList);
|
||||
int32_t size = tableListGetSize(pTableList);
|
||||
|
|
@ -237,18 +238,18 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
|
|||
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
|
||||
}
|
||||
|
||||
if (pInfo->indexOfBufferedRes >= pInfo->pBufferedRes->info.rows) {
|
||||
blockDataCleanup(pInfo->pBufferedRes);
|
||||
if (pInfo->indexOfBufferedRes >= pBufRes->info.rows) {
|
||||
blockDataCleanup(pBufRes);
|
||||
taosArrayClear(pInfo->pUidList);
|
||||
|
||||
int32_t code = pReaderFn->retrieveRows(pInfo->pLastrowReader, pInfo->pBufferedRes, pInfo->pSlotIds,
|
||||
pInfo->pDstSlotIds, pInfo->pUidList);
|
||||
int32_t code =
|
||||
pReaderFn->retrieveRows(pInfo->pLastrowReader, pBufRes, pInfo->pSlotIds, pInfo->pDstSlotIds, pInfo->pUidList);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
T_LONG_JMP(pTaskInfo->env, code);
|
||||
}
|
||||
|
||||
// check for tag values
|
||||
int32_t resultRows = pInfo->pBufferedRes->info.rows;
|
||||
int32_t resultRows = pBufRes->info.rows;
|
||||
|
||||
// the results may be null, if last values are all null
|
||||
ASSERT(resultRows == 0 || resultRows == taosArrayGetSize(pInfo->pUidList));
|
||||
|
|
@ -257,12 +258,12 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
|
|||
|
||||
SSDataBlock* pRes = pInfo->pRes;
|
||||
|
||||
if (pInfo->indexOfBufferedRes < pInfo->pBufferedRes->info.rows) {
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pBufferedRes->pDataBlock); ++i) {
|
||||
if (pInfo->indexOfBufferedRes < pBufRes->info.rows) {
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pBufRes->pDataBlock); ++i) {
|
||||
SColumnInfoData* pCol = taosArrayGet(pRes->pDataBlock, i);
|
||||
int32_t slotId = pCol->info.slotId;
|
||||
|
||||
SColumnInfoData* pSrc = taosArrayGet(pInfo->pBufferedRes->pDataBlock, slotId);
|
||||
SColumnInfoData* pSrc = taosArrayGet(pBufRes->pDataBlock, slotId);
|
||||
SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, slotId);
|
||||
|
||||
if (colDataIsNull_s(pSrc, pInfo->indexOfBufferedRes)) {
|
||||
|
|
|
|||
|
|
@ -2837,7 +2837,7 @@ static int32_t firstLastTransferInfoImpl(SFirstLastRes* pInput, SFirstLastRes* p
|
|||
memcpy(pOutput->buf, pInput->buf, pOutput->bytes);
|
||||
if (pInput->pkData) {
|
||||
pOutput->pkBytes = pInput->pkBytes;
|
||||
memcpy(pOutput->buf+pOutput->bytes, pInput->pkData, pOutput->pkBytes);
|
||||
memcpy(pOutput->buf + pOutput->bytes, pInput->pkData, pOutput->pkBytes);
|
||||
pOutput->pkData = pOutput->buf + pOutput->bytes;
|
||||
}
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
|
@ -2885,7 +2885,8 @@ static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuer
|
|||
} else {
|
||||
pInputInfo->pkData = NULL;
|
||||
}
|
||||
int32_t code = firstLastTransferInfo(pCtx, pInputInfo, pInfo, isFirstQuery, i);
|
||||
|
||||
int32_t code = firstLastTransferInfo(pCtx, pInputInfo, pInfo, isFirstQuery, i);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return code;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2343,7 +2343,7 @@ static EDealRes collectFuncs(SNode* pNode, void* pContext) {
|
|||
return DEAL_RES_CONTINUE;
|
||||
}
|
||||
}
|
||||
SExprNode* pExpr = (SExprNode*)pNode;
|
||||
|
||||
bool bFound = false;
|
||||
SNode* pn = NULL;
|
||||
FOREACH(pn, pCxt->pFuncs) {
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ bool keysHasCol(SNodeList* pKeys);
|
|||
bool keysHasTbname(SNodeList* pKeys);
|
||||
SFunctionNode* createGroupKeyAggFunc(SColumnNode* pGroupCol);
|
||||
int32_t getTimeRangeFromNode(SNode** pPrimaryKeyCond, STimeWindow* pTimeRange, bool* pIsStrict);
|
||||
int32_t tagScanSetExecutionMode(SScanLogicNode* pScan);
|
||||
|
||||
#define CLONE_LIMIT 1
|
||||
#define CLONE_SLIMIT 1 << 1
|
||||
|
|
|
|||
|
|
@ -19,6 +19,9 @@
|
|||
#include "tglobal.h"
|
||||
#include "parser.h"
|
||||
|
||||
// primary key column always the second column if exists
|
||||
#define PRIMARY_COLUMN_SLOT 1
|
||||
|
||||
typedef struct SLogicPlanContext {
|
||||
SPlanContext* pPlanCxt;
|
||||
SLogicNode* pCurrRoot;
|
||||
|
|
@ -304,7 +307,7 @@ static SNode* createFirstCol(SRealTableNode* pTable, const SSchema* pSchema) {
|
|||
return (SNode*)pCol;
|
||||
}
|
||||
|
||||
static int32_t addPrimaryKeyCol(SRealTableNode* pTable, SNodeList** pCols) {
|
||||
static int32_t addPrimaryTsCol(SRealTableNode* pTable, SNodeList** pCols) {
|
||||
bool found = false;
|
||||
SNode* pCol = NULL;
|
||||
FOREACH(pCol, *pCols) {
|
||||
|
|
@ -327,10 +330,10 @@ static int32_t addSystableFirstCol(SRealTableNode* pTable, SNodeList** pCols) {
|
|||
return nodesListMakeStrictAppend(pCols, createFirstCol(pTable, pTable->pMeta->schema));
|
||||
}
|
||||
|
||||
static int32_t addPkCol(SRealTableNode* pTable, SNodeList** pCols) {
|
||||
static int32_t addPrimaryKeyCol(SRealTableNode* pTable, SNodeList** pCols) {
|
||||
bool found = false;
|
||||
SNode* pCol = NULL;
|
||||
SSchema* pSchema = &pTable->pMeta->schema[1];
|
||||
SSchema* pSchema = &pTable->pMeta->schema[PRIMARY_COLUMN_SLOT];
|
||||
FOREACH(pCol, *pCols) {
|
||||
if (pSchema->colId == ((SColumnNode*)pCol)->colId) {
|
||||
found = true;
|
||||
|
|
@ -348,9 +351,9 @@ static int32_t addDefaultScanCol(SRealTableNode* pTable, SNodeList** pCols) {
|
|||
if (TSDB_SYSTEM_TABLE == pTable->pMeta->tableType) {
|
||||
return addSystableFirstCol(pTable, pCols);
|
||||
}
|
||||
int32_t code = addPrimaryKeyCol(pTable, pCols);
|
||||
int32_t code = addPrimaryTsCol(pTable, pCols);
|
||||
if (code == TSDB_CODE_SUCCESS && hasPkInTable(pTable->pMeta)) {
|
||||
code = addPkCol(pTable, pCols);
|
||||
code = addPrimaryKeyCol(pTable, pCols);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
|
@ -389,60 +392,6 @@ static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealT
|
|||
|
||||
static bool needScanDefaultCol(EScanType scanType) { return SCAN_TYPE_TABLE_COUNT != scanType; }
|
||||
|
||||
static EDealRes tagScanNodeHasTbnameFunc(SNode* pNode, void* pContext) {
|
||||
if (QUERY_NODE_FUNCTION == nodeType(pNode) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pNode)->funcType ||
|
||||
(QUERY_NODE_COLUMN == nodeType(pNode) && COLUMN_TYPE_TBNAME == ((SColumnNode*)pNode)->colType)) {
|
||||
*(bool*)pContext = true;
|
||||
return DEAL_RES_END;
|
||||
}
|
||||
return DEAL_RES_CONTINUE;
|
||||
}
|
||||
|
||||
static bool tagScanNodeListHasTbname(SNodeList* pCols) {
|
||||
bool hasTbname = false;
|
||||
nodesWalkExprs(pCols, tagScanNodeHasTbnameFunc, &hasTbname);
|
||||
return hasTbname;
|
||||
}
|
||||
|
||||
static bool tagScanNodeHasTbname(SNode* pKeys) {
|
||||
bool hasTbname = false;
|
||||
nodesWalkExpr(pKeys, tagScanNodeHasTbnameFunc, &hasTbname);
|
||||
return hasTbname;
|
||||
}
|
||||
|
||||
static int32_t tagScanSetExecutionMode(SScanLogicNode* pScan) {
|
||||
pScan->onlyMetaCtbIdx = false;
|
||||
|
||||
if (pScan->tableType == TSDB_CHILD_TABLE) {
|
||||
pScan->onlyMetaCtbIdx = false;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (tagScanNodeListHasTbname(pScan->pScanPseudoCols)) {
|
||||
pScan->onlyMetaCtbIdx = false;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (pScan->node.pConditions == NULL) {
|
||||
pScan->onlyMetaCtbIdx = true;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SNode* pCond = nodesCloneNode(pScan->node.pConditions);
|
||||
SNode* pTagCond = NULL;
|
||||
SNode* pTagIndexCond = NULL;
|
||||
filterPartitionCond(&pCond, NULL, &pTagIndexCond, &pTagCond, NULL);
|
||||
if (pTagIndexCond || tagScanNodeHasTbname(pTagCond)) {
|
||||
pScan->onlyMetaCtbIdx = false;
|
||||
} else {
|
||||
pScan->onlyMetaCtbIdx = true;
|
||||
}
|
||||
nodesDestroyNode(pCond);
|
||||
nodesDestroyNode(pTagIndexCond);
|
||||
nodesDestroyNode(pTagCond);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable,
|
||||
SLogicNode** pLogicNode) {
|
||||
SScanLogicNode* pScan = NULL;
|
||||
|
|
@ -1802,7 +1751,7 @@ static int32_t createDeleteScanLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* p
|
|||
|
||||
STableMeta* pMeta = ((SRealTableNode*)pDelete->pFromTable)->pMeta;
|
||||
if (TSDB_CODE_SUCCESS == code && hasPkInTable(pMeta)) {
|
||||
code = addPkCol((SRealTableNode*)pDelete->pFromTable, &pScan->pScanCols);
|
||||
code = addPrimaryKeyCol((SRealTableNode*)pDelete->pFromTable, &pScan->pScanCols);
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code && NULL != pDelete->pTagCond) {
|
||||
|
|
|
|||
|
|
@ -3966,21 +3966,25 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic
|
|||
if (NULL != cxt.pLastCols) {
|
||||
cxt.doAgg = false;
|
||||
cxt.funcType = FUNCTION_TYPE_CACHE_LAST;
|
||||
|
||||
lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols, pLastRowCols, true, cxt.pkBytes);
|
||||
nodesWalkExprs(pScan->pScanPseudoCols, lastRowScanOptSetColDataType, &cxt);
|
||||
|
||||
lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols, pLastRowCols, false, cxt.pkBytes);
|
||||
lastRowScanOptRemoveUslessTargets(pScan->node.pTargets, cxt.pLastCols, cxt.pOtherCols, pLastRowCols);
|
||||
if (pPKTsCol && pScan->node.pTargets->length == 1) {
|
||||
if (pPKTsCol && ((pScan->node.pTargets->length == 1) || (pScan->node.pTargets->length == 2 && cxt.pkBytes > 0))) {
|
||||
// when select last(ts),ts from ..., we add another ts to targets
|
||||
sprintf(pPKTsCol->colName, "#sel_val.%p", pPKTsCol);
|
||||
nodesListAppend(pScan->node.pTargets, nodesCloneNode((SNode*)pPKTsCol));
|
||||
}
|
||||
|
||||
if (pNonPKCol && cxt.pLastCols->length == 1 &&
|
||||
nodesEqualNode((SNode*)pNonPKCol, nodesListGetNode(cxt.pLastCols, 0))) {
|
||||
// when select last(c1), c1 from ..., we add c1 to targets
|
||||
sprintf(pNonPKCol->colName, "#sel_val.%p", pNonPKCol);
|
||||
nodesListAppend(pScan->node.pTargets, nodesCloneNode((SNode*)pNonPKCol));
|
||||
}
|
||||
|
||||
nodesClearList(cxt.pLastCols);
|
||||
}
|
||||
nodesClearList(cxt.pOtherCols);
|
||||
|
|
@ -5138,7 +5142,6 @@ int32_t stbJoinOptRewriteToTagScan(SLogicNode* pJoin, SNode* pNode) {
|
|||
NODES_DESTORY_NODE(pScan->node.pConditions);
|
||||
pScan->node.requireDataOrder = DATA_ORDER_LEVEL_NONE;
|
||||
pScan->node.resultDataOrder = DATA_ORDER_LEVEL_NONE;
|
||||
pScan->onlyMetaCtbIdx = true;
|
||||
|
||||
SNodeList* pTags = nodesMakeList();
|
||||
int32_t code = nodesCollectColumnsFromNode(pJoinNode->pTagEqCond, NULL, COLLECT_COL_TYPE_TAG, &pTags);
|
||||
|
|
@ -5173,6 +5176,8 @@ int32_t stbJoinOptRewriteToTagScan(SLogicNode* pJoin, SNode* pNode) {
|
|||
code = stbJoinOptAddFuncToScanNode("_vgid", pScan);
|
||||
}
|
||||
|
||||
tagScanSetExecutionMode(pScan);
|
||||
|
||||
if (code) {
|
||||
nodesDestroyList(pTags);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -615,3 +615,61 @@ int32_t getTimeRangeFromNode(SNode** pPrimaryKeyCond, STimeWindow* pTimeRange, b
|
|||
}
|
||||
|
||||
|
||||
static EDealRes tagScanNodeHasTbnameFunc(SNode* pNode, void* pContext) {
|
||||
if (QUERY_NODE_FUNCTION == nodeType(pNode) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pNode)->funcType ||
|
||||
(QUERY_NODE_COLUMN == nodeType(pNode) && COLUMN_TYPE_TBNAME == ((SColumnNode*)pNode)->colType)) {
|
||||
*(bool*)pContext = true;
|
||||
return DEAL_RES_END;
|
||||
}
|
||||
return DEAL_RES_CONTINUE;
|
||||
}
|
||||
|
||||
static bool tagScanNodeListHasTbname(SNodeList* pCols) {
|
||||
bool hasTbname = false;
|
||||
nodesWalkExprs(pCols, tagScanNodeHasTbnameFunc, &hasTbname);
|
||||
return hasTbname;
|
||||
}
|
||||
|
||||
static bool tagScanNodeHasTbname(SNode* pKeys) {
|
||||
bool hasTbname = false;
|
||||
nodesWalkExpr(pKeys, tagScanNodeHasTbnameFunc, &hasTbname);
|
||||
return hasTbname;
|
||||
}
|
||||
|
||||
|
||||
|
||||
int32_t tagScanSetExecutionMode(SScanLogicNode* pScan) {
|
||||
pScan->onlyMetaCtbIdx = false;
|
||||
|
||||
if (pScan->tableType == TSDB_CHILD_TABLE) {
|
||||
pScan->onlyMetaCtbIdx = false;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (tagScanNodeListHasTbname(pScan->pScanPseudoCols)) {
|
||||
pScan->onlyMetaCtbIdx = false;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (pScan->node.pConditions == NULL) {
|
||||
pScan->onlyMetaCtbIdx = true;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SNode* pCond = nodesCloneNode(pScan->node.pConditions);
|
||||
SNode* pTagCond = NULL;
|
||||
SNode* pTagIndexCond = NULL;
|
||||
filterPartitionCond(&pCond, NULL, &pTagIndexCond, &pTagCond, NULL);
|
||||
if (pTagIndexCond || tagScanNodeHasTbname(pTagCond)) {
|
||||
pScan->onlyMetaCtbIdx = false;
|
||||
} else {
|
||||
pScan->onlyMetaCtbIdx = true;
|
||||
}
|
||||
nodesDestroyNode(pCond);
|
||||
nodesDestroyNode(pTagIndexCond);
|
||||
nodesDestroyNode(pTagCond);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@
|
|||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define CHECK_RSP_INTERVAL 300
|
||||
#define CHECK_RSP_CHECK_INTERVAL 300
|
||||
#define LAUNCH_HTASK_INTERVAL 100
|
||||
#define WAIT_FOR_MINIMAL_INTERVAL 100.00
|
||||
#define MAX_RETRY_LAUNCH_HISTORY_TASK 40
|
||||
|
|
@ -69,6 +69,7 @@ typedef struct {
|
|||
int64_t chkpId;
|
||||
char* dbPrefixPath;
|
||||
} SStreamTaskSnap;
|
||||
|
||||
struct STokenBucket {
|
||||
int32_t numCapacity; // total capacity, available token per second
|
||||
int32_t numOfToken; // total available tokens
|
||||
|
|
@ -148,18 +149,19 @@ int32_t streamQueueGetItemSize(const SStreamQueue* pQueue);
|
|||
|
||||
void streamMetaRemoveDB(void* arg, char* key);
|
||||
|
||||
typedef enum UPLOAD_TYPE {
|
||||
UPLOAD_DISABLE = -1,
|
||||
UPLOAD_S3 = 0,
|
||||
UPLOAD_RSYNC = 1,
|
||||
} UPLOAD_TYPE;
|
||||
typedef enum ECHECKPOINT_BACKUP_TYPE {
|
||||
DATA_UPLOAD_DISABLE = -1,
|
||||
DATA_UPLOAD_S3 = 0,
|
||||
DATA_UPLOAD_RSYNC = 1,
|
||||
} ECHECKPOINT_BACKUP_TYPE;
|
||||
|
||||
UPLOAD_TYPE getUploadType();
|
||||
int uploadCheckpoint(char* id, char* path);
|
||||
int downloadCheckpoint(char* id, char* path);
|
||||
int deleteCheckpoint(char* id);
|
||||
int deleteCheckpointFile(char* id, char* name);
|
||||
int downloadCheckpointByName(char* id, char* fname, char* dstName);
|
||||
ECHECKPOINT_BACKUP_TYPE streamGetCheckpointBackupType();
|
||||
|
||||
int32_t streamTaskBackupCheckpoint(char* id, char* path);
|
||||
int32_t downloadCheckpoint(char* id, char* path);
|
||||
int32_t deleteCheckpoint(char* id);
|
||||
int32_t deleteCheckpointFile(char* id, char* name);
|
||||
int32_t downloadCheckpointByName(char* id, char* fname, char* dstName);
|
||||
|
||||
int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask);
|
||||
int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask);
|
||||
|
|
|
|||
|
|
@ -376,10 +376,10 @@ int32_t rebuildFromRemoteChkp_s3(char* key, char* chkpPath, int64_t chkpId, char
|
|||
return code;
|
||||
}
|
||||
int32_t rebuildFromRemoteChkp(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) {
|
||||
UPLOAD_TYPE type = getUploadType();
|
||||
if (type == UPLOAD_S3) {
|
||||
ECHECKPOINT_BACKUP_TYPE type = streamGetCheckpointBackupType();
|
||||
if (type == DATA_UPLOAD_S3) {
|
||||
return rebuildFromRemoteChkp_s3(key, chkpPath, chkpId, defaultPath);
|
||||
} else if (type == UPLOAD_RSYNC) {
|
||||
} else if (type == DATA_UPLOAD_RSYNC) {
|
||||
return rebuildFromRemoteChkp_rsync(key, chkpPath, chkpId, defaultPath);
|
||||
}
|
||||
return -1;
|
||||
|
|
@ -2111,11 +2111,11 @@ int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64
|
|||
}
|
||||
int32_t taskDbGenChkpUploadData(void* arg, void* mgt, int64_t chkpId, int8_t type, char** path, SArray* list) {
|
||||
STaskDbWrapper* pDb = arg;
|
||||
UPLOAD_TYPE utype = type;
|
||||
ECHECKPOINT_BACKUP_TYPE utype = type;
|
||||
|
||||
if (utype == UPLOAD_RSYNC) {
|
||||
if (utype == DATA_UPLOAD_RSYNC) {
|
||||
return taskDbGenChkpUploadData__rsync(pDb, chkpId, path);
|
||||
} else if (utype == UPLOAD_S3) {
|
||||
} else if (utype == DATA_UPLOAD_S3) {
|
||||
return taskDbGenChkpUploadData__s3(pDb, mgt, chkpId, path, list);
|
||||
}
|
||||
return -1;
|
||||
|
|
@ -2179,6 +2179,7 @@ int32_t copyDataAt(RocksdbCfInst* pSrc, STaskDbWrapper* pDst, int8_t i) {
|
|||
}
|
||||
|
||||
_EXIT:
|
||||
rocksdb_writebatch_destroy(wb);
|
||||
rocksdb_iter_destroy(pIter);
|
||||
rocksdb_readoptions_destroy(pRdOpt);
|
||||
taosMemoryFree(err);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,705 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "cos.h"
|
||||
#include "rsync.h"
|
||||
#include "streamBackendRocksdb.h"
|
||||
#include "streamInt.h"
|
||||
|
||||
#define CHECK_NOT_RSP_DURATION 10 * 1000 // 10 sec
|
||||
|
||||
static void processDownstreamReadyRsp(SStreamTask* pTask);
|
||||
static void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId);
|
||||
static void rspMonitorFn(void* param, void* tmrId);
|
||||
static int32_t streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutputInfo, int64_t startTs);
|
||||
static int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id);
|
||||
static int32_t streamTaskCompleteCheckRsp(STaskCheckInfo* pInfo, bool lock, const char* id);
|
||||
static int32_t streamTaskAddReqInfo(STaskCheckInfo* pInfo, int64_t reqId, int32_t taskId, int32_t vgId, const char* id);
|
||||
static void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p);
|
||||
static void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList);
|
||||
static void handleNotReadyDownstreamTask(SStreamTask* pTask, SArray* pNotReadyList);
|
||||
static int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t status, int64_t rspTs,
|
||||
int64_t reqId, int32_t* pNotReady, const char* id);
|
||||
static void setCheckDownstreamReqInfo(SStreamTaskCheckReq* pReq, int64_t reqId, int32_t dstTaskId, int32_t dstNodeId);
|
||||
static void getCheckRspStatus(STaskCheckInfo* pInfo, int64_t el, int32_t* numOfReady, int32_t* numOfFault,
|
||||
int32_t* numOfNotRsp, SArray* pTimeoutList, SArray* pNotReadyList, const char* id);
|
||||
static SDownstreamStatusInfo* findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId);
|
||||
|
||||
// check status
|
||||
void streamTaskCheckDownstream(SStreamTask* pTask) {
|
||||
SDataRange* pRange = &pTask->dataRange;
|
||||
STimeWindow* pWindow = &pRange->window;
|
||||
const char* idstr = pTask->id.idStr;
|
||||
|
||||
SStreamTaskCheckReq req = {
|
||||
.streamId = pTask->id.streamId,
|
||||
.upstreamTaskId = pTask->id.taskId,
|
||||
.upstreamNodeId = pTask->info.nodeId,
|
||||
.childId = pTask->info.selfChildId,
|
||||
.stage = pTask->pMeta->stage,
|
||||
};
|
||||
|
||||
ASSERT(pTask->status.downstreamReady == 0);
|
||||
|
||||
// serialize streamProcessScanHistoryFinishRsp
|
||||
if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) {
|
||||
streamTaskStartMonitorCheckRsp(pTask);
|
||||
|
||||
STaskDispatcherFixed* pDispatch = &pTask->outputInfo.fixedDispatcher;
|
||||
|
||||
setCheckDownstreamReqInfo(&req, tGenIdPI64(), pDispatch->taskId, pDispatch->nodeId);
|
||||
streamTaskAddReqInfo(&pTask->taskCheckInfo, req.reqId, pDispatch->taskId, pDispatch->nodeId, idstr);
|
||||
|
||||
stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " check single downstream task:0x%x(vgId:%d) ver:%" PRId64 "-%" PRId64
|
||||
" window:%" PRId64 "-%" PRId64 " reqId:0x%" PRIx64,
|
||||
idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, pRange->range.minVer,
|
||||
pRange->range.maxVer, pWindow->skey, pWindow->ekey, req.reqId);
|
||||
|
||||
streamSendCheckMsg(pTask, &req, pTask->outputInfo.fixedDispatcher.nodeId, &pTask->outputInfo.fixedDispatcher.epSet);
|
||||
|
||||
} else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
|
||||
streamTaskStartMonitorCheckRsp(pTask);
|
||||
|
||||
SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos;
|
||||
|
||||
int32_t numOfVgs = taosArrayGetSize(vgInfo);
|
||||
stDebug("s-task:%s check %d downstream tasks, ver:%" PRId64 "-%" PRId64 " window:%" PRId64 "-%" PRId64, idstr,
|
||||
numOfVgs, pRange->range.minVer, pRange->range.maxVer, pWindow->skey, pWindow->ekey);
|
||||
|
||||
for (int32_t i = 0; i < numOfVgs; i++) {
|
||||
SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i);
|
||||
|
||||
setCheckDownstreamReqInfo(&req, tGenIdPI64(), pVgInfo->taskId, pVgInfo->vgId);
|
||||
streamTaskAddReqInfo(&pTask->taskCheckInfo, req.reqId, pVgInfo->taskId, pVgInfo->vgId, idstr);
|
||||
|
||||
stDebug("s-task:%s (vgId:%d) stage:%" PRId64
|
||||
" check downstream task:0x%x (vgId:%d) (shuffle), idx:%d, reqId:0x%" PRIx64,
|
||||
idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, req.reqId);
|
||||
streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet);
|
||||
}
|
||||
} else { // for sink task, set it ready directly.
|
||||
stDebug("s-task:%s (vgId:%d) set downstream ready, since no downstream", idstr, pTask->info.nodeId);
|
||||
streamTaskStopMonitorCheckRsp(&pTask->taskCheckInfo, idstr);
|
||||
processDownstreamReadyRsp(pTask);
|
||||
}
|
||||
}
|
||||
|
||||
int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) {
|
||||
ASSERT(pTask->id.taskId == pRsp->upstreamTaskId);
|
||||
|
||||
int64_t now = taosGetTimestampMs();
|
||||
const char* id = pTask->id.idStr;
|
||||
STaskCheckInfo* pInfo = &pTask->taskCheckInfo;
|
||||
int32_t total = streamTaskGetNumOfDownstream(pTask);
|
||||
int32_t left = -1;
|
||||
|
||||
if (streamTaskShouldStop(pTask)) {
|
||||
stDebug("s-task:%s should stop, do not do check downstream again", id);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (pRsp->status == TASK_DOWNSTREAM_READY) {
|
||||
int32_t code = streamTaskUpdateCheckInfo(pInfo, pRsp->downstreamTaskId, pRsp->status, now, pRsp->reqId, &left, id);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (left == 0) {
|
||||
processDownstreamReadyRsp(pTask); // all downstream tasks are ready, set the complete check downstream flag
|
||||
streamTaskStopMonitorCheckRsp(pInfo, id);
|
||||
} else {
|
||||
stDebug("s-task:%s (vgId:%d) recv check rsp from task:0x%x (vgId:%d) status:%d, total:%d not ready:%d", id,
|
||||
pRsp->upstreamNodeId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->status, total, left);
|
||||
}
|
||||
} else { // not ready, wait for 100ms and retry
|
||||
int32_t code = streamTaskUpdateCheckInfo(pInfo, pRsp->downstreamTaskId, pRsp->status, now, pRsp->reqId, &left, id);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return TSDB_CODE_SUCCESS; // return success in any cases.
|
||||
}
|
||||
|
||||
if (pRsp->status == TASK_UPSTREAM_NEW_STAGE || pRsp->status == TASK_DOWNSTREAM_NOT_LEADER) {
|
||||
if (pRsp->status == TASK_UPSTREAM_NEW_STAGE) {
|
||||
stError("s-task:%s vgId:%d self vnode-transfer/leader-change/restart detected, old stage:%" PRId64
|
||||
", current stage:%" PRId64 ", not check wait for downstream task nodeUpdate, and all tasks restart",
|
||||
id, pRsp->upstreamNodeId, pRsp->oldStage, pTask->pMeta->stage);
|
||||
addIntoNodeUpdateList(pTask, pRsp->upstreamNodeId);
|
||||
} else {
|
||||
stError(
|
||||
"s-task:%s downstream taskId:0x%x (vgId:%d) not leader, self dispatch epset needs to be updated, not check "
|
||||
"downstream again, nodeUpdate needed",
|
||||
id, pRsp->downstreamTaskId, pRsp->downstreamNodeId);
|
||||
addIntoNodeUpdateList(pTask, pRsp->downstreamNodeId);
|
||||
}
|
||||
|
||||
int32_t startTs = pTask->execInfo.checkTs;
|
||||
streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, startTs, now, false);
|
||||
|
||||
// automatically set the related fill-history task to be failed.
|
||||
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
|
||||
STaskId* pId = &pTask->hTaskInfo.id;
|
||||
streamMetaAddTaskLaunchResult(pTask->pMeta, pId->streamId, pId->taskId, startTs, now, false);
|
||||
}
|
||||
} else { // TASK_DOWNSTREAM_NOT_READY, let's retry in 100ms
|
||||
ASSERT(left > 0);
|
||||
stDebug("s-task:%s (vgId:%d) recv check rsp from task:0x%x (vgId:%d) status:%d, total:%d not ready:%d", id,
|
||||
pRsp->upstreamNodeId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->status, total, left);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t streamTaskStartMonitorCheckRsp(SStreamTask* pTask) {
|
||||
STaskCheckInfo* pInfo = &pTask->taskCheckInfo;
|
||||
taosThreadMutexLock(&pInfo->checkInfoLock);
|
||||
|
||||
int32_t code = streamTaskStartCheckDownstream(pInfo, pTask->id.idStr);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
/*SStreamTask* p = */ streamMetaAcquireOneTask(pTask); // add task ref here
|
||||
streamTaskInitTaskCheckInfo(pInfo, &pTask->outputInfo, taosGetTimestampMs());
|
||||
|
||||
int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
|
||||
stDebug("s-task:%s start check rsp monit, ref:%d ", pTask->id.idStr, ref);
|
||||
|
||||
if (pInfo->checkRspTmr == NULL) {
|
||||
pInfo->checkRspTmr = taosTmrStart(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer);
|
||||
} else {
|
||||
taosTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr);
|
||||
}
|
||||
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id) {
|
||||
taosThreadMutexLock(&pInfo->checkInfoLock);
|
||||
streamTaskCompleteCheckRsp(pInfo, false, id);
|
||||
|
||||
pInfo->stopCheckProcess = 1;
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
|
||||
stDebug("s-task:%s set stop check rsp mon", id);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) {
|
||||
ASSERT(pInfo->inCheckProcess == 0);
|
||||
|
||||
pInfo->pList = taosArrayDestroy(pInfo->pList);
|
||||
if (pInfo->checkRspTmr != NULL) {
|
||||
/*bool ret = */ taosTmrStop(pInfo->checkRspTmr);
|
||||
pInfo->checkRspTmr = NULL;
|
||||
}
|
||||
|
||||
taosThreadMutexDestroy(&pInfo->checkInfoLock);
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
void processDownstreamReadyRsp(SStreamTask* pTask) {
|
||||
EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST;
|
||||
streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL);
|
||||
|
||||
int64_t checkTs = pTask->execInfo.checkTs;
|
||||
int64_t readyTs = pTask->execInfo.readyTs;
|
||||
streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, checkTs, readyTs, true);
|
||||
|
||||
if (pTask->status.taskStatus == TASK_STATUS__HALT) {
|
||||
ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask) && (pTask->info.fillHistory == 0));
|
||||
|
||||
// halt it self for count window stream task until the related fill history task completed.
|
||||
stDebug("s-task:%s level:%d initial status is %s from mnode, set it to be halt", pTask->id.idStr,
|
||||
pTask->info.taskLevel, streamTaskGetStatusStr(pTask->status.taskStatus));
|
||||
streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_HALT);
|
||||
}
|
||||
|
||||
// start the related fill-history task, when current task is ready
|
||||
// not invoke in success callback due to the deadlock.
|
||||
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
|
||||
stDebug("s-task:%s try to launch related fill-history task", pTask->id.idStr);
|
||||
streamLaunchFillHistoryTask(pTask);
|
||||
}
|
||||
}
|
||||
|
||||
void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) {
|
||||
int32_t vgId = pTask->pMeta->vgId;
|
||||
|
||||
taosThreadMutexLock(&pTask->lock);
|
||||
int32_t num = taosArrayGetSize(pTask->outputInfo.pNodeEpsetUpdateList);
|
||||
bool existed = false;
|
||||
for (int i = 0; i < num; ++i) {
|
||||
SDownstreamTaskEpset* p = taosArrayGet(pTask->outputInfo.pNodeEpsetUpdateList, i);
|
||||
if (p->nodeId == nodeId) {
|
||||
existed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!existed) {
|
||||
SDownstreamTaskEpset t = {.nodeId = nodeId};
|
||||
taosArrayPush(pTask->outputInfo.pNodeEpsetUpdateList, &t);
|
||||
|
||||
stInfo("s-task:%s vgId:%d downstream nodeId:%d needs to be updated, total needs updated:%d", pTask->id.idStr, vgId,
|
||||
t.nodeId, (num + 1));
|
||||
}
|
||||
|
||||
taosThreadMutexUnlock(&pTask->lock);
|
||||
}
|
||||
|
||||
int32_t streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutputInfo, int64_t startTs) {
|
||||
taosArrayClear(pInfo->pList);
|
||||
|
||||
if (pOutputInfo->type == TASK_OUTPUT__FIXED_DISPATCH) {
|
||||
pInfo->notReadyTasks = 1;
|
||||
} else if (pOutputInfo->type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
|
||||
pInfo->notReadyTasks = taosArrayGetSize(pOutputInfo->shuffleDispatcher.dbInfo.pVgroupInfos);
|
||||
ASSERT(pInfo->notReadyTasks == pOutputInfo->shuffleDispatcher.dbInfo.vgNum);
|
||||
}
|
||||
|
||||
pInfo->startTs = startTs;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SDownstreamStatusInfo* findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId) {
|
||||
for (int32_t j = 0; j < taosArrayGetSize(pInfo->pList); ++j) {
|
||||
SDownstreamStatusInfo* p = taosArrayGet(pInfo->pList, j);
|
||||
if (p->taskId == taskId) {
|
||||
return p;
|
||||
}
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t status, int64_t rspTs, int64_t reqId,
|
||||
int32_t* pNotReady, const char* id) {
|
||||
taosThreadMutexLock(&pInfo->checkInfoLock);
|
||||
|
||||
SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId);
|
||||
if (p != NULL) {
|
||||
if (reqId != p->reqId) {
|
||||
stError("s-task:%s reqId:%" PRIx64 " expected:%" PRIx64 " expired check-rsp recv from downstream task:0x%x, discarded",
|
||||
id, reqId, p->reqId, taskId);
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
// subtract one not-ready-task, since it is ready now
|
||||
if ((p->status != TASK_DOWNSTREAM_READY) && (status == TASK_DOWNSTREAM_READY)) {
|
||||
*pNotReady = atomic_sub_fetch_32(&pInfo->notReadyTasks, 1);
|
||||
} else {
|
||||
*pNotReady = pInfo->notReadyTasks;
|
||||
}
|
||||
|
||||
p->status = status;
|
||||
p->rspTs = rspTs;
|
||||
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
stError("s-task:%s unexpected check rsp msg, invalid downstream task:0x%x, reqId:%" PRIx64 " discarded", id, taskId,
|
||||
reqId);
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id) {
|
||||
if (pInfo->inCheckProcess == 0) {
|
||||
pInfo->inCheckProcess = 1;
|
||||
} else {
|
||||
ASSERT(pInfo->startTs > 0);
|
||||
stError("s-task:%s already in check procedure, checkTs:%" PRId64 ", start monitor check rsp failed", id,
|
||||
pInfo->startTs);
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
stDebug("s-task:%s set the in-check-procedure flag", id);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t streamTaskCompleteCheckRsp(STaskCheckInfo* pInfo, bool lock, const char* id) {
|
||||
if (lock) {
|
||||
taosThreadMutexLock(&pInfo->checkInfoLock);
|
||||
}
|
||||
|
||||
if (!pInfo->inCheckProcess) {
|
||||
// stWarn("s-task:%s already not in-check-procedure", id);
|
||||
}
|
||||
|
||||
int64_t el = (pInfo->startTs != 0) ? (taosGetTimestampMs() - pInfo->startTs) : 0;
|
||||
stDebug("s-task:%s clear the in-check-procedure flag, not in-check-procedure elapsed time:%" PRId64 " ms", id, el);
|
||||
|
||||
pInfo->startTs = 0;
|
||||
pInfo->notReadyTasks = 0;
|
||||
pInfo->inCheckProcess = 0;
|
||||
pInfo->stopCheckProcess = 0;
|
||||
|
||||
pInfo->notReadyRetryCount = 0;
|
||||
pInfo->timeoutRetryCount = 0;
|
||||
|
||||
taosArrayClear(pInfo->pList);
|
||||
|
||||
if (lock) {
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t streamTaskAddReqInfo(STaskCheckInfo* pInfo, int64_t reqId, int32_t taskId, int32_t vgId, const char* id) {
|
||||
SDownstreamStatusInfo info = {.taskId = taskId, .status = -1, .vgId = vgId, .reqId = reqId, .rspTs = 0};
|
||||
taosThreadMutexLock(&pInfo->checkInfoLock);
|
||||
|
||||
SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId);
|
||||
if (p != NULL) {
|
||||
stDebug("s-task:%s check info to task:0x%x already sent", id, taskId);
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
taosArrayPush(pInfo->pList, &info);
|
||||
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) {
|
||||
SStreamTaskCheckReq req = {
|
||||
.streamId = pTask->id.streamId,
|
||||
.upstreamTaskId = pTask->id.taskId,
|
||||
.upstreamNodeId = pTask->info.nodeId,
|
||||
.childId = pTask->info.selfChildId,
|
||||
.stage = pTask->pMeta->stage,
|
||||
};
|
||||
|
||||
STaskOutputInfo* pOutputInfo = &pTask->outputInfo;
|
||||
if (pOutputInfo->type == TASK_OUTPUT__FIXED_DISPATCH) {
|
||||
STaskDispatcherFixed* pDispatch = &pOutputInfo->fixedDispatcher;
|
||||
setCheckDownstreamReqInfo(&req, p->reqId, pDispatch->taskId, pDispatch->taskId);
|
||||
|
||||
stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) reqId:0x%" PRIx64,
|
||||
pTask->id.idStr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, req.reqId);
|
||||
|
||||
streamSendCheckMsg(pTask, &req, pOutputInfo->fixedDispatcher.nodeId, &pOutputInfo->fixedDispatcher.epSet);
|
||||
} else if (pOutputInfo->type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
|
||||
SArray* vgInfo = pOutputInfo->shuffleDispatcher.dbInfo.pVgroupInfos;
|
||||
int32_t numOfVgs = taosArrayGetSize(vgInfo);
|
||||
|
||||
for (int32_t i = 0; i < numOfVgs; i++) {
|
||||
SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i);
|
||||
|
||||
if (p->taskId == pVgInfo->taskId) {
|
||||
setCheckDownstreamReqInfo(&req, p->reqId, pVgInfo->taskId, pVgInfo->vgId);
|
||||
|
||||
stDebug("s-task:%s (vgId:%d) stage:%" PRId64
|
||||
" re-send check downstream task:0x%x(vgId:%d) (shuffle), idx:%d reqId:0x%" PRIx64,
|
||||
pTask->id.idStr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i,
|
||||
p->reqId);
|
||||
streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet);
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ASSERT(0);
|
||||
}
|
||||
}
|
||||
|
||||
void getCheckRspStatus(STaskCheckInfo* pInfo, int64_t el, int32_t* numOfReady, int32_t* numOfFault,
|
||||
int32_t* numOfNotRsp, SArray* pTimeoutList, SArray* pNotReadyList, const char* id) {
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pList); ++i) {
|
||||
SDownstreamStatusInfo* p = taosArrayGet(pInfo->pList, i);
|
||||
if (p->status == TASK_DOWNSTREAM_READY) {
|
||||
(*numOfReady) += 1;
|
||||
} else if (p->status == TASK_UPSTREAM_NEW_STAGE || p->status == TASK_DOWNSTREAM_NOT_LEADER) {
|
||||
stDebug("s-task:%s recv status:NEW_STAGE/NOT_LEADER from downstream, task:0x%x, quit from check downstream", id,
|
||||
p->taskId);
|
||||
(*numOfFault) += 1;
|
||||
} else { // TASK_DOWNSTREAM_NOT_READY
|
||||
if (p->rspTs == 0) { // not response yet
|
||||
ASSERT(p->status == -1);
|
||||
if (el >= CHECK_NOT_RSP_DURATION) { // not receive info for 10 sec.
|
||||
taosArrayPush(pTimeoutList, &p->taskId);
|
||||
} else { // el < CHECK_NOT_RSP_DURATION
|
||||
(*numOfNotRsp) += 1; // do nothing and continue waiting for their rsp
|
||||
}
|
||||
} else {
|
||||
taosArrayPush(pNotReadyList, &p->taskId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void setCheckDownstreamReqInfo(SStreamTaskCheckReq* pReq, int64_t reqId, int32_t dstTaskId, int32_t dstNodeId) {
|
||||
pReq->reqId = reqId;
|
||||
pReq->downstreamTaskId = dstTaskId;
|
||||
pReq->downstreamNodeId = dstNodeId;
|
||||
}
|
||||
|
||||
void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) {
|
||||
STaskCheckInfo* pInfo = &pTask->taskCheckInfo;
|
||||
const char* id = pTask->id.idStr;
|
||||
int32_t vgId = pTask->pMeta->vgId;
|
||||
int32_t numOfTimeout = taosArrayGetSize(pTimeoutList);
|
||||
|
||||
ASSERT(pTask->status.downstreamReady == 0);
|
||||
|
||||
for (int32_t i = 0; i < numOfTimeout; ++i) {
|
||||
int32_t taskId = *(int32_t*)taosArrayGet(pTimeoutList, i);
|
||||
|
||||
SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId);
|
||||
if (p != NULL) {
|
||||
ASSERT(p->status == -1 && p->rspTs == 0);
|
||||
doSendCheckMsg(pTask, p);
|
||||
}
|
||||
}
|
||||
|
||||
pInfo->timeoutRetryCount += 1;
|
||||
|
||||
// timeout more than 100 sec, add into node update list
|
||||
if (pInfo->timeoutRetryCount > 10) {
|
||||
pInfo->timeoutRetryCount = 0;
|
||||
|
||||
for (int32_t i = 0; i < numOfTimeout; ++i) {
|
||||
int32_t taskId = *(int32_t*)taosArrayGet(pTimeoutList, i);
|
||||
SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId);
|
||||
if (p != NULL) {
|
||||
addIntoNodeUpdateList(pTask, p->vgId);
|
||||
stDebug("s-task:%s vgId:%d downstream task:0x%x (vgId:%d) timeout more than 100sec, add into nodeUpate list",
|
||||
id, vgId, p->taskId, p->vgId);
|
||||
}
|
||||
}
|
||||
|
||||
stDebug("s-task:%s vgId:%d %d downstream task(s) all add into nodeUpate list", id, vgId, numOfTimeout);
|
||||
} else {
|
||||
stDebug("s-task:%s vgId:%d %d downstream task(s) timeout, send check msg again, retry:%d start time:%" PRId64, id,
|
||||
vgId, numOfTimeout, pInfo->timeoutRetryCount, pInfo->startTs);
|
||||
}
|
||||
}
|
||||
|
||||
void handleNotReadyDownstreamTask(SStreamTask* pTask, SArray* pNotReadyList) {
|
||||
STaskCheckInfo* pInfo = &pTask->taskCheckInfo;
|
||||
const char* id = pTask->id.idStr;
|
||||
int32_t vgId = pTask->pMeta->vgId;
|
||||
int32_t numOfNotReady = taosArrayGetSize(pNotReadyList);
|
||||
|
||||
ASSERT(pTask->status.downstreamReady == 0);
|
||||
|
||||
// reset the info, and send the check msg to failure downstream again
|
||||
for (int32_t i = 0; i < numOfNotReady; ++i) {
|
||||
int32_t taskId = *(int32_t*)taosArrayGet(pNotReadyList, i);
|
||||
|
||||
SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId);
|
||||
if (p != NULL) {
|
||||
p->rspTs = 0;
|
||||
p->status = -1;
|
||||
doSendCheckMsg(pTask, p);
|
||||
}
|
||||
}
|
||||
|
||||
pInfo->notReadyRetryCount += 1;
|
||||
stDebug("s-task:%s vgId:%d %d downstream task(s) not ready, send check msg again, retry:%d start time:%" PRId64, id,
|
||||
vgId, numOfNotReady, pInfo->notReadyRetryCount, pInfo->startTs);
|
||||
}
|
||||
|
||||
void rspMonitorFn(void* param, void* tmrId) {
|
||||
SStreamTask* pTask = param;
|
||||
SStreamMeta* pMeta = pTask->pMeta;
|
||||
SStreamTaskState* pStat = streamTaskGetStatus(pTask);
|
||||
STaskCheckInfo* pInfo = &pTask->taskCheckInfo;
|
||||
int32_t vgId = pTask->pMeta->vgId;
|
||||
int64_t now = taosGetTimestampMs();
|
||||
int64_t el = now - pInfo->startTs;
|
||||
ETaskStatus state = pStat->state;
|
||||
const char* id = pTask->id.idStr;
|
||||
int32_t numOfReady = 0;
|
||||
int32_t numOfFault = 0;
|
||||
int32_t numOfNotRsp = 0;
|
||||
int32_t numOfNotReady = 0;
|
||||
int32_t numOfTimeout = 0;
|
||||
int32_t total = taosArrayGetSize(pInfo->pList);
|
||||
|
||||
stDebug("s-task:%s start to do check-downstream-rsp check in tmr", id);
|
||||
|
||||
if (state == TASK_STATUS__STOP) {
|
||||
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
|
||||
stDebug("s-task:%s status:%s vgId:%d quit from monitor check-rsp tmr, ref:%d", id, pStat->name, vgId, ref);
|
||||
|
||||
streamTaskCompleteCheckRsp(pInfo, true, id);
|
||||
|
||||
streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, pInfo->startTs, now, false);
|
||||
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
|
||||
STaskId* pHId = &pTask->hTaskInfo.id;
|
||||
streamMetaAddTaskLaunchResult(pTask->pMeta, pHId->streamId, pHId->taskId, pInfo->startTs, now, false);
|
||||
}
|
||||
|
||||
streamMetaReleaseTask(pMeta, pTask);
|
||||
return;
|
||||
}
|
||||
|
||||
if (state == TASK_STATUS__DROPPING || state == TASK_STATUS__READY || state == TASK_STATUS__PAUSE) {
|
||||
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
|
||||
stDebug("s-task:%s status:%s vgId:%d quit from monitor check-rsp tmr, ref:%d", id, pStat->name, vgId, ref);
|
||||
|
||||
streamTaskCompleteCheckRsp(pInfo, true, id);
|
||||
streamMetaReleaseTask(pMeta, pTask);
|
||||
return;
|
||||
}
|
||||
|
||||
taosThreadMutexLock(&pInfo->checkInfoLock);
|
||||
if (pInfo->notReadyTasks == 0) {
|
||||
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
|
||||
stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", id, pStat->name,
|
||||
vgId, ref);
|
||||
|
||||
streamTaskCompleteCheckRsp(pInfo, false, id);
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
streamMetaReleaseTask(pMeta, pTask);
|
||||
return;
|
||||
}
|
||||
|
||||
SArray* pNotReadyList = taosArrayInit(4, sizeof(int64_t));
|
||||
SArray* pTimeoutList = taosArrayInit(4, sizeof(int64_t));
|
||||
|
||||
if (pStat->state == TASK_STATUS__UNINIT) {
|
||||
getCheckRspStatus(pInfo, el, &numOfReady, &numOfFault, &numOfNotRsp, pTimeoutList, pNotReadyList, id);
|
||||
} else { // unexpected status
|
||||
stError("s-task:%s unexpected task status:%s during waiting for check rsp", id, pStat->name);
|
||||
}
|
||||
|
||||
numOfNotReady = (int32_t)taosArrayGetSize(pNotReadyList);
|
||||
numOfTimeout = (int32_t)taosArrayGetSize(pTimeoutList);
|
||||
|
||||
// fault tasks detected, not try anymore
|
||||
ASSERT((numOfReady + numOfFault + numOfNotReady + numOfTimeout + numOfNotRsp) == total);
|
||||
if (numOfFault > 0) {
|
||||
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
|
||||
stDebug(
|
||||
"s-task:%s status:%s vgId:%d all rsp. quit from monitor rsp tmr, since vnode-transfer/leader-change/restart "
|
||||
"detected, notRsp:%d, notReady:%d, fault:%d, timeout:%d, ready:%d ref:%d",
|
||||
id, pStat->name, vgId, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref);
|
||||
|
||||
streamTaskCompleteCheckRsp(pInfo, false, id);
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
streamMetaReleaseTask(pMeta, pTask);
|
||||
|
||||
taosArrayDestroy(pNotReadyList);
|
||||
taosArrayDestroy(pTimeoutList);
|
||||
return;
|
||||
}
|
||||
|
||||
// checking of downstream tasks has been stopped by other threads
|
||||
if (pInfo->stopCheckProcess == 1) {
|
||||
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
|
||||
stDebug(
|
||||
"s-task:%s status:%s vgId:%d stopped by other threads to check downstream process, notRsp:%d, notReady:%d, "
|
||||
"fault:%d, timeout:%d, ready:%d ref:%d",
|
||||
id, pStat->name, vgId, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref);
|
||||
|
||||
streamTaskCompleteCheckRsp(pInfo, false, id);
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
|
||||
// add the not-ready tasks into the final task status result buf, along with related fill-history task if exists.
|
||||
streamMetaAddTaskLaunchResult(pMeta, pTask->id.streamId, pTask->id.taskId, pInfo->startTs, now, false);
|
||||
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
|
||||
STaskId* pHId = &pTask->hTaskInfo.id;
|
||||
streamMetaAddTaskLaunchResult(pMeta, pHId->streamId, pHId->taskId, pInfo->startTs, now, false);
|
||||
}
|
||||
|
||||
streamMetaReleaseTask(pMeta, pTask);
|
||||
|
||||
taosArrayDestroy(pNotReadyList);
|
||||
taosArrayDestroy(pTimeoutList);
|
||||
return;
|
||||
}
|
||||
|
||||
if (numOfNotReady > 0) { // check to make sure not in recheck timer
|
||||
handleNotReadyDownstreamTask(pTask, pNotReadyList);
|
||||
}
|
||||
|
||||
if (numOfTimeout > 0) {
|
||||
handleTimeoutDownstreamTasks(pTask, pTimeoutList);
|
||||
}
|
||||
|
||||
taosTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr);
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
|
||||
stDebug("s-task:%s continue checking rsp in 300ms, total:%d, notRsp:%d, notReady:%d, fault:%d, timeout:%d, ready:%d",
|
||||
id, total, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady);
|
||||
|
||||
taosArrayDestroy(pNotReadyList);
|
||||
taosArrayDestroy(pTimeoutList);
|
||||
}
|
||||
|
||||
int32_t tEncodeStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq) {
|
||||
if (tStartEncode(pEncoder) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pReq->reqId) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->upstreamNodeId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->upstreamTaskId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->downstreamNodeId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->downstreamTaskId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->childId) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pReq->stage) < 0) return -1;
|
||||
tEndEncode(pEncoder);
|
||||
return pEncoder->pos;
|
||||
}
|
||||
|
||||
int32_t tDecodeStreamTaskCheckReq(SDecoder* pDecoder, SStreamTaskCheckReq* pReq) {
|
||||
if (tStartDecode(pDecoder) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pReq->reqId) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->upstreamNodeId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->upstreamTaskId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->downstreamNodeId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->downstreamTaskId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->childId) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pReq->stage) < 0) return -1;
|
||||
tEndDecode(pDecoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tEncodeStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp* pRsp) {
|
||||
if (tStartEncode(pEncoder) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pRsp->reqId) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->upstreamNodeId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->upstreamTaskId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->downstreamNodeId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->downstreamTaskId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->childId) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pRsp->oldStage) < 0) return -1;
|
||||
if (tEncodeI8(pEncoder, pRsp->status) < 0) return -1;
|
||||
tEndEncode(pEncoder);
|
||||
return pEncoder->pos;
|
||||
}
|
||||
|
||||
int32_t tDecodeStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp) {
|
||||
if (tStartDecode(pDecoder) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pRsp->reqId) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pRsp->streamId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->upstreamNodeId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->upstreamTaskId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->downstreamNodeId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->downstreamTaskId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->childId) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pRsp->oldStage) < 0) return -1;
|
||||
if (tDecodeI8(pDecoder, &pRsp->status) < 0) return -1;
|
||||
tEndDecode(pDecoder);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -19,7 +19,7 @@
|
|||
#include "streamInt.h"
|
||||
|
||||
typedef struct {
|
||||
UPLOAD_TYPE type;
|
||||
ECHECKPOINT_BACKUP_TYPE type;
|
||||
char* taskId;
|
||||
int64_t chkpId;
|
||||
|
||||
|
|
@ -94,6 +94,24 @@ int32_t tDecodeStreamCheckpointReadyMsg(SDecoder* pDecoder, SStreamCheckpointRea
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t tEncodeStreamTaskCheckpointReq(SEncoder* pEncoder, const SStreamTaskCheckpointReq* pReq) {
|
||||
if (tStartEncode(pEncoder) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->nodeId) < 0) return -1;
|
||||
tEndEncode(pEncoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tDecodeStreamTaskCheckpointReq(SDecoder* pDecoder, SStreamTaskCheckpointReq* pReq) {
|
||||
if (tStartDecode(pDecoder) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->nodeId) < 0) return -1;
|
||||
tEndDecode(pDecoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t streamAlignCheckpoint(SStreamTask* pTask) {
|
||||
int32_t num = taosArrayGetSize(pTask->upstreamInfo.pList);
|
||||
int64_t old = atomic_val_compare_exchange_32(&pTask->chkInfo.downstreamAlignNum, 0, num);
|
||||
|
|
@ -398,7 +416,7 @@ int32_t getChkpMeta(char* id, char* path, SArray* list) {
|
|||
return code;
|
||||
}
|
||||
|
||||
int32_t doUploadChkp(void* param) {
|
||||
int32_t uploadCheckpointData(void* param) {
|
||||
SAsyncUploadArg* arg = param;
|
||||
char* path = NULL;
|
||||
int32_t code = 0;
|
||||
|
|
@ -408,13 +426,13 @@ int32_t doUploadChkp(void* param) {
|
|||
(int8_t)(arg->type), &path, toDelFiles)) != 0) {
|
||||
stError("s-task:%s failed to gen upload checkpoint:%" PRId64 "", arg->pTask->id.idStr, arg->chkpId);
|
||||
}
|
||||
if (arg->type == UPLOAD_S3) {
|
||||
if (arg->type == DATA_UPLOAD_S3) {
|
||||
if (code == 0 && (code = getChkpMeta(arg->taskId, path, toDelFiles)) != 0) {
|
||||
stError("s-task:%s failed to get checkpoint:%" PRId64 " meta", arg->pTask->id.idStr, arg->chkpId);
|
||||
}
|
||||
}
|
||||
|
||||
if (code == 0 && (code = uploadCheckpoint(arg->taskId, path)) != 0) {
|
||||
if (code == 0 && (code = streamTaskBackupCheckpoint(arg->taskId, path)) != 0) {
|
||||
stError("s-task:%s failed to upload checkpoint:%" PRId64, arg->pTask->id.idStr, arg->chkpId);
|
||||
}
|
||||
|
||||
|
|
@ -441,8 +459,8 @@ int32_t doUploadChkp(void* param) {
|
|||
|
||||
int32_t streamTaskUploadChkp(SStreamTask* pTask, int64_t chkpId, char* taskId) {
|
||||
// async upload
|
||||
UPLOAD_TYPE type = getUploadType();
|
||||
if (type == UPLOAD_DISABLE) {
|
||||
ECHECKPOINT_BACKUP_TYPE type = streamGetCheckpointBackupType();
|
||||
if (type == DATA_UPLOAD_DISABLE) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -456,7 +474,7 @@ int32_t streamTaskUploadChkp(SStreamTask* pTask, int64_t chkpId, char* taskId) {
|
|||
arg->chkpId = chkpId;
|
||||
arg->pTask = pTask;
|
||||
|
||||
return streamMetaAsyncExec(pTask->pMeta, doUploadChkp, arg, NULL);
|
||||
return streamMetaAsyncExec(pTask->pMeta, uploadCheckpointData, arg, NULL);
|
||||
}
|
||||
|
||||
int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) {
|
||||
|
|
@ -540,7 +558,7 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) {
|
|||
return code;
|
||||
}
|
||||
|
||||
static int uploadCheckpointToS3(char* id, char* path) {
|
||||
static int32_t uploadCheckpointToS3(char* id, char* path) {
|
||||
TdDirPtr pDir = taosOpenDir(path);
|
||||
if (pDir == NULL) return -1;
|
||||
|
||||
|
|
@ -572,8 +590,8 @@ static int uploadCheckpointToS3(char* id, char* path) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int downloadCheckpointByNameS3(char* id, char* fname, char* dstName) {
|
||||
int code = 0;
|
||||
static int32_t downloadCheckpointByNameS3(char* id, char* fname, char* dstName) {
|
||||
int32_t code = 0;
|
||||
char* buf = taosMemoryCalloc(1, strlen(id) + strlen(dstName) + 4);
|
||||
sprintf(buf, "%s/%s", id, fname);
|
||||
if (s3GetObjectToFile(buf, dstName) != 0) {
|
||||
|
|
@ -583,19 +601,19 @@ static int downloadCheckpointByNameS3(char* id, char* fname, char* dstName) {
|
|||
return code;
|
||||
}
|
||||
|
||||
UPLOAD_TYPE getUploadType() {
|
||||
ECHECKPOINT_BACKUP_TYPE streamGetCheckpointBackupType() {
|
||||
if (strlen(tsSnodeAddress) != 0) {
|
||||
return UPLOAD_RSYNC;
|
||||
return DATA_UPLOAD_RSYNC;
|
||||
} else if (tsS3StreamEnabled) {
|
||||
return UPLOAD_S3;
|
||||
return DATA_UPLOAD_S3;
|
||||
} else {
|
||||
return UPLOAD_DISABLE;
|
||||
return DATA_UPLOAD_DISABLE;
|
||||
}
|
||||
}
|
||||
|
||||
int uploadCheckpoint(char* id, char* path) {
|
||||
int32_t streamTaskBackupCheckpoint(char* id, char* path) {
|
||||
if (id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX) {
|
||||
stError("uploadCheckpoint parameters invalid");
|
||||
stError("streamTaskBackupCheckpoint parameters invalid");
|
||||
return -1;
|
||||
}
|
||||
if (strlen(tsSnodeAddress) != 0) {
|
||||
|
|
@ -607,7 +625,7 @@ int uploadCheckpoint(char* id, char* path) {
|
|||
}
|
||||
|
||||
// fileName: CURRENT
|
||||
int downloadCheckpointByName(char* id, char* fname, char* dstName) {
|
||||
int32_t downloadCheckpointByName(char* id, char* fname, char* dstName) {
|
||||
if (id == NULL || fname == NULL || strlen(id) == 0 || strlen(fname) == 0 || strlen(fname) >= PATH_MAX) {
|
||||
stError("uploadCheckpointByName parameters invalid");
|
||||
return -1;
|
||||
|
|
@ -620,7 +638,7 @@ int downloadCheckpointByName(char* id, char* fname, char* dstName) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int downloadCheckpoint(char* id, char* path) {
|
||||
int32_t downloadCheckpoint(char* id, char* path) {
|
||||
if (id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX) {
|
||||
stError("downloadCheckpoint parameters invalid");
|
||||
return -1;
|
||||
|
|
@ -633,7 +651,7 @@ int downloadCheckpoint(char* id, char* path) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int deleteCheckpoint(char* id) {
|
||||
int32_t deleteCheckpoint(char* id) {
|
||||
if (id == NULL || strlen(id) == 0) {
|
||||
stError("deleteCheckpoint parameters invalid");
|
||||
return -1;
|
||||
|
|
@ -646,7 +664,7 @@ int deleteCheckpoint(char* id) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int deleteCheckpointFile(char* id, char* name) {
|
||||
int32_t deleteCheckpointFile(char* id, char* name) {
|
||||
char object[128] = {0};
|
||||
snprintf(object, sizeof(object), "%s/%s", id, name);
|
||||
char* tmp = object;
|
||||
|
|
|
|||
|
|
@ -1073,9 +1073,9 @@ static void addUpdateNodeIntoHbMsg(SStreamTask* pTask, SStreamHbMsg* pMsg) {
|
|||
|
||||
taosThreadMutexLock(&pTask->lock);
|
||||
|
||||
int32_t num = taosArrayGetSize(pTask->outputInfo.pDownstreamUpdateList);
|
||||
int32_t num = taosArrayGetSize(pTask->outputInfo.pNodeEpsetUpdateList);
|
||||
for (int j = 0; j < num; ++j) {
|
||||
SDownstreamTaskEpset* pTaskEpset = taosArrayGet(pTask->outputInfo.pDownstreamUpdateList, j);
|
||||
SDownstreamTaskEpset* pTaskEpset = taosArrayGet(pTask->outputInfo.pNodeEpsetUpdateList, j);
|
||||
|
||||
bool exist = existInHbMsg(pMsg, pTaskEpset);
|
||||
if (!exist) {
|
||||
|
|
@ -1085,7 +1085,7 @@ static void addUpdateNodeIntoHbMsg(SStreamTask* pTask, SStreamHbMsg* pMsg) {
|
|||
}
|
||||
}
|
||||
|
||||
taosArrayClear(pTask->outputInfo.pDownstreamUpdateList);
|
||||
taosArrayClear(pTask->outputInfo.pNodeEpsetUpdateList);
|
||||
taosThreadMutexUnlock(&pTask->lock);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@
|
|||
|
||||
#define MAX_STREAM_EXEC_BATCH_NUM 32
|
||||
#define MAX_SMOOTH_BURST_RATIO 5 // 5 sec
|
||||
#define WAIT_FOR_DURATION 10
|
||||
|
||||
// todo refactor:
|
||||
// read data from input queue
|
||||
|
|
|
|||
|
|
@ -29,19 +29,15 @@ typedef struct SLaunchHTaskInfo {
|
|||
STaskId hTaskId;
|
||||
} SLaunchHTaskInfo;
|
||||
|
||||
typedef struct STaskRecheckInfo {
|
||||
SStreamTask* pTask;
|
||||
SStreamTaskCheckReq req;
|
||||
void* checkTimer;
|
||||
} STaskRecheckInfo;
|
||||
|
||||
static int32_t streamSetParamForScanHistory(SStreamTask* pTask);
|
||||
static void streamTaskSetRangeStreamCalc(SStreamTask* pTask);
|
||||
static int32_t initScanHistoryReq(SStreamTask* pTask, SStreamScanHistoryReq* pReq, int8_t igUntreated);
|
||||
static SLaunchHTaskInfo* createHTaskLaunchInfo(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, int64_t hStreamId,
|
||||
static SLaunchHTaskInfo* createHTaskLaunchInfo(SStreamMeta* pMeta, STaskId* pTaskId, int64_t hStreamId,
|
||||
int32_t hTaskId);
|
||||
static void tryLaunchHistoryTask(void* param, void* tmrId);
|
||||
static void doProcessDownstreamReadyRsp(SStreamTask* pTask);
|
||||
static void doExecScanhistoryInFuture(void* param, void* tmrId);
|
||||
static int32_t doStartScanHistoryTask(SStreamTask* pTask);
|
||||
static int32_t streamTaskStartScanHistory(SStreamTask* pTask);
|
||||
|
||||
int32_t streamTaskSetReady(SStreamTask* pTask) {
|
||||
int32_t numOfDowns = streamTaskGetNumOfDownstream(pTask);
|
||||
|
|
@ -83,15 +79,16 @@ int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void doExecScanhistoryInFuture(void* param, void* tmrId) {
|
||||
void doExecScanhistoryInFuture(void* param, void* tmrId) {
|
||||
SStreamTask* pTask = param;
|
||||
pTask->schedHistoryInfo.numOfTicks -= 1;
|
||||
|
||||
SStreamTaskState* p = streamTaskGetStatus(pTask);
|
||||
if (p->state == TASK_STATUS__DROPPING || p->state == TASK_STATUS__STOP) {
|
||||
streamMetaReleaseTask(pTask->pMeta, pTask);
|
||||
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
|
||||
stDebug("s-task:%s status:%s not start scan-history again, ref:%d", pTask->id.idStr, p->name, ref);
|
||||
|
||||
streamMetaReleaseTask(pTask->pMeta, pTask);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -139,7 +136,7 @@ int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration)
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t doStartScanHistoryTask(SStreamTask* pTask) {
|
||||
int32_t doStartScanHistoryTask(SStreamTask* pTask) {
|
||||
SVersionRange* pRange = &pTask->dataRange.range;
|
||||
if (pTask->info.fillHistory) {
|
||||
streamSetParamForScanHistory(pTask);
|
||||
|
|
@ -169,67 +166,6 @@ int32_t streamTaskStartScanHistory(SStreamTask* pTask) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
// check status
|
||||
void streamTaskCheckDownstream(SStreamTask* pTask) {
|
||||
SDataRange* pRange = &pTask->dataRange;
|
||||
STimeWindow* pWindow = &pRange->window;
|
||||
|
||||
SStreamTaskCheckReq req = {
|
||||
.streamId = pTask->id.streamId,
|
||||
.upstreamTaskId = pTask->id.taskId,
|
||||
.upstreamNodeId = pTask->info.nodeId,
|
||||
.childId = pTask->info.selfChildId,
|
||||
.stage = pTask->pMeta->stage,
|
||||
};
|
||||
|
||||
ASSERT(pTask->status.downstreamReady == 0);
|
||||
|
||||
// serialize streamProcessScanHistoryFinishRsp
|
||||
if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) {
|
||||
streamTaskStartMonitorCheckRsp(pTask);
|
||||
|
||||
req.reqId = tGenIdPI64();
|
||||
req.downstreamNodeId = pTask->outputInfo.fixedDispatcher.nodeId;
|
||||
req.downstreamTaskId = pTask->outputInfo.fixedDispatcher.taskId;
|
||||
|
||||
streamTaskAddReqInfo(&pTask->taskCheckInfo, req.reqId, req.downstreamTaskId, pTask->id.idStr);
|
||||
|
||||
stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " check single downstream task:0x%x(vgId:%d) ver:%" PRId64 "-%" PRId64
|
||||
" window:%" PRId64 "-%" PRId64 " reqId:0x%" PRIx64,
|
||||
pTask->id.idStr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId,
|
||||
pRange->range.minVer, pRange->range.maxVer, pWindow->skey, pWindow->ekey, req.reqId);
|
||||
|
||||
streamSendCheckMsg(pTask, &req, pTask->outputInfo.fixedDispatcher.nodeId, &pTask->outputInfo.fixedDispatcher.epSet);
|
||||
|
||||
} else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
|
||||
streamTaskStartMonitorCheckRsp(pTask);
|
||||
|
||||
SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos;
|
||||
|
||||
int32_t numOfVgs = taosArrayGetSize(vgInfo);
|
||||
stDebug("s-task:%s check %d downstream tasks, ver:%" PRId64 "-%" PRId64 " window:%" PRId64 "-%" PRId64,
|
||||
pTask->id.idStr, numOfVgs, pRange->range.minVer, pRange->range.maxVer, pWindow->skey, pWindow->ekey);
|
||||
|
||||
for (int32_t i = 0; i < numOfVgs; i++) {
|
||||
SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i);
|
||||
req.reqId = tGenIdPI64();
|
||||
req.downstreamNodeId = pVgInfo->vgId;
|
||||
req.downstreamTaskId = pVgInfo->taskId;
|
||||
|
||||
streamTaskAddReqInfo(&pTask->taskCheckInfo, req.reqId, req.downstreamTaskId, pTask->id.idStr);
|
||||
|
||||
stDebug("s-task:%s (vgId:%d) stage:%" PRId64
|
||||
" check downstream task:0x%x (vgId:%d) (shuffle), idx:%d, reqId:0x%" PRIx64,
|
||||
pTask->id.idStr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, req.reqId);
|
||||
streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet);
|
||||
}
|
||||
} else { // for sink task, set it ready directly.
|
||||
stDebug("s-task:%s (vgId:%d) set downstream ready, since no downstream", pTask->id.idStr, pTask->info.nodeId);
|
||||
streamTaskStopMonitorCheckRsp(&pTask->taskCheckInfo, pTask->id.idStr);
|
||||
doProcessDownstreamReadyRsp(pTask);
|
||||
}
|
||||
}
|
||||
|
||||
int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage,
|
||||
int64_t* oldStage) {
|
||||
SStreamChildEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId);
|
||||
|
|
@ -331,122 +267,6 @@ int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
void doProcessDownstreamReadyRsp(SStreamTask* pTask) {
|
||||
EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST;
|
||||
streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL);
|
||||
|
||||
int64_t checkTs = pTask->execInfo.checkTs;
|
||||
int64_t readyTs = pTask->execInfo.readyTs;
|
||||
streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, checkTs, readyTs, true);
|
||||
|
||||
if (pTask->status.taskStatus == TASK_STATUS__HALT) {
|
||||
ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask) && (pTask->info.fillHistory == 0));
|
||||
|
||||
// halt it self for count window stream task until the related fill history task completed.
|
||||
stDebug("s-task:%s level:%d initial status is %s from mnode, set it to be halt", pTask->id.idStr,
|
||||
pTask->info.taskLevel, streamTaskGetStatusStr(pTask->status.taskStatus));
|
||||
streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_HALT);
|
||||
}
|
||||
|
||||
// start the related fill-history task, when current task is ready
|
||||
// not invoke in success callback due to the deadlock.
|
||||
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
|
||||
stDebug("s-task:%s try to launch related fill-history task", pTask->id.idStr);
|
||||
streamLaunchFillHistoryTask(pTask);
|
||||
}
|
||||
}
|
||||
|
||||
static void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) {
|
||||
int32_t vgId = pTask->pMeta->vgId;
|
||||
|
||||
taosThreadMutexLock(&pTask->lock);
|
||||
int32_t num = taosArrayGetSize(pTask->outputInfo.pDownstreamUpdateList);
|
||||
bool existed = false;
|
||||
for (int i = 0; i < num; ++i) {
|
||||
SDownstreamTaskEpset* p = taosArrayGet(pTask->outputInfo.pDownstreamUpdateList, i);
|
||||
if (p->nodeId == nodeId) {
|
||||
existed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!existed) {
|
||||
SDownstreamTaskEpset t = {.nodeId = nodeId};
|
||||
taosArrayPush(pTask->outputInfo.pDownstreamUpdateList, &t);
|
||||
|
||||
stInfo("s-task:%s vgId:%d downstream nodeId:%d needs to be updated, total needs updated:%d", pTask->id.idStr, vgId,
|
||||
t.nodeId, (int32_t)taosArrayGetSize(pTask->outputInfo.pDownstreamUpdateList));
|
||||
}
|
||||
|
||||
taosThreadMutexUnlock(&pTask->lock);
|
||||
}
|
||||
|
||||
int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) {
|
||||
ASSERT(pTask->id.taskId == pRsp->upstreamTaskId);
|
||||
|
||||
int64_t now = taosGetTimestampMs();
|
||||
const char* id = pTask->id.idStr;
|
||||
STaskCheckInfo* pInfo = &pTask->taskCheckInfo;
|
||||
int32_t total = streamTaskGetNumOfDownstream(pTask);
|
||||
int32_t left = -1;
|
||||
|
||||
if (streamTaskShouldStop(pTask)) {
|
||||
stDebug("s-task:%s should stop, do not do check downstream again", id);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (pRsp->status == TASK_DOWNSTREAM_READY) {
|
||||
int32_t code = streamTaskUpdateCheckInfo(pInfo, pRsp->downstreamTaskId, pRsp->status, now, pRsp->reqId, &left, id);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (left == 0) {
|
||||
doProcessDownstreamReadyRsp(pTask); // all downstream tasks are ready, set the complete check downstream flag
|
||||
streamTaskStopMonitorCheckRsp(pInfo, id);
|
||||
} else {
|
||||
stDebug("s-task:%s (vgId:%d) recv check rsp from task:0x%x (vgId:%d) status:%d, total:%d not ready:%d", id,
|
||||
pRsp->upstreamNodeId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->status, total, left);
|
||||
}
|
||||
} else { // not ready, wait for 100ms and retry
|
||||
int32_t code = streamTaskUpdateCheckInfo(pInfo, pRsp->downstreamTaskId, pRsp->status, now, pRsp->reqId, &left, id);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return TSDB_CODE_SUCCESS; // return success in any cases.
|
||||
}
|
||||
|
||||
if (pRsp->status == TASK_UPSTREAM_NEW_STAGE || pRsp->status == TASK_DOWNSTREAM_NOT_LEADER) {
|
||||
if (pRsp->status == TASK_UPSTREAM_NEW_STAGE) {
|
||||
stError("s-task:%s vgId:%d self vnode-transfer/leader-change/restart detected, old stage:%" PRId64
|
||||
", current stage:%" PRId64 ", not check wait for downstream task nodeUpdate, and all tasks restart",
|
||||
id, pRsp->upstreamNodeId, pRsp->oldStage, pTask->pMeta->stage);
|
||||
addIntoNodeUpdateList(pTask, pRsp->upstreamNodeId);
|
||||
} else {
|
||||
stError(
|
||||
"s-task:%s downstream taskId:0x%x (vgId:%d) not leader, self dispatch epset needs to be updated, not check "
|
||||
"downstream again, nodeUpdate needed",
|
||||
id, pRsp->downstreamTaskId, pRsp->downstreamNodeId);
|
||||
addIntoNodeUpdateList(pTask, pRsp->downstreamNodeId);
|
||||
}
|
||||
|
||||
int32_t startTs = pTask->execInfo.checkTs;
|
||||
streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, startTs, now, false);
|
||||
|
||||
// automatically set the related fill-history task to be failed.
|
||||
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
|
||||
STaskId* pId = &pTask->hTaskInfo.id;
|
||||
streamMetaAddTaskLaunchResult(pTask->pMeta, pId->streamId, pId->taskId, startTs, now, false);
|
||||
}
|
||||
|
||||
} else { // TASK_DOWNSTREAM_NOT_READY, let's retry in 100ms
|
||||
ASSERT(left > 0);
|
||||
stDebug("s-task:%s (vgId:%d) recv check rsp from task:0x%x (vgId:%d) status:%d, total:%d not ready:%d", id,
|
||||
pRsp->upstreamNodeId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->status, total, left);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t streamSendCheckRsp(const SStreamMeta* pMeta, const SStreamTaskCheckReq* pReq, SStreamTaskCheckRsp* pRsp,
|
||||
SRpcHandleInfo* pRpcInfo, int32_t taskId) {
|
||||
SEncoder encoder;
|
||||
|
|
@ -663,16 +483,15 @@ static void tryLaunchHistoryTask(void* param, void* tmrId) {
|
|||
taosMemoryFree(pInfo);
|
||||
}
|
||||
|
||||
SLaunchHTaskInfo* createHTaskLaunchInfo(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, int64_t hStreamId,
|
||||
int32_t hTaskId) {
|
||||
SLaunchHTaskInfo* createHTaskLaunchInfo(SStreamMeta* pMeta, STaskId* pTaskId, int64_t hStreamId, int32_t hTaskId) {
|
||||
SLaunchHTaskInfo* pInfo = taosMemoryCalloc(1, sizeof(SLaunchHTaskInfo));
|
||||
if (pInfo == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
pInfo->id.streamId = streamId;
|
||||
pInfo->id.taskId = taskId;
|
||||
pInfo->id.streamId = pTaskId->streamId;
|
||||
pInfo->id.taskId = pTaskId->taskId;
|
||||
|
||||
pInfo->hTaskId.streamId = hStreamId;
|
||||
pInfo->hTaskId.taskId = hTaskId;
|
||||
|
|
@ -691,7 +510,8 @@ static int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) {
|
|||
|
||||
stWarn("s-task:%s vgId:%d failed to launch history task:0x%x, since not built yet", idStr, pMeta->vgId, hTaskId);
|
||||
|
||||
SLaunchHTaskInfo* pInfo = createHTaskLaunchInfo(pMeta, pTask->id.streamId, pTask->id.taskId, hStreamId, hTaskId);
|
||||
STaskId id = streamTaskGetTaskId(pTask);
|
||||
SLaunchHTaskInfo* pInfo = createHTaskLaunchInfo(pMeta, &id, hStreamId, hTaskId);
|
||||
if (pInfo == NULL) {
|
||||
stError("s-task:%s failed to launch related fill-history task, since Out Of Memory", idStr);
|
||||
streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false);
|
||||
|
|
@ -802,82 +622,6 @@ bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t nextProcessVe
|
|||
}
|
||||
}
|
||||
|
||||
int32_t tEncodeStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq) {
|
||||
if (tStartEncode(pEncoder) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pReq->reqId) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->upstreamNodeId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->upstreamTaskId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->downstreamNodeId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->downstreamTaskId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->childId) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pReq->stage) < 0) return -1;
|
||||
tEndEncode(pEncoder);
|
||||
return pEncoder->pos;
|
||||
}
|
||||
|
||||
int32_t tDecodeStreamTaskCheckReq(SDecoder* pDecoder, SStreamTaskCheckReq* pReq) {
|
||||
if (tStartDecode(pDecoder) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pReq->reqId) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->upstreamNodeId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->upstreamTaskId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->downstreamNodeId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->downstreamTaskId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->childId) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pReq->stage) < 0) return -1;
|
||||
tEndDecode(pDecoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tEncodeStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp* pRsp) {
|
||||
if (tStartEncode(pEncoder) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pRsp->reqId) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->upstreamNodeId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->upstreamTaskId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->downstreamNodeId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->downstreamTaskId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pRsp->childId) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pRsp->oldStage) < 0) return -1;
|
||||
if (tEncodeI8(pEncoder, pRsp->status) < 0) return -1;
|
||||
tEndEncode(pEncoder);
|
||||
return pEncoder->pos;
|
||||
}
|
||||
|
||||
int32_t tDecodeStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp) {
|
||||
if (tStartDecode(pDecoder) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pRsp->reqId) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pRsp->streamId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->upstreamNodeId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->upstreamTaskId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->downstreamNodeId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->downstreamTaskId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pRsp->childId) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pRsp->oldStage) < 0) return -1;
|
||||
if (tDecodeI8(pDecoder, &pRsp->status) < 0) return -1;
|
||||
tEndDecode(pDecoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tEncodeStreamTaskCheckpointReq(SEncoder* pEncoder, const SStreamTaskCheckpointReq* pReq) {
|
||||
if (tStartEncode(pEncoder) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->nodeId) < 0) return -1;
|
||||
tEndEncode(pEncoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tDecodeStreamTaskCheckpointReq(SDecoder* pDecoder, SStreamTaskCheckpointReq* pReq) {
|
||||
if (tStartDecode(pDecoder) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->nodeId) < 0) return -1;
|
||||
tEndDecode(pDecoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void streamTaskSetRangeStreamCalc(SStreamTask* pTask) {
|
||||
SDataRange* pRange = &pTask->dataRange;
|
||||
|
||||
|
|
@ -1094,13 +1094,10 @@ _end:
|
|||
|
||||
int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]) {
|
||||
#ifdef USE_ROCKSDB
|
||||
if (tSimpleHashGetSize(pState->parNameMap) > MAX_TABLE_NAME_NUM) {
|
||||
if (tSimpleHashGet(pState->parNameMap, &groupId, sizeof(int64_t)) == NULL) {
|
||||
streamStatePutParName_rocksdb(pState, groupId, tbname);
|
||||
}
|
||||
return TSDB_CODE_SUCCESS;
|
||||
if (tSimpleHashGet(pState->parNameMap, &groupId, sizeof(int64_t)) == NULL) {
|
||||
tSimpleHashPut(pState->parNameMap, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN);
|
||||
streamStatePutParName_rocksdb(pState, groupId, tbname);
|
||||
}
|
||||
tSimpleHashPut(pState->parNameMap, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
#else
|
||||
return tdbTbUpsert(pState->pTdbState->pParNameDb, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN,
|
||||
|
|
@ -1112,10 +1109,11 @@ int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal
|
|||
#ifdef USE_ROCKSDB
|
||||
void* pStr = tSimpleHashGet(pState->parNameMap, &groupId, sizeof(int64_t));
|
||||
if (!pStr) {
|
||||
if (tSimpleHashGetSize(pState->parNameMap) > MAX_TABLE_NAME_NUM) {
|
||||
return streamStateGetParName_rocksdb(pState, groupId, pVal);
|
||||
int32_t code = streamStateGetParName_rocksdb(pState, groupId, pVal);
|
||||
if (code == TSDB_CODE_SUCCESS) {
|
||||
tSimpleHashPut(pState->parNameMap, &groupId, sizeof(int64_t), *pVal, TSDB_TABLE_NAME_LEN);
|
||||
}
|
||||
return TSDB_CODE_FAILED;
|
||||
return code;
|
||||
}
|
||||
*pVal = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN);
|
||||
memcpy(*pVal, pStr, TSDB_TABLE_NAME_LEN);
|
||||
|
|
|
|||
|
|
@ -21,8 +21,6 @@
|
|||
#include "ttimer.h"
|
||||
#include "wal.h"
|
||||
|
||||
#define CHECK_NOT_RSP_DURATION 10*1000 // 10 sec
|
||||
|
||||
static void streamTaskDestroyUpstreamInfo(SUpstreamInfo* pUpstreamInfo);
|
||||
static void streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool* pUpdated);
|
||||
static void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool* pUpdate);
|
||||
|
|
@ -36,15 +34,20 @@ static int32_t addToTaskset(SArray* pArray, SStreamTask* pTask) {
|
|||
|
||||
static int32_t doUpdateTaskEpset(SStreamTask* pTask, int32_t nodeId, SEpSet* pEpSet, bool* pUpdated) {
|
||||
char buf[512] = {0};
|
||||
|
||||
if (pTask->info.nodeId == nodeId) { // execution task should be moved away
|
||||
if (!(*pUpdated)) {
|
||||
*pUpdated = isEpsetEqual(&pTask->info.epSet, pEpSet);
|
||||
}
|
||||
|
||||
epsetAssign(&pTask->info.epSet, pEpSet);
|
||||
bool isEqual = isEpsetEqual(&pTask->info.epSet, pEpSet);
|
||||
epsetToStr(pEpSet, buf, tListLen(buf));
|
||||
stDebug("s-task:0x%x (vgId:%d) self node epset is updated %s", pTask->id.taskId, nodeId, buf);
|
||||
|
||||
if (!isEqual) {
|
||||
(*pUpdated) = true;
|
||||
char tmp[512] = {0};
|
||||
epsetToStr(&pTask->info.epSet, tmp, tListLen(tmp));
|
||||
|
||||
epsetAssign(&pTask->info.epSet, pEpSet);
|
||||
stDebug("s-task:0x%x (vgId:%d) self node epset is updated %s, old:%s", pTask->id.taskId, nodeId, buf, tmp);
|
||||
} else {
|
||||
stDebug("s-task:0x%x (vgId:%d) not updated task epset, since epset identical, %s", pTask->id.taskId, nodeId, buf);
|
||||
}
|
||||
}
|
||||
|
||||
// check for the dispatch info and the upstream task info
|
||||
|
|
@ -256,8 +259,8 @@ int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
|
|||
if (tDecodeI32(pDecoder, &taskId)) return -1;
|
||||
pTask->streamTaskId.taskId = taskId;
|
||||
|
||||
if (tDecodeU64(pDecoder, &pTask->dataRange.range.minVer)) return -1;
|
||||
if (tDecodeU64(pDecoder, &pTask->dataRange.range.maxVer)) return -1;
|
||||
if (tDecodeU64(pDecoder, (uint64_t*)&pTask->dataRange.range.minVer)) return -1;
|
||||
if (tDecodeU64(pDecoder, (uint64_t*)&pTask->dataRange.range.maxVer)) return -1;
|
||||
if (tDecodeI64(pDecoder, &pTask->dataRange.window.skey)) return -1;
|
||||
if (tDecodeI64(pDecoder, &pTask->dataRange.window.ekey)) return -1;
|
||||
|
||||
|
|
@ -437,7 +440,7 @@ void tFreeStreamTask(SStreamTask* pTask) {
|
|||
taosArrayDestroy(pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos);
|
||||
}
|
||||
|
||||
streamTaskCleanCheckInfo(&pTask->taskCheckInfo);
|
||||
streamTaskCleanupCheckInfo(&pTask->taskCheckInfo);
|
||||
|
||||
if (pTask->pState) {
|
||||
stDebug("s-task:0x%x start to free task state", taskId);
|
||||
|
|
@ -465,7 +468,7 @@ void tFreeStreamTask(SStreamTask* pTask) {
|
|||
taosMemoryFree(pTask->outputInfo.pTokenBucket);
|
||||
taosThreadMutexDestroy(&pTask->lock);
|
||||
|
||||
pTask->outputInfo.pDownstreamUpdateList = taosArrayDestroy(pTask->outputInfo.pDownstreamUpdateList);
|
||||
pTask->outputInfo.pNodeEpsetUpdateList = taosArrayDestroy(pTask->outputInfo.pNodeEpsetUpdateList);
|
||||
|
||||
taosMemoryFree(pTask);
|
||||
stDebug("s-task:0x%x free task completed", taskId);
|
||||
|
|
@ -566,8 +569,8 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i
|
|||
// 2MiB per second for sink task
|
||||
// 50 times sink operator per second
|
||||
streamTaskInitTokenBucket(pOutputInfo->pTokenBucket, 35, 35, tsSinkDataRate, pTask->id.idStr);
|
||||
pOutputInfo->pDownstreamUpdateList = taosArrayInit(4, sizeof(SDownstreamTaskEpset));
|
||||
if (pOutputInfo->pDownstreamUpdateList == NULL) {
|
||||
pOutputInfo->pNodeEpsetUpdateList = taosArrayInit(4, sizeof(SDownstreamTaskEpset));
|
||||
if (pOutputInfo->pNodeEpsetUpdateList == NULL) {
|
||||
stError("s-task:%s failed to prepare downstreamUpdateList, code:%s", pTask->id.idStr, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
|
@ -620,13 +623,21 @@ void streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpS
|
|||
for (int32_t i = 0; i < numOfUpstream; ++i) {
|
||||
SStreamChildEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i);
|
||||
if (pInfo->nodeId == nodeId) {
|
||||
if (!(*pUpdated)) {
|
||||
*pUpdated = isEpsetEqual(&pInfo->epSet, pEpSet);
|
||||
bool equal = isEpsetEqual(&pInfo->epSet, pEpSet);
|
||||
if (!equal) {
|
||||
*pUpdated = true;
|
||||
|
||||
char tmp[512] = {0};
|
||||
epsetToStr(&pInfo->epSet, tmp, tListLen(tmp));
|
||||
|
||||
epsetAssign(&pInfo->epSet, pEpSet);
|
||||
stDebug("s-task:0x%x update the upstreamInfo taskId:0x%x(nodeId:%d) newEpset:%s old:%s", pTask->id.taskId,
|
||||
pInfo->taskId, nodeId, buf, tmp);
|
||||
} else {
|
||||
stDebug("s-task:0x%x not update upstreamInfo, since identical, task:0x%x(nodeId:%d) epset:%s", pTask->id.taskId,
|
||||
pInfo->taskId, nodeId, buf);
|
||||
}
|
||||
|
||||
epsetAssign(&pInfo->epSet, pEpSet);
|
||||
stDebug("s-task:0x%x update the upstreamInfo taskId:0x%x(nodeId:%d) newEpset:%s", pTask->id.taskId, pInfo->taskId,
|
||||
nodeId, buf);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -653,7 +664,6 @@ void streamTaskSetFixedDownstreamInfo(SStreamTask* pTask, const SStreamTask* pDo
|
|||
void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool *pUpdated) {
|
||||
char buf[512] = {0};
|
||||
epsetToStr(pEpSet, buf, tListLen(buf));
|
||||
*pUpdated = false;
|
||||
|
||||
int32_t id = pTask->id.taskId;
|
||||
int8_t type = pTask->outputInfo.type;
|
||||
|
|
@ -661,29 +671,43 @@ void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SE
|
|||
if (type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
|
||||
SArray* pVgs = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos;
|
||||
|
||||
int32_t numOfVgroups = taosArrayGetSize(pVgs);
|
||||
for (int32_t i = 0; i < numOfVgroups; i++) {
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pVgs); i++) {
|
||||
SVgroupInfo* pVgInfo = taosArrayGet(pVgs, i);
|
||||
|
||||
if (pVgInfo->vgId == nodeId) {
|
||||
if (!(*pUpdated)) {
|
||||
(*pUpdated) = isEpsetEqual(&pVgInfo->epSet, pEpSet);
|
||||
}
|
||||
bool isEqual = isEpsetEqual(&pVgInfo->epSet, pEpSet);
|
||||
if (!isEqual) {
|
||||
*pUpdated = true;
|
||||
char tmp[512] = {0};
|
||||
epsetToStr(&pVgInfo->epSet, tmp, tListLen(tmp));
|
||||
|
||||
epsetAssign(&pVgInfo->epSet, pEpSet);
|
||||
stDebug("s-task:0x%x update dispatch info, task:0x%x(nodeId:%d) newEpset:%s", id, pVgInfo->taskId, nodeId, buf);
|
||||
epsetAssign(&pVgInfo->epSet, pEpSet);
|
||||
stDebug("s-task:0x%x update dispatch info, task:0x%x(nodeId:%d) newEpset:%s old:%s", id, pVgInfo->taskId,
|
||||
nodeId, buf, tmp);
|
||||
} else {
|
||||
stDebug("s-task:0x%x not update dispatch info, since identical, task:0x%x(nodeId:%d) epset:%s", id,
|
||||
pVgInfo->taskId, nodeId, buf);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else if (type == TASK_OUTPUT__FIXED_DISPATCH) {
|
||||
STaskDispatcherFixed* pDispatcher = &pTask->outputInfo.fixedDispatcher;
|
||||
if (pDispatcher->nodeId == nodeId) {
|
||||
if (!(*pUpdated)) {
|
||||
*pUpdated = isEpsetEqual(&pDispatcher->epSet, pEpSet);
|
||||
}
|
||||
bool equal = isEpsetEqual(&pDispatcher->epSet, pEpSet);
|
||||
if (!equal) {
|
||||
*pUpdated = true;
|
||||
|
||||
epsetAssign(&pDispatcher->epSet, pEpSet);
|
||||
stDebug("s-task:0x%x update dispatch info, task:0x%x(nodeId:%d) newEpset:%s", id, pDispatcher->taskId, nodeId, buf);
|
||||
char tmp[512] = {0};
|
||||
epsetToStr(&pDispatcher->epSet, tmp, tListLen(tmp));
|
||||
|
||||
epsetAssign(&pDispatcher->epSet, pEpSet);
|
||||
stDebug("s-task:0x%x update dispatch info, task:0x%x(nodeId:%d) newEpset:%s old:%s", id, pDispatcher->taskId,
|
||||
nodeId, buf, tmp);
|
||||
} else {
|
||||
stDebug("s-task:0x%x not update dispatch info, since identical, task:0x%x(nodeId:%d) epset:%s", id,
|
||||
pDispatcher->taskId, nodeId, buf);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -968,379 +992,3 @@ int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) {
|
|||
tmsgSendReq(&pTask->info.mnodeEpset, &msg);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutputInfo, int64_t startTs) {
|
||||
taosArrayClear(pInfo->pList);
|
||||
|
||||
if (pOutputInfo->type == TASK_OUTPUT__FIXED_DISPATCH) {
|
||||
pInfo->notReadyTasks = 1;
|
||||
} else if (pOutputInfo->type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
|
||||
pInfo->notReadyTasks = taosArrayGetSize(pOutputInfo->shuffleDispatcher.dbInfo.pVgroupInfos);
|
||||
ASSERT(pInfo->notReadyTasks == pOutputInfo->shuffleDispatcher.dbInfo.vgNum);
|
||||
}
|
||||
|
||||
pInfo->startTs = startTs;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static SDownstreamStatusInfo* findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId) {
|
||||
for (int32_t j = 0; j < taosArrayGetSize(pInfo->pList); ++j) {
|
||||
SDownstreamStatusInfo* p = taosArrayGet(pInfo->pList, j);
|
||||
if (p->taskId == taskId) {
|
||||
return p;
|
||||
}
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int32_t streamTaskAddReqInfo(STaskCheckInfo* pInfo, int64_t reqId, int32_t taskId, const char* id) {
|
||||
SDownstreamStatusInfo info = {.taskId = taskId, .status = -1, .reqId = reqId, .rspTs = 0};
|
||||
|
||||
taosThreadMutexLock(&pInfo->checkInfoLock);
|
||||
|
||||
SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId);
|
||||
if (p != NULL) {
|
||||
stDebug("s-task:%s check info to task:0x%x already sent", id, taskId);
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
taosArrayPush(pInfo->pList, &info);
|
||||
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t status, int64_t rspTs, int64_t reqId,
|
||||
int32_t* pNotReady, const char* id) {
|
||||
taosThreadMutexLock(&pInfo->checkInfoLock);
|
||||
|
||||
SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId);
|
||||
if (p != NULL) {
|
||||
|
||||
if (reqId != p->reqId) {
|
||||
stError("s-task:%s reqId:%" PRIx64 " expected:%" PRIx64
|
||||
" expired check-rsp recv from downstream task:0x%x, discarded",
|
||||
id, reqId, p->reqId, taskId);
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
// subtract one not-ready-task, since it is ready now
|
||||
if ((p->status != TASK_DOWNSTREAM_READY) && (status == TASK_DOWNSTREAM_READY)) {
|
||||
*pNotReady = atomic_sub_fetch_32(&pInfo->notReadyTasks, 1);
|
||||
} else {
|
||||
*pNotReady = pInfo->notReadyTasks;
|
||||
}
|
||||
|
||||
p->status = status;
|
||||
p->rspTs = rspTs;
|
||||
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
stError("s-task:%s unexpected check rsp msg, invalid downstream task:0x%x, reqId:%" PRIx64 " discarded", id, taskId,
|
||||
reqId);
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
static int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id) {
|
||||
if (pInfo->inCheckProcess == 0) {
|
||||
pInfo->inCheckProcess = 1;
|
||||
} else {
|
||||
ASSERT(pInfo->startTs > 0);
|
||||
stError("s-task:%s already in check procedure, checkTs:%"PRId64", start monitor check rsp failed", id, pInfo->startTs);
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
stDebug("s-task:%s set the in-check-procedure flag", id);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t streamTaskCompleteCheckRsp(STaskCheckInfo* pInfo, const char* id) {
|
||||
if (!pInfo->inCheckProcess) {
|
||||
stWarn("s-task:%s already not in-check-procedure", id);
|
||||
}
|
||||
|
||||
int64_t el = (pInfo->startTs != 0) ? (taosGetTimestampMs() - pInfo->startTs) : 0;
|
||||
stDebug("s-task:%s clear the in-check-procedure flag, not in-check-procedure elapsed time:%" PRId64 " ms", id, el);
|
||||
|
||||
pInfo->startTs = 0;
|
||||
pInfo->notReadyTasks = 0;
|
||||
pInfo->inCheckProcess = 0;
|
||||
pInfo->stopCheckProcess = 0;
|
||||
taosArrayClear(pInfo->pList);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) {
|
||||
SStreamTaskCheckReq req = {
|
||||
.streamId = pTask->id.streamId,
|
||||
.upstreamTaskId = pTask->id.taskId,
|
||||
.upstreamNodeId = pTask->info.nodeId,
|
||||
.childId = pTask->info.selfChildId,
|
||||
.stage = pTask->pMeta->stage,
|
||||
};
|
||||
|
||||
STaskOutputInfo* pOutputInfo = &pTask->outputInfo;
|
||||
if (pOutputInfo->type == TASK_OUTPUT__FIXED_DISPATCH) {
|
||||
req.reqId = p->reqId;
|
||||
req.downstreamNodeId = pOutputInfo->fixedDispatcher.nodeId;
|
||||
req.downstreamTaskId = pOutputInfo->fixedDispatcher.taskId;
|
||||
stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) reqId:0x%" PRIx64,
|
||||
pTask->id.idStr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, req.reqId);
|
||||
|
||||
streamSendCheckMsg(pTask, &req, pOutputInfo->fixedDispatcher.nodeId, &pOutputInfo->fixedDispatcher.epSet);
|
||||
} else if (pOutputInfo->type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
|
||||
SArray* vgInfo = pOutputInfo->shuffleDispatcher.dbInfo.pVgroupInfos;
|
||||
int32_t numOfVgs = taosArrayGetSize(vgInfo);
|
||||
|
||||
for (int32_t i = 0; i < numOfVgs; i++) {
|
||||
SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i);
|
||||
|
||||
if (p->taskId == pVgInfo->taskId) {
|
||||
req.reqId = p->reqId;
|
||||
req.downstreamNodeId = pVgInfo->vgId;
|
||||
req.downstreamTaskId = pVgInfo->taskId;
|
||||
|
||||
stDebug("s-task:%s (vgId:%d) stage:%" PRId64
|
||||
" re-send check downstream task:0x%x(vgId:%d) (shuffle), idx:%d reqId:0x%" PRIx64,
|
||||
pTask->id.idStr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i,
|
||||
p->reqId);
|
||||
streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet);
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ASSERT(0);
|
||||
}
|
||||
}
|
||||
|
||||
static void getCheckRspStatus(STaskCheckInfo* pInfo, int64_t el, int32_t* numOfReady, int32_t* numOfFault,
|
||||
int32_t* numOfNotRsp, SArray* pTimeoutList, SArray* pNotReadyList, const char* id) {
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pList); ++i) {
|
||||
SDownstreamStatusInfo* p = taosArrayGet(pInfo->pList, i);
|
||||
if (p->status == TASK_DOWNSTREAM_READY) {
|
||||
(*numOfReady) += 1;
|
||||
} else if (p->status == TASK_UPSTREAM_NEW_STAGE || p->status == TASK_DOWNSTREAM_NOT_LEADER) {
|
||||
stDebug("s-task:%s recv status:NEW_STAGE/NOT_LEADER from downstream, task:0x%x, quit from check downstream", id,
|
||||
p->taskId);
|
||||
(*numOfFault) += 1;
|
||||
} else { // TASK_DOWNSTREAM_NOT_READY
|
||||
if (p->rspTs == 0) { // not response yet
|
||||
ASSERT(p->status == -1);
|
||||
if (el >= CHECK_NOT_RSP_DURATION) { // not receive info for 10 sec.
|
||||
taosArrayPush(pTimeoutList, &p->taskId);
|
||||
} else { // el < CHECK_NOT_RSP_DURATION
|
||||
(*numOfNotRsp) += 1; // do nothing and continue waiting for their rsp
|
||||
}
|
||||
} else {
|
||||
taosArrayPush(pNotReadyList, &p->taskId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void rspMonitorFn(void* param, void* tmrId) {
|
||||
SStreamTask* pTask = param;
|
||||
SStreamTaskState* pStat = streamTaskGetStatus(pTask);
|
||||
STaskCheckInfo* pInfo = &pTask->taskCheckInfo;
|
||||
int32_t vgId = pTask->pMeta->vgId;
|
||||
int64_t now = taosGetTimestampMs();
|
||||
int64_t el = now - pInfo->startTs;
|
||||
ETaskStatus state = pStat->state;
|
||||
const char* id = pTask->id.idStr;
|
||||
int32_t numOfReady = 0;
|
||||
int32_t numOfFault = 0;
|
||||
int32_t numOfNotRsp = 0;
|
||||
int32_t numOfNotReady = 0;
|
||||
int32_t numOfTimeout = 0;
|
||||
|
||||
stDebug("s-task:%s start to do check-downstream-rsp check in tmr", id);
|
||||
|
||||
if (state == TASK_STATUS__STOP) {
|
||||
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
|
||||
stDebug("s-task:%s status:%s vgId:%d quit from monitor check-rsp tmr, ref:%d", id, pStat->name, vgId, ref);
|
||||
|
||||
taosThreadMutexLock(&pInfo->checkInfoLock);
|
||||
streamTaskCompleteCheckRsp(pInfo, id);
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
|
||||
streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, pInfo->startTs, now, false);
|
||||
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
|
||||
STaskId* pHId = &pTask->hTaskInfo.id;
|
||||
streamMetaAddTaskLaunchResult(pTask->pMeta, pHId->streamId, pHId->taskId, pInfo->startTs, now, false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (state == TASK_STATUS__DROPPING || state == TASK_STATUS__READY || state == TASK_STATUS__PAUSE) {
|
||||
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
|
||||
stDebug("s-task:%s status:%s vgId:%d quit from monitor check-rsp tmr, ref:%d", id, pStat->name, vgId, ref);
|
||||
|
||||
taosThreadMutexLock(&pInfo->checkInfoLock);
|
||||
streamTaskCompleteCheckRsp(pInfo, id);
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
return;
|
||||
}
|
||||
|
||||
taosThreadMutexLock(&pInfo->checkInfoLock);
|
||||
if (pInfo->notReadyTasks == 0) {
|
||||
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
|
||||
stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", id, pStat->name,
|
||||
vgId, ref);
|
||||
|
||||
streamTaskCompleteCheckRsp(pInfo, id);
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
return;
|
||||
}
|
||||
|
||||
SArray* pNotReadyList = taosArrayInit(4, sizeof(int64_t));
|
||||
SArray* pTimeoutList = taosArrayInit(4, sizeof(int64_t));
|
||||
|
||||
if (pStat->state == TASK_STATUS__UNINIT) {
|
||||
getCheckRspStatus(pInfo, el, &numOfReady, &numOfFault, &numOfNotRsp, pTimeoutList, pNotReadyList, id);
|
||||
} else { // unexpected status
|
||||
stError("s-task:%s unexpected task status:%s during waiting for check rsp", id, pStat->name);
|
||||
}
|
||||
|
||||
numOfNotReady = (int32_t)taosArrayGetSize(pNotReadyList);
|
||||
numOfTimeout = (int32_t)taosArrayGetSize(pTimeoutList);
|
||||
|
||||
// fault tasks detected, not try anymore
|
||||
ASSERT((numOfReady + numOfFault + numOfNotReady + numOfTimeout + numOfNotRsp) == taosArrayGetSize(pInfo->pList));
|
||||
if (numOfFault > 0) {
|
||||
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
|
||||
stDebug(
|
||||
"s-task:%s status:%s vgId:%d all rsp. quit from monitor rsp tmr, since vnode-transfer/leader-change/restart "
|
||||
"detected, notRsp:%d, notReady:%d, fault:%d, timeout:%d, ready:%d ref:%d",
|
||||
id, pStat->name, vgId, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref);
|
||||
|
||||
streamTaskCompleteCheckRsp(pInfo, id);
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
|
||||
taosArrayDestroy(pNotReadyList);
|
||||
taosArrayDestroy(pTimeoutList);
|
||||
return;
|
||||
}
|
||||
|
||||
// checking of downstream tasks has been stopped by other threads
|
||||
if (pInfo->stopCheckProcess == 1) {
|
||||
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
|
||||
stDebug(
|
||||
"s-task:%s status:%s vgId:%d stopped by other threads to check downstream process, notRsp:%d, notReady:%d, "
|
||||
"fault:%d, timeout:%d, ready:%d ref:%d",
|
||||
id, pStat->name, vgId, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref);
|
||||
|
||||
streamTaskCompleteCheckRsp(pInfo, id);
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
|
||||
// add the not-ready tasks into the final task status result buf, along with related fill-history task if exists.
|
||||
streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, pInfo->startTs, now, false);
|
||||
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
|
||||
STaskId* pHId = &pTask->hTaskInfo.id;
|
||||
streamMetaAddTaskLaunchResult(pTask->pMeta, pHId->streamId, pHId->taskId, pInfo->startTs, now, false);
|
||||
}
|
||||
|
||||
taosArrayDestroy(pNotReadyList);
|
||||
taosArrayDestroy(pTimeoutList);
|
||||
return;
|
||||
}
|
||||
|
||||
if (numOfNotReady > 0) { // check to make sure not in recheck timer
|
||||
ASSERT(pTask->status.downstreamReady == 0);
|
||||
|
||||
// reset the info, and send the check msg to failure downstream again
|
||||
for (int32_t i = 0; i < numOfNotReady; ++i) {
|
||||
int32_t taskId = *(int32_t*)taosArrayGet(pNotReadyList, i);
|
||||
|
||||
SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId);
|
||||
if (p != NULL) {
|
||||
p->rspTs = 0;
|
||||
p->status = -1;
|
||||
doSendCheckMsg(pTask, p);
|
||||
}
|
||||
}
|
||||
|
||||
stDebug("s-task:%s %d downstream task(s) not ready, send check msg again", id, numOfNotReady);
|
||||
}
|
||||
|
||||
if (numOfTimeout > 0) {
|
||||
pInfo->startTs = now;
|
||||
ASSERT(pTask->status.downstreamReady == 0);
|
||||
|
||||
for (int32_t i = 0; i < numOfTimeout; ++i) {
|
||||
int32_t taskId = *(int32_t*)taosArrayGet(pTimeoutList, i);
|
||||
|
||||
SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId);
|
||||
if (p != NULL) {
|
||||
ASSERT(p->status == -1 && p->rspTs == 0);
|
||||
doSendCheckMsg(pTask, p);
|
||||
}
|
||||
}
|
||||
|
||||
stDebug("s-task:%s %d downstream tasks timeout, send check msg again, start ts:%" PRId64, id, numOfTimeout, now);
|
||||
}
|
||||
|
||||
taosTmrReset(rspMonitorFn, CHECK_RSP_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr);
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
|
||||
stDebug("s-task:%s continue checking rsp in 300ms, notRsp:%d, notReady:%d, fault:%d, timeout:%d, ready:%d", id,
|
||||
numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady);
|
||||
|
||||
taosArrayDestroy(pNotReadyList);
|
||||
taosArrayDestroy(pTimeoutList);
|
||||
}
|
||||
|
||||
int32_t streamTaskStartMonitorCheckRsp(SStreamTask* pTask) {
|
||||
STaskCheckInfo* pInfo = &pTask->taskCheckInfo;
|
||||
|
||||
taosThreadMutexLock(&pInfo->checkInfoLock);
|
||||
int32_t code = streamTaskStartCheckDownstream(pInfo, pTask->id.idStr);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
streamTaskInitTaskCheckInfo(pInfo, &pTask->outputInfo, taosGetTimestampMs());
|
||||
|
||||
int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
|
||||
stDebug("s-task:%s start check rsp monit, ref:%d ", pTask->id.idStr, ref);
|
||||
|
||||
if (pInfo->checkRspTmr == NULL) {
|
||||
pInfo->checkRspTmr = taosTmrStart(rspMonitorFn, CHECK_RSP_INTERVAL, pTask, streamTimer);
|
||||
} else {
|
||||
taosTmrReset(rspMonitorFn, CHECK_RSP_INTERVAL, pTask, streamTimer, pInfo->checkRspTmr);
|
||||
}
|
||||
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id) {
|
||||
taosThreadMutexLock(&pInfo->checkInfoLock);
|
||||
streamTaskCompleteCheckRsp(pInfo, id);
|
||||
|
||||
pInfo->stopCheckProcess = 1;
|
||||
taosThreadMutexUnlock(&pInfo->checkInfoLock);
|
||||
|
||||
stDebug("s-task:%s set stop check rsp mon", id);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
void streamTaskCleanCheckInfo(STaskCheckInfo* pInfo) {
|
||||
ASSERT(pInfo->inCheckProcess == 0);
|
||||
|
||||
pInfo->pList = taosArrayDestroy(pInfo->pList);
|
||||
if (pInfo->checkRspTmr != NULL) {
|
||||
/*bool ret = */ taosTmrStop(pInfo->checkRspTmr);
|
||||
pInfo->checkRspTmr = NULL;
|
||||
}
|
||||
|
||||
taosThreadMutexDestroy(&pInfo->checkInfoLock);
|
||||
}
|
||||
|
|
@ -188,7 +188,7 @@ int32_t l2ComressInitImpl_zlib(char *lossyColumns, float fPrecision, double dPre
|
|||
int32_t l2CompressImpl_zlib(const char *const input, const int32_t inputSize, char *const output, int32_t outputSize,
|
||||
const char type, int8_t lvl) {
|
||||
uLongf dstLen = outputSize - 1;
|
||||
int32_t ret = compress2((Bytef *)(output + 1), (uLongf *)&dstLen, (Bytef *)input, (uLong)inputSize, 9);
|
||||
int32_t ret = compress2((Bytef *)(output + 1), (uLongf *)&dstLen, (Bytef *)input, (uLong)inputSize, lvl);
|
||||
if (ret == Z_OK) {
|
||||
output[0] = 1;
|
||||
return dstLen + 1;
|
||||
|
|
|
|||
|
|
@ -27,12 +27,17 @@
|
|||
#define CFG_NAME_PRINT_LEN 24
|
||||
#define CFG_SRC_PRINT_LEN 12
|
||||
|
||||
struct SConfig {
|
||||
ECfgSrcType stype;
|
||||
SArray *array;
|
||||
TdThreadMutex lock;
|
||||
};
|
||||
|
||||
int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath);
|
||||
int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath);
|
||||
int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile);
|
||||
int32_t cfgLoadFromEnvVar(SConfig *pConfig);
|
||||
int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd);
|
||||
int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url);
|
||||
int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSrcType stype);
|
||||
|
||||
extern char **environ;
|
||||
|
||||
|
|
@ -50,6 +55,7 @@ SConfig *cfgInit() {
|
|||
return NULL;
|
||||
}
|
||||
|
||||
taosThreadMutexInit(&pCfg->lock, NULL);
|
||||
return pCfg;
|
||||
}
|
||||
|
||||
|
|
@ -74,7 +80,7 @@ int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs) {
|
|||
int32_t size = taosArrayGetSize(pArgs);
|
||||
for (int32_t i = 0; i < size; ++i) {
|
||||
SConfigPair *pPair = taosArrayGet(pArgs, i);
|
||||
if (cfgSetItem(pCfg, pPair->name, pPair->value, CFG_STYPE_ARG_LIST) != 0) {
|
||||
if (cfgSetItem(pCfg, pPair->name, pPair->value, CFG_STYPE_ARG_LIST, true) != 0) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
|
@ -82,57 +88,41 @@ int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void cfgFreeItem(SConfigItem *pItem) {
|
||||
void cfgItemFreeVal(SConfigItem *pItem) {
|
||||
if (pItem->dtype == CFG_DTYPE_STRING || pItem->dtype == CFG_DTYPE_DIR || pItem->dtype == CFG_DTYPE_LOCALE ||
|
||||
pItem->dtype == CFG_DTYPE_CHARSET || pItem->dtype == CFG_DTYPE_TIMEZONE) {
|
||||
taosMemoryFreeClear(pItem->str);
|
||||
}
|
||||
|
||||
if (pItem->array) {
|
||||
taosArrayDestroy(pItem->array);
|
||||
pItem->array = NULL;
|
||||
pItem->array = taosArrayDestroy(pItem->array);
|
||||
}
|
||||
}
|
||||
|
||||
void cfgCleanup(SConfig *pCfg) {
|
||||
if (pCfg != NULL) {
|
||||
int32_t size = taosArrayGetSize(pCfg->array);
|
||||
for (int32_t i = 0; i < size; ++i) {
|
||||
SConfigItem *pItem = taosArrayGet(pCfg->array, i);
|
||||
cfgFreeItem(pItem);
|
||||
taosMemoryFreeClear(pItem->name);
|
||||
}
|
||||
taosArrayDestroy(pCfg->array);
|
||||
taosMemoryFree(pCfg);
|
||||
if (pCfg == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
int32_t size = taosArrayGetSize(pCfg->array);
|
||||
for (int32_t i = 0; i < size; ++i) {
|
||||
SConfigItem *pItem = taosArrayGet(pCfg->array, i);
|
||||
cfgItemFreeVal(pItem);
|
||||
taosMemoryFreeClear(pItem->name);
|
||||
}
|
||||
|
||||
taosArrayDestroy(pCfg->array);
|
||||
taosThreadMutexDestroy(&pCfg->lock);
|
||||
taosMemoryFree(pCfg);
|
||||
}
|
||||
|
||||
int32_t cfgGetSize(SConfig *pCfg) { return taosArrayGetSize(pCfg->array); }
|
||||
|
||||
static int32_t cfgCheckAndSetTimezone(SConfigItem *pItem, const char *timezone) {
|
||||
cfgFreeItem(pItem);
|
||||
pItem->str = taosStrdup(timezone);
|
||||
if (pItem->str == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
static int32_t cfgCheckAndSetConf(SConfigItem *pItem, const char *conf) {
|
||||
cfgItemFreeVal(pItem);
|
||||
ASSERT(pItem->str == NULL);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t cfgCheckAndSetCharset(SConfigItem *pItem, const char *charset) {
|
||||
cfgFreeItem(pItem);
|
||||
pItem->str = taosStrdup(charset);
|
||||
if (pItem->str == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t cfgCheckAndSetLocale(SConfigItem *pItem, const char *locale) {
|
||||
cfgFreeItem(pItem);
|
||||
pItem->str = taosStrdup(locale);
|
||||
pItem->str = taosStrdup(conf);
|
||||
if (pItem->str == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
|
|
@ -229,7 +219,7 @@ static int32_t cfgSetString(SConfigItem *pItem, const char *value, ECfgSrcType s
|
|||
return -1;
|
||||
}
|
||||
|
||||
taosMemoryFree(pItem->str);
|
||||
taosMemoryFreeClear(pItem->str);
|
||||
pItem->str = tmp;
|
||||
pItem->stype = stype;
|
||||
return 0;
|
||||
|
|
@ -246,20 +236,8 @@ static int32_t cfgSetDir(SConfigItem *pItem, const char *value, ECfgSrcType styp
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int32_t cfgSetLocale(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
|
||||
if (cfgCheckAndSetLocale(pItem, value) != 0) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s", pItem->name, cfgDtypeStr(pItem->dtype),
|
||||
cfgStypeStr(stype), value, terrstr());
|
||||
return -1;
|
||||
}
|
||||
|
||||
pItem->stype = stype;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t cfgSetCharset(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
|
||||
if (cfgCheckAndSetCharset(pItem, value) != 0) {
|
||||
static int32_t doSetConf(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
|
||||
if (cfgCheckAndSetConf(pItem, value) != 0) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s", pItem->name, cfgDtypeStr(pItem->dtype),
|
||||
cfgStypeStr(stype), value, terrstr());
|
||||
|
|
@ -271,29 +249,32 @@ static int32_t cfgSetCharset(SConfigItem *pItem, const char *value, ECfgSrcType
|
|||
}
|
||||
|
||||
static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
|
||||
if (cfgCheckAndSetTimezone(pItem, value) != 0) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s", pItem->name, cfgDtypeStr(pItem->dtype),
|
||||
cfgStypeStr(stype), value, terrstr());
|
||||
return -1;
|
||||
int32_t code = doSetConf(pItem, value, stype);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return code;
|
||||
}
|
||||
pItem->stype = stype;
|
||||
|
||||
// apply new timezone
|
||||
osSetTimezone(value);
|
||||
|
||||
return 0;
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, const char *level, const char *primary,
|
||||
ECfgSrcType stype) {
|
||||
taosThreadMutexLock(&pCfg->lock);
|
||||
|
||||
SConfigItem *pItem = cfgGetItem(pCfg, name);
|
||||
if (pItem == NULL) return -1;
|
||||
if (pItem == NULL) {
|
||||
taosThreadMutexUnlock(&pCfg->lock);
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (pItem->array == NULL) {
|
||||
pItem->array = taosArrayInit(16, sizeof(SDiskCfg));
|
||||
if (pItem->array == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
taosThreadMutexUnlock(&pCfg->lock);
|
||||
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
|
@ -305,10 +286,14 @@ static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value,
|
|||
void *ret = taosArrayPush(pItem->array, &cfg);
|
||||
if (ret == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
taosThreadMutexUnlock(&pCfg->lock);
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
pItem->stype = stype;
|
||||
taosThreadMutexUnlock(&pCfg->lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -340,65 +325,108 @@ static int32_t cfgUpdateDebugFlagItem(SConfig *pCfg, const char *name, bool rese
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype) {
|
||||
int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype, bool lock) {
|
||||
// GRANT_CFG_SET;
|
||||
int32_t code = 0;
|
||||
|
||||
if (lock) {
|
||||
taosThreadMutexLock(&pCfg->lock);
|
||||
}
|
||||
|
||||
SConfigItem *pItem = cfgGetItem(pCfg, name);
|
||||
if (pItem == NULL) {
|
||||
terrno = TSDB_CODE_CFG_NOT_FOUND;
|
||||
taosThreadMutexUnlock(&pCfg->lock);
|
||||
return -1;
|
||||
}
|
||||
|
||||
switch (pItem->dtype) {
|
||||
case CFG_DTYPE_BOOL:
|
||||
return cfgSetBool(pItem, value, stype);
|
||||
case CFG_DTYPE_INT32:
|
||||
return cfgSetInt32(pItem, value, stype);
|
||||
case CFG_DTYPE_INT64:
|
||||
return cfgSetInt64(pItem, value, stype);
|
||||
case CFG_DTYPE_BOOL: {
|
||||
code = cfgSetBool(pItem, value, stype);
|
||||
break;
|
||||
}
|
||||
case CFG_DTYPE_INT32: {
|
||||
code = cfgSetInt32(pItem, value, stype);
|
||||
break;
|
||||
}
|
||||
case CFG_DTYPE_INT64: {
|
||||
code = cfgSetInt64(pItem, value, stype);
|
||||
break;
|
||||
}
|
||||
case CFG_DTYPE_FLOAT:
|
||||
case CFG_DTYPE_DOUBLE:
|
||||
return cfgSetFloat(pItem, value, stype);
|
||||
case CFG_DTYPE_STRING:
|
||||
return cfgSetString(pItem, value, stype);
|
||||
case CFG_DTYPE_DIR:
|
||||
return cfgSetDir(pItem, value, stype);
|
||||
case CFG_DTYPE_TIMEZONE:
|
||||
return cfgSetTimezone(pItem, value, stype);
|
||||
case CFG_DTYPE_CHARSET:
|
||||
return cfgSetCharset(pItem, value, stype);
|
||||
case CFG_DTYPE_LOCALE:
|
||||
return cfgSetLocale(pItem, value, stype);
|
||||
case CFG_DTYPE_DOUBLE: {
|
||||
code = cfgSetFloat(pItem, value, stype);
|
||||
break;
|
||||
}
|
||||
case CFG_DTYPE_STRING: {
|
||||
code = cfgSetString(pItem, value, stype);
|
||||
break;
|
||||
}
|
||||
case CFG_DTYPE_DIR: {
|
||||
code = cfgSetDir(pItem, value, stype);
|
||||
break;
|
||||
}
|
||||
case CFG_DTYPE_TIMEZONE: {
|
||||
code = cfgSetTimezone(pItem, value, stype);
|
||||
break;
|
||||
}
|
||||
case CFG_DTYPE_CHARSET: {
|
||||
code = doSetConf(pItem, value, stype);
|
||||
break;
|
||||
}
|
||||
case CFG_DTYPE_LOCALE: {
|
||||
code = doSetConf(pItem, value, stype);
|
||||
break;
|
||||
}
|
||||
case CFG_DTYPE_NONE:
|
||||
default:
|
||||
terrno = TSDB_CODE_INVALID_CFG;
|
||||
break;
|
||||
}
|
||||
|
||||
_err_out:
|
||||
terrno = TSDB_CODE_INVALID_CFG;
|
||||
return -1;
|
||||
if (lock) {
|
||||
taosThreadMutexUnlock(&pCfg->lock);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
SConfigItem *cfgGetItem(SConfig *pCfg, const char *name) {
|
||||
SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName) {
|
||||
if (pCfg == NULL) return NULL;
|
||||
int32_t size = taosArrayGetSize(pCfg->array);
|
||||
for (int32_t i = 0; i < size; ++i) {
|
||||
SConfigItem *pItem = taosArrayGet(pCfg->array, i);
|
||||
if (strcasecmp(pItem->name, name) == 0) {
|
||||
if (strcasecmp(pItem->name, pName) == 0) {
|
||||
return pItem;
|
||||
}
|
||||
}
|
||||
|
||||
// uError("name:%s, cfg not found", name);
|
||||
terrno = TSDB_CODE_CFG_NOT_FOUND;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void cfgLock(SConfig *pCfg) {
|
||||
if (pCfg == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
taosThreadMutexLock(&pCfg->lock);
|
||||
}
|
||||
|
||||
void cfgUnLock(SConfig *pCfg) {
|
||||
taosThreadMutexUnlock(&pCfg->lock);
|
||||
}
|
||||
|
||||
int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer) {
|
||||
ECfgDynType dynType = isServer ? CFG_DYN_SERVER : CFG_DYN_CLIENT;
|
||||
|
||||
cfgLock(pCfg);
|
||||
|
||||
SConfigItem *pItem = cfgGetItem(pCfg, name);
|
||||
if (!pItem || (pItem->dynScope & dynType) == 0) {
|
||||
uError("failed to config:%s, not support", name);
|
||||
uError("failed to config:%s, not support update this config", name);
|
||||
terrno = TSDB_CODE_INVALID_CFG;
|
||||
cfgUnLock(pCfg);
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -408,28 +436,37 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p
|
|||
if (ival != 0 && ival != 1) {
|
||||
uError("cfg:%s, type:%s value:%d out of range[0, 1]", pItem->name, cfgDtypeStr(pItem->dtype), ival);
|
||||
terrno = TSDB_CODE_OUT_OF_RANGE;
|
||||
cfgUnLock(pCfg);
|
||||
return -1;
|
||||
}
|
||||
} break;
|
||||
case CFG_DTYPE_INT32: {
|
||||
int32_t ival;
|
||||
int32_t code = (int32_t)taosStrHumanToInt32(pVal, &ival);
|
||||
if (code != TSDB_CODE_SUCCESS) return code;
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
cfgUnLock(pCfg);
|
||||
return code;
|
||||
}
|
||||
if (ival < pItem->imin || ival > pItem->imax) {
|
||||
uError("cfg:%s, type:%s value:%d out of range[%" PRId64 ", %" PRId64 "]", pItem->name,
|
||||
cfgDtypeStr(pItem->dtype), ival, pItem->imin, pItem->imax);
|
||||
terrno = TSDB_CODE_OUT_OF_RANGE;
|
||||
cfgUnLock(pCfg);
|
||||
return -1;
|
||||
}
|
||||
} break;
|
||||
case CFG_DTYPE_INT64: {
|
||||
int64_t ival;
|
||||
int32_t code = taosStrHumanToInt64(pVal, &ival);
|
||||
if (code != TSDB_CODE_SUCCESS) return code;
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
cfgUnLock(pCfg);
|
||||
return code;
|
||||
}
|
||||
if (ival < pItem->imin || ival > pItem->imax) {
|
||||
uError("cfg:%s, type:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64 "]", pItem->name,
|
||||
cfgDtypeStr(pItem->dtype), ival, pItem->imin, pItem->imax);
|
||||
terrno = TSDB_CODE_OUT_OF_RANGE;
|
||||
cfgUnLock(pCfg);
|
||||
return -1;
|
||||
}
|
||||
} break;
|
||||
|
|
@ -437,17 +474,23 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p
|
|||
case CFG_DTYPE_DOUBLE: {
|
||||
double dval;
|
||||
int32_t code = parseCfgReal(pVal, &dval);
|
||||
if (code != TSDB_CODE_SUCCESS) return code;
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
cfgUnLock(pCfg);
|
||||
return code;
|
||||
}
|
||||
if (dval < pItem->fmin || dval > pItem->fmax) {
|
||||
uError("cfg:%s, type:%s value:%f out of range[%f, %f]", pItem->name, cfgDtypeStr(pItem->dtype), dval,
|
||||
pItem->fmin, pItem->fmax);
|
||||
terrno = TSDB_CODE_OUT_OF_RANGE;
|
||||
cfgUnLock(pCfg);
|
||||
return -1;
|
||||
}
|
||||
} break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
cfgUnLock(pCfg);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -459,7 +502,7 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
int size = pCfg->array->size;
|
||||
int32_t size = taosArrayGetSize(pCfg->array);
|
||||
for (int32_t i = 0; i < size; ++i) {
|
||||
SConfigItem *existItem = taosArrayGet(pCfg->array, i);
|
||||
if (existItem != NULL && strcmp(existItem->name, pItem->name) == 0) {
|
||||
|
|
@ -559,7 +602,7 @@ int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_
|
|||
|
||||
int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) {
|
||||
SConfigItem item = {.dtype = CFG_DTYPE_LOCALE, .scope = scope, .dynScope = dynScope};
|
||||
if (cfgCheckAndSetLocale(&item, defaultVal) != 0) {
|
||||
if (cfgCheckAndSetConf(&item, defaultVal) != 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -568,7 +611,7 @@ int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, in
|
|||
|
||||
int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) {
|
||||
SConfigItem item = {.dtype = CFG_DTYPE_CHARSET, .scope = scope, .dynScope = dynScope};
|
||||
if (cfgCheckAndSetCharset(&item, defaultVal) != 0) {
|
||||
if (cfgCheckAndSetConf(&item, defaultVal) != 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -577,7 +620,7 @@ int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, i
|
|||
|
||||
int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) {
|
||||
SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE, .scope = scope, .dynScope = dynScope};
|
||||
if (cfgCheckAndSetTimezone(&item, defaultVal) != 0) {
|
||||
if (cfgCheckAndSetConf(&item, defaultVal) != 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -886,7 +929,7 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) {
|
|||
if (vlen3 != 0) value3[vlen3] = 0;
|
||||
}
|
||||
|
||||
code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_VAR);
|
||||
code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_VAR, true);
|
||||
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
|
||||
|
||||
if (strcasecmp(name, "dataDir") == 0) {
|
||||
|
|
@ -929,7 +972,7 @@ int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) {
|
|||
if (vlen3 != 0) value3[vlen3] = 0;
|
||||
}
|
||||
|
||||
code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_CMD);
|
||||
code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_CMD, true);
|
||||
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
|
||||
|
||||
if (strcasecmp(name, "dataDir") == 0) {
|
||||
|
|
@ -994,7 +1037,7 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) {
|
|||
if (vlen3 != 0) value3[vlen3] = 0;
|
||||
}
|
||||
|
||||
code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_FILE);
|
||||
code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_FILE, true);
|
||||
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
|
||||
|
||||
if (strcasecmp(name, "dataDir") == 0) {
|
||||
|
|
@ -1046,28 +1089,27 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
|
|||
paGetToken(name + olen + 1, &value, &vlen);
|
||||
if (vlen == 0) continue;
|
||||
value[vlen] = 0;
|
||||
|
||||
|
||||
if (strcasecmp(name, "encryptScope") == 0) {
|
||||
char* tmp = NULL;
|
||||
char *tmp = NULL;
|
||||
int32_t len = 0;
|
||||
char newValue[1024] = {0};
|
||||
char newValue[1024] = {0};
|
||||
|
||||
strcpy(newValue, value);
|
||||
|
||||
|
||||
int32_t count = 1;
|
||||
while(vlen < 1024){
|
||||
while (vlen < 1024) {
|
||||
paGetToken(value + vlen + 1 * count, &tmp, &len);
|
||||
if(len == 0) break;
|
||||
if (len == 0) break;
|
||||
tmp[len] = 0;
|
||||
strcpy(newValue + vlen, tmp);
|
||||
vlen += len;
|
||||
count++;
|
||||
}
|
||||
|
||||
code = cfgSetItem(pConfig, name, newValue, CFG_STYPE_CFG_FILE);
|
||||
code = cfgSetItem(pConfig, name, newValue, CFG_STYPE_CFG_FILE, true);
|
||||
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
|
||||
}
|
||||
else{
|
||||
} else {
|
||||
paGetToken(value + vlen + 1, &value2, &vlen2);
|
||||
if (vlen2 != 0) {
|
||||
value2[vlen2] = 0;
|
||||
|
|
@ -1075,7 +1117,7 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
|
|||
if (vlen3 != 0) value3[vlen3] = 0;
|
||||
}
|
||||
|
||||
code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE);
|
||||
code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE, true);
|
||||
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
|
||||
}
|
||||
|
||||
|
|
@ -1250,7 +1292,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
|
|||
if (vlen3 != 0) value3[vlen3] = 0;
|
||||
}
|
||||
|
||||
code = cfgSetItem(pConfig, name, value, CFG_STYPE_APOLLO_URL);
|
||||
code = cfgSetItem(pConfig, name, value, CFG_STYPE_APOLLO_URL, true);
|
||||
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
|
||||
|
||||
if (strcasecmp(name, "dataDir") == 0) {
|
||||
|
|
@ -1356,3 +1398,35 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl
|
|||
uInfo("fail get apollo url from cmd env file");
|
||||
return -1;
|
||||
}
|
||||
|
||||
struct SConfigIter {
|
||||
int32_t index;
|
||||
SConfig *pConf;
|
||||
};
|
||||
|
||||
SConfigIter *cfgCreateIter(SConfig *pConf) {
|
||||
SConfigIter* pIter = taosMemoryCalloc(1, sizeof(SConfigIter));
|
||||
if (pIter == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
pIter->pConf = pConf;
|
||||
return pIter;
|
||||
}
|
||||
|
||||
SConfigItem *cfgNextIter(SConfigIter* pIter) {
|
||||
if (pIter->index < cfgGetSize(pIter->pConf)) {
|
||||
return taosArrayGet(pIter->pConf->array, pIter->index++);
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void cfgDestroyIter(SConfigIter *pIter) {
|
||||
if (pIter == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
taosMemoryFree(pIter);
|
||||
}
|
||||
|
|
@ -63,9 +63,11 @@ TEST_F(CfgTest, 02_Basic) {
|
|||
|
||||
EXPECT_EQ(cfgGetSize(pConfig), 6);
|
||||
|
||||
int32_t size = taosArrayGetSize(pConfig->array);
|
||||
for (int32_t i = 0; i < size; ++i) {
|
||||
SConfigItem *pItem = (SConfigItem *)taosArrayGet(pConfig->array, i);
|
||||
int32_t size = cfgGetSize(pConfig);
|
||||
|
||||
SConfigItem* pItem = NULL;
|
||||
SConfigIter* pIter = cfgCreateIter(pConfig);
|
||||
while((pItem == cfgNextIter(pIter)) != NULL) {
|
||||
switch (pItem->dtype) {
|
||||
case CFG_DTYPE_BOOL:
|
||||
printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->bval);
|
||||
|
|
@ -90,9 +92,12 @@ TEST_F(CfgTest, 02_Basic) {
|
|||
break;
|
||||
}
|
||||
}
|
||||
|
||||
cfgDestroyIter(pIter);
|
||||
|
||||
EXPECT_EQ(cfgGetSize(pConfig), 6);
|
||||
|
||||
SConfigItem *pItem = cfgGetItem(pConfig, "test_bool");
|
||||
pItem = cfgGetItem(pConfig, "test_bool");
|
||||
EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT);
|
||||
EXPECT_EQ(pItem->dtype, CFG_DTYPE_BOOL);
|
||||
EXPECT_STREQ(pItem->name, "test_bool");
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ class TDTestCase(TBase):
|
|||
|
||||
def checkQueryOK(self, rets):
|
||||
if rets[-2][:9] != "Query OK,":
|
||||
tdLog.exit(f"check taos -s return unecpect: {rets}")
|
||||
tdLog.exit(f"check taos -s return unexpect: {rets}")
|
||||
|
||||
def doTaos(self):
|
||||
tdLog.info(f"check taos command options...")
|
||||
|
|
|
|||
|
|
@ -194,4 +194,12 @@ if $rows != 144 then
|
|||
return -1
|
||||
endi
|
||||
|
||||
sql select a.ts, b.ts from tba1 a join sta b on a.ts = b.ts and a.t1 = b.t1;
|
||||
if $rows != 4 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql select a.ts, b.ts from sta a join sta b on a.ts = b.ts and a.t1 = b.t1;
|
||||
if $rows != 8 then
|
||||
return -1
|
||||
endi
|
||||
|
|
|
|||
|
|
@ -185,8 +185,8 @@ class TDTestCase:
|
|||
# baseVersion = "3.0.1.8"
|
||||
|
||||
tdLog.printNoPrefix(f"==========step1:prepare and check data in old version-{BASEVERSION}")
|
||||
tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ")
|
||||
os.system(f"LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ")
|
||||
tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -v 1 -y ")
|
||||
os.system(f"LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -v 1 -y ")
|
||||
os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database test '")
|
||||
os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/com_alltypedata.json -y")
|
||||
os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database curdb '")
|
||||
|
|
@ -196,49 +196,81 @@ class TDTestCase:
|
|||
os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'select min(ui) from curdb.meters '")
|
||||
os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'select max(bi) from curdb.meters '")
|
||||
|
||||
# os.system(f"LD_LIBRARY_PATH=/usr/lib taos -s 'use test;create stream current_stream into current_stream_output_stb as select _wstart as `start`, _wend as wend, max(current) as max_current from meters where voltage <= 220 interval (5s);' ")
|
||||
# os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;create stream power_stream into power_stream_output_stb as select ts, concat_ws(\\".\\", location, tbname) as meter_location, current*voltage*cos(phase) as active_power, current*voltage*sin(phase) as reactive_power from meters partition by tbname;" ')
|
||||
# os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;show streams;" ')
|
||||
os.system(f"LD_LIBRARY_PATH=/usr/lib taos -s 'use test;create stream current_stream into current_stream_output_stb as select _wstart as `start`, _wend as wend, max(current) as max_current from meters where voltage <= 220 interval (5s);' ")
|
||||
os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;create stream power_stream trigger at_once into power_stream_output_stb as select ts, concat_ws(\\".\\", location, tbname) as meter_location, current*voltage*cos(phase) as active_power, current*voltage*sin(phase) as reactive_power from meters partition by tbname;" ')
|
||||
os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;show streams;" ')
|
||||
|
||||
self.alter_string_in_file("0-others/tmqBasic.json", "/etc/taos/", cPath)
|
||||
# os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/tmqBasic.json -y ")
|
||||
os.system('LD_LIBRARY_PATH=/usr/lib taos -s "create topic if not exists tmq_test_topic as select current,voltage,phase from test.meters where voltage <= 106 and current <= 5;" ')
|
||||
# create db/stb/select topic
|
||||
|
||||
db_topic = "db_test_topic"
|
||||
os.system(f'LD_LIBRARY_PATH=/usr/lib taos -s "create topic if not exists {db_topic} with meta as database test" ')
|
||||
|
||||
stable_topic = "stable_test_meters_topic"
|
||||
os.system(f'LD_LIBRARY_PATH=/usr/lib taos -s "create topic if not exists {stable_topic} as stable test.meters where tbname like \\"d3\\";" ')
|
||||
|
||||
select_topic = "select_test_meters_topic"
|
||||
os.system(f'LD_LIBRARY_PATH=/usr/lib taos -s "create topic if not exists {select_topic} as select current,voltage,phase from test.meters where voltage >= 170;" ')
|
||||
|
||||
os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;show topics;" ')
|
||||
os.system(f" /usr/bin/taosadapter --version " )
|
||||
consumer_dict = {
|
||||
"group.id": "g1",
|
||||
"td.connect.websocket.scheme": "ws",
|
||||
"td.connect.user": "root",
|
||||
"td.connect.pass": "taosdata",
|
||||
"auto.offset.reset": "earliest",
|
||||
"enable.auto.commit": "false",
|
||||
}
|
||||
|
||||
consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"})
|
||||
consumer = taosws.Consumer(consumer_dict)
|
||||
try:
|
||||
consumer.subscribe(["tmq_test_topic"])
|
||||
consumer.subscribe([select_topic])
|
||||
except TmqError:
|
||||
tdLog.exit(f"subscribe error")
|
||||
|
||||
first_consumer_rows = 0
|
||||
while True:
|
||||
message = consumer.poll(timeout=1.0)
|
||||
if message:
|
||||
print("message")
|
||||
id = message.vgroup()
|
||||
topic = message.topic()
|
||||
database = message.database()
|
||||
|
||||
for block in message:
|
||||
nrows = block.nrows()
|
||||
ncols = block.ncols()
|
||||
for row in block:
|
||||
print(row)
|
||||
values = block.fetchall()
|
||||
print(nrows, ncols)
|
||||
|
||||
consumer.commit(message)
|
||||
first_consumer_rows += block.nrows()
|
||||
else:
|
||||
print("break")
|
||||
tdLog.notice("message is null and break")
|
||||
break
|
||||
consumer.commit(message)
|
||||
tdLog.debug(f"topic:{select_topic} ,first consumer rows is {first_consumer_rows} in old version")
|
||||
break
|
||||
|
||||
consumer.close()
|
||||
# consumer_dict2 = {
|
||||
# "group.id": "g2",
|
||||
# "td.connect.websocket.scheme": "ws",
|
||||
# "td.connect.user": "root",
|
||||
# "td.connect.pass": "taosdata",
|
||||
# "auto.offset.reset": "earliest",
|
||||
# "enable.auto.commit": "false",
|
||||
# }
|
||||
# consumer = taosws.Consumer(consumer_dict2)
|
||||
# try:
|
||||
# consumer.subscribe([db_topic,stable_topic])
|
||||
# except TmqError:
|
||||
# tdLog.exit(f"subscribe error")
|
||||
# first_consumer_rows = 0
|
||||
# while True:
|
||||
# message = consumer.poll(timeout=1.0)
|
||||
# if message:
|
||||
# for block in message:
|
||||
# first_consumer_rows += block.nrows()
|
||||
# else:
|
||||
# tdLog.notice("message is null and break")
|
||||
# break
|
||||
# consumer.commit(message)
|
||||
# tdLog.debug(f"topic:{select_topic} ,first consumer rows is {first_consumer_rows} in old version")
|
||||
# break
|
||||
|
||||
|
||||
|
||||
tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ")
|
||||
os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y")
|
||||
os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database db4096 '")
|
||||
|
|
@ -279,11 +311,10 @@ class TDTestCase:
|
|||
tdLog.printNoPrefix(f"==========step3:prepare and check data in new version-{nowServerVersion}")
|
||||
tdsql.query(f"select count(*) from {stb}")
|
||||
tdsql.checkData(0,0,tableNumbers*recordNumbers1)
|
||||
# tdsql.query("show streams;")
|
||||
# os.system(f"taosBenchmark -t {tableNumbers} -n {recordNumbers2} -y ")
|
||||
# tdsql.query("show streams;")
|
||||
# tdsql.query(f"select count(*) from {stb}")
|
||||
# tdsql.checkData(0,0,tableNumbers*recordNumbers2)
|
||||
tdsql.query("show streams;")
|
||||
tdsql.checkRows(2)
|
||||
|
||||
|
||||
|
||||
# checkout db4096
|
||||
tdsql.query("select count(*) from db4096.stb0")
|
||||
|
|
@ -334,7 +365,7 @@ class TDTestCase:
|
|||
|
||||
# check stream
|
||||
tdsql.query("show streams;")
|
||||
tdsql.checkRows(0)
|
||||
tdsql.checkRows(2)
|
||||
|
||||
#check TS-3131
|
||||
tdsql.query("select *,tbname from d0.almlog where mcid='m0103';")
|
||||
|
|
@ -348,39 +379,48 @@ class TDTestCase:
|
|||
print("The unordered list is the same as the ordered list.")
|
||||
else:
|
||||
tdLog.exit("The unordered list is not the same as the ordered list.")
|
||||
tdsql.execute("insert into test.d80 values (now+1s, 11, 103, 0.21);")
|
||||
tdsql.execute("insert into test.d9 values (now+5s, 4.3, 104, 0.4);")
|
||||
|
||||
|
||||
# check tmq
|
||||
tdsql.execute("insert into test.d80 values (now+1s, 11, 190, 0.21);")
|
||||
tdsql.execute("insert into test.d9 values (now+5s, 4.3, 104, 0.4);")
|
||||
conn = taos.connect()
|
||||
|
||||
consumer = Consumer(
|
||||
{
|
||||
"group.id": "tg75",
|
||||
"client.id": "124",
|
||||
"group.id": "g1",
|
||||
"td.connect.user": "root",
|
||||
"td.connect.pass": "taosdata",
|
||||
"enable.auto.commit": "true",
|
||||
"experimental.snapshot.enable": "true",
|
||||
}
|
||||
)
|
||||
consumer.subscribe(["tmq_test_topic"])
|
||||
|
||||
consumer.subscribe([select_topic])
|
||||
consumer_rows = 0
|
||||
while True:
|
||||
res = consumer.poll(10)
|
||||
if not res:
|
||||
message = consumer.poll(timeout=1.0)
|
||||
tdLog.info(f" null {message}")
|
||||
if message:
|
||||
for block in message:
|
||||
consumer_rows += block.nrows()
|
||||
tdLog.info(f"consumer rows is {consumer_rows}")
|
||||
else:
|
||||
print("consumer has completed and break")
|
||||
break
|
||||
err = res.error()
|
||||
if err is not None:
|
||||
raise err
|
||||
val = res.value()
|
||||
|
||||
for block in val:
|
||||
print(block.fetchall())
|
||||
consumer.close()
|
||||
tdsql.query("select current,voltage,phase from test.meters where voltage >= 170;")
|
||||
all_rows = tdsql.queryRows
|
||||
if consumer_rows < all_rows - first_consumer_rows :
|
||||
tdLog.exit(f"consumer rows is {consumer_rows}, less than {all_rows - first_consumer_rows}")
|
||||
tdsql.query("show topics;")
|
||||
tdsql.checkRows(1)
|
||||
tdsql.checkRows(3)
|
||||
tdsql.execute(f"drop topic {select_topic};",queryTimes=10)
|
||||
tdsql.execute(f"drop topic {db_topic};",queryTimes=10)
|
||||
tdsql.execute(f"drop topic {stable_topic};",queryTimes=10)
|
||||
|
||||
os.system(f" LD_LIBRARY_PATH={bPath}/build/lib {bPath}/build/bin/taosBenchmark -t {tableNumbers} -n {recordNumbers2} -y ")
|
||||
tdsql.query(f"select count(*) from {stb}")
|
||||
tdsql.checkData(0,0,tableNumbers*recordNumbers2)
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
|
|
|
|||
|
|
@ -182,10 +182,10 @@ class TDTestCase:
|
|||
else:
|
||||
checkFiles("%s/*/*" % i, 0)
|
||||
|
||||
def more_than_16_disks(self):
|
||||
tdLog.info("============== more_than_16_disks test ===============")
|
||||
def more_than_128_disks(self):
|
||||
tdLog.info("============== more_than_128_disks test ===============")
|
||||
cfg={}
|
||||
for i in range(17):
|
||||
for i in range(129):
|
||||
if i == 0 :
|
||||
datadir = '/mnt/data%d 0 1' % (i+1)
|
||||
else:
|
||||
|
|
@ -272,7 +272,7 @@ class TDTestCase:
|
|||
self.dir_permission_denied()
|
||||
self.file_distribution_same_level()
|
||||
self.three_level_basic()
|
||||
self.more_than_16_disks()
|
||||
self.more_than_128_disks()
|
||||
self.trim_database()
|
||||
self.missing_middle_level()
|
||||
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ class TDTestCase:
|
|||
while True:
|
||||
res = consumer.poll(1)
|
||||
if not res:
|
||||
break
|
||||
continue
|
||||
val = res.value()
|
||||
if val is None:
|
||||
continue
|
||||
|
|
@ -173,7 +173,7 @@ class TDTestCase:
|
|||
while True:
|
||||
res = consumer.poll(1)
|
||||
if not res:
|
||||
break
|
||||
continue
|
||||
val = res.value()
|
||||
if val is None:
|
||||
continue
|
||||
|
|
@ -282,7 +282,7 @@ class TDTestCase:
|
|||
while True:
|
||||
res = consumer.poll(1)
|
||||
if not res:
|
||||
break
|
||||
continue
|
||||
val = res.value()
|
||||
if val is None:
|
||||
continue
|
||||
|
|
@ -391,7 +391,7 @@ class TDTestCase:
|
|||
while True:
|
||||
res = consumer.poll(1)
|
||||
if not res:
|
||||
break
|
||||
continue
|
||||
val = res.value()
|
||||
if val is None:
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -1866,6 +1866,29 @@ int sml_td29691_Test() {
|
|||
printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes));
|
||||
ASSERT(code == TSDB_CODE_PAR_DUPLICATED_COLUMN);
|
||||
taos_free_result(pRes);
|
||||
|
||||
//check column tag name duplication when update
|
||||
const char *sql7[] = {
|
||||
"vbin,t1=1,t2=2,f1=ewe f2=b\"hello\" 1632299372003",
|
||||
};
|
||||
pRes = taos_schemaless_insert(taos, (char **)sql7, sizeof(sql7) / sizeof(sql7[0]), TSDB_SML_LINE_PROTOCOL,
|
||||
TSDB_SML_TIMESTAMP_MILLI_SECONDS);
|
||||
code = taos_errno(pRes);
|
||||
printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes));
|
||||
ASSERT(code == TSDB_CODE_PAR_DUPLICATED_COLUMN);
|
||||
taos_free_result(pRes);
|
||||
|
||||
//check column tag name duplication when update
|
||||
const char *sql6[] = {
|
||||
"vbin,t1=1 t2=2,f1=1,f2=b\"hello\" 1632299372004",
|
||||
};
|
||||
pRes = taos_schemaless_insert(taos, (char **)sql6, sizeof(sql6) / sizeof(sql6[0]), TSDB_SML_LINE_PROTOCOL,
|
||||
TSDB_SML_TIMESTAMP_MILLI_SECONDS);
|
||||
code = taos_errno(pRes);
|
||||
printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes));
|
||||
ASSERT(code == TSDB_CODE_PAR_DUPLICATED_COLUMN);
|
||||
taos_free_result(pRes);
|
||||
|
||||
taos_close(taos);
|
||||
|
||||
return code;
|
||||
|
|
|
|||
Loading…
Reference in New Issue