Merge branch '3.0' of github.com:taosdata/TDengine into szhou/fixbugs

This commit is contained in:
slzhou 2022-09-28 16:39:33 +08:00
commit 00fc74acc9
133 changed files with 10882 additions and 4917 deletions

View File

@ -2,7 +2,7 @@
IF (DEFINED VERNUMBER) IF (DEFINED VERNUMBER)
SET(TD_VER_NUMBER ${VERNUMBER}) SET(TD_VER_NUMBER ${VERNUMBER})
ELSE () ELSE ()
SET(TD_VER_NUMBER "3.0.1.2") SET(TD_VER_NUMBER "3.0.1.3")
ENDIF () ENDIF ()
IF (DEFINED VERCOMPATIBLE) IF (DEFINED VERCOMPATIBLE)

View File

@ -2,7 +2,7 @@
# taos-tools # taos-tools
ExternalProject_Add(taos-tools ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG cf1df1c GIT_TAG 70f5a1c
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR "" BINARY_DIR ""
#BUILD_IN_SOURCE TRUE #BUILD_IN_SOURCE TRUE

View File

@ -13,7 +13,7 @@ Single row functions return a result for each row.
#### ABS #### ABS
```sql ```sql
SELECT ABS(field_name) FROM { tb_name | stb_name } [WHERE clause] ABS(expr)
``` ```
**Description**: The absolute value of a specific field. **Description**: The absolute value of a specific field.
@ -31,7 +31,7 @@ SELECT ABS(field_name) FROM { tb_name | stb_name } [WHERE clause]
#### ACOS #### ACOS
```sql ```sql
SELECT ACOS(field_name) FROM { tb_name | stb_name } [WHERE clause] ACOS(expr)
``` ```
**Description**: The arc cosine of a specific field. **Description**: The arc cosine of a specific field.
@ -49,7 +49,7 @@ SELECT ACOS(field_name) FROM { tb_name | stb_name } [WHERE clause]
#### ASIN #### ASIN
```sql ```sql
SELECT ASIN(field_name) FROM { tb_name | stb_name } [WHERE clause] ASIN(expr)
``` ```
**Description**: The arc sine of a specific field. **Description**: The arc sine of a specific field.
@ -68,7 +68,7 @@ SELECT ASIN(field_name) FROM { tb_name | stb_name } [WHERE clause]
#### ATAN #### ATAN
```sql ```sql
SELECT ATAN(field_name) FROM { tb_name | stb_name } [WHERE clause] ATAN(expr)
``` ```
**Description**: The arc tangent of a specific field. **Description**: The arc tangent of a specific field.
@ -87,7 +87,7 @@ SELECT ATAN(field_name) FROM { tb_name | stb_name } [WHERE clause]
#### CEIL #### CEIL
```sql ```sql
SELECT CEIL(field_name) FROM { tb_name | stb_name } [WHERE clause]; CEIL(expr)
``` ```
**Description**: The rounded up value of a specific field **Description**: The rounded up value of a specific field
@ -105,7 +105,7 @@ SELECT CEIL(field_name) FROM { tb_name | stb_name } [WHERE clause];
#### COS #### COS
```sql ```sql
SELECT COS(field_name) FROM { tb_name | stb_name } [WHERE clause] COS(expr)
``` ```
**Description**: The cosine of a specific field. **Description**: The cosine of a specific field.
@ -123,7 +123,7 @@ SELECT COS(field_name) FROM { tb_name | stb_name } [WHERE clause]
#### FLOOR #### FLOOR
```sql ```sql
SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause]; FLOOR(expr)
``` ```
**Description**: The rounded down value of a specific field **Description**: The rounded down value of a specific field
@ -132,7 +132,7 @@ SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause];
#### LOG #### LOG
```sql ```sql
SELECT LOG(field_name[, base]) FROM { tb_name | stb_name } [WHERE clause] LOG(expr [, base])
``` ```
**Description**: The logarithm of a specific field with `base` as the radix. If you do not enter a base, the natural logarithm of the field is returned. **Description**: The logarithm of a specific field with `base` as the radix. If you do not enter a base, the natural logarithm of the field is returned.
@ -151,7 +151,7 @@ SELECT LOG(field_name[, base]) FROM { tb_name | stb_name } [WHERE clause]
#### POW #### POW
```sql ```sql
SELECT POW(field_name, power) FROM { tb_name | stb_name } [WHERE clause] POW(expr, power)
``` ```
**Description**: The power of a specific field with `power` as the exponent. **Description**: The power of a specific field with `power` as the exponent.
@ -170,7 +170,7 @@ SELECT POW(field_name, power) FROM { tb_name | stb_name } [WHERE clause]
#### ROUND #### ROUND
```sql ```sql
SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; ROUND(expr)
``` ```
**Description**: The rounded value of a specific field. **Description**: The rounded value of a specific field.
@ -180,7 +180,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause];
#### SIN #### SIN
```sql ```sql
SELECT SIN(field_name) FROM { tb_name | stb_name } [WHERE clause] SIN(expr)
``` ```
**Description**: The sine of a specific field. **Description**: The sine of a specific field.
@ -198,7 +198,7 @@ SELECT SIN(field_name) FROM { tb_name | stb_name } [WHERE clause]
#### SQRT #### SQRT
```sql ```sql
SELECT SQRT(field_name) FROM { tb_name | stb_name } [WHERE clause] SQRT(expr)
``` ```
**Description**: The square root of a specific field. **Description**: The square root of a specific field.
@ -216,7 +216,7 @@ SELECT SQRT(field_name) FROM { tb_name | stb_name } [WHERE clause]
#### TAN #### TAN
```sql ```sql
SELECT TAN(field_name) FROM { tb_name | stb_name } [WHERE clause] TAN(expr)
``` ```
**Description**: The tangent of a specific field. **Description**: The tangent of a specific field.
@ -238,7 +238,7 @@ Concatenation functions take strings as input and produce string or numeric valu
#### CHAR_LENGTH #### CHAR_LENGTH
```sql ```sql
SELECT CHAR_LENGTH(str|column) FROM { tb_name | stb_name } [WHERE clause] CHAR_LENGTH(expr)
``` ```
**Description**: The length in number of characters of a string **Description**: The length in number of characters of a string
@ -254,7 +254,7 @@ SELECT CHAR_LENGTH(str|column) FROM { tb_name | stb_name } [WHERE clause]
#### CONCAT #### CONCAT
```sql ```sql
SELECT CONCAT(str1|column1, str2|column2, ...) FROM { tb_name | stb_name } [WHERE clause] CONCAT(expr1, expr2 [, expr] ...)
``` ```
**Description**: The concatenation result of two or more strings **Description**: The concatenation result of two or more strings
@ -271,7 +271,7 @@ SELECT CONCAT(str1|column1, str2|column2, ...) FROM { tb_name | stb_name } [WHER
#### CONCAT_WS #### CONCAT_WS
```sql ```sql
SELECT CONCAT_WS(separator, str1|column1, str2|column2, ...) FROM { tb_name | stb_name } [WHERE clause] CONCAT_WS(separator_expr, expr1, expr2 [, expr] ...)
``` ```
**Description**: The concatenation result of two or more strings with separator **Description**: The concatenation result of two or more strings with separator
@ -288,7 +288,7 @@ SELECT CONCAT_WS(separator, str1|column1, str2|column2, ...) FROM { tb_name | st
#### LENGTH #### LENGTH
```sql ```sql
SELECT LENGTH(str|column) FROM { tb_name | stb_name } [WHERE clause] LENGTH(expr)
``` ```
**Description**: The length in bytes of a string **Description**: The length in bytes of a string
@ -305,7 +305,7 @@ SELECT LENGTH(str|column) FROM { tb_name | stb_name } [WHERE clause]
#### LOWER #### LOWER
```sql ```sql
SELECT LOWER(str|column) FROM { tb_name | stb_name } [WHERE clause] LOWER(expr)
``` ```
**Description**: Convert the input string to lower case **Description**: Convert the input string to lower case
@ -322,7 +322,7 @@ SELECT LOWER(str|column) FROM { tb_name | stb_name } [WHERE clause]
#### LTRIM #### LTRIM
```sql ```sql
SELECT LTRIM(str|column) FROM { tb_name | stb_name } [WHERE clause] LTRIM(expr)
``` ```
**Description**: Remove the left leading blanks of a string **Description**: Remove the left leading blanks of a string
@ -339,7 +339,7 @@ SELECT LTRIM(str|column) FROM { tb_name | stb_name } [WHERE clause]
#### RTRIM #### RTRIM
```sql ```sql
SELECT LTRIM(str|column) FROM { tb_name | stb_name } [WHERE clause] LTRIM(expr)
``` ```
**Description**: Remove the right tailing blanks of a string **Description**: Remove the right tailing blanks of a string
@ -356,7 +356,7 @@ SELECT LTRIM(str|column) FROM { tb_name | stb_name } [WHERE clause]
#### SUBSTR #### SUBSTR
```sql ```sql
SELECT SUBSTR(str,pos[,len]) FROM { tb_name | stb_name } [WHERE clause] SUBSTR(expr, pos [, len])
``` ```
**Description**: The sub-string starting from `pos` with length of `len` from the original string `str` - If `len` is not specified, it means from `pos` to the end. **Description**: The sub-string starting from `pos` with length of `len` from the original string `str` - If `len` is not specified, it means from `pos` to the end.
@ -373,7 +373,7 @@ SELECT SUBSTR(str,pos[,len]) FROM { tb_name | stb_name } [WHERE clause]
#### UPPER #### UPPER
```sql ```sql
SELECT UPPER(str|column) FROM { tb_name | stb_name } [WHERE clause] UPPER(expr)
``` ```
**Description**: Convert the input string to upper case **Description**: Convert the input string to upper case
@ -394,10 +394,10 @@ Conversion functions change the data type of a value.
#### CAST #### CAST
```sql ```sql
SELECT CAST(expression AS type_name) FROM { tb_name | stb_name } [WHERE clause] CAST(expr AS type_name)
``` ```
**Description**: Convert the input data `expression` into the type specified by `type_name`. This function can be used only in SELECT statements. **Description**: Convert the input data `expr` into the type specified by `type_name`. This function can be used only in SELECT statements.
**Return value type**: The type specified by parameter `type_name` **Return value type**: The type specified by parameter `type_name`
@ -418,7 +418,7 @@ SELECT CAST(expression AS type_name) FROM { tb_name | stb_name } [WHERE clause]
#### TO_ISO8601 #### TO_ISO8601
```sql ```sql
SELECT TO_ISO8601(ts[, timezone]) FROM { tb_name | stb_name } [WHERE clause]; TO_ISO8601(expr [, timezone])
``` ```
**Description**: The ISO8601 date/time format converted from a UNIX timestamp, plus the timezone. You can specify any time zone with the timezone parameter. If you do not enter this parameter, the time zone on the client is used. **Description**: The ISO8601 date/time format converted from a UNIX timestamp, plus the timezone. You can specify any time zone with the timezone parameter. If you do not enter this parameter, the time zone on the client is used.
@ -441,7 +441,7 @@ SELECT TO_ISO8601(ts[, timezone]) FROM { tb_name | stb_name } [WHERE clause];
#### TO_JSON #### TO_JSON
```sql ```sql
SELECT TO_JSON(str_literal) FROM { tb_name | stb_name } [WHERE clause]; TO_JSON(str_literal)
``` ```
**Description**: Converts a string into JSON. **Description**: Converts a string into JSON.
@ -458,7 +458,7 @@ SELECT TO_JSON(str_literal) FROM { tb_name | stb_name } [WHERE clause];
#### TO_UNIXTIMESTAMP #### TO_UNIXTIMESTAMP
```sql ```sql
SELECT TO_UNIXTIMESTAMP(datetime_string) FROM { tb_name | stb_name } [WHERE clause]; TO_UNIXTIMESTAMP(expr)
``` ```
**Description**: UNIX timestamp converted from a string of date/time format **Description**: UNIX timestamp converted from a string of date/time format
@ -486,9 +486,7 @@ All functions that return the current time, such as `NOW`, `TODAY`, and `TIMEZON
#### NOW #### NOW
```sql ```sql
SELECT NOW() FROM { tb_name | stb_name } [WHERE clause]; NOW()
SELECT select_expr FROM { tb_name | stb_name } WHERE ts_col cond_operatior NOW();
INSERT INTO tb_name VALUES (NOW(), ...);
``` ```
**Description**: The current time of the client side system **Description**: The current time of the client side system
@ -511,7 +509,7 @@ INSERT INTO tb_name VALUES (NOW(), ...);
#### TIMEDIFF #### TIMEDIFF
```sql ```sql
SELECT TIMEDIFF(ts | datetime_string1, ts | datetime_string2 [, time_unit]) FROM { tb_name | stb_name } [WHERE clause]; TIMEDIFF(expr1, expr2 [, time_unit])
``` ```
**Description**: The difference between two timestamps, and rounded to the time unit specified by `time_unit` **Description**: The difference between two timestamps, and rounded to the time unit specified by `time_unit`
@ -534,7 +532,7 @@ SELECT TIMEDIFF(ts | datetime_string1, ts | datetime_string2 [, time_unit]) FROM
#### TIMETRUNCATE #### TIMETRUNCATE
```sql ```sql
SELECT TIMETRUNCATE(ts | datetime_string , time_unit) FROM { tb_name | stb_name } [WHERE clause]; TIMETRUNCATE(expr, time_unit)
``` ```
**Description**: Truncate the input timestamp with unit specified by `time_unit` **Description**: Truncate the input timestamp with unit specified by `time_unit`
@ -555,7 +553,7 @@ SELECT TIMETRUNCATE(ts | datetime_string , time_unit) FROM { tb_name | stb_name
#### TIMEZONE #### TIMEZONE
```sql ```sql
SELECT TIMEZONE() FROM { tb_name | stb_name } [WHERE clause]; TIMEZONE()
``` ```
**Description**: The timezone of the client side system **Description**: The timezone of the client side system
@ -570,9 +568,7 @@ SELECT TIMEZONE() FROM { tb_name | stb_name } [WHERE clause];
#### TODAY #### TODAY
```sql ```sql
SELECT TODAY() FROM { tb_name | stb_name } [WHERE clause]; TODAY()
SELECT select_expr FROM { tb_name | stb_name } WHERE ts_col cond_operatior TODAY()];
INSERT INTO tb_name VALUES (TODAY(), ...);
``` ```
**Description**: The timestamp of 00:00:00 of the client side system **Description**: The timestamp of 00:00:00 of the client side system
@ -599,7 +595,12 @@ TDengine supports the following aggregate functions:
### APERCENTILE ### APERCENTILE
```sql ```sql
SELECT APERCENTILE(field_name, P[, algo_type]) FROM { tb_name | stb_name } [WHERE clause] APERCENTILE(expr, p [, algo_type])
algo_type: {
"default"
| "t-digest"
}
``` ```
**Description**: Similar to `PERCENTILE`, but a simulated result is returned **Description**: Similar to `PERCENTILE`, but a simulated result is returned
@ -611,14 +612,14 @@ SELECT APERCENTILE(field_name, P[, algo_type]) FROM { tb_name | stb_name } [WHER
**Applicable table types**: standard tables and supertables **Applicable table types**: standard tables and supertables
**Explanations** **Explanations**
- _P_ is in range [0,100], when _P_ is 0, the result is same as using function MIN; when _P_ is 100, the result is same as function MAX. - _p_ is in range [0,100], when _p_ is 0, the result is same as using function MIN; when _p_ is 100, the result is same as function MAX.
- `algo_type` can only be input as `default` or `t-digest` Enter `default` to use a histogram-based algorithm. Enter `t-digest` to use the t-digest algorithm to calculate the approximation of the quantile. `default` is used by default. - `algo_type` can only be input as `default` or `t-digest` Enter `default` to use a histogram-based algorithm. Enter `t-digest` to use the t-digest algorithm to calculate the approximation of the quantile. `default` is used by default.
- The approximation result of `t-digest` algorithm is sensitive to input data order. For example, when querying STable with different input data order there might be minor differences in calculated results. - The approximation result of `t-digest` algorithm is sensitive to input data order. For example, when querying STable with different input data order there might be minor differences in calculated results.
### AVG ### AVG
```sql ```sql
SELECT AVG(field_name) FROM tb_name [WHERE clause]; AVG(expr)
``` ```
**Description**: The average value of the specified fields. **Description**: The average value of the specified fields.
@ -633,7 +634,7 @@ SELECT AVG(field_name) FROM tb_name [WHERE clause];
### COUNT ### COUNT
```sql ```sql
SELECT COUNT([*|field_name]) FROM tb_name [WHERE clause]; COUNT({* | expr})
``` ```
**Description**: The number of records in the specified fields. **Description**: The number of records in the specified fields.
@ -653,7 +654,7 @@ If you input a specific column, the number of non-null values in the column is r
### ELAPSED ### ELAPSED
```sql ```sql
SELECT ELAPSED(ts_primary_key [, time_unit]) FROM { tb_name | stb_name } [WHERE clause] [INTERVAL(interval [, offset]) [SLIDING sliding]]; ELAPSED(ts_primary_key [, time_unit])
``` ```
**Description**`elapsed` function can be used to calculate the continuous time length in which there is valid data. If it's used with `INTERVAL` clause, the returned result is the calcualted time length within each time window. If it's used without `INTERVAL` caluse, the returned result is the calculated time length within the specified time range. Please be noted that the return value of `elapsed` is the number of `time_unit` in the calculated time length. **Description**`elapsed` function can be used to calculate the continuous time length in which there is valid data. If it's used with `INTERVAL` clause, the returned result is the calcualted time length within each time window. If it's used without `INTERVAL` caluse, the returned result is the calculated time length within the specified time range. Please be noted that the return value of `elapsed` is the number of `time_unit` in the calculated time length.
@ -665,7 +666,7 @@ SELECT ELAPSED(ts_primary_key [, time_unit]) FROM { tb_name | stb_name } [WHERE
**Applicable tables**: table, STable, outter in nested query **Applicable tables**: table, STable, outter in nested query
**Explanations** **Explanations**
- `field_name` parameter can only be the first column of a table, i.e. timestamp primary key. - `ts_primary_key` parameter can only be the first column of a table, i.e. timestamp primary key.
- The minimum value of `time_unit` is the time precision of the database. If `time_unit` is not specified, the time precision of the database is used as the default time unit. Time unit specified by `time_unit` can be: - The minimum value of `time_unit` is the time precision of the database. If `time_unit` is not specified, the time precision of the database is used as the default time unit. Time unit specified by `time_unit` can be:
1b (nanoseconds), 1u (microseconds), 1a (milliseconds), 1s (seconds), 1m (minutes), 1h (hours), 1d (days), or 1w (weeks) 1b (nanoseconds), 1u (microseconds), 1a (milliseconds), 1s (seconds), 1m (minutes), 1h (hours), 1d (days), or 1w (weeks)
- It can be used with `INTERVAL` to get the time valid time length of each time window. Please be noted that the return value is same as the time window for all time windows except for the first and the last time window. - It can be used with `INTERVAL` to get the time valid time length of each time window. Please be noted that the return value is same as the time window for all time windows except for the first and the last time window.
@ -679,7 +680,7 @@ SELECT ELAPSED(ts_primary_key [, time_unit]) FROM { tb_name | stb_name } [WHERE
### LEASTSQUARES ### LEASTSQUARES
```sql ```sql
SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause]; LEASTSQUARES(expr, start_val, step_val)
``` ```
**Description**: The linear regression function of the specified column and the timestamp column (primary key), `start_val` is the initial value and `step_val` is the step value. **Description**: The linear regression function of the specified column and the timestamp column (primary key), `start_val` is the initial value and `step_val` is the step value.
@ -694,7 +695,7 @@ SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause]
### SPREAD ### SPREAD
```sql ```sql
SELECT SPREAD(field_name) FROM { tb_name | stb_name } [WHERE clause]; SPREAD(expr)
``` ```
**Description**: The difference between the max and the min of a specific column **Description**: The difference between the max and the min of a specific column
@ -709,7 +710,7 @@ SELECT SPREAD(field_name) FROM { tb_name | stb_name } [WHERE clause];
### STDDEV ### STDDEV
```sql ```sql
SELECT STDDEV(field_name) FROM tb_name [WHERE clause]; STDDEV(expr)
``` ```
**Description**: Standard deviation of a specific column in a table or STable **Description**: Standard deviation of a specific column in a table or STable
@ -724,7 +725,7 @@ SELECT STDDEV(field_name) FROM tb_name [WHERE clause];
### SUM ### SUM
```sql ```sql
SELECT SUM(field_name) FROM tb_name [WHERE clause]; SUM(expr)
``` ```
**Description**: The sum of a specific column in a table or STable **Description**: The sum of a specific column in a table or STable
@ -739,7 +740,7 @@ SELECT SUM(field_name) FROM tb_name [WHERE clause];
### HYPERLOGLOG ### HYPERLOGLOG
```sql ```sql
SELECT HYPERLOGLOG(field_name) FROM { tb_name | stb_name } [WHERE clause]; HYPERLOGLOG(expr)
``` ```
**Description** **Description**
@ -756,7 +757,7 @@ SELECT HYPERLOGLOG(field_name) FROM { tb_name | stb_name } [WHERE clause];
### HISTOGRAM ### HISTOGRAM
```sql ```sql
SELECT HISTOGRAM(field_namebin_type, bin_description, normalized) FROM tb_name [WHERE clause]; HISTOGRAM(exprbin_type, bin_description, normalized)
``` ```
**Description**Returns count of data points in user-specified ranges. **Description**Returns count of data points in user-specified ranges.
@ -786,7 +787,7 @@ SELECT HISTOGRAM(field_namebin_type, bin_description, normalized) FROM tb_nam
### PERCENTILE ### PERCENTILE
```sql ```sql
SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause]; PERCENTILE(expr, p)
``` ```
**Description**: The value whose rank in a specific column matches the specified percentage. If such a value matching the specified percentage doesn't exist in the column, an interpolation value will be returned. **Description**: The value whose rank in a specific column matches the specified percentage. If such a value matching the specified percentage doesn't exist in the column, an interpolation value will be returned.
@ -797,7 +798,7 @@ SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause];
**Applicable table types**: table only **Applicable table types**: table only
**More explanations**: _P_ is in range [0,100], when _P_ is 0, the result is same as using function MIN; when _P_ is 100, the result is same as function MAX. **More explanations**: _p_ is in range [0,100], when _p_ is 0, the result is same as using function MIN; when _p_ is 100, the result is same as function MAX.
## Selection Functions ## Selection Functions
@ -807,7 +808,7 @@ Selection functions return one or more results depending. You can specify the ti
### BOTTOM ### BOTTOM
```sql ```sql
SELECT BOTTOM(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; BOTTOM(expr, k)
``` ```
**Description**: The least _k_ values of a specific column in a table or STable. If a value has multiple occurrences in the column but counting all of them in will exceed the upper limit _k_, then a part of them will be returned randomly. **Description**: The least _k_ values of a specific column in a table or STable. If a value has multiple occurrences in the column but counting all of them in will exceed the upper limit _k_, then a part of them will be returned randomly.
@ -827,7 +828,7 @@ SELECT BOTTOM(field_name, K) FROM { tb_name | stb_name } [WHERE clause];
### FIRST ### FIRST
```sql ```sql
SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause]; FIRST(expr)
``` ```
**Description**: The first non-null value of a specific column in a table or STable **Description**: The first non-null value of a specific column in a table or STable
@ -847,7 +848,7 @@ SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause];
### INTERP ### INTERP
```sql ```sql
SELECT INTERP(field_name) FROM { tb_name | stb_name } [WHERE where_condition] RANGE(timestamp1,timestamp2) EVERY(interval) FILL({ VALUE | PREV | NULL | LINEAR | NEXT}); INTERP(expr)
``` ```
**Description**: The value that matches the specified timestamp range is returned, if existing; or an interpolation value is returned. **Description**: The value that matches the specified timestamp range is returned, if existing; or an interpolation value is returned.
@ -866,11 +867,12 @@ SELECT INTERP(field_name) FROM { tb_name | stb_name } [WHERE where_condition] RA
- The number of rows in the result set of `INTERP` is determined by the parameter `EVERY`. Starting from timestamp1, one interpolation is performed for every time interval specified `EVERY` parameter. - The number of rows in the result set of `INTERP` is determined by the parameter `EVERY`. Starting from timestamp1, one interpolation is performed for every time interval specified `EVERY` parameter.
- Interpolation is performed based on `FILL` parameter. - Interpolation is performed based on `FILL` parameter.
- `INTERP` can only be used to interpolate in single timeline. So it must be used with `partition by tbname` when it's used on a STable. - `INTERP` can only be used to interpolate in single timeline. So it must be used with `partition by tbname` when it's used on a STable.
- Pseudo column `_irowts` can be used along with `INTERP` to return the timestamps associated with interpolation points(support after version 3.0.1.4).
### LAST ### LAST
```sql ```sql
SELECT LAST(field_name) FROM { tb_name | stb_name } [WHERE clause]; LAST(expr)
``` ```
**Description**: The last non-NULL value of a specific column in a table or STable **Description**: The last non-NULL value of a specific column in a table or STable
@ -891,7 +893,7 @@ SELECT LAST(field_name) FROM { tb_name | stb_name } [WHERE clause];
### LAST_ROW ### LAST_ROW
```sql ```sql
SELECT LAST_ROW(field_name) FROM { tb_name | stb_name }; LAST_ROW(expr)
``` ```
**Description**: The last row of a table or STable **Description**: The last row of a table or STable
@ -910,7 +912,7 @@ SELECT LAST_ROW(field_name) FROM { tb_name | stb_name };
### MAX ### MAX
```sql ```sql
SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause]; MAX(expr)
``` ```
**Description**: The maximum value of a specific column of a table or STable **Description**: The maximum value of a specific column of a table or STable
@ -925,7 +927,7 @@ SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause];
### MIN ### MIN
```sql ```sql
SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause]; MIN(expr)
``` ```
**Description**: The minimum value of a specific column in a table or STable **Description**: The minimum value of a specific column in a table or STable
@ -940,7 +942,7 @@ SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause];
### MODE ### MODE
```sql ```sql
SELECT MODE(field_name) FROM tb_name [WHERE clause]; MODE(expr)
``` ```
**Description**:The value which has the highest frequency of occurrence. NULL is returned if there are multiple values which have highest frequency of occurrence. **Description**:The value which has the highest frequency of occurrence. NULL is returned if there are multiple values which have highest frequency of occurrence.
@ -955,7 +957,7 @@ SELECT MODE(field_name) FROM tb_name [WHERE clause];
### SAMPLE ### SAMPLE
```sql ```sql
SELECT SAMPLE(field_name, K) FROM { tb_name | stb_name } [WHERE clause] SAMPLE(expr, k)
``` ```
**Description**: _k_ sampling values of a specific column. The applicable range of _k_ is [1,1000]. **Description**: _k_ sampling values of a specific column. The applicable range of _k_ is [1,1000].
@ -977,7 +979,7 @@ This function cannot be used in expression calculation.
### TAIL ### TAIL
```sql ```sql
SELECT TAIL(field_name, k, offset_val) FROM {tb_name | stb_name} [WHERE clause]; TAIL(expr, k, offset_val)
``` ```
**Description**: The next _k_ rows are returned after skipping the last `offset_val` rows, NULL values are not ignored. `offset_val` is optional parameter. When it's not specified, the last _k_ rows are returned. When `offset_val` is used, the effect is same as `order by ts desc LIMIT k OFFSET offset_val`. **Description**: The next _k_ rows are returned after skipping the last `offset_val` rows, NULL values are not ignored. `offset_val` is optional parameter. When it's not specified, the last _k_ rows are returned. When `offset_val` is used, the effect is same as `order by ts desc LIMIT k OFFSET offset_val`.
@ -994,7 +996,7 @@ SELECT TAIL(field_name, k, offset_val) FROM {tb_name | stb_name} [WHERE clause];
### TOP ### TOP
```sql ```sql
SELECT TOP(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; TOP(expr, k)
``` ```
**Description**: The greatest _k_ values of a specific column in a table or STable. If a value has multiple occurrences in the column but counting all of them in will exceed the upper limit _k_, then a part of them will be returned randomly. **Description**: The greatest _k_ values of a specific column in a table or STable. If a value has multiple occurrences in the column but counting all of them in will exceed the upper limit _k_, then a part of them will be returned randomly.
@ -1014,7 +1016,7 @@ SELECT TOP(field_name, K) FROM { tb_name | stb_name } [WHERE clause];
### UNIQUE ### UNIQUE
```sql ```sql
SELECT UNIQUE(field_name) FROM {tb_name | stb_name} [WHERE clause]; UNIQUE(expr)
``` ```
**Description**: The values that occur the first time in the specified column. The effect is similar to `distinct` keyword, but it can also be used to match tags or timestamp. The first occurrence of a timestamp or tag is used. **Description**: The values that occur the first time in the specified column. The effect is similar to `distinct` keyword, but it can also be used to match tags or timestamp. The first occurrence of a timestamp or tag is used.
@ -1033,7 +1035,7 @@ TDengine includes extensions to standard SQL that are intended specifically for
### CSUM ### CSUM
```sql ```sql
SELECT CSUM(field_name) FROM { tb_name | stb_name } [WHERE clause] CSUM(expr)
``` ```
**Description**: The cumulative sum of each row for a specific column. The number of output rows is same as that of the input rows. **Description**: The cumulative sum of each row for a specific column. The number of output rows is same as that of the input rows.
@ -1056,7 +1058,12 @@ SELECT CSUM(field_name) FROM { tb_name | stb_name } [WHERE clause]
### DERIVATIVE ### DERIVATIVE
```sql ```sql
SELECT DERIVATIVE(field_name, time_interval, ignore_negative) FROM tb_name [WHERE clause]; DERIVATIVE(expr, time_inerval, ignore_negative)
ignore_negative: {
0
| 1
}
``` ```
**Description**: The derivative of a specific column. The time rage can be specified by parameter `time_interval`, the minimum allowed time range is 1 second (1s); the value of `ignore_negative` can be 0 or 1, 1 means negative values are ignored. **Description**: The derivative of a specific column. The time rage can be specified by parameter `time_interval`, the minimum allowed time range is 1 second (1s); the value of `ignore_negative` can be 0 or 1, 1 means negative values are ignored.
@ -1075,7 +1082,12 @@ SELECT DERIVATIVE(field_name, time_interval, ignore_negative) FROM tb_name [WHER
### DIFF ### DIFF
```sql ```sql
SELECT {DIFF(field_name, ignore_negative) | DIFF(field_name)} FROM tb_name [WHERE clause]; DIFF(expr [, ignore_negative])
ignore_negative: {
0
| 1
}
``` ```
**Description**: The different of each row with its previous row for a specific column. `ignore_negative` can be specified as 0 or 1, the default value is 1 if it's not specified. `1` means negative values are ignored. **Description**: The different of each row with its previous row for a specific column. `ignore_negative` can be specified as 0 or 1, the default value is 1 if it's not specified. `1` means negative values are ignored.
@ -1095,7 +1107,7 @@ SELECT {DIFF(field_name, ignore_negative) | DIFF(field_name)} FROM tb_name [WHER
### IRATE ### IRATE
```sql ```sql
SELECT IRATE(field_name) FROM tb_name WHERE clause; IRATE(expr)
``` ```
**Description**: instantaneous rate on a specific column. The last two samples in the specified time range are used to calculate instantaneous rate. If the last sample value is smaller, then only the last sample value is used instead of the difference between the last two sample values. **Description**: instantaneous rate on a specific column. The last two samples in the specified time range are used to calculate instantaneous rate. If the last sample value is smaller, then only the last sample value is used instead of the difference between the last two sample values.
@ -1110,7 +1122,7 @@ SELECT IRATE(field_name) FROM tb_name WHERE clause;
### MAVG ### MAVG
```sql ```sql
SELECT MAVG(field_name, K) FROM { tb_name | stb_name } [WHERE clause] MAVG(expr, k)
``` ```
**Description**: The moving average of continuous _k_ values of a specific column. If the number of input rows is less than _k_, nothing is returned. The applicable range of _k_ is [1,1000]. **Description**: The moving average of continuous _k_ values of a specific column. If the number of input rows is less than _k_, nothing is returned. The applicable range of _k_ is [1,1000].
@ -1133,7 +1145,7 @@ SELECT MAVG(field_name, K) FROM { tb_name | stb_name } [WHERE clause]
### STATECOUNT ### STATECOUNT
```sql ```sql
SELECT STATECOUNT(field_name, oper, val) FROM { tb_name | stb_name } [WHERE clause]; STATECOUNT(expr, oper, val)
``` ```
**Description**: The number of continuous rows satisfying the specified conditions for a specific column. The result is shown as an extra column for each row. If the specified condition is evaluated as true, the number is increased by 1; otherwise the number is reset to -1. If the input value is NULL, then the corresponding row is skipped. **Description**: The number of continuous rows satisfying the specified conditions for a specific column. The result is shown as an extra column for each row. If the specified condition is evaluated as true, the number is increased by 1; otherwise the number is reset to -1. If the input value is NULL, then the corresponding row is skipped.
@ -1160,7 +1172,7 @@ SELECT STATECOUNT(field_name, oper, val) FROM { tb_name | stb_name } [WHERE clau
### STATEDURATION ### STATEDURATION
```sql ```sql
SELECT stateDuration(field_name, oper, val, unit) FROM { tb_name | stb_name } [WHERE clause]; STATEDURATION(expr, oper, val, unit)
``` ```
**Description**: The length of time range in which all rows satisfy the specified condition for a specific column. The result is shown as an extra column for each row. The length for the first row that satisfies the condition is 0. Next, if the condition is evaluated as true for a row, the time interval between current row and its previous row is added up to the time range; otherwise the time range length is reset to -1. If the value of the column is NULL, the corresponding row is skipped. **Description**: The length of time range in which all rows satisfy the specified condition for a specific column. The result is shown as an extra column for each row. The length for the first row that satisfies the condition is 0. Next, if the condition is evaluated as true for a row, the time interval between current row and its previous row is added up to the time range; otherwise the time range length is reset to -1. If the value of the column is NULL, the corresponding row is skipped.
@ -1188,7 +1200,7 @@ SELECT stateDuration(field_name, oper, val, unit) FROM { tb_name | stb_name } [W
### TWA ### TWA
```sql ```sql
SELECT TWA(field_name) FROM tb_name WHERE clause; TWA(expr)
``` ```
**Description**: Time weighted average on a specific column within a time range **Description**: Time weighted average on a specific column within a time range

View File

@ -11,7 +11,15 @@ TDengine includes a built-in database named `INFORMATION_SCHEMA` to provide acce
4. Future versions of TDengine can add new columns to INFORMATION_SCHEMA tables without affecting existing business systems. 4. Future versions of TDengine can add new columns to INFORMATION_SCHEMA tables without affecting existing business systems.
5. It is easier for users coming from other database management systems. For example, Oracle users can query data dictionary tables. 5. It is easier for users coming from other database management systems. For example, Oracle users can query data dictionary tables.
Note: SHOW statements are still supported for the convenience of existing users. :::info
- SHOW statements are still supported for the convenience of existing users.
- Some columns in the system table may be keywords, and you need to use the escape character '\`' when querying, for example, to query the VGROUPS in the database `test`:
```sql
select `vgroups` from ins_databases where name = 'test';
```
:::
This document introduces the tables of INFORMATION_SCHEMA and their structure. This document introduces the tables of INFORMATION_SCHEMA and their structure.
@ -102,7 +110,11 @@ Provides information about user-created databases. Similar to SHOW DATABASES.
| 24 | wal_retention_period | INT | WAL retention period | | 24 | wal_retention_period | INT | WAL retention period |
| 25 | wal_retention_size | INT | Maximum WAL size | | 25 | wal_retention_size | INT | Maximum WAL size |
| 26 | wal_roll_period | INT | WAL rotation period | | 26 | wal_roll_period | INT | WAL rotation period |
| 27 | wal_segment_size | WAL file size | | 27 | wal_segment_size | BIGINT | WAL file size |
| 28 | stt_trigger | SMALLINT | The threshold for number of files to trigger file merging |
| 29 | table_prefix | SMALLINT | The prefix length in the table name that is ignored when distributing table to vnode based on table name |
| 30 | table_suffix | SMALLINT | The suffix length in the table name that is ignored when distributing table to vnode based on table name |
| 31 | tsdb_pagesize | INT | The page size for internal storage engine, its unit is KB |
## INS_FUNCTIONS ## INS_FUNCTIONS

View File

@ -177,12 +177,21 @@ The parameters described in this document by the effect that they have on the sy
### maxNumOfDistinctRes ### maxNumOfDistinctRes
| Attribute | Description | | Attribute | Description |
| -------- | -------------------------------- | --- | | -------- | -------------------------------- |
| Applicable | Server Only | | Applicable | Server Only |
| Meaning | The maximum number of distinct rows returned | | Meaning | The maximum number of distinct rows returned |
| Value Range | [100,000 - 100,000,000] | | Value Range | [100,000 - 100,000,000] |
| Default Value | 100,000 | | Default Value | 100,000 |
### keepColumnName
| Attribute | Description |
| -------- | -------------------------------- |
| Applicable | Client only |
| Meaning | When the Last, First, LastRow function is queried, whether the returned column name contains the function name. |
| Value Range | 0 means including the function name, 1 means not including the function name. |
| Default Value | 0 |
## Locale Parameters ## Locale Parameters
### timezone ### timezone

View File

@ -6,6 +6,10 @@ description: TDengine release history, Release Notes and download links.
import Release from "/components/ReleaseV3"; import Release from "/components/ReleaseV3";
## 3.0.1.3
<Release type="tdengine" version="3.0.1.3" />
## 3.0.1.2 ## 3.0.1.2
<Release type="tdengine" version="3.0.1.2" /> <Release type="tdengine" version="3.0.1.2" />

View File

@ -6,6 +6,10 @@ description: taosTools release history, Release Notes, download links.
import Release from "/components/ReleaseV3"; import Release from "/components/ReleaseV3";
## 2.2.3
<Release type="tools" version="2.2.3" />
## 2.2.2 ## 2.2.2
<Release type="tools" version="2.2.2" /> <Release type="tools" version="2.2.2" />

View File

@ -14,7 +14,7 @@ toc_max_heading_level: 4
#### ABS #### ABS
```sql ```sql
SELECT ABS(field_name) FROM { tb_name | stb_name } [WHERE clause] ABS(expr)
``` ```
**功能说明**:获得指定字段的绝对值。 **功能说明**:获得指定字段的绝对值。
@ -32,7 +32,7 @@ SELECT ABS(field_name) FROM { tb_name | stb_name } [WHERE clause]
#### ACOS #### ACOS
```sql ```sql
SELECT ACOS(field_name) FROM { tb_name | stb_name } [WHERE clause] ACOS(expr)
``` ```
**功能说明**:获得指定字段的反余弦结果。 **功能说明**:获得指定字段的反余弦结果。
@ -50,7 +50,7 @@ SELECT ACOS(field_name) FROM { tb_name | stb_name } [WHERE clause]
#### ASIN #### ASIN
```sql ```sql
SELECT ASIN(field_name) FROM { tb_name | stb_name } [WHERE clause] ASIN(expr)
``` ```
**功能说明**:获得指定字段的反正弦结果。 **功能说明**:获得指定字段的反正弦结果。
@ -69,7 +69,7 @@ SELECT ASIN(field_name) FROM { tb_name | stb_name } [WHERE clause]
#### ATAN #### ATAN
```sql ```sql
SELECT ATAN(field_name) FROM { tb_name | stb_name } [WHERE clause] ATAN(expr)
``` ```
**功能说明**:获得指定字段的反正切结果。 **功能说明**:获得指定字段的反正切结果。
@ -88,7 +88,7 @@ SELECT ATAN(field_name) FROM { tb_name | stb_name } [WHERE clause]
#### CEIL #### CEIL
```sql ```sql
SELECT CEIL(field_name) FROM { tb_name | stb_name } [WHERE clause]; CEIL(expr)
``` ```
**功能说明**:获得指定字段的向上取整数的结果。 **功能说明**:获得指定字段的向上取整数的结果。
@ -106,7 +106,7 @@ SELECT CEIL(field_name) FROM { tb_name | stb_name } [WHERE clause];
#### COS #### COS
```sql ```sql
SELECT COS(field_name) FROM { tb_name | stb_name } [WHERE clause] COS(expr)
``` ```
**功能说明**:获得指定字段的余弦结果。 **功能说明**:获得指定字段的余弦结果。
@ -124,7 +124,7 @@ SELECT COS(field_name) FROM { tb_name | stb_name } [WHERE clause]
#### FLOOR #### FLOOR
```sql ```sql
SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause]; FLOOR(expr)
``` ```
**功能说明**:获得指定字段的向下取整数的结果。 **功能说明**:获得指定字段的向下取整数的结果。
@ -133,10 +133,10 @@ SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause];
#### LOG #### LOG
```sql ```sql
SELECT LOG(field_name[, base]) FROM { tb_name | stb_name } [WHERE clause] LOG(expr1[, expr2])
``` ```
**功能说明**:获得指定字段对于底数 base 的对数。如果 base 参数省略,则返回指定字段的自然对数值。 **功能说明**:获得 expr1 对于底数 expr2 的对数。如果 expr2 参数省略,则返回指定字段的自然对数值。
**返回结果类型**DOUBLE。 **返回结果类型**DOUBLE。
@ -152,10 +152,10 @@ SELECT LOG(field_name[, base]) FROM { tb_name | stb_name } [WHERE clause]
#### POW #### POW
```sql ```sql
SELECT POW(field_name, power) FROM { tb_name | stb_name } [WHERE clause] POW(expr1, expr2)
``` ```
**功能说明**:获得指定字段的指数为 power 的幂。 **功能说明**:获得 expr1 的指数为 expr2 的幂。
**返回结果类型**DOUBLE。 **返回结果类型**DOUBLE。
@ -171,7 +171,7 @@ SELECT POW(field_name, power) FROM { tb_name | stb_name } [WHERE clause]
#### ROUND #### ROUND
```sql ```sql
SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; ROUND(expr)
``` ```
**功能说明**:获得指定字段的四舍五入的结果。 **功能说明**:获得指定字段的四舍五入的结果。
@ -181,7 +181,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause];
#### SIN #### SIN
```sql ```sql
SELECT SIN(field_name) FROM { tb_name | stb_name } [WHERE clause] SIN(expr)
``` ```
**功能说明**:获得指定字段的正弦结果。 **功能说明**:获得指定字段的正弦结果。
@ -199,7 +199,7 @@ SELECT SIN(field_name) FROM { tb_name | stb_name } [WHERE clause]
#### SQRT #### SQRT
```sql ```sql
SELECT SQRT(field_name) FROM { tb_name | stb_name } [WHERE clause] SQRT(expr)
``` ```
**功能说明**:获得指定字段的平方根。 **功能说明**:获得指定字段的平方根。
@ -217,7 +217,7 @@ SELECT SQRT(field_name) FROM { tb_name | stb_name } [WHERE clause]
#### TAN #### TAN
```sql ```sql
SELECT TAN(field_name) FROM { tb_name | stb_name } [WHERE clause] TAN(expr)
``` ```
**功能说明**:获得指定字段的正切结果。 **功能说明**:获得指定字段的正切结果。
@ -239,7 +239,7 @@ SELECT TAN(field_name) FROM { tb_name | stb_name } [WHERE clause]
#### CHAR_LENGTH #### CHAR_LENGTH
```sql ```sql
SELECT CHAR_LENGTH(str|column) FROM { tb_name | stb_name } [WHERE clause] CHAR_LENGTH(expr)
``` ```
**功能说明**:以字符计数的字符串长度。 **功能说明**:以字符计数的字符串长度。
@ -255,7 +255,7 @@ SELECT CHAR_LENGTH(str|column) FROM { tb_name | stb_name } [WHERE clause]
#### CONCAT #### CONCAT
```sql ```sql
SELECT CONCAT(str1|column1, str2|column2, ...) FROM { tb_name | stb_name } [WHERE clause] CONCAT(expr1, expr2 [, expr] ... )
``` ```
**功能说明**:字符串连接函数。 **功能说明**:字符串连接函数。
@ -272,7 +272,7 @@ SELECT CONCAT(str1|column1, str2|column2, ...) FROM { tb_name | stb_name } [WHER
#### CONCAT_WS #### CONCAT_WS
```sql ```sql
SELECT CONCAT_WS(separator, str1|column1, str2|column2, ...) FROM { tb_name | stb_name } [WHERE clause] CONCAT_WS(separator_expr, expr1, expr2 [, expr] ...)
``` ```
**功能说明**:带分隔符的字符串连接函数。 **功能说明**:带分隔符的字符串连接函数。
@ -289,7 +289,7 @@ SELECT CONCAT_WS(separator, str1|column1, str2|column2, ...) FROM { tb_name | st
#### LENGTH #### LENGTH
```sql ```sql
SELECT LENGTH(str|column) FROM { tb_name | stb_name } [WHERE clause] LENGTH(expr)
``` ```
**功能说明**:以字节计数的字符串长度。 **功能说明**:以字节计数的字符串长度。
@ -306,7 +306,7 @@ SELECT LENGTH(str|column) FROM { tb_name | stb_name } [WHERE clause]
#### LOWER #### LOWER
```sql ```sql
SELECT LOWER(str|column) FROM { tb_name | stb_name } [WHERE clause] LOWER(expr)
``` ```
**功能说明**:将字符串参数值转换为全小写字母。 **功能说明**:将字符串参数值转换为全小写字母。
@ -323,7 +323,7 @@ SELECT LOWER(str|column) FROM { tb_name | stb_name } [WHERE clause]
#### LTRIM #### LTRIM
```sql ```sql
SELECT LTRIM(str|column) FROM { tb_name | stb_name } [WHERE clause] LTRIM(expr)
``` ```
**功能说明**:返回清除左边空格后的字符串。 **功能说明**:返回清除左边空格后的字符串。
@ -340,7 +340,7 @@ SELECT LTRIM(str|column) FROM { tb_name | stb_name } [WHERE clause]
#### RTRIM #### RTRIM
```sql ```sql
SELECT LTRIM(str|column) FROM { tb_name | stb_name } [WHERE clause] LTRIM(expr)
``` ```
**功能说明**:返回清除右边空格后的字符串。 **功能说明**:返回清除右边空格后的字符串。
@ -357,7 +357,7 @@ SELECT LTRIM(str|column) FROM { tb_name | stb_name } [WHERE clause]
#### SUBSTR #### SUBSTR
```sql ```sql
SELECT SUBSTR(str,pos[,len]) FROM { tb_name | stb_name } [WHERE clause] SUBSTR(expr, pos [,len])
``` ```
**功能说明**:从源字符串 str 中的指定位置 pos 开始取一个长度为 len 的子串并返回。如果输入参数 len 被忽略,返回的子串包含从 pos 开始的整个字串。 **功能说明**:从源字符串 str 中的指定位置 pos 开始取一个长度为 len 的子串并返回。如果输入参数 len 被忽略,返回的子串包含从 pos 开始的整个字串。
@ -374,7 +374,7 @@ SELECT SUBSTR(str,pos[,len]) FROM { tb_name | stb_name } [WHERE clause]
#### UPPER #### UPPER
```sql ```sql
SELECT UPPER(str|column) FROM { tb_name | stb_name } [WHERE clause] UPPER(expr)
``` ```
**功能说明**:将字符串参数值转换为全大写字母。 **功能说明**:将字符串参数值转换为全大写字母。
@ -395,10 +395,10 @@ SELECT UPPER(str|column) FROM { tb_name | stb_name } [WHERE clause]
#### CAST #### CAST
```sql ```sql
SELECT CAST(expression AS type_name) FROM { tb_name | stb_name } [WHERE clause] CAST(expr AS type_name)
``` ```
**功能说明**:数据类型转换函数,返回 expression 转换为 type_name 指定的类型后的结果。只适用于 select 子句中。 **功能说明**:数据类型转换函数,返回 expr 转换为 type_name 指定的类型后的结果。只适用于 select 子句中。
**返回结果类型**CAST 中指定的类型type_name)。 **返回结果类型**CAST 中指定的类型type_name)。
@ -419,7 +419,7 @@ SELECT CAST(expression AS type_name) FROM { tb_name | stb_name } [WHERE clause]
#### TO_ISO8601 #### TO_ISO8601
```sql ```sql
SELECT TO_ISO8601(ts[, timezone]) FROM { tb_name | stb_name } [WHERE clause]; TO_ISO8601(expr [, timezone])
``` ```
**功能说明**:将 UNIX 时间戳转换成为 ISO8601 标准的日期时间格式并附加时区信息。timezone 参数允许用户为输出结果指定附带任意时区信息。如果 timezone 参数省略,输出结果则附带当前客户端的系统时区信息。 **功能说明**:将 UNIX 时间戳转换成为 ISO8601 标准的日期时间格式并附加时区信息。timezone 参数允许用户为输出结果指定附带任意时区信息。如果 timezone 参数省略,输出结果则附带当前客户端的系统时区信息。
@ -442,7 +442,7 @@ SELECT TO_ISO8601(ts[, timezone]) FROM { tb_name | stb_name } [WHERE clause];
#### TO_JSON #### TO_JSON
```sql ```sql
SELECT TO_JSON(str_literal) FROM { tb_name | stb_name } [WHERE clause]; TO_JSON(str_literal)
``` ```
**功能说明**: 将字符串常量转换为 JSON 类型。 **功能说明**: 将字符串常量转换为 JSON 类型。
@ -459,7 +459,7 @@ SELECT TO_JSON(str_literal) FROM { tb_name | stb_name } [WHERE clause];
#### TO_UNIXTIMESTAMP #### TO_UNIXTIMESTAMP
```sql ```sql
SELECT TO_UNIXTIMESTAMP(datetime_string) FROM { tb_name | stb_name } [WHERE clause]; TO_UNIXTIMESTAMP(expr)
``` ```
**功能说明**:将日期时间格式的字符串转换成为 UNIX 时间戳。 **功能说明**:将日期时间格式的字符串转换成为 UNIX 时间戳。
@ -487,9 +487,7 @@ SELECT TO_UNIXTIMESTAMP(datetime_string) FROM { tb_name | stb_name } [WHERE clau
#### NOW #### NOW
```sql ```sql
SELECT NOW() FROM { tb_name | stb_name } [WHERE clause]; NOW()
SELECT select_expr FROM { tb_name | stb_name } WHERE ts_col cond_operatior NOW();
INSERT INTO tb_name VALUES (NOW(), ...);
``` ```
**功能说明**:返回客户端当前系统时间。 **功能说明**:返回客户端当前系统时间。
@ -512,7 +510,7 @@ INSERT INTO tb_name VALUES (NOW(), ...);
#### TIMEDIFF #### TIMEDIFF
```sql ```sql
SELECT TIMEDIFF(ts | datetime_string1, ts | datetime_string2 [, time_unit]) FROM { tb_name | stb_name } [WHERE clause]; TIMEDIFF(expr1, expr2 [, time_unit])
``` ```
**功能说明**:计算两个时间戳之间的差值,并近似到时间单位 time_unit 指定的精度。 **功能说明**:计算两个时间戳之间的差值,并近似到时间单位 time_unit 指定的精度。
@ -535,7 +533,7 @@ SELECT TIMEDIFF(ts | datetime_string1, ts | datetime_string2 [, time_unit]) FROM
#### TIMETRUNCATE #### TIMETRUNCATE
```sql ```sql
SELECT TIMETRUNCATE(ts | datetime_string , time_unit) FROM { tb_name | stb_name } [WHERE clause]; TIMETRUNCATE(expr, time_unit)
``` ```
**功能说明**:将时间戳按照指定时间单位 time_unit 进行截断。 **功能说明**:将时间戳按照指定时间单位 time_unit 进行截断。
@ -556,7 +554,7 @@ SELECT TIMETRUNCATE(ts | datetime_string , time_unit) FROM { tb_name | stb_name
#### TIMEZONE #### TIMEZONE
```sql ```sql
SELECT TIMEZONE() FROM { tb_name | stb_name } [WHERE clause]; TIMEZONE()
``` ```
**功能说明**:返回客户端当前时区信息。 **功能说明**:返回客户端当前时区信息。
@ -571,9 +569,7 @@ SELECT TIMEZONE() FROM { tb_name | stb_name } [WHERE clause];
#### TODAY #### TODAY
```sql ```sql
SELECT TODAY() FROM { tb_name | stb_name } [WHERE clause]; TODAY()
SELECT select_expr FROM { tb_name | stb_name } WHERE ts_col cond_operatior TODAY()];
INSERT INTO tb_name VALUES (TODAY(), ...);
``` ```
**功能说明**:返回客户端当日零时的系统时间。 **功能说明**:返回客户端当日零时的系统时间。
@ -600,7 +596,12 @@ TDengine 支持针对数据的聚合查询。提供如下聚合函数。
### APERCENTILE ### APERCENTILE
```sql ```sql
SELECT APERCENTILE(field_name, P[, algo_type]) FROM { tb_name | stb_name } [WHERE clause] APERCENTILE(expr, p [, algo_type])
algo_type: {
"default"
| "t-digest"
}
``` ```
**功能说明**:统计表/超级表中指定列的值的近似百分比分位数,与 PERCENTILE 函数相似,但是返回近似结果。 **功能说明**:统计表/超级表中指定列的值的近似百分比分位数,与 PERCENTILE 函数相似,但是返回近似结果。
@ -612,14 +613,14 @@ SELECT APERCENTILE(field_name, P[, algo_type]) FROM { tb_name | stb_name } [WHER
**适用于**:表和超级表。 **适用于**:表和超级表。
**说明** **说明**
- P值范围是[0,100]当为0时等同于MIN为100时等同于MAX。 - p值范围是[0,100]当为0时等同于MIN为100时等同于MAX。
- algo_type 取值为 "default" 或 "t-digest"。 输入为 "default" 时函数使用基于直方图算法进行计算。输入为 "t-digest" 时使用t-digest算法计算分位数的近似结果。如果不指定 algo_type 则使用 "default" 算法。 - algo_type 取值为 "default" 或 "t-digest"。 输入为 "default" 时函数使用基于直方图算法进行计算。输入为 "t-digest" 时使用t-digest算法计算分位数的近似结果。如果不指定 algo_type 则使用 "default" 算法。
- "t-digest"算法的近似结果对于输入数据顺序敏感,对超级表查询时不同的输入排序结果可能会有微小的误差。 - "t-digest"算法的近似结果对于输入数据顺序敏感,对超级表查询时不同的输入排序结果可能会有微小的误差。
### AVG ### AVG
```sql ```sql
SELECT AVG(field_name) FROM tb_name [WHERE clause]; AVG(expr)
``` ```
**功能说明**:统计指定字段的平均值。 **功能说明**:统计指定字段的平均值。
@ -634,7 +635,7 @@ SELECT AVG(field_name) FROM tb_name [WHERE clause];
### COUNT ### COUNT
```sql ```sql
SELECT COUNT([*|field_name]) FROM tb_name [WHERE clause]; COUNT({* | expr})
``` ```
**功能说明**:统计指定字段的记录行数。 **功能说明**:统计指定字段的记录行数。
@ -654,7 +655,7 @@ SELECT COUNT([*|field_name]) FROM tb_name [WHERE clause];
### ELAPSED ### ELAPSED
```sql ```sql
SELECT ELAPSED(ts_primary_key [, time_unit]) FROM { tb_name | stb_name } [WHERE clause] [INTERVAL(interval [, offset]) [SLIDING sliding]]; ELAPSED(ts_primary_key [, time_unit])
``` ```
**功能说明**elapsed函数表达了统计周期内连续的时间长度和twa函数配合使用可以计算统计曲线下的面积。在通过INTERVAL子句指定窗口的情况下统计在给定时间范围内的每个窗口内有数据覆盖的时间范围如果没有INTERVAL子句则返回整个给定时间范围内的有数据覆盖的时间范围。注意ELAPSED返回的并不是时间范围的绝对值而是绝对值除以time_unit所得到的单位个数。 **功能说明**elapsed函数表达了统计周期内连续的时间长度和twa函数配合使用可以计算统计曲线下的面积。在通过INTERVAL子句指定窗口的情况下统计在给定时间范围内的每个窗口内有数据覆盖的时间范围如果没有INTERVAL子句则返回整个给定时间范围内的有数据覆盖的时间范围。注意ELAPSED返回的并不是时间范围的绝对值而是绝对值除以time_unit所得到的单位个数。
@ -666,7 +667,7 @@ SELECT ELAPSED(ts_primary_key [, time_unit]) FROM { tb_name | stb_name } [WHERE
**适用于**: 表,超级表,嵌套查询的外层查询 **适用于**: 表,超级表,嵌套查询的外层查询
**说明** **说明**
- field_name参数只能是表的第一列,即 TIMESTAMP 类型的主键列。 - ts_primary_key参数只能是表的第一列,即 TIMESTAMP 类型的主键列。
- 按time_unit参数指定的时间单位返回最小是数据库的时间分辨率。time_unit 参数未指定时,以数据库的时间分辨率为时间单位。支持的时间单位 time_unit 如下: - 按time_unit参数指定的时间单位返回最小是数据库的时间分辨率。time_unit 参数未指定时,以数据库的时间分辨率为时间单位。支持的时间单位 time_unit 如下:
1b(纳秒), 1u(微秒)1a(毫秒)1s(秒)1m(分)1h(小时)1d(天), 1w(周)。 1b(纳秒), 1u(微秒)1a(毫秒)1s(秒)1m(分)1h(小时)1d(天), 1w(周)。
- 可以和interval组合使用返回每个时间窗口的时间戳差值。需要特别注意的是除第一个时间窗口和最后一个时间窗口外中间窗口的时间戳差值均为窗口长度。 - 可以和interval组合使用返回每个时间窗口的时间戳差值。需要特别注意的是除第一个时间窗口和最后一个时间窗口外中间窗口的时间戳差值均为窗口长度。
@ -680,14 +681,14 @@ SELECT ELAPSED(ts_primary_key [, time_unit]) FROM { tb_name | stb_name } [WHERE
### LEASTSQUARES ### LEASTSQUARES
```sql ```sql
SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause]; LEASTSQUARES(expr, start_val, step_val)
``` ```
**功能说明**统计表中某列的值是主键时间戳的拟合直线方程。start_val 是自变量初始值step_val 是自变量的步长值。 **功能说明**统计表中某列的值是主键时间戳的拟合直线方程。start_val 是自变量初始值step_val 是自变量的步长值。
**返回数据类型**:字符串表达式(斜率, 截距)。 **返回数据类型**:字符串表达式(斜率, 截距)。
**适用数据类型**field_name 必须是数值类型。 **适用数据类型**expr 必须是数值类型。
**适用于**:表。 **适用于**:表。
@ -695,7 +696,7 @@ SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause]
### SPREAD ### SPREAD
```sql ```sql
SELECT SPREAD(field_name) FROM { tb_name | stb_name } [WHERE clause]; SPREAD(expr)
``` ```
**功能说明**:统计表中某列的最大值和最小值之差。 **功能说明**:统计表中某列的最大值和最小值之差。
@ -710,7 +711,7 @@ SELECT SPREAD(field_name) FROM { tb_name | stb_name } [WHERE clause];
### STDDEV ### STDDEV
```sql ```sql
SELECT STDDEV(field_name) FROM tb_name [WHERE clause]; STDDEV(expr)
``` ```
**功能说明**:统计表中某列的均方差。 **功能说明**:统计表中某列的均方差。
@ -725,7 +726,7 @@ SELECT STDDEV(field_name) FROM tb_name [WHERE clause];
### SUM ### SUM
```sql ```sql
SELECT SUM(field_name) FROM tb_name [WHERE clause]; SUM(expr)
``` ```
**功能说明**:统计表/超级表中某列的和。 **功能说明**:统计表/超级表中某列的和。
@ -740,7 +741,7 @@ SELECT SUM(field_name) FROM tb_name [WHERE clause];
### HYPERLOGLOG ### HYPERLOGLOG
```sql ```sql
SELECT HYPERLOGLOG(field_name) FROM { tb_name | stb_name } [WHERE clause]; HYPERLOGLOG(expr)
``` ```
**功能说明** **功能说明**
@ -757,7 +758,7 @@ SELECT HYPERLOGLOG(field_name) FROM { tb_name | stb_name } [WHERE clause];
### HISTOGRAM ### HISTOGRAM
```sql ```sql
SELECT HISTOGRAM(field_namebin_type, bin_description, normalized) FROM tb_name [WHERE clause]; HISTOGRAM(exprbin_type, bin_description, normalized)
``` ```
**功能说明**:统计数据按照用户指定区间的分布。 **功能说明**:统计数据按照用户指定区间的分布。
@ -787,7 +788,7 @@ SELECT HISTOGRAM(field_namebin_type, bin_description, normalized) FROM tb_nam
### PERCENTILE ### PERCENTILE
```sql ```sql
SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause]; PERCENTILE(expr, p)
``` ```
**功能说明**:统计表中某列的值百分比分位数。 **功能说明**:统计表中某列的值百分比分位数。
@ -808,7 +809,7 @@ SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause];
### BOTTOM ### BOTTOM
```sql ```sql
SELECT BOTTOM(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; BOTTOM(expr, k)
``` ```
**功能说明**:统计表/超级表中某列的值最小 _k_ 个非 NULL 值。如果多条数据取值一样,全部取用又会超出 k 条限制时,系统会从相同值中随机选取符合要求的数量返回。 **功能说明**:统计表/超级表中某列的值最小 _k_ 个非 NULL 值。如果多条数据取值一样,全部取用又会超出 k 条限制时,系统会从相同值中随机选取符合要求的数量返回。
@ -828,7 +829,7 @@ SELECT BOTTOM(field_name, K) FROM { tb_name | stb_name } [WHERE clause];
### FIRST ### FIRST
```sql ```sql
SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause]; FIRST(expr)
``` ```
**功能说明**:统计表/超级表中某列的值最先写入的非 NULL 值。 **功能说明**:统计表/超级表中某列的值最先写入的非 NULL 值。
@ -848,7 +849,7 @@ SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause];
### INTERP ### INTERP
```sql ```sql
SELECT INTERP(field_name) FROM { tb_name | stb_name } [WHERE where_condition] RANGE(timestamp1,timestamp2) EVERY(interval) FILL({ VALUE | PREV | NULL | LINEAR | NEXT}); INTERP(expr)
``` ```
**功能说明**:返回指定时间截面指定列的记录值或插值。 **功能说明**:返回指定时间截面指定列的记录值或插值。
@ -867,11 +868,12 @@ SELECT INTERP(field_name) FROM { tb_name | stb_name } [WHERE where_condition] RA
- INTERP 根据 EVERY 字段来确定输出时间范围内的结果条数,即从 timestamp1 开始每隔固定长度的时间EVERY 值)进行插值。如果没有指定 EVERY则默认窗口大小为无穷大即从 timestamp1 开始只有一个窗口。 - INTERP 根据 EVERY 字段来确定输出时间范围内的结果条数,即从 timestamp1 开始每隔固定长度的时间EVERY 值)进行插值。如果没有指定 EVERY则默认窗口大小为无穷大即从 timestamp1 开始只有一个窗口。
- INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。 - INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。
- INTERP 只能在一个时间序列内进行插值,因此当作用于超级表时必须跟 partition by tbname 一起使用。 - INTERP 只能在一个时间序列内进行插值,因此当作用于超级表时必须跟 partition by tbname 一起使用。
- INTERP 可以与伪列 _irowts 一起使用,返回插值点所对应的时间戳(3.0.1.4版本以后支持)。
### LAST ### LAST
```sql ```sql
SELECT LAST(field_name) FROM { tb_name | stb_name } [WHERE clause]; LAST(expr)
``` ```
**功能说明**:统计表/超级表中某列的值最后写入的非 NULL 值。 **功能说明**:统计表/超级表中某列的值最后写入的非 NULL 值。
@ -892,7 +894,7 @@ SELECT LAST(field_name) FROM { tb_name | stb_name } [WHERE clause];
### LAST_ROW ### LAST_ROW
```sql ```sql
SELECT LAST_ROW(field_name) FROM { tb_name | stb_name }; LAST_ROW(expr)
``` ```
**功能说明**:返回表/超级表的最后一条记录。 **功能说明**:返回表/超级表的最后一条记录。
@ -911,7 +913,7 @@ SELECT LAST_ROW(field_name) FROM { tb_name | stb_name };
### MAX ### MAX
```sql ```sql
SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause]; MAX(expr)
``` ```
**功能说明**:统计表/超级表中某列的值最大值。 **功能说明**:统计表/超级表中某列的值最大值。
@ -926,7 +928,7 @@ SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause];
### MIN ### MIN
```sql ```sql
SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause]; MIN(expr)
``` ```
**功能说明**:统计表/超级表中某列的值最小值。 **功能说明**:统计表/超级表中某列的值最小值。
@ -941,7 +943,7 @@ SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause];
### MODE ### MODE
```sql ```sql
SELECT MODE(field_name) FROM tb_name [WHERE clause]; MODE(expr)
``` ```
**功能说明**返回出现频率最高的值若存在多个频率相同的最高值输出NULL。 **功能说明**返回出现频率最高的值若存在多个频率相同的最高值输出NULL。
@ -956,7 +958,7 @@ SELECT MODE(field_name) FROM tb_name [WHERE clause];
### SAMPLE ### SAMPLE
```sql ```sql
SELECT SAMPLE(field_name, K) FROM { tb_name | stb_name } [WHERE clause] SAMPLE(expr, k)
``` ```
**功能说明** 获取数据的 k 个采样值。参数 k 的合法输入范围是 1≤ k ≤ 1000。 **功能说明** 获取数据的 k 个采样值。参数 k 的合法输入范围是 1≤ k ≤ 1000。
@ -978,7 +980,7 @@ SELECT SAMPLE(field_name, K) FROM { tb_name | stb_name } [WHERE clause]
### TAIL ### TAIL
```sql ```sql
SELECT TAIL(field_name, k, offset_val) FROM {tb_name | stb_name} [WHERE clause]; TAIL(expr, k [, offset_rows])
``` ```
**功能说明**:返回跳过最后 offset_val 个,然后取连续 k 个记录,不忽略 NULL 值。offset_val 可以不输入。此时返回最后的 k 个记录。当有 offset_val 输入的情况下,该函数功能等效于 `order by ts desc LIMIT k OFFSET offset_val` **功能说明**:返回跳过最后 offset_val 个,然后取连续 k 个记录,不忽略 NULL 值。offset_val 可以不输入。此时返回最后的 k 个记录。当有 offset_val 输入的情况下,该函数功能等效于 `order by ts desc LIMIT k OFFSET offset_val`
@ -995,7 +997,7 @@ SELECT TAIL(field_name, k, offset_val) FROM {tb_name | stb_name} [WHERE clause];
### TOP ### TOP
```sql ```sql
SELECT TOP(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; TOP(expr, k)
``` ```
**功能说明** 统计表/超级表中某列的值最大 _k_ 个非 NULL 值。如果多条数据取值一样,全部取用又会超出 k 条限制时,系统会从相同值中随机选取符合要求的数量返回。 **功能说明** 统计表/超级表中某列的值最大 _k_ 个非 NULL 值。如果多条数据取值一样,全部取用又会超出 k 条限制时,系统会从相同值中随机选取符合要求的数量返回。
@ -1015,7 +1017,7 @@ SELECT TOP(field_name, K) FROM { tb_name | stb_name } [WHERE clause];
### UNIQUE ### UNIQUE
```sql ```sql
SELECT UNIQUE(field_name) FROM {tb_name | stb_name} [WHERE clause]; UNIQUE(expr)
``` ```
**功能说明**:返回该列的数值首次出现的值。该函数功能与 distinct 相似,但是可以匹配标签和时间戳信息。可以针对除时间列以外的字段进行查询,可以匹配标签和时间戳,其中的标签和时间戳是第一次出现时刻的标签和时间戳。 **功能说明**:返回该列的数值首次出现的值。该函数功能与 distinct 相似,但是可以匹配标签和时间戳信息。可以针对除时间列以外的字段进行查询,可以匹配标签和时间戳,其中的标签和时间戳是第一次出现时刻的标签和时间戳。
@ -1034,7 +1036,7 @@ SELECT UNIQUE(field_name) FROM {tb_name | stb_name} [WHERE clause];
### CSUM ### CSUM
```sql ```sql
SELECT CSUM(field_name) FROM { tb_name | stb_name } [WHERE clause] CSUM(expr)
``` ```
**功能说明**累加和Cumulative sum输出行与输入行数相同。 **功能说明**累加和Cumulative sum输出行与输入行数相同。
@ -1057,7 +1059,12 @@ SELECT CSUM(field_name) FROM { tb_name | stb_name } [WHERE clause]
### DERIVATIVE ### DERIVATIVE
```sql ```sql
SELECT DERIVATIVE(field_name, time_interval, ignore_negative) FROM tb_name [WHERE clause]; DERIVATIVE(expr, time_interval, ignore_negative)
ignore_negative: {
0
| 1
}
``` ```
**功能说明**:统计表中某列数值的单位变化率。其中单位时间区间的长度可以通过 time_interval 参数指定,最小可以是 1 秒1signore_negative 参数的值可以是 0 或 1为 1 时表示忽略负值。 **功能说明**:统计表中某列数值的单位变化率。其中单位时间区间的长度可以通过 time_interval 参数指定,最小可以是 1 秒1signore_negative 参数的值可以是 0 或 1为 1 时表示忽略负值。
@ -1076,7 +1083,12 @@ SELECT DERIVATIVE(field_name, time_interval, ignore_negative) FROM tb_name [WHER
### DIFF ### DIFF
```sql ```sql
SELECT {DIFF(field_name, ignore_negative) | DIFF(field_name)} FROM tb_name [WHERE clause]; DIFF(expr [, ignore_negative])
ignore_negative: {
0
| 1
}
``` ```
**功能说明**:统计表中某列的值与前一行对应值的差。 ignore_negative 取值为 0|1 , 可以不填,默认值为 0. 不忽略负值。ignore_negative 为 1 时表示忽略负数。 **功能说明**:统计表中某列的值与前一行对应值的差。 ignore_negative 取值为 0|1 , 可以不填,默认值为 0. 不忽略负值。ignore_negative 为 1 时表示忽略负数。
@ -1096,7 +1108,7 @@ SELECT {DIFF(field_name, ignore_negative) | DIFF(field_name)} FROM tb_name [WHER
### IRATE ### IRATE
```sql ```sql
SELECT IRATE(field_name) FROM tb_name WHERE clause; IRATE(expr)
``` ```
**功能说明**:计算瞬时增长率。使用时间区间中最后两个样本数据来计算瞬时增长速率;如果这两个值呈递减关系,那么只取最后一个数用于计算,而不是使用二者差值。 **功能说明**:计算瞬时增长率。使用时间区间中最后两个样本数据来计算瞬时增长速率;如果这两个值呈递减关系,那么只取最后一个数用于计算,而不是使用二者差值。
@ -1111,7 +1123,7 @@ SELECT IRATE(field_name) FROM tb_name WHERE clause;
### MAVG ### MAVG
```sql ```sql
SELECT MAVG(field_name, K) FROM { tb_name | stb_name } [WHERE clause] MAVG(expr, k)
``` ```
**功能说明** 计算连续 k 个值的移动平均数moving average。如果输入行数小于 k则无结果输出。参数 k 的合法输入范围是 1≤ k ≤ 1000。 **功能说明** 计算连续 k 个值的移动平均数moving average。如果输入行数小于 k则无结果输出。参数 k 的合法输入范围是 1≤ k ≤ 1000。
@ -1134,7 +1146,7 @@ SELECT MAVG(field_name, K) FROM { tb_name | stb_name } [WHERE clause]
### STATECOUNT ### STATECOUNT
```sql ```sql
SELECT STATECOUNT(field_name, oper, val) FROM { tb_name | stb_name } [WHERE clause]; STATECOUNT(expr, oper, val)
``` ```
**功能说明**:返回满足某个条件的连续记录的个数,结果作为新的一列追加在每行后面。条件根据参数计算,如果条件为 true 则加 1条件为 false 则重置为-1如果数据为 NULL跳过该条数据。 **功能说明**:返回满足某个条件的连续记录的个数,结果作为新的一列追加在每行后面。条件根据参数计算,如果条件为 true 则加 1条件为 false 则重置为-1如果数据为 NULL跳过该条数据。
@ -1161,7 +1173,7 @@ SELECT STATECOUNT(field_name, oper, val) FROM { tb_name | stb_name } [WHERE clau
### STATEDURATION ### STATEDURATION
```sql ```sql
SELECT stateDuration(field_name, oper, val, unit) FROM { tb_name | stb_name } [WHERE clause]; STATEDURATION(expr, oper, val, unit)
``` ```
**功能说明**:返回满足某个条件的连续记录的时间长度,结果作为新的一列追加在每行后面。条件根据参数计算,如果条件为 true 则加上两个记录之间的时间长度(第一个满足条件的记录时间长度记为 0条件为 false 则重置为-1如果数据为 NULL跳过该条数据。 **功能说明**:返回满足某个条件的连续记录的时间长度,结果作为新的一列追加在每行后面。条件根据参数计算,如果条件为 true 则加上两个记录之间的时间长度(第一个满足条件的记录时间长度记为 0条件为 false 则重置为-1如果数据为 NULL跳过该条数据。
@ -1189,7 +1201,7 @@ SELECT stateDuration(field_name, oper, val, unit) FROM { tb_name | stb_name } [W
### TWA ### TWA
```sql ```sql
SELECT TWA(field_name) FROM tb_name WHERE clause; TWA(expr)
``` ```
**功能说明**:时间加权平均函数。统计表中某列在一段时间内的时间加权平均。 **功能说明**:时间加权平均函数。统计表中某列在一段时间内的时间加权平均。

View File

@ -12,7 +12,15 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库,提供对数
4. TDengine 在后续演进中可以灵活的添加已有 INFORMATION_SCHEMA 中表的列,而不用担心对既有业务系统造成影响 4. TDengine 在后续演进中可以灵活的添加已有 INFORMATION_SCHEMA 中表的列,而不用担心对既有业务系统造成影响
5. 与其他数据库系统更具互操作性。例如Oracle 数据库用户熟悉查询 Oracle 数据字典中的表 5. 与其他数据库系统更具互操作性。例如Oracle 数据库用户熟悉查询 Oracle 数据字典中的表
Note: 由于 SHOW 语句已经被开发者熟悉和广泛使用,所以它们仍然被保留。 :::info
- 由于 SHOW 语句已经被开发者熟悉和广泛使用,所以它们仍然被保留。
- 系统表中的一些列可能是关键字,在查询时需要使用转义符'\`',例如查询数据库 test 有几个 VGROUP
```sql
select `vgroups` from ins_databases where name = 'test';
```
:::
本章将详细介绍 `INFORMATION_SCHEMA` 这个内置元数据库中的表和表结构。 本章将详细介绍 `INFORMATION_SCHEMA` 这个内置元数据库中的表和表结构。
@ -103,7 +111,11 @@ Note: 由于 SHOW 语句已经被开发者熟悉和广泛使用,所以它们
| 24 | wal_retention_period | INT | WAL 的保存时长 | | 24 | wal_retention_period | INT | WAL 的保存时长 |
| 25 | wal_retention_size | INT | WAL 的保存上限 | | 25 | wal_retention_size | INT | WAL 的保存上限 |
| 26 | wal_roll_period | INT | wal 文件切换时长 | | 26 | wal_roll_period | INT | wal 文件切换时长 |
| 27 | wal_segment_size | wal 单个文件大小 | | 27 | wal_segment_size | BIGINT | wal 单个文件大小 |
| 28 | stt_trigger | SMALLINT | 触发文件合并的落盘文件的个数 |
| 29 | table_prefix | SMALLINT | 内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的前缀的长度 |
| 30 | table_suffix | SMALLINT | 内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的后缀的长度 |
| 31 | tsdb_pagesize | INT | 时序数据存储引擎中的页大小 |
## INS_FUNCTIONS ## INS_FUNCTIONS

View File

@ -177,12 +177,21 @@ taos --dump-config
### maxNumOfDistinctRes ### maxNumOfDistinctRes
| 属性 | 说明 | | 属性 | 说明 |
| -------- | -------------------------------- | --- | | -------- | -------------------------------- |
| 适用范围 | 仅服务端适用 | | 适用范围 | 仅服务端适用 |
| 含义 | 允许返回的 distinct 结果最大行数 | | 含义 | 允许返回的 distinct 结果最大行数 |
| 取值范围 | 默认值为 10 万,最大值 1 亿 | | 取值范围 | 默认值为 10 万,最大值 1 亿 |
| 缺省值 | 10 万 | | 缺省值 | 10 万 |
### keepColumnName
| 属性 | 说明 |
| -------- | -------------------------------- |
| 适用范围 | 仅客户端适用 |
| 含义 | Last、First、LastRow 函数查询时,返回的列名是否包含函数名。 |
| 取值范围 | 0 表示包含函数名1 表示不包含函数名。 |
| 缺省值 | 0 |
## 区域相关 ## 区域相关
### timezone ### timezone

View File

@ -6,6 +6,10 @@ description: TDengine 发布历史、Release Notes 及下载链接
import Release from "/components/ReleaseV3"; import Release from "/components/ReleaseV3";
## 3.0.1.3
<Release type="tdengine" version="3.0.1.3" />
## 3.0.1.2 ## 3.0.1.2
<Release type="tdengine" version="3.0.1.2" /> <Release type="tdengine" version="3.0.1.2" />

View File

@ -6,6 +6,10 @@ description: taosTools 的发布历史、Release Notes 和下载链接
import Release from "/components/ReleaseV3"; import Release from "/components/ReleaseV3";
## 2.2.3
<Release type="tools" version="2.2.3" />
## 2.2.2 ## 2.2.2
<Release type="tools" version="2.2.2" /> <Release type="tools" version="2.2.2" />

View File

@ -98,6 +98,7 @@ extern int32_t tsQueryRsmaTolerance;
extern bool tsQueryPlannerTrace; extern bool tsQueryPlannerTrace;
extern int32_t tsQueryNodeChunkSize; extern int32_t tsQueryNodeChunkSize;
extern bool tsQueryUseNodeAllocator; extern bool tsQueryUseNodeAllocator;
extern bool tsKeepColumnName;
// client // client
extern int32_t tsMinSlidingTime; extern int32_t tsMinSlidingTime;

View File

@ -56,9 +56,8 @@ extern int32_t tMsgDict[];
#define TMSG_SEG_CODE(TYPE) (((TYPE)&0xff00) >> 8) #define TMSG_SEG_CODE(TYPE) (((TYPE)&0xff00) >> 8)
#define TMSG_SEG_SEQ(TYPE) ((TYPE)&0xff) #define TMSG_SEG_SEQ(TYPE) ((TYPE)&0xff)
#define TMSG_INFO(TYPE) \ #define TMSG_INFO(TYPE) \
((TYPE) >= 0 && ((TYPE) < TDMT_DND_MAX_MSG || (TYPE) < TDMT_MND_MAX_MSG || (TYPE) < TDMT_VND_MAX_MSG || \ ((TYPE) < TDMT_DND_MAX_MSG || (TYPE) < TDMT_MND_MAX_MSG || (TYPE) < TDMT_VND_MAX_MSG || (TYPE) < TDMT_SCH_MAX_MSG || \
(TYPE) < TDMT_SCH_MAX_MSG || (TYPE) < TDMT_STREAM_MAX_MSG || (TYPE) < TDMT_MON_MAX_MSG || \ (TYPE) < TDMT_STREAM_MAX_MSG || (TYPE) < TDMT_MON_MAX_MSG || (TYPE) < TDMT_SYNC_MAX_MSG) \
(TYPE) < TDMT_SYNC_MAX_MSG)) \
? tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)] \ ? tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)] \
: 0 : 0
#define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)) #define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE))
@ -867,13 +866,15 @@ int32_t tDeserializeSDbCfgReq(void* buf, int32_t bufLen, SDbCfgReq* pReq);
typedef struct { typedef struct {
char db[TSDB_DB_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN];
int32_t maxSpeed;
} STrimDbReq; } STrimDbReq;
int32_t tSerializeSTrimDbReq(void* buf, int32_t bufLen, STrimDbReq* pReq); int32_t tSerializeSTrimDbReq(void* buf, int32_t bufLen, STrimDbReq* pReq);
int32_t tDeserializeSTrimDbReq(void* buf, int32_t bufLen, STrimDbReq* pReq); int32_t tDeserializeSTrimDbReq(void* buf, int32_t bufLen, STrimDbReq* pReq);
typedef struct { typedef struct {
int32_t timestamp; int64_t timestamp; // unit: millisecond
int64_t maxSpeed; // 0 no limit, unit: Byte/s
} SVTrimDbReq; } SVTrimDbReq;
int32_t tSerializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq); int32_t tSerializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq);
@ -1446,7 +1447,7 @@ typedef struct STableScanAnalyzeInfo {
int32_t tSerializeSExplainRsp(void* buf, int32_t bufLen, SExplainRsp* pRsp); int32_t tSerializeSExplainRsp(void* buf, int32_t bufLen, SExplainRsp* pRsp);
int32_t tDeserializeSExplainRsp(void* buf, int32_t bufLen, SExplainRsp* pRsp); int32_t tDeserializeSExplainRsp(void* buf, int32_t bufLen, SExplainRsp* pRsp);
void tFreeSExplainRsp(SExplainRsp *pRsp); void tFreeSExplainRsp(SExplainRsp* pRsp);
typedef struct { typedef struct {
char fqdn[TSDB_FQDN_LEN]; // end point, hostname:port char fqdn[TSDB_FQDN_LEN]; // end point, hostname:port
@ -1729,6 +1730,8 @@ typedef struct {
int64_t maxDelay; int64_t maxDelay;
int64_t watermark; int64_t watermark;
int8_t igExpired; int8_t igExpired;
int32_t numOfTags;
SArray* pTags; // array of SField
} SCMCreateStreamReq; } SCMCreateStreamReq;
typedef struct { typedef struct {

View File

@ -107,227 +107,230 @@
#define TK_TABLE_PREFIX 89 #define TK_TABLE_PREFIX 89
#define TK_TABLE_SUFFIX 90 #define TK_TABLE_SUFFIX 90
#define TK_NK_COLON 91 #define TK_NK_COLON 91
#define TK_TABLE 92 #define TK_MAX_SPEED 92
#define TK_NK_LP 93 #define TK_TABLE 93
#define TK_NK_RP 94 #define TK_NK_LP 94
#define TK_STABLE 95 #define TK_NK_RP 95
#define TK_ADD 96 #define TK_STABLE 96
#define TK_COLUMN 97 #define TK_ADD 97
#define TK_MODIFY 98 #define TK_COLUMN 98
#define TK_RENAME 99 #define TK_MODIFY 99
#define TK_TAG 100 #define TK_RENAME 100
#define TK_SET 101 #define TK_TAG 101
#define TK_NK_EQ 102 #define TK_SET 102
#define TK_USING 103 #define TK_NK_EQ 103
#define TK_TAGS 104 #define TK_USING 104
#define TK_COMMENT 105 #define TK_TAGS 105
#define TK_BOOL 106 #define TK_COMMENT 106
#define TK_TINYINT 107 #define TK_BOOL 107
#define TK_SMALLINT 108 #define TK_TINYINT 108
#define TK_INT 109 #define TK_SMALLINT 109
#define TK_INTEGER 110 #define TK_INT 110
#define TK_BIGINT 111 #define TK_INTEGER 111
#define TK_FLOAT 112 #define TK_BIGINT 112
#define TK_DOUBLE 113 #define TK_FLOAT 113
#define TK_BINARY 114 #define TK_DOUBLE 114
#define TK_TIMESTAMP 115 #define TK_BINARY 115
#define TK_NCHAR 116 #define TK_TIMESTAMP 116
#define TK_UNSIGNED 117 #define TK_NCHAR 117
#define TK_JSON 118 #define TK_UNSIGNED 118
#define TK_VARCHAR 119 #define TK_JSON 119
#define TK_MEDIUMBLOB 120 #define TK_VARCHAR 120
#define TK_BLOB 121 #define TK_MEDIUMBLOB 121
#define TK_VARBINARY 122 #define TK_BLOB 122
#define TK_DECIMAL 123 #define TK_VARBINARY 123
#define TK_MAX_DELAY 124 #define TK_DECIMAL 124
#define TK_WATERMARK 125 #define TK_MAX_DELAY 125
#define TK_ROLLUP 126 #define TK_WATERMARK 126
#define TK_TTL 127 #define TK_ROLLUP 127
#define TK_SMA 128 #define TK_TTL 128
#define TK_FIRST 129 #define TK_SMA 129
#define TK_LAST 130 #define TK_FIRST 130
#define TK_SHOW 131 #define TK_LAST 131
#define TK_DATABASES 132 #define TK_SHOW 132
#define TK_TABLES 133 #define TK_DATABASES 133
#define TK_STABLES 134 #define TK_TABLES 134
#define TK_MNODES 135 #define TK_STABLES 135
#define TK_MODULES 136 #define TK_MNODES 136
#define TK_QNODES 137 #define TK_MODULES 137
#define TK_FUNCTIONS 138 #define TK_QNODES 138
#define TK_INDEXES 139 #define TK_FUNCTIONS 139
#define TK_ACCOUNTS 140 #define TK_INDEXES 140
#define TK_APPS 141 #define TK_ACCOUNTS 141
#define TK_CONNECTIONS 142 #define TK_APPS 142
#define TK_LICENCES 143 #define TK_CONNECTIONS 143
#define TK_GRANTS 144 #define TK_LICENCES 144
#define TK_QUERIES 145 #define TK_GRANTS 145
#define TK_SCORES 146 #define TK_QUERIES 146
#define TK_TOPICS 147 #define TK_SCORES 147
#define TK_VARIABLES 148 #define TK_TOPICS 148
#define TK_BNODES 149 #define TK_VARIABLES 149
#define TK_SNODES 150 #define TK_BNODES 150
#define TK_CLUSTER 151 #define TK_SNODES 151
#define TK_TRANSACTIONS 152 #define TK_CLUSTER 152
#define TK_DISTRIBUTED 153 #define TK_TRANSACTIONS 153
#define TK_CONSUMERS 154 #define TK_DISTRIBUTED 154
#define TK_SUBSCRIPTIONS 155 #define TK_CONSUMERS 155
#define TK_VNODES 156 #define TK_SUBSCRIPTIONS 156
#define TK_LIKE 157 #define TK_VNODES 157
#define TK_INDEX 158 #define TK_LIKE 158
#define TK_FUNCTION 159 #define TK_INDEX 159
#define TK_INTERVAL 160 #define TK_FUNCTION 160
#define TK_TOPIC 161 #define TK_INTERVAL 161
#define TK_AS 162 #define TK_TOPIC 162
#define TK_WITH 163 #define TK_AS 163
#define TK_META 164 #define TK_WITH 164
#define TK_CONSUMER 165 #define TK_META 165
#define TK_GROUP 166 #define TK_CONSUMER 166
#define TK_DESC 167 #define TK_GROUP 167
#define TK_DESCRIBE 168 #define TK_DESC 168
#define TK_RESET 169 #define TK_DESCRIBE 169
#define TK_QUERY 170 #define TK_RESET 170
#define TK_CACHE 171 #define TK_QUERY 171
#define TK_EXPLAIN 172 #define TK_CACHE 172
#define TK_ANALYZE 173 #define TK_EXPLAIN 173
#define TK_VERBOSE 174 #define TK_ANALYZE 174
#define TK_NK_BOOL 175 #define TK_VERBOSE 175
#define TK_RATIO 176 #define TK_NK_BOOL 176
#define TK_NK_FLOAT 177 #define TK_RATIO 177
#define TK_OUTPUTTYPE 178 #define TK_NK_FLOAT 178
#define TK_AGGREGATE 179 #define TK_OUTPUTTYPE 179
#define TK_BUFSIZE 180 #define TK_AGGREGATE 180
#define TK_STREAM 181 #define TK_BUFSIZE 181
#define TK_INTO 182 #define TK_STREAM 182
#define TK_TRIGGER 183 #define TK_INTO 183
#define TK_AT_ONCE 184 #define TK_TRIGGER 184
#define TK_WINDOW_CLOSE 185 #define TK_AT_ONCE 185
#define TK_IGNORE 186 #define TK_WINDOW_CLOSE 186
#define TK_EXPIRED 187 #define TK_IGNORE 187
#define TK_KILL 188 #define TK_EXPIRED 188
#define TK_CONNECTION 189 #define TK_SUBTABLE 189
#define TK_TRANSACTION 190 #define TK_KILL 190
#define TK_BALANCE 191 #define TK_CONNECTION 191
#define TK_VGROUP 192 #define TK_TRANSACTION 192
#define TK_MERGE 193 #define TK_BALANCE 193
#define TK_REDISTRIBUTE 194 #define TK_VGROUP 194
#define TK_SPLIT 195 #define TK_MERGE 195
#define TK_DELETE 196 #define TK_REDISTRIBUTE 196
#define TK_INSERT 197 #define TK_SPLIT 197
#define TK_NULL 198 #define TK_DELETE 198
#define TK_NK_QUESTION 199 #define TK_INSERT 199
#define TK_NK_ARROW 200 #define TK_NULL 200
#define TK_ROWTS 201 #define TK_NK_QUESTION 201
#define TK_TBNAME 202 #define TK_NK_ARROW 202
#define TK_QSTART 203 #define TK_ROWTS 203
#define TK_QEND 204 #define TK_TBNAME 204
#define TK_QDURATION 205 #define TK_QSTART 205
#define TK_WSTART 206 #define TK_QEND 206
#define TK_WEND 207 #define TK_QDURATION 207
#define TK_WDURATION 208 #define TK_WSTART 208
#define TK_CAST 209 #define TK_WEND 209
#define TK_NOW 210 #define TK_WDURATION 210
#define TK_TODAY 211 #define TK_IROWTS 211
#define TK_TIMEZONE 212 #define TK_CAST 212
#define TK_CLIENT_VERSION 213 #define TK_NOW 213
#define TK_SERVER_VERSION 214 #define TK_TODAY 214
#define TK_SERVER_STATUS 215 #define TK_TIMEZONE 215
#define TK_CURRENT_USER 216 #define TK_CLIENT_VERSION 216
#define TK_COUNT 217 #define TK_SERVER_VERSION 217
#define TK_LAST_ROW 218 #define TK_SERVER_STATUS 218
#define TK_CASE 219 #define TK_CURRENT_USER 219
#define TK_END 220 #define TK_COUNT 220
#define TK_WHEN 221 #define TK_LAST_ROW 221
#define TK_THEN 222 #define TK_CASE 222
#define TK_ELSE 223 #define TK_END 223
#define TK_BETWEEN 224 #define TK_WHEN 224
#define TK_IS 225 #define TK_THEN 225
#define TK_NK_LT 226 #define TK_ELSE 226
#define TK_NK_GT 227 #define TK_BETWEEN 227
#define TK_NK_LE 228 #define TK_IS 228
#define TK_NK_GE 229 #define TK_NK_LT 229
#define TK_NK_NE 230 #define TK_NK_GT 230
#define TK_MATCH 231 #define TK_NK_LE 231
#define TK_NMATCH 232 #define TK_NK_GE 232
#define TK_CONTAINS 233 #define TK_NK_NE 233
#define TK_IN 234 #define TK_MATCH 234
#define TK_JOIN 235 #define TK_NMATCH 235
#define TK_INNER 236 #define TK_CONTAINS 236
#define TK_SELECT 237 #define TK_IN 237
#define TK_DISTINCT 238 #define TK_JOIN 238
#define TK_WHERE 239 #define TK_INNER 239
#define TK_PARTITION 240 #define TK_SELECT 240
#define TK_BY 241 #define TK_DISTINCT 241
#define TK_SESSION 242 #define TK_WHERE 242
#define TK_STATE_WINDOW 243 #define TK_PARTITION 243
#define TK_SLIDING 244 #define TK_BY 244
#define TK_FILL 245 #define TK_SESSION 245
#define TK_VALUE 246 #define TK_STATE_WINDOW 246
#define TK_NONE 247 #define TK_SLIDING 247
#define TK_PREV 248 #define TK_FILL 248
#define TK_LINEAR 249 #define TK_VALUE 249
#define TK_NEXT 250 #define TK_NONE 250
#define TK_HAVING 251 #define TK_PREV 251
#define TK_RANGE 252 #define TK_LINEAR 252
#define TK_EVERY 253 #define TK_NEXT 253
#define TK_ORDER 254 #define TK_HAVING 254
#define TK_SLIMIT 255 #define TK_RANGE 255
#define TK_SOFFSET 256 #define TK_EVERY 256
#define TK_LIMIT 257 #define TK_ORDER 257
#define TK_OFFSET 258 #define TK_SLIMIT 258
#define TK_ASC 259 #define TK_SOFFSET 259
#define TK_NULLS 260 #define TK_LIMIT 260
#define TK_ABORT 261 #define TK_OFFSET 261
#define TK_AFTER 262 #define TK_ASC 262
#define TK_ATTACH 263 #define TK_NULLS 263
#define TK_BEFORE 264 #define TK_ABORT 264
#define TK_BEGIN 265 #define TK_AFTER 265
#define TK_BITAND 266 #define TK_ATTACH 266
#define TK_BITNOT 267 #define TK_BEFORE 267
#define TK_BITOR 268 #define TK_BEGIN 268
#define TK_BLOCKS 269 #define TK_BITAND 269
#define TK_CHANGE 270 #define TK_BITNOT 270
#define TK_COMMA 271 #define TK_BITOR 271
#define TK_COMPACT 272 #define TK_BLOCKS 272
#define TK_CONCAT 273 #define TK_CHANGE 273
#define TK_CONFLICT 274 #define TK_COMMA 274
#define TK_COPY 275 #define TK_COMPACT 275
#define TK_DEFERRED 276 #define TK_CONCAT 276
#define TK_DELIMITERS 277 #define TK_CONFLICT 277
#define TK_DETACH 278 #define TK_COPY 278
#define TK_DIVIDE 279 #define TK_DEFERRED 279
#define TK_DOT 280 #define TK_DELIMITERS 280
#define TK_EACH 281 #define TK_DETACH 281
#define TK_FAIL 282 #define TK_DIVIDE 282
#define TK_FILE 283 #define TK_DOT 283
#define TK_FOR 284 #define TK_EACH 284
#define TK_GLOB 285 #define TK_FAIL 285
#define TK_ID 286 #define TK_FILE 286
#define TK_IMMEDIATE 287 #define TK_FOR 287
#define TK_IMPORT 288 #define TK_GLOB 288
#define TK_INITIALLY 289 #define TK_ID 289
#define TK_INSTEAD 290 #define TK_IMMEDIATE 290
#define TK_ISNULL 291 #define TK_IMPORT 291
#define TK_KEY 292 #define TK_INITIALLY 292
#define TK_NK_BITNOT 293 #define TK_INSTEAD 293
#define TK_NK_SEMI 294 #define TK_ISNULL 294
#define TK_NOTNULL 295 #define TK_KEY 295
#define TK_OF 296 #define TK_NK_BITNOT 296
#define TK_PLUS 297 #define TK_NK_SEMI 297
#define TK_PRIVILEGE 298 #define TK_NOTNULL 298
#define TK_RAISE 299 #define TK_OF 299
#define TK_REPLACE 300 #define TK_PLUS 300
#define TK_RESTRICT 301 #define TK_PRIVILEGE 301
#define TK_ROW 302 #define TK_RAISE 302
#define TK_SEMI 303 #define TK_REPLACE 303
#define TK_STAR 304 #define TK_RESTRICT 304
#define TK_STATEMENT 305 #define TK_ROW 305
#define TK_STRING 306 #define TK_SEMI 306
#define TK_TIMES 307 #define TK_STAR 307
#define TK_UPDATE 308 #define TK_STATEMENT 308
#define TK_VALUES 309 #define TK_STRING 309
#define TK_VARIABLE 310 #define TK_TIMES 310
#define TK_VIEW 311 #define TK_UPDATE 311
#define TK_WAL 312 #define TK_VALUES 312
#define TK_VARIABLE 313
#define TK_VIEW 314
#define TK_WAL 315
#define TK_NK_SPACE 300 #define TK_NK_SPACE 300
#define TK_NK_COMMENT 301 #define TK_NK_COMMENT 301

View File

@ -106,6 +106,8 @@ void mndPostProcessQueryMsg(SRpcMsg *pMsg);
*/ */
void mndGenerateMachineCode(); void mndGenerateMachineCode();
void mndDumpSdb();
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -119,9 +119,10 @@ typedef enum EFunctionType {
FUNCTION_TYPE_WSTART, FUNCTION_TYPE_WSTART,
FUNCTION_TYPE_WEND, FUNCTION_TYPE_WEND,
FUNCTION_TYPE_WDURATION, FUNCTION_TYPE_WDURATION,
FUNCTION_TYPE_IROWTS,
// internal function // internal function
FUNCTION_TYPE_SELECT_VALUE, FUNCTION_TYPE_SELECT_VALUE = 3750,
FUNCTION_TYPE_BLOCK_DIST, // block distribution aggregate function FUNCTION_TYPE_BLOCK_DIST, // block distribution aggregate function
FUNCTION_TYPE_BLOCK_DIST_INFO, // block distribution pseudo column function FUNCTION_TYPE_BLOCK_DIST_INFO, // block distribution pseudo column function
FUNCTION_TYPE_TO_COLUMN, FUNCTION_TYPE_TO_COLUMN,
@ -212,6 +213,7 @@ bool fmIsClientPseudoColumnFunc(int32_t funcId);
bool fmIsMultiRowsFunc(int32_t funcId); bool fmIsMultiRowsFunc(int32_t funcId);
bool fmIsKeepOrderFunc(int32_t funcId); bool fmIsKeepOrderFunc(int32_t funcId);
bool fmIsCumulativeFunc(int32_t funcId); bool fmIsCumulativeFunc(int32_t funcId);
bool fmIsInterpPseudoColumnFunc(int32_t funcId);
int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc); int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc);

View File

@ -119,6 +119,7 @@ typedef struct SFlushDatabaseStmt {
typedef struct STrimDatabaseStmt { typedef struct STrimDatabaseStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
int32_t maxSpeed;
} STrimDatabaseStmt; } STrimDatabaseStmt;
typedef struct STableOptions { typedef struct STableOptions {
@ -383,6 +384,8 @@ typedef struct SCreateStreamStmt {
bool ignoreExists; bool ignoreExists;
SStreamOptions* pOptions; SStreamOptions* pOptions;
SNode* pQuery; SNode* pQuery;
SNodeList* pTags;
SNode* pSubtable;
} SCreateStreamStmt; } SCreateStreamStmt;
typedef struct SDropStreamStmt { typedef struct SDropStreamStmt {

View File

@ -28,8 +28,8 @@ extern "C" {
#define LIST_LENGTH(l) (NULL != (l) ? (l)->length : 0) #define LIST_LENGTH(l) (NULL != (l) ? (l)->length : 0)
#define FOREACH(node, list) \ #define FOREACH(node, list) \
for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \ for (SListCell *cell = (NULL != (list) ? (list)->pHead : NULL), *pNext; \
(NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) (NULL != cell ? (node = cell->pNode, pNext = cell->pNext, true) : (node = NULL, pNext = NULL, false)); cell = pNext)
#define REPLACE_NODE(newNode) cell->pNode = (SNode*)(newNode) #define REPLACE_NODE(newNode) cell->pNode = (SNode*)(newNode)
@ -239,6 +239,7 @@ typedef enum ENodeType {
QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_FILL, QUERY_NODE_PHYSICAL_PLAN_FILL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL,
QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION,
QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION,
QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION, QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION,

View File

@ -94,6 +94,8 @@ typedef struct SScanLogicNode {
SArray* pSmaIndexes; SArray* pSmaIndexes;
SNodeList* pGroupTags; SNodeList* pGroupTags;
bool groupSort; bool groupSort;
SNodeList* pTags; // for create stream
SNode* pSubtable; // for create stream
int8_t cacheLastMode; int8_t cacheLastMode;
bool hasNormalCols; // neither tag column nor primary key tag column bool hasNormalCols; // neither tag column nor primary key tag column
bool sortPrimaryKey; bool sortPrimaryKey;
@ -233,6 +235,8 @@ typedef struct SSortLogicNode {
typedef struct SPartitionLogicNode { typedef struct SPartitionLogicNode {
SLogicNode node; SLogicNode node;
SNodeList* pPartitionKeys; SNodeList* pPartitionKeys;
SNodeList* pTags;
SNode* pSubtable;
} SPartitionLogicNode; } SPartitionLogicNode;
typedef enum ESubplanType { typedef enum ESubplanType {
@ -332,6 +336,8 @@ typedef struct STableScanPhysiNode {
SNodeList* pDynamicScanFuncs; SNodeList* pDynamicScanFuncs;
SNodeList* pGroupTags; SNodeList* pGroupTags;
bool groupSort; bool groupSort;
SNodeList* pTags;
SNode* pSubtable;
int64_t interval; int64_t interval;
int64_t offset; int64_t offset;
int64_t sliding; int64_t sliding;
@ -458,6 +464,8 @@ typedef struct SFillPhysiNode {
EOrder inputTsOrder; EOrder inputTsOrder;
} SFillPhysiNode; } SFillPhysiNode;
typedef SFillPhysiNode SStreamFillPhysiNode;
typedef struct SMultiTableIntervalPhysiNode { typedef struct SMultiTableIntervalPhysiNode {
SIntervalPhysiNode interval; SIntervalPhysiNode interval;
SNodeList* pPartitionKeys; SNodeList* pPartitionKeys;
@ -495,7 +503,11 @@ typedef struct SPartitionPhysiNode {
SNodeList* pTargets; SNodeList* pTargets;
} SPartitionPhysiNode; } SPartitionPhysiNode;
typedef SPartitionPhysiNode SStreamPartitionPhysiNode; typedef struct SStreamPartitionPhysiNode {
SPartitionPhysiNode part;
SNodeList* pTags;
SNode* pSubtable;
} SStreamPartitionPhysiNode;
typedef struct SDataSinkNode { typedef struct SDataSinkNode {
ENodeType type; ENodeType type;

View File

@ -261,6 +261,8 @@ typedef struct SSelectStmt {
SNode* pFromTable; SNode* pFromTable;
SNode* pWhere; SNode* pWhere;
SNodeList* pPartitionByList; SNodeList* pPartitionByList;
SNodeList* pTags; // for create stream
SNode* pSubtable; // for create stream
SNode* pWindow; SNode* pWindow;
SNodeList* pGroupByList; // SGroupingSetNode SNodeList* pGroupByList; // SGroupingSetNode
SNode* pHaving; SNode* pHaving;

View File

@ -31,6 +31,7 @@ typedef struct {
TDB* db; TDB* db;
TTB* pStateDb; TTB* pStateDb;
TTB* pFuncStateDb; TTB* pFuncStateDb;
TTB* pFillStateDb; // todo refactor
TXN txn; TXN txn;
} SStreamState; } SStreamState;
@ -51,15 +52,22 @@ int32_t streamStateFuncDel(SStreamState* pState, const STupleKey* key);
int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t streamStateDel(SStreamState* pState, const SWinKey* key); int32_t streamStateDel(SStreamState* pState, const SWinKey* key);
int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key);
int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal); int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal);
void streamFreeVal(void* val); void streamFreeVal(void* val);
SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key); SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key);
SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key); SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key);
SStreamStateCur* streamStateSeekKeyPrev(SStreamState* pState, const SWinKey* key); SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* key);
SStreamStateCur* streamStateFillSeekKeyPrev(SStreamState* pState, const SWinKey* key);
void streamStateFreeCur(SStreamStateCur* pCur); void streamStateFreeCur(SStreamStateCur* pCur);
int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t streamStateSeekFirst(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStateSeekFirst(SStreamState* pState, SStreamStateCur* pCur);

View File

@ -167,7 +167,7 @@ uint32_t ip2uint(const char *const ip_addr);
void taosIgnSIGPIPE(); void taosIgnSIGPIPE();
void taosSetMaskSIGPIPE(); void taosSetMaskSIGPIPE();
uint32_t taosInetAddr(const char *ipAddr); uint32_t taosInetAddr(const char *ipAddr);
const char *taosInetNtoa(struct in_addr ipInt); const char *taosInetNtoa(struct in_addr ipInt, char *dstStr, int32_t len);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -77,7 +77,6 @@ int32_t taosWcharsWidth(TdWchar *pWchar, int32_t size);
int32_t taosMbToWchar(TdWchar *pWchar, const char *pStr, int32_t size); int32_t taosMbToWchar(TdWchar *pWchar, const char *pStr, int32_t size);
int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size); int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size);
int32_t taosWcharToMb(char *pStr, TdWchar wchar); int32_t taosWcharToMb(char *pStr, TdWchar wchar);
int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size);
char *taosStrCaseStr(const char *str, const char *pattern); char *taosStrCaseStr(const char *str, const char *pattern);

View File

@ -1,22 +0,0 @@
[Unit]
Description=Nginx For TDengine Service
After=network-online.target
Wants=network-online.target
[Service]
Type=forking
PIDFile=/usr/local/nginxd/logs/nginx.pid
ExecStart=/usr/local/nginxd/sbin/nginx
ExecStop=/usr/local/nginxd/sbin/nginx -s stop
TimeoutStopSec=1000000s
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TimeoutStartSec=0
StandardOutput=null
Restart=always
StartLimitBurst=3
StartLimitInterval=60s
[Install]
WantedBy=multi-user.target

View File

@ -38,8 +38,6 @@ temp_version=""
fin_result="" fin_result=""
service_config_dir="/etc/systemd/system" service_config_dir="/etc/systemd/system"
nginx_port=6060
nginx_dir="/usr/local/nginxd"
# Color setting # Color setting
RED='\033[0;31m' RED='\033[0;31m'
@ -132,10 +130,7 @@ function check_main_path() {
check_file ${install_main_dir} $i check_file ${install_main_dir} $i
done done
if [ "$verMode" == "cluster" ]; then if [ "$verMode" == "cluster" ]; then
nginx_main_dir=("admin" "conf" "html" "sbin" "logs") check_file ${install_main_dir} "share/admin"
for i in "${nginx_main_dir[@]}";do
check_file ${nginx_dir} $i
done
fi fi
echo -e "Check main path:\033[32mOK\033[0m!" echo -e "Check main path:\033[32mOK\033[0m!"
} }
@ -150,9 +145,6 @@ function check_bin_path() {
for i in "${lbin_dir[@]}";do for i in "${lbin_dir[@]}";do
check_link ${bin_link_dir}/$i check_link ${bin_link_dir}/$i
done done
if [ "$verMode" == "cluster" ]; then
check_file ${nginx_dir}/sbin nginx
fi
echo -e "Check bin path:\033[32mOK\033[0m!" echo -e "Check bin path:\033[32mOK\033[0m!"
} }

View File

@ -50,8 +50,7 @@ install_main_dir=${installDir}
bin_dir="${installDir}/bin" bin_dir="${installDir}/bin"
service_config_dir="/etc/systemd/system" service_config_dir="/etc/systemd/system"
nginx_port=6060 web_port=6041
nginx_dir="/usr/local/nginxd"
# Color setting # Color setting
RED='\033[0;31m' RED='\033[0;31m'
@ -182,7 +181,7 @@ function install_main_path() {
${csudo}mkdir -p ${install_main_dir}/include ${csudo}mkdir -p ${install_main_dir}/include
# ${csudo}mkdir -p ${install_main_dir}/init.d # ${csudo}mkdir -p ${install_main_dir}/init.d
if [ "$verMode" == "cluster" ]; then if [ "$verMode" == "cluster" ]; then
${csudo}mkdir -p ${nginx_dir} ${csudo}mkdir -p ${install_main_dir}/share
fi fi
if [[ -e ${script_dir}/email ]]; then if [[ -e ${script_dir}/email ]]; then
@ -218,12 +217,6 @@ function install_bin() {
[ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || :
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :
[ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
if [ "$verMode" == "cluster" ]; then
${csudo}cp -r ${script_dir}/nginxd/* ${nginx_dir} && ${csudo}chmod 0555 ${nginx_dir}/*
${csudo}mkdir -p ${nginx_dir}/logs
${csudo}chmod 777 ${nginx_dir}/sbin/nginx
fi
} }
function install_lib() { function install_lib() {
@ -574,6 +567,13 @@ function install_examples() {
fi fi
} }
function install_web() {
if [ -d "${script_dir}/share" ]; then
${csudo}cp -rf ${script_dir}/share/* ${install_main_dir}/share
fi
}
function clean_service_on_sysvinit() { function clean_service_on_sysvinit() {
if pidof ${serverName} &>/dev/null; then if pidof ${serverName} &>/dev/null; then
${csudo}service ${serverName} stop || : ${csudo}service ${serverName} stop || :
@ -654,16 +654,6 @@ function clean_service_on_systemd() {
fi fi
${csudo}systemctl disable tarbitratord &>/dev/null || echo &>/dev/null ${csudo}systemctl disable tarbitratord &>/dev/null || echo &>/dev/null
${csudo}rm -f ${tarbitratord_service_config} ${csudo}rm -f ${tarbitratord_service_config}
if [ "$verMode" == "cluster" ]; then
nginx_service_config="${service_config_dir}/nginxd.service"
if systemctl is-active --quiet nginxd; then
echo "Nginx for ${productName} is running, stopping it..."
${csudo}systemctl stop nginxd &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable nginxd &>/dev/null || echo &>/dev/null
${csudo}rm -f ${nginx_service_config}
fi
} }
function install_service_on_systemd() { function install_service_on_systemd() {
@ -677,19 +667,6 @@ function install_service_on_systemd() {
${csudo}systemctl enable ${serverName} ${csudo}systemctl enable ${serverName}
${csudo}systemctl daemon-reload ${csudo}systemctl daemon-reload
if [ "$verMode" == "cluster" ]; then
[ -f ${script_dir}/cfg/nginxd.service ] &&
${csudo}cp ${script_dir}/cfg/nginxd.service \
${service_config_dir}/ || :
${csudo}systemctl daemon-reload
if ! ${csudo}systemctl enable nginxd &>/dev/null; then
${csudo}systemctl daemon-reexec
${csudo}systemctl enable nginxd
fi
${csudo}systemctl start nginxd
fi
} }
function install_adapter_service() { function install_adapter_service() {
@ -793,19 +770,6 @@ function updateProduct() {
sleep 1 sleep 1
fi fi
if [ "$verMode" == "cluster" ]; then
if pidof nginx &>/dev/null; then
if ((${service_mod} == 0)); then
${csudo}systemctl stop nginxd || :
elif ((${service_mod} == 1)); then
${csudo}service nginxd stop || :
else
kill_process nginx
fi
sleep 1
fi
fi
install_main_path install_main_path
install_log install_log
@ -817,6 +781,7 @@ function updateProduct() {
fi fi
install_examples install_examples
install_web
if [ -z $1 ]; then if [ -z $1 ]; then
install_bin install_bin
install_service install_service
@ -825,18 +790,6 @@ function updateProduct() {
install_adapter_config install_adapter_config
openresty_work=false openresty_work=false
if [ "$verMode" == "cluster" ]; then
# Check if openresty is installed
# Check if nginx is installed successfully
if type curl &>/dev/null; then
if curl -sSf http://127.0.0.1:${nginx_port} &>/dev/null; then
echo -e "\033[44;32;1mNginx for ${productName} is updated successfully!${NC}"
openresty_work=true
else
echo -e "\033[44;31;5mNginx for ${productName} does not work! Please try again!\033[0m"
fi
fi
fi
echo echo
echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${cfg_install_dir}/${configFile}" echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${cfg_install_dir}/${configFile}"
@ -857,7 +810,7 @@ function updateProduct() {
fi fi
if [ ${openresty_work} = 'true' ]; then if [ ${openresty_work} = 'true' ]; then
echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${web_port}${NC}"
else else
echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell${NC}" echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell${NC}"
fi fi
@ -906,6 +859,7 @@ function installProduct() {
install_connector install_connector
fi fi
install_examples install_examples
install_web
if [ -z $1 ]; then # install service and client if [ -z $1 ]; then # install service and client
# For installing new # For installing new
@ -915,17 +869,6 @@ function installProduct() {
install_adapter_config install_adapter_config
openresty_work=false openresty_work=false
if [ "$verMode" == "cluster" ]; then
# Check if nginx is installed successfully
if type curl &>/dev/null; then
if curl -sSf http://127.0.0.1:${nginx_port} &>/dev/null; then
echo -e "\033[44;32;1mNginx for ${productName} is installed successfully!${NC}"
openresty_work=true
else
echo -e "\033[44;31;5mNginx for ${productName} does not work! Please try again!\033[0m"
fi
fi
fi
install_config install_config

View File

@ -151,6 +151,7 @@ function install_main_path() {
${csudo}mkdir -p ${install_main_dir}/driver ${csudo}mkdir -p ${install_main_dir}/driver
${csudo}mkdir -p ${install_main_dir}/examples ${csudo}mkdir -p ${install_main_dir}/examples
${csudo}mkdir -p ${install_main_dir}/include ${csudo}mkdir -p ${install_main_dir}/include
${csudo}mkdir -p ${install_main_dir}/share
# ${csudo}mkdir -p ${install_main_dir}/init.d # ${csudo}mkdir -p ${install_main_dir}/init.d
else else
${csudo}rm -rf ${install_main_dir} || ${csudo}rm -rf ${install_main_2_dir} || : ${csudo}rm -rf ${install_main_dir} || ${csudo}rm -rf ${install_main_2_dir} || :
@ -161,6 +162,7 @@ function install_main_path() {
${csudo}mkdir -p ${install_main_dir}/driver || ${csudo}mkdir -p ${install_main_2_dir}/driver ${csudo}mkdir -p ${install_main_dir}/driver || ${csudo}mkdir -p ${install_main_2_dir}/driver
${csudo}mkdir -p ${install_main_dir}/examples || ${csudo}mkdir -p ${install_main_2_dir}/examples ${csudo}mkdir -p ${install_main_dir}/examples || ${csudo}mkdir -p ${install_main_2_dir}/examples
${csudo}mkdir -p ${install_main_dir}/include || ${csudo}mkdir -p ${install_main_2_dir}/include ${csudo}mkdir -p ${install_main_dir}/include || ${csudo}mkdir -p ${install_main_2_dir}/include
${csudo}mkdir -p ${install_main_dir}/share || ${csudo}mkdir -p ${install_main_2_dir}/share
fi fi
} }
@ -469,6 +471,16 @@ function install_examples() {
fi fi
} }
function install_web() {
if [ -d "${binary_dir}/build/share" ]; then
if [ "$osType" != "Darwin" ]; then
${csudo}cp -rf ${binary_dir}/build/share/* ${install_main_dir}/share || :
else
${csudo}cp -rf ${binary_dir}/build/share/* ${install_main_dir}/share || ${csudo}cp -rf ${binary_dir}/build/share/* ${install_main_2_dir}/share || :
fi
fi
}
function clean_service_on_sysvinit() { function clean_service_on_sysvinit() {
if pidof ${serverName} &>/dev/null; then if pidof ${serverName} &>/dev/null; then
${csudo}service ${serverName} stop || : ${csudo}service ${serverName} stop || :
@ -596,6 +608,7 @@ function update_TDengine() {
install_lib install_lib
# install_connector # install_connector
install_examples install_examples
install_web
install_bin install_bin
install_service install_service

View File

@ -107,7 +107,7 @@ else
fi fi
install_files="${script_dir}/install.sh" install_files="${script_dir}/install.sh"
nginx_dir="${top_dir}/../enterprise/src/plugins/web" web_dir="${top_dir}/../enterprise/src/plugins/web"
init_file_deb=${script_dir}/../deb/taosd init_file_deb=${script_dir}/../deb/taosd
init_file_rpm=${script_dir}/../rpm/taosd init_file_rpm=${script_dir}/../rpm/taosd
@ -132,10 +132,6 @@ if [ -f "${cfg_dir}/${serverName}.service" ]; then
cp ${cfg_dir}/${serverName}.service ${install_dir}/cfg || : cp ${cfg_dir}/${serverName}.service ${install_dir}/cfg || :
fi fi
if [ -f "${top_dir}/packaging/cfg/nginxd.service" ]; then
cp ${top_dir}/packaging/cfg/nginxd.service ${install_dir}/cfg || :
fi
mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || :
mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/${serverName}.deb mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/${serverName}.deb
mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/${serverName}.rpm mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/${serverName}.rpm
@ -222,16 +218,6 @@ if [ "$verMode" == "cluster" ]; then
sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh
mv remove_temp.sh ${install_dir}/bin/remove.sh mv remove_temp.sh ${install_dir}/bin/remove.sh
mkdir -p ${install_dir}/nginxd && cp -r ${nginx_dir}/* ${install_dir}/nginxd
cp ${nginx_dir}/png/taos.png ${install_dir}/nginxd/admin/images/taos.png
rm -rf ${install_dir}/nginxd/png
if [ "$cpuType" == "aarch64" ]; then
cp -f ${install_dir}/nginxd/sbin/arm/64bit/nginx ${install_dir}/nginxd/sbin/
elif [ "$cpuType" == "aarch32" ]; then
cp -f ${install_dir}/nginxd/sbin/arm/32bit/nginx ${install_dir}/nginxd/sbin/
fi
rm -rf ${install_dir}/nginxd/sbin/arm
fi fi
cd ${install_dir} cd ${install_dir}
@ -288,6 +274,13 @@ if [[ $dbName == "taos" ]]; then
cp -r ${examples_dir}/C# ${install_dir}/examples cp -r ${examples_dir}/C# ${install_dir}/examples
mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../tools/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../tools/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json
fi fi
# Add web files
if [ -d "${web_dir}/admin" ]; then
mkdir -p ${install_dir}/share/
cp ${web_dir}/admin ${install_dir}/share/ -r
cp ${web_dir}/png/taos.png ${install_dir}/share/admin/images/taos.png
fi
fi fi
# Copy driver # Copy driver

View File

@ -27,13 +27,11 @@ local_bin_link_dir="/usr/local/bin"
lib_link_dir="/usr/lib" lib_link_dir="/usr/lib"
lib64_link_dir="/usr/lib64" lib64_link_dir="/usr/lib64"
inc_link_dir="/usr/include" inc_link_dir="/usr/include"
install_nginxd_dir="/usr/local/nginxd"
service_config_dir="/etc/systemd/system" service_config_dir="/etc/systemd/system"
taos_service_name=${serverName} taos_service_name=${serverName}
taosadapter_service_name="taosadapter" taosadapter_service_name="taosadapter"
tarbitrator_service_name="tarbitratord" tarbitrator_service_name="tarbitratord"
nginx_service_name="nginxd"
csudo="" csudo=""
if command -v sudo >/dev/null; then if command -v sudo >/dev/null; then
csudo="sudo " csudo="sudo "
@ -153,18 +151,6 @@ function clean_service_on_systemd() {
fi fi
${csudo}systemctl disable ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null ${csudo}systemctl disable ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null
${csudo}rm -f ${tarbitratord_service_config} ${csudo}rm -f ${tarbitratord_service_config}
if [ "$verMode" == "cluster" ]; then
nginx_service_config="${service_config_dir}/${nginx_service_name}.service"
if [ -d ${install_nginxd_dir} ]; then
if systemctl is-active --quiet ${nginx_service_name}; then
echo "Nginx for ${productName} is running, stopping it..."
${csudo}systemctl stop ${nginx_service_name} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${nginx_service_name} &>/dev/null || echo &>/dev/null
${csudo}rm -f ${nginx_service_config}
fi
fi
} }
function clean_service_on_sysvinit() { function clean_service_on_sysvinit() {
@ -239,7 +225,6 @@ clean_config
${csudo}rm -rf ${data_link_dir} || : ${csudo}rm -rf ${data_link_dir} || :
${csudo}rm -rf ${install_main_dir} ${csudo}rm -rf ${install_main_dir}
${csudo}rm -rf ${install_nginxd_dir}
if [[ -e /etc/os-release ]]; then if [[ -e /etc/os-release ]]; then
osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release)
else else

View File

@ -19,6 +19,7 @@
#include "functionMgt.h" #include "functionMgt.h"
#include "os.h" #include "os.h"
#include "query.h" #include "query.h"
#include "qworker.h"
#include "scheduler.h" #include "scheduler.h"
#include "tcache.h" #include "tcache.h"
#include "tglobal.h" #include "tglobal.h"
@ -27,7 +28,6 @@
#include "trpc.h" #include "trpc.h"
#include "tsched.h" #include "tsched.h"
#include "ttime.h" #include "ttime.h"
#include "qworker.h"
#define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0 #define TSC_VAR_RELEASED 0

View File

@ -483,7 +483,7 @@ void setResPrecision(SReqResultInfo* pResInfo, int32_t precision) {
int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList, SArray* pDbVgList) { int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList, SArray* pDbVgList) {
SArray* nodeList = taosArrayInit(4, sizeof(SQueryNodeLoad)); SArray* nodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
char *policy = (tsQueryPolicy == QUERY_POLICY_VNODE) ? "vnode" : "client"; char* policy = (tsQueryPolicy == QUERY_POLICY_VNODE) ? "vnode" : "client";
int32_t dbNum = taosArrayGetSize(pDbVgList); int32_t dbNum = taosArrayGetSize(pDbVgList);
for (int32_t i = 0; i < dbNum; ++i) { for (int32_t i = 0; i < dbNum; ++i) {
@ -906,6 +906,8 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId);
pRequest->prevCode = code; pRequest->prevCode = code;
schedulerFreeJob(&pRequest->body.queryJob, 0); schedulerFreeJob(&pRequest->body.queryJob, 0);
qDestroyQuery(pRequest->pQuery);
pRequest->pQuery = NULL;
doAsyncQuery(pRequest, true); doAsyncQuery(pRequest, true);
return; return;
} }

View File

@ -20,13 +20,13 @@
#include "functionMgt.h" #include "functionMgt.h"
#include "os.h" #include "os.h"
#include "query.h" #include "query.h"
#include "qworker.h"
#include "scheduler.h" #include "scheduler.h"
#include "tglobal.h" #include "tglobal.h"
#include "tmsg.h" #include "tmsg.h"
#include "tref.h" #include "tref.h"
#include "trpc.h" #include "trpc.h"
#include "version.h" #include "version.h"
#include "qworker.h"
#define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0 #define TSC_VAR_RELEASED 0

View File

@ -79,7 +79,7 @@
#define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" " #define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" "
#define MAX_RETRY_TIMES 5 #define MAX_RETRY_TIMES 5
#define LINE_BATCH 20000 #define LINE_BATCH 2000
//================================================================================================= //=================================================================================================
typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType;
@ -151,13 +151,14 @@ typedef struct {
typedef struct { typedef struct {
SRequestObj *request; SRequestObj *request;
tsem_t sem; tsem_t sem;
int32_t cnt;
int32_t total;
TdThreadSpinlock lock; TdThreadSpinlock lock;
} Params; } Params;
typedef struct { typedef struct {
int64_t id; int64_t id;
Params *params; Params *params;
bool isLast;
SMLProtocolType protocol; SMLProtocolType protocol;
int8_t precision; int8_t precision;
@ -1531,6 +1532,7 @@ static SSmlHandle* smlBuildSmlInfo(STscObj* pTscObj, SRequestObj* request, SMLPr
info->pRequest = request; info->pRequest = request;
info->msgBuf.buf = info->pRequest->msgBuf; info->msgBuf.buf = info->pRequest->msgBuf;
info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE;
info->pRequest->stmtType = info->pQuery->pRoot->type;
} }
info->exec = smlInitHandle(info->pQuery); info->exec = smlInitHandle(info->pQuery);
@ -2331,6 +2333,9 @@ static int32_t smlInsertData(SSmlHandle *info) {
// info->affectedRows = taos_affected_rows(info->pRequest); // info->affectedRows = taos_affected_rows(info->pRequest);
// return info->pRequest->code; // return info->pRequest->code;
SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary;
atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1);
launchAsyncQuery(info->pRequest, info->pQuery, NULL); launchAsyncQuery(info->pRequest, info->pQuery, NULL);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -2449,28 +2454,26 @@ static void smlInsertCallback(void *param, void *res, int32_t code) {
int32_t rows = taos_affected_rows(pRequest); int32_t rows = taos_affected_rows(pRequest);
uDebug("SML:0x%" PRIx64 " result. code:%d, msg:%s", info->id, pRequest->code, pRequest->msgBuf); uDebug("SML:0x%" PRIx64 " result. code:%d, msg:%s", info->id, pRequest->code, pRequest->msgBuf);
// lock
taosThreadSpinLock(&info->params->lock);
if (code != TSDB_CODE_SUCCESS) {
info->params->request->code = code;
info->params->request->body.resInfo.numOfRows += rows;
}else{
info->params->request->body.resInfo.numOfRows += info->affectedRows;
}
taosThreadSpinUnlock(&info->params->lock);
// unlock
uDebug("SML:0x%" PRIx64 " insert finished, code: %d, rows: %d, total: %d", info->id, code, rows, info->affectedRows);
Params *pParam = info->params; Params *pParam = info->params;
bool isLast = info->isLast; // lock
taosThreadSpinLock(&pParam->lock);
pParam->cnt++;
if (code != TSDB_CODE_SUCCESS) {
pParam->request->code = code;
pParam->request->body.resInfo.numOfRows += rows;
}else{
pParam->request->body.resInfo.numOfRows += info->affectedRows;
}
if (pParam->cnt == pParam->total) {
tsem_post(&pParam->sem);
}
taosThreadSpinUnlock(&pParam->lock);
// unlock
uDebug("SML:0x%" PRIx64 " insert finished, code: %d, rows: %d, total: %d", info->id, code, rows, info->affectedRows);
info->cost.endTime = taosGetTimestampUs(); info->cost.endTime = taosGetTimestampUs();
info->cost.code = code; info->cost.code = code;
smlPrintStatisticInfo(info); smlPrintStatisticInfo(info);
smlDestroyInfo(info); smlDestroyInfo(info);
if (isLast) {
tsem_post(&pParam->sem);
}
} }
/** /**
@ -2512,7 +2515,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
pTscObj->schemalessType = 1; pTscObj->schemalessType = 1;
SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf};
Params params; Params params = {0};
params.request = request; params.request = request;
tsem_init(&params.sem, 0, 0); tsem_init(&params.sem, 0, 0);
taosThreadSpinInit(&(params.lock), 0); taosThreadSpinInit(&(params.lock), 0);
@ -2557,6 +2560,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
} }
batchs = ceil(((double)numLines) / LINE_BATCH); batchs = ceil(((double)numLines) / LINE_BATCH);
params.total = batchs;
for (int i = 0; i < batchs; ++i) { for (int i = 0; i < batchs; ++i) {
SRequestObj* req = (SRequestObj*)createRequest(pTscObj->id, TSDB_SQL_INSERT); SRequestObj* req = (SRequestObj*)createRequest(pTscObj->id, TSDB_SQL_INSERT);
if(!req){ if(!req){
@ -2575,11 +2579,9 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
if (numLines > perBatch) { if (numLines > perBatch) {
numLines -= perBatch; numLines -= perBatch;
info->isLast = false;
} else { } else {
perBatch = numLines; perBatch = numLines;
numLines = 0; numLines = 0;
info->isLast = true;
} }
info->params = &params; info->params = &params;

View File

@ -85,8 +85,7 @@ uint16_t tsTelemPort = 80;
char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null"; char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null";
char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; // user defined child table name can be specified in tag value. char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; // user defined child table name can be specified in tag value.
// If set to empty system will generate table name using MD5 hash. // If set to empty system will generate table name using MD5 hash.
bool tsSmlDataFormat = bool tsSmlDataFormat = false; // true means that the name and order of cols in each line are the same(only for influx protocol)
true; // true means that the name and order of cols in each line are the same(only for influx protocol)
// query // query
int32_t tsQueryPolicy = 1; int32_t tsQueryPolicy = 1;
@ -95,6 +94,7 @@ int32_t tsQueryRsmaTolerance = 1000; // the tolerance time (ms) to judge from w
bool tsQueryPlannerTrace = false; bool tsQueryPlannerTrace = false;
int32_t tsQueryNodeChunkSize = 32 * 1024; int32_t tsQueryNodeChunkSize = 32 * 1024;
bool tsQueryUseNodeAllocator = true; bool tsQueryUseNodeAllocator = true;
bool tsKeepColumnName = false;
/* /*
* denote if the server needs to compress response message at the application layer to client, including query rsp, * denote if the server needs to compress response message at the application layer to client, including query rsp,
@ -205,7 +205,9 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input
tstrncpy(cfgFile, cfgDir, sizeof(cfgDir)); tstrncpy(cfgFile, cfgDir, sizeof(cfgDir));
} }
if (apolloUrl == NULL || apolloUrl[0] == '\0') cfgGetApollUrl(envCmd, envFile, apolloUrl); if (apolloUrl != NULL && apolloUrl[0] == '\0') {
cfgGetApollUrl(envCmd, envFile, apolloUrl);
}
if (cfgLoad(pCfg, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) { if (cfgLoad(pCfg, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) {
uError("failed to load from apollo url:%s since %s", apolloUrl, terrstr()); uError("failed to load from apollo url:%s since %s", apolloUrl, terrstr());
@ -290,6 +292,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, true) != 0) return -1; if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, true) != 0) return -1;
if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, true) != 0) return -1; if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, true) != 0) return -1;
if (cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, true) != 0) return -1; if (cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, true) != 0) return -1;
if (cfgAddBool(pCfg, "keepColumnName", tsKeepColumnName, true) != 0) return -1;
if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1; if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1;
if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1; if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1;
if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1;
@ -652,6 +655,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval; tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval;
tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32; tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32;
tsQueryUseNodeAllocator = cfgGetItem(pCfg, "queryUseNodeAllocator")->bval; tsQueryUseNodeAllocator = cfgGetItem(pCfg, "queryUseNodeAllocator")->bval;
tsKeepColumnName = cfgGetItem(pCfg, "keepColumnName")->bval;
return 0; return 0;
} }
@ -685,7 +689,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32; tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32;
tsPrintAuth = cfgGetItem(pCfg, "printAuth")->bval; tsPrintAuth = cfgGetItem(pCfg, "printAuth")->bval;
#if !defined(WINDOWS) && !defined(DARWIN)
tsMultiProcess = cfgGetItem(pCfg, "multiProcess")->bval; tsMultiProcess = cfgGetItem(pCfg, "multiProcess")->bval;
#endif
tsMnodeShmSize = cfgGetItem(pCfg, "mnodeShmSize")->i32; tsMnodeShmSize = cfgGetItem(pCfg, "mnodeShmSize")->i32;
tsVnodeShmSize = cfgGetItem(pCfg, "vnodeShmSize")->i32; tsVnodeShmSize = cfgGetItem(pCfg, "vnodeShmSize")->i32;
tsQnodeShmSize = cfgGetItem(pCfg, "qnodeShmSize")->i32; tsQnodeShmSize = cfgGetItem(pCfg, "qnodeShmSize")->i32;
@ -843,6 +849,9 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) {
break; break;
} }
case 'k': { case 'k': {
if (strcasecmp("keepColumnName", name) == 0) {
tsKeepColumnName = cfgGetItem(pCfg, "keepColumnName")->bval;
}
break; break;
} }
case 'l': { case 'l': {
@ -919,7 +928,9 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) {
} }
case 'u': { case 'u': {
if (strcasecmp("multiProcess", name) == 0) { if (strcasecmp("multiProcess", name) == 0) {
#if !defined(WINDOWS) && !defined(DARWIN)
tsMultiProcess = cfgGetItem(pCfg, "multiProcess")->bval; tsMultiProcess = cfgGetItem(pCfg, "multiProcess")->bval;
#endif
} else if (strcasecmp("udfDebugFlag", name) == 0) { } else if (strcasecmp("udfDebugFlag", name) == 0) {
udfDebugFlag = cfgGetItem(pCfg, "udfDebugFlag")->i32; udfDebugFlag = cfgGetItem(pCfg, "udfDebugFlag")->i32;
} }
@ -1122,11 +1133,20 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
if (tsc) { if (tsc) {
tsLogEmbedded = 0; tsLogEmbedded = 0;
if (taosAddClientLogCfg(pCfg) != 0) return -1; if (taosAddClientLogCfg(pCfg) != 0) {
cfgCleanup(pCfg);
return -1;
}
} else { } else {
tsLogEmbedded = 1; tsLogEmbedded = 1;
if (taosAddClientLogCfg(pCfg) != 0) return -1; if (taosAddClientLogCfg(pCfg) != 0) {
if (taosAddServerLogCfg(pCfg) != 0) return -1; cfgCleanup(pCfg);
return -1;
}
if (taosAddServerLogCfg(pCfg) != 0) {
cfgCleanup(pCfg);
return -1;
}
} }
if (taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) { if (taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) {

View File

@ -2682,6 +2682,7 @@ int32_t tSerializeSTrimDbReq(void *buf, int32_t bufLen, STrimDbReq *pReq) {
if (tStartEncode(&encoder) < 0) return -1; if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
if (tEncodeI32(&encoder, pReq->maxSpeed) < 0) return -1;
tEndEncode(&encoder); tEndEncode(&encoder);
int32_t tlen = encoder.pos; int32_t tlen = encoder.pos;
@ -2695,6 +2696,7 @@ int32_t tDeserializeSTrimDbReq(void *buf, int32_t bufLen, STrimDbReq *pReq) {
if (tStartDecode(&decoder) < 0) return -1; if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->maxSpeed) < 0) return -1;
tEndDecode(&decoder); tEndDecode(&decoder);
tDecoderClear(&decoder); tDecoderClear(&decoder);
@ -2706,7 +2708,8 @@ int32_t tSerializeSVTrimDbReq(void *buf, int32_t bufLen, SVTrimDbReq *pReq) {
tEncoderInit(&encoder, buf, bufLen); tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1; if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->timestamp) < 0) return -1; if (tEncodeI64(&encoder, pReq->timestamp) < 0) return -1;
if (tEncodeI64(&encoder, pReq->maxSpeed) < 0) return -1;
tEndEncode(&encoder); tEndEncode(&encoder);
int32_t tlen = encoder.pos; int32_t tlen = encoder.pos;
@ -2719,7 +2722,8 @@ int32_t tDeserializeSVTrimDbReq(void *buf, int32_t bufLen, SVTrimDbReq *pReq) {
tDecoderInit(&decoder, buf, bufLen); tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1; if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->timestamp) < 0) return -1; if (tDecodeI64(&decoder, &pReq->timestamp) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->maxSpeed) < 0) return -1;
tEndDecode(&decoder); tEndDecode(&decoder);
tDecoderClear(&decoder); tDecoderClear(&decoder);
@ -4373,8 +4377,7 @@ int32_t tDeserializeSExplainRsp(void *buf, int32_t bufLen, SExplainRsp *pRsp) {
if (tDecodeDouble(&decoder, &pRsp->subplanInfo[i].totalCost) < 0) return -1; if (tDecodeDouble(&decoder, &pRsp->subplanInfo[i].totalCost) < 0) return -1;
if (tDecodeU64(&decoder, &pRsp->subplanInfo[i].numOfRows) < 0) return -1; if (tDecodeU64(&decoder, &pRsp->subplanInfo[i].numOfRows) < 0) return -1;
if (tDecodeU32(&decoder, &pRsp->subplanInfo[i].verboseLen) < 0) return -1; if (tDecodeU32(&decoder, &pRsp->subplanInfo[i].verboseLen) < 0) return -1;
if (tDecodeBinaryAlloc(&decoder, &pRsp->subplanInfo[i].verboseInfo, NULL) < 0) if (tDecodeBinaryAlloc(&decoder, &pRsp->subplanInfo[i].verboseInfo, NULL) < 0) return -1;
return -1;
} }
tEndDecode(&decoder); tEndDecode(&decoder);
@ -4826,6 +4829,14 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
if (tEncodeI8(&encoder, pReq->igExpired) < 0) return -1; if (tEncodeI8(&encoder, pReq->igExpired) < 0) return -1;
if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1; if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1; if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1;
if (tEncodeI32(&encoder, pReq->numOfTags) < 0) return -1;
for (int32_t i = 0; i < pReq->numOfTags; ++i) {
SField *pField = taosArrayGet(pReq->pTags, i);
if (tEncodeI8(&encoder, pField->type) < 0) return -1;
if (tEncodeI8(&encoder, pField->flags) < 0) return -1;
if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
}
tEndEncode(&encoder); tEndEncode(&encoder);
@ -4864,6 +4875,28 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea
if (pReq->ast == NULL) return -1; if (pReq->ast == NULL) return -1;
if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1;
} }
if (tDecodeI32(&decoder, &pReq->numOfTags) < 0) return -1;
if (pReq->numOfTags > 0) {
pReq->pTags = taosArrayInit(pReq->numOfTags, sizeof(SField));
if (pReq->pTags == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
for (int32_t i = 0; i < pReq->numOfTags; ++i) {
SField field = {0};
if (tDecodeI8(&decoder, &field.type) < 0) return -1;
if (tDecodeI8(&decoder, &field.flags) < 0) return -1;
if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
if (taosArrayPush(pReq->pTags, &field) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
}
}
tEndDecode(&decoder); tEndDecode(&decoder);
tDecoderClear(&decoder); tDecoderClear(&decoder);

View File

@ -16,10 +16,12 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "dmMgmt.h" #include "dmMgmt.h"
#include "tconfig.h" #include "tconfig.h"
#include "mnode.h"
#define DM_APOLLO_URL "The apollo string to use when configuring the server, such as: -a 'jsonFile:./tests/cfg.json', cfg.json text can be '{\"fqdn\":\"td1\"}'." #define DM_APOLLO_URL "The apollo string to use when configuring the server, such as: -a 'jsonFile:./tests/cfg.json', cfg.json text can be '{\"fqdn\":\"td1\"}'."
#define DM_CFG_DIR "Configuration directory." #define DM_CFG_DIR "Configuration directory."
#define DM_DMP_CFG "Dump configuration." #define DM_DMP_CFG "Dump configuration."
#define DM_SDB_INFO "Dump sdb info."
#define DM_ENV_CMD "The env cmd variable string to use when configuring the server, such as: -e 'TAOS_FQDN=td1'." #define DM_ENV_CMD "The env cmd variable string to use when configuring the server, such as: -e 'TAOS_FQDN=td1'."
#define DM_ENV_FILE "The env variable file path to use when configuring the server, default is './.env', .env text can be 'TAOS_FQDN=td1'." #define DM_ENV_FILE "The env variable file path to use when configuring the server, default is './.env', .env text can be 'TAOS_FQDN=td1'."
#define DM_NODE_TYPE "Startup type of the node, default is 0." #define DM_NODE_TYPE "Startup type of the node, default is 0."
@ -31,6 +33,7 @@ static struct {
bool winServiceMode; bool winServiceMode;
#endif #endif
bool dumpConfig; bool dumpConfig;
bool dumpSdb;
bool generateGrant; bool generateGrant;
bool printAuth; bool printAuth;
bool printVersion; bool printVersion;
@ -82,6 +85,8 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
} }
} else if (strcmp(argv[i], "-a") == 0) { } else if (strcmp(argv[i], "-a") == 0) {
tstrncpy(global.apolloUrl, argv[++i], PATH_MAX); tstrncpy(global.apolloUrl, argv[++i], PATH_MAX);
} else if (strcmp(argv[i], "-s") == 0) {
global.dumpSdb = true;
} else if (strcmp(argv[i], "-E") == 0) { } else if (strcmp(argv[i], "-E") == 0) {
tstrncpy(global.envFile, argv[++i], PATH_MAX); tstrncpy(global.envFile, argv[++i], PATH_MAX);
} else if (strcmp(argv[i], "-n") == 0) { } else if (strcmp(argv[i], "-n") == 0) {
@ -131,6 +136,7 @@ static void dmPrintHelp() {
printf("Usage: taosd [OPTION...] \n\n"); printf("Usage: taosd [OPTION...] \n\n");
printf("%s%s%s%s\n", indent, "-a,", indent, DM_APOLLO_URL); printf("%s%s%s%s\n", indent, "-a,", indent, DM_APOLLO_URL);
printf("%s%s%s%s\n", indent, "-c,", indent, DM_CFG_DIR); printf("%s%s%s%s\n", indent, "-c,", indent, DM_CFG_DIR);
printf("%s%s%s%s\n", indent, "-s,", indent, DM_SDB_INFO);
printf("%s%s%s%s\n", indent, "-C,", indent, DM_DMP_CFG); printf("%s%s%s%s\n", indent, "-C,", indent, DM_DMP_CFG);
printf("%s%s%s%s\n", indent, "-e,", indent, DM_ENV_CMD); printf("%s%s%s%s\n", indent, "-e,", indent, DM_ENV_CMD);
printf("%s%s%s%s\n", indent, "-E,", indent, DM_ENV_FILE); printf("%s%s%s%s\n", indent, "-E,", indent, DM_ENV_FILE);
@ -229,6 +235,14 @@ int mainWindows(int argc,char** argv) {
return 0; return 0;
} }
if (global.dumpSdb) {
mndDumpSdb();
taosCleanupCfg();
taosCloseLog();
taosCleanupArgs();
return 0;
}
dmSetProcInfo(argc, (char **)argv); dmSetProcInfo(argc, (char **)argv);
taosCleanupArgs(); taosCleanupArgs();

View File

@ -87,6 +87,7 @@ int32_t qmPutRpcMsgToQueue(SQnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
return 0; return 0;
default: default:
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
taosFreeQitem(pMsg);
return -1; return -1;
} }
} }

View File

@ -135,12 +135,14 @@ _OVER:
if (content != NULL) taosMemoryFree(content); if (content != NULL) taosMemoryFree(content);
if (root != NULL) cJSON_Delete(root); if (root != NULL) cJSON_Delete(root);
if (pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
if (*ppCfgs == NULL && pCfgs != NULL) taosMemoryFree(pCfgs);
terrno = code; terrno = code;
return code; return code;
} }
int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) { int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
int32_t ret = 0;
char file[PATH_MAX] = {0}; char file[PATH_MAX] = {0};
char realfile[PATH_MAX] = {0}; char realfile[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%svnodes.json.bak", pMgmt->path, TD_DIRSEP); snprintf(file, sizeof(file), "%s%svnodes.json.bak", pMgmt->path, TD_DIRSEP);
@ -161,13 +163,16 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
char *content = taosMemoryCalloc(1, maxLen + 1); char *content = taosMemoryCalloc(1, maxLen + 1);
if (content == NULL) { if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; ret = -1;
goto _OVER;
} }
len += snprintf(content + len, maxLen - len, "{\n"); len += snprintf(content + len, maxLen - len, "{\n");
len += snprintf(content + len, maxLen - len, " \"vnodes\": [\n"); len += snprintf(content + len, maxLen - len, " \"vnodes\": [\n");
for (int32_t i = 0; i < numOfVnodes; ++i) { for (int32_t i = 0; i < numOfVnodes; ++i) {
SVnodeObj *pVnode = pVnodes[i]; SVnodeObj *pVnode = pVnodes[i];
if (pVnode == NULL) continue;
len += snprintf(content + len, maxLen - len, " {\n"); len += snprintf(content + len, maxLen - len, " {\n");
len += snprintf(content + len, maxLen - len, " \"vgId\": %d,\n", pVnode->vgId); len += snprintf(content + len, maxLen - len, " \"vgId\": %d,\n", pVnode->vgId);
len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pVnode->dropped); len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pVnode->dropped);
@ -180,12 +185,13 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
} }
len += snprintf(content + len, maxLen - len, " ]\n"); len += snprintf(content + len, maxLen - len, " ]\n");
len += snprintf(content + len, maxLen - len, "}\n"); len += snprintf(content + len, maxLen - len, "}\n");
terrno = 0;
_OVER:
taosWriteFile(pFile, content, len); taosWriteFile(pFile, content, len);
taosFsyncFile(pFile); taosFsyncFile(pFile);
taosCloseFile(&pFile); taosCloseFile(&pFile);
taosMemoryFree(content); taosMemoryFree(content);
terrno = 0;
for (int32_t i = 0; i < numOfVnodes; ++i) { for (int32_t i = 0; i < numOfVnodes; ++i) {
SVnodeObj *pVnode = pVnodes[i]; SVnodeObj *pVnode = pVnodes[i];
@ -196,6 +202,8 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
taosMemoryFree(pVnodes); taosMemoryFree(pVnodes);
} }
if (ret != 0) return -1;
dDebug("successed to write %s, numOfVnodes:%d", realfile, numOfVnodes); dDebug("successed to write %s, numOfVnodes:%d", realfile, numOfVnodes);
return taosRenameFile(file, realfile); return taosRenameFile(file, realfile);
} }

View File

@ -175,7 +175,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->hashSuffix = pCreate->hashSuffix; pCfg->hashSuffix = pCreate->hashSuffix;
pCfg->tsdbPageSize = pCreate->tsdbPageSize * 1024; pCfg->tsdbPageSize = pCreate->tsdbPageSize * 1024;
pCfg->standby = pCfg->standby; pCfg->standby = 0;
pCfg->syncCfg.myIndex = pCreate->selfIndex; pCfg->syncCfg.myIndex = pCreate->selfIndex;
pCfg->syncCfg.replicaNum = pCreate->replica; pCfg->syncCfg.replicaNum = pCreate->replica;
memset(&pCfg->syncCfg.nodeInfo, 0, sizeof(pCfg->syncCfg.nodeInfo)); memset(&pCfg->syncCfg.nodeInfo, 0, sizeof(pCfg->syncCfg.nodeInfo));

View File

@ -58,11 +58,14 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
if (pVnode->path == NULL) { if (pVnode->path == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pVnode);
return -1; return -1;
} }
if (vmAllocQueue(pMgmt, pVnode) != 0) { if (vmAllocQueue(pMgmt, pVnode) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pVnode->path);
taosMemoryFree(pVnode);
return -1; return -1;
} }
@ -221,6 +224,7 @@ static void vmCloseVnodes(SVnodeMgmt *pMgmt) {
SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes);
for (int32_t i = 0; i < numOfVnodes; ++i) { for (int32_t i = 0; i < numOfVnodes; ++i) {
if (ppVnodes == NULL || ppVnodes[i] == NULL) continue;
vmCloseVnode(pMgmt, ppVnodes[i]); vmCloseVnode(pMgmt, ppVnodes[i]);
} }
@ -380,8 +384,10 @@ static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) {
for (int32_t v = 0; v < numOfVnodes; ++v) { for (int32_t v = 0; v < numOfVnodes; ++v) {
int32_t t = v % threadNum; int32_t t = v % threadNum;
SVnodeThread *pThread = &threads[t]; SVnodeThread *pThread = &threads[t];
if (pThread->ppVnodes != NULL) {
pThread->ppVnodes[pThread->vnodeNum++] = ppVnodes[v]; pThread->ppVnodes[pThread->vnodeNum++] = ppVnodes[v];
} }
}
pMgmt->state.openVnodes = 0; pMgmt->state.openVnodes = 0;
dInfo("restore %d vnodes with %d threads", numOfVnodes, threadNum); dInfo("restore %d vnodes with %d threads", numOfVnodes, threadNum);
@ -411,8 +417,8 @@ static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) {
taosMemoryFree(threads); taosMemoryFree(threads);
for (int32_t i = 0; i < numOfVnodes; ++i) { for (int32_t i = 0; i < numOfVnodes; ++i) {
SVnodeObj *pVnode = ppVnodes[i]; if (ppVnodes == NULL || ppVnodes[i] == NULL) continue;
vmReleaseVnode(pMgmt, pVnode); vmReleaseVnode(pMgmt, ppVnodes[i]);
} }
if (ppVnodes != NULL) { if (ppVnodes != NULL) {

View File

@ -40,6 +40,8 @@ int32_t mndBuildSMCreateStbRsp(SMnode *pMnode, char* dbFName, char* stbFName, vo
void mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst); void mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst);
void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t dstSize); void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t dstSize);
const char *mndGetStbStr(const char *src);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -315,7 +315,7 @@ static int32_t mndProcessUptimeTimer(SRpcMsg *pReq) {
return 0; return 0;
} }
mTrace("update cluster uptime to %" PRId64, clusterObj.upTime); mInfo("update cluster uptime to %" PRId64, clusterObj.upTime);
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "update-uptime"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "update-uptime");
if (pTrans == NULL) return -1; if (pTrans == NULL) return -1;

View File

@ -54,13 +54,15 @@ static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg);
static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg); static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg);
int32_t mndInitConsumer(SMnode *pMnode) { int32_t mndInitConsumer(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_CONSUMER, SSdbTable table = {
.sdbType = SDB_CONSUMER,
.keyType = SDB_KEY_INT64, .keyType = SDB_KEY_INT64,
.encodeFp = (SdbEncodeFp)mndConsumerActionEncode, .encodeFp = (SdbEncodeFp)mndConsumerActionEncode,
.decodeFp = (SdbDecodeFp)mndConsumerActionDecode, .decodeFp = (SdbDecodeFp)mndConsumerActionDecode,
.insertFp = (SdbInsertFp)mndConsumerActionInsert, .insertFp = (SdbInsertFp)mndConsumerActionInsert,
.updateFp = (SdbUpdateFp)mndConsumerActionUpdate, .updateFp = (SdbUpdateFp)mndConsumerActionUpdate,
.deleteFp = (SdbDeleteFp)mndConsumerActionDelete}; .deleteFp = (SdbDeleteFp)mndConsumerActionDelete,
};
mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq);
mndSetMsgHandle(pMnode, TDMT_MND_MQ_HB, mndProcessMqHbReq); mndSetMsgHandle(pMnode, TDMT_MND_MQ_HB, mndProcessMqHbReq);
@ -176,6 +178,8 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
SMqConsumerObj *pConsumer; SMqConsumerObj *pConsumer;
void *pIter = NULL; void *pIter = NULL;
mTrace("start to process mq timer");
// rebalance cannot be parallel // rebalance cannot be parallel
if (!mndRebTryStart()) { if (!mndRebTryStart()) {
mInfo("mq rebalance already in progress, do nothing"); mInfo("mq rebalance already in progress, do nothing");

View File

@ -1385,11 +1385,19 @@ _OVER:
return code; return code;
} }
static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb) { /**
* @brief trim database
*
* @param pMnode
* @param pDb
* @param maxSpeed MB/s
* @return int32_t
*/
static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb, int32_t maxSpeed) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL; SVgObj *pVgroup = NULL;
void *pIter = NULL; void *pIter = NULL;
SVTrimDbReq trimReq = {.timestamp = taosGetTimestampSec()}; SVTrimDbReq trimReq = {.timestamp = taosGetTimestampMs(), .maxSpeed = maxSpeed << 20};
int32_t reqLen = tSerializeSVTrimDbReq(NULL, 0, &trimReq); int32_t reqLen = tSerializeSVTrimDbReq(NULL, 0, &trimReq);
int32_t contLen = reqLen + sizeof(SMsgHead); int32_t contLen = reqLen + sizeof(SMsgHead);
@ -1413,7 +1421,8 @@ static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb) {
if (code != 0) { if (code != 0) {
mError("vgId:%d, failed to send vnode-trim request to vnode since 0x%x", pVgroup->vgId, code); mError("vgId:%d, failed to send vnode-trim request to vnode since 0x%x", pVgroup->vgId, code);
} else { } else {
mInfo("vgId:%d, send vnode-trim request to vnode, time:%d", pVgroup->vgId, trimReq.timestamp); mInfo("vgId:%d, send vnode-trim request to vnode, time:%" PRIi64 ", max speed:%" PRIi64, pVgroup->vgId,
trimReq.timestamp, trimReq.maxSpeed);
} }
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
} }
@ -1443,7 +1452,7 @@ static int32_t mndProcessTrimDbReq(SRpcMsg *pReq) {
goto _OVER; goto _OVER;
} }
code = mndTrimDb(pMnode, pDb); code = mndTrimDb(pMnode, pDb, trimReq.maxSpeed);
_OVER: _OVER:
if (code != 0) { if (code != 0) {

View File

@ -0,0 +1,645 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "mndDb.h"
#include "mndInt.h"
#include "mndStb.h"
#include "sdb.h"
#include "tconfig.h"
#include "tjson.h"
#include "ttypes.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
void reportStartup(const char *name, const char *desc) {}
void sendRsp(SRpcMsg *pMsg) { rpcFreeCont(pMsg->pCont); }
int32_t sendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
}
char *i642str(int64_t val) {
static char str[24] = {0};
snprintf(str, sizeof(str), "%" PRId64, val);
return str;
}
void dumpFunc(SSdb *pSdb, SJson *json) {
void *pIter = NULL;
SJson *items = tjsonAddArrayToObject(json, "funcs");
while (1) {
SFuncObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_FUNC, pIter, (void **)&pObj);
if (pIter == NULL) break;
SJson *item = tjsonCreateObject();
tjsonAddItemToArray(items, item);
tjsonAddStringToObject(item, "name", pObj->name);
tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
tjsonAddStringToObject(item, "funcType", i642str(pObj->funcType));
tjsonAddStringToObject(item, "scriptType", i642str(pObj->scriptType));
tjsonAddStringToObject(item, "align", i642str(pObj->align));
tjsonAddStringToObject(item, "outputType", i642str(pObj->outputType));
tjsonAddStringToObject(item, "outputLen", i642str(pObj->outputLen));
tjsonAddStringToObject(item, "bufSize", i642str(pObj->bufSize));
tjsonAddStringToObject(item, "signature", i642str(pObj->signature));
tjsonAddStringToObject(item, "commentSize", i642str(pObj->commentSize));
tjsonAddStringToObject(item, "codeSize", i642str(pObj->codeSize));
sdbRelease(pSdb, pObj);
}
}
void dumpDb(SSdb *pSdb, SJson *json) {
void *pIter = NULL;
SJson *items = tjsonCreateObject();
tjsonAddItemToObject(json, "dbs", items);
while (1) {
SDbObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pObj);
if (pIter == NULL) break;
SJson *item = tjsonCreateObject();
tjsonAddItemToObject(items, "db", item);
tjsonAddStringToObject(item, "name", mndGetDbStr(pObj->name));
tjsonAddStringToObject(item, "acct", pObj->acct);
tjsonAddStringToObject(item, "createUser", pObj->createUser);
tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
tjsonAddStringToObject(item, "uid", i642str(pObj->uid));
tjsonAddStringToObject(item, "cfgVersion", i642str(pObj->cfgVersion));
tjsonAddStringToObject(item, "vgVersion", i642str(pObj->vgVersion));
tjsonAddStringToObject(item, "numOfVgroups", i642str(pObj->cfg.numOfVgroups));
tjsonAddStringToObject(item, "numOfStables", i642str(pObj->cfg.numOfStables));
tjsonAddStringToObject(item, "buffer", i642str(pObj->cfg.buffer));
tjsonAddStringToObject(item, "pageSize", i642str(pObj->cfg.pageSize));
tjsonAddStringToObject(item, "pages", i642str(pObj->cfg.pages));
tjsonAddStringToObject(item, "cacheLastSize", i642str(pObj->cfg.cacheLastSize));
tjsonAddStringToObject(item, "daysPerFile", i642str(pObj->cfg.daysPerFile));
tjsonAddStringToObject(item, "daysToKeep0", i642str(pObj->cfg.daysToKeep0));
tjsonAddStringToObject(item, "daysToKeep1", i642str(pObj->cfg.daysToKeep1));
tjsonAddStringToObject(item, "daysToKeep2", i642str(pObj->cfg.daysToKeep2));
tjsonAddStringToObject(item, "minRows", i642str(pObj->cfg.minRows));
tjsonAddStringToObject(item, "maxRows", i642str(pObj->cfg.maxRows));
tjsonAddStringToObject(item, "precision", i642str(pObj->cfg.precision));
tjsonAddStringToObject(item, "compression", i642str(pObj->cfg.compression));
tjsonAddStringToObject(item, "replications", i642str(pObj->cfg.replications));
tjsonAddStringToObject(item, "strict", i642str(pObj->cfg.strict));
tjsonAddStringToObject(item, "cacheLast",i642str( pObj->cfg.cacheLast));
tjsonAddStringToObject(item, "hashMethod", i642str(pObj->cfg.hashMethod));
tjsonAddStringToObject(item, "hashPrefix", i642str(pObj->cfg.hashPrefix));
tjsonAddStringToObject(item, "hashSuffix", i642str(pObj->cfg.hashSuffix));
tjsonAddStringToObject(item, "sstTrigger", i642str(pObj->cfg.sstTrigger));
tjsonAddStringToObject(item, "tsdbPageSize",i642str( pObj->cfg.tsdbPageSize));
tjsonAddStringToObject(item, "schemaless", i642str(pObj->cfg.schemaless));
tjsonAddStringToObject(item, "walLevel",i642str( pObj->cfg.walLevel));
tjsonAddStringToObject(item, "walFsyncPeriod", i642str(pObj->cfg.walFsyncPeriod));
tjsonAddStringToObject(item, "walRetentionPeriod", i642str(pObj->cfg.walRetentionPeriod));
tjsonAddStringToObject(item, "walRetentionSize",i642str( pObj->cfg.walRetentionSize));
tjsonAddStringToObject(item, "walRollPeriod", i642str(pObj->cfg.walRollPeriod));
tjsonAddStringToObject(item, "walSegmentSize", i642str(pObj->cfg.walSegmentSize));
tjsonAddStringToObject(item, "numOfRetensions",i642str( pObj->cfg.numOfRetensions));
for (int32_t i = 0; i < pObj->cfg.numOfRetensions; ++i) {
SJson *rentensions = tjsonAddArrayToObject(item, "rentensions");
SJson *rentension = tjsonCreateObject();
tjsonAddItemToArray(rentensions, rentension);
SRetention *pRetension = taosArrayGet(pObj->cfg.pRetensions, i);
tjsonAddStringToObject(item, "freq", i642str(pRetension->freq));
tjsonAddStringToObject(item, "freqUnit", i642str(pRetension->freqUnit));
tjsonAddStringToObject(item, "keep", i642str(pRetension->keep));
tjsonAddStringToObject(item, "keepUnit",i642str( pRetension->keepUnit));
}
sdbRelease(pSdb, pObj);
}
}
void dumpStb(SSdb *pSdb, SJson *json) {
void *pIter = NULL;
SJson *items = tjsonAddArrayToObject(json, "stbs");
while (1) {
SStbObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_STB, pIter, (void **)&pObj);
if (pIter == NULL) break;
SJson *item = tjsonCreateObject();
tjsonAddItemToArray(items, item);
tjsonAddStringToObject(item, "name", mndGetStbStr(pObj->name));
tjsonAddStringToObject(item, "db", mndGetDbStr(pObj->db));
tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
tjsonAddStringToObject(item, "uid", i642str(pObj->uid));
tjsonAddStringToObject(item, "dbUid", i642str(pObj->dbUid));
tjsonAddStringToObject(item, "tagVer",i642str( pObj->tagVer));
tjsonAddStringToObject(item, "colVer", i642str(pObj->colVer));
tjsonAddStringToObject(item, "smaVer", i642str(pObj->smaVer));
tjsonAddStringToObject(item, "nextColId", i642str(pObj->nextColId));
tjsonAddStringToObject(item, "watermark1", i642str(pObj->watermark[0]));
tjsonAddStringToObject(item, "watermark2", i642str(pObj->watermark[1]));
tjsonAddStringToObject(item, "maxdelay0",i642str( pObj->maxdelay[0]));
tjsonAddStringToObject(item, "maxdelay1",i642str( pObj->maxdelay[1]));
tjsonAddStringToObject(item, "ttl",i642str( pObj->ttl));
tjsonAddStringToObject(item, "numOfFuncs",i642str( pObj->numOfFuncs));
tjsonAddStringToObject(item, "commentLen", i642str(pObj->commentLen));
tjsonAddStringToObject(item, "ast1Len", i642str(pObj->ast1Len));
tjsonAddStringToObject(item, "ast2Len",i642str( pObj->ast2Len));
tjsonAddStringToObject(item, "numOfColumns",i642str( pObj->numOfColumns));
SJson *columns = tjsonAddArrayToObject(item, "columns");
for (int32_t i = 0; i < pObj->numOfColumns; ++i) {
SJson *column = tjsonCreateObject();
tjsonAddItemToArray(columns, column);
SSchema *pColumn = &pObj->pColumns[i];
tjsonAddStringToObject(column, "type", i642str(pColumn->type));
tjsonAddStringToObject(column, "typestr", tDataTypes[pColumn->type].name);
tjsonAddStringToObject(column, "flags", i642str(pColumn->flags));
tjsonAddStringToObject(column, "colId", i642str(pColumn->colId));
tjsonAddStringToObject(column, "bytes", i642str(pColumn->bytes));
tjsonAddStringToObject(column, "name", pColumn->name);
}
tjsonAddStringToObject(item, "numOfTags", i642str(pObj->numOfTags));
SJson *tags = tjsonAddArrayToObject(item, "tags");
for (int32_t i = 0; i < pObj->numOfTags; ++i) {
SJson *tag = tjsonCreateObject();
tjsonAddItemToArray(tags, tag);
SSchema *pTag = &pObj->pTags[i];
tjsonAddStringToObject(tag, "type", i642str(pTag->type));
tjsonAddStringToObject(tag, "typestr", tDataTypes[pTag->type].name);
tjsonAddStringToObject(tag, "flags",i642str( pTag->flags));
tjsonAddStringToObject(tag, "colId", i642str(pTag->colId));
tjsonAddStringToObject(tag, "bytes", i642str(pTag->bytes));
tjsonAddStringToObject(tag, "name", pTag->name);
}
sdbRelease(pSdb, pObj);
}
}
void dumpSma(SSdb *pSdb, SJson *json) {
void *pIter = NULL;
SJson *items = tjsonAddArrayToObject(json, "smas");
while (1) {
SSmaObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pObj);
if (pIter == NULL) break;
SJson *item = tjsonCreateObject();
tjsonAddItemToArray(items, item);
tjsonAddStringToObject(item, "name", mndGetStbStr(pObj->name));
tjsonAddStringToObject(item, "stb", mndGetStbStr(pObj->stb));
tjsonAddStringToObject(item, "db", mndGetDbStr(pObj->db));
tjsonAddStringToObject(item, "dstTbName", mndGetStbStr(pObj->dstTbName));
tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
tjsonAddStringToObject(item, "uid", i642str(pObj->uid));
tjsonAddStringToObject(item, "stbUid", i642str(pObj->stbUid));
tjsonAddStringToObject(item, "dbUid", i642str(pObj->dbUid));
tjsonAddStringToObject(item, "dstTbUid", i642str(pObj->dstTbUid));
tjsonAddStringToObject(item, "intervalUnit", i642str(pObj->intervalUnit));
tjsonAddStringToObject(item, "slidingUnit",i642str( pObj->slidingUnit));
tjsonAddStringToObject(item, "timezone", i642str(pObj->timezone));
tjsonAddStringToObject(item, "dstVgId",i642str( pObj->dstVgId));
tjsonAddStringToObject(item, "interval", i642str(pObj->interval));
tjsonAddStringToObject(item, "offset", i642str(pObj->offset));
tjsonAddStringToObject(item, "sliding", i642str(pObj->sliding));
tjsonAddStringToObject(item, "exprLen",i642str( pObj->exprLen));
tjsonAddStringToObject(item, "tagsFilterLen", i642str(pObj->tagsFilterLen));
tjsonAddStringToObject(item, "sqlLen",i642str( pObj->sqlLen));
tjsonAddStringToObject(item, "astLen",i642str( pObj->astLen));
sdbRelease(pSdb, pObj);
}
}
void dumpVgroup(SSdb *pSdb, SJson *json) {
void *pIter = NULL;
SJson *items = tjsonAddArrayToObject(json, "vgroups");
while (1) {
SVgObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pObj);
if (pIter == NULL) break;
SJson *item = tjsonCreateObject();
tjsonAddItemToArray(items, item);
tjsonAddStringToObject(item, "vgId", i642str(pObj->vgId));
tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
tjsonAddStringToObject(item, "version",i642str(pObj->version));
tjsonAddStringToObject(item, "hashBegin", i642str(pObj->hashBegin));
tjsonAddStringToObject(item, "hashEnd", i642str(pObj->hashEnd));
tjsonAddStringToObject(item, "db", mndGetDbStr(pObj->dbName));
tjsonAddStringToObject(item, "dbUid", i642str(pObj->dbUid));
tjsonAddStringToObject(item, "isTsma", i642str(pObj->isTsma));
tjsonAddStringToObject(item, "replica",i642str( pObj->replica));
for (int32_t i = 0; i < pObj->replica; ++i) {
SJson *replicas = tjsonAddArrayToObject(item, "replicas");
SJson *replica = tjsonCreateObject();
tjsonAddItemToArray(replicas, replica);
tjsonAddStringToObject(replica, "dnodeId", i642str(pObj->vnodeGid[i].dnodeId));
}
sdbRelease(pSdb, pObj);
}
}
void dumpTopic(SSdb *pSdb, SJson *json) {
void *pIter = NULL;
SJson *items = tjsonAddArrayToObject(json, "topics");
while (1) {
SMqTopicObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pObj);
if (pIter == NULL) break;
SJson *item = tjsonCreateObject();
tjsonAddItemToArray(items, item);
tjsonAddStringToObject(item, "name", mndGetDbStr(pObj->name));
tjsonAddStringToObject(item, "name", mndGetDbStr(pObj->db));
tjsonAddStringToObject(item, "createTime", i642str(pObj->createTime));
tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
tjsonAddStringToObject(item, "uid", i642str(pObj->uid));
tjsonAddStringToObject(item, "dbUid", i642str(pObj->dbUid));
tjsonAddStringToObject(item, "version",i642str( pObj->version));
tjsonAddStringToObject(item, "subType",i642str( pObj->subType));
tjsonAddStringToObject(item, "withMeta", i642str(pObj->withMeta));
tjsonAddStringToObject(item, "stbUid", i642str(pObj->stbUid));
tjsonAddStringToObject(item, "sqlLen", i642str(pObj->sqlLen));
tjsonAddStringToObject(item, "astLen",i642str( pObj->astLen));
tjsonAddStringToObject(item, "sqlLen",i642str( pObj->sqlLen));
tjsonAddStringToObject(item, "ntbUid", i642str(pObj->ntbUid));
tjsonAddStringToObject(item, "ctbStbUid", i642str(pObj->ctbStbUid));
sdbRelease(pSdb, pObj);
}
}
void dumpConsumer(SSdb *pSdb, SJson *json) {
void *pIter = NULL;
SJson *items = tjsonAddArrayToObject(json, "consumers");
while (1) {
SMqConsumerObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pObj);
if (pIter == NULL) break;
SJson *item = tjsonCreateObject();
tjsonAddItemToArray(items, item);
tjsonAddStringToObject(item, "consumerId", i642str(pObj->consumerId));
tjsonAddStringToObject(item, "cgroup", pObj->cgroup);
sdbRelease(pSdb, pObj);
}
}
void dumpSubscribe(SSdb *pSdb, SJson *json) {
void *pIter = NULL;
SJson *items = tjsonAddArrayToObject(json, "subscribes");
while (1) {
SMqSubscribeObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, pIter, (void **)&pObj);
if (pIter == NULL) break;
SJson *item = tjsonCreateObject();
tjsonAddItemToArray(items, item);
tjsonAddStringToObject(item, "key", pObj->key);
tjsonAddStringToObject(item, "dbUid", i642str(pObj->dbUid));
tjsonAddStringToObject(item, "stbUid", i642str(pObj->stbUid));
sdbRelease(pSdb, pObj);
}
}
void dumpOffset(SSdb *pSdb, SJson *json) {
void *pIter = NULL;
SJson *items = tjsonAddArrayToObject(json, "offsets");
while (1) {
SMqOffsetObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_OFFSET, pIter, (void **)&pObj);
if (pIter == NULL) break;
SJson *item = tjsonCreateObject();
tjsonAddItemToArray(items, item);
tjsonAddStringToObject(item, "key", pObj->key);
tjsonAddStringToObject(item, "dbUid", i642str(pObj->dbUid));
tjsonAddStringToObject(item, "offset", i642str(pObj->offset));
sdbRelease(pSdb, pObj);
}
}
void dumpStream(SSdb *pSdb, SJson *json) {
void *pIter = NULL;
SJson *items = tjsonAddArrayToObject(json, "streams");
while (1) {
SStreamObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pObj);
if (pIter == NULL) break;
SJson *item = tjsonCreateObject();
tjsonAddItemToArray(items, item);
tjsonAddStringToObject(item, "name", mndGetDbStr(pObj->name));
tjsonAddStringToObject(item, "createTime", i642str(pObj->createTime));
tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
tjsonAddStringToObject(item, "version", i642str(pObj->version));
tjsonAddStringToObject(item, "totalLevel", i642str(pObj->totalLevel));
tjsonAddStringToObject(item, "smaId", i642str(pObj->smaId));
tjsonAddStringToObject(item, "uid", i642str(pObj->uid));
tjsonAddStringToObject(item, "status",i642str( pObj->status));
tjsonAddStringToObject(item, "igExpired",i642str( pObj->igExpired));
tjsonAddStringToObject(item, "trigger",i642str( pObj->trigger));
tjsonAddStringToObject(item, "triggerParam", i642str(pObj->triggerParam));
tjsonAddStringToObject(item, "watermark", i642str(pObj->watermark));
tjsonAddStringToObject(item, "sourceDbUid", i642str(pObj->sourceDbUid));
tjsonAddStringToObject(item, "targetDbUid", i642str(pObj->targetDbUid));
tjsonAddStringToObject(item, "sourceDb", mndGetDbStr(pObj->sourceDb));
tjsonAddStringToObject(item, "targetDb", mndGetDbStr(pObj->targetDb));
tjsonAddStringToObject(item, "targetSTbName", mndGetStbStr(pObj->targetSTbName));
tjsonAddStringToObject(item, "targetStbUid", i642str(pObj->targetStbUid));
tjsonAddStringToObject(item, "fixedSinkVgId", i642str(pObj->fixedSinkVgId));
sdbRelease(pSdb, pObj);
}
}
void dumpAcct(SSdb *pSdb, SJson *json) {
void *pIter = NULL;
SJson *items = tjsonAddArrayToObject(json, "accts");
while (1) {
SAcctObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_ACCT, pIter, (void **)&pObj);
if (pIter == NULL) break;
SJson *item = tjsonCreateObject();
tjsonAddItemToArray(items, item);
tjsonAddStringToObject(item, "acct", pObj->acct);
tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
tjsonAddStringToObject(item, "acctId", i642str(pObj->acctId));
sdbRelease(pSdb, pObj);
}
}
void dumpAuth(SSdb *pSdb, SJson *json) {
// todo
}
void dumpUser(SSdb *pSdb, SJson *json) {
void *pIter = NULL;
SJson *items = tjsonAddArrayToObject(json, "users");
while (1) {
SUserObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pObj);
if (pIter == NULL) break;
SJson *item = tjsonCreateObject();
tjsonAddItemToArray(items, item);
tjsonAddStringToObject(item, "name", pObj->user);
tjsonAddStringToObject(item, "acct", pObj->acct);
tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
tjsonAddStringToObject(item, "superUser",i642str( pObj->superUser));
tjsonAddStringToObject(item, "authVersion", i642str(pObj->authVersion));
tjsonAddStringToObject(item, "numOfReadDbs",i642str( taosHashGetSize(pObj->readDbs)));
tjsonAddStringToObject(item, "numOfWriteDbs", i642str(taosHashGetSize(pObj->writeDbs)));
sdbRelease(pSdb, pObj);
}
}
void dumpDnode(SSdb *pSdb, SJson *json) {
void *pIter = NULL;
SJson *items = tjsonAddArrayToObject(json, "dnodes");
while (1) {
SDnodeObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pObj);
if (pIter == NULL) break;
SJson *item = tjsonCreateObject();
tjsonAddItemToArray(items, item);
tjsonAddStringToObject(item, "id",i642str( pObj->id));
tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
tjsonAddStringToObject(item, "port",i642str( pObj->port));
tjsonAddStringToObject(item, "fqdn", pObj->fqdn);
sdbRelease(pSdb, pObj);
}
}
void dumpBnode(SSdb *pSdb, SJson *json) {
// not implemented yet
}
void dumpSnode(SSdb *pSdb, SJson *json) {
void *pIter = NULL;
SJson *items = tjsonAddArrayToObject(json, "snodes");
while (1) {
SSnodeObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_QNODE, pIter, (void **)&pObj);
if (pIter == NULL) break;
SJson *item = tjsonCreateObject();
tjsonAddItemToArray(items, item);
tjsonAddStringToObject(item, "id",i642str( pObj->id));
tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
sdbRelease(pSdb, pObj);
}
}
void dumpQnode(SSdb *pSdb, SJson *json) {
void *pIter = NULL;
SJson *items = tjsonAddArrayToObject(json, "qnodes");
while (1) {
SQnodeObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_QNODE, pIter, (void **)&pObj);
if (pIter == NULL) break;
SJson *item = tjsonCreateObject();
tjsonAddItemToArray(items, item);
tjsonAddStringToObject(item, "id", i642str(pObj->id));
tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
sdbRelease(pSdb, pObj);
}
}
void dumpMnode(SSdb *pSdb, SJson *json) {
void *pIter = NULL;
SJson *items = tjsonAddArrayToObject(json, "mnodes");
while (1) {
SMnodeObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj);
if (pIter == NULL) break;
SJson *item = tjsonCreateObject();
tjsonAddItemToArray(items, item);
tjsonAddStringToObject(item, "id", i642str(pObj->id));
tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
sdbRelease(pSdb, pObj);
}
}
void dumpCluster(SSdb *pSdb, SJson *json) {
void *pIter = NULL;
SJson *items = tjsonAddArrayToObject(json, "clusters");
while (1) {
SClusterObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_CLUSTER, pIter, (void **)&pObj);
if (pIter == NULL) break;
SJson *item = tjsonCreateObject();
tjsonAddItemToArray(items, item);
tjsonAddStringToObject(item, "id", i642str(pObj->id));
tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
tjsonAddStringToObject(item, "name", pObj->name);
sdbRelease(pSdb, pObj);
}
}
void dumpTrans(SSdb *pSdb, SJson *json) {
void *pIter = NULL;
SJson *items = tjsonAddArrayToObject(json, "transactions");
while (1) {
STrans *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_TRANS, pIter, (void **)&pObj);
if (pIter == NULL) break;
SJson *item = tjsonCreateObject();
tjsonAddItemToArray(items, item);
tjsonAddStringToObject(item, "id", i642str(pObj->id));
tjsonAddStringToObject(item, "stage", i642str(pObj->stage));
tjsonAddStringToObject(item, "policy", i642str(pObj->policy));
tjsonAddStringToObject(item, "conflict",i642str( pObj->conflict));
tjsonAddStringToObject(item, "exec", i642str(pObj->exec));
tjsonAddStringToObject(item, "oper", i642str(pObj->oper));
tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
tjsonAddStringToObject(item, "dbname", pObj->dbname);
tjsonAddStringToObject(item, "stbname", pObj->stbname);
tjsonAddStringToObject(item, "opername", pObj->opername);
tjsonAddStringToObject(item, "commitLogNum",i642str( taosArrayGetSize(pObj->commitActions)));
tjsonAddStringToObject(item, "redoActionNum",i642str(taosArrayGetSize(pObj->redoActions)));
tjsonAddStringToObject(item, "undoActionNum", i642str(taosArrayGetSize(pObj->undoActions)));
sdbRelease(pSdb, pObj);
}
}
void dumpHeader(SSdb *pSdb, SJson *json) {
tjsonAddStringToObject(json, "sver", i642str(1));
tjsonAddStringToObject(json, "applyIndex", i642str(pSdb->applyIndex));
tjsonAddStringToObject(json, "applyTerm", i642str(pSdb->applyTerm));
tjsonAddStringToObject(json, "applyConfig", i642str(pSdb->applyConfig));
SJson *maxIdsJson = tjsonCreateObject();
tjsonAddItemToObject(json, "maxIds", maxIdsJson);
for (int32_t i = 0; i < SDB_MAX; ++i) {
int64_t maxId = 0;
if (i < SDB_MAX) {
maxId = pSdb->maxId[i];
}
tjsonAddStringToObject(maxIdsJson, sdbTableName(i), i642str(maxId));
}
SJson *tableVersJson = tjsonCreateObject();
tjsonAddItemToObject(json, "tableVers", tableVersJson);
for (int32_t i = 0; i < SDB_MAX; ++i) {
int64_t tableVer = 0;
if (i < SDB_MAX) {
tableVer = pSdb->tableVer[i];
}
tjsonAddStringToObject(tableVersJson, sdbTableName(i), i642str(tableVer));
}
}
void mndDumpSdb() {
mInfo("start to dump sdb info to sdb.json");
char path[PATH_MAX * 2] = {0};
snprintf(path, sizeof(path), "%s%smnode", tsDataDir, TD_DIRSEP);
SMsgCb msgCb = {0};
msgCb.reportStartupFp = reportStartup;
msgCb.sendReqFp = sendReq;
msgCb.sendRspFp = sendRsp;
msgCb.mgmt = (SMgmtWrapper *)(&msgCb); // hack
tmsgSetDefault(&msgCb);
walInit();
syncInit();
SMnodeOpt opt = {.msgCb = msgCb};
SMnode *pMnode = mndOpen(path, &opt);
if (pMnode == NULL) return;
SSdb *pSdb = pMnode->pSdb;
SJson *json = tjsonCreateObject();
dumpHeader(pSdb, json);
dumpFunc(pSdb, json);
dumpDb(pSdb, json);
dumpStb(pSdb, json);
dumpSma(pSdb, json);
dumpVgroup(pSdb, json);
dumpTopic(pSdb, json);
dumpConsumer(pSdb, json);
dumpSubscribe(pSdb, json);
dumpOffset(pSdb, json);
dumpStream(pSdb, json);
dumpAcct(pSdb, json);
dumpAuth(pSdb, json);
dumpUser(pSdb, json);
dumpDnode(pSdb, json);
dumpBnode(pSdb, json);
dumpSnode(pSdb, json);
dumpQnode(pSdb, json);
dumpMnode(pSdb, json);
dumpCluster(pSdb, json);
dumpTrans(pSdb, json);
char *pCont = tjsonToString(json);
int32_t contLen = strlen(pCont);
char file[] = "sdb.json";
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to write %s since %s", file, terrstr());
return;
}
taosWriteFile(pFile, pCont, contLen);
taosWriteFile(pFile, "\n", 1);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
tjsonDelete(json);
taosMemoryFree(pCont);
mInfo("dump sdb info success");
}
#pragma GCC diagnostic pop

View File

@ -38,13 +38,15 @@ static int32_t mndRetrieveFuncs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
static void mndCancelGetNextFunc(SMnode *pMnode, void *pIter); static void mndCancelGetNextFunc(SMnode *pMnode, void *pIter);
int32_t mndInitFunc(SMnode *pMnode) { int32_t mndInitFunc(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_FUNC, SSdbTable table = {
.sdbType = SDB_FUNC,
.keyType = SDB_KEY_BINARY, .keyType = SDB_KEY_BINARY,
.encodeFp = (SdbEncodeFp)mndFuncActionEncode, .encodeFp = (SdbEncodeFp)mndFuncActionEncode,
.decodeFp = (SdbDecodeFp)mndFuncActionDecode, .decodeFp = (SdbDecodeFp)mndFuncActionDecode,
.insertFp = (SdbInsertFp)mndFuncActionInsert, .insertFp = (SdbInsertFp)mndFuncActionInsert,
.updateFp = (SdbUpdateFp)mndFuncActionUpdate, .updateFp = (SdbUpdateFp)mndFuncActionUpdate,
.deleteFp = (SdbDeleteFp)mndFuncActionDelete}; .deleteFp = (SdbDeleteFp)mndFuncActionDelete,
};
mndSetMsgHandle(pMnode, TDMT_MND_CREATE_FUNC, mndProcessCreateFuncReq); mndSetMsgHandle(pMnode, TDMT_MND_CREATE_FUNC, mndProcessCreateFuncReq);
mndSetMsgHandle(pMnode, TDMT_MND_DROP_FUNC, mndProcessDropFuncReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_FUNC, mndProcessDropFuncReq);

View File

@ -119,28 +119,30 @@ static void *mndThreadFp(void *param) {
lastTime++; lastTime++;
taosMsleep(100); taosMsleep(100);
if (mndGetStop(pMnode)) break; if (mndGetStop(pMnode)) break;
if (lastTime % 10 != 0) continue;
if (lastTime % (tsTtlPushInterval * 10) == 1) { int64_t sec = lastTime / 10;
if (sec % tsTtlPushInterval == 0) {
mndPullupTtl(pMnode); mndPullupTtl(pMnode);
} }
if (lastTime % (tsTransPullupInterval * 10) == 0) { if (sec % tsTransPullupInterval == 0) {
mndPullupTrans(pMnode); mndPullupTrans(pMnode);
} }
if (lastTime % (tsMqRebalanceInterval * 10) == 0) { if (sec % tsMqRebalanceInterval == 0) {
mndCalMqRebalance(pMnode); mndCalMqRebalance(pMnode);
} }
if (lastTime % (tsTelemInterval * 10) == ((tsTelemInterval - 1) * 10)) { if (sec % tsTelemInterval == (TMIN(60, (tsTelemInterval - 1)))) {
mndPullupTelem(pMnode); mndPullupTelem(pMnode);
} }
if (lastTime % (tsGrantHBInterval * 10) == 0) { if (sec % tsGrantHBInterval == 0) {
mndPullupGrant(pMnode); mndPullupGrant(pMnode);
} }
if ((lastTime % (tsUptimeInterval * 10)) == ((tsUptimeInterval - 1) * 10)) { if (sec % tsUptimeInterval == 0) {
mndIncreaseUpTime(pMnode); mndIncreaseUpTime(pMnode);
} }
} }
@ -399,7 +401,7 @@ void mndPreClose(SMnode *pMnode) {
atomic_store_8(&(pMnode->syncMgmt.leaderTransferFinish), 0); atomic_store_8(&(pMnode->syncMgmt.leaderTransferFinish), 0);
syncLeaderTransfer(pMnode->syncMgmt.sync); syncLeaderTransfer(pMnode->syncMgmt.sync);
/* #if 0
mInfo("vgId:1, mnode start leader transfer"); mInfo("vgId:1, mnode start leader transfer");
// wait for leader transfer finish // wait for leader transfer finish
while (!atomic_load_8(&(pMnode->syncMgmt.leaderTransferFinish))) { while (!atomic_load_8(&(pMnode->syncMgmt.leaderTransferFinish))) {
@ -407,7 +409,7 @@ void mndPreClose(SMnode *pMnode) {
mInfo("vgId:1, mnode waiting for leader transfer"); mInfo("vgId:1, mnode waiting for leader transfer");
} }
mInfo("vgId:1, mnode finish leader transfer"); mInfo("vgId:1, mnode finish leader transfer");
*/ #endif
} }
} }

View File

@ -834,6 +834,8 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) {
int32_t reqLen = tSerializeSVDropTtlTableReq(NULL, 0, &ttlReq); int32_t reqLen = tSerializeSVDropTtlTableReq(NULL, 0, &ttlReq);
int32_t contLen = reqLen + sizeof(SMsgHead); int32_t contLen = reqLen + sizeof(SMsgHead);
mInfo("start to process ttl timer");
while (1) { while (1) {
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break; if (pIter == NULL) break;
@ -2579,3 +2581,14 @@ static void mndCancelGetNextStb(SMnode *pMnode, void *pIter) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
} }
const char *mndGetStbStr(const char *src) {
char *posDb = strstr(src, TS_PATH_DELIMITER);
if (posDb != NULL) ++posDb;
if (posDb == NULL) return src;
char *posStb = strstr(posDb, TS_PATH_DELIMITER);
if (posStb != NULL) ++posStb;
if (posStb == NULL) return posDb;
return posStb;
}

View File

@ -133,7 +133,7 @@ static int32_t mndProcessTelemTimer(SRpcMsg* pReq) {
if (taosSendHttpReport(tsTelemServer, tsTelemPort, pCont, strlen(pCont), HTTP_FLAT) != 0) { if (taosSendHttpReport(tsTelemServer, tsTelemPort, pCont, strlen(pCont), HTTP_FLAT) != 0) {
mError("failed to send telemetry report"); mError("failed to send telemetry report");
} else { } else {
mTrace("succeed to send telemetry report"); mInfo("succeed to send telemetry report");
} }
taosMemoryFree(pCont); taosMemoryFree(pCont);
} }

View File

@ -1478,6 +1478,7 @@ void mndTransExecute(SMnode *pMnode, STrans *pTrans) {
} }
static int32_t mndProcessTransTimer(SRpcMsg *pReq) { static int32_t mndProcessTransTimer(SRpcMsg *pReq) {
mTrace("start to process trans timer");
mndTransPullup(pReq->info.node); mndTransPullup(pReq->info.node);
return 0; return 0;
} }
@ -1604,7 +1605,7 @@ static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)dbname, false); colDataAppend(pColInfo, numOfRows, (const char *)dbname, false);
char stbname[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char stbname[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(stbname, mndGetDbStr(pTrans->stbname), pShow->pMeta->pSchemas[cols].bytes); STR_WITH_MAXSIZE_TO_VARSTR(stbname, mndGetDbStr(pTrans->stbname), pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)stbname, false); colDataAppend(pColInfo, numOfRows, (const char *)stbname, false);

View File

@ -514,7 +514,7 @@ static void sdbCloseIter(SSdbIter *pIter) {
} }
if (pIter->name != NULL) { if (pIter->name != NULL) {
taosRemoveFile(pIter->name); (void)taosRemoveFile(pIter->name);
taosMemoryFree(pIter->name); taosMemoryFree(pIter->name);
pIter->name = NULL; pIter->name = NULL;
} }
@ -606,6 +606,7 @@ int32_t sdbStartWrite(SSdb *pSdb, SSdbIter **ppIter) {
if (pIter->file == NULL) { if (pIter->file == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to open %s since %s", pIter->name, terrstr()); mError("failed to open %s since %s", pIter->name, terrstr());
sdbCloseIter(pIter);
return -1; return -1;
} }
@ -636,9 +637,9 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, i
return -1; return -1;
} }
sdbCloseIter(pIter);
if (sdbReadFile(pSdb) != 0) { if (sdbReadFile(pSdb) != 0) {
mError("sdbiter:%p, failed to read from %s since %s", pIter, datafile, terrstr()); mError("sdbiter:%p, failed to read from %s since %s", pIter, datafile, terrstr());
sdbCloseIter(pIter);
return -1; return -1;
} }
@ -656,6 +657,7 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, i
} }
mInfo("sdbiter:%p, success applyed to sdb", pIter); mInfo("sdbiter:%p, success applyed to sdb", pIter);
sdbCloseIter(pIter);
return 0; return 0;
} }

View File

@ -40,6 +40,8 @@ const char *sdbTableName(ESdbType type) {
return "auth"; return "auth";
case SDB_ACCT: case SDB_ACCT:
return "acct"; return "acct";
case SDB_STREAM_CK:
return "stream_ck";
case SDB_STREAM: case SDB_STREAM:
return "stream"; return "stream";
case SDB_OFFSET: case SDB_OFFSET:
@ -219,14 +221,15 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
return terrno; return terrno;
} }
SSdbRow *pOldRow = *ppOldRow; SSdbRow *pOldRow = *ppOldRow;
pOldRow->status = pRaw->status; pOldRow->status = pRaw->status;
atomic_add_fetch_32(&pOldRow->refCount, 1);
sdbPrintOper(pSdb, pOldRow, "delete"); sdbPrintOper(pSdb, pOldRow, "delete");
taosHashRemove(hash, pOldRow->pObj, keySize); taosHashRemove(hash, pOldRow->pObj, keySize);
pSdb->tableVer[pOldRow->type]++;
taosThreadRwlockUnlock(pLock); taosThreadRwlockUnlock(pLock);
pSdb->tableVer[pOldRow->type]++;
sdbFreeRow(pSdb, pRow, false); sdbFreeRow(pSdb, pRow, false);
sdbCheckRow(pSdb, pOldRow); sdbCheckRow(pSdb, pOldRow);
@ -315,7 +318,7 @@ static void sdbCheckRow(SSdb *pSdb, SSdbRow *pRow) {
TdThreadRwlock *pLock = &pSdb->locks[pRow->type]; TdThreadRwlock *pLock = &pSdb->locks[pRow->type];
taosThreadRwlockWrlock(pLock); taosThreadRwlockWrlock(pLock);
int32_t ref = atomic_load_32(&pRow->refCount); int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1);
sdbPrintOper(pSdb, pRow, "check"); sdbPrintOper(pSdb, pRow, "check");
if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) { if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) {
sdbFreeRow(pSdb, pRow, true); sdbFreeRow(pSdb, pRow, true);

View File

@ -32,6 +32,12 @@ extern "C" {
#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSD ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) #define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSD ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
// clang-format on // clang-format on
#define TSDB_CHECK_CODE(CODE, LINO, LABEL) \
if (CODE) { \
LINO = __LINE__; \
goto LABEL; \
}
typedef struct TSDBROW TSDBROW; typedef struct TSDBROW TSDBROW;
typedef struct TABLEID TABLEID; typedef struct TABLEID TABLEID;
typedef struct TSDBKEY TSDBKEY; typedef struct TSDBKEY TSDBKEY;
@ -58,6 +64,7 @@ typedef struct SDelFWriter SDelFWriter;
typedef struct SDelFReader SDelFReader; typedef struct SDelFReader SDelFReader;
typedef struct SRowIter SRowIter; typedef struct SRowIter SRowIter;
typedef struct STsdbFS STsdbFS; typedef struct STsdbFS STsdbFS;
typedef struct STsdbTrimHdl STsdbTrimHdl;
typedef struct SRowMerger SRowMerger; typedef struct SRowMerger SRowMerger;
typedef struct STsdbReadSnap STsdbReadSnap; typedef struct STsdbReadSnap STsdbReadSnap;
typedef struct SBlockInfo SBlockInfo; typedef struct SBlockInfo SBlockInfo;
@ -88,9 +95,8 @@ typedef struct SLDataIter SLDataIter;
static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) { static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) {
int64_t fOffSet = LOGIC_TO_FILE_OFFSET(lSize, szPage); int64_t fOffSet = LOGIC_TO_FILE_OFFSET(lSize, szPage);
int64_t pgno = OFFSET_PGNO(fOffSet, szPage); int64_t pgno = OFFSET_PGNO(fOffSet, szPage);
int32_t szPageCont = PAGE_CONTENT_SIZE(szPage);
if (fOffSet % szPageCont == 0) { if (fOffSet % szPage == 0) {
pgno--; pgno--;
} }
@ -151,7 +157,7 @@ int32_t tCmprBlockL(void const *lhs, void const *rhs);
int32_t tBlockDataCreate(SBlockData *pBlockData); int32_t tBlockDataCreate(SBlockData *pBlockData);
void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear); void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear);
int32_t tBlockDataInit(SBlockData *pBlockData, int64_t suid, int64_t uid, STSchema *pTSchema); int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid);
int32_t tBlockDataInitEx(SBlockData *pBlockData, SBlockData *pBlockDataFrom); int32_t tBlockDataInitEx(SBlockData *pBlockData, SBlockData *pBlockDataFrom);
void tBlockDataReset(SBlockData *pBlockData); void tBlockDataReset(SBlockData *pBlockData);
int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid);
@ -243,6 +249,7 @@ int32_t tsdbFSClose(STsdb *pTsdb);
int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS); int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS);
void tsdbFSDestroy(STsdbFS *pFS); void tsdbFSDestroy(STsdbFS *pFS);
int32_t tDFileSetCmprFn(const void *p1, const void *p2); int32_t tDFileSetCmprFn(const void *p1, const void *p2);
int32_t tsdbFSUpdDel(STsdb *pTsdb, STsdbFS *pFS, STsdbFS *pFSNew, int32_t maxFid);
int32_t tsdbFSCommit1(STsdb *pTsdb, STsdbFS *pFS); int32_t tsdbFSCommit1(STsdb *pTsdb, STsdbFS *pFS);
int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFS); int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFS);
int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS); int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS);
@ -263,7 +270,7 @@ int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk);
int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo,
int8_t cmprAlg, int8_t toLast); int8_t cmprAlg, int8_t toLast);
int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, int64_t maxSpeed);
// SDataFReader // SDataFReader
int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet);
int32_t tsdbDataFReaderClose(SDataFReader **ppReader); int32_t tsdbDataFReaderClose(SDataFReader **ppReader);
@ -273,6 +280,7 @@ int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk);
int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pBlock, SArray *aColumnDataAgg); int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pBlock, SArray *aColumnDataAgg);
int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pBlock, SBlockData *pBlockData); int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pBlock, SBlockData *pBlockData);
int32_t tsdbReadSttBlock(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData); int32_t tsdbReadSttBlock(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData);
int32_t tsdbReadSttBlockEx(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData);
// SDelFWriter // SDelFWriter
int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb);
int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync);
@ -285,8 +293,8 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader);
int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData); int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData);
int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx);
// tsdbRead.c ============================================================================================== // tsdbRead.c ==============================================================================================
int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap, const char* id); int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap, const char *id);
void tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap, const char* id); void tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap, const char *id);
// tsdbMerge.c ============================================================================================== // tsdbMerge.c ==============================================================================================
int32_t tsdbMerge(STsdb *pTsdb); int32_t tsdbMerge(STsdb *pTsdb);
@ -314,10 +322,18 @@ int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSc
// structs ======================= // structs =======================
struct STsdbFS { struct STsdbFS {
int64_t version;
SDelFile *pDelFile; SDelFile *pDelFile;
SArray *aDFileSet; // SArray<SDFileSet> SArray *aDFileSet; // SArray<SDFileSet>
}; };
struct STsdbTrimHdl {
volatile int8_t state; // 0 idle 1 in use
volatile int8_t commitInWait; // 0 not in wait, 1 in wait
volatile int32_t maxRetentFid;
volatile int32_t minCommitFid;
};
struct STsdb { struct STsdb {
char *path; char *path;
SVnode *pVnode; SVnode *pVnode;
@ -326,6 +342,7 @@ struct STsdb {
SMemTable *mem; SMemTable *mem;
SMemTable *imem; SMemTable *imem;
STsdbFS fs; STsdbFS fs;
STsdbTrimHdl trimHdl;
SLRUCache *lruCache; SLRUCache *lruCache;
TdThreadMutex lruMutex; TdThreadMutex lruMutex;
}; };
@ -634,6 +651,9 @@ typedef struct SSttBlockLoadInfo {
int32_t currentLoadBlockIndex; int32_t currentLoadBlockIndex;
int32_t loadBlocks; int32_t loadBlocks;
double elapsedTime; double elapsedTime;
STSchema *pSchema;
int16_t *colIds;
int32_t numOfCols;
} SSttBlockLoadInfo; } SSttBlockLoadInfo;
typedef struct SMergeTree { typedef struct SMergeTree {
@ -653,13 +673,14 @@ typedef struct {
} SSkmInfo; } SSkmInfo;
int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
STimeWindow *pTimeWindow, SVersionRange *pVerRange, void *pLoadInfo, const char *idStr); STimeWindow *pTimeWindow, SVersionRange *pVerRange, void *pBlockLoadInfo, STSchema *pSchema,
int16_t *pCols, int32_t numOfCols, const char *idStr);
void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter);
bool tMergeTreeNext(SMergeTree *pMTree); bool tMergeTreeNext(SMergeTree *pMTree);
TSDBROW tMergeTreeGetRow(SMergeTree *pMTree); TSDBROW tMergeTreeGetRow(SMergeTree *pMTree);
void tMergeTreeClose(SMergeTree *pMTree); void tMergeTreeClose(SMergeTree *pMTree);
SSttBlockLoadInfo *tCreateLastBlockLoadInfo(); SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols);
void resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); void resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
void getLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, int64_t *blocks, double *el); void getLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, int64_t *blocks, double *el);
void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);

View File

@ -58,6 +58,7 @@ typedef struct STQ STQ;
typedef struct SVState SVState; typedef struct SVState SVState;
typedef struct SVBufPool SVBufPool; typedef struct SVBufPool SVBufPool;
typedef struct SQWorker SQHandle; typedef struct SQWorker SQHandle;
typedef struct SVTrimDbHdl SVTrimDbHdl;
typedef struct STsdbKeepCfg STsdbKeepCfg; typedef struct STsdbKeepCfg STsdbKeepCfg;
typedef struct SMetaSnapReader SMetaSnapReader; typedef struct SMetaSnapReader SMetaSnapReader;
typedef struct SMetaSnapWriter SMetaSnapWriter; typedef struct SMetaSnapWriter SMetaSnapWriter;
@ -143,7 +144,7 @@ int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepC
int tsdbClose(STsdb** pTsdb); int tsdbClose(STsdb** pTsdb);
int32_t tsdbBegin(STsdb* pTsdb); int32_t tsdbBegin(STsdb* pTsdb);
int32_t tsdbCommit(STsdb* pTsdb); int32_t tsdbCommit(STsdb* pTsdb);
int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now); int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now, int64_t maxSpeed);
int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg);
int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp);
int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock,
@ -198,7 +199,7 @@ int32_t smaSyncPostCommit(SSma* pSma);
int32_t smaAsyncPreCommit(SSma* pSma); int32_t smaAsyncPreCommit(SSma* pSma);
int32_t smaAsyncCommit(SSma* pSma); int32_t smaAsyncCommit(SSma* pSma);
int32_t smaAsyncPostCommit(SSma* pSma); int32_t smaAsyncPostCommit(SSma* pSma);
int32_t smaDoRetention(SSma* pSma, int64_t now); int32_t smaDoRetention(SSma* pSma, int64_t now, int64_t maxSpeed);
int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg);
int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg);
@ -300,6 +301,9 @@ struct STsdbKeepCfg {
int32_t keep1; int32_t keep1;
int32_t keep2; int32_t keep2;
}; };
struct SVTrimDbHdl {
volatile int8_t state; // 0 not in trim, 1 in trim
};
struct SVnode { struct SVnode {
char* path; char* path;
@ -324,6 +328,7 @@ struct SVnode {
bool restored; bool restored;
tsem_t syncSem; tsem_t syncSem;
SQHandle* pQuery; SQHandle* pQuery;
SVTrimDbHdl trimDbH;
}; };
#define TD_VID(PVNODE) ((PVNODE)->config.vgId) #define TD_VID(PVNODE) ((PVNODE)->config.vgId)

View File

@ -661,9 +661,10 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) {
* *
* @param pSma * @param pSma
* @param now * @param now
* @param maxSpeed
* @return int32_t * @return int32_t
*/ */
int32_t smaDoRetention(SSma *pSma, int64_t now) { int32_t smaDoRetention(SSma *pSma, int64_t now, int64_t maxSpeed) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (!VND_IS_RSMA(pSma->pVnode)) { if (!VND_IS_RSMA(pSma->pVnode)) {
return code; return code;
@ -671,7 +672,7 @@ int32_t smaDoRetention(SSma *pSma, int64_t now) {
for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
if (pSma->pRSmaTsdb[i]) { if (pSma->pRSmaTsdb[i]) {
code = tsdbDoRetention(pSma->pRSmaTsdb[i], now); code = tsdbDoRetention(pSma->pRSmaTsdb[i], now, maxSpeed);
if (code) goto _end; if (code) goto _end;
} }
} }

View File

@ -457,7 +457,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) {
tMergeTreeOpen(&state->mergeTree, 1, state->pDataFReader, state->suid, state->uid, tMergeTreeOpen(&state->mergeTree, 1, state->pDataFReader, state->suid, state->uid,
&(STimeWindow){.skey = TSKEY_MIN, .ekey = TSKEY_MAX}, &(STimeWindow){.skey = TSKEY_MIN, .ekey = TSKEY_MAX},
&(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, NULL, NULL); &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, NULL, NULL, NULL, 0, NULL);
bool hasVal = tMergeTreeNext(&state->mergeTree); bool hasVal = tMergeTreeNext(&state->mergeTree);
if (!hasVal) { if (!hasVal) {
state->state = SFSLASTNEXTROW_FILESET; state->state = SFSLASTNEXTROW_FILESET;
@ -612,7 +612,8 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetDataBlk); tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetDataBlk);
/* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); */ /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); */
tBlockDataReset(state->pBlockData); tBlockDataReset(state->pBlockData);
code = tBlockDataInit(state->pBlockData, state->suid, state->uid, state->pTSchema); TABLEID tid = {.suid = state->suid, .uid = state->uid};
code = tBlockDataInit(state->pBlockData, &tid, state->pTSchema, NULL, 0);
if (code) goto _err; if (code) goto _err;
code = tsdbReadDataBlock(state->pDataFReader, &block, state->pBlockData); code = tsdbReadDataBlock(state->pDataFReader, &block, state->pBlockData);

View File

@ -437,7 +437,7 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) {
pIter->iSttBlk = 0; pIter->iSttBlk = 0;
SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, 0); SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, 0);
code = tsdbReadSttBlock(pCommitter->dReader.pReader, iStt, pSttBlk, &pIter->bData); code = tsdbReadSttBlockEx(pCommitter->dReader.pReader, iStt, pSttBlk, &pIter->bData);
if (code) goto _err; if (code) goto _err;
pIter->iRow = 0; pIter->iRow = 0;
@ -756,6 +756,32 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
if (pTsdb->imem->nRow > 0) {
int32_t minCommitFid = tsdbKeyFid(pTsdb->imem->minKey, pCommitter->minutes, pCommitter->precision);
int32_t nLoops = 0;
_wait_retention_end:
while (atomic_load_32(&pTsdb->trimHdl.maxRetentFid) >= minCommitFid) {
atomic_val_compare_exchange_8(&pTsdb->trimHdl.commitInWait, 0, 1);
if (++nLoops > 1000) {
nLoops = 0;
sched_yield();
}
}
if (atomic_val_compare_exchange_8(&pTsdb->trimHdl.state, 0, 1) == 0) {
if (atomic_load_32(&pTsdb->trimHdl.maxRetentFid) >= minCommitFid) {
atomic_store_8(&pTsdb->trimHdl.state, 0);
goto _wait_retention_end;
}
atomic_store_32(&pTsdb->trimHdl.minCommitFid, minCommitFid);
atomic_store_8(&pTsdb->trimHdl.state, 0);
} else {
goto _wait_retention_end;
}
atomic_val_compare_exchange_8(&pTsdb->trimHdl.commitInWait, 1, 0);
}
code = tsdbFSCopy(pTsdb, &pCommitter->fs); code = tsdbFSCopy(pTsdb, &pCommitter->fs);
if (code) goto _err; if (code) goto _err;
@ -962,20 +988,38 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) {
int32_t code = 0; int32_t code = 0;
STsdb *pTsdb = pCommitter->pTsdb; STsdb *pTsdb = pCommitter->pTsdb;
SMemTable *pMemTable = pTsdb->imem; SMemTable *pMemTable = pTsdb->imem;
STsdbFS fsLatest = {0};
ASSERT(eno == 0); ASSERT(eno == 0);
code = tsdbFSCommit1(pTsdb, &pCommitter->fs);
if (code) goto _err;
// lock // lock
taosThreadRwlockWrlock(&pTsdb->rwLock); taosThreadRwlockWrlock(&pTsdb->rwLock);
ASSERT(pCommitter->fs.version <= pTsdb->fs.version);
if (pCommitter->fs.version < pTsdb->fs.version) {
if ((code = tsdbFSCopy(pTsdb, &fsLatest))) {
taosThreadRwlockUnlock(&pTsdb->rwLock);
goto _exit;
}
if ((code = tsdbFSUpdDel(pTsdb, &pCommitter->fs, &fsLatest, pTsdb->trimHdl.minCommitFid - 1))) {
taosThreadRwlockUnlock(&pTsdb->rwLock);
goto _exit;
}
}
code = tsdbFSCommit1(pTsdb, &pCommitter->fs);
if (code) {
taosThreadRwlockUnlock(&pTsdb->rwLock);
goto _exit;
}
// commit or rollback // commit or rollback
code = tsdbFSCommit2(pTsdb, &pCommitter->fs); code = tsdbFSCommit2(pTsdb, &pCommitter->fs);
if (code) { if (code) {
taosThreadRwlockUnlock(&pTsdb->rwLock); taosThreadRwlockUnlock(&pTsdb->rwLock);
goto _err; goto _exit;
} }
pTsdb->imem = NULL; pTsdb->imem = NULL;
@ -983,20 +1027,23 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) {
// unlock // unlock
taosThreadRwlockUnlock(&pTsdb->rwLock); taosThreadRwlockUnlock(&pTsdb->rwLock);
_exit:
tsdbUnrefMemTable(pMemTable); tsdbUnrefMemTable(pMemTable);
tsdbFSDestroy(&pCommitter->fs); tsdbFSDestroy(&pCommitter->fs);
tsdbFSDestroy(&fsLatest);
taosArrayDestroy(pCommitter->aTbDataP); taosArrayDestroy(pCommitter->aTbDataP);
atomic_store_32(&pTsdb->trimHdl.minCommitFid, INT32_MAX);
// if (pCommitter->toMerge) { // if (pCommitter->toMerge) {
// code = tsdbMerge(pTsdb); // code = tsdbMerge(pTsdb);
// if (code) goto _err; // if (code) goto _err;
// } // }
if(code == 0) {
tsdbInfo("vgId:%d, tsdb end commit", TD_VID(pTsdb->pVnode)); tsdbInfo("vgId:%d, tsdb end commit", TD_VID(pTsdb->pVnode));
return code; } else {
_err:
tsdbError("vgId:%d, tsdb end commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); tsdbError("vgId:%d, tsdb end commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
}
return code; return code;
} }
@ -1049,7 +1096,7 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) {
if (pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk)) { if (pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk)) {
SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk); SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk);
code = tsdbReadSttBlock(pCommitter->dReader.pReader, pIter->iStt, pSttBlk, &pIter->bData); code = tsdbReadSttBlockEx(pCommitter->dReader.pReader, pIter->iStt, pSttBlk, &pIter->bData);
if (code) goto _exit; if (code) goto _exit;
pIter->iRow = 0; pIter->iRow = 0;
@ -1305,7 +1352,8 @@ static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) {
if (!pBDatal->suid && !pBDatal->uid) { if (!pBDatal->suid && !pBDatal->uid) {
ASSERT(pCommitter->skmTable.suid == id.suid); ASSERT(pCommitter->skmTable.suid == id.suid);
ASSERT(pCommitter->skmTable.uid == id.uid); ASSERT(pCommitter->skmTable.uid == id.uid);
code = tBlockDataInit(pBDatal, id.suid, id.suid ? 0 : id.uid, pCommitter->skmTable.pTSchema); TABLEID tid = {.suid = id.suid, .uid = id.suid ? 0 : id.uid};
code = tBlockDataInit(pBDatal, &tid, pCommitter->skmTable.pTSchema, NULL, 0);
if (code) goto _exit; if (code) goto _exit;
} }
@ -1428,9 +1476,9 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) {
// impl // impl
code = tsdbUpdateTableSchema(pCommitter->pTsdb->pVnode->pMeta, id.suid, id.uid, &pCommitter->skmTable); code = tsdbUpdateTableSchema(pCommitter->pTsdb->pVnode->pMeta, id.suid, id.uid, &pCommitter->skmTable);
if (code) goto _err; if (code) goto _err;
code = tBlockDataInit(&pCommitter->dReader.bData, id.suid, id.uid, pCommitter->skmTable.pTSchema); code = tBlockDataInit(&pCommitter->dReader.bData, &id, pCommitter->skmTable.pTSchema, NULL, 0);
if (code) goto _err; if (code) goto _err;
code = tBlockDataInit(&pCommitter->dWriter.bData, id.suid, id.uid, pCommitter->skmTable.pTSchema); code = tBlockDataInit(&pCommitter->dWriter.bData, &id, pCommitter->skmTable.pTSchema, NULL, 0);
if (code) goto _err; if (code) goto _err;
/* merge with data in .data file */ /* merge with data in .data file */

View File

@ -250,7 +250,7 @@ _err:
void tsdbFSDestroy(STsdbFS *pFS) { void tsdbFSDestroy(STsdbFS *pFS) {
if (pFS->pDelFile) { if (pFS->pDelFile) {
taosMemoryFree(pFS->pDelFile); taosMemoryFreeClear(pFS->pDelFile);
} }
for (int32_t iSet = 0; iSet < taosArrayGetSize(pFS->aDFileSet); iSet++) { for (int32_t iSet = 0; iSet < taosArrayGetSize(pFS->aDFileSet); iSet++) {
@ -263,7 +263,7 @@ void tsdbFSDestroy(STsdbFS *pFS) {
} }
} }
taosArrayDestroy(pFS->aDFileSet); pFS->aDFileSet = taosArrayDestroy(pFS->aDFileSet);
} }
static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
@ -419,6 +419,7 @@ int32_t tsdbFSOpen(STsdb *pTsdb) {
int32_t code = 0; int32_t code = 0;
// open handle // open handle
pTsdb->fs.version = 0;
pTsdb->fs.pDelFile = NULL; pTsdb->fs.pDelFile = NULL;
pTsdb->fs.aDFileSet = taosArrayInit(0, sizeof(SDFileSet)); pTsdb->fs.aDFileSet = taosArrayInit(0, sizeof(SDFileSet));
if (pTsdb->fs.aDFileSet == NULL) { if (pTsdb->fs.aDFileSet == NULL) {
@ -534,6 +535,7 @@ int32_t tsdbFSClose(STsdb *pTsdb) {
int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) {
int32_t code = 0; int32_t code = 0;
pFS->version = pTsdb->fs.version;
pFS->pDelFile = NULL; pFS->pDelFile = NULL;
pFS->aDFileSet = taosArrayInit(taosArrayGetSize(pTsdb->fs.aDFileSet), sizeof(SDFileSet)); pFS->aDFileSet = taosArrayInit(taosArrayGetSize(pTsdb->fs.aDFileSet), sizeof(SDFileSet));
if (pFS->aDFileSet == NULL) { if (pFS->aDFileSet == NULL) {
@ -664,6 +666,9 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) {
} }
} }
// update the diskId
pDFileSet->diskId = pSet->diskId;
goto _exit; goto _exit;
} }
} }
@ -712,6 +717,108 @@ _exit:
return code; return code;
} }
/**
* @brief Update or delete DFileSet in pFS according to DFileSet (fid <= maxFid) in pFSNew.
*
* @param pTsdb
* @param pFS
* @param pFSNew
* @param maxFid
* @return int32_t
*/
int32_t tsdbFSUpdDel(STsdb *pTsdb, STsdbFS *pFS, STsdbFS *pFSNew, int32_t maxFid) {
int32_t code = 0;
int32_t nRef = 0;
char fname[TSDB_FILENAME_LEN];
int32_t iOld = 0;
int32_t iNew = 0;
while (true) {
int32_t nOld = taosArrayGetSize(pFS->aDFileSet);
int32_t nNew = taosArrayGetSize(pFSNew->aDFileSet);
SDFileSet fSet;
int8_t sameDisk;
if (iOld >= nOld && iNew >= nNew) break;
SDFileSet *pSetOld = (iOld < nOld) ? taosArrayGet(pFS->aDFileSet, iOld) : NULL;
SDFileSet *pSetNew = (iNew < nNew) ? taosArrayGet(pFSNew->aDFileSet, iNew) : NULL;
if (pSetNew && (pSetNew->fid > maxFid)) break;
if (pSetOld && pSetNew) {
if (pSetOld->fid == pSetNew->fid) {
goto _merge_migrate;
} else if (pSetOld->fid > pSetNew->fid) {
goto _remove_old;
} else {
++iOld;
ASSERT(0);
}
continue;
} else {
break;
}
_merge_migrate:
sameDisk = ((pSetOld->diskId.level == pSetNew->diskId.level) && (pSetOld->diskId.id == pSetNew->diskId.id));
ASSERT(pSetOld->pHeadF->commitID == pSetNew->pHeadF->commitID);
ASSERT(pSetOld->pHeadF->size == pSetNew->pHeadF->size);
ASSERT(pSetOld->pHeadF->offset == pSetNew->pHeadF->offset);
if (!sameDisk) {
// head
*pSetOld->pHeadF = *pSetNew->pHeadF;
pSetOld->pHeadF->nRef = 1;
// data
ASSERT(pSetOld->pDataF->size == pSetNew->pDataF->size);
*pSetOld->pDataF = *pSetNew->pDataF;
pSetOld->pDataF->nRef = 1;
// sma
ASSERT(pSetOld->pSmaF->size == pSetNew->pSmaF->size);
*pSetOld->pSmaF = *pSetNew->pSmaF;
pSetOld->pSmaF->nRef = 1;
// stt
ASSERT(pSetOld->nSttF == pSetNew->nSttF);
for (int32_t iStt = 0; iStt < pSetOld->nSttF; ++iStt) {
ASSERT(pSetOld->aSttF[iStt]->size == pSetNew->aSttF[iStt]->size);
ASSERT(pSetOld->aSttF[iStt]->offset == pSetNew->aSttF[iStt]->offset);
*pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt];
pSetOld->aSttF[iStt]->nRef = 1;
}
// set diskId
pSetOld->diskId = pSetNew->diskId;
}
++iOld;
++iNew;
continue;
_remove_old:
taosMemoryFree(pSetOld->pHeadF);
taosMemoryFree(pSetOld->pDataF);
for (int32_t iStt = 0; iStt < pSetOld->nSttF; ++iStt) {
taosMemoryFree(pSetOld->aSttF[iStt]);
}
taosMemoryFree(pSetOld->pSmaF);
taosArrayRemove(pFS->aDFileSet, iOld);
++iNew;
continue;
}
return code;
_err:
tsdbError("vgId:%d, tsdb fs upd/del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code;
}
int32_t tsdbFSCommit1(STsdb *pTsdb, STsdbFS *pFSNew) { int32_t tsdbFSCommit1(STsdb *pTsdb, STsdbFS *pFSNew) {
int32_t code = 0; int32_t code = 0;
char tfname[TSDB_FILENAME_LEN]; char tfname[TSDB_FILENAME_LEN];
@ -745,6 +852,8 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
int32_t nRef; int32_t nRef;
char fname[TSDB_FILENAME_LEN]; char fname[TSDB_FILENAME_LEN];
++pTsdb->fs.version;
// del // del
if (pFSNew->pDelFile) { if (pFSNew->pDelFile) {
SDelFile *pDelFile = pTsdb->fs.pDelFile; SDelFile *pDelFile = pTsdb->fs.pDelFile;
@ -921,8 +1030,8 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
*pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt]; *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt];
pSetOld->aSttF[iStt]->nRef = 1; pSetOld->aSttF[iStt]->nRef = 1;
} else { } else {
ASSERT(pSetOld->aSttF[iStt]->size == pSetOld->aSttF[iStt]->size); ASSERT(pSetOld->aSttF[iStt]->size == pSetNew->aSttF[iStt]->size);
ASSERT(pSetOld->aSttF[iStt]->offset == pSetOld->aSttF[iStt]->offset); ASSERT(pSetOld->aSttF[iStt]->offset == pSetNew->aSttF[iStt]->offset);
} }
} }
} }

View File

@ -28,11 +28,10 @@ struct SLDataIter {
uint64_t uid; uint64_t uid;
STimeWindow timeWindow; STimeWindow timeWindow;
SVersionRange verRange; SVersionRange verRange;
SSttBlockLoadInfo* pBlockLoadInfo; SSttBlockLoadInfo* pBlockLoadInfo;
}; };
SSttBlockLoadInfo* tCreateLastBlockLoadInfo() { SSttBlockLoadInfo* tCreateLastBlockLoadInfo(STSchema* pSchema, int16_t* colList, int32_t numOfCols) {
SSttBlockLoadInfo* pLoadInfo = taosMemoryCalloc(TSDB_DEFAULT_STT_FILE, sizeof(SSttBlockLoadInfo)); SSttBlockLoadInfo* pLoadInfo = taosMemoryCalloc(TSDB_DEFAULT_STT_FILE, sizeof(SSttBlockLoadInfo));
if (pLoadInfo == NULL) { if (pLoadInfo == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -55,6 +54,9 @@ SSttBlockLoadInfo* tCreateLastBlockLoadInfo() {
} }
pLoadInfo[i].aSttBlk = taosArrayInit(4, sizeof(SSttBlk)); pLoadInfo[i].aSttBlk = taosArrayInit(4, sizeof(SSttBlk));
pLoadInfo[i].pSchema = pSchema;
pLoadInfo[i].colIds = colList;
pLoadInfo[i].numOfCols = numOfCols;
} }
return pLoadInfo; return pLoadInfo;
@ -111,7 +113,19 @@ static SBlockData* loadLastBlock(SLDataIter *pIter, const char* idStr) {
pInfo->currentLoadBlockIndex ^= 1; pInfo->currentLoadBlockIndex ^= 1;
if (pIter->pSttBlk != NULL) { // current block not loaded yet if (pIter->pSttBlk != NULL) { // current block not loaded yet
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, &pInfo->blockData[pInfo->currentLoadBlockIndex]);
SBlockData* pBlock = &pInfo->blockData[pInfo->currentLoadBlockIndex];
TABLEID id = {0};
if (pIter->pSttBlk->suid != 0) {
id.suid = pIter->pSttBlk->suid;
} else {
id.uid = pIter->uid;
}
tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols);
code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlock);
double el = (taosGetTimestampUs() - st)/ 1000.0; double el = (taosGetTimestampUs() - st)/ 1000.0;
pInfo->elapsedTime += el; pInfo->elapsedTime += el;
pInfo->loadBlocks += 1; pInfo->loadBlocks += 1;
@ -460,7 +474,8 @@ static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) {
} }
int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
STimeWindow *pTimeWindow, SVersionRange *pVerRange, void* pBlockLoadInfo, const char* idStr) { STimeWindow *pTimeWindow, SVersionRange *pVerRange, void* pBlockLoadInfo, STSchema* pSchema,
int16_t* pCols, int32_t numOfCols, const char* idStr) {
pMTree->backward = backward; pMTree->backward = backward;
pMTree->pIter = NULL; pMTree->pIter = NULL;
pMTree->pIterList = taosArrayInit(4, POINTER_BYTES); pMTree->pIterList = taosArrayInit(4, POINTER_BYTES);
@ -475,9 +490,10 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead
SSttBlockLoadInfo* pLoadInfo = NULL; SSttBlockLoadInfo* pLoadInfo = NULL;
if (pBlockLoadInfo == NULL) { if (pBlockLoadInfo == NULL) {
ASSERT(0);
if (pMTree->pLoadInfo == NULL) { if (pMTree->pLoadInfo == NULL) {
pMTree->destroyLoadInfo = true; pMTree->destroyLoadInfo = true;
pMTree->pLoadInfo = tCreateLastBlockLoadInfo(); pMTree->pLoadInfo = tCreateLastBlockLoadInfo(pSchema, pCols, numOfCols);
} }
pLoadInfo = pMTree->pLoadInfo; pLoadInfo = pMTree->pLoadInfo;

View File

@ -71,6 +71,10 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
goto _err; goto _err;
} }
pTsdb->trimHdl.maxRetentFid = INT32_MIN;
pTsdb->trimHdl.minCommitFid = INT32_MAX;
pTsdb->trimHdl.commitInWait = 0;
tsdbDebug("vgId:%d, tsdb is opened at %s, days:%d, keep:%d,%d,%d", TD_VID(pVnode), pTsdb->path, pTsdb->keepCfg.days, tsdbDebug("vgId:%d, tsdb is opened at %s, days:%d, keep:%d,%d,%d", TD_VID(pVnode), pTsdb->path, pTsdb->keepCfg.days,
pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2); pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2);

View File

@ -79,6 +79,7 @@ typedef struct SBlockLoadSuppInfo {
SColumnDataAgg tsColAgg; SColumnDataAgg tsColAgg;
SColumnDataAgg** plist; SColumnDataAgg** plist;
int16_t* colIds; // column ids for loading file block data int16_t* colIds; // column ids for loading file block data
int32_t numOfCols;
char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated.
} SBlockLoadSuppInfo; } SBlockLoadSuppInfo;
@ -203,6 +204,7 @@ static int32_t setColumnIdSlotList(STsdbReader* pReader, SSDataBlock* pBlock) {
size_t numOfCols = blockDataGetNumOfCols(pBlock); size_t numOfCols = blockDataGetNumOfCols(pBlock);
pSupInfo->numOfCols = numOfCols;
pSupInfo->colIds = taosMemoryMalloc(numOfCols * sizeof(int16_t)); pSupInfo->colIds = taosMemoryMalloc(numOfCols * sizeof(int16_t));
pSupInfo->buildBuf = taosMemoryCalloc(numOfCols, POINTER_BYTES); pSupInfo->buildBuf = taosMemoryCalloc(numOfCols, POINTER_BYTES);
if (pSupInfo->buildBuf == NULL || pSupInfo->colIds == NULL) { if (pSupInfo->buildBuf == NULL || pSupInfo->colIds == NULL) {
@ -352,7 +354,8 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb
tMergeTreeClose(&pLReader->mergeTree); tMergeTreeClose(&pLReader->mergeTree);
if (pLReader->pInfo == NULL) { if (pLReader->pInfo == NULL) {
pLReader->pInfo = tCreateLastBlockLoadInfo(); // here we ignore the first column, which is always be the primary timestamp column
pLReader->pInfo = tCreateLastBlockLoadInfo(pReader->pSchema, &pReader->suppInfo.colIds[1], pReader->suppInfo.numOfCols - 1);
if (pLReader->pInfo == NULL) { if (pLReader->pInfo == NULL) {
tsdbDebug("init fileset iterator failed, code:%s %s", tstrerror(terrno), pReader->idStr); tsdbDebug("init fileset iterator failed, code:%s %s", tstrerror(terrno), pReader->idStr);
return terrno; return terrno;
@ -1279,14 +1282,8 @@ static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, cons
if (p->version >= pBlock->minVer) { if (p->version >= pBlock->minVer) {
if (i < num - 1) { if (i < num - 1) {
TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1); TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1);
// if (i + 1 == num - 1) { // pnext is the last point
if (pnext->ts >= pBlock->minKey.ts) { if (pnext->ts >= pBlock->minKey.ts) {
return true; return true;
// }
// } else {
// if (pnext->ts >= pBlock->minKey.ts) {
// return true;
// }
} }
} else { // it must be the last point } else { // it must be the last point
ASSERT(p->version == 0); ASSERT(p->version == 0);
@ -2001,7 +1998,8 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan
int32_t code = int32_t code =
tMergeTreeOpen(&pLBlockReader->mergeTree, (pLBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader, tMergeTreeOpen(&pLBlockReader->mergeTree, (pLBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader,
pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange, pLBlockReader->pInfo, pReader->idStr); pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange, pLBlockReader->pInfo,
pReader->pSchema, pReader->suppInfo.colIds, pReader->suppInfo.numOfCols, pReader->idStr);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return false; return false;
} }
@ -2458,7 +2456,8 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
code = buildComposedDataBlock(pReader); code = buildComposedDataBlock(pReader);
} else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) { } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) {
tBlockDataReset(&pStatus->fileBlockData); tBlockDataReset(&pStatus->fileBlockData);
code = tBlockDataInit(&pStatus->fileBlockData, pReader->suid, pScanInfo->uid, pReader->pSchema); TABLEID tid = {.suid = pReader->suid, .uid = pScanInfo->uid};
code = tBlockDataInit(&pStatus->fileBlockData, &tid, pReader->pSchema, NULL, 0);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
@ -2938,7 +2937,8 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn
// 3. load the neighbor block, and set it to be the currently accessed file data block // 3. load the neighbor block, and set it to be the currently accessed file data block
tBlockDataReset(&pStatus->fileBlockData); tBlockDataReset(&pStatus->fileBlockData);
int32_t code = tBlockDataInit(&pStatus->fileBlockData, pReader->suid, pFBlock->uid, pReader->pSchema); TABLEID tid = {.suid = pReader->suid, .uid = pFBlock->uid};
int32_t code = tBlockDataInit(&pStatus->fileBlockData, &tid, pReader->pSchema, NULL, 0);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
@ -3221,8 +3221,8 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
tColDataGetValue(pData, rowIndex, &cv); tColDataGetValue(pData, rowIndex, &cv);
doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo); doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo);
j += 1; j += 1;
} else if (pData->cid > } else if (pData->cid > pCol->info.colId) {
pCol->info.colId) { // the specified column does not exist in file block, fill with null data // the specified column does not exist in file block, fill with null data
colDataAppendNULL(pCol, outputRowIndex); colDataAppendNULL(pCol, outputRowIndex);
} }
@ -3410,10 +3410,14 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
// we need only one row // we need only one row
pPrevReader->capacity = 1; pPrevReader->capacity = 1;
pPrevReader->status.pTableMap = pReader->status.pTableMap; pPrevReader->status.pTableMap = pReader->status.pTableMap;
pPrevReader->pSchema = pReader->pSchema;
pPrevReader->pMemSchema = pReader->pMemSchema;
pPrevReader->pReadSnap = pReader->pReadSnap; pPrevReader->pReadSnap = pReader->pReadSnap;
pNextReader->capacity = 1; pNextReader->capacity = 1;
pNextReader->status.pTableMap = pReader->status.pTableMap; pNextReader->status.pTableMap = pReader->status.pTableMap;
pNextReader->pSchema = pReader->pSchema;
pNextReader->pMemSchema = pReader->pMemSchema;
pNextReader->pReadSnap = pReader->pReadSnap; pNextReader->pReadSnap = pReader->pReadSnap;
code = doOpenReaderImpl(pPrevReader); code = doOpenReaderImpl(pPrevReader);
@ -3447,11 +3451,19 @@ void tsdbReaderClose(STsdbReader* pReader) {
{ {
if (pReader->innerReader[0] != NULL) { if (pReader->innerReader[0] != NULL) {
pReader->innerReader[0]->status.pTableMap = NULL; STsdbReader* p = pReader->innerReader[0];
pReader->innerReader[0]->pReadSnap = NULL;
pReader->innerReader[1]->status.pTableMap = NULL; p->status.pTableMap = NULL;
pReader->innerReader[1]->pReadSnap = NULL; p->pReadSnap = NULL;
p->pSchema = NULL;
p->pMemSchema = NULL;
p = pReader->innerReader[1];
p->status.pTableMap = NULL;
p->pReadSnap = NULL;
p->pSchema = NULL;
p->pMemSchema = NULL;
tsdbReaderClose(pReader->innerReader[0]); tsdbReaderClose(pReader->innerReader[0]);
tsdbReaderClose(pReader->innerReader[1]); tsdbReaderClose(pReader->innerReader[1]);
@ -3690,7 +3702,8 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) {
STableBlockScanInfo* pBlockScanInfo = taosHashGet(pStatus->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); STableBlockScanInfo* pBlockScanInfo = taosHashGet(pStatus->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid));
tBlockDataReset(&pStatus->fileBlockData); tBlockDataReset(&pStatus->fileBlockData);
int32_t code = tBlockDataInit(&pStatus->fileBlockData, pReader->suid, pBlockScanInfo->uid, pReader->pSchema); TABLEID tid = {.suid = pReader->suid, .uid = pBlockScanInfo->uid};
int32_t code = tBlockDataInit(&pStatus->fileBlockData, &tid, pReader->pSchema, NULL, 0);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
terrno = code; terrno = code;
return NULL; return NULL;

View File

@ -607,7 +607,66 @@ _err:
return code; return code;
} }
int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { /**
* @brief send file with limited speed(rough control)
*
* @param pTsdb
* @param pFileOut
* @param pFileIn
* @param size
* @param speed 0 no limit, unit: B/s
* @return int64_t
*/
static int64_t tsdbFSendFile(STsdb *pTsdb, TdFilePtr pOutFD, TdFilePtr pInFD, int64_t size, int64_t speed) {
if (speed <= 0) {
return taosFSendFile(pOutFD, pInFD, 0, size);
}
int64_t offset = 0;
int64_t tBytes = 0;
int64_t nBytes = 0;
int64_t startMs = 0;
int64_t cost = 0;
while ((offset + nBytes) < size) {
if (atomic_load_8(&pTsdb->trimHdl.commitInWait) == 1) {
tsdbDebug("vgId:%d sendFile without limit since conflicts, fSize:%" PRIi64 ", maxSpeed:%" PRIi64,
TD_VID(pTsdb->pVnode), size, speed);
goto _send_remain;
}
startMs = taosGetTimestampMs();
if ((nBytes = taosFSendFile(pOutFD, pInFD, &offset, speed)) < 0) {
return nBytes;
}
cost = taosGetTimestampMs() - startMs;
tBytes += nBytes;
int64_t nSleep = 0;
if (cost < 0) {
nSleep = 1000;
} else if (cost < 1000) {
nSleep = 1000 - cost;
}
if (nSleep > 0) {
taosMsleep(nSleep);
tsdbDebug("vgId:%d sendFile and msleep:%" PRIi64 ", fSize:%" PRIi64 ", tBytes:%" PRIi64 " maxSpeed:%" PRIi64,
TD_VID(pTsdb->pVnode), nSleep, size, tBytes, speed);
}
}
_send_remain:
if (offset < size) {
if ((nBytes = taosFSendFile(pOutFD, pInFD, &offset, size - offset)) < 0) {
return nBytes;
}
tBytes += nBytes;
tsdbDebug("vgId:%d sendFile remain, fSize:%" PRIi64 ", tBytes:%" PRIi64 " maxSpeed:%" PRIi64, TD_VID(pTsdb->pVnode),
size, tBytes, speed);
}
return tBytes;
}
int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, int64_t maxSpeed) {
int32_t code = 0; int32_t code = 0;
int64_t n; int64_t n;
int64_t size; int64_t size;
@ -620,7 +679,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
// head // head
tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom); tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom);
tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo); tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo);
pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
if (pOutFD == NULL) { if (pOutFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
@ -630,7 +689,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pHeadF->size, szPage)); n = tsdbFSendFile(pTsdb, pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->pHeadF->size, szPage), maxSpeed);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
@ -641,7 +700,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
// data // data
tsdbDataFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pDataF, fNameFrom); tsdbDataFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pDataF, fNameFrom);
tsdbDataFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pDataF, fNameTo); tsdbDataFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pDataF, fNameTo);
pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
if (pOutFD == NULL) { if (pOutFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
@ -651,7 +710,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
n = taosFSendFile(pOutFD, PInFD, 0, LOGIC_TO_FILE_OFFSET(pSetFrom->pDataF->size, szPage)); n = tsdbFSendFile(pTsdb, pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->pDataF->size, szPage), maxSpeed);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
@ -662,7 +721,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
// sma // sma
tsdbSmaFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pSmaF, fNameFrom); tsdbSmaFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pSmaF, fNameFrom);
tsdbSmaFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pSmaF, fNameTo); tsdbSmaFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pSmaF, fNameTo);
pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
if (pOutFD == NULL) { if (pOutFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
@ -672,7 +731,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pSmaF->size, szPage)); n = tsdbFSendFile(pTsdb, pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->pSmaF->size, szPage), maxSpeed);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
@ -684,7 +743,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
for (int8_t iStt = 0; iStt < pSetFrom->nSttF; iStt++) { for (int8_t iStt = 0; iStt < pSetFrom->nSttF; iStt++) {
tsdbSttFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSttF[iStt], fNameFrom); tsdbSttFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSttF[iStt], fNameFrom);
tsdbSttFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSttF[iStt], fNameTo); tsdbSttFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSttF[iStt], fNameTo);
pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
if (pOutFD == NULL) { if (pOutFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
@ -694,7 +753,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->aSttF[iStt]->size, szPage)); n = tsdbFSendFile(pTsdb, pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->aSttF[iStt]->size, szPage), maxSpeed);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
@ -926,12 +985,13 @@ _err:
return code; return code;
} }
static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo, SBlockData *pBlockData) { static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo, SBlockData *pBlockData,
int32_t iStt) {
int32_t code = 0; int32_t code = 0;
tBlockDataClear(pBlockData); tBlockDataClear(pBlockData);
STsdbFD *pFD = pReader->pDataFD; STsdbFD *pFD = (iStt < 0) ? pReader->pDataFD : pReader->aSttFD[iStt];
// uid + version + tskey // uid + version + tskey
code = tRealloc(&pReader->aBuf[0], pBlkInfo->szKey); code = tRealloc(&pReader->aBuf[0], pBlkInfo->szKey);
@ -1070,9 +1130,12 @@ _err:
int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData) { int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData) {
int32_t code = 0; int32_t code = 0;
code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[0], pBlockData); code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[0], pBlockData, -1);
if (code) goto _err; if (code) goto _err;
ASSERT(pDataBlk->nSubBlock == 1);
#if 0
if (pDataBlk->nSubBlock > 1) { if (pDataBlk->nSubBlock > 1) {
SBlockData bData1; SBlockData bData1;
SBlockData bData2; SBlockData bData2;
@ -1113,6 +1176,7 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData
tBlockDataDestroy(&bData1, 1); tBlockDataDestroy(&bData1, 1);
tBlockDataDestroy(&bData2, 1); tBlockDataDestroy(&bData2, 1);
} }
#endif
return code; return code;
@ -1123,23 +1187,38 @@ _err:
int32_t tsdbReadSttBlock(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData) { int32_t tsdbReadSttBlock(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
code = tsdbReadBlockDataImpl(pReader, &pSttBlk->bInfo, pBlockData, iStt);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pReader->pTsdb->pVnode), __func__, lino, tstrerror(code));
}
return code;
}
int32_t tsdbReadSttBlockEx(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData) {
int32_t code = 0;
int32_t lino = 0;
// alloc // alloc
code = tRealloc(&pReader->aBuf[0], pSttBlk->bInfo.szBlock); code = tRealloc(&pReader->aBuf[0], pSttBlk->bInfo.szBlock);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// read // read
code = tsdbReadFile(pReader->aSttFD[iStt], pSttBlk->bInfo.offset, pReader->aBuf[0], pSttBlk->bInfo.szBlock); code = tsdbReadFile(pReader->aSttFD[iStt], pSttBlk->bInfo.offset, pReader->aBuf[0], pSttBlk->bInfo.szBlock);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// decmpr // decmpr
code = tDecmprBlockData(pReader->aBuf[0], pSttBlk->bInfo.szBlock, pBlockData, &pReader->aBuf[1]); code = tDecmprBlockData(pReader->aBuf[0], pSttBlk->bInfo.szBlock, pBlockData, &pReader->aBuf[1]);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
return code; _exit:
if (code) {
_err: tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pReader->pTsdb->pVnode), __func__, lino, tstrerror(code));
tsdbError("vgId:%d tsdb read stt block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); }
return code; return code;
} }

View File

@ -15,57 +15,189 @@
#include "tsdb.h" #include "tsdb.h"
static bool tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) { enum { RETENTION_NO = 0, RETENTION_EXPIRED = 1, RETENTION_MIGRATE = 2 };
for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) {
SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); #define MIGRATE_MAX_SPEED (1048576 << 4) // 16 MB, vnode level
#define MIGRATE_MIN_COST (5) // second
static bool tsdbShouldDoMigrate(STsdb *pTsdb);
static int32_t tsdbShouldDoRetention(STsdb *pTsdb, int64_t now);
static int32_t tsdbProcessRetention(STsdb *pTsdb, int64_t now, int64_t maxSpeed, int32_t retention, int8_t type);
static bool tsdbShouldDoMigrate(STsdb *pTsdb) {
if (tfsGetLevel(pTsdb->pVnode->pTfs) < 2) {
return false;
}
STsdbKeepCfg *keepCfg = &pTsdb->keepCfg;
if (keepCfg->keep0 == keepCfg->keep1 && keepCfg->keep1 == keepCfg->keep2) {
return false;
}
return true;
}
static int32_t tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) {
int32_t retention = RETENTION_NO;
if (taosArrayGetSize(pTsdb->fs.aDFileSet) == 0) {
return retention;
}
SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, 0);
if (tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now) < 0) {
retention |= RETENTION_EXPIRED;
}
if (!tsdbShouldDoMigrate(pTsdb)) {
return retention;
}
for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); ++iSet) {
pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet);
int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now);
SDiskID did;
if (expLevel == pSet->diskId.level) continue; if (expLevel == pSet->diskId.level) continue;
if (expLevel < 0) { if (expLevel > 0) {
return true; retention |= RETENTION_MIGRATE;
} else { break;
if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) {
return false;
}
if (did.level == pSet->diskId.level) continue;
return true;
} }
} }
return false; return retention;
} }
int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { /**
* @brief process retention
*
* @param pTsdb
* @param now
* @param maxSpeed
* @param retention
* @param type 0 RETENTION_EXPIRED, 1 RETENTION_MIGRATE
* @return int32_t
*/
static int32_t tsdbProcessRetention(STsdb *pTsdb, int64_t now, int64_t maxSpeed, int32_t retention, int8_t type) {
int32_t code = 0; int32_t code = 0;
int32_t nBatch = 0;
int32_t nLoops = 0;
int32_t maxFid = 0;
int64_t fSize = 0;
int64_t speed = maxSpeed > 0 ? maxSpeed : MIGRATE_MAX_SPEED;
STsdbFS fs = {0};
STsdbFS fsLatest = {0};
if (!tsdbShouldDoRetention(pTsdb, now)) { if (!(retention & type)) {
return code; goto _exit;
} }
// do retention _retention_loop:
STsdbFS fs; // reset
maxFid = INT32_MIN;
fSize = 0;
tsdbFSDestroy(&fs);
tsdbFSDestroy(&fsLatest);
if (atomic_load_8(&pTsdb->trimHdl.commitInWait) == 1) {
atomic_store_32(&pTsdb->trimHdl.maxRetentFid, INT32_MIN);
taosMsleep(50);
}
code = tsdbFSCopy(pTsdb, &fs); code = tsdbFSCopy(pTsdb, &fs);
if (code) goto _err; if (code) goto _exit;
for (int32_t iSet = 0; iSet < taosArrayGetSize(fs.aDFileSet); iSet++) { if (type == RETENTION_MIGRATE) {
int32_t fsSize = taosArrayGetSize(fs.aDFileSet);
for (int32_t iSet = 0; iSet < fsSize; ++iSet) {
SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet);
int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now);
SDiskID did;
if (pSet->diskId.level == expLevel) continue;
if (expLevel > 0) {
ASSERT(pSet->fid > maxFid);
maxFid = pSet->fid;
fSize += (pSet->pDataF->size + pSet->pHeadF->size + pSet->pSmaF->size);
if (fSize / speed > MIGRATE_MIN_COST) {
tsdbDebug("vgId:%d migrate loop %d with maxFid:%d", TD_VID(pTsdb->pVnode), nBatch, maxFid);
break;
}
for (int32_t iStt = 0; iStt < pSet->nSttF; ++iStt) {
fSize += pSet->aSttF[iStt]->size;
}
if (fSize / speed > MIGRATE_MIN_COST) {
tsdbDebug("vgId:%d migrate loop %d with maxFid:%d", TD_VID(pTsdb->pVnode), nBatch, maxFid);
break;
}
}
}
} else if (type == RETENTION_EXPIRED) {
for (int32_t iSet = 0; iSet < taosArrayGetSize(fs.aDFileSet); ++iSet) {
SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet);
int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now);
SDiskID did; SDiskID did;
if (expLevel < 0) { if (expLevel < 0) {
ASSERT(pSet->fid > maxFid);
if (pSet->fid > maxFid) maxFid = pSet->fid;
taosMemoryFree(pSet->pHeadF); taosMemoryFree(pSet->pHeadF);
taosMemoryFree(pSet->pDataF); taosMemoryFree(pSet->pDataF);
taosMemoryFree(pSet->aSttF[0]);
taosMemoryFree(pSet->pSmaF); taosMemoryFree(pSet->pSmaF);
for (int32_t iStt = 0; iStt < pSet->nSttF; ++iStt) {
taosMemoryFree(pSet->aSttF[iStt]);
}
taosArrayRemove(fs.aDFileSet, iSet); taosArrayRemove(fs.aDFileSet, iSet);
iSet--; --iSet;
} else { } else {
if (expLevel == 0) continue; break;
}
}
}
if (maxFid == INT32_MIN) goto _exit;
_commit_conflict_check:
while (atomic_load_32(&pTsdb->trimHdl.minCommitFid) <= maxFid) {
if (++nLoops > 1000) {
nLoops = 0;
sched_yield();
}
}
if (atomic_val_compare_exchange_8(&pTsdb->trimHdl.state, 0, 1) == 0) {
if (atomic_load_32(&pTsdb->trimHdl.minCommitFid) <= maxFid) {
atomic_store_8(&pTsdb->trimHdl.state, 0);
goto _commit_conflict_check;
}
atomic_store_32(&pTsdb->trimHdl.maxRetentFid, maxFid);
atomic_store_8(&pTsdb->trimHdl.state, 0);
} else {
goto _commit_conflict_check;
}
// migrate
if (type == RETENTION_MIGRATE) {
for (int32_t iSet = 0; iSet < taosArrayGetSize(fs.aDFileSet); ++iSet) {
SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet);
int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now);
SDiskID did;
if (pSet->fid > maxFid) break;
tsdbDebug("vgId:%d migrate loop %d with maxFid:%d, fid:%d, did:%d, level:%d, expLevel:%d", TD_VID(pTsdb->pVnode),
nBatch, maxFid, pSet->fid, pSet->diskId.id, pSet->diskId.level, expLevel);
if (expLevel < 0) {
taosMemoryFree(pSet->pHeadF);
taosMemoryFree(pSet->pDataF);
taosMemoryFree(pSet->pSmaF);
for (int32_t iStt = 0; iStt < pSet->nSttF; ++iStt) {
taosMemoryFree(pSet->aSttF[iStt]);
}
taosArrayRemove(fs.aDFileSet, iSet);
--iSet;
} else {
if (expLevel == pSet->diskId.level) continue;
if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) {
code = terrno; code = terrno;
goto _exit; goto _exit;
@ -73,40 +205,106 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) {
if (did.level == pSet->diskId.level) continue; if (did.level == pSet->diskId.level) continue;
// copy file to new disk (todo) // copy file to new disk
SDFileSet fSet = *pSet; SDFileSet fSet = *pSet;
fSet.diskId = did; fSet.diskId = did;
code = tsdbDFileSetCopy(pTsdb, pSet, &fSet); code = tsdbDFileSetCopy(pTsdb, pSet, &fSet, maxSpeed);
if (code) goto _err; if (code) goto _exit;
code = tsdbFSUpsertFSet(&fs, &fSet); code = tsdbFSUpsertFSet(&fs, &fSet);
if (code) goto _err; if (code) goto _exit;
}
} }
} }
// do change fs _merge_fs:
code = tsdbFSCommit1(pTsdb, &fs);
if (code) goto _err;
taosThreadRwlockWrlock(&pTsdb->rwLock); taosThreadRwlockWrlock(&pTsdb->rwLock);
code = tsdbFSCommit2(pTsdb, &fs); // 1) prepare fs, merge tsdbFSNew and pTsdb->fs if needed
if (code) { STsdbFS *pTsdbFS = &fs;
ASSERT(fs.version <= pTsdb->fs.version);
if (fs.version < pTsdb->fs.version) {
if ((code = tsdbFSCopy(pTsdb, &fsLatest))) {
taosThreadRwlockUnlock(&pTsdb->rwLock); taosThreadRwlockUnlock(&pTsdb->rwLock);
goto _err; goto _exit;
} }
if ((code = tsdbFSUpdDel(pTsdb, &fsLatest, &fs, maxFid))) {
taosThreadRwlockUnlock(&pTsdb->rwLock);
goto _exit;
}
pTsdbFS = &fsLatest;
}
// 2) save CURRENT
if ((code = tsdbFSCommit1(pTsdb, pTsdbFS))) {
taosThreadRwlockUnlock(&pTsdb->rwLock);
goto _exit;
}
// 3) apply the tsdbFS to pTsdb->fs
if ((code = tsdbFSCommit2(pTsdb, pTsdbFS))) {
taosThreadRwlockUnlock(&pTsdb->rwLock);
goto _exit;
}
taosThreadRwlockUnlock(&pTsdb->rwLock); taosThreadRwlockUnlock(&pTsdb->rwLock);
tsdbFSDestroy(&fs); if (type == RETENTION_MIGRATE) {
++nBatch;
goto _retention_loop;
}
_exit: _exit:
return code; tsdbFSDestroy(&fs);
tsdbFSDestroy(&fsLatest);
_err: if (code != 0) {
tsdbError("vgId:%d, tsdb do retention failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); tsdbError("vgId:%d, tsdb do retention %" PRIi8 " failed since %s", TD_VID(pTsdb->pVnode), type, tstrerror(code));
ASSERT(0); ASSERT(0);
// tsdbFSRollback(pTsdb->pFS); }
return code;
}
/**
* @brief Data migration between multi-tier storage, including remove expired data.
* 1) firstly, remove expired DFileSet;
* 2) partition the tsdbFS by the expLevel and the estimated cost(e.g. 5s) to copy, and migrate
* DFileSet groups between multi-tier storage;
* 3) update the tsdbFS and CURRENT in the same transaction;
* 4) finish the migration
* @param pTsdb
* @param now
* @param maxSpeed
* @return int32_t
*/
int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now, int64_t maxSpeed) {
int32_t code = 0;
int32_t retention = RETENTION_NO;
retention = tsdbShouldDoRetention(pTsdb, now);
if (retention == RETENTION_NO) {
goto _exit;
}
// step 1: process expire
code = tsdbProcessRetention(pTsdb, now, maxSpeed, retention, RETENTION_EXPIRED);
if (code < 0) goto _exit;
// step 2: process multi-tier migration
code = tsdbProcessRetention(pTsdb, now, maxSpeed, retention, RETENTION_MIGRATE);
if (code < 0) goto _exit;
_exit:
pTsdb->trimHdl.maxRetentFid = INT32_MIN;
if (code != 0) {
tsdbError("vgId:%d, tsdb do retention %d failed since %s, time:%" PRIi64 ", max speed:%" PRIi64,
TD_VID(pTsdb->pVnode), retention, tstrerror(code), now, maxSpeed);
ASSERT(0);
// tsdbFSRollback(pTsdb->pFS);
} else {
tsdbInfo("vgId:%d, tsdb do retention %d succeed, time:%" PRIi64 ", max speed:%" PRIi64, TD_VID(pTsdb->pVnode),
retention, now, maxSpeed);
}
return code; return code;
} }

View File

@ -140,7 +140,7 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) {
if (pSttBlk->minVer > pReader->ever) continue; if (pSttBlk->minVer > pReader->ever) continue;
if (pSttBlk->maxVer < pReader->sver) continue; if (pSttBlk->maxVer < pReader->sver) continue;
code = tsdbReadSttBlock(pReader->pDataFReader, iStt, pSttBlk, &pIter->bData); code = tsdbReadSttBlockEx(pReader->pDataFReader, iStt, pSttBlk, &pIter->bData);
if (code) goto _err; if (code) goto _err;
for (pIter->iRow = 0; pIter->iRow < pIter->bData.nRow; pIter->iRow++) { for (pIter->iRow = 0; pIter->iRow < pIter->bData.nRow; pIter->iRow++) {
@ -223,7 +223,7 @@ static int32_t tsdbSnapNextRow(STsdbSnapReader* pReader) {
if (pSttBlk->minVer > pReader->ever || pSttBlk->maxVer < pReader->sver) continue; if (pSttBlk->minVer > pReader->ever || pSttBlk->maxVer < pReader->sver) continue;
code = tsdbReadSttBlock(pReader->pDataFReader, pIter->iStt, pSttBlk, &pIter->bData); code = tsdbReadSttBlockEx(pReader->pDataFReader, pIter->iStt, pSttBlk, &pIter->bData);
if (code) goto _err; if (code) goto _err;
pIter->iRow = -1; pIter->iRow = -1;
@ -319,7 +319,7 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
code = tsdbUpdateTableSchema(pTsdb->pVnode->pMeta, id.suid, id.uid, &pReader->skmTable); code = tsdbUpdateTableSchema(pTsdb->pVnode->pMeta, id.suid, id.uid, &pReader->skmTable);
if (code) goto _err; if (code) goto _err;
code = tBlockDataInit(pBlockData, id.suid, id.uid, pReader->skmTable.pTSchema); code = tBlockDataInit(pBlockData, &id, pReader->skmTable.pTSchema, NULL, 0);
if (code) goto _err; if (code) goto _err;
while (pRowInfo->suid == id.suid && pRowInfo->uid == id.uid) { while (pRowInfo->suid == id.suid && pRowInfo->uid == id.uid) {
@ -715,7 +715,7 @@ static int32_t tsdbSnapWriteTableDataStart(STsdbSnapWriter* pWriter, TABLEID* pI
if (code) goto _err; if (code) goto _err;
tMapDataReset(&pWriter->dWriter.mDataBlk); tMapDataReset(&pWriter->dWriter.mDataBlk);
code = tBlockDataInit(&pWriter->dWriter.bData, pId->suid, pId->uid, pWriter->skmTable.pTSchema); code = tBlockDataInit(&pWriter->dWriter.bData, pId, pWriter->skmTable.pTSchema, NULL, 0);
if (code) goto _err; if (code) goto _err;
return code; return code;
@ -1000,7 +1000,8 @@ static int32_t tsdbSnapWriteToSttFile(STsdbSnapWriter* pWriter, int32_t iRow) {
code = tsdbUpdateTableSchema(pWriter->pTsdb->pVnode->pMeta, pWriter->id.suid, pWriter->id.uid, &pWriter->skmTable); code = tsdbUpdateTableSchema(pWriter->pTsdb->pVnode->pMeta, pWriter->id.suid, pWriter->id.uid, &pWriter->skmTable);
if (code) goto _err; if (code) goto _err;
code = tBlockDataInit(pBData, pWriter->id.suid, pWriter->id.suid ? 0 : pWriter->id.uid, pWriter->skmTable.pTSchema); TABLEID tid = {.suid = pWriter->id.suid, .uid = pWriter->id.suid ? 0 : pWriter->id.uid};
code = tBlockDataInit(pBData, &tid, pWriter->skmTable.pTSchema, NULL, 0);
if (code) goto _err; if (code) goto _err;
} }

View File

@ -509,16 +509,24 @@ void tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *minK
*maxKey = *minKey + minutes * tsTickPerMin[precision] - 1; *maxKey = *minKey + minutes * tsTickPerMin[precision] - 1;
} }
/**
* @brief get fid level by keep and days.
*
* @param fid
* @param pKeepCfg
* @param now millisecond
* @return int32_t
*/
int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t now) { int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t now) {
int32_t aFid[3]; int32_t aFid[3];
TSKEY key; TSKEY key;
if (pKeepCfg->precision == TSDB_TIME_PRECISION_MILLI) { if (pKeepCfg->precision == TSDB_TIME_PRECISION_MILLI) {
now = now * 1000; // now = now * 1000;
} else if (pKeepCfg->precision == TSDB_TIME_PRECISION_MICRO) { } else if (pKeepCfg->precision == TSDB_TIME_PRECISION_MICRO) {
now = now * 1000000l; now = now * 1000l;
} else if (pKeepCfg->precision == TSDB_TIME_PRECISION_NANO) { } else if (pKeepCfg->precision == TSDB_TIME_PRECISION_NANO) {
now = now * 1000000000l; now = now * 1000000l;
} else { } else {
ASSERT(0); ASSERT(0);
} }
@ -948,16 +956,38 @@ void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear) {
pBlockData->aColData = NULL; pBlockData->aColData = NULL;
} }
int32_t tBlockDataInit(SBlockData *pBlockData, int64_t suid, int64_t uid, STSchema *pTSchema) { int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid) {
int32_t code = 0; int32_t code = 0;
ASSERT(suid || uid); ASSERT(pId->suid || pId->uid);
pBlockData->suid = suid; pBlockData->suid = pId->suid;
pBlockData->uid = uid; pBlockData->uid = pId->uid;
pBlockData->nRow = 0; pBlockData->nRow = 0;
taosArrayClear(pBlockData->aIdx); taosArrayClear(pBlockData->aIdx);
if (aCid) {
int32_t iColumn = 1;
STColumn *pTColumn = &pTSchema->columns[iColumn];
for (int32_t iCid = 0; iCid < nCid; iCid++) {
while (pTColumn && pTColumn->colId < aCid[iCid]) {
iColumn++;
pTColumn = (iColumn < pTSchema->numOfCols) ? &pTSchema->columns[iColumn] : NULL;
}
if (pTColumn == NULL) {
break;
} else if (pTColumn->colId == aCid[iCid]) {
SColData *pColData;
code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData);
if (code) goto _exit;
tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) ? 1 : 0);
iColumn++;
pTColumn = (iColumn < pTSchema->numOfCols) ? &pTSchema->columns[iColumn] : NULL;
}
}
} else {
for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) {
STColumn *pTColumn = &pTSchema->columns[iColumn]; STColumn *pTColumn = &pTSchema->columns[iColumn];
@ -967,6 +997,7 @@ int32_t tBlockDataInit(SBlockData *pBlockData, int64_t suid, int64_t uid, STSche
tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) ? 1 : 0); tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) ? 1 : 0);
} }
}
_exit: _exit:
return code; return code;

View File

@ -186,6 +186,17 @@ void vnodePreClose(SVnode *pVnode) {
} }
} }
static void vnodeTrimDbClose(SVnode *pVnode) {
int32_t nLoops = 0;
while (atomic_load_8(&pVnode->trimDbH.state) != 0) {
if (++nLoops > 1000) {
vTrace("vgId:%d, wait for trimDb task to finish", TD_VID(pVnode));
sched_yield();
nLoops = 0;
}
}
}
void vnodeClose(SVnode *pVnode) { void vnodeClose(SVnode *pVnode) {
if (pVnode) { if (pVnode) {
vnodeCommit(pVnode); vnodeCommit(pVnode);
@ -197,6 +208,7 @@ void vnodeClose(SVnode *pVnode) {
smaClose(pVnode->pSma); smaClose(pVnode->pSma);
metaClose(pVnode->pMeta); metaClose(pVnode->pMeta);
vnodeCloseBufPool(pVnode); vnodeCloseBufPool(pVnode);
vnodeTrimDbClose(pVnode);
// destroy handle // destroy handle
tsem_destroy(&(pVnode->canCommit)); tsem_destroy(&(pVnode->canCommit));
tsem_destroy(&pVnode->syncSem); tsem_destroy(&pVnode->syncSem);

View File

@ -378,6 +378,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) {
pMetaRsp->precision = pVnode->config.tsdbCfg.precision; pMetaRsp->precision = pVnode->config.tsdbCfg.precision;
} }
#if 0
static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
int32_t code = 0; int32_t code = 0;
SVTrimDbReq trimReq = {0}; SVTrimDbReq trimReq = {0};
@ -388,7 +389,7 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq,
goto _exit; goto _exit;
} }
vInfo("vgId:%d, trim vnode request will be processed, time:%d", pVnode->config.vgId, trimReq.timestamp); vInfo("vgId:%d, trim vnode request will be processed, time:%" PRIi64, pVnode->config.vgId, trimReq.timestamp);
// process // process
code = tsdbDoRetention(pVnode->pTsdb, trimReq.timestamp); code = tsdbDoRetention(pVnode->pTsdb, trimReq.timestamp);
@ -400,6 +401,91 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq,
_exit: _exit:
return code; return code;
} }
#endif
typedef struct {
SVnode *pVnode;
SVTrimDbReq trimReq;
} SVndTrimDbReq;
void *vnodeProcessTrimReqFunc(void *param) {
int32_t code = 0;
int8_t oldVal = 0;
SVndTrimDbReq *pReq = (SVndTrimDbReq *)param;
SVnode *pVnode = pReq->pVnode;
setThreadName("vnode-trim");
// process
code = tsdbDoRetention(pVnode->pTsdb, pReq->trimReq.timestamp, pReq->trimReq.maxSpeed);
if (code) goto _exit;
code = smaDoRetention(pVnode->pSma, pReq->trimReq.timestamp, pReq->trimReq.maxSpeed);
if (code) goto _exit;
_exit:
oldVal = atomic_val_compare_exchange_8(&pVnode->trimDbH.state, 1, 0);
ASSERT(oldVal == 1);
if (code) {
vError("vgId:%d, trim vnode thread failed since %s, time:%" PRIi64 ", max speed:%" PRIi64, TD_VID(pVnode),
tstrerror(code), pReq->trimReq.timestamp, pReq->trimReq.maxSpeed);
} else {
vInfo("vgId:%d, trim vnode thread finish, time:%" PRIi64 ", max speed:%" PRIi64, TD_VID(pVnode),
pReq->trimReq.timestamp, pReq->trimReq.maxSpeed);
}
taosMemoryFree(pReq);
return NULL;
}
static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
int32_t code = 0;
SVndTrimDbReq *pVndTrimReq = taosMemoryMalloc(sizeof(SVndTrimDbReq));
SVTrimDbHdl *pHandle = &pVnode->trimDbH;
if (!pVndTrimReq) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
pVndTrimReq->pVnode = pVnode;
if (tDeserializeSVTrimDbReq(pReq, len, &pVndTrimReq->trimReq) != 0) {
taosMemoryFree(pVndTrimReq);
code = TSDB_CODE_INVALID_MSG;
goto _exit;
}
if (atomic_val_compare_exchange_8(&pHandle->state, 0, 1) != 0) {
vInfo("vgId:%d, trim vnode request ignored since duplicated req, time:%" PRIi64 ", max speed:%" PRIi64,
TD_VID(pVnode), pVndTrimReq->trimReq.timestamp, pVndTrimReq->trimReq.maxSpeed);
taosMemoryFree(pVndTrimReq);
goto _exit;
}
vInfo("vgId:%d, trim vnode request will be processed, time:%" PRIi64 ", max speed:%" PRIi64, TD_VID(pVnode),
pVndTrimReq->trimReq.timestamp, pVndTrimReq->trimReq.maxSpeed);
TdThreadAttr thAttr = {0};
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_DETACHED);
TdThread tid;
if (taosThreadCreate(&tid, &thAttr, vnodeProcessTrimReqFunc, (void *)pVndTrimReq) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(pVndTrimReq);
taosThreadAttrDestroy(&thAttr);
int8_t oldVal = atomic_val_compare_exchange_8(&pHandle->state, 1, 0);
ASSERT(oldVal == 1);
vError("vgId:%d, failed to create pthread to trim vnode since %s", TD_VID(pVnode), tstrerror(code));
goto _exit;
}
vDebug("vgId:%d, success to create pthread to trim vnode", TD_VID(pVnode));
taosThreadAttrDestroy(&thAttr);
_exit:
terrno = code;
return code;
}
static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
SArray *tbUids = taosArrayInit(8, sizeof(int64_t)); SArray *tbUids = taosArrayInit(8, sizeof(int64_t));
@ -921,7 +1007,8 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
} }
if (taosArrayGetSize(newTbUids) > 0) { if (taosArrayGetSize(newTbUids) > 0) {
vDebug("vgId:%d, add %d table into query table list in handling submit", TD_VID(pVnode), (int32_t)taosArrayGetSize(newTbUids)); vDebug("vgId:%d, add %d table into query table list in handling submit", TD_VID(pVnode),
(int32_t)taosArrayGetSize(newTbUids));
} }
tqUpdateTbUidList(pVnode->pTq, newTbUids, true); tqUpdateTbUidList(pVnode->pTq, newTbUids, true);

View File

@ -688,6 +688,8 @@ static void vnodeRestoreFinish(struct SSyncFSM *pFsm) {
} }
} while (true); } while (true);
walApplyVer(pVnode->pWal, pVnode->state.applied);
pVnode->restored = true; pVnode->restored = true;
vDebug("vgId:%d, sync restore finished", pVnode->config.vgId); vDebug("vgId:%d, sync restore finished", pVnode->config.vgId);
} }

View File

@ -34,6 +34,7 @@ extern "C" {
#include "scalar.h" #include "scalar.h"
#include "taosdef.h" #include "taosdef.h"
#include "tarray.h" #include "tarray.h"
#include "tfill.h"
#include "thash.h" #include "thash.h"
#include "tlockfree.h" #include "tlockfree.h"
#include "tmsg.h" #include "tmsg.h"
@ -798,6 +799,22 @@ typedef struct SStreamPartitionOperatorInfo {
SSDataBlock* pDelRes; SSDataBlock* pDelRes;
} SStreamPartitionOperatorInfo; } SStreamPartitionOperatorInfo;
typedef struct SStreamFillOperatorInfo {
SStreamFillSupporter* pFillSup;
SSDataBlock* pRes;
SSDataBlock* pSrcBlock;
int32_t srcRowIndex;
SSDataBlock* pPrevSrcBlock;
SSDataBlock* pSrcDelBlock;
int32_t srcDelRowIndex;
SSDataBlock* pDelRes;
SNode* pCondition;
SArray* pColMatchColInfo;
int32_t primaryTsCol;
int32_t primarySrcSlotId;
SStreamFillInfo* pFillInfo;
} SStreamFillOperatorInfo;
typedef struct STimeSliceOperatorInfo { typedef struct STimeSliceOperatorInfo {
SSDataBlock* pRes; SSDataBlock* pRes;
STimeWindow win; STimeWindow win;
@ -990,7 +1007,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi
SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode,
SExecTaskInfo* pTaskInfo); SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode,
SExecTaskInfo* pTaskInfo); SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo);
@ -1006,6 +1023,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream,
SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
SExecTaskInfo* pTaskInfo); SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode,
SExecTaskInfo* pTaskInfo);
int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx,
int32_t numOfOutput, SArray* pPseudoList); int32_t numOfOutput, SArray* pPseudoList);
@ -1094,6 +1113,7 @@ int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupI
SExecTaskInfo* pTaskInfo); SExecTaskInfo* pTaskInfo);
int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult); int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult);
int32_t saveOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize); int32_t saveOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize);
void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -23,11 +23,12 @@ extern "C" {
#include "os.h" #include "os.h"
#include "taosdef.h" #include "taosdef.h"
#include "tcommon.h" #include "tcommon.h"
#include "tsimplehash.h"
struct SSDataBlock; struct SSDataBlock;
typedef struct SFillColInfo { typedef struct SFillColInfo {
SExprInfo *pExpr; SExprInfo* pExpr;
bool notFillCol; // denote if this column needs fill operation bool notFillCol; // denote if this column needs fill operation
SVariant fillVal; SVariant fillVal;
} SFillColInfo; } SFillColInfo;
@ -66,7 +67,7 @@ typedef struct SFillInfo {
SInterval interval; SInterval interval;
SRowVal prev; SRowVal prev;
SRowVal next; SRowVal next;
SSDataBlock *pSrcBlock; SSDataBlock* pSrcBlock;
int32_t alloc; // data buffer size in rows int32_t alloc; // data buffer size in rows
SFillColInfo* pFillCol; // column info for fill operations SFillColInfo* pFillCol; // column info for fill operations
@ -74,23 +75,73 @@ typedef struct SFillInfo {
const char* id; const char* id;
} SFillInfo; } SFillInfo;
int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows); typedef struct SResultCellData {
bool isNull;
int8_t type;
int32_t bytes;
char pData[];
} SResultCellData;
typedef struct SResultRowData {
TSKEY key;
SResultCellData* pRowVal;
} SResultRowData;
typedef struct SStreamFillLinearInfo {
TSKEY nextEnd;
SArray* pDeltaVal; // double. value for Fill(linear).
SArray* pNextDeltaVal; // double. value for Fill(linear).
int64_t winIndex;
bool hasNext;
} SStreamFillLinearInfo;
typedef struct SStreamFillInfo {
TSKEY start; // startKey for fill
TSKEY end; // endKey for fill
TSKEY current; // current Key for fill
TSKEY preRowKey;
TSKEY nextRowKey;
SResultRowData* pResRow;
SStreamFillLinearInfo* pLinearInfo;
bool needFill;
int32_t type; // fill type
int32_t pos;
SArray* delRanges;
int32_t delIndex;
} SStreamFillInfo;
typedef struct SStreamFillSupporter {
int32_t type; // fill type
SInterval interval;
SResultRowData prev;
SResultRowData cur;
SResultRowData next;
SResultRowData nextNext;
SFillColInfo* pAllColInfo; // fill exprs and not fill exprs
int32_t numOfAllCols; // number of all exprs, including the tags columns
int32_t numOfFillCols;
int32_t numOfNotFillCols;
int32_t rowSize;
SSHashObj* pResMap;
bool hasDelete;
} SStreamFillSupporter;
int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows);
void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp); void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp);
void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput); void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput);
struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprInfo* pNotFillExpr, int32_t numOfNotFillCols, const struct SNodeListNode* val); struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprInfo* pNotFillExpr,
int32_t numOfNotFillCols, const struct SNodeListNode* val);
bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); bool taosFillHasMoreResults(struct SFillInfo* pFillInfo);
SFillInfo* taosCreateFillInfo(TSKEY skey, int32_t numOfFillCols, int32_t numOfNotFillCols, int32_t capacity, SFillInfo* taosCreateFillInfo(TSKEY skey, int32_t numOfFillCols, int32_t numOfNotFillCols, int32_t capacity,
SInterval* pInterval, int32_t fillType, struct SFillColInfo* pCol, int32_t slotId, SInterval* pInterval, int32_t fillType, struct SFillColInfo* pCol, int32_t slotId,
int32_t order, const char* id); int32_t order, const char* id);
void* taosDestroyFillInfo(struct SFillInfo *pFillInfo); void* taosDestroyFillInfo(struct SFillInfo* pFillInfo);
int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity); int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity);
int64_t getFillInfoStart(struct SFillInfo *pFillInfo); int64_t getFillInfoStart(struct SFillInfo* pFillInfo);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -1110,7 +1110,8 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO
} }
} }
static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, bool keep, int32_t status); static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, bool keep,
int32_t status);
void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo) { void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo) {
if (pFilterNode == NULL || pBlock->info.rows == 0) { if (pFilterNode == NULL || pBlock->info.rows == 0) {
@ -1201,7 +1202,7 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD
} }
} }
void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId) { void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId) {
// for simple group by query without interval, all the tables belong to one group result. // for simple group by query without interval, all the tables belong to one group result.
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SAggOperatorInfo* pAggInfo = pOperator->info; SAggOperatorInfo* pAggInfo = pOperator->info;
@ -1787,7 +1788,9 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf
if (pSource->localExec) { if (pSource->localExec) {
SDataBuf pBuf = {0}; SDataBuf pBuf = {0};
int32_t code = (*pTaskInfo->localFetch.fp)(pTaskInfo->localFetch.handle, pSource->schedId, pTaskInfo->id.queryId, pSource->taskId, 0, pSource->execId, &pBuf.pData, pTaskInfo->localFetch.explainRes); int32_t code =
(*pTaskInfo->localFetch.fp)(pTaskInfo->localFetch.handle, pSource->schedId, pTaskInfo->id.queryId,
pSource->taskId, 0, pSource->execId, &pBuf.pData, pTaskInfo->localFetch.explainRes);
loadRemoteDataCallback(pWrapper, &pBuf, code); loadRemoteDataCallback(pWrapper, &pBuf, code);
taosMemoryFree(pWrapper); taosMemoryFree(pWrapper);
} else { } else {
@ -1798,8 +1801,8 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf
} }
qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, pSource->execId, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId,
sourceIndex, totalSources); pSource->execId, sourceIndex, totalSources);
pMsg->header.vgId = htonl(pSource->addr.nodeId); pMsg->header.vgId = htonl(pSource->addr.nodeId);
pMsg->sId = htobe64(pSource->schedId); pMsg->sId = htobe64(pSource->schedId);
@ -1824,7 +1827,8 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf
pMsgSendInfo->fp = loadRemoteDataCallback; pMsgSendInfo->fp = loadRemoteDataCallback;
int64_t transporterId = 0; int64_t transporterId = 0;
int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epSet, &transporterId, pMsgSendInfo); int32_t code =
asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epSet, &transporterId, pMsgSendInfo);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -3356,9 +3360,7 @@ static void cleanupTableSchemaInfo(SSchemaInfo* pSchemaInfo) {
tDeleteSSchemaWrapper(pSchemaInfo->qsw); tDeleteSSchemaWrapper(pSchemaInfo->qsw);
} }
static void cleanupStreamInfo(SStreamTaskInfo* pStreamInfo) { static void cleanupStreamInfo(SStreamTaskInfo* pStreamInfo) { tDeleteSSchemaWrapper(pStreamInfo->schema); }
tDeleteSSchemaWrapper(pStreamInfo->schema);
}
static int32_t sortTableGroup(STableListInfo* pTableListInfo) { static int32_t sortTableGroup(STableListInfo* pTableListInfo) {
taosArrayClear(pTableListInfo->pGroupList); taosArrayClear(pTableListInfo->pGroupList);
@ -3539,7 +3541,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
STableScanInfo* pScanInfo = pOperator->info; STableScanInfo* pScanInfo = pOperator->info;
pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder;
} else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) {
pOperator = createExchangeOperatorInfo(pHandle ? pHandle->pMsgCb->clientRpc : NULL, (SExchangePhysiNode*)pPhyNode, pTaskInfo); pOperator = createExchangeOperatorInfo(pHandle ? pHandle->pMsgCb->clientRpc : NULL, (SExchangePhysiNode*)pPhyNode,
pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) {
STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
if (pHandle->vnode) { if (pHandle->vnode) {
@ -3729,7 +3732,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
} else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) {
pOptr = createPartitionOperatorInfo(ops[0], (SPartitionPhysiNode*)pPhyNode, pTaskInfo); pOptr = createPartitionOperatorInfo(ops[0], (SPartitionPhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION == type) {
pOptr = createStreamPartitionOperatorInfo(ops[0], (SPartitionPhysiNode*)pPhyNode, pTaskInfo); pOptr = createStreamPartitionOperatorInfo(ops[0], (SStreamPartitionPhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE == type) {
SStateWinodwPhysiNode* pStateNode = (SStateWinodwPhysiNode*)pPhyNode; SStateWinodwPhysiNode* pStateNode = (SStateWinodwPhysiNode*)pPhyNode;
pOptr = createStatewindowOperatorInfo(ops[0], pStateNode, pTaskInfo); pOptr = createStatewindowOperatorInfo(ops[0], pStateNode, pTaskInfo);
@ -3739,6 +3742,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
pOptr = createMergeJoinOperatorInfo(ops, size, (SSortMergeJoinPhysiNode*)pPhyNode, pTaskInfo); pOptr = createMergeJoinOperatorInfo(ops, size, (SSortMergeJoinPhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_FILL == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_FILL == type) {
pOptr = createFillOperatorInfo(ops[0], (SFillPhysiNode*)pPhyNode, pTaskInfo); pOptr = createFillOperatorInfo(ops[0], (SFillPhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL == type) {
pOptr = createStreamFillOperatorInfo(ops[0], (SStreamFillPhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC == type) {
pOptr = createIndefinitOutputOperatorInfo(ops[0], pPhyNode, pTaskInfo); pOptr = createIndefinitOutputOperatorInfo(ops[0], pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC == type) {

View File

@ -989,11 +989,11 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
goto _error; goto _error;
} }
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
pInfo->partitionSup.pGroupCols = extractPartitionColInfo(pPartNode->pPartitionKeys); pInfo->partitionSup.pGroupCols = extractPartitionColInfo(pPartNode->part.pPartitionKeys);
if (pPartNode->pExprs != NULL) { if (pPartNode->part.pExprs != NULL) {
int32_t num = 0; int32_t num = 0;
SExprInfo* pCalExprInfo = createExprInfo(pPartNode->pExprs, NULL, &num); SExprInfo* pCalExprInfo = createExprInfo(pPartNode->part.pExprs, NULL, &num);
code = initExprSupp(&pInfo->scalarSup, pCalExprInfo, num); code = initExprSupp(&pInfo->scalarSup, pCalExprInfo, num);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
@ -1008,7 +1008,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
} }
pInfo->partitionSup.needCalc = true; pInfo->partitionSup.needCalc = true;
SSDataBlock* pResBlock = createResDataBlock(pPartNode->node.pOutputDataBlockDesc); SSDataBlock* pResBlock = createResDataBlock(pPartNode->part.node.pOutputDataBlockDesc);
if (!pResBlock) { if (!pResBlock) {
goto _error; goto _error;
} }
@ -1022,7 +1022,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
int32_t numOfCols = 0; int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pPartNode->pTargets, NULL, &numOfCols); SExprInfo* pExprInfo = createExprInfo(pPartNode->part.pTargets, NULL, &numOfCols);
pOperator->name = "StreamPartitionOperator"; pOperator->name = "StreamPartitionOperator";
pOperator->blocking = false; pOperator->blocking = false;

View File

@ -1027,11 +1027,7 @@ static uint64_t getGroupIdByCol(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts,
return calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pPreRes, 0); return calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pPreRes, 0);
} }
static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) { static uint64_t getGroupIdByUid(SStreamScanInfo* pInfo, uint64_t uid) {
if (pInfo->partitionSup.needCalc) {
return getGroupIdByCol(pInfo, uid, ts, maxVersion);
}
SHashObj* map = pInfo->pTableScanOp->pTaskInfo->tableqinfoList.map; SHashObj* map = pInfo->pTableScanOp->pTaskInfo->tableqinfoList.map;
uint64_t* groupId = taosHashGet(map, &uid, sizeof(int64_t)); uint64_t* groupId = taosHashGet(map, &uid, sizeof(int64_t));
if (groupId) { if (groupId) {
@ -1040,6 +1036,14 @@ static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts,
return 0; return 0;
} }
static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) {
if (pInfo->partitionSup.needCalc) {
return getGroupIdByCol(pInfo, uid, ts, maxVersion);
}
return getGroupIdByUid(pInfo, uid);
}
static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t* pRowIndex) { static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t* pRowIndex) {
if ((*pRowIndex) == pBlock->info.rows) { if ((*pRowIndex) == pBlock->info.rows) {
return false; return false;
@ -1086,26 +1090,32 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_
return true; return true;
} }
static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, SInterval* pInterval, static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, uint64_t* gpIdCol, SInterval* pInterval,
SDataBlockInfo* pDataBlockInfo, int32_t* pRowIndex, bool hasGroup) { SDataBlockInfo* pDataBlockInfo, int32_t* pRowIndex, bool hasGroup) {
SResultRowInfo dumyInfo; SResultRowInfo dumyInfo;
dumyInfo.cur.pageId = -1; dumyInfo.cur.pageId = -1;
STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC); STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC);
STimeWindow endWin = win; STimeWindow endWin = win;
STimeWindow preWin = win; STimeWindow preWin = win;
uint64_t groupId = gpIdCol[*pRowIndex];
while (1) { while (1) {
if (hasGroup) { if (hasGroup) {
(*pRowIndex) += 1; (*pRowIndex) += 1;
} else { } else {
(*pRowIndex) += getNumOfRowsInTimeWindow(pDataBlockInfo, startTsCol, *pRowIndex, endWin.ekey, binarySearchForKey, while ((groupId == gpIdCol[(*pRowIndex)] && startTsCol[*pRowIndex] < endWin.ekey)) {
NULL, TSDB_ORDER_ASC); (*pRowIndex) += 1;
if ((*pRowIndex) == pDataBlockInfo->rows) {
break;
} }
}
}
do { do {
preWin = endWin; preWin = endWin;
getNextTimeWindow(pInterval, &endWin, TSDB_ORDER_ASC); getNextTimeWindow(pInterval, &endWin, TSDB_ORDER_ASC);
} while (endTsCol[(*pRowIndex) - 1] >= endWin.skey); } while (endTsCol[(*pRowIndex) - 1] >= endWin.skey);
endWin = preWin; endWin = preWin;
if (win.ekey == endWin.ekey || (*pRowIndex) == pDataBlockInfo->rows) { if (win.ekey == endWin.ekey || (*pRowIndex) == pDataBlockInfo->rows || groupId != gpIdCol[*pRowIndex]) {
win.ekey = endWin.ekey; win.ekey = endWin.ekey;
return win; return win;
} }
@ -1240,11 +1250,13 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS
int64_t version = pSrcBlock->info.version - 1; int64_t version = pSrcBlock->info.version - 1;
for (int32_t i = 0; i < rows;) { for (int32_t i = 0; i < rows;) {
uint64_t srcUid = srcUidData[i]; uint64_t srcUid = srcUidData[i];
uint64_t groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version); uint64_t groupId = srcGp[i];
uint64_t srcGpId = srcGp[i]; if (groupId == 0) {
groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version);
}
TSKEY calStartTs = srcStartTsCol[i]; TSKEY calStartTs = srcStartTsCol[i];
colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false); colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false);
STimeWindow win = getSlidingWindow(srcStartTsCol, srcEndTsCol, &pInfo->interval, &pSrcBlock->info, &i, STimeWindow win = getSlidingWindow(srcStartTsCol, srcEndTsCol, srcGp, &pInfo->interval, &pSrcBlock->info, &i,
pInfo->partitionSup.needCalc); pInfo->partitionSup.needCalc);
TSKEY calEndTs = srcStartTsCol[i - 1]; TSKEY calEndTs = srcStartTsCol[i - 1];
colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(&calEndTs), false); colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(&calEndTs), false);
@ -1253,15 +1265,6 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS
colDataAppend(pEndTsCol, pDestBlock->info.rows, (const char*)(&win.ekey), false); colDataAppend(pEndTsCol, pDestBlock->info.rows, (const char*)(&win.ekey), false);
colDataAppend(pGpCol, pDestBlock->info.rows, (const char*)(&groupId), false); colDataAppend(pGpCol, pDestBlock->info.rows, (const char*)(&groupId), false);
pDestBlock->info.rows++; pDestBlock->info.rows++;
if (pInfo->partitionSup.needCalc && srcGpId != 0 && groupId != srcGpId) {
colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false);
colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(&calEndTs), false);
colDataAppend(pDeUidCol, pDestBlock->info.rows, (const char*)(&srcUid), false);
colDataAppend(pStartTsCol, pDestBlock->info.rows, (const char*)(&win.skey), false);
colDataAppend(pEndTsCol, pDestBlock->info.rows, (const char*)(&win.ekey), false);
colDataAppend(pGpCol, pDestBlock->info.rows, (const char*)(&srcGpId), false);
pDestBlock->info.rows++;
}
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1336,7 +1339,7 @@ void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t*
static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock* pBlock, bool out) { static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock* pBlock, bool out) {
if (out) { if (out) {
blockDataCleanup(pInfo->pUpdateDataRes); blockDataCleanup(pInfo->pUpdateDataRes);
blockDataEnsureCapacity(pInfo->pUpdateDataRes, pBlock->info.rows); blockDataEnsureCapacity(pInfo->pUpdateDataRes, pBlock->info.rows * 2);
} }
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP);
@ -1357,10 +1360,12 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock
isDeletedStreamWindow(&win, pBlock->info.groupId, pInfo->pTableScanOp, &pInfo->twAggSup); isDeletedStreamWindow(&win, pBlock->info.groupId, pInfo->pTableScanOp, &pInfo->twAggSup);
if ((update || closedWin) && out) { if ((update || closedWin) && out) {
qDebug("stream update check not pass, update %d, closedWin %d", update, closedWin); qDebug("stream update check not pass, update %d, closedWin %d", update, closedWin);
uint64_t gpId = closedWin && pInfo->partitionSup.needCalc uint64_t gpId = 0;
? calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pBlock, rowId)
: 0;
appendOneRow(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.uid, &gpId); appendOneRow(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.uid, &gpId);
if (closedWin && pInfo->partitionSup.needCalc) {
gpId = calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pBlock, rowId);
appendOneRow(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.uid, &gpId);
}
} }
} }
if (out && pInfo->pUpdateDataRes->info.rows > 0) { if (out && pInfo->pUpdateDataRes->info.rows > 0) {
@ -1537,6 +1542,30 @@ static int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, S
return 0; return 0;
} }
// for partition by tag
static void setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) {
SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
TSKEY* startTsCol = (TSKEY*)pStartTsCol->pData;
SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
uint64_t* gpCol = (uint64_t*)pGpCol->pData;
SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
uint64_t* uidCol = (uint64_t*)pUidCol->pData;
int32_t rows = pBlock->info.rows;
if (!pInfo->partitionSup.needCalc) {
for (int32_t i = 0; i < rows; i++) {
uint64_t groupId = getGroupIdByUid(pInfo, uidCol[i]);
colDataAppend(pGpCol, i, (const char*)&groupId, false);
}
} else {
// SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, uidCol[i], startTsCol, ts, maxVersion);
// if (!pPreRes || pPreRes->info.rows == 0) {
// return 0;
// }
// ASSERT(pPreRes->info.rows == 1);
// return calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pPreRes, 0);
}
}
static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
// NOTE: this operator does never check if current status is done or not // NOTE: this operator does never check if current status is done or not
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -1633,7 +1662,8 @@ FETCH_NEXT_BLOCK:
} else { } else {
pDelBlock = pBlock; pDelBlock = pBlock;
} }
printDataBlock(pBlock, "stream scan delete recv filtered"); setBlockGroupIdByUid(pInfo, pDelBlock);
printDataBlock(pDelBlock, "stream scan delete recv filtered");
if (!isIntervalWindow(pInfo) && !isSessionWindow(pInfo) && !isStateWindow(pInfo)) { if (!isIntervalWindow(pInfo) && !isSessionWindow(pInfo) && !isStateWindow(pInfo)) {
generateDeleteResultBlock(pInfo, pDelBlock, pInfo->pDeleteDataRes); generateDeleteResultBlock(pInfo, pDelBlock, pInfo->pDeleteDataRes);
pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT; pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT;

File diff suppressed because it is too large Load Diff

View File

@ -271,6 +271,10 @@ static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t o
tw->ekey -= 1; tw->ekey -= 1;
} }
void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) {
getNextTimeWindow(pInterval, pInterval->precision, order, tw);
}
void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs,
int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) { int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) {
SqlFunctionCtx* pCtx = pSup->pCtx; SqlFunctionCtx* pCtx = pSup->pCtx;
@ -2095,12 +2099,17 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
bool hasInterp = true; bool hasInterp = true;
for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) { for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
SExprInfo* pExprInfo = &pExprSup->pExprInfo[j]; SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
int32_t dstSlot = pExprInfo->base.resSchema.slotId;
// SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); int32_t dstSlot = pExprInfo->base.resSchema.slotId;
SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);
if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) {
colDataAppend(pDst, rows, (char*)&pSliceInfo->current, false);
continue;
}
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
// SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
switch (pSliceInfo->fillType) { switch (pSliceInfo->fillType) {
case TSDB_FILL_NULL: { case TSDB_FILL_NULL: {
colDataAppendNULL(pDst, rows); colDataAppendNULL(pDst, rows);
@ -2346,12 +2355,16 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
if (ts == pSliceInfo->current) { if (ts == pSliceInfo->current) {
for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j];
int32_t dstSlot = pExprInfo->base.resSchema.slotId;
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); int32_t dstSlot = pExprInfo->base.resSchema.slotId;
SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);
if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) {
colDataAppend(pDst, pResBlock->info.rows, (char *)&pSliceInfo->current, false);
} else {
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
if (colDataIsNull_s(pSrc, i)) { if (colDataIsNull_s(pSrc, i)) {
colDataAppendNULL(pDst, pResBlock->info.rows); colDataAppendNULL(pDst, pResBlock->info.rows);
continue; continue;
@ -2360,6 +2373,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
char* v = colDataGetData(pSrc, i); char* v = colDataGetData(pSrc, i);
colDataAppend(pDst, pResBlock->info.rows, v, false); colDataAppend(pDst, pResBlock->info.rows, v, false);
} }
}
pResBlock->info.rows += 1; pResBlock->info.rows += 1;
doKeepPrevRows(pSliceInfo, pBlock, i); doKeepPrevRows(pSliceInfo, pBlock, i);
@ -2478,15 +2492,25 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) { if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) {
for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j];
int32_t dstSlot = pExprInfo->base.resSchema.slotId;
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); int32_t dstSlot = pExprInfo->base.resSchema.slotId;
SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);
if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) {
colDataAppend(pDst, pResBlock->info.rows, (char *)&pSliceInfo->current, false);
} else {
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
if (colDataIsNull_s(pSrc, i)) {
colDataAppendNULL(pDst, pResBlock->info.rows);
continue;
}
char* v = colDataGetData(pSrc, i); char* v = colDataGetData(pSrc, i);
colDataAppend(pDst, pResBlock->info.rows, v, false); colDataAppend(pDst, pResBlock->info.rows, v, false);
} }
}
pResBlock->info.rows += 1; pResBlock->info.rows += 1;
doKeepPrevRows(pSliceInfo, pBlock, i); doKeepPrevRows(pSliceInfo, pBlock, i);
@ -3146,8 +3170,12 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
} }
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
if (pInfo->binfo.pRes->info.rows == 0) { if (pInfo->binfo.pRes->info.rows != 0) {
pOperator->status = OP_EXEC_DONE; printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
return pInfo->binfo.pRes;
}
doSetOperatorCompleted(pOperator);
if (!IS_FINAL_OP(pInfo)) { if (!IS_FINAL_OP(pInfo)) {
clearFunctionContext(&pOperator->exprSupp); clearFunctionContext(&pOperator->exprSupp);
// semi interval operator clear disk buffer // semi interval operator clear disk buffer
@ -3157,9 +3185,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
} }
return NULL; return NULL;
}
printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
return pInfo->binfo.pRes;
} else { } else {
if (!IS_FINAL_OP(pInfo)) { if (!IS_FINAL_OP(pInfo)) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
@ -3316,7 +3341,13 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
return pInfo->pPullDataRes; return pInfo->pPullDataRes;
} }
// we should send result first. doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
if (pInfo->pDelRes->info.rows != 0) {
// process the rest of the data
printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
return pInfo->pDelRes;
}
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
if (pInfo->binfo.pRes->info.rows != 0) { if (pInfo->binfo.pRes->info.rows != 0) {
printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
@ -3330,13 +3361,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
// process the rest of the data // process the rest of the data
return pInfo->pUpdateRes; return pInfo->pUpdateRes;
} }
doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
if (pInfo->pDelRes->info.rows != 0) {
// process the rest of the data
printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
return pInfo->pDelRes;
}
return NULL; return NULL;
} }
@ -5744,19 +5768,18 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
if (pOperator->status == OP_RES_TO_RETURN) { if (pOperator->status == OP_RES_TO_RETURN) {
doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
if (pInfo->pDelRes->info.rows > 0) { if (pInfo->pDelRes->info.rows > 0) {
printDataBlock(pInfo->pDelRes, "single interval"); printDataBlock(pInfo->pDelRes, "single interval delete");
return pInfo->pDelRes; return pInfo->pDelRes;
} }
doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo); doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo);
// doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pInfo->binfo.pRes->info.rows > 0) {
if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
pOperator->status = OP_EXEC_DONE;
qDebug("===stream===single interval is done");
freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
}
printDataBlock(pInfo->binfo.pRes, "single interval"); printDataBlock(pInfo->binfo.pRes, "single interval");
return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; return pInfo->binfo.pRes;
}
doSetOperatorCompleted(pOperator);
return NULL;
} }
SOperatorInfo* downstream = pOperator->pDownstream[0]; SOperatorInfo* downstream = pOperator->pDownstream[0];
@ -5823,24 +5846,24 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
} }
taosArraySort(pUpdated, resultrowComparAsc); taosArraySort(pUpdated, resultrowComparAsc);
// new disc buf
// finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated,
// pSup->rowEntryInfoOffset);
initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
removeDeleteResults(pUpdatedMap, pInfo->pDelWins); removeDeleteResults(pUpdatedMap, pInfo->pDelWins);
taosHashCleanup(pUpdatedMap); taosHashCleanup(pUpdatedMap);
doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
if (pInfo->pDelRes->info.rows > 0) { if (pInfo->pDelRes->info.rows > 0) {
printDataBlock(pInfo->pDelRes, "single interval"); printDataBlock(pInfo->pDelRes, "single interval delete");
return pInfo->pDelRes; return pInfo->pDelRes;
} }
// doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
// new disc buf
doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo); doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo);
if (pInfo->binfo.pRes->info.rows > 0) {
printDataBlock(pInfo->binfo.pRes, "single interval"); printDataBlock(pInfo->binfo.pRes, "single interval");
return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; return pInfo->binfo.pRes;
}
return NULL;
} }
void destroyStreamIntervalOperatorInfo(void* param) { void destroyStreamIntervalOperatorInfo(void* param) {

View File

@ -50,6 +50,7 @@ extern "C" {
#define FUNC_MGT_KEEP_ORDER_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(21) #define FUNC_MGT_KEEP_ORDER_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(21)
#define FUNC_MGT_CUMULATIVE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(22) #define FUNC_MGT_CUMULATIVE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(22)
#define FUNC_MGT_FORBID_STABLE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(23) #define FUNC_MGT_FORBID_STABLE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(23)
#define FUNC_MGT_INTERP_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(24)
#define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0) #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0)

View File

@ -3146,6 +3146,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.classification = FUNC_MGT_SYSTEM_INFO_FUNC | FUNC_MGT_SCALAR_FUNC, .classification = FUNC_MGT_SYSTEM_INFO_FUNC | FUNC_MGT_SCALAR_FUNC,
.translateFunc = translateUserFunc, .translateFunc = translateUserFunc,
}, },
{
.name = "_irowts",
.type = FUNCTION_TYPE_IROWTS,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_INTERP_PC_FUNC,
.translateFunc = translateTimePseudoColumn,
.getEnvFunc = getTimePseudoFuncEnv,
.initFunc = NULL,
.sprocessFunc = NULL,
.finalizeFunc = NULL
},
}; };
// clang-format on // clang-format on

View File

@ -224,6 +224,8 @@ bool fmIsInterpFunc(int32_t funcId) {
return FUNCTION_TYPE_INTERP == funcMgtBuiltins[funcId].type; return FUNCTION_TYPE_INTERP == funcMgtBuiltins[funcId].type;
} }
bool fmIsInterpPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_INTERP_PC_FUNC); }
bool fmIsLastRowFunc(int32_t funcId) { bool fmIsLastRowFunc(int32_t funcId) {
if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
return false; return false;

View File

@ -302,6 +302,7 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR
char* p = taosMemoryCalloc(1, strlen(c->colVal) + 1); char* p = taosMemoryCalloc(1, strlen(c->colVal) + 1);
memcpy(p, c->colVal, strlen(c->colVal)); memcpy(p, c->colVal, strlen(c->colVal));
cond = cmpFn(p + skip, term->colVal, dType); cond = cmpFn(p + skip, term->colVal, dType);
taosMemoryFree(p);
} }
} }
if (cond == MATCH) { if (cond == MATCH) {

View File

@ -69,6 +69,8 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of
int32_t blkOffset = offset % kBlockSize; int32_t blkOffset = offset % kBlockSize;
int32_t blkLeft = kBlockSize - blkOffset; int32_t blkLeft = kBlockSize - blkOffset;
if (offset >= ctx->file.size) return 0;
do { do {
char key[128] = {0}; char key[128] = {0};
idxGenLRUKey(key, ctx->file.buf, blkId); idxGenLRUKey(key, ctx->file.buf, blkId);
@ -79,6 +81,15 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of
nread = TMIN(blkLeft, len); nread = TMIN(blkLeft, len);
memcpy(buf + total, blk->buf + blkOffset, nread); memcpy(buf + total, blk->buf + blkOffset, nread);
taosLRUCacheRelease(ctx->lru, h, false); taosLRUCacheRelease(ctx->lru, h, false);
} else {
int32_t left = ctx->file.size - offset;
if (left < kBlockSize) {
nread = TMIN(left, len);
int32_t bytes = taosPReadFile(ctx->file.pFile, buf + total, nread, offset);
assert(bytes == nread);
total += bytes;
return total;
} else { } else {
int32_t cacheMemSize = sizeof(SDataBlock) + kBlockSize; int32_t cacheMemSize = sizeof(SDataBlock) + kBlockSize;
@ -100,6 +111,7 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of
return -1; return -1;
} }
} }
}
total += nread; total += nread;
len -= nread; len -= nread;
offset += nread; offset += nread;
@ -146,9 +158,7 @@ IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int
} else { } else {
ctx->file.pFile = taosOpenFile(path, TD_FILE_READ); ctx->file.pFile = taosOpenFile(path, TD_FILE_READ);
int64_t size = 0;
taosFStatFile(ctx->file.pFile, &ctx->file.size, NULL); taosFStatFile(ctx->file.pFile, &ctx->file.size, NULL);
ctx->file.size = (int)size;
#ifdef USE_MMAP #ifdef USE_MMAP
ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size); ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size);
#endif #endif

View File

@ -172,9 +172,9 @@ TEST_F(JsonEnv, testWriteMillonData) {
{ {
std::string colName("voltagefdadfa"); std::string colName("voltagefdadfa");
std::string colVal("abxxxxxxxxxxxx"); std::string colVal("abxxxxxxxxxxxx");
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10000; i++) {
colVal[i % colVal.size()] = '0' + i % 128; colVal[i % colVal.size()] = '0' + i % 128;
for (size_t i = 0; i < 100; i++) { for (size_t i = 0; i < 10; i++) {
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size()); colVal.c_str(), colVal.size());

View File

@ -381,6 +381,8 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) {
COPY_SCALAR_FIELD(igExpired); COPY_SCALAR_FIELD(igExpired);
CLONE_NODE_LIST_FIELD(pGroupTags); CLONE_NODE_LIST_FIELD(pGroupTags);
COPY_SCALAR_FIELD(groupSort); COPY_SCALAR_FIELD(groupSort);
CLONE_NODE_LIST_FIELD(pTags);
CLONE_NODE_FIELD(pSubtable);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -488,6 +490,8 @@ static int32_t logicSortCopy(const SSortLogicNode* pSrc, SSortLogicNode* pDst) {
static int32_t logicPartitionCopy(const SPartitionLogicNode* pSrc, SPartitionLogicNode* pDst) { static int32_t logicPartitionCopy(const SPartitionLogicNode* pSrc, SPartitionLogicNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
CLONE_NODE_LIST_FIELD(pPartitionKeys); CLONE_NODE_LIST_FIELD(pPartitionKeys);
CLONE_NODE_LIST_FIELD(pTags);
CLONE_NODE_FIELD(pSubtable);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

View File

@ -254,6 +254,7 @@ const char* nodesNodeName(ENodeType type) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
return "PhysiStreamSemiInterval"; return "PhysiStreamSemiInterval";
case QUERY_NODE_PHYSICAL_PLAN_FILL: case QUERY_NODE_PHYSICAL_PLAN_FILL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
return "PhysiFill"; return "PhysiFill";
case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION:
return "PhysiSessionWindow"; return "PhysiSessionWindow";
@ -1538,6 +1539,8 @@ static const char* jkTableScanPhysiPlanWatermark = "Watermark";
static const char* jkTableScanPhysiPlanIgnoreExpired = "IgnoreExpired"; static const char* jkTableScanPhysiPlanIgnoreExpired = "IgnoreExpired";
static const char* jkTableScanPhysiPlanGroupTags = "GroupTags"; static const char* jkTableScanPhysiPlanGroupTags = "GroupTags";
static const char* jkTableScanPhysiPlanGroupSort = "GroupSort"; static const char* jkTableScanPhysiPlanGroupSort = "GroupSort";
static const char* jkTableScanPhysiPlanTags = "Tags";
static const char* jkTableScanPhysiPlanSubtable = "Subtable";
static const char* jkTableScanPhysiPlanAssignBlockUid = "AssignBlockUid"; static const char* jkTableScanPhysiPlanAssignBlockUid = "AssignBlockUid";
static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
@ -1595,6 +1598,12 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanGroupSort, pNode->groupSort); code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanGroupSort, pNode->groupSort);
} }
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkTableScanPhysiPlanTags, pNode->pTags);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkTableScanPhysiPlanSubtable, nodeToJson, pNode->pSubtable);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanAssignBlockUid, pNode->assignBlockUid); code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanAssignBlockUid, pNode->assignBlockUid);
} }
@ -1657,6 +1666,12 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanGroupSort, &pNode->groupSort); code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanGroupSort, &pNode->groupSort);
} }
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkTableScanPhysiPlanTags, &pNode->pTags);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkTableScanPhysiPlanSubtable, &pNode->pSubtable);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanAssignBlockUid, &pNode->assignBlockUid); code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanAssignBlockUid, &pNode->assignBlockUid);
} }
@ -2270,6 +2285,37 @@ static int32_t jsonToPhysiPartitionNode(const SJson* pJson, void* pObj) {
return code; return code;
} }
static const char* jkStreamPartitionPhysiPlanTags = "Tags";
static const char* jkStreamPartitionPhysiPlanSubtable = "Subtable";
static int32_t physiStreamPartitionNodeToJson(const void* pObj, SJson* pJson) {
const SStreamPartitionPhysiNode* pNode = (const SStreamPartitionPhysiNode*)pObj;
int32_t code = physiPartitionNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkStreamPartitionPhysiPlanTags, pNode->pTags);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkStreamPartitionPhysiPlanSubtable, nodeToJson, pNode->pSubtable);
}
return code;
}
static int32_t jsonToPhysiStreamPartitionNode(const SJson* pJson, void* pObj) {
SStreamPartitionPhysiNode* pNode = (SStreamPartitionPhysiNode*)pObj;
int32_t code = jsonToPhysiPartitionNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkStreamPartitionPhysiPlanTags, &pNode->pTags);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkStreamPartitionPhysiPlanSubtable, &pNode->pSubtable);
}
return code;
}
static const char* jkIndefRowsFuncPhysiPlanExprs = "Exprs"; static const char* jkIndefRowsFuncPhysiPlanExprs = "Exprs";
static const char* jkIndefRowsFuncPhysiPlanFuncs = "Funcs"; static const char* jkIndefRowsFuncPhysiPlanFuncs = "Funcs";
@ -4109,6 +4155,8 @@ static const char* jkSelectStmtProjections = "Projections";
static const char* jkSelectStmtFrom = "From"; static const char* jkSelectStmtFrom = "From";
static const char* jkSelectStmtWhere = "Where"; static const char* jkSelectStmtWhere = "Where";
static const char* jkSelectStmtPartitionBy = "PartitionBy"; static const char* jkSelectStmtPartitionBy = "PartitionBy";
static const char* jkSelectStmtTags = "Tags";
static const char* jkSelectStmtSubtable = "Subtable";
static const char* jkSelectStmtWindow = "Window"; static const char* jkSelectStmtWindow = "Window";
static const char* jkSelectStmtGroupBy = "GroupBy"; static const char* jkSelectStmtGroupBy = "GroupBy";
static const char* jkSelectStmtHaving = "Having"; static const char* jkSelectStmtHaving = "Having";
@ -4134,6 +4182,12 @@ static int32_t selectStmtToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkSelectStmtPartitionBy, pNode->pPartitionByList); code = nodeListToJson(pJson, jkSelectStmtPartitionBy, pNode->pPartitionByList);
} }
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkSelectStmtTags, pNode->pTags);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkSelectStmtSubtable, nodeToJson, pNode->pSubtable);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkSelectStmtWindow, nodeToJson, pNode->pWindow); code = tjsonAddObject(pJson, jkSelectStmtWindow, nodeToJson, pNode->pWindow);
} }
@ -4178,6 +4232,12 @@ static int32_t jsonToSelectStmt(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkSelectStmtPartitionBy, &pNode->pPartitionByList); code = jsonToNodeList(pJson, jkSelectStmtPartitionBy, &pNode->pPartitionByList);
} }
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkSelectStmtTags, &pNode->pTags);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSelectStmtSubtable, &pNode->pSubtable);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSelectStmtWindow, &pNode->pWindow); code = jsonToNodeObject(pJson, jkSelectStmtWindow, &pNode->pWindow);
} }
@ -4576,6 +4636,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
return physiIntervalNodeToJson(pObj, pJson); return physiIntervalNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_FILL: case QUERY_NODE_PHYSICAL_PLAN_FILL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
return physiFillNodeToJson(pObj, pJson); return physiFillNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION:
@ -4586,8 +4647,9 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE:
return physiStateWindowNodeToJson(pObj, pJson); return physiStateWindowNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_PARTITION: case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
return physiPartitionNodeToJson(pObj, pJson); return physiPartitionNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
return physiStreamPartitionNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC:
return physiIndefRowsFuncNodeToJson(pObj, pJson); return physiIndefRowsFuncNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC: case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC:
@ -4728,6 +4790,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
return jsonToPhysiIntervalNode(pJson, pObj); return jsonToPhysiIntervalNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_FILL: case QUERY_NODE_PHYSICAL_PLAN_FILL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
return jsonToPhysiFillNode(pJson, pObj); return jsonToPhysiFillNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION:
@ -4738,8 +4801,9 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE:
return jsonToPhysiStateWindowNode(pJson, pObj); return jsonToPhysiStateWindowNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_PARTITION: case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
return jsonToPhysiPartitionNode(pJson, pObj); return jsonToPhysiPartitionNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
return jsonToPhysiStreamPartitionNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC:
return jsonToPhysiIndefRowsFuncNode(pJson, pObj); return jsonToPhysiIndefRowsFuncNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC: case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC:

View File

@ -1993,7 +1993,9 @@ enum {
PHY_TABLE_SCAN_CODE_SCAN = 1, PHY_TABLE_SCAN_CODE_SCAN = 1,
PHY_TABLE_SCAN_CODE_INLINE_ATTRS, PHY_TABLE_SCAN_CODE_INLINE_ATTRS,
PHY_TABLE_SCAN_CODE_DYN_SCAN_FUNCS, PHY_TABLE_SCAN_CODE_DYN_SCAN_FUNCS,
PHY_TABLE_SCAN_CODE_GROUP_TAGS PHY_TABLE_SCAN_CODE_GROUP_TAGS,
PHY_TABLE_SCAN_CODE_TAGS,
PHY_TABLE_SCAN_CODE_SUBTABLE
}; };
static int32_t physiTableScanNodeInlineToMsg(const void* pObj, STlvEncoder* pEncoder) { static int32_t physiTableScanNodeInlineToMsg(const void* pObj, STlvEncoder* pEncoder) {
@ -2062,6 +2064,12 @@ static int32_t physiTableScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder)
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_TABLE_SCAN_CODE_GROUP_TAGS, nodeListToMsg, pNode->pGroupTags); code = tlvEncodeObj(pEncoder, PHY_TABLE_SCAN_CODE_GROUP_TAGS, nodeListToMsg, pNode->pGroupTags);
} }
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_TABLE_SCAN_CODE_TAGS, nodeListToMsg, pNode->pTags);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_TABLE_SCAN_CODE_SUBTABLE, nodeToMsg, pNode->pSubtable);
}
return code; return code;
} }
@ -2138,6 +2146,12 @@ static int32_t msgToPhysiTableScanNode(STlvDecoder* pDecoder, void* pObj) {
case PHY_TABLE_SCAN_CODE_GROUP_TAGS: case PHY_TABLE_SCAN_CODE_GROUP_TAGS:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pGroupTags); code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pGroupTags);
break; break;
case PHY_TABLE_SCAN_CODE_TAGS:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTags);
break;
case PHY_TABLE_SCAN_CODE_SUBTABLE:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pSubtable);
break;
default: default:
break; break;
} }
@ -2914,6 +2928,46 @@ static int32_t msgToPhysiPartitionNode(STlvDecoder* pDecoder, void* pObj) {
return code; return code;
} }
enum { PHY_STREAM_PARTITION_CODE_BASE_NODE = 1, PHY_STREAM_PARTITION_CODE_TAGS, PHY_STREAM_PARTITION_CODE_SUBTABLE };
static int32_t physiStreamPartitionNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
const SStreamPartitionPhysiNode* pNode = (const SStreamPartitionPhysiNode*)pObj;
int32_t code = tlvEncodeObj(pEncoder, PHY_STREAM_PARTITION_CODE_BASE_NODE, physiPartitionNodeToMsg, &pNode->part);
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_STREAM_PARTITION_CODE_TAGS, nodeListToMsg, pNode->pTags);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_STREAM_PARTITION_CODE_SUBTABLE, nodeToMsg, pNode->pSubtable);
}
return code;
}
static int32_t msgToPhysiStreamPartitionNode(STlvDecoder* pDecoder, void* pObj) {
SStreamPartitionPhysiNode* pNode = (SStreamPartitionPhysiNode*)pObj;
int32_t code = TSDB_CODE_SUCCESS;
STlv* pTlv = NULL;
tlvForEach(pDecoder, pTlv, code) {
switch (pTlv->type) {
case PHY_STREAM_PARTITION_CODE_BASE_NODE:
code = tlvDecodeObjFromTlv(pTlv, msgToPhysiPartitionNode, &pNode->part);
break;
case PHY_STREAM_PARTITION_CODE_TAGS:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTags);
break;
case PHY_STREAM_PARTITION_CODE_SUBTABLE:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pSubtable);
break;
default:
break;
}
}
return code;
}
enum { PHY_INDEF_ROWS_FUNC_CODE_BASE_NODE = 1, PHY_INDEF_ROWS_FUNC_CODE_EXPRS, PHY_INDEF_ROWS_FUNC_CODE_FUNCS }; enum { PHY_INDEF_ROWS_FUNC_CODE_BASE_NODE = 1, PHY_INDEF_ROWS_FUNC_CODE_EXPRS, PHY_INDEF_ROWS_FUNC_CODE_FUNCS };
static int32_t physiIndefRowsFuncNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { static int32_t physiIndefRowsFuncNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
@ -3579,6 +3633,7 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
code = physiIntervalNodeToMsg(pObj, pEncoder); code = physiIntervalNodeToMsg(pObj, pEncoder);
break; break;
case QUERY_NODE_PHYSICAL_PLAN_FILL: case QUERY_NODE_PHYSICAL_PLAN_FILL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
code = physiFillNodeToMsg(pObj, pEncoder); code = physiFillNodeToMsg(pObj, pEncoder);
break; break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION:
@ -3592,9 +3647,11 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
code = physiStateWindowNodeToMsg(pObj, pEncoder); code = physiStateWindowNodeToMsg(pObj, pEncoder);
break; break;
case QUERY_NODE_PHYSICAL_PLAN_PARTITION: case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
code = physiPartitionNodeToMsg(pObj, pEncoder); code = physiPartitionNodeToMsg(pObj, pEncoder);
break; break;
case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
code = physiStreamPartitionNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC:
code = physiIndefRowsFuncNodeToMsg(pObj, pEncoder); code = physiIndefRowsFuncNodeToMsg(pObj, pEncoder);
break; break;
@ -3714,6 +3771,7 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) {
code = msgToPhysiIntervalNode(pDecoder, pObj); code = msgToPhysiIntervalNode(pDecoder, pObj);
break; break;
case QUERY_NODE_PHYSICAL_PLAN_FILL: case QUERY_NODE_PHYSICAL_PLAN_FILL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
code = msgToPhysiFillNode(pDecoder, pObj); code = msgToPhysiFillNode(pDecoder, pObj);
break; break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION:
@ -3727,9 +3785,11 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) {
code = msgToPhysiStateWindowNode(pDecoder, pObj); code = msgToPhysiStateWindowNode(pDecoder, pObj);
break; break;
case QUERY_NODE_PHYSICAL_PLAN_PARTITION: case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
code = msgToPhysiPartitionNode(pDecoder, pObj); code = msgToPhysiPartitionNode(pDecoder, pObj);
break; break;
case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
code = msgToPhysiStreamPartitionNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC:
code = msgToPhysiIndefRowsFuncNode(pDecoder, pObj); code = msgToPhysiIndefRowsFuncNode(pDecoder, pObj);
break; break;

View File

@ -378,6 +378,8 @@ void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker wa
nodesWalkExpr(pSelect->pWhere, walker, pContext); nodesWalkExpr(pSelect->pWhere, walker, pContext);
case SQL_CLAUSE_WHERE: case SQL_CLAUSE_WHERE:
nodesWalkExprs(pSelect->pPartitionByList, walker, pContext); nodesWalkExprs(pSelect->pPartitionByList, walker, pContext);
nodesWalkExprs(pSelect->pTags, walker, pContext);
nodesWalkExpr(pSelect->pSubtable, walker, pContext);
case SQL_CLAUSE_PARTITION_BY: case SQL_CLAUSE_PARTITION_BY:
nodesWalkExpr(pSelect->pWindow, walker, pContext); nodesWalkExpr(pSelect->pWindow, walker, pContext);
case SQL_CLAUSE_WINDOW: case SQL_CLAUSE_WINDOW:
@ -412,6 +414,8 @@ void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewrit
nodesRewriteExpr(&(pSelect->pWhere), rewriter, pContext); nodesRewriteExpr(&(pSelect->pWhere), rewriter, pContext);
case SQL_CLAUSE_WHERE: case SQL_CLAUSE_WHERE:
nodesRewriteExprs(pSelect->pPartitionByList, rewriter, pContext); nodesRewriteExprs(pSelect->pPartitionByList, rewriter, pContext);
nodesRewriteExprs(pSelect->pTags, rewriter, pContext);
nodesRewriteExpr(&(pSelect->pSubtable), rewriter, pContext);
case SQL_CLAUSE_PARTITION_BY: case SQL_CLAUSE_PARTITION_BY:
nodesRewriteExpr(&(pSelect->pWindow), rewriter, pContext); nodesRewriteExpr(&(pSelect->pWindow), rewriter, pContext);
case SQL_CLAUSE_WINDOW: case SQL_CLAUSE_WINDOW:

View File

@ -511,6 +511,7 @@ SNode* nodesMakeNode(ENodeType type) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
return makeNode(type, sizeof(SStreamSemiIntervalPhysiNode)); return makeNode(type, sizeof(SStreamSemiIntervalPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_FILL: case QUERY_NODE_PHYSICAL_PLAN_FILL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
return makeNode(type, sizeof(SFillPhysiNode)); return makeNode(type, sizeof(SFillPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION:
return makeNode(type, sizeof(SSessionWinodwPhysiNode)); return makeNode(type, sizeof(SSessionWinodwPhysiNode));
@ -772,6 +773,8 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(pStmt->pFromTable); nodesDestroyNode(pStmt->pFromTable);
nodesDestroyNode(pStmt->pWhere); nodesDestroyNode(pStmt->pWhere);
nodesDestroyList(pStmt->pPartitionByList); nodesDestroyList(pStmt->pPartitionByList);
nodesDestroyList(pStmt->pTags);
nodesDestroyNode(pStmt->pSubtable);
nodesDestroyNode(pStmt->pWindow); nodesDestroyNode(pStmt->pWindow);
nodesDestroyList(pStmt->pGroupByList); nodesDestroyList(pStmt->pGroupByList);
nodesDestroyNode(pStmt->pHaving); nodesDestroyNode(pStmt->pHaving);
@ -1154,7 +1157,8 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode);
break; break;
case QUERY_NODE_PHYSICAL_PLAN_FILL: { case QUERY_NODE_PHYSICAL_PLAN_FILL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: {
SFillPhysiNode* pPhyNode = (SFillPhysiNode*)pNode; SFillPhysiNode* pPhyNode = (SFillPhysiNode*)pNode;
destroyPhysiNode((SPhysiNode*)pPhyNode); destroyPhysiNode((SPhysiNode*)pPhyNode);
nodesDestroyList(pPhyNode->pFillExprs); nodesDestroyList(pPhyNode->pFillExprs);

View File

@ -147,7 +147,7 @@ SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, STok
SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pDbName); SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pDbName);
SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pOptions); SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pOptions);
SNode* createFlushDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName); SNode* createFlushDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName);
SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName); SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, int32_t maxSpeed);
SNode* createDefaultTableOptions(SAstCreateContext* pCxt); SNode* createDefaultTableOptions(SAstCreateContext* pCxt);
SNode* createAlterTableOptions(SAstCreateContext* pCxt); SNode* createAlterTableOptions(SAstCreateContext* pCxt);
SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, void* pVal); SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, void* pVal);
@ -212,7 +212,7 @@ SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool
SNode* createDropFunctionStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pFuncName); SNode* createDropFunctionStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pFuncName);
SNode* createStreamOptions(SAstCreateContext* pCxt); SNode* createStreamOptions(SAstCreateContext* pCxt);
SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pStreamName, SNode* pRealTable, SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pStreamName, SNode* pRealTable,
SNode* pOptions, SNode* pQuery); SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery);
SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pStreamName); SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pStreamName);
SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId); SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId);
SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId); SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId);

View File

@ -159,7 +159,7 @@ cmd ::= DROP DATABASE exists_opt(A) db_name(B).
cmd ::= USE db_name(A). { pCxt->pRootNode = createUseDatabaseStmt(pCxt, &A); } cmd ::= USE db_name(A). { pCxt->pRootNode = createUseDatabaseStmt(pCxt, &A); }
cmd ::= ALTER DATABASE db_name(A) alter_db_options(B). { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &A, B); } cmd ::= ALTER DATABASE db_name(A) alter_db_options(B). { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &A, B); }
cmd ::= FLUSH DATABASE db_name(A). { pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &A); } cmd ::= FLUSH DATABASE db_name(A). { pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &A); }
cmd ::= TRIM DATABASE db_name(A). { pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &A); } cmd ::= TRIM DATABASE db_name(A) speed_opt(B). { pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &A, B); }
%type not_exists_opt { bool } %type not_exists_opt { bool }
%destructor not_exists_opt { } %destructor not_exists_opt { }
@ -246,6 +246,11 @@ retention_list(A) ::= retention_list(B) NK_COMMA retention(C).
retention(A) ::= NK_VARIABLE(B) NK_COLON NK_VARIABLE(C). { A = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &B), createDurationValueNode(pCxt, &C)); } retention(A) ::= NK_VARIABLE(B) NK_COLON NK_VARIABLE(C). { A = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &B), createDurationValueNode(pCxt, &C)); }
%type speed_opt { int32_t }
%destructor speed_opt { }
speed_opt(A) ::= . { A = 0; }
speed_opt(A) ::= MAX_SPEED NK_INTEGER(B). { A = taosStr2Int32(B.z, NULL, 10); }
/************************************************ create/drop table/stable ********************************************/ /************************************************ create/drop table/stable ********************************************/
cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B) cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B)
NK_LP column_def_list(C) NK_RP tags_def_opt(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E); } NK_LP column_def_list(C) NK_RP tags_def_opt(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E); }
@ -501,8 +506,8 @@ bufsize_opt(A) ::= .
bufsize_opt(A) ::= BUFSIZE NK_INTEGER(B). { A = taosStr2Int32(B.z, NULL, 10); } bufsize_opt(A) ::= BUFSIZE NK_INTEGER(B). { A = taosStr2Int32(B.z, NULL, 10); }
/************************************************ create/drop stream **************************************************/ /************************************************ create/drop stream **************************************************/
cmd ::= CREATE STREAM not_exists_opt(E) stream_name(A) cmd ::= CREATE STREAM not_exists_opt(E) stream_name(A) stream_options(B) INTO
stream_options(B) INTO full_table_name(C) AS query_or_subquery(D). { pCxt->pRootNode = createCreateStreamStmt(pCxt, E, &A, C, B, D); } full_table_name(C) tags_def_opt(F) subtable_opt(G) AS query_or_subquery(D). { pCxt->pRootNode = createCreateStreamStmt(pCxt, E, &A, C, B, F, G, D); }
cmd ::= DROP STREAM exists_opt(A) stream_name(B). { pCxt->pRootNode = createDropStreamStmt(pCxt, A, &B); } cmd ::= DROP STREAM exists_opt(A) stream_name(B). { pCxt->pRootNode = createDropStreamStmt(pCxt, A, &B); }
stream_options(A) ::= . { A = createStreamOptions(pCxt); } stream_options(A) ::= . { A = createStreamOptions(pCxt); }
@ -512,6 +517,9 @@ stream_options(A) ::= stream_options(B) TRIGGER MAX_DELAY duration_literal(C).
stream_options(A) ::= stream_options(B) WATERMARK duration_literal(C). { ((SStreamOptions*)B)->pWatermark = releaseRawExprNode(pCxt, C); A = B; } stream_options(A) ::= stream_options(B) WATERMARK duration_literal(C). { ((SStreamOptions*)B)->pWatermark = releaseRawExprNode(pCxt, C); A = B; }
stream_options(A) ::= stream_options(B) IGNORE EXPIRED NK_INTEGER(C). { ((SStreamOptions*)B)->ignoreExpired = taosStr2Int8(C.z, NULL, 10); A = B; } stream_options(A) ::= stream_options(B) IGNORE EXPIRED NK_INTEGER(C). { ((SStreamOptions*)B)->ignoreExpired = taosStr2Int8(C.z, NULL, 10); A = B; }
subtable_opt(A) ::= . { A = NULL; }
subtable_opt(A) ::= SUBTABLE NK_LP expression(B) NK_RP. { A = releaseRawExprNode(pCxt, B); }
/************************************************ kill connection/query ***********************************************/ /************************************************ kill connection/query ***********************************************/
cmd ::= KILL CONNECTION NK_INTEGER(A). { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &A); } cmd ::= KILL CONNECTION NK_INTEGER(A). { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &A); }
cmd ::= KILL QUERY NK_STRING(A). { pCxt->pRootNode = createKillQueryStmt(pCxt, &A); } cmd ::= KILL QUERY NK_STRING(A). { pCxt->pRootNode = createKillQueryStmt(pCxt, &A); }
@ -699,6 +707,7 @@ pseudo_column(A) ::= QDURATION(B).
pseudo_column(A) ::= WSTART(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= WSTART(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= WEND(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= WEND(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= WDURATION(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= WDURATION(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= IROWTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
function_expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } function_expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); }
function_expression(A) ::= star_func(B) NK_LP star_func_para_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } function_expression(A) ::= star_func(B) NK_LP star_func_para_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); }
@ -909,7 +918,16 @@ where_clause_opt(A) ::= WHERE search_condition(B).
%type partition_by_clause_opt { SNodeList* } %type partition_by_clause_opt { SNodeList* }
%destructor partition_by_clause_opt { nodesDestroyList($$); } %destructor partition_by_clause_opt { nodesDestroyList($$); }
partition_by_clause_opt(A) ::= . { A = NULL; } partition_by_clause_opt(A) ::= . { A = NULL; }
partition_by_clause_opt(A) ::= PARTITION BY expression_list(B). { A = B; } partition_by_clause_opt(A) ::= PARTITION BY partition_list(B). { A = B; }
%type partition_list { SNodeList* }
%destructor partition_list { nodesDestroyList($$); }
partition_list(A) ::= partition_item(B). { A = createNodeList(pCxt, B); }
partition_list(A) ::= partition_list(B) NK_COMMA partition_item(C). { A = addNodeToList(pCxt, B, C); }
partition_item(A) ::= expr_or_subquery(B). { A = releaseRawExprNode(pCxt, B); }
partition_item(A) ::= expr_or_subquery(B) column_alias(C). { A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); }
partition_item(A) ::= expr_or_subquery(B) AS column_alias(C). { A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); }
twindow_clause_opt(A) ::= . { A = NULL; } twindow_clause_opt(A) ::= . { A = NULL; }
twindow_clause_opt(A) ::= twindow_clause_opt(A) ::=

View File

@ -1055,7 +1055,7 @@ SNode* createFlushDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName) {
return (SNode*)pStmt; return (SNode*)pStmt;
} }
SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName) { SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, int32_t maxSpeed) {
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
if (!checkDbName(pCxt, pDbName, false)) { if (!checkDbName(pCxt, pDbName, false)) {
return NULL; return NULL;
@ -1063,6 +1063,7 @@ SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName) {
STrimDatabaseStmt* pStmt = (STrimDatabaseStmt*)nodesMakeNode(QUERY_NODE_TRIM_DATABASE_STMT); STrimDatabaseStmt* pStmt = (STrimDatabaseStmt*)nodesMakeNode(QUERY_NODE_TRIM_DATABASE_STMT);
CHECK_OUT_OF_MEM(pStmt); CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName); COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName);
pStmt->maxSpeed = maxSpeed;
return (SNode*)pStmt; return (SNode*)pStmt;
} }
@ -1700,7 +1701,7 @@ SNode* createStreamOptions(SAstCreateContext* pCxt) {
} }
SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pStreamName, SNode* pRealTable, SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pStreamName, SNode* pRealTable,
SNode* pOptions, SNode* pQuery) { SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery) {
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
SCreateStreamStmt* pStmt = (SCreateStreamStmt*)nodesMakeNode(QUERY_NODE_CREATE_STREAM_STMT); SCreateStreamStmt* pStmt = (SCreateStreamStmt*)nodesMakeNode(QUERY_NODE_CREATE_STREAM_STMT);
CHECK_OUT_OF_MEM(pStmt); CHECK_OUT_OF_MEM(pStmt);
@ -1713,6 +1714,8 @@ SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, const
pStmt->ignoreExists = ignoreExists; pStmt->ignoreExists = ignoreExists;
pStmt->pOptions = (SStreamOptions*)pOptions; pStmt->pOptions = (SStreamOptions*)pOptions;
pStmt->pQuery = pQuery; pStmt->pQuery = pQuery;
pStmt->pTags = pTags;
pStmt->pSubtable = pSubtable;
return (SNode*)pStmt; return (SNode*)pStmt;
} }

View File

@ -275,6 +275,12 @@ static int32_t calcConstSelectFrom(SCalcConstContext* pCxt, SSelectStmt* pSelect
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = calcConstList(pSelect->pPartitionByList); code = calcConstList(pSelect->pPartitionByList);
} }
if (TSDB_CODE_SUCCESS == code) {
code = calcConstList(pSelect->pTags);
}
if (TSDB_CODE_SUCCESS == code) {
code = calcConstNode(&pSelect->pSubtable);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = calcConstNode(&pSelect->pWindow); code = calcConstNode(&pSelect->pWindow);
} }

View File

@ -129,6 +129,7 @@ static SKeyword keywordTable[] = {
{"MATCH", TK_MATCH}, {"MATCH", TK_MATCH},
{"MAXROWS", TK_MAXROWS}, {"MAXROWS", TK_MAXROWS},
{"MAX_DELAY", TK_MAX_DELAY}, {"MAX_DELAY", TK_MAX_DELAY},
{"MAX_SPEED", TK_MAX_SPEED},
{"MERGE", TK_MERGE}, {"MERGE", TK_MERGE},
{"META", TK_META}, {"META", TK_META},
{"MINROWS", TK_MINROWS}, {"MINROWS", TK_MINROWS},
@ -200,6 +201,7 @@ static SKeyword keywordTable[] = {
{"STREAMS", TK_STREAMS}, {"STREAMS", TK_STREAMS},
{"STRICT", TK_STRICT}, {"STRICT", TK_STRICT},
{"SUBSCRIPTIONS", TK_SUBSCRIPTIONS}, {"SUBSCRIPTIONS", TK_SUBSCRIPTIONS},
{"SUBTABLE", TK_SUBTABLE},
{"SYSINFO", TK_SYSINFO}, {"SYSINFO", TK_SYSINFO},
{"TABLE", TK_TABLE}, {"TABLE", TK_TABLE},
{"TABLES", TK_TABLES}, {"TABLES", TK_TABLES},
@ -250,6 +252,7 @@ static SKeyword keywordTable[] = {
{"WITH", TK_WITH}, {"WITH", TK_WITH},
{"WRITE", TK_WRITE}, {"WRITE", TK_WRITE},
{"_C0", TK_ROWTS}, {"_C0", TK_ROWTS},
{"_IROWTS", TK_IROWTS},
{"_QDURATION", TK_QDURATION}, {"_QDURATION", TK_QDURATION},
{"_QEND", TK_QEND}, {"_QEND", TK_QEND},
{"_QSTART", TK_QSTART}, {"_QSTART", TK_QSTART},

View File

@ -1556,6 +1556,9 @@ static int32_t translateMultiResFunc(STranslateContext* pCxt, SFunctionNode* pFu
"%s(*) is only supported in SELECTed list", pFunc->functionName); "%s(*) is only supported in SELECTed list", pFunc->functionName);
} }
} }
if (tsKeepColumnName) {
strcpy(pFunc->node.userAlias, ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->userAlias);
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -2388,8 +2391,12 @@ static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) {
SColumnNode* pCol = (SColumnNode*)pExpr; SColumnNode* pCol = (SColumnNode*)pExpr;
len = snprintf(buf, sizeof(buf), "%s(%s.%s)", pSrcFunc->functionName, pCol->tableAlias, pCol->colName); len = snprintf(buf, sizeof(buf), "%s(%s.%s)", pSrcFunc->functionName, pCol->tableAlias, pCol->colName);
strncpy(pFunc->node.aliasName, buf, TMIN(len, sizeof(pFunc->node.aliasName) - 1)); strncpy(pFunc->node.aliasName, buf, TMIN(len, sizeof(pFunc->node.aliasName) - 1));
if (tsKeepColumnName) {
strcpy(pFunc->node.userAlias, pCol->colName);
} else {
len = snprintf(buf, sizeof(buf), "%s(%s)", pSrcFunc->functionName, pCol->colName); len = snprintf(buf, sizeof(buf), "%s(%s)", pSrcFunc->functionName, pCol->colName);
strncpy(pFunc->node.userAlias, buf, TMIN(len, sizeof(pFunc->node.userAlias) - 1)); strncpy(pFunc->node.userAlias, buf, TMIN(len, sizeof(pFunc->node.userAlias) - 1));
}
} else { } else {
len = snprintf(buf, sizeof(buf), "%s(%s)", pSrcFunc->functionName, pExpr->aliasName); len = snprintf(buf, sizeof(buf), "%s(%s)", pSrcFunc->functionName, pExpr->aliasName);
strncpy(pFunc->node.aliasName, buf, TMIN(len, sizeof(pFunc->node.aliasName) - 1)); strncpy(pFunc->node.aliasName, buf, TMIN(len, sizeof(pFunc->node.aliasName) - 1));
@ -3076,7 +3083,14 @@ static int32_t translatePartitionBy(STranslateContext* pCxt, SSelectStmt* pSelec
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
pCxt->currClause = SQL_CLAUSE_PARTITION_BY; pCxt->currClause = SQL_CLAUSE_PARTITION_BY;
return translateExprList(pCxt, pSelect->pPartitionByList); int32_t code = translateExprList(pCxt, pSelect->pPartitionByList);
if (TSDB_CODE_SUCCESS == code) {
code = translateExprList(pCxt, pSelect->pTags);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateExpr(pCxt, &pSelect->pSubtable);
}
return code;
} }
static int32_t translateWhere(STranslateContext* pCxt, SSelectStmt* pSelect) { static int32_t translateWhere(STranslateContext* pCxt, SSelectStmt* pSelect) {
@ -3968,7 +3982,7 @@ static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStm
} }
static int32_t translateTrimDatabase(STranslateContext* pCxt, STrimDatabaseStmt* pStmt) { static int32_t translateTrimDatabase(STranslateContext* pCxt, STrimDatabaseStmt* pStmt) {
STrimDbReq req = {0}; STrimDbReq req = {.maxSpeed = pStmt->maxSpeed};
SName name = {0}; SName name = {0};
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
tNameGetFullDbName(&name, req.db); tNameGetFullDbName(&name, req.db);
@ -5211,6 +5225,93 @@ static int32_t addWstartTsToCreateStreamQuery(SNode* pStmt) {
return code; return code;
} }
static int32_t addTagsToCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SSelectStmt* pSelect) {
if (NULL == pStmt->pTags) {
return TSDB_CODE_SUCCESS;
}
SNode* pTag = NULL;
FOREACH(pTag, pStmt->pTags) {
bool found = false;
SNode* pPart = NULL;
FOREACH(pPart, pSelect->pPartitionByList) {
if (0 == strcmp(((SColumnDefNode*)pTag)->colName, ((SExprNode*)pPart)->userAlias)) {
if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSelect->pTags, nodesCloneNode(pPart))) {
return TSDB_CODE_OUT_OF_MEMORY;
}
found = true;
break;
}
}
if (!found) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, ((SColumnDefNode*)pTag)->colName);
}
}
return TSDB_CODE_SUCCESS;
}
typedef struct SRewriteSubtableCxt {
STranslateContext* pCxt;
SNodeList* pPartitionList;
} SRewriteSubtableCxt;
static EDealRes rewriteSubtable(SNode** pNode, void* pContext) {
if (QUERY_NODE_COLUMN == nodeType(*pNode)) {
SRewriteSubtableCxt* pCxt = pContext;
bool found = false;
SNode* pPart = NULL;
FOREACH(pPart, pCxt->pPartitionList) {
if (0 == strcmp(((SColumnNode*)*pNode)->colName, ((SExprNode*)pPart)->userAlias)) {
SNode* pNew = nodesCloneNode(pPart);
if (NULL == pNew) {
pCxt->pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
return DEAL_RES_ERROR;
}
nodesDestroyNode(*pNode);
*pNode = pNew;
found = true;
break;
}
if (!found) {
return generateDealNodeErrMsg(pCxt->pCxt, TSDB_CODE_PAR_INVALID_COLUMN, ((SColumnNode*)*pNode)->colName);
}
}
return DEAL_RES_IGNORE_CHILD;
}
return DEAL_RES_CONTINUE;
}
static int32_t addSubtableNameToCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt,
SSelectStmt* pSelect) {
if (NULL == pStmt->pSubtable) {
return TSDB_CODE_SUCCESS;
}
pSelect->pSubtable = nodesCloneNode(pStmt->pSubtable);
if (NULL == pSelect->pSubtable) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SRewriteSubtableCxt cxt = {.pCxt = pCxt, .pPartitionList = pSelect->pPartitionByList};
nodesRewriteExpr(&pSelect->pSubtable, rewriteSubtable, &cxt);
return pCxt->errCode;
}
static int32_t addSubtableInfoToCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt) {
SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery;
if (NULL == pSelect->pPartitionByList) {
if (NULL != pStmt->pTags || NULL != pStmt->pSubtable) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query");
}
return TSDB_CODE_SUCCESS;
}
int32_t code = addTagsToCreateStreamQuery(pCxt, pStmt, pSelect);
if (TSDB_CODE_SUCCESS == code) {
code = addSubtableNameToCreateStreamQuery(pCxt, pStmt, pSelect);
}
return code;
}
static int32_t checkStreamQuery(STranslateContext* pCxt, SSelectStmt* pSelect) { static int32_t checkStreamQuery(STranslateContext* pCxt, SSelectStmt* pSelect) {
if (TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type || if (TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type ||
!pSelect->isTimeLineResult || crossTableWithoutAggOper(pSelect) || NULL != pSelect->pOrderByList || !pSelect->isTimeLineResult || crossTableWithoutAggOper(pSelect) || NULL != pSelect->pOrderByList ||
@ -5220,18 +5321,21 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SSelectStmt* pSelect) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SNode* pStmt, SCMCreateStreamReq* pReq) { static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SCMCreateStreamReq* pReq) {
pCxt->createStream = true; pCxt->createStream = true;
int32_t code = addWstartTsToCreateStreamQuery(pStmt); int32_t code = addWstartTsToCreateStreamQuery(pStmt->pQuery);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = translateQuery(pCxt, pStmt); code = addSubtableInfoToCreateStreamQuery(pCxt, pStmt);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkStreamQuery(pCxt, (SSelectStmt*)pStmt); code = translateQuery(pCxt, pStmt->pQuery);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
getSourceDatabase(pStmt, pCxt->pParseCxt->acctId, pReq->sourceDB); code = checkStreamQuery(pCxt, (SSelectStmt*)pStmt->pQuery);
code = nodesNodeToString(pStmt, false, &pReq->ast, NULL); }
if (TSDB_CODE_SUCCESS == code) {
getSourceDatabase(pStmt->pQuery, pCxt->pParseCxt->acctId, pReq->sourceDB);
code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL);
} }
return code; return code;
} }
@ -5249,7 +5353,7 @@ static int32_t buildCreateStreamReq(STranslateContext* pCxt, SCreateStreamStmt*
tNameExtractFullName(&name, pReq->targetStbFullName); tNameExtractFullName(&name, pReq->targetStbFullName);
} }
int32_t code = buildCreateStreamQuery(pCxt, pStmt->pQuery, pReq); int32_t code = buildCreateStreamQuery(pCxt, pStmt, pReq);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
pReq->sql = strdup(pCxt->pParseCxt->pSql); pReq->sql = strdup(pCxt->pParseCxt->pSql);
if (NULL == pReq->sql) { if (NULL == pReq->sql) {
@ -5262,6 +5366,8 @@ static int32_t buildCreateStreamReq(STranslateContext* pCxt, SCreateStreamStmt*
pReq->maxDelay = (NULL != pStmt->pOptions->pDelay ? ((SValueNode*)pStmt->pOptions->pDelay)->datum.i : 0); pReq->maxDelay = (NULL != pStmt->pOptions->pDelay ? ((SValueNode*)pStmt->pOptions->pDelay)->datum.i : 0);
pReq->watermark = (NULL != pStmt->pOptions->pWatermark ? ((SValueNode*)pStmt->pOptions->pWatermark)->datum.i : 0); pReq->watermark = (NULL != pStmt->pOptions->pWatermark ? ((SValueNode*)pStmt->pOptions->pWatermark)->datum.i : 0);
pReq->igExpired = pStmt->pOptions->ignoreExpired; pReq->igExpired = pStmt->pOptions->ignoreExpired;
columnDefNodeToField(pStmt->pTags, &pReq->pTags);
pReq->numOfTags = LIST_LENGTH(pStmt->pTags);
} }
return code; return code;

File diff suppressed because it is too large Load Diff

View File

@ -610,6 +610,20 @@ TEST_F(ParserInitialCTest, createStream) {
expect.igExpired = igExpired; expect.igExpired = igExpired;
}; };
auto addTag = [&](const char* pFieldName, uint8_t type, int32_t bytes = 0) {
SField field = {0};
strcpy(field.name, pFieldName);
field.type = type;
field.bytes = bytes > 0 ? bytes : tDataTypes[type].bytes;
field.flags |= COL_SMA_ON;
if (NULL == expect.pTags) {
expect.pTags = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SField));
}
taosArrayPush(expect.pTags, &field);
expect.numOfTags += 1;
};
setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_STREAM_STMT); ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_STREAM_STMT);
SCMCreateStreamReq req = {0}; SCMCreateStreamReq req = {0};
@ -625,6 +639,19 @@ TEST_F(ParserInitialCTest, createStream) {
ASSERT_EQ(req.maxDelay, expect.maxDelay); ASSERT_EQ(req.maxDelay, expect.maxDelay);
ASSERT_EQ(req.watermark, expect.watermark); ASSERT_EQ(req.watermark, expect.watermark);
ASSERT_EQ(req.igExpired, expect.igExpired); ASSERT_EQ(req.igExpired, expect.igExpired);
ASSERT_EQ(req.numOfTags, expect.numOfTags);
if (expect.numOfTags > 0) {
ASSERT_EQ(taosArrayGetSize(req.pTags), expect.numOfTags);
ASSERT_EQ(taosArrayGetSize(req.pTags), taosArrayGetSize(expect.pTags));
for (int32_t i = 0; i < expect.numOfTags; ++i) {
SField* pField = (SField*)taosArrayGet(req.pTags, i);
SField* pExpectField = (SField*)taosArrayGet(expect.pTags, i);
ASSERT_EQ(std::string(pField->name), std::string(pExpectField->name));
ASSERT_EQ(pField->type, pExpectField->type);
ASSERT_EQ(pField->bytes, pExpectField->bytes);
ASSERT_EQ(pField->flags, pExpectField->flags);
}
}
tFreeSCMCreateStreamReq(&req); tFreeSCMCreateStreamReq(&req);
}); });
@ -640,6 +667,17 @@ TEST_F(ParserInitialCTest, createStream) {
run("CREATE STREAM IF NOT EXISTS s1 TRIGGER MAX_DELAY 20s WATERMARK 10s IGNORE EXPIRED 0 INTO st1 AS SELECT COUNT(*) " run("CREATE STREAM IF NOT EXISTS s1 TRIGGER MAX_DELAY 20s WATERMARK 10s IGNORE EXPIRED 0 INTO st1 AS SELECT COUNT(*) "
"FROM t1 INTERVAL(10S)"); "FROM t1 INTERVAL(10S)");
clearCreateStreamReq(); clearCreateStreamReq();
setCreateStreamReqFunc(
"s1", "test",
"create stream s1 into st3 tags(tname varchar(10), id int) subtable(concat('new-', tname)) as "
"select _wstart wstart, count(*) cnt from st1 partition by tbname tname, tag1 id interval(10s)",
"st3");
addTag("tname", TSDB_DATA_TYPE_VARCHAR, 10 + VARSTR_HEADER_SIZE);
addTag("id", TSDB_DATA_TYPE_INT);
run("CREATE STREAM s1 INTO st3 TAGS(tname VARCHAR(10), id INT) SUBTABLE(CONCAT('new-', tname)) "
"AS SELECT _WSTART wstart, COUNT(*) cnt FROM st1 PARTITION BY TBNAME tname, tag1 id INTERVAL(10S)");
clearCreateStreamReq();
} }
TEST_F(ParserInitialCTest, createStreamSemanticCheck) { TEST_F(ParserInitialCTest, createStreamSemanticCheck) {

View File

@ -250,7 +250,10 @@ TEST_F(ParserShowToUseTest, trimDatabase) {
STrimDbReq expect = {0}; STrimDbReq expect = {0};
auto setTrimDbReq = [&](const char* pDb) { snprintf(expect.db, sizeof(expect.db), "0.%s", pDb); }; auto setTrimDbReq = [&](const char* pDb, int32_t maxSpeed = 0) {
snprintf(expect.db, sizeof(expect.db), "0.%s", pDb);
expect.maxSpeed = maxSpeed;
};
setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_TRIM_DATABASE_STMT); ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_TRIM_DATABASE_STMT);
@ -258,10 +261,14 @@ TEST_F(ParserShowToUseTest, trimDatabase) {
STrimDbReq req = {0}; STrimDbReq req = {0};
ASSERT_EQ(tDeserializeSTrimDbReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS); ASSERT_EQ(tDeserializeSTrimDbReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS);
ASSERT_EQ(std::string(req.db), std::string(expect.db)); ASSERT_EQ(std::string(req.db), std::string(expect.db));
ASSERT_EQ(req.maxSpeed, expect.maxSpeed);
}); });
setTrimDbReq("wxy_db"); setTrimDbReq("wxy_db");
run("TRIM DATABASE wxy_db"); run("TRIM DATABASE wxy_db");
setTrimDbReq("wxy_db", 100);
run("TRIM DATABASE wxy_db MAX_SPEED 100");
} }
TEST_F(ParserShowToUseTest, useDatabase) { TEST_F(ParserShowToUseTest, useDatabase) {

View File

@ -611,6 +611,8 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt
return code; return code;
} }
static bool isInterpFunc(int32_t funcId) { return fmIsInterpFunc(funcId) || fmIsInterpPseudoColumnFunc(funcId); }
static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
if (!pSelect->hasInterpFunc) { if (!pSelect->hasInterpFunc) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -625,7 +627,7 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p
pInterpFunc->node.requireDataOrder = getRequireDataOrder(true, pSelect); pInterpFunc->node.requireDataOrder = getRequireDataOrder(true, pSelect);
pInterpFunc->node.resultDataOrder = pInterpFunc->node.requireDataOrder; pInterpFunc->node.resultDataOrder = pInterpFunc->node.requireDataOrder;
int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsInterpFunc, &pInterpFunc->pFuncs); int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, isInterpFunc, &pInterpFunc->pFuncs);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = rewriteExprsForSelect(pInterpFunc->pFuncs, pSelect, SQL_CLAUSE_SELECT); code = rewriteExprsForSelect(pInterpFunc->pFuncs, pSelect, SQL_CLAUSE_SELECT);
} }
@ -1022,6 +1024,20 @@ static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pS
} }
} }
if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pTags) {
pPartition->pTags = nodesCloneList(pSelect->pTags);
if (NULL == pPartition->pTags) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pSubtable) {
pPartition->pSubtable = nodesCloneNode(pSelect->pSubtable);
if (NULL == pPartition->pSubtable) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pPartition; *pLogicNode = (SLogicNode*)pPartition;
} else { } else {

View File

@ -1615,6 +1615,8 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub
SScanLogicNode* pScan = (SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0); SScanLogicNode* pScan = (SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0);
if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) {
TSWAP(((SPartitionLogicNode*)pNode)->pPartitionKeys, pScan->pGroupTags); TSWAP(((SPartitionLogicNode*)pNode)->pPartitionKeys, pScan->pGroupTags);
TSWAP(((SPartitionLogicNode*)pNode)->pTags, pScan->pTags);
TSWAP(((SPartitionLogicNode*)pNode)->pSubtable, pScan->pSubtable);
int32_t code = replaceLogicNode(pLogicSubplan, pNode, (SLogicNode*)pScan); int32_t code = replaceLogicNode(pLogicSubplan, pNode, (SLogicNode*)pScan);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = adjustLogicNodeDataRequirement((SLogicNode*)pScan, pNode->resultDataOrder); code = adjustLogicNodeDataRequirement((SLogicNode*)pScan, pNode->resultDataOrder);

View File

@ -563,7 +563,16 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp
pTableScan->igExpired = pScanLogicNode->igExpired; pTableScan->igExpired = pScanLogicNode->igExpired;
pTableScan->assignBlockUid = pCxt->pPlanCxt->rSmaQuery ? true : false; pTableScan->assignBlockUid = pCxt->pPlanCxt->rSmaQuery ? true : false;
return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode); int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode);
if (TSDB_CODE_SUCCESS == code) {
code = setListSlotId(pCxt, pTableScan->scan.node.pOutputDataBlockDesc->dataBlockId, -1, pScanLogicNode->pTags,
&pTableScan->pTags);
}
if (TSDB_CODE_SUCCESS == code) {
code = setNodeSlotId(pCxt, pTableScan->scan.node.pOutputDataBlockDesc->dataBlockId, -1, pScanLogicNode->pSubtable,
&pTableScan->pSubtable);
}
return code;
} }
static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan,
@ -1322,11 +1331,10 @@ static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren
return code; return code;
} }
static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, static int32_t createPartitionPhysiNodeImpl(SPhysiPlanContext* pCxt, SNodeList* pChildren,
SPartitionLogicNode* pPartLogicNode, SPhysiNode** pPhyNode) { SPartitionLogicNode* pPartLogicNode, ENodeType type,
SPartitionPhysiNode* pPart = (SPartitionPhysiNode*)makePhysiNode( SPhysiNode** pPhyNode) {
pCxt, (SLogicNode*)pPartLogicNode, SPartitionPhysiNode* pPart = (SPartitionPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pPartLogicNode, type);
pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION : QUERY_NODE_PHYSICAL_PLAN_PARTITION);
if (NULL == pPart) { if (NULL == pPart) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
@ -1371,9 +1379,39 @@ static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChi
return code; return code;
} }
static int32_t createStreamPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
SPartitionLogicNode* pPartLogicNode, SPhysiNode** pPhyNode) {
SStreamPartitionPhysiNode* pPart = NULL;
int32_t code = createPartitionPhysiNodeImpl(pCxt, pChildren, pPartLogicNode,
QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION, (SPhysiNode**)&pPart);
SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc);
if (TSDB_CODE_SUCCESS == code) {
code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPartLogicNode->pTags, &pPart->pTags);
}
if (TSDB_CODE_SUCCESS == code) {
code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pPartLogicNode->pSubtable, &pPart->pSubtable);
}
if (TSDB_CODE_SUCCESS == code) {
*pPhyNode = (SPhysiNode*)pPart;
} else {
nodesDestroyNode((SNode*)pPart);
}
return code;
}
static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
SPartitionLogicNode* pPartLogicNode, SPhysiNode** pPhyNode) {
if (pCxt->pPlanCxt->streamQuery) {
return createStreamPartitionPhysiNode(pCxt, pChildren, pPartLogicNode, pPhyNode);
}
return createPartitionPhysiNodeImpl(pCxt, pChildren, pPartLogicNode, QUERY_NODE_PHYSICAL_PLAN_PARTITION, pPhyNode);
}
static int32_t createFillPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SFillLogicNode* pFillNode, static int32_t createFillPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SFillLogicNode* pFillNode,
SPhysiNode** pPhyNode) { SPhysiNode** pPhyNode) {
SFillPhysiNode* pFill = (SFillPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pFillNode, QUERY_NODE_PHYSICAL_PLAN_FILL); SFillPhysiNode* pFill = (SFillPhysiNode*)makePhysiNode(
pCxt, (SLogicNode*)pFillNode,
pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL : QUERY_NODE_PHYSICAL_PLAN_FILL);
if (NULL == pFill) { if (NULL == pFill) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }

View File

@ -101,6 +101,8 @@ TEST_F(PlanBasicTest, interpFunc) {
useDb("root", "test"); useDb("root", "test");
run("SELECT INTERP(c1) FROM t1 RANGE('2017-7-14 18:00:00', '2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR)"); run("SELECT INTERP(c1) FROM t1 RANGE('2017-7-14 18:00:00', '2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR)");
run("SELECT _IROWTS, INTERP(c1) FROM t1 RANGE('2017-7-14 18:00:00', '2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR)");
} }
TEST_F(PlanBasicTest, lastRowFunc) { TEST_F(PlanBasicTest, lastRowFunc) {

Some files were not shown because too many files have changed in this diff Show More