This commit is contained in:
yihaoDeng 2022-09-29 11:09:02 +08:00
commit 88ac809f87
115 changed files with 5627 additions and 599 deletions

View File

@ -2,7 +2,7 @@
IF (DEFINED VERNUMBER) IF (DEFINED VERNUMBER)
SET(TD_VER_NUMBER ${VERNUMBER}) SET(TD_VER_NUMBER ${VERNUMBER})
ELSE () ELSE ()
SET(TD_VER_NUMBER "3.0.1.2") SET(TD_VER_NUMBER "3.0.1.3")
ENDIF () ENDIF ()
IF (DEFINED VERCOMPATIBLE) IF (DEFINED VERCOMPATIBLE)

View File

@ -2,7 +2,7 @@
# taos-tools # taos-tools
ExternalProject_Add(taos-tools ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG f03c09a GIT_TAG 70f5a1c
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR "" BINARY_DIR ""
#BUILD_IN_SOURCE TRUE #BUILD_IN_SOURCE TRUE

View File

@ -37,7 +37,8 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0
- All the data in `tag_set` will be converted to nchar type automatically . - All the data in `tag_set` will be converted to nchar type automatically .
- Each data in `field_set` must be self-descriptive for its data type. For example 1.2f32 means a value 1.2 of float type. Without the "f" type suffix, it will be treated as type double. - Each data in `field_set` must be self-descriptive for its data type. For example 1.2f32 means a value 1.2 of float type. Without the "f" type suffix, it will be treated as type double.
- Multiple kinds of precision can be used for the `timestamp` field. Time precision can be from nanosecond (ns) to hour (h). - Multiple kinds of precision can be used for the `timestamp` field. Time precision can be from nanosecond (ns) to hour (h).
- You can configure smlChildTableName in taos.cfg to specify table names, for example, `smlChildTableName=tname`. You can insert `st,tname=cpul,t1=4 c1=3 1626006833639000000` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored.
- It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.(smlDataFormat in taos.cfg default to false after version of 3.0.1.3)
::: :::
For more details please refer to [InfluxDB Line Protocol](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) and [TDengine Schemaless](/reference/schemaless/#Schemaless-Line-Protocol) For more details please refer to [InfluxDB Line Protocol](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) and [TDengine Schemaless](/reference/schemaless/#Schemaless-Line-Protocol)
@ -64,3 +65,7 @@ For more details please refer to [InfluxDB Line Protocol](https://docs.influxdat
<CLine /> <CLine />
</TabItem> </TabItem>
</Tabs> </Tabs>
## Query Examples
If you want query the data of `location=California.LosAngeles,groupid=2`here is the query sql
select * from `meters.voltage` where location="California.LosAngeles" and groupid=2

View File

@ -31,7 +31,7 @@ For example:
```txt ```txt
meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3 meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3
``` ```
- The defult child table name is generated by rules.You can configure smlChildTableName in taos.cfg to specify chile table names, for example, `smlChildTableName=tname`. You can insert `meters.current 1648432611250 11.3 tname=cpu1 location=California.LosAngeles groupid=3` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored.
Please refer to [OpenTSDB Telnet API](http://opentsdb.net/docs/build/html/api_telnet/put.html) for more details. Please refer to [OpenTSDB Telnet API](http://opentsdb.net/docs/build/html/api_telnet/put.html) for more details.
## Examples ## Examples
@ -79,3 +79,6 @@ taos> select tbname, * from `meters.current`;
t_7e7b26dd860280242c6492a16... | 2022-03-28 09:56:51.250 | 12.600000000 | 2 | California.SanFrancisco | t_7e7b26dd860280242c6492a16... | 2022-03-28 09:56:51.250 | 12.600000000 | 2 | California.SanFrancisco |
Query OK, 4 row(s) in set (0.005399s) Query OK, 4 row(s) in set (0.005399s)
``` ```
## Query Examples
If you want query the data of `location=California.LosAngeles groupid=3`here is the query sql
select * from `meters.voltage` where location="California.LosAngeles" and groupid=3

View File

@ -48,7 +48,7 @@ Please refer to [OpenTSDB HTTP API](http://opentsdb.net/docs/build/html/api_http
- In JSON protocol, strings will be converted to nchar type and numeric values will be converted to double type. - In JSON protocol, strings will be converted to nchar type and numeric values will be converted to double type.
- Only data in array format is accepted and so an array must be used even if there is only one row. - Only data in array format is accepted and so an array must be used even if there is only one row.
- The defult child table name is generated by rules.You can configure smlChildTableName in taos.cfg to specify chile table names, for example, `smlChildTableName=tname`. You can insert `"tags": { "host": "web02","dc": "lga","tname":"cpu1"}` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored.
::: :::
## Examples ## Examples
@ -94,3 +94,6 @@ taos> select * from `meters.current`;
2022-03-28 09:56:51.250 | 12.600000000 | 2.000000000 | California.SanFrancisco | 2022-03-28 09:56:51.250 | 12.600000000 | 2.000000000 | California.SanFrancisco |
Query OK, 2 row(s) in set (0.004076s) Query OK, 2 row(s) in set (0.004076s)
``` ```
## Query Examples
If you want query the data of "tags": {"location": "California.LosAngeles", "groupid": 1}here is the query sql
select * from `meters.voltage` where location="California.LosAngeles" and groupid=1

View File

@ -181,6 +181,14 @@ In TDengine, the first column of all tables must be a timestamp. This column is
select _rowts, max(current) from meters; select _rowts, max(current) from meters;
``` ```
**\_IROWTS**
The \_IROWTS pseudocolumn can only be used with INTERP function. This pseudocolumn can be used to retrieve the corresponding timestamp column associated with the interpolation results.
```sql
select _irowts, interp(current) from meters range('2020-01-01 10:00:00', '2020-01-01 10:30:00') every(1s) fill(linear);
```
## Query Objects ## Query Objects
`FROM` can be followed by a number of tables or super tables, or can be followed by a sub-query. `FROM` can be followed by a number of tables or super tables, or can be followed by a sub-query.

View File

@ -867,6 +867,7 @@ INTERP(expr)
- The number of rows in the result set of `INTERP` is determined by the parameter `EVERY`. Starting from timestamp1, one interpolation is performed for every time interval specified `EVERY` parameter. - The number of rows in the result set of `INTERP` is determined by the parameter `EVERY`. Starting from timestamp1, one interpolation is performed for every time interval specified `EVERY` parameter.
- Interpolation is performed based on `FILL` parameter. - Interpolation is performed based on `FILL` parameter.
- `INTERP` can only be used to interpolate in single timeline. So it must be used with `partition by tbname` when it's used on a STable. - `INTERP` can only be used to interpolate in single timeline. So it must be used with `partition by tbname` when it's used on a STable.
- Pseudo column `_irowts` can be used along with `INTERP` to return the timestamps associated with interpolation points(support after version 3.0.1.4).
### LAST ### LAST

View File

@ -343,6 +343,7 @@ The following list shows all reserved keywords:
### \_ ### \_
- \_C0 - \_C0
- \_IROWTS
- \_QDURATION - \_QDURATION
- \_QEND - \_QEND
- \_QSTART - \_QSTART

View File

@ -11,7 +11,15 @@ TDengine includes a built-in database named `INFORMATION_SCHEMA` to provide acce
4. Future versions of TDengine can add new columns to INFORMATION_SCHEMA tables without affecting existing business systems. 4. Future versions of TDengine can add new columns to INFORMATION_SCHEMA tables without affecting existing business systems.
5. It is easier for users coming from other database management systems. For example, Oracle users can query data dictionary tables. 5. It is easier for users coming from other database management systems. For example, Oracle users can query data dictionary tables.
Note: SHOW statements are still supported for the convenience of existing users. :::info
- SHOW statements are still supported for the convenience of existing users.
- Some columns in the system table may be keywords, and you need to use the escape character '\`' when querying, for example, to query the VGROUPS in the database `test`:
```sql
select `vgroups` from ins_databases where name = 'test';
```
:::
This document introduces the tables of INFORMATION_SCHEMA and their structure. This document introduces the tables of INFORMATION_SCHEMA and their structure.
@ -102,7 +110,11 @@ Provides information about user-created databases. Similar to SHOW DATABASES.
| 24 | wal_retention_period | INT | WAL retention period | | 24 | wal_retention_period | INT | WAL retention period |
| 25 | wal_retention_size | INT | Maximum WAL size | | 25 | wal_retention_size | INT | Maximum WAL size |
| 26 | wal_roll_period | INT | WAL rotation period | | 26 | wal_roll_period | INT | WAL rotation period |
| 27 | wal_segment_size | WAL file size | | 27 | wal_segment_size | BIGINT | WAL file size |
| 28 | stt_trigger | SMALLINT | The threshold for number of files to trigger file merging |
| 29 | table_prefix | SMALLINT | The prefix length in the table name that is ignored when distributing table to vnode based on table name |
| 30 | table_suffix | SMALLINT | The suffix length in the table name that is ignored when distributing table to vnode based on table name |
| 31 | tsdb_pagesize | INT | The page size for internal storage engine, its unit is KB |
## INS_FUNCTIONS ## INS_FUNCTIONS

View File

@ -11,12 +11,13 @@ description: "This document explains how TDengine SQL has changed in version 3.0
| 1 | VARCHAR | Added | Alias of BINARY. | 1 | VARCHAR | Added | Alias of BINARY.
| 2 | TIMESTAMP literal | Added | TIMESTAMP 'timestamp format' syntax now supported. | 2 | TIMESTAMP literal | Added | TIMESTAMP 'timestamp format' syntax now supported.
| 3 | _ROWTS pseudocolumn | Added | Indicates the primary key. Alias of _C0. | 3 | _ROWTS pseudocolumn | Added | Indicates the primary key. Alias of _C0.
| 4 | INFORMATION_SCHEMA | Added | Database for system metadata containing all schema definitions | 4 | _IROWTS pseudocolumn | Added | Used to retrieve timestamps with INTERP function.
| 5 | PERFORMANCE_SCHEMA | Added | Database for system performance information. | 5 | INFORMATION_SCHEMA | Added | Database for system metadata containing all schema definitions
| 6 | Connection queries | Deprecated | Connection queries are no longer supported. The syntax and interfaces are deprecated. | 6 | PERFORMANCE_SCHEMA | Added | Database for system performance information.
| 7 | Mixed operations | Enhanced | Mixing scalar and vector operations in queries has been enhanced and is supported in all SELECT clauses. | 7 | Connection queries | Deprecated | Connection queries are no longer supported. The syntax and interfaces are deprecated.
| 8 | Tag operations | Added | Tag columns can be used in queries and clauses like data columns. | 8 | Mixed operations | Enhanced | Mixing scalar and vector operations in queries has been enhanced and is supported in all SELECT clauses.
| 9 | Timeline clauses and time functions in supertables | Enhanced | When PARTITION BY is not used, data in supertables is merged into a single timeline. | 9 | Tag operations | Added | Tag columns can be used in queries and clauses like data columns.
| 10 | Timeline clauses and time functions in supertables | Enhanced | When PARTITION BY is not used, data in supertables is merged into a single timeline.
## SQL Syntax ## SQL Syntax

View File

@ -177,12 +177,21 @@ The parameters described in this document by the effect that they have on the sy
### maxNumOfDistinctRes ### maxNumOfDistinctRes
| Attribute | Description | | Attribute | Description |
| -------- | -------------------------------- | --- | | -------- | -------------------------------- |
| Applicable | Server Only | | Applicable | Server Only |
| Meaning | The maximum number of distinct rows returned | | Meaning | The maximum number of distinct rows returned |
| Value Range | [100,000 - 100,000,000] | | Value Range | [100,000 - 100,000,000] |
| Default Value | 100,000 | | Default Value | 100,000 |
### keepColumnName
| Attribute | Description |
| -------- | -------------------------------- |
| Applicable | Client only |
| Meaning | When the Last, First, LastRow function is queried, whether the returned column name contains the function name. |
| Value Range | 0 means including the function name, 1 means not including the function name. |
| Default Value | 0 |
## Locale Parameters ## Locale Parameters
### timezone ### timezone

View File

@ -47,9 +47,8 @@ In the schemaless writing data line protocol, each data item in the field_set ne
- `t`, `T`, `true`, `True`, `TRUE`, `f`, `F`, `false`, and `False` will be handled directly as BOOL types. - `t`, `T`, `true`, `True`, `TRUE`, `f`, `F`, `false`, and `False` will be handled directly as BOOL types.
For example, the following data rows indicate that the t1 label is "3" (NCHAR), the t2 label is "4" (NCHAR), and the t3 label For example, the following data rows write c1 column as 3 (BIGINT), c2 column as false (BOOL), c3 column
is "t3" to the super table named `st` labeled "t3" (NCHAR), write c1 column as 3 (BIGINT), c2 column as false (BOOL), c3 column as "passit" (BINARY), c4 column as 4 (DOUBLE), and the primary key timestamp as 1626006833639000000 to child table with the t1 label as "3" (NCHAR), the t2 label as "4" (NCHAR), and the t3 label as "t3" (NCHAR) and the super table named `st`.
is "passit" (BINARY), c4 column is 4 (DOUBLE), and the primary key timestamp is 1626006833639000000 in one row.
```json ```json
st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000
@ -69,7 +68,7 @@ Schemaless writes process row data according to the following principles.
Note that tag_key1, tag_key2 are not the original order of the tags entered by the user but the result of using the tag names in ascending order of the strings. Therefore, tag_key1 is not the first tag entered in the line protocol. Note that tag_key1, tag_key2 are not the original order of the tags entered by the user but the result of using the tag names in ascending order of the strings. Therefore, tag_key1 is not the first tag entered in the line protocol.
The string's MD5 hash value "md5_val" is calculated after the ranking is completed. The calculation result is then combined with the string to generate the table name: "t_md5_val". "t_" is a fixed prefix that every table generated by this mapping relationship has. The string's MD5 hash value "md5_val" is calculated after the ranking is completed. The calculation result is then combined with the string to generate the table name: "t_md5_val". "t_" is a fixed prefix that every table generated by this mapping relationship has.
You can configure smlChildTableName to specify table names, for example, `smlChildTableName=tname`. You can insert `st,tname=cpul,t1=4 c1=3 1626006833639000000` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. You can configure smlChildTableName in taos.cfg to specify table names, for example, `smlChildTableName=tname`. You can insert `st,tname=cpul,t1=4 c1=3 1626006833639000000` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored.
2. If the super table obtained by parsing the line protocol does not exist, this super table is created. 2. If the super table obtained by parsing the line protocol does not exist, this super table is created.
3. If the subtable obtained by the parse line protocol does not exist, Schemaless creates the sub-table according to the subtable name determined in steps 1 or 2. 3. If the subtable obtained by the parse line protocol does not exist, Schemaless creates the sub-table according to the subtable name determined in steps 1 or 2.
@ -78,7 +77,7 @@ You can configure smlChildTableName to specify table names, for example, `smlChi
NULL. NULL.
6. For BINARY or NCHAR columns, if the length of the value provided in a data row exceeds the column type limit, the maximum length of characters allowed to be stored in the column is automatically increased (only incremented and not decremented) to ensure complete preservation of the data. 6. For BINARY or NCHAR columns, if the length of the value provided in a data row exceeds the column type limit, the maximum length of characters allowed to be stored in the column is automatically increased (only incremented and not decremented) to ensure complete preservation of the data.
7. Errors encountered throughout the processing will interrupt the writing process and return an error code. 7. Errors encountered throughout the processing will interrupt the writing process and return an error code.
8. It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat to false. Otherwise, data will be written out of order and a database error will occur. 8. It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.(smlDataFormat in taos.cfg default to false after version of 3.0.1.3)
:::tip :::tip
All processing logic of schemaless will still follow TDengine's underlying restrictions on data structures, such as the total length of each row of data cannot exceed All processing logic of schemaless will still follow TDengine's underlying restrictions on data structures, such as the total length of each row of data cannot exceed

View File

@ -6,6 +6,10 @@ description: TDengine release history, Release Notes and download links.
import Release from "/components/ReleaseV3"; import Release from "/components/ReleaseV3";
## 3.0.1.3
<Release type="tdengine" version="3.0.1.3" />
## 3.0.1.2 ## 3.0.1.2
<Release type="tdengine" version="3.0.1.2" /> <Release type="tdengine" version="3.0.1.2" />

View File

@ -6,6 +6,10 @@ description: taosTools release history, Release Notes, download links.
import Release from "/components/ReleaseV3"; import Release from "/components/ReleaseV3";
## 2.2.3
<Release type="tools" version="2.2.3" />
## 2.2.2 ## 2.2.2
<Release type="tools" version="2.2.2" /> <Release type="tools" version="2.2.2" />

View File

@ -37,7 +37,9 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0
- tag_set 中的所有的数据自动转化为 nchar 数据类型; - tag_set 中的所有的数据自动转化为 nchar 数据类型;
- field_set 中的每个数据项都需要对自身的数据类型进行描述, 比如 1.2f32 代表 float 类型的数值 1.2, 如果不带类型后缀会被当作 double 处理; - field_set 中的每个数据项都需要对自身的数据类型进行描述, 比如 1.2f32 代表 float 类型的数值 1.2, 如果不带类型后缀会被当作 double 处理;
- timestamp 支持多种时间精度。写入数据的时候需要用参数指定时间精度,支持从小时到纳秒的 6 种时间精度。 - timestamp 支持多种时间精度。写入数据的时候需要用参数指定时间精度,支持从小时到纳秒的 6 种时间精度。
- 为了提高写入的效率,默认假设同一个超级表中 field_set 的顺序是一样的(第一条数据包含所有的 field后面的数据按照这个顺序如果顺序不一样需要配置参数 smlDataFormat 为 false否则数据写入按照相同顺序写入库中数据会异常。3.0.1.3之后的版本 smlDataFormat 默认为 false [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议)
- 默认生产的子表名是根据规则生成的唯一ID值。为了让用户可以指定生成的表名可以通过在taos.cfg里配置 smlChildTableName 参数来指定。
举例如下:配置 smlChildTableName=tname 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1注意如果多行数据 tname 相同,但是后面的 tag_set 不同,则使用第一行自动建表时指定的 tag_set其他的行会忽略。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议)
::: :::
要了解更多可参考:[InfluxDB Line 协议官方文档](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) 和 [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) 要了解更多可参考:[InfluxDB Line 协议官方文档](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) 和 [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议)
@ -64,3 +66,7 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0
<CLine /> <CLine />
</TabItem> </TabItem>
</Tabs> </Tabs>
## 查询示例
比如查询 location=California.LosAngeles,groupid=2 子表的数据可以通过如下sql
select * from meters where location=California.LosAngeles and groupid=2

View File

@ -32,6 +32,8 @@ OpenTSDB 行协议同样采用一行字符串来表示一行数据。OpenTSDB
meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3 meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3
``` ```
- 默认生产的子表名是根据规则生成的唯一ID值。为了让用户可以指定生成的表名可以通过在taos.cfg里配置 smlChildTableName 参数来指定。
举例如下:配置 smlChildTableName=tname 插入数据为 meters.current 1648432611250 11.3 tname=cpu1 location=California.LosAngeles groupid=3 则创建的表名为 cpu1注意如果多行数据 tname 相同,但是后面的 tag_set 不同,则使用第一行自动建表时指定的 tag_set其他的行会忽略
参考[OpenTSDB Telnet API 文档](http://opentsdb.net/docs/build/html/api_telnet/put.html)。 参考[OpenTSDB Telnet API 文档](http://opentsdb.net/docs/build/html/api_telnet/put.html)。
## 示例代码 ## 示例代码
@ -79,3 +81,6 @@ taos> select tbname, * from `meters.current`;
t_7e7b26dd860280242c6492a16... | 2022-03-28 09:56:51.250 | 12.600000000 | 2 | California.SanFrancisco | t_7e7b26dd860280242c6492a16... | 2022-03-28 09:56:51.250 | 12.600000000 | 2 | California.SanFrancisco |
Query OK, 4 row(s) in set (0.005399s) Query OK, 4 row(s) in set (0.005399s)
``` ```
## 查询示例:
想要查询 location=California.LosAngeles groupid=3 的数据可以通过如下sql:
select * from `meters.voltage` where location="California.LosAngeles" and groupid=3

View File

@ -48,7 +48,8 @@ OpenTSDB JSON 格式协议采用一个 JSON 字符串表示一行或多行数据
- 对于 JSON 格式协议TDengine 并不会自动把所有标签转成 nchar 类型, 字符串将将转为 nchar 类型, 数值将同样转换为 double 类型。 - 对于 JSON 格式协议TDengine 并不会自动把所有标签转成 nchar 类型, 字符串将将转为 nchar 类型, 数值将同样转换为 double 类型。
- TDengine 只接收 JSON **数组格式**的字符串,即使一行数据也需要转换成数组形式。 - TDengine 只接收 JSON **数组格式**的字符串,即使一行数据也需要转换成数组形式。
- 默认生产的子表名是根据规则生成的唯一ID值。为了让用户可以指定生成的表名可以通过在taos.cfg里配置 smlChildTableName 参数来指定。
举例如下:配置 smlChildTableName=tname 插入数据为 "tags": { "host": "web02","dc": "lga","tname":"cpu1"} 则创建的表名为 cpu1注意如果多行数据 tname 相同,但是后面的 tag_set 不同,则使用第一行自动建表时指定的 tag_set其他的行会忽略
::: :::
## 示例代码 ## 示例代码
@ -94,3 +95,7 @@ taos> select * from `meters.current`;
2022-03-28 09:56:51.250 | 12.600000000 | 2.000000000 | California.SanFrancisco | 2022-03-28 09:56:51.250 | 12.600000000 | 2.000000000 | California.SanFrancisco |
Query OK, 2 row(s) in set (0.004076s) Query OK, 2 row(s) in set (0.004076s)
``` ```
## 查询示例
想要查询"tags": {"location": "California.LosAngeles", "groupid": 1} 的数据可以通过如下sql
select * from `meters.voltage` where location="California.LosAngeles" and groupid=1

View File

@ -182,6 +182,14 @@ TDengine 中,所有表的第一列都必须是时间戳类型,且为其主
select _rowts, max(current) from meters; select _rowts, max(current) from meters;
``` ```
**\_IROWTS**
\_irowts 伪列只能与 interp 函数一起使用,用于返回 interp 函数插值结果对应的时间戳列。
```sql
select _irowts, interp(current) from meters range('2020-01-01 10:00:00', '2020-01-01 10:30:00') every(1s) fill(linear);
```
## 查询对象 ## 查询对象
FROM 关键字后面可以是若干个表(超级表)列表,也可以是子查询的结果。 FROM 关键字后面可以是若干个表(超级表)列表,也可以是子查询的结果。

View File

@ -868,6 +868,7 @@ INTERP(expr)
- INTERP 根据 EVERY 字段来确定输出时间范围内的结果条数,即从 timestamp1 开始每隔固定长度的时间EVERY 值)进行插值。如果没有指定 EVERY则默认窗口大小为无穷大即从 timestamp1 开始只有一个窗口。 - INTERP 根据 EVERY 字段来确定输出时间范围内的结果条数,即从 timestamp1 开始每隔固定长度的时间EVERY 值)进行插值。如果没有指定 EVERY则默认窗口大小为无穷大即从 timestamp1 开始只有一个窗口。
- INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。 - INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。
- INTERP 只能在一个时间序列内进行插值,因此当作用于超级表时必须跟 partition by tbname 一起使用。 - INTERP 只能在一个时间序列内进行插值,因此当作用于超级表时必须跟 partition by tbname 一起使用。
- INTERP 可以与伪列 _irowts 一起使用,返回插值点所对应的时间戳(3.0.1.4版本以后支持)。
### LAST ### LAST

View File

@ -344,6 +344,7 @@ description: TDengine 保留关键字的详细列表
### \_ ### \_
- \_C0 - \_C0
- \_IROWTS
- \_QDURATION - \_QDURATION
- \_QEND - \_QEND
- \_QSTART - \_QSTART

View File

@ -12,7 +12,15 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库,提供对数
4. TDengine 在后续演进中可以灵活的添加已有 INFORMATION_SCHEMA 中表的列,而不用担心对既有业务系统造成影响 4. TDengine 在后续演进中可以灵活的添加已有 INFORMATION_SCHEMA 中表的列,而不用担心对既有业务系统造成影响
5. 与其他数据库系统更具互操作性。例如Oracle 数据库用户熟悉查询 Oracle 数据字典中的表 5. 与其他数据库系统更具互操作性。例如Oracle 数据库用户熟悉查询 Oracle 数据字典中的表
Note: 由于 SHOW 语句已经被开发者熟悉和广泛使用,所以它们仍然被保留。 :::info
- 由于 SHOW 语句已经被开发者熟悉和广泛使用,所以它们仍然被保留。
- 系统表中的一些列可能是关键字,在查询时需要使用转义符'\`',例如查询数据库 test 有几个 VGROUP
```sql
select `vgroups` from ins_databases where name = 'test';
```
:::
本章将详细介绍 `INFORMATION_SCHEMA` 这个内置元数据库中的表和表结构。 本章将详细介绍 `INFORMATION_SCHEMA` 这个内置元数据库中的表和表结构。
@ -103,7 +111,11 @@ Note: 由于 SHOW 语句已经被开发者熟悉和广泛使用,所以它们
| 24 | wal_retention_period | INT | WAL 的保存时长 | | 24 | wal_retention_period | INT | WAL 的保存时长 |
| 25 | wal_retention_size | INT | WAL 的保存上限 | | 25 | wal_retention_size | INT | WAL 的保存上限 |
| 26 | wal_roll_period | INT | wal 文件切换时长 | | 26 | wal_roll_period | INT | wal 文件切换时长 |
| 27 | wal_segment_size | wal 单个文件大小 | | 27 | wal_segment_size | BIGINT | wal 单个文件大小 |
| 28 | stt_trigger | SMALLINT | 触发文件合并的落盘文件的个数 |
| 29 | table_prefix | SMALLINT | 内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的前缀的长度 |
| 30 | table_suffix | SMALLINT | 内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的后缀的长度 |
| 31 | tsdb_pagesize | INT | 时序数据存储引擎中的页大小 |
## INS_FUNCTIONS ## INS_FUNCTIONS

View File

@ -11,12 +11,13 @@ description: "TDengine 3.0 版本的语法变更说明"
| 1 | VARCHAR | 新增 | BINARY类型的别名。 | 1 | VARCHAR | 新增 | BINARY类型的别名。
| 2 | TIMESTAMP字面量 | 新增 | 新增支持 TIMESTAMP 'timestamp format' 语法。 | 2 | TIMESTAMP字面量 | 新增 | 新增支持 TIMESTAMP 'timestamp format' 语法。
| 3 | _ROWTS伪列 | 新增 | 表示时间戳主键。是_C0伪列的别名。 | 3 | _ROWTS伪列 | 新增 | 表示时间戳主键。是_C0伪列的别名。
| 4 | INFORMATION_SCHEMA | 新增 | 包含各种SCHEMA定义的系统数据库。 | 4 | _IROWTS伪列 | 新增 | 用于返回 interp 函数插值结果对应的时间戳列。
| 5 | PERFORMANCE_SCHEMA | 新增 | 包含运行信息的系统数据库。 | 5 | INFORMATION_SCHEMA | 新增 | 包含各种SCHEMA定义的系统数据库。
| 6 | 连续查询 | 废除 | 不再支持连续查询。相关的各种语法和接口废除。 | 6 | PERFORMANCE_SCHEMA | 新增 | 包含运行信息的系统数据库。
| 7 | 混合运算 | 增强 | 查询中的混合运算标量运算和矢量运算混合全面增强SELECT的各个子句均全面支持符合语法语义的混合运算。 | 7 | 连续查询 | 废除 | 不再支持连续查询。相关的各种语法和接口废除。
| 8 | 标签运算 | 新增 |在查询中,标签列可以像普通列一样参与各种运算,用于各种子句。 | 8 | 混合运算 | 增强 | 查询中的混合运算标量运算和矢量运算混合全面增强SELECT的各个子句均全面支持符合语法语义的混合运算。
| 9 | 时间线子句和时间函数用于超级表查询 | 增强 |没有PARTITION BY时超级表的数据会被合并成一条时间线。 | 9 | 标签运算 | 新增 |在查询中,标签列可以像普通列一样参与各种运算,用于各种子句。
| 10 | 时间线子句和时间函数用于超级表查询 | 增强 |没有PARTITION BY时超级表的数据会被合并成一条时间线。
## SQL 语句变更 ## SQL 语句变更

View File

@ -177,12 +177,21 @@ taos --dump-config
### maxNumOfDistinctRes ### maxNumOfDistinctRes
| 属性 | 说明 | | 属性 | 说明 |
| -------- | -------------------------------- | --- | | -------- | -------------------------------- |
| 适用范围 | 仅服务端适用 | | 适用范围 | 仅服务端适用 |
| 含义 | 允许返回的 distinct 结果最大行数 | | 含义 | 允许返回的 distinct 结果最大行数 |
| 取值范围 | 默认值为 10 万,最大值 1 亿 | | 取值范围 | 默认值为 10 万,最大值 1 亿 |
| 缺省值 | 10 万 | | 缺省值 | 10 万 |
### keepColumnName
| 属性 | 说明 |
| -------- | -------------------------------- |
| 适用范围 | 仅客户端适用 |
| 含义 | Last、First、LastRow 函数查询时,返回的列名是否包含函数名。 |
| 取值范围 | 0 表示包含函数名1 表示不包含函数名。 |
| 缺省值 | 0 |
## 区域相关 ## 区域相关
### timezone ### timezone

View File

@ -67,9 +67,12 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000
"measurement,tag_key1=tag_value1,tag_key2=tag_value2" "measurement,tag_key1=tag_value1,tag_key2=tag_value2"
``` ```
:::tip
需要注意的是,这里的 tag_key1, tag_key2 并不是用户输入的标签的原始顺序而是使用了标签名称按照字符串升序排列后的结果。所以tag_key1 并不是在行协议中输入的第一个标签。 需要注意的是,这里的 tag_key1, tag_key2 并不是用户输入的标签的原始顺序而是使用了标签名称按照字符串升序排列后的结果。所以tag_key1 并不是在行协议中输入的第一个标签。
排列完成以后计算该字符串的 MD5 散列值 "md5_val"。然后将计算的结果与字符串组合生成表名“t_md5_val”。其中的 “t_” 是固定的前缀,每个通过该映射关系自动生成的表都具有该前缀。 排列完成以后计算该字符串的 MD5 散列值 "md5_val"。然后将计算的结果与字符串组合生成表名“t_md5_val”。其中的 “t_” 是固定的前缀,每个通过该映射关系自动生成的表都具有该前缀。
为了让用户可以指定生成的表名,可以通过配置 smlChildTableName 来指定(比如 配置 smlChildTableName=tname 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1注意如果多行数据 tname 相同,但是后面的 tag_set 不同,则使用第一次自动建表时指定的 tag_set其他的会忽略 :::tip
为了让用户可以指定生成的表名可以通过在taos.cfg里配置 smlChildTableName 参数来指定。
举例如下:配置 smlChildTableName=tname 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1注意如果多行数据 tname 相同,但是后面的 tag_set 不同,则使用第一行自动建表时指定的 tag_set其他的行会忽略
2. 如果解析行协议获得的超级表不存在,则会创建这个超级表(不建议手动创建超级表,不然插入数据可能异常)。 2. 如果解析行协议获得的超级表不存在,则会创建这个超级表(不建议手动创建超级表,不然插入数据可能异常)。
3. 如果解析行协议获得子表不存在,则 Schemaless 会按照步骤 1 或 2 确定的子表名来创建子表。 3. 如果解析行协议获得子表不存在,则 Schemaless 会按照步骤 1 或 2 确定的子表名来创建子表。

View File

@ -6,6 +6,10 @@ description: TDengine 发布历史、Release Notes 及下载链接
import Release from "/components/ReleaseV3"; import Release from "/components/ReleaseV3";
## 3.0.1.3
<Release type="tdengine" version="3.0.1.3" />
## 3.0.1.2 ## 3.0.1.2
<Release type="tdengine" version="3.0.1.2" /> <Release type="tdengine" version="3.0.1.2" />

View File

@ -6,6 +6,10 @@ description: taosTools 的发布历史、Release Notes 和下载链接
import Release from "/components/ReleaseV3"; import Release from "/components/ReleaseV3";
## 2.2.3
<Release type="tools" version="2.2.3" />
## 2.2.2 ## 2.2.2
<Release type="tools" version="2.2.2" /> <Release type="tools" version="2.2.2" />

View File

@ -1,6 +1,12 @@
# TDengine driver connector for Lua # TDengine driver connector for Lua
It's a Lua implementation for [TDengine](https://github.com/taosdata/TDengine), an open-sourced big data platform designed and optimized for the Internet of Things (IoT), Connected Cars, Industrial IoT, and IT Infrastructure and Application Monitoring. You may need to install Lua5.3 . It's a Lua implementation for [TDengine](https://github.com/taosdata/TDengine), an open-sourced big data platform designed and optimized for the Internet of Things (IoT), Connected Cars, Industrial IoT, and IT Infrastructure and Application Monitoring. You may need to install Lua5.3 .
As TDengine is built with lua-enable with default configure, the built-in lua lib conflicts with external lua lib. The following commands require TDengine built with lua-disable.
To disable built-in lua:
```
mkdir debug && cd debug
cmake .. -DBUILD_LUA=false && cmake --build .
```
## Lua Dependencies ## Lua Dependencies
- Lua: - Lua:

View File

@ -60,6 +60,7 @@ static int l_connect(lua_State *L){
lua_settop(L, 0); lua_settop(L, 0);
taos_init();
lua_newtable(L); lua_newtable(L);
int table_index = lua_gettop(L); int table_index = lua_gettop(L);
@ -136,15 +137,19 @@ static int l_query(lua_State *L){
lua_pushstring(L,fields[i].name); lua_pushstring(L,fields[i].name);
int32_t* length = taos_fetch_lengths(result); int32_t* length = taos_fetch_lengths(result);
switch (fields[i].type) { switch (fields[i].type) {
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_TINYINT:
lua_pushinteger(L,*((char *)row[i])); lua_pushinteger(L,*((char *)row[i]));
break; break;
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_SMALLINT:
lua_pushinteger(L,*((short *)row[i])); lua_pushinteger(L,*((short *)row[i]));
break; break;
case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_INT:
lua_pushinteger(L,*((int *)row[i])); lua_pushinteger(L,*((int *)row[i]));
break; break;
case TSDB_DATA_TYPE_UBIGINT:
case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_BIGINT:
lua_pushinteger(L,*((int64_t *)row[i])); lua_pushinteger(L,*((int64_t *)row[i]));
break; break;
@ -154,6 +159,7 @@ static int l_query(lua_State *L){
case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_DOUBLE:
lua_pushnumber(L,*((double *)row[i])); lua_pushnumber(L,*((double *)row[i]));
break; break;
case TSDB_DATA_TYPE_JSON:
case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_NCHAR:
//printf("type:%d, max len:%d, current len:%d\n",fields[i].type, fields[i].bytes, length[i]); //printf("type:%d, max len:%d, current len:%d\n",fields[i].type, fields[i].bytes, length[i]);
@ -197,7 +203,7 @@ void async_query_callback(void *param, TAOS_RES *result, int code){
printf("failed, reason:%s\n", taos_errstr(result)); printf("failed, reason:%s\n", taos_errstr(result));
lua_pushinteger(L, -1); lua_pushinteger(L, -1);
lua_setfield(L, table_index, "code"); lua_setfield(L, table_index, "code");
lua_pushstring(L,"something is wrong");// taos_errstr(taos)); lua_pushstring(L, taos_errstr(result));
lua_setfield(L, table_index, "error"); lua_setfield(L, table_index, "error");
}else{ }else{
//printf("success to async query.\n"); //printf("success to async query.\n");

View File

@ -203,7 +203,7 @@ void async_query_callback(void *param, TAOS_RES *result, int code){
printf("failed, reason:%s\n", taos_errstr(result)); printf("failed, reason:%s\n", taos_errstr(result));
lua_pushinteger(L, -1); lua_pushinteger(L, -1);
lua_setfield(L, table_index, "code"); lua_setfield(L, table_index, "code");
lua_pushstring(L,"something is wrong");// taos_errstr(taos)); lua_pushstring(L, taos_errstr(result));
lua_setfield(L, table_index, "error"); lua_setfield(L, table_index, "error");
}else{ }else{
//printf("success to async query.\n"); //printf("success to async query.\n");

View File

@ -176,8 +176,14 @@ end
driver.query_a(conn,"INSERT INTO therm1 VALUES ('2019-09-01 00:00:00.005', 100),('2019-09-01 00:00:00.006', 101),('2019-09-01 00:00:00.007', 102)", async_query_callback) driver.query_a(conn,"INSERT INTO therm1 VALUES ('2019-09-01 00:00:00.005', 100),('2019-09-01 00:00:00.006', 101),('2019-09-01 00:00:00.007', 102)", async_query_callback)
res = driver.query(conn, "create stream stream_avg_degree into avg_degree as select avg(degree) from thermometer interval(5s) sliding(1s)") res = driver.query(conn, "create stream stream_avg_degree into avg_degree as select avg(degree) from thermometer interval(5s) sliding(1s)")
if res.code ~=0 then
print("create stream--- failed:"..res.error)
return
else
print("create stream--- pass")
end
print("From now on we start continous insert in an definite loop, pls wait for about 10 seconds and check stream table for result.") print("From now on we start continous insertion in an definite loop, please wait for about 10 seconds and check stream table avg_degree for result.")
local loop_index = 0 local loop_index = 0
while loop_index < 10 do while loop_index < 10 do
local t = os.time()*1000 local t = os.time()*1000
@ -193,5 +199,5 @@ while loop_index < 10 do
os.execute("sleep " .. 1) os.execute("sleep " .. 1)
loop_index = loop_index + 1 loop_index = loop_index + 1
end end
driver.query(conn,"DROP STREAM IF EXISTS avg_therm_s") driver.query(conn,"DROP STREAM IF EXISTS stream_avg_degree")
driver.close(conn) driver.close(conn)

View File

@ -56,9 +56,8 @@ extern int32_t tMsgDict[];
#define TMSG_SEG_CODE(TYPE) (((TYPE)&0xff00) >> 8) #define TMSG_SEG_CODE(TYPE) (((TYPE)&0xff00) >> 8)
#define TMSG_SEG_SEQ(TYPE) ((TYPE)&0xff) #define TMSG_SEG_SEQ(TYPE) ((TYPE)&0xff)
#define TMSG_INFO(TYPE) \ #define TMSG_INFO(TYPE) \
((TYPE) >= 0 && ((TYPE) < TDMT_DND_MAX_MSG || (TYPE) < TDMT_MND_MAX_MSG || (TYPE) < TDMT_VND_MAX_MSG || \ ((TYPE) < TDMT_DND_MAX_MSG || (TYPE) < TDMT_MND_MAX_MSG || (TYPE) < TDMT_VND_MAX_MSG || (TYPE) < TDMT_SCH_MAX_MSG || \
(TYPE) < TDMT_SCH_MAX_MSG || (TYPE) < TDMT_STREAM_MAX_MSG || (TYPE) < TDMT_MON_MAX_MSG || \ (TYPE) < TDMT_STREAM_MAX_MSG || (TYPE) < TDMT_MON_MAX_MSG || (TYPE) < TDMT_SYNC_MAX_MSG) \
(TYPE) < TDMT_SYNC_MAX_MSG)) \
? tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)] \ ? tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)] \
: 0 : 0
#define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)) #define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE))

View File

@ -136,7 +136,7 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table
* @param handle * @param handle
* @return * @return
*/ */
int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, SLocalFetch *pLocal); int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bool* hasMore, SLocalFetch *pLocal);
int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pBlock, uint64_t* useconds); int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pBlock, uint64_t* useconds);
/** /**

View File

@ -239,6 +239,7 @@ typedef enum ENodeType {
QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_FILL, QUERY_NODE_PHYSICAL_PLAN_FILL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL,
QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION,
QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION,
QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION, QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION,

View File

@ -464,6 +464,8 @@ typedef struct SFillPhysiNode {
EOrder inputTsOrder; EOrder inputTsOrder;
} SFillPhysiNode; } SFillPhysiNode;
typedef SFillPhysiNode SStreamFillPhysiNode;
typedef struct SMultiTableIntervalPhysiNode { typedef struct SMultiTableIntervalPhysiNode {
SIntervalPhysiNode interval; SIntervalPhysiNode interval;
SNodeList* pPartitionKeys; SNodeList* pPartitionKeys;

View File

@ -31,6 +31,7 @@ typedef struct {
TDB* db; TDB* db;
TTB* pStateDb; TTB* pStateDb;
TTB* pFuncStateDb; TTB* pFuncStateDb;
TTB* pFillStateDb; // todo refactor
TXN txn; TXN txn;
} SStreamState; } SStreamState;
@ -51,15 +52,22 @@ int32_t streamStateFuncDel(SStreamState* pState, const STupleKey* key);
int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t streamStateDel(SStreamState* pState, const SWinKey* key); int32_t streamStateDel(SStreamState* pState, const SWinKey* key);
int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key);
int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal); int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal);
void streamFreeVal(void* val); void streamFreeVal(void* val);
SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key); SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key);
SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key); SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key);
SStreamStateCur* streamStateSeekKeyPrev(SStreamState* pState, const SWinKey* key); SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* key);
SStreamStateCur* streamStateFillSeekKeyPrev(SStreamState* pState, const SWinKey* key);
void streamStateFreeCur(SStreamStateCur* pCur); void streamStateFreeCur(SStreamStateCur* pCur);
int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t streamStateSeekFirst(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStateSeekFirst(SStreamState* pState, SStreamStateCur* pCur);

View File

@ -1,22 +0,0 @@
[Unit]
Description=Nginx For TDengine Service
After=network-online.target
Wants=network-online.target
[Service]
Type=forking
PIDFile=/usr/local/nginxd/logs/nginx.pid
ExecStart=/usr/local/nginxd/sbin/nginx
ExecStop=/usr/local/nginxd/sbin/nginx -s stop
TimeoutStopSec=1000000s
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TimeoutStartSec=0
StandardOutput=null
Restart=always
StartLimitBurst=3
StartLimitInterval=60s
[Install]
WantedBy=multi-user.target

View File

@ -38,8 +38,6 @@ temp_version=""
fin_result="" fin_result=""
service_config_dir="/etc/systemd/system" service_config_dir="/etc/systemd/system"
nginx_port=6060
nginx_dir="/usr/local/nginxd"
# Color setting # Color setting
RED='\033[0;31m' RED='\033[0;31m'
@ -132,10 +130,7 @@ function check_main_path() {
check_file ${install_main_dir} $i check_file ${install_main_dir} $i
done done
if [ "$verMode" == "cluster" ]; then if [ "$verMode" == "cluster" ]; then
nginx_main_dir=("admin" "conf" "html" "sbin" "logs") check_file ${install_main_dir} "share/admin"
for i in "${nginx_main_dir[@]}";do
check_file ${nginx_dir} $i
done
fi fi
echo -e "Check main path:\033[32mOK\033[0m!" echo -e "Check main path:\033[32mOK\033[0m!"
} }
@ -150,9 +145,6 @@ function check_bin_path() {
for i in "${lbin_dir[@]}";do for i in "${lbin_dir[@]}";do
check_link ${bin_link_dir}/$i check_link ${bin_link_dir}/$i
done done
if [ "$verMode" == "cluster" ]; then
check_file ${nginx_dir}/sbin nginx
fi
echo -e "Check bin path:\033[32mOK\033[0m!" echo -e "Check bin path:\033[32mOK\033[0m!"
} }

View File

@ -50,8 +50,7 @@ install_main_dir=${installDir}
bin_dir="${installDir}/bin" bin_dir="${installDir}/bin"
service_config_dir="/etc/systemd/system" service_config_dir="/etc/systemd/system"
nginx_port=6060 web_port=6041
nginx_dir="/usr/local/nginxd"
# Color setting # Color setting
RED='\033[0;31m' RED='\033[0;31m'
@ -182,7 +181,7 @@ function install_main_path() {
${csudo}mkdir -p ${install_main_dir}/include ${csudo}mkdir -p ${install_main_dir}/include
# ${csudo}mkdir -p ${install_main_dir}/init.d # ${csudo}mkdir -p ${install_main_dir}/init.d
if [ "$verMode" == "cluster" ]; then if [ "$verMode" == "cluster" ]; then
${csudo}mkdir -p ${nginx_dir} ${csudo}mkdir -p ${install_main_dir}/share
fi fi
if [[ -e ${script_dir}/email ]]; then if [[ -e ${script_dir}/email ]]; then
@ -218,12 +217,6 @@ function install_bin() {
[ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || :
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :
[ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
if [ "$verMode" == "cluster" ]; then
${csudo}cp -r ${script_dir}/nginxd/* ${nginx_dir} && ${csudo}chmod 0555 ${nginx_dir}/*
${csudo}mkdir -p ${nginx_dir}/logs
${csudo}chmod 777 ${nginx_dir}/sbin/nginx
fi
} }
function install_lib() { function install_lib() {
@ -574,6 +567,13 @@ function install_examples() {
fi fi
} }
function install_web() {
if [ -d "${script_dir}/share" ]; then
${csudo}cp -rf ${script_dir}/share/* ${install_main_dir}/share
fi
}
function clean_service_on_sysvinit() { function clean_service_on_sysvinit() {
if pidof ${serverName} &>/dev/null; then if pidof ${serverName} &>/dev/null; then
${csudo}service ${serverName} stop || : ${csudo}service ${serverName} stop || :
@ -654,16 +654,6 @@ function clean_service_on_systemd() {
fi fi
${csudo}systemctl disable tarbitratord &>/dev/null || echo &>/dev/null ${csudo}systemctl disable tarbitratord &>/dev/null || echo &>/dev/null
${csudo}rm -f ${tarbitratord_service_config} ${csudo}rm -f ${tarbitratord_service_config}
if [ "$verMode" == "cluster" ]; then
nginx_service_config="${service_config_dir}/nginxd.service"
if systemctl is-active --quiet nginxd; then
echo "Nginx for ${productName} is running, stopping it..."
${csudo}systemctl stop nginxd &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable nginxd &>/dev/null || echo &>/dev/null
${csudo}rm -f ${nginx_service_config}
fi
} }
function install_service_on_systemd() { function install_service_on_systemd() {
@ -677,19 +667,6 @@ function install_service_on_systemd() {
${csudo}systemctl enable ${serverName} ${csudo}systemctl enable ${serverName}
${csudo}systemctl daemon-reload ${csudo}systemctl daemon-reload
if [ "$verMode" == "cluster" ]; then
[ -f ${script_dir}/cfg/nginxd.service ] &&
${csudo}cp ${script_dir}/cfg/nginxd.service \
${service_config_dir}/ || :
${csudo}systemctl daemon-reload
if ! ${csudo}systemctl enable nginxd &>/dev/null; then
${csudo}systemctl daemon-reexec
${csudo}systemctl enable nginxd
fi
${csudo}systemctl start nginxd
fi
} }
function install_adapter_service() { function install_adapter_service() {
@ -793,19 +770,6 @@ function updateProduct() {
sleep 1 sleep 1
fi fi
if [ "$verMode" == "cluster" ]; then
if pidof nginx &>/dev/null; then
if ((${service_mod} == 0)); then
${csudo}systemctl stop nginxd || :
elif ((${service_mod} == 1)); then
${csudo}service nginxd stop || :
else
kill_process nginx
fi
sleep 1
fi
fi
install_main_path install_main_path
install_log install_log
@ -817,6 +781,7 @@ function updateProduct() {
fi fi
install_examples install_examples
install_web
if [ -z $1 ]; then if [ -z $1 ]; then
install_bin install_bin
install_service install_service
@ -825,18 +790,6 @@ function updateProduct() {
install_adapter_config install_adapter_config
openresty_work=false openresty_work=false
if [ "$verMode" == "cluster" ]; then
# Check if openresty is installed
# Check if nginx is installed successfully
if type curl &>/dev/null; then
if curl -sSf http://127.0.0.1:${nginx_port} &>/dev/null; then
echo -e "\033[44;32;1mNginx for ${productName} is updated successfully!${NC}"
openresty_work=true
else
echo -e "\033[44;31;5mNginx for ${productName} does not work! Please try again!\033[0m"
fi
fi
fi
echo echo
echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${cfg_install_dir}/${configFile}" echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${cfg_install_dir}/${configFile}"
@ -857,7 +810,7 @@ function updateProduct() {
fi fi
if [ ${openresty_work} = 'true' ]; then if [ ${openresty_work} = 'true' ]; then
echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${web_port}${NC}"
else else
echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell${NC}" echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell${NC}"
fi fi
@ -906,6 +859,7 @@ function installProduct() {
install_connector install_connector
fi fi
install_examples install_examples
install_web
if [ -z $1 ]; then # install service and client if [ -z $1 ]; then # install service and client
# For installing new # For installing new
@ -915,17 +869,6 @@ function installProduct() {
install_adapter_config install_adapter_config
openresty_work=false openresty_work=false
if [ "$verMode" == "cluster" ]; then
# Check if nginx is installed successfully
if type curl &>/dev/null; then
if curl -sSf http://127.0.0.1:${nginx_port} &>/dev/null; then
echo -e "\033[44;32;1mNginx for ${productName} is installed successfully!${NC}"
openresty_work=true
else
echo -e "\033[44;31;5mNginx for ${productName} does not work! Please try again!\033[0m"
fi
fi
fi
install_config install_config

View File

@ -151,6 +151,7 @@ function install_main_path() {
${csudo}mkdir -p ${install_main_dir}/driver ${csudo}mkdir -p ${install_main_dir}/driver
${csudo}mkdir -p ${install_main_dir}/examples ${csudo}mkdir -p ${install_main_dir}/examples
${csudo}mkdir -p ${install_main_dir}/include ${csudo}mkdir -p ${install_main_dir}/include
${csudo}mkdir -p ${install_main_dir}/share
# ${csudo}mkdir -p ${install_main_dir}/init.d # ${csudo}mkdir -p ${install_main_dir}/init.d
else else
${csudo}rm -rf ${install_main_dir} || ${csudo}rm -rf ${install_main_2_dir} || : ${csudo}rm -rf ${install_main_dir} || ${csudo}rm -rf ${install_main_2_dir} || :
@ -161,6 +162,7 @@ function install_main_path() {
${csudo}mkdir -p ${install_main_dir}/driver || ${csudo}mkdir -p ${install_main_2_dir}/driver ${csudo}mkdir -p ${install_main_dir}/driver || ${csudo}mkdir -p ${install_main_2_dir}/driver
${csudo}mkdir -p ${install_main_dir}/examples || ${csudo}mkdir -p ${install_main_2_dir}/examples ${csudo}mkdir -p ${install_main_dir}/examples || ${csudo}mkdir -p ${install_main_2_dir}/examples
${csudo}mkdir -p ${install_main_dir}/include || ${csudo}mkdir -p ${install_main_2_dir}/include ${csudo}mkdir -p ${install_main_dir}/include || ${csudo}mkdir -p ${install_main_2_dir}/include
${csudo}mkdir -p ${install_main_dir}/share || ${csudo}mkdir -p ${install_main_2_dir}/share
fi fi
} }
@ -469,6 +471,16 @@ function install_examples() {
fi fi
} }
function install_web() {
if [ -d "${binary_dir}/build/share" ]; then
if [ "$osType" != "Darwin" ]; then
${csudo}cp -rf ${binary_dir}/build/share/* ${install_main_dir}/share || :
else
${csudo}cp -rf ${binary_dir}/build/share/* ${install_main_dir}/share || ${csudo}cp -rf ${binary_dir}/build/share/* ${install_main_2_dir}/share || :
fi
fi
}
function clean_service_on_sysvinit() { function clean_service_on_sysvinit() {
if pidof ${serverName} &>/dev/null; then if pidof ${serverName} &>/dev/null; then
${csudo}service ${serverName} stop || : ${csudo}service ${serverName} stop || :
@ -596,6 +608,7 @@ function update_TDengine() {
install_lib install_lib
# install_connector # install_connector
install_examples install_examples
install_web
install_bin install_bin
install_service install_service

View File

@ -107,7 +107,7 @@ else
fi fi
install_files="${script_dir}/install.sh" install_files="${script_dir}/install.sh"
nginx_dir="${top_dir}/../enterprise/src/plugins/web" web_dir="${top_dir}/../enterprise/src/plugins/web"
init_file_deb=${script_dir}/../deb/taosd init_file_deb=${script_dir}/../deb/taosd
init_file_rpm=${script_dir}/../rpm/taosd init_file_rpm=${script_dir}/../rpm/taosd
@ -132,10 +132,6 @@ if [ -f "${cfg_dir}/${serverName}.service" ]; then
cp ${cfg_dir}/${serverName}.service ${install_dir}/cfg || : cp ${cfg_dir}/${serverName}.service ${install_dir}/cfg || :
fi fi
if [ -f "${top_dir}/packaging/cfg/nginxd.service" ]; then
cp ${top_dir}/packaging/cfg/nginxd.service ${install_dir}/cfg || :
fi
mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || :
mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/${serverName}.deb mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/${serverName}.deb
mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/${serverName}.rpm mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/${serverName}.rpm
@ -222,16 +218,6 @@ if [ "$verMode" == "cluster" ]; then
sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh
mv remove_temp.sh ${install_dir}/bin/remove.sh mv remove_temp.sh ${install_dir}/bin/remove.sh
mkdir -p ${install_dir}/nginxd && cp -r ${nginx_dir}/* ${install_dir}/nginxd
cp ${nginx_dir}/png/taos.png ${install_dir}/nginxd/admin/images/taos.png
rm -rf ${install_dir}/nginxd/png
if [ "$cpuType" == "aarch64" ]; then
cp -f ${install_dir}/nginxd/sbin/arm/64bit/nginx ${install_dir}/nginxd/sbin/
elif [ "$cpuType" == "aarch32" ]; then
cp -f ${install_dir}/nginxd/sbin/arm/32bit/nginx ${install_dir}/nginxd/sbin/
fi
rm -rf ${install_dir}/nginxd/sbin/arm
fi fi
cd ${install_dir} cd ${install_dir}
@ -288,6 +274,13 @@ if [[ $dbName == "taos" ]]; then
cp -r ${examples_dir}/C# ${install_dir}/examples cp -r ${examples_dir}/C# ${install_dir}/examples
mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../tools/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../tools/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json
fi fi
# Add web files
if [ -d "${web_dir}/admin" ]; then
mkdir -p ${install_dir}/share/
cp ${web_dir}/admin ${install_dir}/share/ -r
cp ${web_dir}/png/taos.png ${install_dir}/share/admin/images/taos.png
fi
fi fi
# Copy driver # Copy driver

View File

@ -27,13 +27,11 @@ local_bin_link_dir="/usr/local/bin"
lib_link_dir="/usr/lib" lib_link_dir="/usr/lib"
lib64_link_dir="/usr/lib64" lib64_link_dir="/usr/lib64"
inc_link_dir="/usr/include" inc_link_dir="/usr/include"
install_nginxd_dir="/usr/local/nginxd"
service_config_dir="/etc/systemd/system" service_config_dir="/etc/systemd/system"
taos_service_name=${serverName} taos_service_name=${serverName}
taosadapter_service_name="taosadapter" taosadapter_service_name="taosadapter"
tarbitrator_service_name="tarbitratord" tarbitrator_service_name="tarbitratord"
nginx_service_name="nginxd"
csudo="" csudo=""
if command -v sudo >/dev/null; then if command -v sudo >/dev/null; then
csudo="sudo " csudo="sudo "
@ -153,18 +151,6 @@ function clean_service_on_systemd() {
fi fi
${csudo}systemctl disable ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null ${csudo}systemctl disable ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null
${csudo}rm -f ${tarbitratord_service_config} ${csudo}rm -f ${tarbitratord_service_config}
if [ "$verMode" == "cluster" ]; then
nginx_service_config="${service_config_dir}/${nginx_service_name}.service"
if [ -d ${install_nginxd_dir} ]; then
if systemctl is-active --quiet ${nginx_service_name}; then
echo "Nginx for ${productName} is running, stopping it..."
${csudo}systemctl stop ${nginx_service_name} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${nginx_service_name} &>/dev/null || echo &>/dev/null
${csudo}rm -f ${nginx_service_config}
fi
fi
} }
function clean_service_on_sysvinit() { function clean_service_on_sysvinit() {
@ -239,7 +225,6 @@ clean_config
${csudo}rm -rf ${data_link_dir} || : ${csudo}rm -rf ${data_link_dir} || :
${csudo}rm -rf ${install_main_dir} ${csudo}rm -rf ${install_main_dir}
${csudo}rm -rf ${install_nginxd_dir}
if [[ -e /etc/os-release ]]; then if [[ -e /etc/os-release ]]; then
osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release)
else else

View File

@ -19,6 +19,7 @@
#include "functionMgt.h" #include "functionMgt.h"
#include "os.h" #include "os.h"
#include "query.h" #include "query.h"
#include "qworker.h"
#include "scheduler.h" #include "scheduler.h"
#include "tcache.h" #include "tcache.h"
#include "tglobal.h" #include "tglobal.h"
@ -27,7 +28,6 @@
#include "trpc.h" #include "trpc.h"
#include "tsched.h" #include "tsched.h"
#include "ttime.h" #include "ttime.h"
#include "qworker.h"
#define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0 #define TSC_VAR_RELEASED 0
@ -70,11 +70,10 @@ static void deregisterRequest(SRequestObj *pRequest) {
int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1); int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1);
int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1);
int64_t nowUs = taosGetTimestampUs(); int64_t duration = taosGetTimestampUs() - pRequest->metric.start;
int64_t duration = nowUs - pRequest->metric.start; tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%.2f ms, "
tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64 "current:%d, app current:%d",
" ms, current:%d, app current:%d", pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000.0, num, currentInst);
pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst);
if (QUERY_NODE_VNODE_MODIF_STMT == pRequest->stmtType) { if (QUERY_NODE_VNODE_MODIF_STMT == pRequest->stmtType) {
tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64
@ -85,11 +84,12 @@ static void deregisterRequest(SRequestObj *pRequest) {
atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration);
} else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) {
tscPerf("select duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 tscPerf("select duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64
"us, planner:%" PRId64 "us, exec:%" PRId64 "us", "us, planner:%" PRId64 "us, exec:%" PRId64 "us, reqId:0x%"PRIx64,
duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart,
pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - pRequest->metric.ctgEnd, pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - pRequest->metric.ctgEnd,
pRequest->metric.planEnd - pRequest->metric.semanticEnd, pRequest->metric.planEnd - pRequest->metric.semanticEnd,
pRequest->metric.resultReady - pRequest->metric.planEnd); pRequest->metric.resultReady - pRequest->metric.planEnd, pRequest->requestId);
atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration); atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration);
} }

View File

@ -815,7 +815,7 @@ int32_t handleCreateTbExecRes(void* res, SCatalog* pCatalog) {
int32_t handleQueryExecRsp(SRequestObj* pRequest) { int32_t handleQueryExecRsp(SRequestObj* pRequest) {
if (NULL == pRequest->body.resInfo.execRes.res) { if (NULL == pRequest->body.resInfo.execRes.res) {
return TSDB_CODE_SUCCESS; return pRequest->code;
} }
SCatalog* pCatalog = NULL; SCatalog* pCatalog = NULL;
@ -868,10 +868,12 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) {
return code; return code;
} }
//todo refacto the error code mgmt
void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
SRequestObj* pRequest = (SRequestObj*)param; SRequestObj* pRequest = (SRequestObj*)param;
pRequest->code = code; STscObj* pTscObj = pRequest->pTscObj;
pRequest->code = code;
pRequest->metric.resultReady = taosGetTimestampUs(); pRequest->metric.resultReady = taosGetTimestampUs();
if (pResult) { if (pResult) {
@ -879,33 +881,32 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
memcpy(&pRequest->body.resInfo.execRes, pResult, sizeof(*pResult)); memcpy(&pRequest->body.resInfo.execRes, pResult, sizeof(*pResult));
} }
if (TDMT_VND_SUBMIT == pRequest->type || TDMT_VND_DELETE == pRequest->type || int32_t type = pRequest->type;
TDMT_VND_CREATE_TABLE == pRequest->type) { if (TDMT_VND_SUBMIT == type || TDMT_VND_DELETE == type || TDMT_VND_CREATE_TABLE == type) {
if (pResult) { if (pResult) {
pRequest->body.resInfo.numOfRows = pResult->numOfRows; pRequest->body.resInfo.numOfRows = pResult->numOfRows;
if (TDMT_VND_SUBMIT == pRequest->type) {
STscObj* pTscObj = pRequest->pTscObj; // record the insert rows
if (TDMT_VND_SUBMIT == type) {
SAppClusterSummary* pActivity = &pTscObj->pAppInfo->summary; SAppClusterSummary* pActivity = &pTscObj->pAppInfo->summary;
atomic_add_fetch_64((int64_t*)&pActivity->numOfInsertRows, pResult->numOfRows); atomic_add_fetch_64((int64_t*)&pActivity->numOfInsertRows, pResult->numOfRows);
} }
} }
schedulerFreeJob(&pRequest->body.queryJob, 0); schedulerFreeJob(&pRequest->body.queryJob, 0);
pRequest->metric.execEnd = taosGetTimestampUs();
} }
taosMemoryFree(pResult); taosMemoryFree(pResult);
tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code),
pRequest->requestId);
tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code,
tstrerror(code), pRequest->requestId);
STscObj* pTscObj = pRequest->pTscObj;
if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code) && pRequest->sqlstr != NULL) { if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code) && pRequest->sqlstr != NULL) {
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, reqId:0x%" PRIx64,
pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); pRequest->self, tstrerror(code), pRequest->retry, pRequest->requestId);
pRequest->prevCode = code; pRequest->prevCode = code;
schedulerFreeJob(&pRequest->body.queryJob, 0); schedulerFreeJob(&pRequest->body.queryJob, 0);
qDestroyQuery(pRequest->pQuery);
pRequest->pQuery = NULL;
doAsyncQuery(pRequest, true); doAsyncQuery(pRequest, true);
return; return;
} }
@ -915,7 +916,11 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
removeMeta(pTscObj, pRequest->targetTableList); removeMeta(pTscObj, pRequest->targetTableList);
} }
handleQueryExecRsp(pRequest); pRequest->metric.execEnd = taosGetTimestampUs();
int32_t code1 = handleQueryExecRsp(pRequest);
if (pRequest->code == TSDB_CODE_SUCCESS && pRequest->code != code1) {
pRequest->code = code1;
}
// return to client // return to client
pRequest->body.queryFp(pRequest->body.param, pRequest, code); pRequest->body.queryFp(pRequest->body.param, pRequest, code);

View File

@ -20,13 +20,13 @@
#include "functionMgt.h" #include "functionMgt.h"
#include "os.h" #include "os.h"
#include "query.h" #include "query.h"
#include "qworker.h"
#include "scheduler.h" #include "scheduler.h"
#include "tglobal.h" #include "tglobal.h"
#include "tmsg.h" #include "tmsg.h"
#include "tref.h" #include "tref.h"
#include "trpc.h" #include "trpc.h"
#include "version.h" #include "version.h"
#include "qworker.h"
#define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0 #define TSC_VAR_RELEASED 0
@ -700,6 +700,7 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
SQuery *pQuery = pRequest->pQuery; SQuery *pQuery = pRequest->pQuery;
pRequest->metric.ctgEnd = taosGetTimestampUs(); pRequest->metric.ctgEnd = taosGetTimestampUs();
qDebug("0x%" PRIx64 " start to semantic analysis, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
code = qAnalyseSqlSemantic(pWrapper->pCtx, &pWrapper->catalogReq, pResultMeta, pQuery); code = qAnalyseSqlSemantic(pWrapper->pCtx, &pWrapper->catalogReq, pResultMeta, pQuery);
@ -723,13 +724,16 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
destorySqlParseWrapper(pWrapper); destorySqlParseWrapper(pWrapper);
tscDebug("0x%" PRIx64 " analysis semantics completed, start async query, reqId:0x%" PRIx64, pRequest->self, double el = (pRequest->metric.semanticEnd - pRequest->metric.ctgEnd)/1000.0;
pRequest->requestId); tscDebug("0x%" PRIx64 " analysis semantics completed, start async query, elapsed time:%.2f ms, reqId:0x%" PRIx64,
pRequest->self, el, pRequest->requestId);
launchAsyncQuery(pRequest, pQuery, pResultMeta); launchAsyncQuery(pRequest, pQuery, pResultMeta);
} else { } else {
destorySqlParseWrapper(pWrapper); destorySqlParseWrapper(pWrapper);
qDestroyQuery(pRequest->pQuery); qDestroyQuery(pRequest->pQuery);
pRequest->pQuery = NULL; pRequest->pQuery = NULL;
if (NEED_CLIENT_HANDLE_ERROR(code)) { if (NEED_CLIENT_HANDLE_ERROR(code)) {
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64,
pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId);

View File

@ -85,8 +85,7 @@ uint16_t tsTelemPort = 80;
char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null"; char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null";
char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; // user defined child table name can be specified in tag value. char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; // user defined child table name can be specified in tag value.
// If set to empty system will generate table name using MD5 hash. // If set to empty system will generate table name using MD5 hash.
bool tsSmlDataFormat = bool tsSmlDataFormat = false; // true means that the name and order of cols in each line are the same(only for influx protocol)
true; // true means that the name and order of cols in each line are the same(only for influx protocol)
// query // query
int32_t tsQueryPolicy = 1; int32_t tsQueryPolicy = 1;
@ -206,7 +205,9 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input
tstrncpy(cfgFile, cfgDir, sizeof(cfgDir)); tstrncpy(cfgFile, cfgDir, sizeof(cfgDir));
} }
if (apolloUrl == NULL || apolloUrl[0] == '\0') cfgGetApollUrl(envCmd, envFile, apolloUrl); if (apolloUrl != NULL && apolloUrl[0] == '\0') {
cfgGetApollUrl(envCmd, envFile, apolloUrl);
}
if (cfgLoad(pCfg, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) { if (cfgLoad(pCfg, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) {
uError("failed to load from apollo url:%s since %s", apolloUrl, terrstr()); uError("failed to load from apollo url:%s since %s", apolloUrl, terrstr());
@ -1132,11 +1133,20 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
if (tsc) { if (tsc) {
tsLogEmbedded = 0; tsLogEmbedded = 0;
if (taosAddClientLogCfg(pCfg) != 0) return -1; if (taosAddClientLogCfg(pCfg) != 0) {
cfgCleanup(pCfg);
return -1;
}
} else { } else {
tsLogEmbedded = 1; tsLogEmbedded = 1;
if (taosAddClientLogCfg(pCfg) != 0) return -1; if (taosAddClientLogCfg(pCfg) != 0) {
if (taosAddServerLogCfg(pCfg) != 0) return -1; cfgCleanup(pCfg);
return -1;
}
if (taosAddServerLogCfg(pCfg) != 0) {
cfgCleanup(pCfg);
return -1;
}
} }
if (taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) { if (taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) {

View File

@ -87,6 +87,7 @@ int32_t qmPutRpcMsgToQueue(SQnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
return 0; return 0;
default: default:
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
taosFreeQitem(pMsg);
return -1; return -1;
} }
} }

View File

@ -135,12 +135,14 @@ _OVER:
if (content != NULL) taosMemoryFree(content); if (content != NULL) taosMemoryFree(content);
if (root != NULL) cJSON_Delete(root); if (root != NULL) cJSON_Delete(root);
if (pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
if (*ppCfgs == NULL && pCfgs != NULL) taosMemoryFree(pCfgs);
terrno = code; terrno = code;
return code; return code;
} }
int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) { int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
int32_t ret = 0;
char file[PATH_MAX] = {0}; char file[PATH_MAX] = {0};
char realfile[PATH_MAX] = {0}; char realfile[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%svnodes.json.bak", pMgmt->path, TD_DIRSEP); snprintf(file, sizeof(file), "%s%svnodes.json.bak", pMgmt->path, TD_DIRSEP);
@ -161,13 +163,16 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
char *content = taosMemoryCalloc(1, maxLen + 1); char *content = taosMemoryCalloc(1, maxLen + 1);
if (content == NULL) { if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; ret = -1;
goto _OVER;
} }
len += snprintf(content + len, maxLen - len, "{\n"); len += snprintf(content + len, maxLen - len, "{\n");
len += snprintf(content + len, maxLen - len, " \"vnodes\": [\n"); len += snprintf(content + len, maxLen - len, " \"vnodes\": [\n");
for (int32_t i = 0; i < numOfVnodes; ++i) { for (int32_t i = 0; i < numOfVnodes; ++i) {
SVnodeObj *pVnode = pVnodes[i]; SVnodeObj *pVnode = pVnodes[i];
if (pVnode == NULL) continue;
len += snprintf(content + len, maxLen - len, " {\n"); len += snprintf(content + len, maxLen - len, " {\n");
len += snprintf(content + len, maxLen - len, " \"vgId\": %d,\n", pVnode->vgId); len += snprintf(content + len, maxLen - len, " \"vgId\": %d,\n", pVnode->vgId);
len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pVnode->dropped); len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pVnode->dropped);
@ -180,12 +185,13 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
} }
len += snprintf(content + len, maxLen - len, " ]\n"); len += snprintf(content + len, maxLen - len, " ]\n");
len += snprintf(content + len, maxLen - len, "}\n"); len += snprintf(content + len, maxLen - len, "}\n");
terrno = 0;
_OVER:
taosWriteFile(pFile, content, len); taosWriteFile(pFile, content, len);
taosFsyncFile(pFile); taosFsyncFile(pFile);
taosCloseFile(&pFile); taosCloseFile(&pFile);
taosMemoryFree(content); taosMemoryFree(content);
terrno = 0;
for (int32_t i = 0; i < numOfVnodes; ++i) { for (int32_t i = 0; i < numOfVnodes; ++i) {
SVnodeObj *pVnode = pVnodes[i]; SVnodeObj *pVnode = pVnodes[i];
@ -196,6 +202,8 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
taosMemoryFree(pVnodes); taosMemoryFree(pVnodes);
} }
if (ret != 0) return -1;
dDebug("successed to write %s, numOfVnodes:%d", realfile, numOfVnodes); dDebug("successed to write %s, numOfVnodes:%d", realfile, numOfVnodes);
return taosRenameFile(file, realfile); return taosRenameFile(file, realfile);
} }

View File

@ -175,7 +175,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->hashSuffix = pCreate->hashSuffix; pCfg->hashSuffix = pCreate->hashSuffix;
pCfg->tsdbPageSize = pCreate->tsdbPageSize * 1024; pCfg->tsdbPageSize = pCreate->tsdbPageSize * 1024;
pCfg->standby = pCfg->standby; pCfg->standby = 0;
pCfg->syncCfg.myIndex = pCreate->selfIndex; pCfg->syncCfg.myIndex = pCreate->selfIndex;
pCfg->syncCfg.replicaNum = pCreate->replica; pCfg->syncCfg.replicaNum = pCreate->replica;
memset(&pCfg->syncCfg.nodeInfo, 0, sizeof(pCfg->syncCfg.nodeInfo)); memset(&pCfg->syncCfg.nodeInfo, 0, sizeof(pCfg->syncCfg.nodeInfo));

View File

@ -58,11 +58,14 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
if (pVnode->path == NULL) { if (pVnode->path == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pVnode);
return -1; return -1;
} }
if (vmAllocQueue(pMgmt, pVnode) != 0) { if (vmAllocQueue(pMgmt, pVnode) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pVnode->path);
taosMemoryFree(pVnode);
return -1; return -1;
} }
@ -221,6 +224,7 @@ static void vmCloseVnodes(SVnodeMgmt *pMgmt) {
SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes);
for (int32_t i = 0; i < numOfVnodes; ++i) { for (int32_t i = 0; i < numOfVnodes; ++i) {
if (ppVnodes == NULL || ppVnodes[i] == NULL) continue;
vmCloseVnode(pMgmt, ppVnodes[i]); vmCloseVnode(pMgmt, ppVnodes[i]);
} }
@ -380,8 +384,10 @@ static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) {
for (int32_t v = 0; v < numOfVnodes; ++v) { for (int32_t v = 0; v < numOfVnodes; ++v) {
int32_t t = v % threadNum; int32_t t = v % threadNum;
SVnodeThread *pThread = &threads[t]; SVnodeThread *pThread = &threads[t];
if (pThread->ppVnodes != NULL) {
pThread->ppVnodes[pThread->vnodeNum++] = ppVnodes[v]; pThread->ppVnodes[pThread->vnodeNum++] = ppVnodes[v];
} }
}
pMgmt->state.openVnodes = 0; pMgmt->state.openVnodes = 0;
dInfo("restore %d vnodes with %d threads", numOfVnodes, threadNum); dInfo("restore %d vnodes with %d threads", numOfVnodes, threadNum);
@ -411,8 +417,8 @@ static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) {
taosMemoryFree(threads); taosMemoryFree(threads);
for (int32_t i = 0; i < numOfVnodes; ++i) { for (int32_t i = 0; i < numOfVnodes; ++i) {
SVnodeObj *pVnode = ppVnodes[i]; if (ppVnodes == NULL || ppVnodes[i] == NULL) continue;
vmReleaseVnode(pMgmt, pVnode); vmReleaseVnode(pMgmt, ppVnodes[i]);
} }
if (ppVnodes != NULL) { if (ppVnodes != NULL) {

View File

@ -514,7 +514,7 @@ static void sdbCloseIter(SSdbIter *pIter) {
} }
if (pIter->name != NULL) { if (pIter->name != NULL) {
taosRemoveFile(pIter->name); (void)taosRemoveFile(pIter->name);
taosMemoryFree(pIter->name); taosMemoryFree(pIter->name);
pIter->name = NULL; pIter->name = NULL;
} }
@ -606,6 +606,7 @@ int32_t sdbStartWrite(SSdb *pSdb, SSdbIter **ppIter) {
if (pIter->file == NULL) { if (pIter->file == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to open %s since %s", pIter->name, terrstr()); mError("failed to open %s since %s", pIter->name, terrstr());
sdbCloseIter(pIter);
return -1; return -1;
} }
@ -636,9 +637,9 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, i
return -1; return -1;
} }
sdbCloseIter(pIter);
if (sdbReadFile(pSdb) != 0) { if (sdbReadFile(pSdb) != 0) {
mError("sdbiter:%p, failed to read from %s since %s", pIter, datafile, terrstr()); mError("sdbiter:%p, failed to read from %s since %s", pIter, datafile, terrstr());
sdbCloseIter(pIter);
return -1; return -1;
} }
@ -656,6 +657,7 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, i
} }
mInfo("sdbiter:%p, success applyed to sdb", pIter); mInfo("sdbiter:%p, success applyed to sdb", pIter);
sdbCloseIter(pIter);
return 0; return 0;
} }

View File

@ -662,8 +662,8 @@ typedef struct {
} SSkmInfo; } SSkmInfo;
int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
STimeWindow *pTimeWindow, SVersionRange *pVerRange, void *pBlockLoadInfo, STSchema *pSchema, STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo,
int16_t *pCols, int32_t numOfCols, const char *idStr); bool destroyLoadInfo, const char *idStr);
void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter);
bool tMergeTreeNext(SMergeTree *pMTree); bool tMergeTreeNext(SMergeTree *pMTree);
TSDBROW tMergeTreeGetRow(SMergeTree *pMTree); TSDBROW tMergeTreeGetRow(SMergeTree *pMTree);

View File

@ -892,7 +892,7 @@ SArray *metaGetSmaIdsByTable(SMeta *pMeta, tb_uid_t uid) {
pSmaIdxKey = (SSmaIdxKey *)pCur->pKey; pSmaIdxKey = (SSmaIdxKey *)pCur->pKey;
if (taosArrayPush(pUids, &pSmaIdxKey->smaUid) < 0) { if (!taosArrayPush(pUids, &pSmaIdxKey->smaUid)) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
metaCloseSmaCursor(pCur); metaCloseSmaCursor(pCur);
taosArrayDestroy(pUids); taosArrayDestroy(pUids);
@ -935,7 +935,7 @@ SArray *metaGetSmaTbUids(SMeta *pMeta) {
} }
} }
if (taosArrayPush(pUids, &uid) < 0) { if (!taosArrayPush(pUids, &uid)) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
metaCloseSmaCursor(pCur); metaCloseSmaCursor(pCur);
taosArrayDestroy(pUids); taosArrayDestroy(pUids);

View File

@ -213,7 +213,7 @@ static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pStat) {
tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), committed, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), committed, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName);
if (taosCheckExistFile(qTaskInfoFullName)) { if (taosCheckExistFile(qTaskInfoFullName)) {
SQTaskFile qFile = {.nRef = 1, .padding = 0, .version = committed, .size = 0}; SQTaskFile qFile = {.nRef = 1, .padding = 0, .version = committed, .size = 0};
if (taosArrayPush(pFS->aQTaskInf, &qFile) < 0) { if (!taosArrayPush(pFS->aQTaskInf, &qFile)) {
taosWUnLockLatch(RSMA_FS_LOCK(pStat)); taosWUnLockLatch(RSMA_FS_LOCK(pStat));
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;

View File

@ -386,7 +386,7 @@ int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) {
} }
break; break;
default: default:
smaError("vgId:%d, undefined smaType:%", SMA_VID(pSma), smaType); smaError("vgId:%d, undefined smaType:%" PRIi8, SMA_VID(pSma), smaType);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }

View File

@ -54,7 +54,7 @@ int32_t tdRSmaFSOpen(SSma *pSma, int64_t version) {
if ((terrno = tdRSmaFSUpsertQTaskFile(RSMA_FS(pStat), &qTaskFile)) < 0) { if ((terrno = tdRSmaFSUpsertQTaskFile(RSMA_FS(pStat), &qTaskFile)) < 0) {
goto _end; goto _end;
} }
smaInfo("vgId:%d, open fs, version:%" PRIi64 ", ref:%" PRIi64, TD_VID(pVnode), qTaskFile.version, qTaskFile.nRef); smaInfo("vgId:%d, open fs, version:%" PRIi64 ", ref:%d", TD_VID(pVnode), qTaskFile.version, qTaskFile.nRef);
} }
_end: _end:

View File

@ -196,7 +196,8 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids,
if (!suid || !tbUids) { if (!suid || !tbUids) {
terrno = TSDB_CODE_INVALID_PTR; terrno = TSDB_CODE_INVALID_PTR;
smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr()); smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64 " since %s", SMA_VID(pSma), suid ? *suid : -1,
terrstr());
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
@ -566,6 +567,7 @@ static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid)
} }
if (!taosArrayPush(pUidArray, uid)) { if (!taosArrayPush(pUidArray, uid)) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
taosArrayDestroy(pUidArray);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) < 0) { if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) < 0) {
@ -690,7 +692,8 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
while (1) { while (1) {
uint64_t ts; uint64_t ts;
int32_t code = qExecTaskOpt(taskInfo, pResList, &ts, NULL); bool hasMore = false;
int32_t code = qExecTaskOpt(taskInfo, pResList, &ts, &hasMore, NULL);
if (code < 0) { if (code < 0) {
if (code == TSDB_CODE_QRY_IN_EXEC) { if (code == TSDB_CODE_QRY_IN_EXEC) {
break; break;
@ -1677,13 +1680,13 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
SRSmaInfoItem *pItem = NULL; SRSmaInfoItem *pItem = NULL;
if (!(pRSmaRef = taosHashGet(smaMgmt.refHash, &param, POINTER_BYTES))) { if (!(pRSmaRef = taosHashGet(smaMgmt.refHash, &param, POINTER_BYTES))) {
smaDebug("rsma fetch task not start since rsma info item:%p not exist in refHash:%p, rsetId:%" PRIi64, param, smaDebug("rsma fetch task not start since rsma info item:%p not exist in refHash:%p, rsetId:%d", param,
*(int64_t *)&param, smaMgmt.refHash, smaMgmt.rsetId); smaMgmt.refHash, smaMgmt.rsetId);
return; return;
} }
if (!(pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pRSmaRef->refId))) { if (!(pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pRSmaRef->refId))) {
smaDebug("rsma fetch task not start since rsma stat already destroyed, rsetId:%" PRIi64 " refId:%d)", smaDebug("rsma fetch task not start since rsma stat already destroyed, rsetId:%d refId:%" PRIi64 ")",
smaMgmt.rsetId, pRSmaRef->refId); // pRSmaRef freed in taosHashRemove smaMgmt.rsetId, pRSmaRef->refId); // pRSmaRef freed in taosHashRemove
taosHashRemove(smaMgmt.refHash, &param, POINTER_BYTES); taosHashRemove(smaMgmt.refHash, &param, POINTER_BYTES);
return; return;
@ -1692,7 +1695,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
pSma = pStat->pSma; pSma = pStat->pSma;
if (!(pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, pRSmaRef->suid))) { if (!(pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, pRSmaRef->suid))) {
smaDebug("rsma fetch task not start since rsma info not exist, rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId, smaDebug("rsma fetch task not start since rsma info not exist, rsetId:%d refId:%" PRIi64 ")", smaMgmt.rsetId,
pRSmaRef->refId); // pRSmaRef freed in taosHashRemove pRSmaRef->refId); // pRSmaRef freed in taosHashRemove
tdReleaseSmaRef(smaMgmt.rsetId, pRSmaRef->refId); tdReleaseSmaRef(smaMgmt.rsetId, pRSmaRef->refId);
taosHashRemove(smaMgmt.refHash, &param, POINTER_BYTES); taosHashRemove(smaMgmt.refHash, &param, POINTER_BYTES);
@ -1700,7 +1703,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
} }
if (RSMA_INFO_IS_DEL(pRSmaInfo)) { if (RSMA_INFO_IS_DEL(pRSmaInfo)) {
smaDebug("rsma fetch task not start since rsma info already deleted, rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId, smaDebug("rsma fetch task not start since rsma info already deleted, rsetId:%d refId:%" PRIi64 ")", smaMgmt.rsetId,
pRSmaRef->refId); // pRSmaRef freed in taosHashRemove pRSmaRef->refId); // pRSmaRef freed in taosHashRemove
tdReleaseRSmaInfo(pSma, pRSmaInfo); tdReleaseRSmaInfo(pSma, pRSmaInfo);
tdReleaseSmaRef(smaMgmt.rsetId, pRSmaRef->refId); tdReleaseSmaRef(smaMgmt.rsetId, pRSmaRef->refId);
@ -1716,7 +1719,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
case TASK_TRIGGER_STAT_PAUSED: case TASK_TRIGGER_STAT_PAUSED:
case TASK_TRIGGER_STAT_CANCELLED: { case TASK_TRIGGER_STAT_CANCELLED: {
smaDebug("vgId:%d, rsma fetch task not start for level %" PRIi8 " since stat is %" PRIi8 smaDebug("vgId:%d, rsma fetch task not start for level %" PRIi8 " since stat is %" PRIi8
", rsetId rsetId:%" PRIi64 " refId:%d", ", rsetId:%d refId:%" PRIi64,
SMA_VID(pSma), pItem->level, rsmaTriggerStat, smaMgmt.rsetId, pRSmaRef->refId); SMA_VID(pSma), pItem->level, rsmaTriggerStat, smaMgmt.rsetId, pRSmaRef->refId);
if (rsmaTriggerStat == TASK_TRIGGER_STAT_PAUSED) { if (rsmaTriggerStat == TASK_TRIGGER_STAT_PAUSED) {
taosTmrReset(tdRSmaFetchTrigger, RSMA_FETCH_INTERVAL, pItem, smaMgmt.tmrHandle, &pItem->tmrId); taosTmrReset(tdRSmaFetchTrigger, RSMA_FETCH_INTERVAL, pItem, smaMgmt.tmrHandle, &pItem->tmrId);
@ -1844,7 +1847,7 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA
void *msg = NULL; void *msg = NULL;
taosGetQitem(qall, (void **)&msg); taosGetQitem(qall, (void **)&msg);
if (msg) { if (msg) {
if (taosArrayPush(pSubmitArr, &msg) < 0) { if (!taosArrayPush(pSubmitArr, &msg)) {
tdFreeRSmaSubmitItems(pSubmitArr); tdFreeRSmaSubmitItems(pSubmitArr);
goto _err; goto _err;
} }

View File

@ -70,6 +70,8 @@ int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapRead
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_err: _err:
if (pReader) rsmaSnapReaderClose(&pReader);
*ppReader = NULL;
smaError("vgId:%d, vnode snapshot rsma reader open failed since %s", TD_VID(pVnode), tstrerror(code)); smaError("vgId:%d, vnode snapshot rsma reader open failed since %s", TD_VID(pVnode), tstrerror(code));
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
@ -101,8 +103,8 @@ static int32_t rsmaQTaskInfSnapReaderOpen(SRSmaSnapReader* pReader, int64_t vers
if (!taosCheckExistFile(qTaskInfoFullName)) { if (!taosCheckExistFile(qTaskInfoFullName)) {
tdRSmaFSUnRef(pSma, pStat, version); tdRSmaFSUnRef(pSma, pStat, version);
smaInfo("vgId:%d, vnode snapshot rsma reader for qtaskinfo version %" PRIi64 " not need as %s not exists", smaInfo("vgId:%d, vnode snapshot rsma reader for qtaskinfo version %" PRIi64 " not need as %s not exist",
TD_VID(pVnode), qTaskInfoFullName); TD_VID(pVnode), version, qTaskInfoFullName);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -336,6 +338,7 @@ int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWrit
tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), 0, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), 0, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName);
TdFilePtr qTaskF = taosCreateFile(qTaskInfoFullName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); TdFilePtr qTaskF = taosCreateFile(qTaskInfoFullName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (!qTaskF) { if (!qTaskF) {
taosMemoryFree(qWriter);
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
smaError("vgId:%d, rsma snapshot writer open %s failed since %s", TD_VID(pSma->pVnode), qTaskInfoFullName, smaError("vgId:%d, rsma snapshot writer open %s failed since %s", TD_VID(pSma->pVnode), qTaskInfoFullName,
tstrerror(code)); tstrerror(code));
@ -356,6 +359,7 @@ int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWrit
_err: _err:
smaError("vgId:%d, rsma snapshot writer open failed since %s", TD_VID(pSma->pVnode), tstrerror(code)); smaError("vgId:%d, rsma snapshot writer open failed since %s", TD_VID(pSma->pVnode), tstrerror(code));
if (pWriter) rsmaSnapWriterClose(&pWriter, 0);
*ppWriter = NULL; *ppWriter = NULL;
return code; return code;
} }
@ -449,11 +453,11 @@ static int32_t rsmaSnapWriteQTaskInfo(SRSmaSnapWriter* pWriter, uint8_t* pData,
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
smaInfo("vgId:%d, vnode snapshot rsma write qtaskinfo %s succeed", SMA_VID(pWriter->pSma), qWriter->fname);
} else { } else {
smaInfo("vgId:%d, vnode snapshot rsma write qtaskinfo is not needed", SMA_VID(pWriter->pSma)); smaInfo("vgId:%d, vnode snapshot rsma write qtaskinfo is not needed", SMA_VID(pWriter->pSma));
} }
smaInfo("vgId:%d, vnode snapshot rsma write qtaskinfo %s succeed", SMA_VID(pWriter->pSma), qWriter->fname);
_exit: _exit:
return code; return code;

View File

@ -290,19 +290,19 @@ int32_t tdRemoveTFile(STFile *pTFile) {
void *tdAcquireSmaRef(int32_t rsetId, int64_t refId) { void *tdAcquireSmaRef(int32_t rsetId, int64_t refId) {
void *pResult = taosAcquireRef(rsetId, refId); void *pResult = taosAcquireRef(rsetId, refId);
if (!pResult) { if (!pResult) {
smaWarn("rsma acquire ref for rsetId:%" PRIi64 " refId:%d failed since %s", rsetId, refId, terrstr()); smaWarn("rsma acquire ref for rsetId:%d refId:%" PRIi64 " failed since %s", rsetId, refId, terrstr());
} else { } else {
smaDebug("rsma acquire ref for rsetId:%" PRIi64 " refId:%d success", rsetId, refId); smaDebug("rsma acquire ref for rsetId:%d refId:%" PRIi64 " success", rsetId, refId);
} }
return pResult; return pResult;
} }
int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId) { int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId) {
if (taosReleaseRef(rsetId, refId) < 0) { if (taosReleaseRef(rsetId, refId) < 0) {
smaWarn("rsma release ref for rsetId:%" PRIi64 " refId:%d failed since %s", rsetId, refId, terrstr()); smaWarn("rsma release ref for rsetId:%d refId:%" PRIi64 " failed since %s", rsetId, refId, terrstr());
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
smaDebug("rsma release ref for rsetId:%" PRIi64 " refId:%d success", rsetId, refId); smaDebug("rsma release ref for rsetId:%d refId:%" PRIi64 " success", rsetId, refId);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

View File

@ -420,6 +420,7 @@ typedef enum {
typedef struct { typedef struct {
SFSLASTNEXTROWSTATES state; // [input] SFSLASTNEXTROWSTATES state; // [input]
STsdb *pTsdb; // [input] STsdb *pTsdb; // [input]
STSchema *pTSchema;// [input]
tb_uid_t suid; tb_uid_t suid;
tb_uid_t uid; tb_uid_t uid;
int32_t nFileSet; int32_t nFileSet;
@ -455,9 +456,10 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) {
code = tsdbDataFReaderOpen(&state->pDataFReader, state->pTsdb, pFileSet); code = tsdbDataFReaderOpen(&state->pDataFReader, state->pTsdb, pFileSet);
if (code) goto _err; if (code) goto _err;
SSttBlockLoadInfo* pLoadInfo = tCreateLastBlockLoadInfo(state->pTSchema, NULL, 0);
tMergeTreeOpen(&state->mergeTree, 1, state->pDataFReader, state->suid, state->uid, tMergeTreeOpen(&state->mergeTree, 1, state->pDataFReader, state->suid, state->uid,
&(STimeWindow){.skey = TSKEY_MIN, .ekey = TSKEY_MAX}, &(STimeWindow){.skey = TSKEY_MIN, .ekey = TSKEY_MAX},
&(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, NULL, NULL, NULL, 0, NULL); &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, pLoadInfo,true, NULL);
bool hasVal = tMergeTreeNext(&state->mergeTree); bool hasVal = tMergeTreeNext(&state->mergeTree);
if (!hasVal) { if (!hasVal) {
state->state = SFSLASTNEXTROW_FILESET; state->state = SFSLASTNEXTROW_FILESET;
@ -892,6 +894,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
pIter->fsLastState.state = (SFSLASTNEXTROWSTATES)SFSNEXTROW_FS; pIter->fsLastState.state = (SFSLASTNEXTROWSTATES)SFSNEXTROW_FS;
pIter->fsLastState.pTsdb = pTsdb; pIter->fsLastState.pTsdb = pTsdb;
pIter->fsLastState.aDFileSet = pIter->pReadSnap->fs.aDFileSet; pIter->fsLastState.aDFileSet = pIter->pReadSnap->fs.aDFileSet;
pIter->fsLastState.pTSchema = pTSchema;
pIter->fsLastState.suid = suid; pIter->fsLastState.suid = suid;
pIter->fsLastState.uid = uid; pIter->fsLastState.uid = uid;

View File

@ -474,8 +474,8 @@ static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) {
} }
int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
STimeWindow *pTimeWindow, SVersionRange *pVerRange, void* pBlockLoadInfo, STSchema* pSchema, STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo,
int16_t* pCols, int32_t numOfCols, const char* idStr) { bool destroyLoadInfo, const char *idStr) {
pMTree->backward = backward; pMTree->backward = backward;
pMTree->pIter = NULL; pMTree->pIter = NULL;
pMTree->pIterList = taosArrayInit(4, POINTER_BYTES); pMTree->pIterList = taosArrayInit(4, POINTER_BYTES);
@ -488,22 +488,12 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead
tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn); tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn);
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SSttBlockLoadInfo* pLoadInfo = NULL; pMTree->pLoadInfo = pBlockLoadInfo;
if (pBlockLoadInfo == NULL) { pMTree->destroyLoadInfo = destroyLoadInfo;
ASSERT(0);
if (pMTree->pLoadInfo == NULL) {
pMTree->destroyLoadInfo = true;
pMTree->pLoadInfo = tCreateLastBlockLoadInfo(pSchema, pCols, numOfCols);
}
pLoadInfo = pMTree->pLoadInfo;
} else {
pLoadInfo = pBlockLoadInfo;
}
for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file
struct SLDataIter* pIter = NULL; struct SLDataIter* pIter = NULL;
code = tLDataIterOpen(&pIter, pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange, &pLoadInfo[i]); code = tLDataIterOpen(&pIter, pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange, &pMTree->pLoadInfo[i]);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _end; goto _end;
} }

View File

@ -85,6 +85,8 @@ _err:
int tsdbClose(STsdb **pTsdb) { int tsdbClose(STsdb **pTsdb) {
if (*pTsdb) { if (*pTsdb) {
taosThreadRwlockDestroy(&(*pTsdb)->rwLock); taosThreadRwlockDestroy(&(*pTsdb)->rwLock);
tsdbMemTableDestroy((*pTsdb)->mem);
(*pTsdb)->mem = NULL;
tsdbFSClose(*pTsdb); tsdbFSClose(*pTsdb);
tsdbCloseCache(*pTsdb); tsdbCloseCache(*pTsdb);
taosMemoryFreeClear(*pTsdb); taosMemoryFreeClear(*pTsdb);

View File

@ -951,15 +951,22 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockIter, SBlockData* pBlockData) { static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockIter, SBlockData* pBlockData, uint64_t uid) {
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
tBlockDataReset(pBlockData);
TABLEID tid = {.suid = pReader->suid, .uid = uid};
int32_t code = tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colIds[1], pReader->suppInfo.numOfCols-1);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter);
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
ASSERT(pBlockInfo != NULL); ASSERT(pBlockInfo != NULL);
SDataBlk* pBlock = getCurrentBlock(pBlockIter); SDataBlk* pBlock = getCurrentBlock(pBlockIter);
int32_t code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData); code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64
", rows:%d, code:%s %s", ", rows:%d, code:%s %s",
@ -1998,8 +2005,7 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan
int32_t code = int32_t code =
tMergeTreeOpen(&pLBlockReader->mergeTree, (pLBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader, tMergeTreeOpen(&pLBlockReader->mergeTree, (pLBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader,
pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange, pLBlockReader->pInfo, pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange, pLBlockReader->pInfo, false, pReader->idStr);
pReader->pSchema, pReader->suppInfo.colIds, pReader->suppInfo.numOfCols, pReader->idStr);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return false; return false;
} }
@ -2455,14 +2461,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
ASSERT(pBlockIter->numOfBlocks == 0); ASSERT(pBlockIter->numOfBlocks == 0);
code = buildComposedDataBlock(pReader); code = buildComposedDataBlock(pReader);
} else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) { } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) {
tBlockDataReset(&pStatus->fileBlockData); code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid);
TABLEID tid = {.suid = pReader->suid, .uid = pScanInfo->uid};
code = tBlockDataInit(&pStatus->fileBlockData, &tid, pReader->pSchema, NULL, 0);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
@ -2936,14 +2935,7 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn
setFileBlockActiveInBlockIter(pBlockIter, neighborIndex, step); setFileBlockActiveInBlockIter(pBlockIter, neighborIndex, step);
// 3. load the neighbor block, and set it to be the currently accessed file data block // 3. load the neighbor block, and set it to be the currently accessed file data block
tBlockDataReset(&pStatus->fileBlockData); int32_t code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pFBlock->uid);
TABLEID tid = {.suid = pReader->suid, .uid = pFBlock->uid};
int32_t code = tBlockDataInit(&pStatus->fileBlockData, &tid, pReader->pSchema, NULL, 0);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
@ -3701,15 +3693,7 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) {
SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pStatus->blockIter); SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pStatus->blockIter);
STableBlockScanInfo* pBlockScanInfo = taosHashGet(pStatus->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); STableBlockScanInfo* pBlockScanInfo = taosHashGet(pStatus->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid));
tBlockDataReset(&pStatus->fileBlockData); int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid);
TABLEID tid = {.suid = pReader->suid, .uid = pBlockScanInfo->uid};
int32_t code = tBlockDataInit(&pStatus->fileBlockData, &tid, pReader->pSchema, NULL, 0);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
return NULL;
}
code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tBlockDataDestroy(&pStatus->fileBlockData, 1); tBlockDataDestroy(&pStatus->fileBlockData, 1);
terrno = code; terrno = code;

View File

@ -53,6 +53,10 @@ int vnodeCloseBufPool(SVnode *pVnode) {
vnodeBufPoolDestroy(pPool); vnodeBufPoolDestroy(pPool);
} }
if (pVnode->inUse) {
vnodeBufPoolDestroy(pVnode->inUse);
pVnode->inUse = NULL;
}
vDebug("vgId:%d, vnode buffer pool is closed", TD_VID(pVnode)); vDebug("vgId:%d, vnode buffer pool is closed", TD_VID(pVnode));
return 0; return 0;

View File

@ -161,7 +161,6 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
// open sync // open sync
if (vnodeSyncOpen(pVnode, dir)) { if (vnodeSyncOpen(pVnode, dir)) {
vError("vgId:%d, failed to open sync since %s", TD_VID(pVnode), tstrerror(terrno)); vError("vgId:%d, failed to open sync since %s", TD_VID(pVnode), tstrerror(terrno));
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
@ -174,6 +173,7 @@ _err:
if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb); if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb);
if (pVnode->pSma) smaClose(pVnode->pSma); if (pVnode->pSma) smaClose(pVnode->pSma);
if (pVnode->pMeta) metaClose(pVnode->pMeta); if (pVnode->pMeta) metaClose(pVnode->pMeta);
if (pVnode->pPool) vnodeCloseBufPool(pVnode);
tsem_destroy(&(pVnode->canCommit)); tsem_destroy(&(pVnode->canCommit));
taosMemoryFree(pVnode); taosMemoryFree(pVnode);

View File

@ -34,6 +34,7 @@ extern "C" {
#include "scalar.h" #include "scalar.h"
#include "taosdef.h" #include "taosdef.h"
#include "tarray.h" #include "tarray.h"
#include "tfill.h"
#include "thash.h" #include "thash.h"
#include "tlockfree.h" #include "tlockfree.h"
#include "tmsg.h" #include "tmsg.h"
@ -798,6 +799,22 @@ typedef struct SStreamPartitionOperatorInfo {
SSDataBlock* pDelRes; SSDataBlock* pDelRes;
} SStreamPartitionOperatorInfo; } SStreamPartitionOperatorInfo;
typedef struct SStreamFillOperatorInfo {
SStreamFillSupporter* pFillSup;
SSDataBlock* pRes;
SSDataBlock* pSrcBlock;
int32_t srcRowIndex;
SSDataBlock* pPrevSrcBlock;
SSDataBlock* pSrcDelBlock;
int32_t srcDelRowIndex;
SSDataBlock* pDelRes;
SNode* pCondition;
SArray* pColMatchColInfo;
int32_t primaryTsCol;
int32_t primarySrcSlotId;
SStreamFillInfo* pFillInfo;
} SStreamFillOperatorInfo;
typedef struct STimeSliceOperatorInfo { typedef struct STimeSliceOperatorInfo {
SSDataBlock* pRes; SSDataBlock* pRes;
STimeWindow win; STimeWindow win;
@ -1006,6 +1023,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream,
SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
SExecTaskInfo* pTaskInfo); SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode,
SExecTaskInfo* pTaskInfo);
int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx,
int32_t numOfOutput, SArray* pPseudoList); int32_t numOfOutput, SArray* pPseudoList);
@ -1094,6 +1113,7 @@ int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupI
SExecTaskInfo* pTaskInfo); SExecTaskInfo* pTaskInfo);
int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult); int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult);
int32_t saveOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize); int32_t saveOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize);
void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -23,6 +23,7 @@ extern "C" {
#include "os.h" #include "os.h"
#include "taosdef.h" #include "taosdef.h"
#include "tcommon.h" #include "tcommon.h"
#include "tsimplehash.h"
struct SSDataBlock; struct SSDataBlock;
@ -74,13 +75,64 @@ typedef struct SFillInfo {
const char* id; const char* id;
} SFillInfo; } SFillInfo;
int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows); typedef struct SResultCellData {
bool isNull;
int8_t type;
int32_t bytes;
char pData[];
} SResultCellData;
typedef struct SResultRowData {
TSKEY key;
SResultCellData* pRowVal;
} SResultRowData;
typedef struct SStreamFillLinearInfo {
TSKEY nextEnd;
SArray* pDeltaVal; // double. value for Fill(linear).
SArray* pNextDeltaVal; // double. value for Fill(linear).
int64_t winIndex;
bool hasNext;
} SStreamFillLinearInfo;
typedef struct SStreamFillInfo {
TSKEY start; // startKey for fill
TSKEY end; // endKey for fill
TSKEY current; // current Key for fill
TSKEY preRowKey;
TSKEY nextRowKey;
SResultRowData* pResRow;
SStreamFillLinearInfo* pLinearInfo;
bool needFill;
int32_t type; // fill type
int32_t pos;
SArray* delRanges;
int32_t delIndex;
} SStreamFillInfo;
typedef struct SStreamFillSupporter {
int32_t type; // fill type
SInterval interval;
SResultRowData prev;
SResultRowData cur;
SResultRowData next;
SResultRowData nextNext;
SFillColInfo* pAllColInfo; // fill exprs and not fill exprs
int32_t numOfAllCols; // number of all exprs, including the tags columns
int32_t numOfFillCols;
int32_t numOfNotFillCols;
int32_t rowSize;
SSHashObj* pResMap;
bool hasDelete;
} SStreamFillSupporter;
int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows);
void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp); void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp);
void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput); void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput);
struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprInfo* pNotFillExpr, int32_t numOfNotFillCols, const struct SNodeListNode* val); struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprInfo* pNotFillExpr,
int32_t numOfNotFillCols, const struct SNodeListNode* val);
bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); bool taosFillHasMoreResults(struct SFillInfo* pFillInfo);
SFillInfo* taosCreateFillInfo(TSKEY skey, int32_t numOfFillCols, int32_t numOfNotFillCols, int32_t capacity, SFillInfo* taosCreateFillInfo(TSKEY skey, int32_t numOfFillCols, int32_t numOfNotFillCols, int32_t capacity,
@ -91,7 +143,6 @@ void* taosDestroyFillInfo(struct SFillInfo *pFillInfo);
int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity); int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity);
int64_t getFillInfoStart(struct SFillInfo* pFillInfo); int64_t getFillInfoStart(struct SFillInfo* pFillInfo);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -479,7 +479,7 @@ static void freeBlock(void* param) {
blockDataDestroy(pBlock); blockDataDestroy(pBlock);
} }
int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, SLocalFetch* pLocal) { int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bool* hasMore, SLocalFetch* pLocal) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
int64_t threadId = taosGetSelfPthreadId(); int64_t threadId = taosGetSelfPthreadId();
@ -536,6 +536,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, SL
} }
} }
*hasMore = (pRes != NULL);
uint64_t el = (taosGetTimestampUs() - st); uint64_t el = (taosGetTimestampUs() - st);
pTaskInfo->cost.elapsedTime += el; pTaskInfo->cost.elapsedTime += el;

View File

@ -3742,6 +3742,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
pOptr = createMergeJoinOperatorInfo(ops, size, (SSortMergeJoinPhysiNode*)pPhyNode, pTaskInfo); pOptr = createMergeJoinOperatorInfo(ops, size, (SSortMergeJoinPhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_FILL == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_FILL == type) {
pOptr = createFillOperatorInfo(ops[0], (SFillPhysiNode*)pPhyNode, pTaskInfo); pOptr = createFillOperatorInfo(ops[0], (SFillPhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL == type) {
pOptr = createStreamFillOperatorInfo(ops[0], (SStreamFillPhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC == type) {
pOptr = createIndefinitOutputOperatorInfo(ops[0], pPhyNode, pTaskInfo); pOptr = createIndefinitOutputOperatorInfo(ops[0], pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC == type) {

View File

@ -1022,11 +1022,7 @@ static uint64_t getGroupIdByCol(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts,
return calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pPreRes, 0); return calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pPreRes, 0);
} }
static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) { static uint64_t getGroupIdByUid(SStreamScanInfo* pInfo, uint64_t uid) {
if (pInfo->partitionSup.needCalc) {
return getGroupIdByCol(pInfo, uid, ts, maxVersion);
}
SHashObj* map = pInfo->pTableScanOp->pTaskInfo->tableqinfoList.map; SHashObj* map = pInfo->pTableScanOp->pTaskInfo->tableqinfoList.map;
uint64_t* groupId = taosHashGet(map, &uid, sizeof(int64_t)); uint64_t* groupId = taosHashGet(map, &uid, sizeof(int64_t));
if (groupId) { if (groupId) {
@ -1035,6 +1031,14 @@ static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts,
return 0; return 0;
} }
static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) {
if (pInfo->partitionSup.needCalc) {
return getGroupIdByCol(pInfo, uid, ts, maxVersion);
}
return getGroupIdByUid(pInfo, uid);
}
static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t* pRowIndex) { static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t* pRowIndex) {
if ((*pRowIndex) == pBlock->info.rows) { if ((*pRowIndex) == pBlock->info.rows) {
return false; return false;
@ -1081,26 +1085,32 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_
return true; return true;
} }
static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, SInterval* pInterval, static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, uint64_t* gpIdCol, SInterval* pInterval,
SDataBlockInfo* pDataBlockInfo, int32_t* pRowIndex, bool hasGroup) { SDataBlockInfo* pDataBlockInfo, int32_t* pRowIndex, bool hasGroup) {
SResultRowInfo dumyInfo; SResultRowInfo dumyInfo;
dumyInfo.cur.pageId = -1; dumyInfo.cur.pageId = -1;
STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC); STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC);
STimeWindow endWin = win; STimeWindow endWin = win;
STimeWindow preWin = win; STimeWindow preWin = win;
uint64_t groupId = gpIdCol[*pRowIndex];
while (1) { while (1) {
if (hasGroup) { if (hasGroup) {
(*pRowIndex) += 1; (*pRowIndex) += 1;
} else { } else {
(*pRowIndex) += getNumOfRowsInTimeWindow(pDataBlockInfo, startTsCol, *pRowIndex, endWin.ekey, binarySearchForKey, while ((groupId == gpIdCol[(*pRowIndex)] && startTsCol[*pRowIndex] < endWin.ekey)) {
NULL, TSDB_ORDER_ASC); (*pRowIndex) += 1;
if ((*pRowIndex) == pDataBlockInfo->rows) {
break;
} }
}
}
do { do {
preWin = endWin; preWin = endWin;
getNextTimeWindow(pInterval, &endWin, TSDB_ORDER_ASC); getNextTimeWindow(pInterval, &endWin, TSDB_ORDER_ASC);
} while (endTsCol[(*pRowIndex) - 1] >= endWin.skey); } while (endTsCol[(*pRowIndex) - 1] >= endWin.skey);
endWin = preWin; endWin = preWin;
if (win.ekey == endWin.ekey || (*pRowIndex) == pDataBlockInfo->rows) { if (win.ekey == endWin.ekey || (*pRowIndex) == pDataBlockInfo->rows || groupId != gpIdCol[*pRowIndex]) {
win.ekey = endWin.ekey; win.ekey = endWin.ekey;
return win; return win;
} }
@ -1235,11 +1245,13 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS
int64_t version = pSrcBlock->info.version - 1; int64_t version = pSrcBlock->info.version - 1;
for (int32_t i = 0; i < rows;) { for (int32_t i = 0; i < rows;) {
uint64_t srcUid = srcUidData[i]; uint64_t srcUid = srcUidData[i];
uint64_t groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version); uint64_t groupId = srcGp[i];
uint64_t srcGpId = srcGp[i]; if (groupId == 0) {
groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version);
}
TSKEY calStartTs = srcStartTsCol[i]; TSKEY calStartTs = srcStartTsCol[i];
colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false); colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false);
STimeWindow win = getSlidingWindow(srcStartTsCol, srcEndTsCol, &pInfo->interval, &pSrcBlock->info, &i, STimeWindow win = getSlidingWindow(srcStartTsCol, srcEndTsCol, srcGp, &pInfo->interval, &pSrcBlock->info, &i,
pInfo->partitionSup.needCalc); pInfo->partitionSup.needCalc);
TSKEY calEndTs = srcStartTsCol[i - 1]; TSKEY calEndTs = srcStartTsCol[i - 1];
colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(&calEndTs), false); colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(&calEndTs), false);
@ -1248,15 +1260,6 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS
colDataAppend(pEndTsCol, pDestBlock->info.rows, (const char*)(&win.ekey), false); colDataAppend(pEndTsCol, pDestBlock->info.rows, (const char*)(&win.ekey), false);
colDataAppend(pGpCol, pDestBlock->info.rows, (const char*)(&groupId), false); colDataAppend(pGpCol, pDestBlock->info.rows, (const char*)(&groupId), false);
pDestBlock->info.rows++; pDestBlock->info.rows++;
if (pInfo->partitionSup.needCalc && srcGpId != 0 && groupId != srcGpId) {
colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false);
colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(&calEndTs), false);
colDataAppend(pDeUidCol, pDestBlock->info.rows, (const char*)(&srcUid), false);
colDataAppend(pStartTsCol, pDestBlock->info.rows, (const char*)(&win.skey), false);
colDataAppend(pEndTsCol, pDestBlock->info.rows, (const char*)(&win.ekey), false);
colDataAppend(pGpCol, pDestBlock->info.rows, (const char*)(&srcGpId), false);
pDestBlock->info.rows++;
}
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1331,7 +1334,7 @@ void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t*
static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock* pBlock, bool out) { static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock* pBlock, bool out) {
if (out) { if (out) {
blockDataCleanup(pInfo->pUpdateDataRes); blockDataCleanup(pInfo->pUpdateDataRes);
blockDataEnsureCapacity(pInfo->pUpdateDataRes, pBlock->info.rows); blockDataEnsureCapacity(pInfo->pUpdateDataRes, pBlock->info.rows * 2);
} }
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP);
@ -1352,10 +1355,12 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock
isDeletedStreamWindow(&win, pBlock->info.groupId, pInfo->pTableScanOp, &pInfo->twAggSup); isDeletedStreamWindow(&win, pBlock->info.groupId, pInfo->pTableScanOp, &pInfo->twAggSup);
if ((update || closedWin) && out) { if ((update || closedWin) && out) {
qDebug("stream update check not pass, update %d, closedWin %d", update, closedWin); qDebug("stream update check not pass, update %d, closedWin %d", update, closedWin);
uint64_t gpId = closedWin && pInfo->partitionSup.needCalc uint64_t gpId = 0;
? calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pBlock, rowId)
: 0;
appendOneRow(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.uid, &gpId); appendOneRow(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.uid, &gpId);
if (closedWin && pInfo->partitionSup.needCalc) {
gpId = calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pBlock, rowId);
appendOneRow(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.uid, &gpId);
}
} }
} }
if (out && pInfo->pUpdateDataRes->info.rows > 0) { if (out && pInfo->pUpdateDataRes->info.rows > 0) {
@ -1532,6 +1537,30 @@ static int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, S
return 0; return 0;
} }
// for partition by tag
static void setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) {
SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
TSKEY* startTsCol = (TSKEY*)pStartTsCol->pData;
SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
uint64_t* gpCol = (uint64_t*)pGpCol->pData;
SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
uint64_t* uidCol = (uint64_t*)pUidCol->pData;
int32_t rows = pBlock->info.rows;
if (!pInfo->partitionSup.needCalc) {
for (int32_t i = 0; i < rows; i++) {
uint64_t groupId = getGroupIdByUid(pInfo, uidCol[i]);
colDataAppend(pGpCol, i, (const char*)&groupId, false);
}
} else {
// SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, uidCol[i], startTsCol, ts, maxVersion);
// if (!pPreRes || pPreRes->info.rows == 0) {
// return 0;
// }
// ASSERT(pPreRes->info.rows == 1);
// return calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pPreRes, 0);
}
}
static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
// NOTE: this operator does never check if current status is done or not // NOTE: this operator does never check if current status is done or not
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -1628,7 +1657,8 @@ FETCH_NEXT_BLOCK:
} else { } else {
pDelBlock = pBlock; pDelBlock = pBlock;
} }
printDataBlock(pBlock, "stream scan delete recv filtered"); setBlockGroupIdByUid(pInfo, pDelBlock);
printDataBlock(pDelBlock, "stream scan delete recv filtered");
if (!isIntervalWindow(pInfo) && !isSessionWindow(pInfo) && !isStateWindow(pInfo)) { if (!isIntervalWindow(pInfo) && !isSessionWindow(pInfo) && !isStateWindow(pInfo)) {
generateDeleteResultBlock(pInfo, pDelBlock, pInfo->pDeleteDataRes); generateDeleteResultBlock(pInfo, pDelBlock, pInfo->pDeleteDataRes);
pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT; pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT;

File diff suppressed because it is too large Load Diff

View File

@ -271,6 +271,10 @@ static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t o
tw->ekey -= 1; tw->ekey -= 1;
} }
void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) {
getNextTimeWindow(pInterval, pInterval->precision, order, tw);
}
void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs,
int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) { int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) {
SqlFunctionCtx* pCtx = pSup->pCtx; SqlFunctionCtx* pCtx = pSup->pCtx;
@ -2095,12 +2099,17 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
bool hasInterp = true; bool hasInterp = true;
for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) { for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
SExprInfo* pExprInfo = &pExprSup->pExprInfo[j]; SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
int32_t dstSlot = pExprInfo->base.resSchema.slotId;
// SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); int32_t dstSlot = pExprInfo->base.resSchema.slotId;
SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);
if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) {
colDataAppend(pDst, rows, (char*)&pSliceInfo->current, false);
continue;
}
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
// SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
switch (pSliceInfo->fillType) { switch (pSliceInfo->fillType) {
case TSDB_FILL_NULL: { case TSDB_FILL_NULL: {
colDataAppendNULL(pDst, rows); colDataAppendNULL(pDst, rows);
@ -2346,12 +2355,16 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
if (ts == pSliceInfo->current) { if (ts == pSliceInfo->current) {
for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j];
int32_t dstSlot = pExprInfo->base.resSchema.slotId;
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); int32_t dstSlot = pExprInfo->base.resSchema.slotId;
SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);
if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) {
colDataAppend(pDst, pResBlock->info.rows, (char *)&pSliceInfo->current, false);
} else {
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
if (colDataIsNull_s(pSrc, i)) { if (colDataIsNull_s(pSrc, i)) {
colDataAppendNULL(pDst, pResBlock->info.rows); colDataAppendNULL(pDst, pResBlock->info.rows);
continue; continue;
@ -2360,6 +2373,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
char* v = colDataGetData(pSrc, i); char* v = colDataGetData(pSrc, i);
colDataAppend(pDst, pResBlock->info.rows, v, false); colDataAppend(pDst, pResBlock->info.rows, v, false);
} }
}
pResBlock->info.rows += 1; pResBlock->info.rows += 1;
doKeepPrevRows(pSliceInfo, pBlock, i); doKeepPrevRows(pSliceInfo, pBlock, i);
@ -2478,15 +2492,25 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) { if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) {
for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j];
int32_t dstSlot = pExprInfo->base.resSchema.slotId;
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); int32_t dstSlot = pExprInfo->base.resSchema.slotId;
SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);
if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) {
colDataAppend(pDst, pResBlock->info.rows, (char *)&pSliceInfo->current, false);
} else {
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
if (colDataIsNull_s(pSrc, i)) {
colDataAppendNULL(pDst, pResBlock->info.rows);
continue;
}
char* v = colDataGetData(pSrc, i); char* v = colDataGetData(pSrc, i);
colDataAppend(pDst, pResBlock->info.rows, v, false); colDataAppend(pDst, pResBlock->info.rows, v, false);
} }
}
pResBlock->info.rows += 1; pResBlock->info.rows += 1;
doKeepPrevRows(pSliceInfo, pBlock, i); doKeepPrevRows(pSliceInfo, pBlock, i);
@ -3146,8 +3170,12 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
} }
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
if (pInfo->binfo.pRes->info.rows == 0) { if (pInfo->binfo.pRes->info.rows != 0) {
pOperator->status = OP_EXEC_DONE; printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
return pInfo->binfo.pRes;
}
doSetOperatorCompleted(pOperator);
if (!IS_FINAL_OP(pInfo)) { if (!IS_FINAL_OP(pInfo)) {
clearFunctionContext(&pOperator->exprSupp); clearFunctionContext(&pOperator->exprSupp);
// semi interval operator clear disk buffer // semi interval operator clear disk buffer
@ -3157,9 +3185,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
} }
return NULL; return NULL;
}
printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
return pInfo->binfo.pRes;
} else { } else {
if (!IS_FINAL_OP(pInfo)) { if (!IS_FINAL_OP(pInfo)) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
@ -3316,7 +3341,13 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
return pInfo->pPullDataRes; return pInfo->pPullDataRes;
} }
// we should send result first. doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
if (pInfo->pDelRes->info.rows != 0) {
// process the rest of the data
printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
return pInfo->pDelRes;
}
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
if (pInfo->binfo.pRes->info.rows != 0) { if (pInfo->binfo.pRes->info.rows != 0) {
printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
@ -3330,13 +3361,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
// process the rest of the data // process the rest of the data
return pInfo->pUpdateRes; return pInfo->pUpdateRes;
} }
doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
if (pInfo->pDelRes->info.rows != 0) {
// process the rest of the data
printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
return pInfo->pDelRes;
}
return NULL; return NULL;
} }
@ -5744,19 +5768,18 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
if (pOperator->status == OP_RES_TO_RETURN) { if (pOperator->status == OP_RES_TO_RETURN) {
doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
if (pInfo->pDelRes->info.rows > 0) { if (pInfo->pDelRes->info.rows > 0) {
printDataBlock(pInfo->pDelRes, "single interval"); printDataBlock(pInfo->pDelRes, "single interval delete");
return pInfo->pDelRes; return pInfo->pDelRes;
} }
doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo); doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo);
// doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pInfo->binfo.pRes->info.rows > 0) {
if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
pOperator->status = OP_EXEC_DONE;
qDebug("===stream===single interval is done");
freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
}
printDataBlock(pInfo->binfo.pRes, "single interval"); printDataBlock(pInfo->binfo.pRes, "single interval");
return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; return pInfo->binfo.pRes;
}
doSetOperatorCompleted(pOperator);
return NULL;
} }
SOperatorInfo* downstream = pOperator->pDownstream[0]; SOperatorInfo* downstream = pOperator->pDownstream[0];
@ -5823,24 +5846,24 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
} }
taosArraySort(pUpdated, resultrowComparAsc); taosArraySort(pUpdated, resultrowComparAsc);
// new disc buf
// finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated,
// pSup->rowEntryInfoOffset);
initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
removeDeleteResults(pUpdatedMap, pInfo->pDelWins); removeDeleteResults(pUpdatedMap, pInfo->pDelWins);
taosHashCleanup(pUpdatedMap); taosHashCleanup(pUpdatedMap);
doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
if (pInfo->pDelRes->info.rows > 0) { if (pInfo->pDelRes->info.rows > 0) {
printDataBlock(pInfo->pDelRes, "single interval"); printDataBlock(pInfo->pDelRes, "single interval delete");
return pInfo->pDelRes; return pInfo->pDelRes;
} }
// doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
// new disc buf
doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo); doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo);
if (pInfo->binfo.pRes->info.rows > 0) {
printDataBlock(pInfo->binfo.pRes, "single interval"); printDataBlock(pInfo->binfo.pRes, "single interval");
return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; return pInfo->binfo.pRes;
}
return NULL;
} }
void destroyStreamIntervalOperatorInfo(void* param) { void destroyStreamIntervalOperatorInfo(void* param) {

View File

@ -302,6 +302,7 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR
char* p = taosMemoryCalloc(1, strlen(c->colVal) + 1); char* p = taosMemoryCalloc(1, strlen(c->colVal) + 1);
memcpy(p, c->colVal, strlen(c->colVal)); memcpy(p, c->colVal, strlen(c->colVal));
cond = cmpFn(p + skip, term->colVal, dType); cond = cmpFn(p + skip, term->colVal, dType);
taosMemoryFree(p);
} }
} }
if (cond == MATCH) { if (cond == MATCH) {

View File

@ -69,6 +69,8 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of
int32_t blkOffset = offset % kBlockSize; int32_t blkOffset = offset % kBlockSize;
int32_t blkLeft = kBlockSize - blkOffset; int32_t blkLeft = kBlockSize - blkOffset;
if (offset >= ctx->file.size) return 0;
do { do {
char key[128] = {0}; char key[128] = {0};
idxGenLRUKey(key, ctx->file.buf, blkId); idxGenLRUKey(key, ctx->file.buf, blkId);
@ -79,6 +81,15 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of
nread = TMIN(blkLeft, len); nread = TMIN(blkLeft, len);
memcpy(buf + total, blk->buf + blkOffset, nread); memcpy(buf + total, blk->buf + blkOffset, nread);
taosLRUCacheRelease(ctx->lru, h, false); taosLRUCacheRelease(ctx->lru, h, false);
} else {
int32_t left = ctx->file.size - offset;
if (left < kBlockSize) {
nread = TMIN(left, len);
int32_t bytes = taosPReadFile(ctx->file.pFile, buf + total, nread, offset);
assert(bytes == nread);
total += bytes;
return total;
} else { } else {
int32_t cacheMemSize = sizeof(SDataBlock) + kBlockSize; int32_t cacheMemSize = sizeof(SDataBlock) + kBlockSize;
@ -100,6 +111,7 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of
return -1; return -1;
} }
} }
}
total += nread; total += nread;
len -= nread; len -= nread;
offset += nread; offset += nread;
@ -146,9 +158,7 @@ IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int
} else { } else {
ctx->file.pFile = taosOpenFile(path, TD_FILE_READ); ctx->file.pFile = taosOpenFile(path, TD_FILE_READ);
int64_t size = 0;
taosFStatFile(ctx->file.pFile, &ctx->file.size, NULL); taosFStatFile(ctx->file.pFile, &ctx->file.size, NULL);
ctx->file.size = (int)size;
#ifdef USE_MMAP #ifdef USE_MMAP
ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size); ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size);
#endif #endif

View File

@ -172,9 +172,9 @@ TEST_F(JsonEnv, testWriteMillonData) {
{ {
std::string colName("voltagefdadfa"); std::string colName("voltagefdadfa");
std::string colVal("abxxxxxxxxxxxx"); std::string colVal("abxxxxxxxxxxxx");
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10000; i++) {
colVal[i % colVal.size()] = '0' + i % 128; colVal[i % colVal.size()] = '0' + i % 128;
for (size_t i = 0; i < 100; i++) { for (size_t i = 0; i < 10; i++) {
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size()); colVal.c_str(), colVal.size());

View File

@ -254,6 +254,7 @@ const char* nodesNodeName(ENodeType type) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
return "PhysiStreamSemiInterval"; return "PhysiStreamSemiInterval";
case QUERY_NODE_PHYSICAL_PLAN_FILL: case QUERY_NODE_PHYSICAL_PLAN_FILL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
return "PhysiFill"; return "PhysiFill";
case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION:
return "PhysiSessionWindow"; return "PhysiSessionWindow";
@ -4635,6 +4636,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
return physiIntervalNodeToJson(pObj, pJson); return physiIntervalNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_FILL: case QUERY_NODE_PHYSICAL_PLAN_FILL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
return physiFillNodeToJson(pObj, pJson); return physiFillNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION:
@ -4788,6 +4790,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
return jsonToPhysiIntervalNode(pJson, pObj); return jsonToPhysiIntervalNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_FILL: case QUERY_NODE_PHYSICAL_PLAN_FILL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
return jsonToPhysiFillNode(pJson, pObj); return jsonToPhysiFillNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION:

View File

@ -3633,6 +3633,7 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
code = physiIntervalNodeToMsg(pObj, pEncoder); code = physiIntervalNodeToMsg(pObj, pEncoder);
break; break;
case QUERY_NODE_PHYSICAL_PLAN_FILL: case QUERY_NODE_PHYSICAL_PLAN_FILL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
code = physiFillNodeToMsg(pObj, pEncoder); code = physiFillNodeToMsg(pObj, pEncoder);
break; break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION:
@ -3770,6 +3771,7 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) {
code = msgToPhysiIntervalNode(pDecoder, pObj); code = msgToPhysiIntervalNode(pDecoder, pObj);
break; break;
case QUERY_NODE_PHYSICAL_PLAN_FILL: case QUERY_NODE_PHYSICAL_PLAN_FILL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
code = msgToPhysiFillNode(pDecoder, pObj); code = msgToPhysiFillNode(pDecoder, pObj);
break; break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION:

View File

@ -511,6 +511,7 @@ SNode* nodesMakeNode(ENodeType type) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
return makeNode(type, sizeof(SStreamSemiIntervalPhysiNode)); return makeNode(type, sizeof(SStreamSemiIntervalPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_FILL: case QUERY_NODE_PHYSICAL_PLAN_FILL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
return makeNode(type, sizeof(SFillPhysiNode)); return makeNode(type, sizeof(SFillPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION:
return makeNode(type, sizeof(SSessionWinodwPhysiNode)); return makeNode(type, sizeof(SSessionWinodwPhysiNode));
@ -1156,7 +1157,8 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode);
break; break;
case QUERY_NODE_PHYSICAL_PLAN_FILL: { case QUERY_NODE_PHYSICAL_PLAN_FILL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: {
SFillPhysiNode* pPhyNode = (SFillPhysiNode*)pNode; SFillPhysiNode* pPhyNode = (SFillPhysiNode*)pNode;
destroyPhysiNode((SPhysiNode*)pPhyNode); destroyPhysiNode((SPhysiNode*)pPhyNode);
nodesDestroyList(pPhyNode->pFillExprs); nodesDestroyList(pPhyNode->pFillExprs);

View File

@ -407,6 +407,10 @@ static int32_t collectMetaKeyFromShowTags(SCollectMetaKeyCxt* pCxt, SShowStmt* p
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache); code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache);
} }
if (TSDB_CODE_SUCCESS == code && NULL != pStmt->pTbName) {
code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal,
((SValueNode*)pStmt->pTbName)->literal, pCxt->pMetaCache);
}
return code; return code;
} }

View File

@ -2141,10 +2141,114 @@ static bool sysTableFromVnode(const char* pTable) {
static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES); } static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES); }
static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTargetName, SName* pName,
SArray** pVgroupList) {
if (0 == pTargetName->type) {
return getDBVgInfoImpl(pCxt, pName, pVgroupList);
}
if (TSDB_DB_NAME_T == pTargetName->type) {
return getDBVgInfoImpl(pCxt, pTargetName, pVgroupList);
}
SVgroupInfo vgInfo = {0};
int32_t code = getTableHashVgroupImpl(pCxt, pTargetName, &vgInfo);
if (TSDB_CODE_SUCCESS == code) {
*pVgroupList = taosArrayInit(1, sizeof(SVgroupInfo));
if (NULL == *pVgroupList) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
if (TSDB_CODE_SUCCESS == code) {
taosArrayPush(*pVgroupList, &vgInfo);
}
return code;
}
static int32_t getTagsTableTargetNameFromOp(STranslateContext* pCxt, SOperatorNode* pOper, SName* pName) {
if (OP_TYPE_EQUAL != pOper->opType) {
return TSDB_CODE_SUCCESS;
}
SColumnNode* pCol = NULL;
SValueNode* pVal = NULL;
if (QUERY_NODE_COLUMN == nodeType(pOper->pLeft)) {
pCol = (SColumnNode*)pOper->pLeft;
} else if (QUERY_NODE_VALUE == nodeType(pOper->pLeft)) {
pVal = (SValueNode*)pOper->pLeft;
}
if (QUERY_NODE_COLUMN == nodeType(pOper->pRight)) {
pCol = (SColumnNode*)pOper->pRight;
} else if (QUERY_NODE_VALUE == nodeType(pOper->pRight)) {
pVal = (SValueNode*)pOper->pRight;
}
if (NULL == pCol || NULL == pVal) {
return TSDB_CODE_SUCCESS;
}
if (0 == strcmp(pCol->colName, "db_name")) {
return tNameSetDbName(pName, pCxt->pParseCxt->acctId, pVal->literal, strlen(pVal->literal));
} else if (0 == strcmp(pCol->colName, "table_name")) {
return tNameAddTbName(pName, pVal->literal, strlen(pVal->literal));
}
return TSDB_CODE_SUCCESS;
}
static void getTagsTableTargetObjName(STranslateContext* pCxt, SNode* pNode, SName* pName) {
if (QUERY_NODE_OPERATOR == nodeType(pNode)) {
getTagsTableTargetNameFromOp(pCxt, (SOperatorNode*)pNode, pName);
}
}
static int32_t getTagsTableTargetNameFromCond(STranslateContext* pCxt, SLogicConditionNode* pCond, SName* pName) {
if (LOGIC_COND_TYPE_AND != pCond->condType) {
return TSDB_CODE_SUCCESS;
}
SNode* pNode = NULL;
FOREACH(pNode, pCond->pParameterList) { getTagsTableTargetObjName(pCxt, pNode, pName); }
if ('\0' == pName->dbname[0]) {
pName->type = 0;
}
return TSDB_CODE_SUCCESS;
}
static int32_t getTagsTableTargetName(STranslateContext* pCxt, SNode* pWhere, SName* pName) {
if (NULL == pWhere) {
return TSDB_CODE_SUCCESS;
}
if (QUERY_NODE_OPERATOR == nodeType(pWhere)) {
return getTagsTableTargetNameFromOp(pCxt, (SOperatorNode*)pWhere, pName);
}
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pWhere)) {
return getTagsTableTargetNameFromCond(pCxt, (SLogicConditionNode*)pWhere, pName);
}
return TSDB_CODE_SUCCESS;
}
static int32_t getTagsTableVgroupList(STranslateContext* pCxt, SName* pName, SArray** pVgroupList) {
if (!isSelectStmt(pCxt->pCurrStmt)) {
return TSDB_CODE_SUCCESS;
}
SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt;
SName targetName = {0};
int32_t code = getTagsTableTargetName(pCxt, pSelect->pWhere, &targetName);
if (TSDB_CODE_SUCCESS == code) {
code = getTagsTableVgroupListImpl(pCxt, &targetName, pName, pVgroupList);
}
return code;
}
static int32_t setVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { static int32_t setVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SArray* vgroupList = NULL; SArray* vgroupList = NULL;
if ('\0' != pRealTable->qualDbName[0]) { if (0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS)) {
code = getTagsTableVgroupList(pCxt, pName, &vgroupList);
} else if ('\0' != pRealTable->qualDbName[0]) {
if (0 != strcmp(pRealTable->qualDbName, TSDB_INFORMATION_SCHEMA_DB)) { if (0 != strcmp(pRealTable->qualDbName, TSDB_INFORMATION_SCHEMA_DB)) {
code = getDBVgInfo(pCxt, pRealTable->qualDbName, &vgroupList); code = getDBVgInfo(pCxt, pRealTable->qualDbName, &vgroupList);
} }
@ -2152,14 +2256,12 @@ static int32_t setVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName,
code = getDBVgInfoImpl(pCxt, pName, &vgroupList); code = getDBVgInfoImpl(pCxt, pName, &vgroupList);
} }
if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.dbName, TSDB_INFORMATION_SCHEMA_DB) && if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS) &&
0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS) && isSelectStmt(pCxt->pCurrStmt) && isSelectStmt(pCxt->pCurrStmt) && 0 == taosArrayGetSize(vgroupList)) {
0 == taosArrayGetSize(vgroupList)) {
((SSelectStmt*)pCxt->pCurrStmt)->isEmptyResult = true; ((SSelectStmt*)pCxt->pCurrStmt)->isEmptyResult = true;
} }
if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.dbName, TSDB_INFORMATION_SCHEMA_DB) && if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES)) {
0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES)) {
code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &vgroupList); code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &vgroupList);
} }

View File

@ -85,6 +85,10 @@ void generateInformationSchema(MockCatalogService* mcs) {
.addColumn("dnode_id", TSDB_DATA_TYPE_INT) .addColumn("dnode_id", TSDB_DATA_TYPE_INT)
.addColumn("dnode_ep", TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN) .addColumn("dnode_ep", TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN)
.done(); .done();
mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TAGS, TSDB_SYSTEM_TABLE, 2)
.addColumn("table_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN)
.addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN)
.done();
} }
void generatePerformanceSchema(MockCatalogService* mcs) { void generatePerformanceSchema(MockCatalogService* mcs) {

View File

@ -196,6 +196,12 @@ TEST_F(ParserShowToUseTest, showTableDistributed) {
run("SHOW TABLE DISTRIBUTED st1"); run("SHOW TABLE DISTRIBUTED st1");
} }
TEST_F(ParserShowToUseTest, showTags) {
useDb("root", "test");
run("SHOW TAGS FROM st1s1");
}
// todo SHOW topics // todo SHOW topics
TEST_F(ParserShowToUseTest, showUsers) { TEST_F(ParserShowToUseTest, showUsers) {
@ -213,9 +219,9 @@ TEST_F(ParserShowToUseTest, showVariables) {
TEST_F(ParserShowToUseTest, showVgroups) { TEST_F(ParserShowToUseTest, showVgroups) {
useDb("root", "test"); useDb("root", "test");
run("SHOW vgroups"); run("SHOW VGROUPS");
run("SHOW test.vgroups"); run("SHOW test.VGROUPS");
} }
TEST_F(ParserShowToUseTest, showVnodes) { TEST_F(ParserShowToUseTest, showVnodes) {

View File

@ -86,6 +86,7 @@ static void parseArg(int argc, char* argv[]) {
{"dump", no_argument, NULL, 'd'}, {"dump", no_argument, NULL, 'd'},
{"async", required_argument, NULL, 'a'}, {"async", required_argument, NULL, 'a'},
{"skipSql", required_argument, NULL, 's'}, {"skipSql", required_argument, NULL, 's'},
{"log", required_argument, NULL, 'l'},
{0, 0, 0, 0} {0, 0, 0, 0}
}; };
// clang-format on // clang-format on
@ -100,6 +101,9 @@ static void parseArg(int argc, char* argv[]) {
case 's': case 's':
setSkipSqlNum(optarg); setSkipSqlNum(optarg);
break; break;
case 'l':
setLogLevel(optarg);
break;
default: default:
break; break;
} }

View File

@ -1409,7 +1409,9 @@ static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChi
static int32_t createFillPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SFillLogicNode* pFillNode, static int32_t createFillPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SFillLogicNode* pFillNode,
SPhysiNode** pPhyNode) { SPhysiNode** pPhyNode) {
SFillPhysiNode* pFill = (SFillPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pFillNode, QUERY_NODE_PHYSICAL_PLAN_FILL); SFillPhysiNode* pFill = (SFillPhysiNode*)makePhysiNode(
pCxt, (SLogicNode*)pFillNode,
pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL : QUERY_NODE_PHYSICAL_PLAN_FILL);
if (NULL == pFill) { if (NULL == pFill) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }

View File

@ -84,6 +84,8 @@ TEST_F(PlanOtherTest, show) {
run("SHOW TABLE DISTRIBUTED st1"); run("SHOW TABLE DISTRIBUTED st1");
run("SHOW DNODE 1 VARIABLES"); run("SHOW DNODE 1 VARIABLES");
run("SHOW TAGS FROM st1s1");
} }
TEST_F(PlanOtherTest, delete) { TEST_F(PlanOtherTest, delete) {

View File

@ -107,9 +107,12 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
QW_TASK_DLOG("start to execTask, loopIdx:%d", i++); QW_TASK_DLOG("start to execTask, loopIdx:%d", i++);
// if *taskHandle is NULL, it's killed right now // if *taskHandle is NULL, it's killed right now
bool hasMore = false;
if (taskHandle) { if (taskHandle) {
qwDbgSimulateSleep(); qwDbgSimulateSleep();
code = qExecTaskOpt(taskHandle, pResList, &useconds, &localFetch);
code = qExecTaskOpt(taskHandle, pResList, &useconds, &hasMore, &localFetch);
if (code) { if (code) {
if (code != TSDB_CODE_OPS_NOT_SUPPORT) { if (code != TSDB_CODE_OPS_NOT_SUPPORT) {
QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code)); QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code));
@ -122,20 +125,8 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
++execNum; ++execNum;
if (taosArrayGetSize(pResList) == 0) { size_t numOfResBlock = taosArrayGetSize(pResList);
QW_TASK_DLOG("qExecTask end with empty res, useconds:%" PRIu64, useconds); for (int32_t j = 0; j < numOfResBlock; ++j) {
dsEndPut(sinkHandle, useconds);
QW_ERR_JRET(qwHandleTaskComplete(QW_FPARAMS(), ctx));
if (queryStop) {
*queryStop = true;
}
break;
}
for (int32_t j = 0; j < taosArrayGetSize(pResList); ++j) {
SSDataBlock *pRes = taosArrayGetP(pResList, j); SSDataBlock *pRes = taosArrayGetP(pResList, j);
ASSERT(pRes->info.rows > 0); ASSERT(pRes->info.rows > 0);
@ -149,6 +140,23 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", pRes->info.rows, qcontinue); QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", pRes->info.rows, qcontinue);
} }
if (numOfResBlock == 0 || (hasMore == false)) {
if (numOfResBlock == 0) {
QW_TASK_DLOG("qExecTask end with empty res, useconds:%" PRIu64, useconds);
} else {
QW_TASK_DLOG("qExecTask done", "");
}
dsEndPut(sinkHandle, useconds);
QW_ERR_JRET(qwHandleTaskComplete(QW_FPARAMS(), ctx));
if (queryStop) {
*queryStop = true;
}
break;
}
if (!qcontinue) { if (!qcontinue) {
if (queryStop) { if (queryStop) {
*queryStop = true; *queryStop = true;

View File

@ -47,8 +47,7 @@ void schUpdateJobErrCode(SSchJob *pJob, int32_t errCode) {
return; return;
_return: _return:
SCH_JOB_DLOG("job errCode updated to %s", tstrerror(errCode));
SCH_JOB_DLOG("job errCode updated to %x - %s", errCode, tstrerror(errCode));
} }
bool schJobDone(SSchJob *pJob) { bool schJobDone(SSchJob *pJob) {
@ -491,7 +490,7 @@ int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) {
int32_t code = atomic_load_32(&pJob->errCode); int32_t code = atomic_load_32(&pJob->errCode);
if (code) { if (code) {
SCH_JOB_DLOG("job failed with error: %s", tstrerror(code)); SCH_JOB_DLOG("job failed with error %s", tstrerror(code));
} }
schPostJobRes(pJob, 0); schPostJobRes(pJob, 0);

View File

@ -29,7 +29,7 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath) {
if (!specPath) { if (!specPath) {
sprintf(statePath, "%s/%d", path, pTask->taskId); sprintf(statePath, "%s/%d", path, pTask->taskId);
} else { } else {
memcpy(statePath, path, 300); strncpy(statePath, path, 300);
} }
if (tdbOpen(statePath, 4096, 256, &pState->db) < 0) { if (tdbOpen(statePath, 4096, 256, &pState->db) < 0) {
goto _err; goto _err;
@ -40,6 +40,11 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath) {
goto _err; goto _err;
} }
// todo refactor
if (tdbTbOpen("func.state.db", sizeof(SWinKey), -1, SWinKeyCmpr, pState->db, &pState->pFillStateDb) < 0) {
goto _err;
}
if (tdbTbOpen("func.state.db", sizeof(STupleKey), -1, STupleKeyCmpr, pState->db, &pState->pFuncStateDb) < 0) { if (tdbTbOpen("func.state.db", sizeof(STupleKey), -1, STupleKeyCmpr, pState->db, &pState->pFuncStateDb) < 0) {
goto _err; goto _err;
} }
@ -55,6 +60,7 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath) {
_err: _err:
tdbTbClose(pState->pStateDb); tdbTbClose(pState->pStateDb);
tdbTbClose(pState->pFuncStateDb); tdbTbClose(pState->pFuncStateDb);
tdbTbClose(pState->pFillStateDb);
tdbClose(pState->db); tdbClose(pState->db);
taosMemoryFree(pState); taosMemoryFree(pState);
return NULL; return NULL;
@ -64,6 +70,7 @@ void streamStateClose(SStreamState* pState) {
tdbCommit(pState->db, &pState->txn); tdbCommit(pState->db, &pState->txn);
tdbTbClose(pState->pStateDb); tdbTbClose(pState->pStateDb);
tdbTbClose(pState->pFuncStateDb); tdbTbClose(pState->pFuncStateDb);
tdbTbClose(pState->pFillStateDb);
tdbClose(pState->db); tdbClose(pState->db);
taosMemoryFree(pState); taosMemoryFree(pState);
@ -126,14 +133,30 @@ int32_t streamStateFuncDel(SStreamState* pState, const STupleKey* key) {
int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) { int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) {
return tdbTbUpsert(pState->pStateDb, key, sizeof(SWinKey), value, vLen, &pState->txn); return tdbTbUpsert(pState->pStateDb, key, sizeof(SWinKey), value, vLen, &pState->txn);
} }
// todo refactor
int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) {
return tdbTbUpsert(pState->pFillStateDb, key, sizeof(SWinKey), value, vLen, &pState->txn);
}
int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) {
return tdbTbGet(pState->pStateDb, key, sizeof(SWinKey), pVal, pVLen); return tdbTbGet(pState->pStateDb, key, sizeof(SWinKey), pVal, pVLen);
} }
// todo refactor
int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) {
return tdbTbGet(pState->pFillStateDb, key, sizeof(SWinKey), pVal, pVLen);
}
int32_t streamStateDel(SStreamState* pState, const SWinKey* key) { int32_t streamStateDel(SStreamState* pState, const SWinKey* key) {
return tdbTbDelete(pState->pStateDb, key, sizeof(SWinKey), &pState->txn); return tdbTbDelete(pState->pStateDb, key, sizeof(SWinKey), &pState->txn);
} }
// todo refactor
int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key) {
return tdbTbDelete(pState->pFillStateDb, key, sizeof(SWinKey), &pState->txn);
}
int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) {
// todo refactor // todo refactor
int32_t size = *pVLen; int32_t size = *pVLen;
@ -165,6 +188,31 @@ SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key) {
return pCur; return pCur;
} }
SStreamStateCur* streamStateFillGetCur(SStreamState* pState, const SWinKey* key) {
SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));
if (pCur == NULL) return NULL;
tdbTbcOpen(pState->pFillStateDb, &pCur->pCur, NULL);
int32_t c;
tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c);
if (c != 0) {
taosMemoryFree(pCur);
return NULL;
}
return pCur;
}
SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key) {
SStreamStateCur* pCur = streamStateFillGetCur(pState, key);
if (pCur) {
int32_t code = streamStateGetGroupKVByCur(pCur, key, NULL, 0);
if (code == 0) {
return pCur;
}
}
return NULL;
}
int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) {
const SWinKey* pKTmp = NULL; const SWinKey* pKTmp = NULL;
int32_t kLen; int32_t kLen;
@ -175,6 +223,17 @@ int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void**
return 0; return 0;
} }
int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) {
uint64_t groupId = pKey->groupId;
int32_t code = streamStateGetKVByCur(pCur, pKey, pVal, pVLen);
if (code == 0) {
if (pKey->groupId == groupId) {
return 0;
}
}
return -1;
}
int32_t streamStateSeekFirst(SStreamState* pState, SStreamStateCur* pCur) { int32_t streamStateSeekFirst(SStreamState* pState, SStreamStateCur* pCur) {
// //
return tdbTbcMoveToFirst(pCur->pCur); return tdbTbcMoveToFirst(pCur->pCur);
@ -185,12 +244,12 @@ int32_t streamStateSeekLast(SStreamState* pState, SStreamStateCur* pCur) {
return tdbTbcMoveToLast(pCur->pCur); return tdbTbcMoveToLast(pCur->pCur);
} }
SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key) { SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* key) {
SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));
if (pCur == NULL) { if (pCur == NULL) {
return NULL; return NULL;
} }
if (tdbTbcOpen(pState->pStateDb, &pCur->pCur, NULL) < 0) { if (tdbTbcOpen(pState->pFillStateDb, &pCur->pCur, NULL) < 0) {
taosMemoryFree(pCur); taosMemoryFree(pCur);
return NULL; return NULL;
} }
@ -211,12 +270,12 @@ SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key
return pCur; return pCur;
} }
SStreamStateCur* streamStateSeekKeyPrev(SStreamState* pState, const SWinKey* key) { SStreamStateCur* streamStateFillSeekKeyPrev(SStreamState* pState, const SWinKey* key) {
SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));
if (pCur == NULL) { if (pCur == NULL) {
return NULL; return NULL;
} }
if (tdbTbcOpen(pState->pStateDb, &pCur->pCur, NULL) < 0) { if (tdbTbcOpen(pState->pFillStateDb, &pCur->pCur, NULL) < 0) {
taosMemoryFree(pCur); taosMemoryFree(pCur);
return NULL; return NULL;
} }

View File

@ -32,7 +32,7 @@ uint64_t syncUtilAddr2U64(const char* host, uint16_t port);
void syncUtilU642Addr(uint64_t u64, char* host, size_t len, uint16_t* port); void syncUtilU642Addr(uint64_t u64, char* host, size_t len, uint16_t* port);
void syncUtilnodeInfo2EpSet(const SNodeInfo* pNodeInfo, SEpSet* pEpSet); void syncUtilnodeInfo2EpSet(const SNodeInfo* pNodeInfo, SEpSet* pEpSet);
void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet); void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet);
void syncUtilnodeInfo2raftId(const SNodeInfo* pNodeInfo, SyncGroupId vgId, SRaftId* raftId); bool syncUtilnodeInfo2raftId(const SNodeInfo* pNodeInfo, SyncGroupId vgId, SRaftId* raftId);
bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2); bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2);
bool syncUtilEmptyId(const SRaftId* pId); bool syncUtilEmptyId(const SRaftId* pId);

View File

@ -20,7 +20,10 @@
SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode) { SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode) {
SSyncIndexMgr *pSyncIndexMgr = taosMemoryMalloc(sizeof(SSyncIndexMgr)); SSyncIndexMgr *pSyncIndexMgr = taosMemoryMalloc(sizeof(SSyncIndexMgr));
ASSERT(pSyncIndexMgr != NULL); if (pSyncIndexMgr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
memset(pSyncIndexMgr, 0, sizeof(SSyncIndexMgr)); memset(pSyncIndexMgr, 0, sizeof(SSyncIndexMgr));
pSyncIndexMgr->replicas = &(pSyncNode->replicasId); pSyncIndexMgr->replicas = &(pSyncNode->replicasId);

View File

@ -51,15 +51,17 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths);
int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg);
int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg);
// life cycle
static void syncFreeNode(void* param);
// --------------------------------- // ---------------------------------
static void syncNodeFreeCb(void *param) {
syncNodeClose(param);
param = NULL;
}
int32_t syncInit() { int32_t syncInit() {
int32_t ret = 0; int32_t ret = 0;
if (!syncEnvIsStart()) { if (!syncEnvIsStart()) {
tsNodeRefId = taosOpenRef(200, syncFreeNode); tsNodeRefId = taosOpenRef(200, syncNodeFreeCb);
if (tsNodeRefId < 0) { if (tsNodeRefId < 0) {
sError("failed to init node ref"); sError("failed to init node ref");
syncCleanUp(); syncCleanUp();
@ -86,11 +88,15 @@ void syncCleanUp() {
int64_t syncOpen(const SSyncInfo* pSyncInfo) { int64_t syncOpen(const SSyncInfo* pSyncInfo) {
SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo); SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo);
ASSERT(pSyncNode != NULL); if (pSyncNode == NULL) {
sError("failed to open sync node. vgId:%d", pSyncInfo->vgId);
return -1;
}
pSyncNode->rid = taosAddRef(tsNodeRefId, pSyncNode); pSyncNode->rid = taosAddRef(tsNodeRefId, pSyncNode);
if (pSyncNode->rid < 0) { if (pSyncNode->rid < 0) {
syncFreeNode(pSyncNode); syncNodeClose(pSyncNode);
pSyncNode = NULL;
return -1; return -1;
} }
@ -136,11 +142,9 @@ void syncStartStandBy(int64_t rid) {
void syncStop(int64_t rid) { void syncStop(int64_t rid) {
SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
if (pSyncNode == NULL) return; if (pSyncNode == NULL) return;
int32_t vgId = pSyncNode->vgId; int32_t vgId = pSyncNode->vgId;
syncNodeClose(pSyncNode);
taosReleaseRef(tsNodeRefId, pSyncNode->rid); taosReleaseRef(tsNodeRefId, pSyncNode->rid);
taosRemoveRef(tsNodeRefId, rid); taosRemoveRef(tsNodeRefId, rid);
sDebug("vgId:%d, sync rid:%" PRId64 " is removed from rsetId:%" PRId64, vgId, rid, tsNodeRefId); sDebug("vgId:%d, sync rid:%" PRId64 " is removed from rsetId:%" PRId64, vgId, rid, tsNodeRefId);
} }
@ -210,7 +214,7 @@ int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg
if (!syncNodeCheckNewConfig(pSyncNode, pNewCfg)) { if (!syncNodeCheckNewConfig(pSyncNode, pNewCfg)) {
taosReleaseRef(tsNodeRefId, pSyncNode->rid); taosReleaseRef(tsNodeRefId, pSyncNode->rid);
terrno = TSDB_CODE_SYN_NEW_CONFIG_ERROR; terrno = TSDB_CODE_SYN_NEW_CONFIG_ERROR;
sError("syncNodeCheckNewConfig error"); sError("invalid new config. vgId:%d", pSyncNode->vgId);
return -1; return -1;
} }
@ -237,7 +241,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) {
if (!syncNodeCheckNewConfig(pSyncNode, pNewCfg)) { if (!syncNodeCheckNewConfig(pSyncNode, pNewCfg)) {
taosReleaseRef(tsNodeRefId, pSyncNode->rid); taosReleaseRef(tsNodeRefId, pSyncNode->rid);
terrno = TSDB_CODE_SYN_NEW_CONFIG_ERROR; terrno = TSDB_CODE_SYN_NEW_CONFIG_ERROR;
sError("syncNodeCheckNewConfig error"); sError("invalid new config. vgId:%d", pSyncNode->vgId);
return -1; return -1;
} }
@ -941,16 +945,18 @@ _END:
SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
SSyncInfo* pSyncInfo = (SSyncInfo*)pOldSyncInfo; SSyncInfo* pSyncInfo = (SSyncInfo*)pOldSyncInfo;
SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); SSyncNode* pSyncNode = (SSyncNode*)taosMemoryCalloc(1, sizeof(SSyncNode));
ASSERT(pSyncNode != NULL); if (pSyncNode == NULL) {
memset(pSyncNode, 0, sizeof(SSyncNode)); terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
}
int32_t ret = 0; int32_t ret = 0;
if (!taosDirExist((char*)(pSyncInfo->path))) { if (!taosDirExist((char*)(pSyncInfo->path))) {
if (taosMkDir(pSyncInfo->path) != 0) { if (taosMkDir(pSyncInfo->path) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
sError("failed to create dir:%s since %s", pSyncInfo->path, terrstr()); sError("failed to create dir:%s since %s", pSyncInfo->path, terrstr());
return NULL; goto _error;
} }
} }
@ -963,15 +969,21 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
meta.lastConfigIndex = SYNC_INDEX_INVALID; meta.lastConfigIndex = SYNC_INDEX_INVALID;
meta.batchSize = pSyncInfo->batchSize; meta.batchSize = pSyncInfo->batchSize;
ret = raftCfgCreateFile((SSyncCfg*)&(pSyncInfo->syncCfg), meta, pSyncNode->configPath); ret = raftCfgCreateFile((SSyncCfg*)&(pSyncInfo->syncCfg), meta, pSyncNode->configPath);
ASSERT(ret == 0); if (ret != 0) {
sError("failed to create raft cfg file. configPath: %s", pSyncNode->configPath);
goto _error;
}
} else { } else {
// update syncCfg by raft_config.json // update syncCfg by raft_config.json
pSyncNode->pRaftCfg = raftCfgOpen(pSyncNode->configPath); pSyncNode->pRaftCfg = raftCfgOpen(pSyncNode->configPath);
ASSERT(pSyncNode->pRaftCfg != NULL); if (pSyncNode->pRaftCfg == NULL) {
sError("failed to open raft cfg file. path:%s", pSyncNode->configPath);
goto _error;
}
pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg; pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg;
raftCfgClose(pSyncNode->pRaftCfg); raftCfgClose(pSyncNode->pRaftCfg);
pSyncNode->pRaftCfg = NULL;
} }
// init by SSyncInfo // init by SSyncInfo
@ -988,11 +1000,17 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
// init raft config // init raft config
pSyncNode->pRaftCfg = raftCfgOpen(pSyncNode->configPath); pSyncNode->pRaftCfg = raftCfgOpen(pSyncNode->configPath);
ASSERT(pSyncNode->pRaftCfg != NULL); if (pSyncNode->pRaftCfg == NULL) {
sError("failed to open raft cfg file. path:%s", pSyncNode->configPath);
goto _error;
}
// init internal // init internal
pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex]; pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex];
syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId); if (!syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId)) {
sError("failed to determine my raft member id. vgId:%d", pSyncNode->vgId);
goto _error;
}
// init peersNum, peers, peersId // init peersNum, peers, peersId
pSyncNode->peersNum = pSyncNode->pRaftCfg->cfg.replicaNum - 1; pSyncNode->peersNum = pSyncNode->pRaftCfg->cfg.replicaNum - 1;
@ -1004,17 +1022,24 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
} }
} }
for (int i = 0; i < pSyncNode->peersNum; ++i) { for (int i = 0; i < pSyncNode->peersNum; ++i) {
syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i]); if (!syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i])) {
sError("failed to determine raft member id. vgId:%d, peer:%d", pSyncNode->vgId, i);
goto _error;
}
} }
// init replicaNum, replicasId // init replicaNum, replicasId
pSyncNode->replicaNum = pSyncNode->pRaftCfg->cfg.replicaNum; pSyncNode->replicaNum = pSyncNode->pRaftCfg->cfg.replicaNum;
for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) {
syncUtilnodeInfo2raftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]); if(!syncUtilnodeInfo2raftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i])) {
sError("failed to determine raft member id. vgId:%d, replica:%d", pSyncNode->vgId, i);
goto _error;
}
} }
// init raft algorithm // init raft algorithm
pSyncNode->pFsm = pSyncInfo->pFsm; pSyncNode->pFsm = pSyncInfo->pFsm;
pSyncInfo->pFsm = NULL;
pSyncNode->quorum = syncUtilQuorum(pSyncNode->pRaftCfg->cfg.replicaNum); pSyncNode->quorum = syncUtilQuorum(pSyncNode->pRaftCfg->cfg.replicaNum);
pSyncNode->leaderCache = EMPTY_RAFT_ID; pSyncNode->leaderCache = EMPTY_RAFT_ID;
@ -1047,29 +1072,50 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
// init TLA+ server vars // init TLA+ server vars
pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER;
pSyncNode->pRaftStore = raftStoreOpen(pSyncNode->raftStorePath); pSyncNode->pRaftStore = raftStoreOpen(pSyncNode->raftStorePath);
ASSERT(pSyncNode->pRaftStore != NULL); if (pSyncNode->pRaftStore == NULL) {
sError("failed to open raft store. path: %s", pSyncNode->raftStorePath);
goto _error;
}
// init TLA+ candidate vars // init TLA+ candidate vars
pSyncNode->pVotesGranted = voteGrantedCreate(pSyncNode); pSyncNode->pVotesGranted = voteGrantedCreate(pSyncNode);
ASSERT(pSyncNode->pVotesGranted != NULL); if (pSyncNode->pVotesGranted == NULL) {
sError("failed to create VotesGranted. vgId:%d", pSyncNode->vgId);
goto _error;
}
pSyncNode->pVotesRespond = votesRespondCreate(pSyncNode); pSyncNode->pVotesRespond = votesRespondCreate(pSyncNode);
ASSERT(pSyncNode->pVotesRespond != NULL); if (pSyncNode->pVotesRespond == NULL) {
sError("failed to create VotesRespond. vgId:%d", pSyncNode->vgId);
goto _error;
}
// init TLA+ leader vars // init TLA+ leader vars
pSyncNode->pNextIndex = syncIndexMgrCreate(pSyncNode); pSyncNode->pNextIndex = syncIndexMgrCreate(pSyncNode);
ASSERT(pSyncNode->pNextIndex != NULL); if (pSyncNode->pNextIndex == NULL) {
sError("failed to create SyncIndexMgr. vgId:%d", pSyncNode->vgId);
goto _error;
}
pSyncNode->pMatchIndex = syncIndexMgrCreate(pSyncNode); pSyncNode->pMatchIndex = syncIndexMgrCreate(pSyncNode);
ASSERT(pSyncNode->pMatchIndex != NULL); if (pSyncNode->pMatchIndex == NULL) {
sError("failed to create SyncIndexMgr. vgId:%d", pSyncNode->vgId);
goto _error;
}
// init TLA+ log vars // init TLA+ log vars
pSyncNode->pLogStore = logStoreCreate(pSyncNode); pSyncNode->pLogStore = logStoreCreate(pSyncNode);
ASSERT(pSyncNode->pLogStore != NULL); if (pSyncNode->pLogStore == NULL) {
sError("failed to create SyncLogStore. vgId:%d", pSyncNode->vgId);
goto _error;
}
SyncIndex commitIndex = SYNC_INDEX_INVALID; SyncIndex commitIndex = SYNC_INDEX_INVALID;
if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpGetSnapshotInfo != NULL) {
SSnapshot snapshot = {0}; SSnapshot snapshot = {0};
int32_t code = pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); int32_t code = pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
ASSERT(code == 0); if (code != 0) {
sError("failed to get snapshot info. vgId:%d, code:%d", pSyncNode->vgId, code);
goto _error;
}
if (snapshot.lastApplyIndex > commitIndex) { if (snapshot.lastApplyIndex > commitIndex) {
commitIndex = snapshot.lastApplyIndex; commitIndex = snapshot.lastApplyIndex;
syncNodeEventLog(pSyncNode, "reset commit index by snapshot"); syncNodeEventLog(pSyncNode, "reset commit index by snapshot");
@ -1132,7 +1178,10 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
// tools // tools
pSyncNode->pSyncRespMgr = syncRespMgrCreate(pSyncNode, SYNC_RESP_TTL_MS); pSyncNode->pSyncRespMgr = syncRespMgrCreate(pSyncNode, SYNC_RESP_TTL_MS);
ASSERT(pSyncNode->pSyncRespMgr != NULL); if (pSyncNode->pSyncRespMgr == NULL) {
sError("failed to create SyncRespMgr. vgId:%d", pSyncNode->vgId);
goto _error;
}
// restore state // restore state
pSyncNode->restoreFinish = false; pSyncNode->restoreFinish = false;
@ -1162,6 +1211,15 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
syncNodeEventLog(pSyncNode, "sync open"); syncNodeEventLog(pSyncNode, "sync open");
return pSyncNode; return pSyncNode;
_error:
if (pSyncInfo->pFsm) {
taosMemoryFree(pSyncInfo->pFsm);
pSyncInfo->pFsm = NULL;
}
syncNodeClose(pSyncNode);
pSyncNode = NULL;
return NULL;
} }
void syncNodeMaybeUpdateCommitBySnapshot(SSyncNode* pSyncNode) { void syncNodeMaybeUpdateCommitBySnapshot(SSyncNode* pSyncNode) {
@ -1214,20 +1272,28 @@ void syncNodeStartStandBy(SSyncNode* pSyncNode) {
void syncNodeClose(SSyncNode* pSyncNode) { void syncNodeClose(SSyncNode* pSyncNode) {
syncNodeEventLog(pSyncNode, "sync close"); syncNodeEventLog(pSyncNode, "sync close");
if (pSyncNode == NULL) {
return;
}
int32_t ret; int32_t ret;
ASSERT(pSyncNode != NULL);
ret = raftStoreClose(pSyncNode->pRaftStore); ret = raftStoreClose(pSyncNode->pRaftStore);
ASSERT(ret == 0); ASSERT(ret == 0);
syncRespMgrDestroy(pSyncNode->pSyncRespMgr); syncRespMgrDestroy(pSyncNode->pSyncRespMgr);
pSyncNode->pSyncRespMgr = NULL;
voteGrantedDestroy(pSyncNode->pVotesGranted); voteGrantedDestroy(pSyncNode->pVotesGranted);
pSyncNode->pVotesGranted = NULL;
votesRespondDestory(pSyncNode->pVotesRespond); votesRespondDestory(pSyncNode->pVotesRespond);
pSyncNode->pVotesRespond = NULL;
syncIndexMgrDestroy(pSyncNode->pNextIndex); syncIndexMgrDestroy(pSyncNode->pNextIndex);
pSyncNode->pNextIndex = NULL;
syncIndexMgrDestroy(pSyncNode->pMatchIndex); syncIndexMgrDestroy(pSyncNode->pMatchIndex);
pSyncNode->pMatchIndex = NULL;
logStoreDestory(pSyncNode->pLogStore); logStoreDestory(pSyncNode->pLogStore);
pSyncNode->pLogStore = NULL;
raftCfgClose(pSyncNode->pRaftCfg); raftCfgClose(pSyncNode->pRaftCfg);
pSyncNode->pRaftCfg = NULL;
syncNodeStopPingTimer(pSyncNode); syncNodeStopPingTimer(pSyncNode);
syncNodeStopElectTimer(pSyncNode); syncNodeStopElectTimer(pSyncNode);
@ -1249,8 +1315,7 @@ void syncNodeClose(SSyncNode* pSyncNode) {
pSyncNode->pNewNodeReceiver = NULL; pSyncNode->pNewNodeReceiver = NULL;
} }
// free memory in syncFreeNode taosMemoryFree(pSyncNode);
// taosMemoryFree(pSyncNode);
} }
// option // option
@ -2534,7 +2599,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) {
return; return;
} }
} else { } else {
sError("syncNodeEqHeartbeatTimer FpEqMsg is NULL"); sError("vgId:%d, enqueue msg cb ptr (i.e. FpEqMsg) not set.", pSyncNode->vgId);
} }
syncTimeoutDestroy(pSyncMsg); syncTimeoutDestroy(pSyncMsg);
@ -2774,14 +2839,6 @@ int32_t syncNodeOnClientRequestBatchCb(SSyncNode* ths, SyncClientRequestBatch* p
return 0; return 0;
} }
static void syncFreeNode(void* param) {
SSyncNode* pNode = param;
// inner object already free
// syncNodePrint2((char*)"==syncFreeNode==", pNode);
taosMemoryFree(pNode);
}
const char* syncStr(ESyncState state) { const char* syncStr(ESyncState state) {
switch (state) { switch (state) {
case TAOS_SYNC_STATE_FOLLOWER: case TAOS_SYNC_STATE_FOLLOWER:

View File

@ -364,8 +364,6 @@ int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path) {
int32_t sysErr = errno; int32_t sysErr = errno;
const char *sysErrStr = strerror(errno); const char *sysErrStr = strerror(errno);
sError("create raft cfg file error, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", err, err, errStr, sysErr, sysErrStr); sError("create raft cfg file error, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", err, err, errStr, sysErr, sysErrStr);
ASSERT(0);
return -1; return -1;
} }

View File

@ -28,7 +28,7 @@ SRaftStore *raftStoreOpen(const char *path) {
SRaftStore *pRaftStore = taosMemoryMalloc(sizeof(SRaftStore)); SRaftStore *pRaftStore = taosMemoryMalloc(sizeof(SRaftStore));
if (pRaftStore == NULL) { if (pRaftStore == NULL) {
sError("raftStoreOpen malloc error"); terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
memset(pRaftStore, 0, sizeof(*pRaftStore)); memset(pRaftStore, 0, sizeof(*pRaftStore));
@ -72,7 +72,9 @@ static int32_t raftStoreInit(SRaftStore *pRaftStore) {
} }
int32_t raftStoreClose(SRaftStore *pRaftStore) { int32_t raftStoreClose(SRaftStore *pRaftStore) {
ASSERT(pRaftStore != NULL); if (pRaftStore == NULL) {
return 0;
}
taosCloseFile(&pRaftStore->pFile); taosCloseFile(&pRaftStore->pFile);
taosMemoryFree(pRaftStore); taosMemoryFree(pRaftStore);

View File

@ -19,6 +19,10 @@
SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) { SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) {
SSyncRespMgr *pObj = (SSyncRespMgr *)taosMemoryMalloc(sizeof(SSyncRespMgr)); SSyncRespMgr *pObj = (SSyncRespMgr *)taosMemoryMalloc(sizeof(SSyncRespMgr));
if (pObj == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
memset(pObj, 0, sizeof(SSyncRespMgr)); memset(pObj, 0, sizeof(SSyncRespMgr));
pObj->pRespHash = pObj->pRespHash =

View File

@ -35,7 +35,10 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI
SSyncSnapshotSender *pSender = NULL; SSyncSnapshotSender *pSender = NULL;
if (condition) { if (condition) {
pSender = taosMemoryMalloc(sizeof(SSyncSnapshotSender)); pSender = taosMemoryMalloc(sizeof(SSyncSnapshotSender));
ASSERT(pSender != NULL); if (pSender == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
memset(pSender, 0, sizeof(*pSender)); memset(pSender, 0, sizeof(*pSender));
pSender->start = false; pSender->start = false;

View File

@ -26,7 +26,8 @@ uint64_t syncUtilAddr2U64(const char* host, uint16_t port) {
uint32_t hostU32 = taosGetIpv4FromFqdn(host); uint32_t hostU32 = taosGetIpv4FromFqdn(host);
if (hostU32 == (uint32_t)-1) { if (hostU32 == (uint32_t)-1) {
sError("Get IP address error"); sError("failed to resolve ipv4 addr. host:%s", host);
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return -1; return -1;
} }
@ -84,13 +85,18 @@ void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) {
addEpIntoEpSet(pEpSet, host, port); addEpIntoEpSet(pEpSet, host, port);
} }
void syncUtilnodeInfo2raftId(const SNodeInfo* pNodeInfo, SyncGroupId vgId, SRaftId* raftId) { bool syncUtilnodeInfo2raftId(const SNodeInfo* pNodeInfo, SyncGroupId vgId, SRaftId* raftId) {
uint32_t ipv4 = taosGetIpv4FromFqdn(pNodeInfo->nodeFqdn); uint32_t ipv4 = taosGetIpv4FromFqdn(pNodeInfo->nodeFqdn);
ASSERT(ipv4 != 0xFFFFFFFF); if (ipv4 == 0xFFFFFFFF || ipv4 == 1) {
sError("failed to resolve ipv4 addr. fqdn: %s", pNodeInfo->nodeFqdn);
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return false;
}
char ipbuf[128] = {0}; char ipbuf[128] = {0};
tinet_ntoa(ipbuf, ipv4); tinet_ntoa(ipbuf, ipv4);
raftId->addr = syncUtilAddr2U64(ipbuf, pNodeInfo->nodePort); raftId->addr = syncUtilAddr2U64(ipbuf, pNodeInfo->nodePort);
raftId->vgId = vgId; raftId->vgId = vgId;
return true;
} }
bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2) { bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2) {

View File

@ -24,7 +24,10 @@ static void voteGrantedClearVotes(SVotesGranted *pVotesGranted) {
SVotesGranted *voteGrantedCreate(SSyncNode *pSyncNode) { SVotesGranted *voteGrantedCreate(SSyncNode *pSyncNode) {
SVotesGranted *pVotesGranted = taosMemoryMalloc(sizeof(SVotesGranted)); SVotesGranted *pVotesGranted = taosMemoryMalloc(sizeof(SVotesGranted));
ASSERT(pVotesGranted != NULL); if (pVotesGranted == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
memset(pVotesGranted, 0, sizeof(SVotesGranted)); memset(pVotesGranted, 0, sizeof(SVotesGranted));
pVotesGranted->replicas = &(pSyncNode->replicasId); pVotesGranted->replicas = &(pSyncNode->replicasId);

View File

@ -151,15 +151,15 @@ int64_t taosQueueMemorySize(STaosQueue *queue) {
void *taosAllocateQitem(int32_t size, EQItype itype) { void *taosAllocateQitem(int32_t size, EQItype itype) {
STaosQnode *pNode = taosMemoryCalloc(1, sizeof(STaosQnode) + size); STaosQnode *pNode = taosMemoryCalloc(1, sizeof(STaosQnode) + size);
pNode->size = size;
pNode->itype = itype;
pNode->timestamp = taosGetTimestampUs();
if (pNode == NULL) { if (pNode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
pNode->size = size;
pNode->itype = itype;
pNode->timestamp = taosGetTimestampUs();
if (itype == RPC_QITEM) { if (itype == RPC_QITEM) {
int64_t alloced = atomic_add_fetch_64(&tsRpcQueueMemoryUsed, size); int64_t alloced = atomic_add_fetch_64(&tsRpcQueueMemoryUsed, size);
if (alloced > tsRpcQueueMemoryAllowed) { if (alloced > tsRpcQueueMemoryAllowed) {

View File

@ -20,8 +20,8 @@ static int32_t tUUIDSerialNo = 0;
int32_t tGenIdPI32(void) { int32_t tGenIdPI32(void) {
if (tUUIDHashId == 0) { if (tUUIDHashId == 0) {
char uid[64]; char uid[65] = {0};
int32_t code = taosGetSystemUUID(uid, tListLen(uid)); int32_t code = taosGetSystemUUID(uid, sizeof(uid));
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
} else { } else {

View File

@ -46,7 +46,7 @@ int32_t tQWorkerInit(SQWorkerPool *pool) {
void tQWorkerCleanup(SQWorkerPool *pool) { void tQWorkerCleanup(SQWorkerPool *pool) {
for (int32_t i = 0; i < pool->max; ++i) { for (int32_t i = 0; i < pool->max; ++i) {
SQWorker *worker = pool->workers + i; SQWorker *worker = pool->workers + i;
if (worker == NULL) continue; // if (worker == NULL) continue;
if (taosCheckPthreadValid(worker->thread)) { if (taosCheckPthreadValid(worker->thread)) {
taosQsetThreadResume(pool->qset); taosQsetThreadResume(pool->qset);
} }
@ -54,7 +54,7 @@ void tQWorkerCleanup(SQWorkerPool *pool) {
for (int32_t i = 0; i < pool->max; ++i) { for (int32_t i = 0; i < pool->max; ++i) {
SQWorker *worker = pool->workers + i; SQWorker *worker = pool->workers + i;
if (worker == NULL) continue; // if (worker == NULL) continue;
if (taosCheckPthreadValid(worker->thread)) { if (taosCheckPthreadValid(worker->thread)) {
taosThreadJoin(worker->thread, NULL); taosThreadJoin(worker->thread, NULL);
taosThreadClear(&worker->thread); taosThreadClear(&worker->thread);
@ -138,8 +138,8 @@ STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) {
} }
void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue) { void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue) {
taosCloseQueue(queue);
uDebug("worker:%s, queue:%p is freed", pool->name, queue); uDebug("worker:%s, queue:%p is freed", pool->name, queue);
taosCloseQueue(queue);
} }
int32_t tWWorkerInit(SWWorkerPool *pool) { int32_t tWWorkerInit(SWWorkerPool *pool) {
@ -283,8 +283,8 @@ STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) {
} }
void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue) { void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue) {
taosCloseQueue(queue);
uDebug("worker:%s, queue:%p is freed", pool->name, queue); uDebug("worker:%s, queue:%p is freed", pool->name, queue);
taosCloseQueue(queue);
} }
int32_t tSingleWorkerInit(SSingleWorker *pWorker, const SSingleWorkerCfg *pCfg) { int32_t tSingleWorkerInit(SSingleWorker *pWorker, const SSingleWorkerCfg *pCfg) {

View File

@ -413,13 +413,8 @@ if $data12 != 3 then
goto loop14 goto loop14
endi endi
return 1
sql drop stream if exists streams3;
sql drop database if exists test3;
sql drop database if exists test;
sql create database test3 vgroups 4; sql create database test3 vgroups 4;
sql create database test vgroups 1;
sql use test3; sql use test3;
sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1); sql create table t1 using st tags(1,1,1);
@ -435,7 +430,7 @@ sql delete from t1;
loop15: loop15:
sleep 200 sleep 200
sql select * from test.streamt2 order by c1, c2, c3; sql select * from test.streamt3 order by c1, c2, c3;
$loop_count = $loop_count + 1 $loop_count = $loop_count + 1
if $loop_count == 10 then if $loop_count == 10 then
@ -453,7 +448,7 @@ sql delete from t1 where ts > 100;
loop16: loop16:
sleep 200 sleep 200
sql select * from test.streamt2 order by c1, c2, c3; sql select * from test.streamt3 order by c1, c2, c3;
$loop_count = $loop_count + 1 $loop_count = $loop_count + 1
if $loop_count == 10 then if $loop_count == 10 then
@ -471,7 +466,7 @@ sql delete from st;
loop17: loop17:
sleep 200 sleep 200
sql select * from test.streamt2 order by c1, c2, c3; sql select * from test.streamt3 order by c1, c2, c3;
$loop_count = $loop_count + 1 $loop_count = $loop_count + 1
if $loop_count == 10 then if $loop_count == 10 then

Some files were not shown because too many files have changed in this diff Show More