xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into feat/TD-22970

This commit is contained in:
dm chen 2023-08-21 10:50:45 +08:00 committed by GitHub
parent e393a98ee9 54e1f69db4
commit dda51d8019
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
119 changed files with 11164 additions and 5093 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
docs/en/07-develop/03-insert-data/30-influxdb-line.mdx
View File

@ -35,11 +35,12 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0
:::note
- All the data in `tag_set` will be converted to NCHAR type automatically .
- Each data in `field_set` must be self-descriptive for its data type. For example 1.2f32 means a value 1.2 of float type. Without the "f" type suffix, it will be treated as type double.
- Multiple kinds of precision can be used for the `timestamp` field. Time precision can be from nanosecond (ns) to hour (h).
- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be created automatically. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored.
- All the data in `tag_set` will be converted to NCHAR type automatically
- Each data in `field_set` must be self-descriptive for its data type. For example 1.2f32 means a value 1.2 of float type. Without the "f" type suffix, it will be treated as type double
- Multiple kinds of precision can be used for the `timestamp` field. Time precision can be from nanosecond (ns) to hour (h)
- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be created automatically. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored
- It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.(smlDataFormat in taos.cfg default to false after version of 3.0.1.3, smlDataFormat is discarded since 3.0.3.0)
:::
For more details please refer to [InfluxDB Line Protocol](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) and [TDengine Schemaless](/reference/schemaless/#Schemaless-Line-Protocol)

9
docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx
View File

@ -34,6 +34,7 @@ meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3
```
- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored.
Please refer to [OpenTSDB Telnet API](http://opentsdb.net/docs/build/html/api_telnet/put.html) for more details.
## Examples
@ -68,11 +69,11 @@ Database changed.
taos> show stables;
name |
=================================
meters.current |
meters.voltage |
meters_current |
meters_voltage |
Query OK, 2 row(s) in set (0.002544s)
taos> select tbname, * from `meters.current`;
taos> select tbname, * from `meters_current`;
tbname | _ts | _value | groupid | location |
==================================================================================================================================
t_0e7bcfa21a02331c06764f275... | 2022-03-28 09:56:51.249 | 10.800000000 | 3 | California.LosAngeles |
@ -87,5 +88,5 @@ Query OK, 4 row(s) in set (0.005399s)
If you want query the data of `location=California.LosAngeles groupid=3`, here is the query SQL:
```sql
SELECT * FROM `meters.current` WHERE location = "California.LosAngeles" AND groupid = 3;
SELECT * FROM `meters_current` WHERE location = "California.LosAngeles" AND groupid = 3;
```

9
docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx
View File

@ -49,6 +49,7 @@ Please refer to [OpenTSDB HTTP API](http://opentsdb.net/docs/build/html/api_http
- In JSON protocol, strings will be converted to NCHAR type and numeric values will be converted to double type.
- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored.
:::
## Examples
@ -83,11 +84,11 @@ Database changed.
taos> show stables;
name |
=================================
meters.current |
meters.voltage |
meters_current |
meters_voltage |
Query OK, 2 row(s) in set (0.001954s)
taos> select * from `meters.current`;
taos> select * from `meters_current`;
_ts | _value | groupid | location |
===================================================================================================================
2022-03-28 09:56:51.249 | 10.300000000 | 2.000000000 | California.SanFrancisco |
@ -100,5 +101,5 @@ Query OK, 2 row(s) in set (0.004076s)
If you want query the data of "tags": {"location": "California.LosAngeles", "groupid": 1}, here is the query SQL:
```sql
SELECT * FROM `meters.current` WHERE location = "California.LosAngeles" AND groupid = 3;
SELECT * FROM `meters_current` WHERE location = "California.LosAngeles" AND groupid = 3;
```

5
docs/en/14-reference/03-connector/03-cpp.mdx
View File

@ -135,7 +135,7 @@ The following describes the basic API, synchronous API, asynchronous API, subscr
The base API is used to do things like create database connections and provide a runtime environment for the execution of other APIs.
- `void taos_init()`
- `int taos_init()`
Initializes the runtime environment. If the API is not actively called, the driver will automatically call the API when `taos_connect()` is called, so the program generally does not need to call it manually.
@ -454,6 +454,7 @@ In addition to writing data using the SQL method or the parameter binding API, w
- zero success,none zero failed, wrong message can be obtained through `char *tmq_err2str(int32_t code)`
- `int64_t tmq_committed(tmq_t *tmq, const char *pTopicName, int32_t vgId)`
**Function description**
- get the committed offset
@ -467,7 +468,7 @@ In addition to writing data using the SQL method or the parameter binding API, w
**Function description**
The commit interface is divided into two types, each with synchronous and asynchronous interfaces:
- The commit interface is divided into two types, each with synchronous and asynchronous interfaces:
- The first type: based on message submission, submit the progress in the message. If the message passes NULL, submit the current progress of all vgroups consumed by the current consumer: tmq_commit_sync/tmq_commit_async
- The second type: submit based on the offset of a Vgroup in a topic: tmq_commit_offset_sync/tmq_commit_offset_async

4
docs/en/14-reference/13-schemaless/13-schemaless.md
View File

@ -32,8 +32,8 @@ All data in tag_set is automatically converted to the NCHAR data type and does n
In the schemaless writing data line protocol, each data item in the field_set needs to be described with its data type. Let's explain in detail:
- If there are English double quotes on both sides, it indicates the BINARY(32) type. For example, `"abc"`.
- If there are double quotes on both sides and an L prefix, it means NCHAR(32) type. For example, `L"error message"`.
- If there are English double quotes on both sides, it indicates the VARCHAR(N) type. For example, `"abc"`.
- If there are double quotes on both sides and an L prefix, it means NCHAR(N) type. For example, `L"error message"`.
- Spaces, equals sign (=), comma (,), double quote ("), and backslash (\\) need to be escaped with a backslash (\\) in front. (All refer to the ASCII character). The rules are as follows:
| **Serial number** | **Element** | **Escape characters** |

7
docs/zh/07-develop/03-insert-data/30-influxdb-line.mdx
View File

@ -34,11 +34,12 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0
:::note
- tag_set 中的所有的数据自动转化为 NCHAR 数据类型;
- field_set 中的每个数据项都需要对自身的数据类型进行描述, 比如 1.2f32 代表 FLOAT 类型的数值 1.2, 如果不带类型后缀会被当作 DOUBLE 处理;
- timestamp 支持多种时间精度。写入数据的时候需要用参数指定时间精度，支持从小时到纳秒的 6 种时间精度。
- tag_set 中的所有的数据自动转化为 NCHAR 数据类型
- field_set 中的每个数据项都需要对自身的数据类型进行描述, 比如 1.2f32 代表 FLOAT 类型的数值 1.2, 如果不带类型后缀会被当作 DOUBLE 处理
- timestamp 支持多种时间精度。写入数据的时候需要用参数指定时间精度，支持从小时到纳秒的 6 种时间精度
- 为了提高写入的效率，默认假设同一个超级表中 field_set 的顺序是一样的（第一条数据包含所有的 field，后面的数据按照这个顺序），如果顺序不一样，需要配置参数 smlDataFormat 为 false，否则，数据写入按照相同顺序写入，库中数据会异常。（3.0.1.3 之后的版本 smlDataFormat 默认为 false，从3.0.3.0开始，该配置废弃） [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议)
- 默认产生的子表名是根据规则生成的唯一 ID 值。用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议)
:::
要了解更多可参考：[InfluxDB Line 协议官方文档](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) 和 [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议)

10
docs/zh/07-develop/03-insert-data/40-opentsdb-telnet.mdx
View File

@ -67,11 +67,11 @@ Database changed.
taos> SHOW STABLES;
name |
=================================
meters.current |
meters.voltage |
meters_current |
meters_voltage |
Query OK, 2 row(s) in set (0.002544s)
taos> SELECT TBNAME, * FROM `meters.current`;
taos> SELECT TBNAME, * FROM `meters_current`;
tbname | _ts | _value | groupid | location |
==================================================================================================================================
t_0e7bcfa21a02331c06764f275... | 2022-03-28 09:56:51.249 | 10.800000000 | 3 | California.LosAngeles |
@ -83,10 +83,10 @@ Query OK, 4 row(s) in set (0.005399s)
## SQL 查询示例
`meters.current` 是插入数据的超级表名。
`meters_current` 是插入数据的超级表名。
可以通过超级表的 TAG 来过滤数据，比如查询 `location=California.LosAngeles groupid=3` 可以通过如下 SQL：
```sql
SELECT * FROM `meters.current` WHERE location = "California.LosAngeles" AND groupid = 3;
SELECT * FROM `meters_current` WHERE location = "California.LosAngeles" AND groupid = 3;
```

9
docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx
View File

@ -48,6 +48,7 @@ OpenTSDB JSON 格式协议采用一个 JSON 字符串表示一行或多行数据
- 对于 JSON 格式协议，TDengine 并不会自动把所有标签转成 NCHAR 类型， 字符串将将转为 NCHAR 类型， 数值将同样转换为 DOUBLE 类型。
- 默认生成的子表名是根据规则生成的唯一 ID 值。用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 `"tags": { "host": "web02","dc": "lga","tname":"cpu1"}` 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。
:::
## 示例代码
@ -82,8 +83,8 @@ Database changed.
taos> SHOW STABLES;
name |
=================================
meters.current |
meters.voltage |
meters_current |
meters_voltage |
Query OK, 2 row(s) in set (0.001954s)
taos> SELECT * FROM `meters.current`;
@ -96,10 +97,10 @@ Query OK, 2 row(s) in set (0.004076s)
## SQL 查询示例
`meters.voltage` 是插入数据的超级表名。
`meters_voltage` 是插入数据的超级表名。
可以通过超级表的 TAG 来过滤数据，比如查询 `location=California.LosAngeles groupid=1` 可以通过如下 SQL：
```sql
SELECT * FROM `meters.current` WHERE location = "California.LosAngeles" AND groupid = 3;
SELECT * FROM `meters_current` WHERE location = "California.LosAngeles" AND groupid = 3;
```

10
docs/zh/08-connector/10-cpp.mdx
View File

@ -223,7 +223,7 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields)
基础 API 用于完成创建数据库连接等工作，为其它 API 的执行提供运行时环境。
- `void taos_init()`
- `int taos_init()`
初始化运行环境。如果没有主动调用该 API，那么调用 `taos_connect()` 时驱动将自动调用该 API，故程序一般无需手动调用。
@ -542,6 +542,7 @@ TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多
- 错误码，0成功，非0失败，可通过 `char *tmq_err2str(int32_t code)` 函数获取错误信息。
- `int64_t tmq_committed(tmq_t *tmq, const char *pTopicName, int32_t vgId)`
**功能说明**
- 获取当前 consumer 在某个 topic 和 vgroup上的 commit 位置。
@ -555,7 +556,7 @@ TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多
**功能说明**
commit接口分为两种类型，每种类型有同步和异步接口：
- commit接口分为两种类型，每种类型有同步和异步接口：
- 第一种类型：根据消息提交，提交消息里的进度，如果消息传NULL，提交当前consumer所有消费的vgroup的当前进度 : tmq_commit_sync/tmq_commit_async
- 第二种类型：根据某个topic的某个vgroup的offset提交 : tmq_commit_offset_sync/tmq_commit_offset_async
@ -584,8 +585,7 @@ TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多
- `int32_t int64_t tmq_get_vgroup_offset(TAOS_RES* res)`
**功能说明**
获取 poll 消费到的数据的起始offset
- 获取 poll 消费到的数据的起始offset
**参数说明**
- msg：消费到的消息结构
@ -596,8 +596,8 @@ TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多
- `int32_t int32_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics)`
**功能说明**
- 获取消费者订阅的 topic 列表
获取消费者订阅的 topic 列表
**参数说明**
- topics: 获取的 topic 列表存储在这个结构中，接口内分配内存，需调用tmq_list_destroy释放

4
docs/zh/14-reference/13-schemaless/13-schemaless.md
View File

@ -33,8 +33,8 @@ tag_set 中的所有的数据自动转化为 nchar 数据类型，并不需要
在无模式写入数据行协议中，field_set 中的每个数据项都需要对自身的数据类型进行描述。具体来说：
- 如果两边有英文双引号，表示 BINARY(32) 类型。例如 `"abc"`。
- 如果两边有英文双引号而且带有 L 前缀，表示 NCHAR(32) 类型。例如 `L"报错信息"`。
- 如果两边有英文双引号，表示 VARCHAR(N) 类型。例如 `"abc"`。
- 如果两边有英文双引号而且带有 L 前缀，表示 NCHAR(N) 类型。例如 `L"报错信息"`。
- 对空格、等号（=）、逗号（,）、双引号（"）、反斜杠（\），前面需要使用反斜杠（\）进行转义。（都指的是英文半角符号）。具体转义规则如下：
| **序号** | **域** | **需转义字符** |

2
examples/c/demo.c
View File

@ -63,7 +63,7 @@ int main(int argc, char *argv[]) {
TAOS *taos = taos_connect(argv[1], "root", "taosdata", NULL, 0);
if (taos == NULL) {
printf("failed to connect to server, reason:%s\n", "null taos"/*taos_errstr(taos)*/);
printf("failed to connect to server, reason:%s\n", taos_errstr(NULL));
exit(1);
}
for (int i = 0; i < 100; i++) {

2
include/common/tdatablock.h
View File

@ -179,6 +179,8 @@ int32_t getJsonValueLen(const char* data);
int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull);
int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx, const char* pData);
int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows, bool trimValue);
int32_t colDataCopyNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData,
uint32_t numOfRows, bool isNull);
int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity,
const SColumnInfoData* pSource, int32_t numOfRow2);
int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows,

237
include/common/tmsg.h
View File

@ -213,6 +213,215 @@ typedef enum _mgmt_table {
#define TD_REQ_FROM_APP 0
#define TD_REQ_FROM_TAOX 1
typedef enum ENodeType {
// Syntax nodes are used in parser and planner module, and some are also used in executor module, such as COLUMN,
// VALUE, OPERATOR, FUNCTION and so on.
QUERY_NODE_COLUMN = 1,
QUERY_NODE_VALUE,
QUERY_NODE_OPERATOR,
QUERY_NODE_LOGIC_CONDITION,
QUERY_NODE_FUNCTION,
QUERY_NODE_REAL_TABLE,
QUERY_NODE_TEMP_TABLE,
QUERY_NODE_JOIN_TABLE,
QUERY_NODE_GROUPING_SET,
QUERY_NODE_ORDER_BY_EXPR,
QUERY_NODE_LIMIT,
QUERY_NODE_STATE_WINDOW,
QUERY_NODE_SESSION_WINDOW,
QUERY_NODE_INTERVAL_WINDOW,
QUERY_NODE_NODE_LIST,
QUERY_NODE_FILL,
QUERY_NODE_RAW_EXPR, // Only be used in parser module.
QUERY_NODE_TARGET,
QUERY_NODE_DATABLOCK_DESC,
QUERY_NODE_SLOT_DESC,
QUERY_NODE_COLUMN_DEF,
QUERY_NODE_DOWNSTREAM_SOURCE,
QUERY_NODE_DATABASE_OPTIONS,
QUERY_NODE_TABLE_OPTIONS,
QUERY_NODE_INDEX_OPTIONS,
QUERY_NODE_EXPLAIN_OPTIONS,
QUERY_NODE_STREAM_OPTIONS,
QUERY_NODE_LEFT_VALUE,
QUERY_NODE_COLUMN_REF,
QUERY_NODE_WHEN_THEN,
QUERY_NODE_CASE_WHEN,
QUERY_NODE_EVENT_WINDOW,
QUERY_NODE_HINT,
// Statement nodes are used in parser and planner module.
QUERY_NODE_SET_OPERATOR = 100,
QUERY_NODE_SELECT_STMT,
QUERY_NODE_VNODE_MODIFY_STMT,
QUERY_NODE_CREATE_DATABASE_STMT,
QUERY_NODE_DROP_DATABASE_STMT,
QUERY_NODE_ALTER_DATABASE_STMT,
QUERY_NODE_FLUSH_DATABASE_STMT,
QUERY_NODE_TRIM_DATABASE_STMT,
QUERY_NODE_CREATE_TABLE_STMT,
QUERY_NODE_CREATE_SUBTABLE_CLAUSE,
QUERY_NODE_CREATE_MULTI_TABLES_STMT,
QUERY_NODE_DROP_TABLE_CLAUSE,
QUERY_NODE_DROP_TABLE_STMT,
QUERY_NODE_DROP_SUPER_TABLE_STMT,
QUERY_NODE_ALTER_TABLE_STMT,
QUERY_NODE_ALTER_SUPER_TABLE_STMT,
QUERY_NODE_CREATE_USER_STMT,
QUERY_NODE_ALTER_USER_STMT,
QUERY_NODE_DROP_USER_STMT,
QUERY_NODE_USE_DATABASE_STMT,
QUERY_NODE_CREATE_DNODE_STMT,
QUERY_NODE_DROP_DNODE_STMT,
QUERY_NODE_ALTER_DNODE_STMT,
QUERY_NODE_CREATE_INDEX_STMT,
QUERY_NODE_DROP_INDEX_STMT,
QUERY_NODE_CREATE_QNODE_STMT,
QUERY_NODE_DROP_QNODE_STMT,
QUERY_NODE_CREATE_BNODE_STMT,
QUERY_NODE_DROP_BNODE_STMT,
QUERY_NODE_CREATE_SNODE_STMT,
QUERY_NODE_DROP_SNODE_STMT,
QUERY_NODE_CREATE_MNODE_STMT,
QUERY_NODE_DROP_MNODE_STMT,
QUERY_NODE_CREATE_TOPIC_STMT,
QUERY_NODE_DROP_TOPIC_STMT,
QUERY_NODE_DROP_CGROUP_STMT,
QUERY_NODE_ALTER_LOCAL_STMT,
QUERY_NODE_EXPLAIN_STMT,
QUERY_NODE_DESCRIBE_STMT,
QUERY_NODE_RESET_QUERY_CACHE_STMT,
QUERY_NODE_COMPACT_DATABASE_STMT,
QUERY_NODE_CREATE_FUNCTION_STMT,
QUERY_NODE_DROP_FUNCTION_STMT,
QUERY_NODE_CREATE_STREAM_STMT,
QUERY_NODE_DROP_STREAM_STMT,
QUERY_NODE_BALANCE_VGROUP_STMT,
QUERY_NODE_MERGE_VGROUP_STMT,
QUERY_NODE_REDISTRIBUTE_VGROUP_STMT,
QUERY_NODE_SPLIT_VGROUP_STMT,
QUERY_NODE_SYNCDB_STMT,
QUERY_NODE_GRANT_STMT,
QUERY_NODE_REVOKE_STMT,
QUERY_NODE_SHOW_DNODES_STMT,
QUERY_NODE_SHOW_MNODES_STMT,
QUERY_NODE_SHOW_MODULES_STMT,
QUERY_NODE_SHOW_QNODES_STMT,
QUERY_NODE_SHOW_SNODES_STMT,
QUERY_NODE_SHOW_BNODES_STMT,
QUERY_NODE_SHOW_CLUSTER_STMT,
QUERY_NODE_SHOW_DATABASES_STMT,
QUERY_NODE_SHOW_FUNCTIONS_STMT,
QUERY_NODE_SHOW_INDEXES_STMT,
QUERY_NODE_SHOW_STABLES_STMT,
QUERY_NODE_SHOW_STREAMS_STMT,
QUERY_NODE_SHOW_TABLES_STMT,
QUERY_NODE_SHOW_TAGS_STMT,
QUERY_NODE_SHOW_USERS_STMT,
QUERY_NODE_SHOW_LICENCES_STMT,
QUERY_NODE_SHOW_VGROUPS_STMT,
QUERY_NODE_SHOW_TOPICS_STMT,
QUERY_NODE_SHOW_CONSUMERS_STMT,
QUERY_NODE_SHOW_CONNECTIONS_STMT,
QUERY_NODE_SHOW_QUERIES_STMT,
QUERY_NODE_SHOW_APPS_STMT,
QUERY_NODE_SHOW_VARIABLES_STMT,
QUERY_NODE_SHOW_DNODE_VARIABLES_STMT,
QUERY_NODE_SHOW_TRANSACTIONS_STMT,
QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT,
QUERY_NODE_SHOW_VNODES_STMT,
QUERY_NODE_SHOW_USER_PRIVILEGES_STMT,
QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
QUERY_NODE_SHOW_CREATE_TABLE_STMT,
QUERY_NODE_SHOW_CREATE_STABLE_STMT,
QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT,
QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT,
QUERY_NODE_SHOW_SCORES_STMT,
QUERY_NODE_SHOW_TABLE_TAGS_STMT,
QUERY_NODE_KILL_CONNECTION_STMT,
QUERY_NODE_KILL_QUERY_STMT,
QUERY_NODE_KILL_TRANSACTION_STMT,
QUERY_NODE_DELETE_STMT,
QUERY_NODE_INSERT_STMT,
QUERY_NODE_QUERY,
QUERY_NODE_SHOW_DB_ALIVE_STMT,
QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT,
QUERY_NODE_BALANCE_VGROUP_LEADER_STMT,
QUERY_NODE_RESTORE_DNODE_STMT,
QUERY_NODE_RESTORE_QNODE_STMT,
QUERY_NODE_RESTORE_MNODE_STMT,
QUERY_NODE_RESTORE_VNODE_STMT,
QUERY_NODE_PAUSE_STREAM_STMT,
QUERY_NODE_RESUME_STREAM_STMT,
// logic plan node
QUERY_NODE_LOGIC_PLAN_SCAN = 1000,
QUERY_NODE_LOGIC_PLAN_JOIN,
QUERY_NODE_LOGIC_PLAN_AGG,
QUERY_NODE_LOGIC_PLAN_PROJECT,
QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY,
QUERY_NODE_LOGIC_PLAN_EXCHANGE,
QUERY_NODE_LOGIC_PLAN_MERGE,
QUERY_NODE_LOGIC_PLAN_WINDOW,
QUERY_NODE_LOGIC_PLAN_FILL,
QUERY_NODE_LOGIC_PLAN_SORT,
QUERY_NODE_LOGIC_PLAN_PARTITION,
QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC,
QUERY_NODE_LOGIC_PLAN_INTERP_FUNC,
QUERY_NODE_LOGIC_SUBPLAN,
QUERY_NODE_LOGIC_PLAN,
QUERY_NODE_LOGIC_PLAN_GROUP_CACHE,
QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL,
// physical plan node
QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN = 1100,
QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN,
QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN,
QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN,
QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN,
QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN,
QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN,
QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN,
QUERY_NODE_PHYSICAL_PLAN_PROJECT,
QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN,
QUERY_NODE_PHYSICAL_PLAN_HASH_AGG,
QUERY_NODE_PHYSICAL_PLAN_EXCHANGE,
QUERY_NODE_PHYSICAL_PLAN_MERGE,
QUERY_NODE_PHYSICAL_PLAN_SORT,
QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT,
QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_FILL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL,
QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION,
QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION,
QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION,
QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION,
QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE,
QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE,
QUERY_NODE_PHYSICAL_PLAN_PARTITION,
QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION,
QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC,
QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC,
QUERY_NODE_PHYSICAL_PLAN_DISPATCH,
QUERY_NODE_PHYSICAL_PLAN_INSERT,
QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT,
QUERY_NODE_PHYSICAL_PLAN_DELETE,
QUERY_NODE_PHYSICAL_SUBPLAN,
QUERY_NODE_PHYSICAL_PLAN,
QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN,
QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT,
QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT,
QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN,
QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE,
QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL
} ENodeType;
typedef struct {
int32_t vgId;
char* dbFName;
@ -744,6 +953,10 @@ typedef struct STimeWindow {
TSKEY ekey;
} STimeWindow;
typedef struct SQueryHint {
bool batchScan;
} SQueryHint;
typedef struct {
int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed
int32_t tsLen; // total length of ts comp block
@ -762,12 +975,18 @@ typedef struct {
int64_t offset;
} SInterval;
typedef struct {
int32_t code;
typedef struct STbVerInfo {
char tbFName[TSDB_TABLE_FNAME_LEN];
int32_t sversion;
int32_t tversion;
} STbVerInfo;
typedef struct {
int32_t code;
int64_t affectedRows;
SArray* tbVerInfo; // STbVerInfo
} SQueryTableRsp;
int32_t tSerializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp);
@ -1841,12 +2060,26 @@ typedef struct {
int32_t tversion;
} SResReadyRsp;
typedef struct SOperatorParam {
int32_t opType;
int32_t downstreamIdx;
void* value;
SArray* pChildren; //SArray<SOperatorParam*>
} SOperatorParam;
typedef struct STableScanOperatorParam {
bool tableSeq;
SArray* pUidList;
} STableScanOperatorParam;
typedef struct {
SMsgHead header;
uint64_t sId;
uint64_t queryId;
uint64_t taskId;
int32_t execId;
SOperatorParam* pOpParam;
} SResFetchReq;
int32_t tSerializeSResFetchReq(void* buf, int32_t bufLen, SResFetchReq* pReq);

162
include/common/ttokendef.h
View File

@ -276,83 +276,88 @@
#define TK_JOIN 258
#define TK_INNER 259
#define TK_SELECT 260
#define TK_DISTINCT 261
#define TK_WHERE 262
#define TK_PARTITION 263
#define TK_BY 264
#define TK_SESSION 265
#define TK_STATE_WINDOW 266
#define TK_EVENT_WINDOW 267
#define TK_SLIDING 268
#define TK_FILL 269
#define TK_VALUE 270
#define TK_VALUE_F 271
#define TK_NONE 272
#define TK_PREV 273
#define TK_NULL_F 274
#define TK_LINEAR 275
#define TK_NEXT 276
#define TK_HAVING 277
#define TK_RANGE 278
#define TK_EVERY 279
#define TK_ORDER 280
#define TK_SLIMIT 281
#define TK_SOFFSET 282
#define TK_LIMIT 283
#define TK_OFFSET 284
#define TK_ASC 285
#define TK_NULLS 286
#define TK_ABORT 287
#define TK_AFTER 288
#define TK_ATTACH 289
#define TK_BEFORE 290
#define TK_BEGIN 291
#define TK_BITAND 292
#define TK_BITNOT 293
#define TK_BITOR 294
#define TK_BLOCKS 295
#define TK_CHANGE 296
#define TK_COMMA 297
#define TK_CONCAT 298
#define TK_CONFLICT 299
#define TK_COPY 300
#define TK_DEFERRED 301
#define TK_DELIMITERS 302
#define TK_DETACH 303
#define TK_DIVIDE 304
#define TK_DOT 305
#define TK_EACH 306
#define TK_FAIL 307
#define TK_FILE 308
#define TK_FOR 309
#define TK_GLOB 310
#define TK_ID 311
#define TK_IMMEDIATE 312
#define TK_IMPORT 313
#define TK_INITIALLY 314
#define TK_INSTEAD 315
#define TK_ISNULL 316
#define TK_KEY 317
#define TK_MODULES 318
#define TK_NK_BITNOT 319
#define TK_NK_SEMI 320
#define TK_NOTNULL 321
#define TK_OF 322
#define TK_PLUS 323
#define TK_PRIVILEGE 324
#define TK_RAISE 325
#define TK_RESTRICT 326
#define TK_ROW 327
#define TK_SEMI 328
#define TK_STAR 329
#define TK_STATEMENT 330
#define TK_STRICT 331
#define TK_STRING 332
#define TK_TIMES 333
#define TK_VALUES 334
#define TK_VARIABLE 335
#define TK_VIEW 336
#define TK_WAL 337
#define TK_NK_HINT 261
#define TK_DISTINCT 262
#define TK_WHERE 263
#define TK_PARTITION 264
#define TK_BY 265
#define TK_SESSION 266
#define TK_STATE_WINDOW 267
#define TK_EVENT_WINDOW 268
#define TK_SLIDING 269
#define TK_FILL 270
#define TK_VALUE 271
#define TK_VALUE_F 272
#define TK_NONE 273
#define TK_PREV 274
#define TK_NULL_F 275
#define TK_LINEAR 276
#define TK_NEXT 277
#define TK_HAVING 278
#define TK_RANGE 279
#define TK_EVERY 280
#define TK_ORDER 281
#define TK_SLIMIT 282
#define TK_SOFFSET 283
#define TK_LIMIT 284
#define TK_OFFSET 285
#define TK_ASC 286
#define TK_NULLS 287
#define TK_ABORT 288
#define TK_AFTER 289
#define TK_ATTACH 290
#define TK_BEFORE 291
#define TK_BEGIN 292
#define TK_BITAND 293
#define TK_BITNOT 294
#define TK_BITOR 295
#define TK_BLOCKS 296
#define TK_CHANGE 297
#define TK_COMMA 298
#define TK_CONCAT 299
#define TK_CONFLICT 300
#define TK_COPY 301
#define TK_DEFERRED 302
#define TK_DELIMITERS 303
#define TK_DETACH 304
#define TK_DIVIDE 305
#define TK_DOT 306
#define TK_EACH 307
#define TK_FAIL 308
#define TK_FILE 309
#define TK_FOR 310
#define TK_GLOB 311
#define TK_ID 312
#define TK_IMMEDIATE 313
#define TK_IMPORT 314
#define TK_INITIALLY 315
#define TK_INSTEAD 316
#define TK_ISNULL 317
#define TK_KEY 318
#define TK_MODULES 319
#define TK_NK_BITNOT 320
#define TK_NK_SEMI 321
#define TK_NOTNULL 322
#define TK_OF 323
#define TK_PLUS 324
#define TK_PRIVILEGE 325
#define TK_RAISE 326
#define TK_RESTRICT 327
#define TK_ROW 328
#define TK_SEMI 329
#define TK_STAR 330
#define TK_STATEMENT 331
#define TK_STRICT 332
#define TK_STRING 333
#define TK_TIMES 334
#define TK_VALUES 335
#define TK_VARIABLE 336
#define TK_VIEW 337
#define TK_WAL 338
#define TK_NK_SPACE 600
#define TK_NK_COMMENT 601
@ -360,6 +365,9 @@
#define TK_NK_HEX 603 // hex number 0x123
#define TK_NK_OCT 604 // oct number
#define TK_NK_BIN 605 // bin format data 0b111
#define TK_BATCH_SCAN 606
#define TK_NO_BATCH_SCAN 607
#define TK_NK_NIL 65535

2
include/libs/executor/dataSinkMgt.h
View File

@ -97,6 +97,8 @@ int32_t dsPutDataBlock(DataSinkHandle handle, const SInputData* pInput, bool* pC
void dsEndPut(DataSinkHandle handle, uint64_t useconds);
void dsReset(DataSinkHandle handle);
/**
* Get the length of the data returned by the next call to dsGetDataBlock.
* @param handle

6
include/libs/executor/executor.h
View File

@ -130,6 +130,10 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks,
*/
int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd);
bool qIsDynamicExecTask(qTaskInfo_t tinfo);
void qUpdateOperatorParam(qTaskInfo_t tinfo, void* pParam);
/**
* Create the exec task object according to task json
* @param readHandle
@ -150,7 +154,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
* @return
*/
int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion,
int32_t* tversion);
int32_t* tversion, int32_t idx);
/**
* The main task execution function, including query on both table and multiple tables,

2
include/libs/function/functionMgt.h
View File

@ -122,6 +122,8 @@ typedef enum EFunctionType {
FUNCTION_TYPE_IROWTS,
FUNCTION_TYPE_ISFILLED,
FUNCTION_TYPE_TAGS,
FUNCTION_TYPE_TBUID,
FUNCTION_TYPE_VGID,
// internal function
FUNCTION_TYPE_SELECT_VALUE = 3750,

203
include/libs/nodes/nodes.h
View File

@ -21,6 +21,7 @@ extern "C" {
#endif
#include "tdef.h"
#include "tmsg.h"
#define nodeType(nodeptr) (((const SNode*)(nodeptr))->type)
#define setNodeType(nodeptr, nodetype) (((SNode*)(nodeptr))->type = (nodetype))
@ -78,208 +79,6 @@ extern "C" {
(list) = NULL; \
} while (0)
typedef enum ENodeType {
// Syntax nodes are used in parser and planner module, and some are also used in executor module, such as COLUMN,
// VALUE, OPERATOR, FUNCTION and so on.
QUERY_NODE_COLUMN = 1,
QUERY_NODE_VALUE,
QUERY_NODE_OPERATOR,
QUERY_NODE_LOGIC_CONDITION,
QUERY_NODE_FUNCTION,
QUERY_NODE_REAL_TABLE,
QUERY_NODE_TEMP_TABLE,
QUERY_NODE_JOIN_TABLE,
QUERY_NODE_GROUPING_SET,
QUERY_NODE_ORDER_BY_EXPR,
QUERY_NODE_LIMIT,
QUERY_NODE_STATE_WINDOW,
QUERY_NODE_SESSION_WINDOW,
QUERY_NODE_INTERVAL_WINDOW,
QUERY_NODE_NODE_LIST,
QUERY_NODE_FILL,
QUERY_NODE_RAW_EXPR, // Only be used in parser module.
QUERY_NODE_TARGET,
QUERY_NODE_DATABLOCK_DESC,
QUERY_NODE_SLOT_DESC,
QUERY_NODE_COLUMN_DEF,
QUERY_NODE_DOWNSTREAM_SOURCE,
QUERY_NODE_DATABASE_OPTIONS,
QUERY_NODE_TABLE_OPTIONS,
QUERY_NODE_INDEX_OPTIONS,
QUERY_NODE_EXPLAIN_OPTIONS,
QUERY_NODE_STREAM_OPTIONS,
QUERY_NODE_LEFT_VALUE,
QUERY_NODE_COLUMN_REF,
QUERY_NODE_WHEN_THEN,
QUERY_NODE_CASE_WHEN,
QUERY_NODE_EVENT_WINDOW,
// Statement nodes are used in parser and planner module.
QUERY_NODE_SET_OPERATOR = 100,
QUERY_NODE_SELECT_STMT,
QUERY_NODE_VNODE_MODIFY_STMT,
QUERY_NODE_CREATE_DATABASE_STMT,
QUERY_NODE_DROP_DATABASE_STMT,
QUERY_NODE_ALTER_DATABASE_STMT,
QUERY_NODE_FLUSH_DATABASE_STMT,
QUERY_NODE_TRIM_DATABASE_STMT,
QUERY_NODE_CREATE_TABLE_STMT,
QUERY_NODE_CREATE_SUBTABLE_CLAUSE,
QUERY_NODE_CREATE_MULTI_TABLES_STMT,
QUERY_NODE_DROP_TABLE_CLAUSE,
QUERY_NODE_DROP_TABLE_STMT,
QUERY_NODE_DROP_SUPER_TABLE_STMT,
QUERY_NODE_ALTER_TABLE_STMT,
QUERY_NODE_ALTER_SUPER_TABLE_STMT,
QUERY_NODE_CREATE_USER_STMT,
QUERY_NODE_ALTER_USER_STMT,
QUERY_NODE_DROP_USER_STMT,
QUERY_NODE_USE_DATABASE_STMT,
QUERY_NODE_CREATE_DNODE_STMT,
QUERY_NODE_DROP_DNODE_STMT,
QUERY_NODE_ALTER_DNODE_STMT,
QUERY_NODE_CREATE_INDEX_STMT,
QUERY_NODE_DROP_INDEX_STMT,
QUERY_NODE_CREATE_QNODE_STMT,
QUERY_NODE_DROP_QNODE_STMT,
QUERY_NODE_CREATE_BNODE_STMT,
QUERY_NODE_DROP_BNODE_STMT,
QUERY_NODE_CREATE_SNODE_STMT,
QUERY_NODE_DROP_SNODE_STMT,
QUERY_NODE_CREATE_MNODE_STMT,
QUERY_NODE_DROP_MNODE_STMT,
QUERY_NODE_CREATE_TOPIC_STMT,
QUERY_NODE_DROP_TOPIC_STMT,
QUERY_NODE_DROP_CGROUP_STMT,
QUERY_NODE_ALTER_LOCAL_STMT,
QUERY_NODE_EXPLAIN_STMT,
QUERY_NODE_DESCRIBE_STMT,
QUERY_NODE_RESET_QUERY_CACHE_STMT,
QUERY_NODE_COMPACT_DATABASE_STMT,
QUERY_NODE_CREATE_FUNCTION_STMT,
QUERY_NODE_DROP_FUNCTION_STMT,
QUERY_NODE_CREATE_STREAM_STMT,
QUERY_NODE_DROP_STREAM_STMT,
QUERY_NODE_BALANCE_VGROUP_STMT,
QUERY_NODE_MERGE_VGROUP_STMT,
QUERY_NODE_REDISTRIBUTE_VGROUP_STMT,
QUERY_NODE_SPLIT_VGROUP_STMT,
QUERY_NODE_SYNCDB_STMT,
QUERY_NODE_GRANT_STMT,
QUERY_NODE_REVOKE_STMT,
QUERY_NODE_SHOW_DNODES_STMT,
QUERY_NODE_SHOW_MNODES_STMT,
// QUERY_NODE_SHOW_MODULES_STMT,
QUERY_NODE_SHOW_QNODES_STMT,
QUERY_NODE_SHOW_SNODES_STMT,
QUERY_NODE_SHOW_BNODES_STMT,
QUERY_NODE_SHOW_CLUSTER_STMT,
QUERY_NODE_SHOW_DATABASES_STMT,
QUERY_NODE_SHOW_FUNCTIONS_STMT,
QUERY_NODE_SHOW_INDEXES_STMT,
QUERY_NODE_SHOW_STABLES_STMT,
QUERY_NODE_SHOW_STREAMS_STMT,
QUERY_NODE_SHOW_TABLES_STMT,
QUERY_NODE_SHOW_TAGS_STMT,
QUERY_NODE_SHOW_USERS_STMT,
QUERY_NODE_SHOW_LICENCES_STMT,
QUERY_NODE_SHOW_VGROUPS_STMT,
QUERY_NODE_SHOW_TOPICS_STMT,
QUERY_NODE_SHOW_CONSUMERS_STMT,
QUERY_NODE_SHOW_CONNECTIONS_STMT,
QUERY_NODE_SHOW_QUERIES_STMT,
QUERY_NODE_SHOW_APPS_STMT,
QUERY_NODE_SHOW_VARIABLES_STMT,
QUERY_NODE_SHOW_DNODE_VARIABLES_STMT,
QUERY_NODE_SHOW_TRANSACTIONS_STMT,
QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT,
QUERY_NODE_SHOW_VNODES_STMT,
QUERY_NODE_SHOW_USER_PRIVILEGES_STMT,
QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
QUERY_NODE_SHOW_CREATE_TABLE_STMT,
QUERY_NODE_SHOW_CREATE_STABLE_STMT,
QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT,
QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT,
QUERY_NODE_SHOW_SCORES_STMT,
QUERY_NODE_SHOW_TABLE_TAGS_STMT,
QUERY_NODE_KILL_CONNECTION_STMT,
QUERY_NODE_KILL_QUERY_STMT,
QUERY_NODE_KILL_TRANSACTION_STMT,
QUERY_NODE_DELETE_STMT,
QUERY_NODE_INSERT_STMT,
QUERY_NODE_QUERY,
QUERY_NODE_SHOW_DB_ALIVE_STMT,
QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT,
QUERY_NODE_BALANCE_VGROUP_LEADER_STMT,
QUERY_NODE_RESTORE_DNODE_STMT,
QUERY_NODE_RESTORE_QNODE_STMT,
QUERY_NODE_RESTORE_MNODE_STMT,
QUERY_NODE_RESTORE_VNODE_STMT,
QUERY_NODE_PAUSE_STREAM_STMT,
QUERY_NODE_RESUME_STREAM_STMT,
// logic plan node
QUERY_NODE_LOGIC_PLAN_SCAN = 1000,
QUERY_NODE_LOGIC_PLAN_JOIN,
QUERY_NODE_LOGIC_PLAN_AGG,
QUERY_NODE_LOGIC_PLAN_PROJECT,
QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY,
QUERY_NODE_LOGIC_PLAN_EXCHANGE,
QUERY_NODE_LOGIC_PLAN_MERGE,
QUERY_NODE_LOGIC_PLAN_WINDOW,
QUERY_NODE_LOGIC_PLAN_FILL,
QUERY_NODE_LOGIC_PLAN_SORT,
QUERY_NODE_LOGIC_PLAN_PARTITION,
QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC,
QUERY_NODE_LOGIC_PLAN_INTERP_FUNC,
QUERY_NODE_LOGIC_SUBPLAN,
QUERY_NODE_LOGIC_PLAN,
// physical plan node
QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN = 1100,
QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN,
QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN,
QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN,
QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN,
QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN,
QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN,
QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN,
QUERY_NODE_PHYSICAL_PLAN_PROJECT,
QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN,
QUERY_NODE_PHYSICAL_PLAN_HASH_AGG,
QUERY_NODE_PHYSICAL_PLAN_EXCHANGE,
QUERY_NODE_PHYSICAL_PLAN_MERGE,
QUERY_NODE_PHYSICAL_PLAN_SORT,
QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT,
QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_FILL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL,
QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION,
QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION,
QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION,
QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION,
QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE,
QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE,
QUERY_NODE_PHYSICAL_PLAN_PARTITION,
QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION,
QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC,
QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC,
QUERY_NODE_PHYSICAL_PLAN_DISPATCH,
QUERY_NODE_PHYSICAL_PLAN_INSERT,
QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT,
QUERY_NODE_PHYSICAL_PLAN_DELETE,
QUERY_NODE_PHYSICAL_SUBPLAN,
QUERY_NODE_PHYSICAL_PLAN,
QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN,
QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT,
QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT
} ENodeType;
/**
* The first field of a node of any type is guaranteed to be the ENodeType.
* Hence the type of any node can be gotten by casting it to SNode.

81
include/libs/nodes/plannodes.h
View File

@ -42,10 +42,13 @@ typedef enum EGroupAction {
typedef struct SLogicNode {
ENodeType type;
bool dynamicOp;
bool stmtRoot;
SNodeList* pTargets; // SColumnNode
SNode* pConditions;
SNodeList* pChildren;
struct SLogicNode* pParent;
SNodeList* pHint;
int32_t optimizedFlag;
uint8_t precision;
SNode* pLimit;
@ -113,10 +116,15 @@ typedef struct SScanLogicNode {
typedef struct SJoinLogicNode {
SLogicNode node;
EJoinType joinType;
SNode* pMergeCondition;
SNode* pOnConditions;
EJoinAlgorithm joinAlgo;
SNode* pPrimKeyEqCond;
SNode* pColEqCond;
SNode* pTagEqCond;
SNode* pTagOnCond;
SNode* pOtherOnCond;
bool isSingleTableJoin;
SNode* pColEqualOnConditions;
bool hasSubQuery;
bool isLowLevelJoin;
} SJoinLogicNode;
typedef struct SAggLogicNode {
@ -155,6 +163,28 @@ typedef struct SInterpFuncLogicNode {
SNode* pTimeSeries; // SColumnNode
} SInterpFuncLogicNode;
typedef struct SGroupCacheLogicNode {
SLogicNode node;
bool grpColsMayBeNull;
bool grpByUid;
bool globalGrp;
bool batchFetch;
SNodeList* pGroupCols;
} SGroupCacheLogicNode;
typedef struct SDynQueryCtrlStbJoin {
bool batchFetch;
SNodeList* pVgList;
SNodeList* pUidList;
bool srcScan[2];
} SDynQueryCtrlStbJoin;
typedef struct SDynQueryCtrlLogicNode {
SLogicNode node;
EDynQueryType qType;
SDynQueryCtrlStbJoin stbJoin;
} SDynQueryCtrlLogicNode;
typedef enum EModifyTableType { MODIFY_TABLE_TYPE_INSERT = 1, MODIFY_TABLE_TYPE_DELETE } EModifyTableType;
typedef struct SVnodeModifyLogicNode {
@ -313,6 +343,7 @@ typedef struct SDataBlockDescNode {
typedef struct SPhysiNode {
ENodeType type;
bool dynamicOp;
EOrder inputTsOrder;
EOrder outputTsOrder;
SDataBlockDescNode* pOutputDataBlockDesc;
@ -414,12 +445,50 @@ typedef struct SInterpFuncPhysiNode {
typedef struct SSortMergeJoinPhysiNode {
SPhysiNode node;
EJoinType joinType;
SNode* pMergeCondition;
SNode* pOnConditions;
SNode* pPrimKeyCond;
SNode* pColEqCond;
SNode* pOtherOnCond;
SNodeList* pTargets;
SNode* pColEqualOnConditions;
} SSortMergeJoinPhysiNode;
typedef struct SHashJoinPhysiNode {
SPhysiNode node;
EJoinType joinType;
SNodeList* pOnLeft;
SNodeList* pOnRight;
SNode* pFilterConditions;
SNodeList* pTargets;
SQueryStat inputStat[2];
SNode* pPrimKeyCond;
SNode* pColEqCond;
SNode* pTagEqCond;
} SHashJoinPhysiNode;
typedef struct SGroupCachePhysiNode {
SPhysiNode node;
bool grpColsMayBeNull;
bool grpByUid;
bool globalGrp;
bool batchFetch;
SNodeList* pGroupCols;
} SGroupCachePhysiNode;
typedef struct SStbJoinDynCtrlBasic {
bool batchFetch;
int32_t vgSlot[2];
int32_t uidSlot[2];
bool srcScan[2];
} SStbJoinDynCtrlBasic;
typedef struct SDynQueryCtrlPhysiNode {
SPhysiNode node;
EDynQueryType qType;
union {
SStbJoinDynCtrlBasic stbJoin;
};
} SDynQueryCtrlPhysiNode;
typedef struct SAggPhysiNode {
SPhysiNode node;
SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function

32
include/libs/nodes/querynodes.h
View File

@ -116,6 +116,17 @@ typedef struct SLeftValueNode {
ENodeType type;
} SLeftValueNode;
typedef enum EHintOption {
HINT_NO_BATCH_SCAN = 1,
HINT_BATCH_SCAN,
} EHintOption;
typedef struct SHintNode {
ENodeType type;
EHintOption option;
void* value;
} SHintNode;
typedef struct SOperatorNode {
SExprNode node; // QUERY_NODE_OPERATOR
EOperatorType opType;
@ -169,11 +180,27 @@ typedef struct STempTableNode {
SNode* pSubquery;
} STempTableNode;
typedef enum EJoinType { JOIN_TYPE_INNER = 1 } EJoinType;
typedef enum EJoinType {
JOIN_TYPE_INNER = 1,
JOIN_TYPE_LEFT,
JOIN_TYPE_RIGHT,
} EJoinType;
typedef enum EJoinAlgorithm {
JOIN_ALGO_UNKNOWN = 0,
JOIN_ALGO_MERGE,
JOIN_ALGO_HASH,
} EJoinAlgorithm;
typedef enum EDynQueryType {
DYN_QTYPE_STB_HASH = 1,
} EDynQueryType;
typedef struct SJoinTableNode {
STableNode table; // QUERY_NODE_JOIN_TABLE
EJoinType joinType;
bool hasSubQuery;
bool isLowLevelJoin;
SNode* pLeft;
SNode* pRight;
SNode* pOnCond;
@ -289,6 +316,7 @@ typedef struct SSelectStmt {
SLimitNode* pLimit;
SLimitNode* pSlimit;
STimeWindow timeRange;
SNodeList* pHint;
char stmtName[TSDB_TABLE_NAME_LEN];
uint8_t precision;
int32_t selectFuncNum;
@ -470,7 +498,7 @@ int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char*
int32_t nodesCollectColumnsFromNode(SNode* node, const char* pTableAlias, ECollectColType type, SNodeList** pCols);
typedef bool (*FFuncClassifier)(int32_t funcId);
int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, FFuncClassifier classifier, SNodeList** pFuncs);
int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, char* tableAlias, FFuncClassifier classifier, SNodeList** pFuncs);
int32_t nodesCollectSpecialNodes(SSelectStmt* pSelect, ESqlClause clause, ENodeType type, SNodeList** pNodes);

10
include/libs/qcom/query.h
View File

@ -90,11 +90,6 @@ typedef struct SExecResult {
void* res;
} SExecResult;
typedef struct STbVerInfo {
char tbFName[TSDB_TABLE_FNAME_LEN];
int32_t sversion;
int32_t tversion;
} STbVerInfo;
#pragma pack(push, 1)
typedef struct SCTableMeta {
@ -212,6 +207,11 @@ typedef struct SQueryNodeStat {
int32_t tableNum; // vg table number, unit is TSDB_TABLE_NUM_UNIT
} SQueryNodeStat;
typedef struct SQueryStat {
int64_t inputRowNum;
int32_t inputRowSize;
} SQueryStat;
int32_t initTaskQueue();
int32_t cleanupTaskQueue();

2
include/libs/scalar/scalar.h
View File

@ -95,6 +95,8 @@ int32_t qStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
int32_t qEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t qTbUidFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t qVgIdFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
/* Aggregation functions */
int32_t countScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

2
include/util/taoserror.h
View File

@ -518,6 +518,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_QRY_JOB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x072F)
#define TSDB_CODE_QRY_QWORKER_QUIT TAOS_DEF_ERROR_CODE(0, 0x0730)
#define TSDB_CODE_QRY_GEO_NOT_SUPPORT_ERROR TAOS_DEF_ERROR_CODE(0, 0x0731)
#define TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x0732)
// grant
#define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800)
@ -714,6 +715,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700)
#define TSDB_CODE_PLAN_EXPECTED_TS_EQUAL TAOS_DEF_ERROR_CODE(0, 0x2701)
#define TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN TAOS_DEF_ERROR_CODE(0, 0x2702)
#define TSDB_CODE_PLAN_NOT_SUPPORT_JOIN_COND TAOS_DEF_ERROR_CODE(0, 0x2703)
//function
#define TSDB_CODE_FUNC_FUNTION_ERROR TAOS_DEF_ERROR_CODE(0, 0x2800)

85
source/common/src/tdatablock.c
View File

@ -146,15 +146,6 @@ int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx,
const char* pData) {
int32_t type = pColumnInfoData->info.type;
if (IS_VAR_DATA_TYPE(type)) {
int32_t dataLen = 0;
if (type == TSDB_DATA_TYPE_JSON) {
dataLen = getJsonValueLen(pData);
} else {
dataLen = varDataTLen(pData);
}
SVarColAttr* pAttr = &pColumnInfoData->varmeta;
pColumnInfoData->varmeta.offset[dstRowIdx] = pColumnInfoData->varmeta.offset[srcRowIdx];
pColumnInfoData->reassigned = true;
} else {
@ -247,6 +238,82 @@ int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow,
return doCopyNItems(pColumnInfoData, currentRow, pData, len, numOfRows, trimValue);
}
void colDataSetNItemsNull(SColumnInfoData* pColumnInfoData, uint32_t currentRow, uint32_t numOfRows) {
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
memset(&pColumnInfoData->varmeta.offset[currentRow], -1, sizeof(int32_t) * numOfRows);
} else {
if (numOfRows < sizeof(char) * 2) {
for (int32_t i = 0; i < numOfRows; ++i) {
colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow + i);
}
} else {
int32_t i = 0;
for (; i < numOfRows; ++i) {
if (BitPos(currentRow + i)) {
colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow + i);
} else {
break;
}
}
memset(&BMCharPos(pColumnInfoData->nullbitmap, currentRow + i), 0xFF, (numOfRows - i) / sizeof(char));
i += (numOfRows - i) / sizeof(char) * sizeof(char);
for (; i < numOfRows; ++i) {
colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow + i);
}
}
}
}
int32_t colDataCopyAndReassign(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, uint32_t numOfRows) {
int32_t code = colDataSetVal(pColumnInfoData, currentRow, pData, false);
if (code) {
return code;
}
if (numOfRows > 1) {
int32_t* pOffset = pColumnInfoData->varmeta.offset;
memset(&pOffset[currentRow + 1], pOffset[currentRow], sizeof(pOffset[0]) * (numOfRows - 1));
pColumnInfoData->reassigned = true;
}
return TSDB_CODE_SUCCESS;
}
int32_t colDataCopyNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData,
uint32_t numOfRows, bool isNull) {
int32_t len = pColumnInfoData->info.bytes;
if (isNull) {
colDataSetNItemsNull(pColumnInfoData, currentRow, numOfRows);
pColumnInfoData->hasNull = true;
return 0;
}
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
return colDataCopyAndReassign(pColumnInfoData, currentRow, pData, numOfRows);
} else {
int32_t colBytes = pColumnInfoData->info.bytes;
int32_t colOffset = currentRow * colBytes;
uint32_t num = 1;
void* pStart = pColumnInfoData->pData + colOffset;
memcpy(pStart, pData, colBytes);
colOffset += num * colBytes;
while (num < numOfRows) {
int32_t maxNum = num << 1;
int32_t tnum = maxNum > numOfRows ? (numOfRows - num) : num;
memcpy(pColumnInfoData->pData + colOffset, pStart, tnum * colBytes);
colOffset += tnum * colBytes;
num += tnum;
}
}
return TSDB_CODE_SUCCESS;
}
static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, const SColumnInfoData* pSource,
int32_t numOfRow2) {
if (numOfRow2 <= 0) return;

117
source/common/src/tmsg.c
View File

@ -5573,6 +5573,82 @@ void tFreeSSubQueryMsg(SSubQueryMsg *pReq) {
taosMemoryFreeClear(pReq->msg);
}
int32_t tSerializeSOperatorParam(SEncoder* pEncoder, SOperatorParam* pOpParam) {
if (tEncodeI32(pEncoder, pOpParam->opType) < 0) return -1;
if (tEncodeI32(pEncoder, pOpParam->downstreamIdx) < 0) return -1;
switch (pOpParam->opType) {
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: {
STableScanOperatorParam* pScan = (STableScanOperatorParam*)pOpParam->value;
if (tEncodeI8(pEncoder, pScan->tableSeq) < 0) return -1;
int32_t uidNum = taosArrayGetSize(pScan->pUidList);
if (tEncodeI32(pEncoder, uidNum) < 0) return -1;
for (int32_t m = 0; m < uidNum; ++m) {
int64_t* pUid = taosArrayGet(pScan->pUidList, m);
if (tEncodeI64(pEncoder, *pUid) < 0) return -1;
}
break;
}
default:
return TSDB_CODE_INVALID_PARA;
}
int32_t n = taosArrayGetSize(pOpParam->pChildren);
if (tEncodeI32(pEncoder, n) < 0) return -1;
for (int32_t i = 0; i < n; ++i) {
SOperatorParam* pChild = *(SOperatorParam**)taosArrayGet(pOpParam->pChildren, i);
if (tSerializeSOperatorParam(pEncoder, pChild) < 0) return -1;
}
return 0;
}
int32_t tDeserializeSOperatorParam(SDecoder *pDecoder, SOperatorParam* pOpParam) {
if (tDecodeI32(pDecoder, &pOpParam->opType) < 0) return -1;
if (tDecodeI32(pDecoder, &pOpParam->downstreamIdx) < 0) return -1;
switch (pOpParam->opType) {
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: {
STableScanOperatorParam* pScan = taosMemoryMalloc(sizeof(STableScanOperatorParam));
if (NULL == pScan) return -1;
if (tDecodeI8(pDecoder, (int8_t*)&pScan->tableSeq) < 0) return -1;
int32_t uidNum = 0;
int64_t uid = 0;
if (tDecodeI32(pDecoder, &uidNum) < 0) return -1;
if (uidNum > 0) {
pScan->pUidList = taosArrayInit(uidNum, sizeof(int64_t));
if (NULL == pScan->pUidList) return -1;
for (int32_t m = 0; m < uidNum; ++m) {
if (tDecodeI64(pDecoder, &uid) < 0) return -1;
taosArrayPush(pScan->pUidList, &uid);
}
} else {
pScan->pUidList = NULL;
}
pOpParam->value = pScan;
break;
}
default:
return TSDB_CODE_INVALID_PARA;
}
int32_t childrenNum = 0;
if (tDecodeI32(pDecoder, &childrenNum) < 0) return -1;
if (childrenNum > 0) {
pOpParam->pChildren = taosArrayInit(childrenNum, POINTER_BYTES);
if (NULL == pOpParam->pChildren) return -1;
for (int32_t i = 0; i < childrenNum; ++i) {
SOperatorParam* pChild = taosMemoryCalloc(1, sizeof(SOperatorParam));
if (NULL == pChild) return -1;
if (tDeserializeSOperatorParam(pDecoder, pChild) < 0) return -1;
taosArrayPush(pOpParam->pChildren, &pChild);
}
} else {
pOpParam->pChildren = NULL;
}
return 0;
}
int32_t tSerializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq) {
int32_t headLen = sizeof(SMsgHead);
if (buf != NULL) {
@ -5588,6 +5664,12 @@ int32_t tSerializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq) {
if (tEncodeU64(&encoder, pReq->queryId) < 0) return -1;
if (tEncodeU64(&encoder, pReq->taskId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->execId) < 0) return -1;
if (pReq->pOpParam) {
if (tEncodeI32(&encoder, 1) < 0) return -1;
if (tSerializeSOperatorParam(&encoder, pReq->pOpParam) < 0) return -1;
} else {
if (tEncodeI32(&encoder, 0) < 0) return -1;
}
tEndEncode(&encoder);
@ -5620,6 +5702,14 @@ int32_t tDeserializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq)
if (tDecodeU64(&decoder, &pReq->taskId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->execId) < 0) return -1;
int32_t paramNum = 0;
if (tDecodeI32(&decoder, &paramNum) < 0) return -1;
if (paramNum > 0) {
pReq->pOpParam = taosMemoryMalloc(sizeof(*pReq->pOpParam));
if (NULL == pReq->pOpParam) return -1;
if (tDeserializeSOperatorParam(&decoder, pReq->pOpParam) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
@ -5764,10 +5854,17 @@ int32_t tSerializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pRsp->code) < 0) return -1;
if (tEncodeCStr(&encoder, pRsp->tbFName) < 0) return -1;
if (tEncodeI32(&encoder, pRsp->sversion) < 0) return -1;
if (tEncodeI32(&encoder, pRsp->tversion) < 0) return -1;
if (tEncodeI64(&encoder, pRsp->affectedRows) < 0) return -1;
int32_t tbNum = taosArrayGetSize(pRsp->tbVerInfo);
if (tEncodeI32(&encoder, tbNum) < 0) return -1;
if (tbNum > 0) {
for (int32_t i = 0; i < tbNum; ++i) {
STbVerInfo *pVer = taosArrayGet(pRsp->tbVerInfo, i);
if (tEncodeCStr(&encoder, pVer->tbFName) < 0) return -1;
if (tEncodeI32(&encoder, pVer->sversion) < 0) return -1;
if (tEncodeI32(&encoder, pVer->tversion) < 0) return -1;
}
}
tEndEncode(&encoder);
@ -5784,10 +5881,18 @@ int32_t tDeserializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pR
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pRsp->code) < 0) return -1;
if (tDecodeCStrTo(&decoder, pRsp->tbFName) < 0) return -1;
if (tDecodeI32(&decoder, &pRsp->sversion) < 0) return -1;
if (tDecodeI32(&decoder, &pRsp->tversion) < 0) return -1;
if (tDecodeI64(&decoder, &pRsp->affectedRows) < 0) return -1;
int32_t tbNum = 0;
if (tDecodeI32(&decoder, &tbNum) < 0) return -1;
if (tbNum > 0) {
pRsp->tbVerInfo = taosArrayInit(tbNum, sizeof(STbVerInfo));
if (NULL == pRsp->tbVerInfo) return -1;
STbVerInfo tbVer;
if (tDecodeCStrTo(&decoder, tbVer.tbFName) < 0) return -1;
if (tDecodeI32(&decoder, &tbVer.sversion) < 0) return -1;
if (tDecodeI32(&decoder, &tbVer.tversion) < 0) return -1;
if (NULL == taosArrayPush(pRsp->tbVerInfo, &tbVer)) return -1;
}
tEndDecode(&decoder);

56
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -202,25 +202,25 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return -1;
}
if(req.learnerReplica == 0)
{
if (req.learnerReplica == 0) {
req.learnerSelfIndex = -1;
}
dInfo("vgId:%d, vnode management handle msgType:%s, start to create vnode, page:%d pageSize:%d buffer:%d szPage:%d szBuf:%" PRIu64
", cacheLast:%d cacheLastSize:%d sstTrigger:%d tsdbPageSize:%d %d dbname:%s dbId:%" PRId64
dInfo(
"vgId:%d, vnode management handle msgType:%s, start to create vnode, page:%d pageSize:%d buffer:%d szPage:%d "
"szBuf:%" PRIu64 ", cacheLast:%d cacheLastSize:%d sstTrigger:%d tsdbPageSize:%d %d dbname:%s dbId:%" PRId64
", days:%d keep0:%d keep1:%d keep2:%d tsma:%d precision:%d compression:%d minRows:%d maxRows:%d"
", wal fsync:%d level:%d retentionPeriod:%d retentionSize:%" PRId64 " rollPeriod:%d segSize:%" PRId64
", hash method:%d begin:%u end:%u prefix:%d surfix:%d replica:%d selfIndex:%d "
"learnerReplica:%d learnerSelfIndex:%d strict:%d changeVersion:%d",
req.vgId, TMSG_INFO(pMsg->msgType), req.pages, req.pageSize, req.buffer, req.pageSize * 1024,
(uint64_t)req.buffer * 1024 * 1024,
req.cacheLast, req.cacheLastSize, req.sstTrigger, req.tsdbPageSize, req.tsdbPageSize * 1024, req.db, req.dbUid,
req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2, req.isTsma, req.precision, req.compression,
req.minRows, req.maxRows, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize,
req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.hashBegin, req.hashEnd, req.hashPrefix,
req.hashSuffix, req.replica, req.selfIndex, req.learnerReplica, req.learnerSelfIndex, req.strict,
req.changeVersion);
(uint64_t)req.buffer * 1024 * 1024, req.cacheLast, req.cacheLastSize, req.sstTrigger, req.tsdbPageSize,
req.tsdbPageSize * 1024, req.db, req.dbUid, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2,
req.isTsma, req.precision, req.compression, req.minRows, req.maxRows, req.walFsyncPeriod, req.walLevel,
req.walRetentionPeriod, req.walRetentionSize, req.walRollPeriod, req.walSegmentSize, req.hashMethod,
req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix, req.replica, req.selfIndex, req.learnerReplica,
req.learnerSelfIndex, req.strict, req.changeVersion);
for (int32_t i = 0; i < req.replica; ++i) {
dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", req.vgId, i, req.replicas[i].fqdn, req.replicas[i].port,
req.replicas[i].id);
@ -233,8 +233,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SReplica *pReplica = NULL;
if (req.selfIndex != -1) {
pReplica = &req.replicas[req.selfIndex];
}
else{
} else {
pReplica = &req.learnerReplicas[req.learnerSelfIndex];
}
if (pReplica->id != pMgmt->pData->dnodeId || pReplica->port != tsServerPort ||
@ -315,10 +314,10 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
_OVER:
if (code != 0) {
vnodeClose(pImpl);
vnodeDestroy(path, pMgmt->pTfs);
vnodeDestroy(0, path, pMgmt->pTfs);
} else {
dInfo("vgId:%d, vnode management handle msgType:%s, end to create vnode, vnode is created",
req.vgId, TMSG_INFO(pMsg->msgType));
dInfo("vgId:%d, vnode management handle msgType:%s, end to create vnode, vnode is created", req.vgId,
TMSG_INFO(pMsg->msgType));
}
tFreeSCreateVnodeReq(&req);
@ -338,8 +337,8 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
req.learnerSelfIndex = -1;
}
dInfo("vgId:%d, vnode management handle msgType:%s, start to process alter-node-type-request",
req.vgId, TMSG_INFO(pMsg->msgType));
dInfo("vgId:%d, vnode management handle msgType:%s, start to process alter-node-type-request", req.vgId,
TMSG_INFO(pMsg->msgType));
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId);
if (pVnode == NULL) {
@ -378,9 +377,8 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dInfo("vgId:%d, learnerReplicas:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->id);
}
if (req.replica <= 0 ||
(req.selfIndex < 0 && req.learnerSelfIndex <0)||
req.selfIndex >= req.replica || req.learnerSelfIndex >= req.learnerReplica) {
if (req.replica <= 0 || (req.selfIndex < 0 && req.learnerSelfIndex < 0) || req.selfIndex >= req.replica ||
req.learnerSelfIndex >= req.learnerReplica) {
terrno = TSDB_CODE_INVALID_MSG;
dError("vgId:%d, failed to alter replica since invalid msg", vgId);
vmReleaseVnode(pMgmt, pVnode);
@ -390,8 +388,7 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SReplica *pReplica = NULL;
if (req.selfIndex != -1) {
pReplica = &req.replicas[req.selfIndex];
}
else{
} else {
pReplica = &req.learnerReplicas[req.learnerSelfIndex];
}
@ -611,10 +608,12 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
}
int32_t vgId = alterReq.vgId;
dInfo("vgId:%d,vnode management handle msgType:%s, start to alter vnode replica:%d selfIndex:%d leanerReplica:%d "
dInfo(
"vgId:%d,vnode management handle msgType:%s, start to alter vnode replica:%d selfIndex:%d leanerReplica:%d "
"learnerSelfIndex:%d strict:%d changeVersion:%d",
vgId, TMSG_INFO(pMsg->msgType), alterReq.replica, alterReq.selfIndex, alterReq.learnerReplica,
alterReq.learnerSelfIndex, alterReq.strict, alterReq.changeVersion);
for (int32_t i = 0; i < alterReq.replica; ++i) {
SReplica *pReplica = &alterReq.replicas[i];
dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->port);
@ -624,8 +623,7 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dInfo("vgId:%d, learnerReplicas:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->port);
}
if (alterReq.replica <= 0 ||
(alterReq.selfIndex < 0 && alterReq.learnerSelfIndex <0)||
if (alterReq.replica <= 0 || (alterReq.selfIndex < 0 && alterReq.learnerSelfIndex < 0) ||
alterReq.selfIndex >= alterReq.replica || alterReq.learnerSelfIndex >= alterReq.learnerReplica) {
terrno = TSDB_CODE_INVALID_MSG;
dError("vgId:%d, failed to alter replica since invalid msg", vgId);
@ -635,8 +633,7 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SReplica *pReplica = NULL;
if (alterReq.selfIndex != -1) {
pReplica = &alterReq.replicas[alterReq.selfIndex];
}
else{
} else {
pReplica = &alterReq.learnerReplicas[alterReq.learnerSelfIndex];
}
@ -692,7 +689,8 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return -1;
}
dInfo("vgId:%d, vnode management handle msgType:%s, end to alter vnode replica:%d selfIndex:%d leanerReplica:%d "
dInfo(
"vgId:%d, vnode management handle msgType:%s, end to alter vnode replica:%d selfIndex:%d leanerReplica:%d "
"learnerSelfIndex:%d strict:%d",
vgId, TMSG_INFO(pMsg->msgType), alterReq.replica, alterReq.selfIndex, alterReq.learnerReplica,
alterReq.learnerSelfIndex, alterReq.strict);

2
source/dnode/mgmt/mgmt_vnode/src/vmInt.c
View File

@ -208,7 +208,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
if (pVnode->dropped) {
dInfo("vgId:%d, vnode is destroyed, dropped:%d", pVnode->vgId, pVnode->dropped);
snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pVnode->vgId);
vnodeDestroy(path, pMgmt->pTfs);
vnodeDestroy(pVnode->vgId, path, pMgmt->pTfs);
}
taosMemoryFree(pVnode->path);

5
source/dnode/mnode/impl/src/mndStb.c
View File

@ -1610,9 +1610,11 @@ static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbO
return -1;
}
col_id_t colId = pOld->pColumns[col].colId;
uint32_t nLen = 0;
for (int32_t i = 0; i < pOld->numOfColumns; ++i) {
nLen += (pOld->pColumns[i].colId == col) ? pField->bytes : pOld->pColumns[i].bytes;
nLen += (pOld->pColumns[i].colId == colId) ? pField->bytes : pOld->pColumns[i].bytes;
}
if (nLen > TSDB_MAX_BYTES_PER_ROW) {
@ -1620,7 +1622,6 @@ static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbO
return -1;
}
col_id_t colId = pOld->pColumns[col].colId;
if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) {
return -1;
}

7
source/dnode/vnode/inc/vnode.h
View File

@ -33,8 +33,8 @@
#include "tmsg.h"
#include "trow.h"
#include "tdb.h"
#include "storageapi.h"
#include "tdb.h"
#ifdef __cplusplus
extern "C" {
@ -57,7 +57,7 @@ int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnod
int32_t diskPrimary, STfs *pTfs);
int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId,
int32_t diskPrimary, STfs *pTfs);
void vnodeDestroy(const char *path, STfs *pTfs);
void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs);
SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgCb);
void vnodePreClose(SVnode *pVnode);
void vnodePostClose(SVnode *pVnode);
@ -80,7 +80,8 @@ ESyncRole vnodeGetRole(SVnode *pVnode);
int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list);
int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg);
int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list);
int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg, void* arg1), void *arg);
int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg, void *arg1),
void *arg);
void *vnodeGetIdx(void *pVnode);
void *vnodeGetIvtIdx(void *pVnode);

24
source/dnode/vnode/src/inc/tsdb.h
View File

@ -708,14 +708,21 @@ typedef struct {
TSDBROW row;
} SRowInfo;
typedef struct SSttBlockLoadCostInfo {
int64_t loadBlocks;
int64_t loadStatisBlocks;
double blockElapsedTime;
double statisElapsedTime;
} SSttBlockLoadCostInfo;
typedef struct SSttBlockLoadInfo {
SBlockData blockData[2];
SBlockData blockData[2]; // buffered block data
int32_t statisBlockIndex; // buffered statistics block index
void *statisBlock; // buffered statistics block data
void *pSttStatisBlkArray;
SArray *aSttBlk;
int32_t blockIndex[2]; // to denote the loaded block in the corresponding position.
int32_t currentLoadBlockIndex;
int32_t loadBlocks;
double elapsedTime;
STSchema *pSchema;
int16_t *colIds;
int32_t numOfCols;
@ -723,12 +730,7 @@ typedef struct SSttBlockLoadInfo {
bool isLast;
bool sttBlockLoaded;
// keep the last access position, this position may be used to reduce the binary times for
// starting last block data for a new table
struct {
int32_t blockIndex;
int32_t rowIndex;
} prevEndPos;
SSttBlockLoadCostInfo cost;
} SSttBlockLoadInfo;
typedef struct SMergeTree {
@ -831,9 +833,9 @@ void tMergeTreeClose(SMergeTree *pMTree);
SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, int32_t numOfStt);
SSttBlockLoadInfo *tCreateOneLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols);
void resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
void getLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, int64_t *blocks, double *el);
void getSttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, SSttBlockLoadCostInfo *pLoadCost);
void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
void *destroySttBlockReader(SArray *pLDataIterArray, int64_t *blocks, double *el);
void *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost);
// tsdbCache ==============================================================================================
typedef enum {

1
source/dnode/vnode/src/inc/vndCos.h
View File

@ -27,6 +27,7 @@ extern int8_t tsS3Enabled;
int32_t s3Init();
void s3CleanUp();
int32_t s3PutObjectFromFile(const char *file, const char *object);
void s3DeleteObjectsByPrefix(const char *prefix);
void s3DeleteObjects(const char *object_name[], int nobject);
bool s3Exists(const char *object_name);
bool s3Get(const char *object_name, const char *path);

5
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -1760,10 +1760,7 @@ typedef struct {
static int32_t lastIterOpen(SFSLastIter *iter, STFileSet *pFileSet, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid,
tb_uid_t uid, SCacheRowsReader *pr, int64_t lastTs, int16_t *aCols, int nCols) {
int32_t code = 0;
int64_t loadBlocks = 0;
double elapse = 0;
pr->pLDataIterArray = destroySttBlockReader(pr->pLDataIterArray, &loadBlocks, &elapse);
pr->pLDataIterArray = destroySttBlockReader(pr->pLDataIterArray, NULL);
pr->pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
SMergeTreeConf conf = {

9
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -125,10 +125,7 @@ int32_t tsdbReuseCacherowsReader(void* reader, void* pTableIdList, int32_t numOf
pReader->pTableList = pTableIdList;
pReader->numOfTables = numOfTables;
pReader->lastTs = INT64_MIN;
int64_t blocks;
double elapse;
pReader->pLDataIterArray = destroySttBlockReader(pReader->pLDataIterArray, &blocks, &elapse);
pReader->pLDataIterArray = destroySttBlockReader(pReader->pLDataIterArray, NULL);
pReader->pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
return TSDB_CODE_SUCCESS;
@ -208,9 +205,7 @@ void* tsdbCacherowsReaderClose(void* pReader) {
taosMemoryFree(p->pCurrSchema);
if (p->pLDataIterArray) {
int64_t loadBlocks = 0;
double elapse = 0;
destroySttBlockReader(p->pLDataIterArray, &loadBlocks, &elapse);
destroySttBlockReader(p->pLDataIterArray, NULL);
}
if (p->pFileReader) {

2
source/dnode/vnode/src/tsdb/tsdbCommit2.c
View File

@ -530,7 +530,9 @@ static int32_t tsdbCloseCommitter(SCommitter2 *committer, int32_t eno) {
ASSERT(committer->tombIterMerger == NULL);
TARRAY2_DESTROY(committer->dataIterArray, NULL);
TARRAY2_DESTROY(committer->tombIterArray, NULL);
TARRAY2_DESTROY(committer->sttReaderArray, NULL);
TARRAY2_DESTROY(committer->fopArray, NULL);
TARRAY2_DESTROY(committer->sttReaderArray, NULL);
tsdbFSDestroyCopySnapshot(&committer->fsetArr);
_exit:

4
source/dnode/vnode/src/tsdb/tsdbFSet2.c
View File

@ -86,7 +86,7 @@ static int32_t tsdbSttLvlApplyEdit(STsdb *pTsdb, const SSttLvl *lvl1, SSttLvl *l
// create a file obj
code = tsdbTFileObjInit(pTsdb, fobj1->f, &fobj2);
if (code) return code;
code = TARRAY2_APPEND(lvl2->fobjArr, fobj2);
code = TARRAY2_INSERT_PTR(lvl2->fobjArr, i2, &fobj2);
if (code) return code;
i1++;
i2++;
@ -112,7 +112,7 @@ static int32_t tsdbSttLvlApplyEdit(STsdb *pTsdb, const SSttLvl *lvl1, SSttLvl *l
// create a file obj
code = tsdbTFileObjInit(pTsdb, fobj1->f, &fobj2);
if (code) return code;
code = TARRAY2_APPEND(lvl2->fobjArr, fobj2);
code = TARRAY2_INSERT_PTR(lvl2->fobjArr, i2, &fobj2);
if (code) return code;
i1++;
i2++;

165
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
View File

@ -91,41 +91,39 @@ void resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
taosArrayClear(pLoadInfo[i].aSttBlk);
pLoadInfo[i].elapsedTime = 0;
pLoadInfo[i].loadBlocks = 0;
pLoadInfo[i].cost.loadBlocks = 0;
pLoadInfo[i].cost.blockElapsedTime = 0;
pLoadInfo[i].cost.statisElapsedTime = 0;
pLoadInfo[i].cost.loadStatisBlocks = 0;
pLoadInfo[i].statisBlockIndex = -1;
tStatisBlockDestroy(pLoadInfo[i].statisBlock);
pLoadInfo[i].sttBlockLoaded = false;
}
}
void getLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, int64_t *blocks, double *el) {
void getSttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, SSttBlockLoadCostInfo* pLoadCost) {
for (int32_t i = 0; i < 1; ++i) {
*el += pLoadInfo[i].elapsedTime;
*blocks += pLoadInfo[i].loadBlocks;
pLoadCost->blockElapsedTime += pLoadInfo[i].cost.blockElapsedTime;
pLoadCost->loadBlocks += pLoadInfo[i].cost.loadBlocks;
pLoadCost->loadStatisBlocks += pLoadInfo[i].cost.loadStatisBlocks;
pLoadCost->statisElapsedTime += pLoadInfo[i].cost.statisElapsedTime;
}
}
static void freeTombBlock(void *param) {
STombBlock **pTombBlock = (STombBlock **)param;
tTombBlockDestroy(*pTombBlock);
taosMemoryFree(*pTombBlock);
}
void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
if (pLoadInfo == NULL) {
return NULL;
}
for (int32_t i = 0; i < 1; ++i) {
pLoadInfo[i].currentLoadBlockIndex = 1;
pLoadInfo[i].blockIndex[0] = -1;
pLoadInfo[i].blockIndex[1] = -1;
pLoadInfo->currentLoadBlockIndex = 1;
pLoadInfo->blockIndex[0] = -1;
pLoadInfo->blockIndex[1] = -1;
tBlockDataDestroy(&pLoadInfo[i].blockData[0]);
tBlockDataDestroy(&pLoadInfo[i].blockData[1]);
taosArrayDestroy(pLoadInfo[i].aSttBlk);
}
tBlockDataDestroy(&pLoadInfo->blockData[0]);
tBlockDataDestroy(&pLoadInfo->blockData[1]);
taosArrayDestroy(pLoadInfo->aSttBlk);
taosMemoryFree(pLoadInfo);
return NULL;
}
@ -136,7 +134,7 @@ static void destroyLDataIter(SLDataIter *pIter) {
taosMemoryFree(pIter);
}
void *destroySttBlockReader(SArray *pLDataIterArray, int64_t *blocks, double *el) {
void *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo* pLoadCost) {
if (pLDataIterArray == NULL) {
return NULL;
}
@ -146,8 +144,13 @@ void *destroySttBlockReader(SArray *pLDataIterArray, int64_t *blocks, double *el
SArray *pList = taosArrayGetP(pLDataIterArray, i);
for (int32_t j = 0; j < taosArrayGetSize(pList); ++j) {
SLDataIter *pIter = taosArrayGetP(pList, j);
*el += pIter->pBlockLoadInfo->elapsedTime;
*blocks += pIter->pBlockLoadInfo->loadBlocks;
if (pLoadCost != NULL) {
pLoadCost->loadBlocks += pIter->pBlockLoadInfo->cost.loadBlocks;
pLoadCost->loadStatisBlocks += pIter->pBlockLoadInfo->cost.loadStatisBlocks;
pLoadCost->blockElapsedTime += pIter->pBlockLoadInfo->cost.blockElapsedTime;
pLoadCost->statisElapsedTime += pIter->pBlockLoadInfo->cost.statisElapsedTime;
}
destroyLDataIter(pIter);
}
taosArrayDestroy(pList);
@ -195,12 +198,12 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) {
}
double el = (taosGetTimestampUs() - st) / 1000.0;
pInfo->elapsedTime += el;
pInfo->loadBlocks += 1;
pInfo->cost.blockElapsedTime += el;
pInfo->cost.loadBlocks += 1;
tsdbDebug("read last block, total load:%d, trigger by uid:%" PRIu64
tsdbDebug("read last block, total load:%"PRId64", trigger by uid:%" PRIu64
", last file index:%d, last block index:%d, entry:%d, rows:%d, %p, elapsed time:%.2f ms, %s",
pInfo->loadBlocks, pIter->uid, pIter->iStt, pIter->iSttBlk, pInfo->currentLoadBlockIndex, pBlock->nRow,
pInfo->cost.loadBlocks, pIter->uid, pIter->iStt, pIter->iSttBlk, pInfo->currentLoadBlockIndex, pBlock->nRow,
pBlock, el, idStr);
pInfo->blockIndex[pInfo->currentLoadBlockIndex] = pIter->iSttBlk;
@ -363,8 +366,9 @@ static int32_t suidComparFn(const void *target, const void *p2) {
}
}
static bool existsFromSttBlkStatis(const TStatisBlkArray *pStatisBlkArray, uint64_t suid, uint64_t uid,
static bool existsFromSttBlkStatis(SSttBlockLoadInfo *pBlockLoadInfo, uint64_t suid, uint64_t uid,
SSttFileReader *pReader) {
const TStatisBlkArray *pStatisBlkArray = pBlockLoadInfo->pSttStatisBlkArray;
if (TARRAY2_SIZE(pStatisBlkArray) <= 0) {
return true;
}
@ -387,23 +391,40 @@ static bool existsFromSttBlkStatis(const TStatisBlkArray *pStatisBlkArray, uint6
return false;
}
STbStatisBlock block = {0};
tsdbSttFileReadStatisBlock(pReader, p, &block);
// if (pBlockLoadInfo->statisBlock == NULL) {
// pBlockLoadInfo->statisBlock = taosMemoryCalloc(1, sizeof(STbStatisBlock));
//
// int64_t st = taosGetTimestampMs();
// tsdbSttFileReadStatisBlock(pReader, p, pBlockLoadInfo->statisBlock);
// pBlockLoadInfo->statisBlockIndex = i;
//
// double el = (taosGetTimestampMs() - st) / 1000.0;
// pBlockLoadInfo->cost.loadStatisBlocks += 1;
// pBlockLoadInfo->cost.statisElapsedTime += el;
// } else if (pBlockLoadInfo->statisBlockIndex != i) {
// tStatisBlockDestroy(pBlockLoadInfo->statisBlock);
//
// int64_t st = taosGetTimestampMs();
// tsdbSttFileReadStatisBlock(pReader, p, pBlockLoadInfo->statisBlock);
// pBlockLoadInfo->statisBlockIndex = i;
//
// double el = (taosGetTimestampMs() - st) / 1000.0;
// pBlockLoadInfo->cost.loadStatisBlocks += 1;
// pBlockLoadInfo->cost.statisElapsedTime += el;
// }
int32_t index = tarray2SearchIdx(block.suid, &suid, sizeof(int64_t), suidComparFn, TD_EQ);
STbStatisBlock* pBlock = pBlockLoadInfo->statisBlock;
int32_t index = tarray2SearchIdx(pBlock->suid, &suid, sizeof(int64_t), suidComparFn, TD_EQ);
if (index == -1) {
tStatisBlockDestroy(&block);
return false;
}
int32_t j = index;
if (block.uid->data[j] == uid) {
tStatisBlockDestroy(&block);
if (pBlock->uid->data[j] == uid) {
return true;
} else if (block.uid->data[j] > uid) {
while (j >= 0 && block.suid->data[j] == suid) {
if (block.uid->data[j] == uid) {
tStatisBlockDestroy(&block);
} else if (pBlock->uid->data[j] > uid) {
while (j >= 0 && pBlock->suid->data[j] == suid) {
if (pBlock->uid->data[j] == uid) {
return true;
} else {
j -= 1;
@ -411,9 +432,8 @@ static bool existsFromSttBlkStatis(const TStatisBlkArray *pStatisBlkArray, uint6
}
} else {
j = index + 1;
while (j < block.suid->size && block.suid->data[j] == suid) {
if (block.uid->data[j] == uid) {
tStatisBlockDestroy(&block);
while (j < pBlock->suid->size && pBlock->suid->data[j] == suid) {
if (pBlock->uid->data[j] == uid) {
return true;
} else {
j += 1;
@ -421,44 +441,21 @@ static bool existsFromSttBlkStatis(const TStatisBlkArray *pStatisBlkArray, uint6
}
}
tStatisBlockDestroy(&block);
i += 1;
}
return false;
}
int32_t tLDataIterOpen2(struct SLDataIter *pIter, SSttFileReader *pSttFileReader, int32_t iStt, int8_t backward,
uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange,
SSttBlockLoadInfo *pBlockLoadInfo, const char *idStr, bool strictTimeRange,
_load_tomb_fn loadTombFn, void *pReader1) {
int32_t code = TSDB_CODE_SUCCESS;
pIter->uid = uid;
pIter->iStt = iStt;
pIter->backward = backward;
pIter->verRange.minVer = pRange->minVer;
pIter->verRange.maxVer = pRange->maxVer;
pIter->timeWindow.skey = pTimeWindow->skey;
pIter->timeWindow.ekey = pTimeWindow->ekey;
pIter->pReader = pSttFileReader;
pIter->pBlockLoadInfo = pBlockLoadInfo;
if (pIter->pReader == NULL) {
tsdbError("stt file reader is null, %s", idStr);
pIter->pSttBlk = NULL;
pIter->iSttBlk = -1;
return TSDB_CODE_SUCCESS;
}
if (!pBlockLoadInfo->sttBlockLoaded) {
static int32_t doLoadSttFilesBlk(SSttBlockLoadInfo *pBlockLoadInfo, SLDataIter *pIter, int64_t suid,
_load_tomb_fn loadTombFn, void *pReader1, const char *idStr) {
int64_t st = taosGetTimestampUs();
const TSttBlkArray *pSttBlkArray = NULL;
pBlockLoadInfo->sttBlockLoaded = true;
// load the stt block info for each stt-block
code = tsdbSttFileReadSttBlk(pIter->pReader, &pSttBlkArray);
int32_t code = tsdbSttFileReadSttBlk(pIter->pReader, &pSttBlkArray);
if (code != TSDB_CODE_SUCCESS) {
tsdbError("load stt blk failed, code:%s, %s", tstrerror(code), idStr);
return code;
@ -481,9 +478,41 @@ int32_t tLDataIterOpen2(struct SLDataIter *pIter, SSttFileReader *pSttFileReader
double el = (taosGetTimestampUs() - st) / 1000.0;
tsdbDebug("load the stt file info completed, elapsed time:%.2fms, %s", el, idStr);
return code;
}
// bool exists = existsFromSttBlkStatis(pBlockLoadInfo->pSttStatisBlkArray, suid, uid, pIter->pReader);
int32_t tLDataIterOpen2(SLDataIter *pIter, SSttFileReader *pSttFileReader, int32_t iStt, int8_t backward,
uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange,
SSttBlockLoadInfo *pBlockLoadInfo, const char *idStr, bool strictTimeRange,
_load_tomb_fn loadTombFn, void *pReader1) {
int32_t code = TSDB_CODE_SUCCESS;
pIter->uid = uid;
pIter->iStt = iStt;
pIter->backward = backward;
pIter->verRange.minVer = pRange->minVer;
pIter->verRange.maxVer = pRange->maxVer;
pIter->timeWindow.skey = pTimeWindow->skey;
pIter->timeWindow.ekey = pTimeWindow->ekey;
pIter->pReader = pSttFileReader;
pIter->pBlockLoadInfo = pBlockLoadInfo;
// open stt file failed, ignore and continue
if (pIter->pReader == NULL) {
tsdbError("stt file reader is null, %s", idStr);
pIter->pSttBlk = NULL;
pIter->iSttBlk = -1;
return TSDB_CODE_SUCCESS;
}
if (!pBlockLoadInfo->sttBlockLoaded) {
code = doLoadSttFilesBlk(pBlockLoadInfo, pIter, suid, loadTombFn, pReader1, idStr);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
}
// bool exists = existsFromSttBlkStatis(pBlockLoadInfo, suid, uid, pIter->pReader);
// if (!exists) {
// pIter->iSttBlk = -1;
// pIter->pSttBlk = NULL;

24
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -63,7 +63,7 @@ typedef struct STableBlockScanInfo {
SIterInfo iiter; // imem buffer skip list iterator
SArray* delSkyline; // delete info for this table
int32_t fileDelIndex; // file block delete index
int32_t lastBlockDelIndex; // delete index for last block
int32_t sttBlockDelIndex; // delete index for last block
bool iterInit; // whether to initialize the in-memory skip list iterator or not
} STableBlockScanInfo;
@ -87,13 +87,13 @@ typedef struct SIOCostSummary {
double headFileLoadTime;
int64_t smaDataLoad;
double smaLoadTime;
int64_t lastBlockLoad;
double lastBlockLoadTime;
int64_t sttStatisBlockLoad;
int64_t sttBlockLoad;
double sttBlockLoadTime;
int64_t composedBlocks;
double buildComposedBlockTime;
double createScanInfoList;
// double getTbFromMemTime;
// double getTbFromIMemTime;
SSttBlockLoadCostInfo sttCost;
double initDelSkylineIterTime;
} SIOCostSummary;
@ -586,8 +586,8 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo
return TSDB_CODE_SUCCESS;
}
SIOCostSummary* pSum = &pReader->cost;
getLastBlockLoadInfo(pIter->pLastBlockReader->pInfo, &pSum->lastBlockLoad, &pReader->cost.lastBlockLoadTime);
SIOCostSummary* pCost = &pReader->cost;
getSttBlockLoadInfo(pIter->pLastBlockReader->pInfo, &pCost->sttCost);
pIter->pLastBlockReader->uid = 0;
tMergeTreeClose(&pIter->pLastBlockReader->mergeTree);
@ -1976,7 +1976,7 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc
pLastBlockReader->currentKey = key;
pScanInfo->lastKeyInStt = key;
if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->lastBlockDelIndex, key, ver, pLastBlockReader->order,
if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, pLastBlockReader->order,
pVerRange)) {
return true;
}
@ -3018,7 +3018,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader*
pBlockScanInfo->iter.index = index;
pBlockScanInfo->iiter.index = index;
pBlockScanInfo->fileDelIndex = index;
pBlockScanInfo->lastBlockDelIndex = index;
pBlockScanInfo->sttBlockDelIndex = index;
return code;
@ -4029,7 +4029,7 @@ int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockSc
tsdbRowMergerAdd(pMerger, pRow1, NULL);
} else {
tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid,
pScanInfo->lastBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline), pScanInfo->lastKeyInStt,
pScanInfo->sttBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline), pScanInfo->lastKeyInStt,
idStr);
break;
}
@ -4697,7 +4697,7 @@ void tsdbReaderClose(STsdbReader* pReader) {
SLastBlockReader* pLReader = pFilesetIter->pLastBlockReader;
tMergeTreeClose(&pLReader->mergeTree);
getLastBlockLoadInfo(pLReader->pInfo, &pCost->lastBlockLoad, &pCost->lastBlockLoadTime);
getSttBlockLoadInfo(pLReader->pInfo, &pCost->sttCost);
pLReader->pInfo = destroyLastBlockLoadInfo(pLReader->pInfo);
taosMemoryFree(pLReader);
@ -4711,7 +4711,7 @@ void tsdbReaderClose(STsdbReader* pReader) {
", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb, createTime:%.2f ms,initDelSkylineIterTime:%.2f "
"ms, %s",
pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, pCost->numOfBlocks,
pCost->blockLoadTime, pCost->buildmemBlock, pCost->lastBlockLoad, pCost->lastBlockLoadTime, pCost->composedBlocks,
pCost->blockLoadTime, pCost->buildmemBlock, pCost->sttBlockLoad, pCost->sttBlockLoadTime, pCost->composedBlocks,
pCost->buildComposedBlockTime, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pCost->createScanInfoList,
pCost->initDelSkylineIterTime, pReader->idStr);

40
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -168,13 +168,11 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo
return TSDB_CODE_SUCCESS;
}
SCostSummary* pSum = &pReader->cost;
SCostSummary* pCost = &pReader->cost;
pIter->pLastBlockReader->uid = 0;
tMergeTreeClose(&pIter->pLastBlockReader->mergeTree);
pReader->status.pLDataIterArray =
destroySttBlockReader(pReader->status.pLDataIterArray, &pSum->lastBlockLoad, &pSum->lastBlockLoadTime);
pReader->status.pLDataIterArray = destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost);
pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
// check file the time range of coverage
@ -1389,7 +1387,7 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc
pLastBlockReader->currentKey = key;
pScanInfo->lastKeyInStt = key;
if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->lastBlockDelIndex, key, ver, pLastBlockReader->order,
if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, pLastBlockReader->order,
pVerRange)) {
return true;
}
@ -2408,7 +2406,7 @@ int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order) {
int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SCostSummary* pCost) {
int32_t code = 0;
int32_t newDelDataInFile = taosArrayGetSize(pBlockScanInfo->pfileDelData);
int32_t newDelDataInFile = taosArrayGetSize(pBlockScanInfo->pFileDelData);
if (newDelDataInFile == 0 &&
((pBlockScanInfo->delSkyline != NULL) || (TARRAY_SIZE(pBlockScanInfo->pMemDelData) == 0))) {
return code;
@ -2422,7 +2420,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t orde
pBlockScanInfo->delSkyline = taosArrayInit(4, sizeof(TSDBKEY));
}
SArray* pSource = pBlockScanInfo->pfileDelData;
SArray* pSource = pBlockScanInfo->pFileDelData;
if (pSource == NULL) {
pSource = pBlockScanInfo->pMemDelData;
} else {
@ -2431,13 +2429,13 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t orde
code = tsdbBuildDeleteSkyline(pSource, 0, taosArrayGetSize(pSource) - 1, pBlockScanInfo->delSkyline);
taosArrayClear(pBlockScanInfo->pfileDelData);
taosArrayClear(pBlockScanInfo->pFileDelData);
int32_t index = getInitialDelIndex(pBlockScanInfo->delSkyline, order);
pBlockScanInfo->iter.index = index;
pBlockScanInfo->iiter.index = index;
pBlockScanInfo->fileDelIndex = index;
pBlockScanInfo->lastBlockDelIndex = index;
pBlockScanInfo->sttBlockDelIndex = index;
double el = taosGetTimestampUs() - st;
pCost->createSkylineIterTime = el / 1000.0;
@ -3411,7 +3409,7 @@ int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockSc
tsdbRowMergerAdd(pMerger, pRow1, NULL);
} else {
tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid,
pScanInfo->lastBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline), pScanInfo->lastKeyInStt,
pScanInfo->sttBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline), pScanInfo->lastKeyInStt,
idStr);
break;
}
@ -4067,18 +4065,20 @@ void tsdbReaderClose2(STsdbReader* pReader) {
taosMemoryFree(pLReader);
}
destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->lastBlockLoad, &pCost->lastBlockLoadTime);
destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost);
taosMemoryFreeClear(pReader->status.uidList.tableUidList);
tsdbDebug(
"%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64
" SMA-time:%.2f ms, fileBlocks:%" PRId64
", fileBlocks-load-time:%.2f ms, "
"build in-memory-block-time:%.2f ms, lastBlocks:%" PRId64 ", lastBlocks-time:%.2f ms, composed-blocks:%" PRId64
"build in-memory-block-time:%.2f ms, sttBlocks:%" PRId64 ", sttBlocks-time:%.2f ms, sttStatisBlock:%" PRId64
", stt-statis-Block-time:%.2f ms, composed-blocks:%" PRId64
", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb, createTime:%.2f ms,createSkylineIterTime:%.2f "
"ms, initLastBlockReader:%.2fms, %s",
pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, pCost->numOfBlocks,
pCost->blockLoadTime, pCost->buildmemBlock, pCost->lastBlockLoad, pCost->lastBlockLoadTime, pCost->composedBlocks,
pCost->blockLoadTime, pCost->buildmemBlock, pCost->sttCost.loadBlocks, pCost->sttCost.blockElapsedTime,
pCost->sttCost.loadStatisBlocks, pCost->sttCost.statisElapsedTime, pCost->composedBlocks,
pCost->buildComposedBlockTime, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pCost->createScanInfoList,
pCost->createSkylineIterTime, pCost->initLastBlockReader, pReader->idStr);
@ -4092,9 +4092,8 @@ void tsdbReaderClose2(STsdbReader* pReader) {
}
int32_t tsdbReaderSuspend2(STsdbReader* pReader) {
int32_t code = 0;
// save reader's base state & reset top state to be reconstructed from base state
int32_t code = 0;
SReaderStatus* pStatus = &pReader->status;
STableBlockScanInfo* pBlockScanInfo = NULL;
@ -4110,9 +4109,9 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) {
}
tsdbDataFileReaderClose(&pReader->pFileReader);
int64_t loadBlocks = 0;
double elapse = 0;
pReader->status.pLDataIterArray = destroySttBlockReader(pReader->status.pLDataIterArray, &loadBlocks, &elapse);
SCostSummary* pCost = &pReader->cost;
pReader->status.pLDataIterArray = destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost);
pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
// resetDataBlockScanInfo excluding lastKey
STableBlockScanInfo** p = NULL;
@ -4134,7 +4133,7 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) {
}
pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline);
pInfo->pfileDelData = taosArrayDestroy(pInfo->pfileDelData);
pInfo->pFileDelData = taosArrayDestroy(pInfo->pFileDelData);
}
} else {
// resetDataBlockScanInfo excluding lastKey
@ -4209,7 +4208,6 @@ static int32_t tsdbSetQueryReseek(void* pQHandle) {
}
tsdbReaderSuspend2(pReader);
tsdbReleaseReader(pReader);
return code;
@ -4223,7 +4221,6 @@ static int32_t tsdbSetQueryReseek(void* pQHandle) {
int32_t tsdbReaderResume2(STsdbReader* pReader) {
int32_t code = 0;
STableBlockScanInfo** pBlockScanInfo = pReader->status.pTableIter;
// restore reader's state
@ -4290,7 +4287,6 @@ static bool tsdbReadRowsCountOnly(STsdbReader* pReader) {
pBlock->info.rows = pReader->rowsNum;
pBlock->info.id.uid = 0;
pBlock->info.dataLoad = 0;
pReader->rowsNum = 0;
return pBlock->info.rows > 0;

14
source/dnode/vnode/src/tsdb/tsdbReadUtil.c
View File

@ -221,7 +221,7 @@ void clearBlockScanInfo(STableBlockScanInfo* p) {
p->delSkyline = taosArrayDestroy(p->delSkyline);
p->pBlockList = taosArrayDestroy(p->pBlockList);
p->pMemDelData = taosArrayDestroy(p->pMemDelData);
p->pfileDelData = taosArrayDestroy(p->pfileDelData);
p->pFileDelData = taosArrayDestroy(p->pFileDelData);
}
void destroyAllBlockScanInfo(SSHashObj* pTableMap) {
@ -238,7 +238,7 @@ void destroyAllBlockScanInfo(SSHashObj* pTableMap) {
static void doCleanupInfoForNextFileset(STableBlockScanInfo* pScanInfo) {
// reset the index in last block when handing a new file
taosArrayClear(pScanInfo->pBlockList);
taosArrayClear(pScanInfo->pfileDelData); // del data from each file set
taosArrayClear(pScanInfo->pFileDelData); // del data from each file set
}
void cleanupInfoFoxNextFileset(SSHashObj* pTableMap) {
@ -502,14 +502,14 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_
if (newTable) {
(*pScanInfo) = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr);
if ((*pScanInfo)->pfileDelData == NULL) {
(*pScanInfo)->pfileDelData = taosArrayInit(4, sizeof(SDelData));
if ((*pScanInfo)->pFileDelData == NULL) {
(*pScanInfo)->pFileDelData = taosArrayInit(4, sizeof(SDelData));
}
}
if (record.version <= pReader->info.verRange.maxVer) {
SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey};
taosArrayPush((*pScanInfo)->pfileDelData, &delData);
taosArrayPush((*pScanInfo)->pFileDelData, &delData);
}
}
@ -556,8 +556,8 @@ static int32_t doLoadTombDataFromTombBlk(const TTombBlkArray* pTombBlkArray, STs
uint64_t uid = pReader->status.uidList.tableUidList[j];
STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr);
if (pScanInfo->pfileDelData == NULL) {
pScanInfo->pfileDelData = taosArrayInit(4, sizeof(SDelData));
if (pScanInfo->pFileDelData == NULL) {
pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData));
}
ETombBlkCheckEnum ret = 0;

7
source/dnode/vnode/src/tsdb/tsdbReadUtil.h
View File

@ -65,12 +65,12 @@ typedef struct STableBlockScanInfo {
TSKEY lastKeyInStt; // last accessed key in stt
SArray* pBlockList; // block data index list, SArray<SBrinRecord>
SArray* pMemDelData; // SArray<SDelData>
SArray* pfileDelData; // SArray<SDelData> from each file set
SArray* pFileDelData; // SArray<SDelData> from each file set
SIterInfo iter; // mem buffer skip list iterator
SIterInfo iiter; // imem buffer skip list iterator
SArray* delSkyline; // delete info for this table
int32_t fileDelIndex; // file block delete index
int32_t lastBlockDelIndex; // delete index for last block
int32_t sttBlockDelIndex; // delete index for last block
bool iterInit; // whether to initialize the in-memory skip list iterator or not
} STableBlockScanInfo;
@ -88,8 +88,7 @@ typedef struct SCostSummary {
double headFileLoadTime;
int64_t smaDataLoad;
double smaLoadTime;
int64_t lastBlockLoad;
double lastBlockLoadTime;
SSttBlockLoadCostInfo sttCost;
int64_t composedBlocks;
double buildComposedBlockTime;
double createScanInfoList;

2
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -44,7 +44,7 @@ int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **p
pFD->pFD = taosOpenFile(path, flag);
if (pFD->pFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
code = TAOS_SYSTEM_ERROR(ENOENT);
taosMemoryFree(pFD);
goto _exit;
}

12
source/dnode/vnode/src/tsdb/tsdbWrite.c
View File

@ -14,6 +14,7 @@
*/
#include "tsdb.h"
#include "vndCos.h"
/**
* @brief max key by precision
@ -76,9 +77,18 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) {
int32_t code = 0;
STsdbKeepCfg *pCfg = &pTsdb->keepCfg;
TSKEY now = taosGetTimestamp(pCfg->precision);
TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep1;
TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2;
TSKEY maxKey = tsMaxKeyByPrecision[pCfg->precision];
int32_t size = taosArrayGetSize(pMsg->aSubmitTbData);
int32_t nlevel = tfsGetLevel(pTsdb->pVnode->pTfs);
if (nlevel > 1 && tsS3Enabled) {
if (nlevel == 3) {
minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep1;
} else if (nlevel == 2) {
minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep0;
}
}
for (int32_t i = 0; i < size; ++i) {
SSubmitTbData *pData = TARRAY_GET_ELEM(pMsg->aSubmitTbData, i);

20
source/dnode/vnode/src/vnd/vnodeCos.c
View File

@ -85,6 +85,25 @@ int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) {
return code;
}
void s3DeleteObjectsByPrefix(const char *prefix_str) {
cos_pool_t *p = NULL;
cos_request_options_t *options = NULL;
int is_cname = 0;
cos_string_t bucket;
cos_status_t *s = NULL;
cos_string_t prefix;
cos_pool_create(&p, NULL);
options = cos_request_options_create(p);
s3InitRequestOptions(options, is_cname);
cos_str_set(&bucket, tsS3BucketName);
cos_str_set(&prefix, prefix_str);
s = cos_delete_objects_by_prefix(options, &bucket, &prefix);
log_status(s);
cos_pool_destroy(p);
}
void s3DeleteObjects(const char *object_name[], int nobject) {
cos_pool_t *p = NULL;
int is_cname = 0;
@ -314,6 +333,7 @@ long s3Size(const char *object_name) {
int32_t s3Init() { return 0; }
void s3CleanUp() {}
int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; }
void s3DeleteObjectsByPrefix(const char *prefix) {}
void s3DeleteObjects(const char *object_name[], int nobject) {}
bool s3Exists(const char *object_name) { return false; }
bool s3Get(const char *object_name, const char *path) { return false; }

18
source/dnode/vnode/src/vnd/vnodeOpen.c
View File

@ -14,6 +14,7 @@
*/
#include "vnd.h"
#include "vndCos.h"
int32_t vnodeGetPrimaryDir(const char *relPath, int32_t diskPrimary, STfs *pTfs, char *buf, size_t bufLen) {
if (pTfs) {
@ -295,9 +296,16 @@ int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t s
return dstVgId;
}
void vnodeDestroy(const char *path, STfs *pTfs) {
void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs) {
vInfo("path:%s is removed while destroy vnode", path);
tfsRmdir(pTfs, path);
int32_t nlevel = tfsGetLevel(pTfs);
if (vgId > 0 && nlevel > 1 && tsS3Enabled) {
char vnode_prefix[TSDB_FILENAME_LEN];
snprintf(vnode_prefix, TSDB_FILENAME_LEN, "v%df", vgId);
s3DeleteObjectsByPrefix(vnode_prefix);
}
}
static int32_t vnodeCheckDisk(int32_t diskPrimary, STfs *pTfs) {
@ -500,13 +508,9 @@ void vnodeClose(SVnode *pVnode) {
// start the sync timer after the queue is ready
int32_t vnodeStart(SVnode *pVnode) { return vnodeSyncStart(pVnode); }
int32_t vnodeIsCatchUp(SVnode *pVnode){
return syncIsCatchUp(pVnode->sync);
}
int32_t vnodeIsCatchUp(SVnode *pVnode) { return syncIsCatchUp(pVnode->sync); }
ESyncRole vnodeGetRole(SVnode *pVnode){
return syncGetRole(pVnode->sync);
}
ESyncRole vnodeGetRole(SVnode *pVnode) { return syncGetRole(pVnode->sync); }
void vnodeStop(SVnode *pVnode) {}

14
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -16,6 +16,7 @@
#include "tencode.h"
#include "tmsg.h"
#include "vnd.h"
#include "vndCos.h"
#include "vnode.h"
#include "vnodeInt.h"
@ -191,7 +192,18 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int
} else if (pVnode->config.tsdbCfg.precision == TSDB_TIME_PRECISION_NANO) {
now *= 1000000;
}
TSKEY minKey = now - tsTickPerMin[pVnode->config.tsdbCfg.precision] * pVnode->config.tsdbCfg.keep2;
int32_t nlevel = tfsGetLevel(pVnode->pTfs);
int32_t keep = pVnode->config.tsdbCfg.keep2;
if (nlevel > 1 && tsS3Enabled) {
if (nlevel == 3) {
keep = pVnode->config.tsdbCfg.keep1;
} else if (nlevel == 2) {
keep = pVnode->config.tsdbCfg.keep0;
}
}
TSKEY minKey = now - tsTickPerMin[pVnode->config.tsdbCfg.precision] * keep;
TSKEY maxKey = tsMaxKeyByPrecision[pVnode->config.tsdbCfg.precision];
if (submitTbData.flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {
uint64_t nColData;

8
source/libs/command/inc/commandInt.h
View File

@ -67,6 +67,8 @@ extern "C" {
#define EXPLAIN_EVENT_FORMAT "Event"
#define EXPLAIN_EVENT_START_FORMAT "Start Cond: "
#define EXPLAIN_EVENT_END_FORMAT "End Cond: "
#define EXPLAIN_GROUP_CACHE_FORMAT "Group Cache"
#define EXPLAIN_DYN_QRY_CTRL_FORMAT "Dynamic Query Control for %s"
#define EXPLAIN_PLANNING_TIME_FORMAT "Planning Time: %.3f ms"
#define EXPLAIN_EXEC_TIME_FORMAT "Execution Time: %.3f ms"
@ -96,6 +98,12 @@ extern "C" {
#define EXPLAIN_OFFSET_FORMAT "offset=%" PRId64
#define EXPLAIN_SOFFSET_FORMAT "soffset=%" PRId64
#define EXPLAIN_PARTITIONS_FORMAT "partitions=%d"
#define EXPLAIN_GLOBAL_GROUP_FORMAT "global_group=%d"
#define EXPLAIN_GROUP_BY_UID_FORMAT "group_by_uid=%d"
#define EXPLAIN_BATCH_SCAN_FORMAT "batch_scan=%d"
#define EXPLAIN_VGROUP_SLOT_FORMAT "vgroup_slot=%d,%d"
#define EXPLAIN_UID_SLOT_FORMAT "uid_slot=%d,%d"
#define EXPLAIN_SRC_SCAN_FORMAT "src_scan=%d,%d"
#define COMMAND_RESET_LOG "resetLog"
#define COMMAND_SCHEDULE_POLICY "schedulePolicy"

179
source/libs/command/src/explain.c
View File

@ -24,6 +24,17 @@
int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pRes);
int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, bool singleChannel);
char *qExplainGetDynQryCtrlType(EDynQueryType type) {
switch (type) {
case DYN_QTYPE_STB_HASH:
return "STable Join";
default:
break;
}
return "unknown task";
}
void qExplainFreeResNode(SExplainResNode *resNode) {
if (NULL == resNode) {
return;
@ -575,11 +586,11 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT);
QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pMergeCondition, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
if (pJoinNode->pOnConditions) {
nodesNodeToSQL(pJoinNode->pPrimKeyCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
if (pJoinNode->pOtherOnCond) {
EXPLAIN_ROW_APPEND(" AND ");
QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pOnConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
nodesNodeToSQL(pJoinNode->pOtherOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
}
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
@ -1522,6 +1533,168 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN:{
SHashJoinPhysiNode *pJoinNode = (SHashJoinPhysiNode *)pNode;
EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType));
EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT);
if (pResNode->pExecInfo) {
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
}
EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pJoinNode->pTargets->length);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pJoinNode->node.pOutputDataBlockDesc->totalRowSize);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_INPUT_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pJoinNode->node.inputTsOrder));
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
if (verbose) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT,
nodesGetOutputNumFromSlotList(pJoinNode->node.pOutputDataBlockDesc->pSlots));
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pJoinNode->node.pOutputDataBlockDesc->outputRowSize);
EXPLAIN_ROW_APPEND_LIMIT(pJoinNode->node.pLimit);
EXPLAIN_ROW_APPEND_SLIMIT(pJoinNode->node.pSlimit);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
if (pJoinNode->node.pConditions || pJoinNode->pFilterConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
if (pJoinNode->node.pConditions) {
QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE,
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
}
if (pJoinNode->pFilterConditions) {
if (pJoinNode->node.pConditions) {
EXPLAIN_ROW_APPEND(" AND ");
}
QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pFilterConditions, tbuf + VARSTR_HEADER_SIZE,
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
}
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
bool conditionsGot = false;
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT);
if (pJoinNode->pPrimKeyCond) {
QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pPrimKeyCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
conditionsGot = true;
}
if (pJoinNode->pColEqCond) {
if (conditionsGot) {
EXPLAIN_ROW_APPEND(" AND ");
}
QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pColEqCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
conditionsGot = true;
}
if (pJoinNode->pTagEqCond) {
if (conditionsGot) {
EXPLAIN_ROW_APPEND(" AND ");
}
QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pTagEqCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
conditionsGot = true;
}
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE:{
SGroupCachePhysiNode *pGroupCache = (SGroupCachePhysiNode *)pNode;
EXPLAIN_ROW_NEW(level, EXPLAIN_GROUP_CACHE_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT);
if (pResNode->pExecInfo) {
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
}
EXPLAIN_ROW_APPEND(EXPLAIN_GLOBAL_GROUP_FORMAT, pGroupCache->globalGrp);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_GROUP_BY_UID_FORMAT, pGroupCache->grpByUid);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_BATCH_SCAN_FORMAT, pGroupCache->batchFetch);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pGroupCache->node.pOutputDataBlockDesc->totalRowSize);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_INPUT_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pGroupCache->node.inputTsOrder));
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
if (verbose) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT,
nodesGetOutputNumFromSlotList(pGroupCache->node.pOutputDataBlockDesc->pSlots));
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pGroupCache->node.pOutputDataBlockDesc->outputRowSize);
EXPLAIN_ROW_APPEND_LIMIT(pGroupCache->node.pLimit);
EXPLAIN_ROW_APPEND_SLIMIT(pGroupCache->node.pSlimit);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
if (pGroupCache->node.pConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pGroupCache->node.pConditions, tbuf + VARSTR_HEADER_SIZE,
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL:{
SDynQueryCtrlPhysiNode *pDyn = (SDynQueryCtrlPhysiNode *)pNode;
EXPLAIN_ROW_NEW(level, EXPLAIN_DYN_QRY_CTRL_FORMAT, qExplainGetDynQryCtrlType(pDyn->qType));
EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT);
if (pResNode->pExecInfo) {
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
}
EXPLAIN_ROW_APPEND(EXPLAIN_BATCH_SCAN_FORMAT, pDyn->stbJoin.batchFetch);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_VGROUP_SLOT_FORMAT, pDyn->stbJoin.vgSlot[0], pDyn->stbJoin.vgSlot[1]);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_UID_SLOT_FORMAT, pDyn->stbJoin.uidSlot[0], pDyn->stbJoin.uidSlot[1]);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_SRC_SCAN_FORMAT, pDyn->stbJoin.srcScan[0], pDyn->stbJoin.srcScan[1]);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pDyn->node.pOutputDataBlockDesc->totalRowSize);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_INPUT_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pDyn->node.inputTsOrder));
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
if (verbose) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT,
nodesGetOutputNumFromSlotList(pDyn->node.pOutputDataBlockDesc->pSlots));
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pDyn->node.pOutputDataBlockDesc->outputRowSize);
EXPLAIN_ROW_APPEND_LIMIT(pDyn->node.pLimit);
EXPLAIN_ROW_APPEND_SLIMIT(pDyn->node.pSlimit);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
if (pDyn->node.pConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pDyn->node.pConditions, tbuf + VARSTR_HEADER_SIZE,
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
}
break;
}
default:
qError("not supported physical node type %d", pNode->type);
return TSDB_CODE_APP_ERROR;

2
source/libs/executor/inc/dataSinkInt.h
View File

@ -35,6 +35,7 @@ typedef struct SDataSinkManager {
typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue);
typedef void (*FEndPut)(struct SDataSinkHandle* pHandle, uint64_t useconds);
typedef void (*FReset)(struct SDataSinkHandle* pHandle);
typedef void (*FGetDataLength)(struct SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryEnd);
typedef int32_t (*FGetDataBlock)(struct SDataSinkHandle* pHandle, SOutputData* pOutput);
typedef int32_t (*FDestroyDataSinker)(struct SDataSinkHandle* pHandle);
@ -43,6 +44,7 @@ typedef int32_t (*FGetCacheSize)(struct SDataSinkHandle* pHandle, uint64_t* size
typedef struct SDataSinkHandle {
FPutDataBlock fPut;
FEndPut fEndPut;
FReset fReset;
FGetDataLength fGetLen;
FGetDataBlock fGetData;
FDestroyDataSinker fDestroy;

87
source/libs/executor/inc/dynqueryctrl.h Executable file
View File

@ -0,0 +1,87 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_DYNQUERYCTRL_H
#define TDENGINE_DYNQUERYCTRL_H
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SDynQueryCtrlExecInfo {
int64_t prevBlkNum;
int64_t prevBlkRows;
int64_t postBlkNum;
int64_t postBlkRows;
int64_t leftCacheNum;
int64_t rightCacheNum;
} SDynQueryCtrlExecInfo;
typedef struct SStbJoinTableList {
void *pNext;
int64_t uidNum;
int64_t readIdx;
int32_t *pLeftVg;
int64_t *pLeftUid;
int32_t *pRightVg;
int64_t *pRightUid;
} SStbJoinTableList;
typedef struct SStbJoinPrevJoinCtx {
bool joinBuild;
SSHashObj* leftHash;
SSHashObj* rightHash;
SSHashObj* leftCache;
SSHashObj* rightCache;
SSHashObj* onceTable;
int64_t tableNum;
SStbJoinTableList* pListHead;
SStbJoinTableList* pListTail;
} SStbJoinPrevJoinCtx;
typedef struct SStbJoinPostJoinCtx {
bool isStarted;
bool leftNeedCache;
bool rightNeedCache;
int32_t leftVgId;
int32_t rightVgId;
int64_t leftCurrUid;
int64_t rightCurrUid;
int64_t rightNextUid;
} SStbJoinPostJoinCtx;
typedef struct SStbJoinDynCtrlCtx {
SStbJoinPrevJoinCtx prev;
SStbJoinPostJoinCtx post;
} SStbJoinDynCtrlCtx;
typedef struct SStbJoinDynCtrlInfo {
SDynQueryCtrlExecInfo execInfo;
SStbJoinDynCtrlBasic basic;
SStbJoinDynCtrlCtx ctx;
int16_t outputBlkId;
} SStbJoinDynCtrlInfo;
typedef struct SDynQueryCtrlOperatorInfo {
EDynQueryType qType;
union {
SStbJoinDynCtrlInfo stbJoin;
};
} SDynQueryCtrlOperatorInfo;
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_DYNQUERYCTRL_H

2
source/libs/executor/inc/executil.h
View File

@ -163,7 +163,7 @@ int32_t getGroupIdFromTagsVal(void* pVnode, uint64_t uid, SNodeList* pGroupNode
size_t getTableTagsBufLen(const SNodeList* pGroups);
SArray* createSortInfo(SNodeList* pNodeList);
SArray* extractPartitionColInfo(SNodeList* pNodeList);
SArray* makeColumnArrayFromList(SNodeList* pNodeList);
int32_t extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols,
int32_t type, SColMatchInfo* pMatchInfo);

65
source/libs/executor/inc/executorInt.h
View File

@ -48,6 +48,14 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int
typedef struct STsdbReader STsdbReader;
typedef struct STqReader STqReader;
typedef enum SOperatorParamType{
OP_GET_PARAM = 1,
OP_NOTIFY_PARAM
} SOperatorParamType;
#define IS_VALID_SESSION_WIN(winInfo) ((winInfo).sessionWin.win.skey > 0)
#define SET_SESSION_WIN_INVALID(winInfo) ((winInfo).sessionWin.win.skey = INT64_MIN)
#define IS_INVALID_SESSION_WIN_KEY(winKey) ((winKey).win.skey <= 0)
@ -105,6 +113,20 @@ typedef struct SExchangeOpStopInfo {
int64_t refId;
} SExchangeOpStopInfo;
typedef struct SGcOperatorParam {
int64_t sessionId;
int32_t downstreamIdx;
int32_t vgId;
int64_t tbUid;
bool needCache;
} SGcOperatorParam;
typedef struct SGcNotifyOperatorParam {
int32_t downstreamIdx;
int32_t vgId;
int64_t tbUid;
} SGcNotifyOperatorParam;
typedef struct SExprSupp {
SExprInfo* pExprInfo;
int32_t numOfExprs; // the number of scalar expression in group operator
@ -115,8 +137,9 @@ typedef struct SExprSupp {
typedef enum {
EX_SOURCE_DATA_NOT_READY = 0x1,
EX_SOURCE_DATA_READY = 0x2,
EX_SOURCE_DATA_EXHAUSTED = 0x3,
EX_SOURCE_DATA_STARTED,
EX_SOURCE_DATA_READY,
EX_SOURCE_DATA_EXHAUSTED,
} EX_SOURCE_STATUS;
#define COL_MATCH_FROM_COL_ID 0x1
@ -138,8 +161,35 @@ typedef struct SLimitInfo {
int64_t numOfOutputRows;
} SLimitInfo;
typedef struct SSortMergeJoinOperatorParam {
bool initDownstream;
} SSortMergeJoinOperatorParam;
typedef struct SExchangeOperatorBasicParam {
int32_t vgId;
int32_t srcOpType;
bool tableSeq;
SArray* uidList;
} SExchangeOperatorBasicParam;
typedef struct SExchangeOperatorBatchParam {
bool multiParams;
SSHashObj* pBatchs; // SExchangeOperatorBasicParam
} SExchangeOperatorBatchParam;
typedef struct SExchangeOperatorParam {
bool multiParams;
SExchangeOperatorBasicParam basic;
} SExchangeOperatorParam;
typedef struct SExchangeSrcIndex {
int32_t srcIdx;
int32_t inUseIdx;
} SExchangeSrcIndex;
typedef struct SExchangeInfo {
SArray* pSources;
SSHashObj* pHashSources;
SArray* pSourceDataInfo;
tsem_t ready;
void* pTransporter;
@ -150,6 +200,7 @@ typedef struct SExchangeInfo {
SArray* pRecycledBlocks; // build a pool for small data block to avoid to repeatly create and then destroy.
SSDataBlock* pDummyBlock; // dummy block, not keep data
bool seqLoadData; // sequential load data or not, false by default
bool dynamicOp;
int32_t current;
SLoadRemoteDataInfo loadInfo;
uint64_t self;
@ -602,7 +653,7 @@ typedef struct SStreamFillOperatorInfo {
SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode);
int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, const char* dbName, SExecTaskInfo* pTaskInfo);
void cleanupQueriedTableScanInfo(SSchemaInfo* pSchemaInfo);
void cleanupQueriedTableScanInfo(void* p);
void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock);
void cleanupBasicInfo(SOptrBasicInfo* pInfo);
@ -707,6 +758,14 @@ uint64_t calcGroupId(char* pData, int32_t len);
void streamOpReleaseState(struct SOperatorInfo* pOperator);
void streamOpReloadState(struct SOperatorInfo* pOperator);
void destroyOperatorParamValue(void* pValues);
int32_t mergeOperatorParams(SOperatorParam* pDst, SOperatorParam* pSrc);
int32_t buildTableScanOperatorParam(SOperatorParam** ppRes, SArray* pUidList, int32_t srcOpType, bool tableSeq);
void freeExchangeGetBasicOperatorParam(void* pParam);
void freeOperatorParam(SOperatorParam* pParam, SOperatorParamType type);
void freeResetOperatorParams(struct SOperatorInfo* pOperator, SOperatorParamType type, bool allFree);
SSDataBlock* getNextBlockFromDownstreamImpl(struct SOperatorInfo* pOperator, int32_t idx, bool clearParam);
bool inSlidingWindow(SInterval* pInterval, STimeWindow* pWin, SDataBlockInfo* pBlockInfo);
bool inCalSlidingWindow(SInterval* pInterval, STimeWindow* pWin, TSKEY calStart, TSKEY calEnd, EStreamType blockType);
bool compareVal(const char* v, const SStateKeys* pKey);

176
source/libs/executor/inc/groupcache.h Executable file
View File

@ -0,0 +1,176 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_GROUPCACHE_H
#define TDENGINE_GROUPCACHE_H
#ifdef __cplusplus
extern "C" {
#endif
#define GROUP_CACHE_DEFAULT_MAX_FILE_SIZE 104857600
#define GROUP_CACHE_MAX_FILE_FDS 10
#define GROUP_CACHE_DEFAULT_VGID 0
#pragma pack(push, 1)
typedef struct SGcBlkBufBasic {
int32_t fileId;
int64_t blkId;
int64_t offset;
int64_t bufSize;
} SGcBlkBufBasic;
#pragma pack(pop)
typedef struct SGroupCacheFileFd {
TdThreadMutex mutex;
TdFilePtr fd;
} SGroupCacheFileFd;
typedef struct SGroupCacheFileInfo {
uint32_t groupNum;
bool deleted;
SGroupCacheFileFd fd;
} SGroupCacheFileInfo;
typedef struct SGcFileCacheCtx {
int64_t fileSize;
int32_t fileId;
SHashObj* pCacheFile;
int32_t baseNameLen;
char baseFilename[256];
} SGcFileCacheCtx;
typedef struct SGcDownstreamCtx {
int32_t id;
SRWLatch grpLock;
int64_t fetchSessionId;
SArray* pNewGrpList; // SArray<SGcNewGroupInfo>
SSHashObj* pVgTbHash; // SHash<SGcVgroupCtx>
SHashObj* pGrpHash;
SRWLatch blkLock;
SSDataBlock* pBaseBlock;
SArray* pFreeBlock;
int64_t lastBlkUid;
SHashObj* pSessions;
SHashObj* pWaitSessions;
SGcFileCacheCtx fileCtx;
} SGcDownstreamCtx;
typedef struct SGcVgroupCtx {
int32_t id;
SArray* pTbList;
uint64_t lastBlkUid;
SGcFileCacheCtx fileCtx;
} SGcVgroupCtx;
typedef struct SGcBlkList {
SRWLatch lock;
SArray* pList;
} SGcBlkList;
typedef struct SGroupCacheData {
TdThreadMutex mutex;
SArray* waitQueue;
bool fetchDone;
bool needCache;
SSDataBlock* pBlock;
SGcVgroupCtx* pVgCtx;
int32_t downstreamIdx;
int32_t vgId;
SGcBlkList blkList;
int32_t fileId;
int64_t startOffset;
} SGroupCacheData;
typedef struct SGroupColInfo {
int32_t slot;
bool vardata;
int32_t bytes;
} SGroupColInfo;
typedef struct SGroupColsInfo {
int32_t colNum;
bool withNull;
SGroupColInfo* pColsInfo;
int32_t bitMapSize;
int32_t bufSize;
char* pBuf;
char* pData;
} SGroupColsInfo;
typedef struct SGcNewGroupInfo {
int32_t vgId;
int64_t uid;
SGroupCacheData* pGroup;
SOperatorParam* pParam;
} SGcNewGroupInfo;
typedef struct SGcSessionCtx {
int32_t downstreamIdx;
SGcOperatorParam* pParam;
SGroupCacheData* pGroupData;
int64_t lastBlkId;
bool semInit;
tsem_t waitSem;
bool newFetch;
int64_t resRows;
} SGcSessionCtx;
typedef struct SGcBlkBufInfo {
SGcBlkBufBasic basic;
void* next;
void* pBuf;
SGcDownstreamCtx* pCtx;
int64_t groupId;
} SGcBlkBufInfo;
typedef struct SGcExecInfo {
int64_t* pDownstreamBlkNum;
} SGcExecInfo;
typedef struct SGcCacheFile {
uint32_t grpNum;
uint32_t grpDone;
int64_t fileSize;
} SGcCacheFile;
typedef struct SGcBlkCacheInfo {
SRWLatch dirtyLock;
SHashObj* pDirtyBlk;
SGcBlkBufInfo* pDirtyHead;
SGcBlkBufInfo* pDirtyTail;
SHashObj* pReadBlk;
int64_t blkCacheSize;
int32_t writeDownstreamId;
} SGcBlkCacheInfo;
typedef struct SGroupCacheOperatorInfo {
int64_t maxCacheSize;
int64_t currentBlkId;
SGroupColsInfo groupColsInfo;
bool globalGrp;
bool grpByUid;
bool batchFetch;
int32_t downstreamNum;
SGcDownstreamCtx* pDownstreams;
SGcBlkCacheInfo blkCache;
SHashObj* pGrpHash;
SGcExecInfo execInfo;
} SGroupCacheOperatorInfo;
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_GROUPCACHE_H

116
source/libs/executor/inc/hashjoin.h Executable file
View File

@ -0,0 +1,116 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_HASHJOIN_H
#define TDENGINE_HASHJOIN_H
#ifdef __cplusplus
extern "C" {
#endif
#define HASH_JOIN_DEFAULT_PAGE_SIZE 10485760
#pragma pack(push, 1)
typedef struct SBufRowInfo {
void* next;
uint16_t pageId;
int32_t offset;
} SBufRowInfo;
#pragma pack(pop)
typedef struct SHJoinCtx {
bool rowRemains;
SBufRowInfo* pBuildRow;
SSDataBlock* pProbeData;
int32_t probeIdx;
} SHJoinCtx;
typedef struct SRowLocation {
SSDataBlock* pDataBlock;
int32_t pos;
} SRowLocation;
typedef struct SHJoinColInfo {
int32_t srcSlot;
int32_t dstSlot;
bool keyCol;
bool vardata;
int32_t* offset;
int32_t bytes;
char* data;
char* bitMap;
} SHJoinColInfo;
typedef struct SBufPageInfo {
int32_t pageSize;
int32_t offset;
char* data;
} SBufPageInfo;
typedef struct SGroupData {
SBufRowInfo* rows;
} SGroupData;
typedef struct SHJoinTableInfo {
int32_t downStreamIdx;
SOperatorInfo* downStream;
int32_t blkId;
SQueryStat inputStat;
int32_t keyNum;
SHJoinColInfo* keyCols;
char* keyBuf;
char* keyData;
int32_t valNum;
SHJoinColInfo* valCols;
char* valData;
int32_t valBitMapSize;
int32_t valBufSize;
SArray* valVarCols;
bool valColExist;
} SHJoinTableInfo;
typedef struct SHJoinExecInfo {
int64_t buildBlkNum;
int64_t buildBlkRows;
int64_t probeBlkNum;
int64_t probeBlkRows;
int64_t resRows;
int64_t expectRows;
} SHJoinExecInfo;
typedef struct SHJoinOperatorInfo {
int32_t joinType;
SHJoinTableInfo tbs[2];
SHJoinTableInfo* pBuild;
SHJoinTableInfo* pProbe;
SSDataBlock* pRes;
int32_t pResColNum;
int8_t* pResColMap;
SArray* pRowBufs;
SNode* pCond;
SSHashObj* pKeyHash;
bool keyHashBuilt;
SHJoinCtx ctx;
SHJoinExecInfo execInfo;
} SHJoinOperatorInfo;
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_HASHJOIN_H

27
source/libs/executor/inc/operator.h
View File

@ -35,6 +35,8 @@ typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* pOptr);
typedef void (*__optr_close_fn_t)(void* param);
typedef int32_t (*__optr_explain_fn_t)(struct SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len);
typedef int32_t (*__optr_reqBuf_fn_t)(struct SOperatorInfo* pOptr);
typedef SSDataBlock* (*__optr_get_ext_fn_t)(struct SOperatorInfo* pOptr, SOperatorParam* param);
typedef int32_t (*__optr_notify_fn_t)(struct SOperatorInfo* pOptr, SOperatorParam* param);
typedef void (*__optr_state_fn_t)(struct SOperatorInfo* pOptr);
typedef struct SOperatorFpSet {
@ -46,6 +48,8 @@ typedef struct SOperatorFpSet {
__optr_encode_fn_t encodeResultRow;
__optr_decode_fn_t decodeResultRow;
__optr_explain_fn_t getExplainFn;
__optr_get_ext_fn_t getNextExtFn;
__optr_notify_fn_t notifyFn;
__optr_state_fn_t releaseStreamStateFn;
__optr_state_fn_t reloadStreamStateFn;
} SOperatorFpSet;
@ -61,6 +65,8 @@ typedef struct SOperatorInfo {
uint16_t operatorType;
int16_t resultDataBlockId;
bool blocking; // block operator or not
bool transparent;
bool dynamicTask;
uint8_t status; // denote if current operator is completed
char* name; // name, for debug purpose
void* info; // extension attribution
@ -68,8 +74,13 @@ typedef struct SOperatorInfo {
SExecTaskInfo* pTaskInfo;
SOperatorCostInfo cost;
SResultInfo resultInfo;
SOperatorParam* pOperatorGetParam;
SOperatorParam* pOperatorNotifyParam;
SOperatorParam** pDownstreamGetParams;
SOperatorParam** pDownstreamNotifyParams;
struct SOperatorInfo** pDownstream; // downstram pointer list
int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator
int32_t numOfRealDownstream;
SOperatorFpSet fpSet;
} SOperatorInfo;
@ -129,6 +140,8 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode
SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createHashJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SHashJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle);
SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild, SReadHandle* pHandle);
@ -142,10 +155,16 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createGroupCacheOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SGroupCachePhysiNode* pPhyciNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SDynQueryCtrlPhysiNode* pPhyciNode, SExecTaskInfo* pTaskInfo);
// clang-format on
SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup,
__optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, __optr_explain_fn_t explain);
__optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn,
__optr_explain_fn_t explain, __optr_get_ext_fn_t nextExtFn, __optr_notify_fn_t notifyFn);
void setOperatorStreamStateFn(SOperatorInfo* pOperator, __optr_state_fn_t relaseFn, __optr_state_fn_t reloadFn);
int32_t optrDummyOpenFn(SOperatorInfo* pOperator);
int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num);
@ -153,6 +172,11 @@ void setOperatorCompleted(SOperatorInfo* pOperator);
void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, bool blocking, int32_t status,
void* pInfo, SExecTaskInfo* pTaskInfo);
int32_t optrDefaultBufFn(SOperatorInfo* pOperator);
SSDataBlock* optrDefaultGetNextExtFn(struct SOperatorInfo* pOperator, SOperatorParam* pParam);
int32_t optrDefaultNotifyFn(struct SOperatorInfo* pOperator, SOperatorParam* pParam);
SSDataBlock* getNextBlockFromDownstream(struct SOperatorInfo* pOperator, int32_t idx);
SSDataBlock* getNextBlockFromDownstreamRemain(struct SOperatorInfo* pOperator, int32_t idx);
int16_t getOperatorResultBlockId(struct SOperatorInfo* pOperator, int32_t idx);
SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SNode* pTagCond,
SNode* pTagIndexCond, const char* pUser, const char* dbname);
@ -162,6 +186,7 @@ SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, con
int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder);
int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr, SStorageAPI* pAPI);
int32_t getOperatorExplainExecInfo(struct SOperatorInfo* operatorInfo, SArray* pExecInfoList);
void * getOperatorParam(int32_t opType, SOperatorParam* param, int32_t idx);
#ifdef __cplusplus
}

5
source/libs/executor/inc/querytask.h
View File

@ -39,6 +39,7 @@ typedef struct STaskIdInfo {
uint64_t templateId;
char* str;
int32_t vgId;
uint64_t taskId;
} STaskIdInfo;
typedef struct STaskCostInfo {
@ -83,6 +84,7 @@ struct SExecTaskInfo {
int32_t qbufQuota; // total available buffer (in KB) during execution query
int64_t version; // used for stream to record wal version, why not move to sschemainfo
SStreamTaskInfo streamInfo;
SArray* schemaInfos;
SSchemaInfo schemaInfo;
const char* sql; // query sql string
jmp_buf env; // jump to this position when error happens.
@ -94,6 +96,9 @@ struct SExecTaskInfo {
STaskStopInfo stopInfo;
SRWLatch lock; // secure the access of STableListInfo
SStorageAPI storageAPI;
int8_t dynamicTask;
SOperatorParam* pOpParam;
bool paramSet;
};
void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst);

4
source/libs/executor/src/aggregateoperator.c
View File

@ -115,7 +115,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG,
!pAggNode->node.forceCreateNonBlockingOptr, OP_NOT_OPENED, pInfo, pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, getAggregateResult, NULL, destroyAggOperatorInfo,
optrDefaultBufFn, NULL);
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
STableScanInfo* pTableScanInfo = downstream->info;
@ -187,7 +187,7 @@ static bool nextGroupedResult(SOperatorInfo* pOperator) {
}
while (1) {
bool blockAllocated = false;
pBlock = downstream->fpSet.getNextFn(downstream);
pBlock = getNextBlockFromDownstream(pOperator, 0);
if (pBlock == NULL) {
if (!pAggInfo->hasValidBlock) {
createDataBlockForEmptyInput(pOperator, &pBlock);

5
source/libs/executor/src/cachescanoperator.c
View File

@ -134,7 +134,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock);
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doScanCache, NULL, destroyCacheScanOperator, optrDefaultBufFn, NULL);
createOperatorFpSet(optrDummyOpenFn, doScanCache, NULL, destroyCacheScanOperator, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
pOperator->cost.openCost = 0;
return pOperator;
@ -330,7 +330,8 @@ int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTask
return TSDB_CODE_OUT_OF_MEMORY;
}
SSchemaWrapper* pWrapper = pTaskInfo->schemaInfo.sw;
SSchemaInfo* pSchemaInfo = taosArrayGetLast(pTaskInfo->schemaInfos);
SSchemaWrapper* pWrapper = pSchemaInfo->sw;
for (int32_t i = 0; i < numOfCols; ++i) {
SColMatchItem* pColMatch = taosArrayGet(pColMatchInfo, i);

9
source/libs/executor/src/dataDispatcher.c
View File

@ -156,6 +156,13 @@ static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) {
taosThreadMutexUnlock(&pDispatcher->mutex);
}
static void resetDispatcher(struct SDataSinkHandle* pHandle) {
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
taosThreadMutexLock(&pDispatcher->mutex);
pDispatcher->queryEnd = false;
taosThreadMutexUnlock(&pDispatcher->mutex);
}
static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryEnd) {
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
if (taosQueueEmpty(pDispatcher->pDataBlocks)) {
@ -182,6 +189,7 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE
((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->numOfRows);
}
static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
if (NULL == pDispatcher->nextOutput.pData) {
@ -244,6 +252,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD
}
dispatcher->sink.fPut = putDataBlock;
dispatcher->sink.fEndPut = endPut;
dispatcher->sink.fReset = resetDispatcher;
dispatcher->sink.fGetLen = getDataLength;
dispatcher->sink.fGetData = getDataBlock;
dispatcher->sink.fDestroy = destroyDataSinker;

7
source/libs/executor/src/dataSinkMgt.c
View File

@ -59,6 +59,13 @@ void dsEndPut(DataSinkHandle handle, uint64_t useconds) {
return pHandleImpl->fEndPut(pHandleImpl, useconds);
}
void dsReset(DataSinkHandle handle) {
SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle;
if (pHandleImpl->fReset) {
return pHandleImpl->fReset(pHandleImpl);
}
}
void dsGetDataLength(DataSinkHandle handle, int64_t* pLen, bool* pQueryEnd) {
SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle;
pHandleImpl->fGetLen(pHandleImpl, pLen, pQueryEnd);

885
source/libs/executor/src/dynqueryctrloperator.c Executable file
View File

@ -0,0 +1,885 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "executorInt.h"
#include "filter.h"
#include "function.h"
#include "operator.h"
#include "os.h"
#include "querynodes.h"
#include "querytask.h"
#include "tcompare.h"
#include "tdatablock.h"
#include "thash.h"
#include "tmsg.h"
#include "ttypes.h"
#include "dynqueryctrl.h"
int64_t gSessionId = 0;
void freeVgTableList(void* ptr) {
taosArrayDestroy(*(SArray**)ptr);
}
static void destroyStbJoinTableList(SStbJoinTableList* pListHead) {
SStbJoinTableList* pNext = NULL;
while (pListHead) {
taosMemoryFree(pListHead->pLeftVg);
taosMemoryFree(pListHead->pLeftUid);
taosMemoryFree(pListHead->pRightVg);
taosMemoryFree(pListHead->pRightUid);
pNext = pListHead->pNext;
taosMemoryFree(pListHead);
pListHead = pNext;
}
}
static void destroyStbJoinDynCtrlInfo(SStbJoinDynCtrlInfo* pStbJoin) {
qError("dynQueryCtrl exec info, prevBlk:%" PRId64 ", prevRows:%" PRId64 ", postBlk:%" PRId64 ", postRows:%" PRId64 ", leftCacheNum:%" PRId64 ", rightCacheNum:%" PRId64,
pStbJoin->execInfo.prevBlkNum, pStbJoin->execInfo.prevBlkRows, pStbJoin->execInfo.postBlkNum,
pStbJoin->execInfo.postBlkRows, pStbJoin->execInfo.leftCacheNum, pStbJoin->execInfo.rightCacheNum);
if (pStbJoin->basic.batchFetch) {
if (pStbJoin->ctx.prev.leftHash) {
tSimpleHashSetFreeFp(pStbJoin->ctx.prev.leftHash, freeVgTableList);
tSimpleHashCleanup(pStbJoin->ctx.prev.leftHash);
}
if (pStbJoin->ctx.prev.rightHash) {
tSimpleHashSetFreeFp(pStbJoin->ctx.prev.rightHash, freeVgTableList);
tSimpleHashCleanup(pStbJoin->ctx.prev.rightHash);
}
} else {
if (pStbJoin->ctx.prev.leftCache) {
tSimpleHashCleanup(pStbJoin->ctx.prev.leftCache);
}
if (pStbJoin->ctx.prev.rightCache) {
tSimpleHashCleanup(pStbJoin->ctx.prev.rightCache);
}
if (pStbJoin->ctx.prev.onceTable) {
tSimpleHashCleanup(pStbJoin->ctx.prev.onceTable);
}
}
destroyStbJoinTableList(pStbJoin->ctx.prev.pListHead);
}
static void destroyDynQueryCtrlOperator(void* param) {
SDynQueryCtrlOperatorInfo* pDyn = (SDynQueryCtrlOperatorInfo*)param;
switch (pDyn->qType) {
case DYN_QTYPE_STB_HASH:
destroyStbJoinDynCtrlInfo(&pDyn->stbJoin);
break;
default:
qError("unsupported dynamic query ctrl type: %d", pDyn->qType);
break;
}
taosMemoryFreeClear(param);
}
static FORCE_INLINE bool tableNeedCache(int64_t uid, SStbJoinPrevJoinCtx* pPrev, SStbJoinPostJoinCtx* pPost, bool rightTable, bool batchFetch) {
if (batchFetch) {
return true;
}
if (rightTable) {
return pPost->rightCurrUid == pPost->rightNextUid;
}
uint32_t* num = tSimpleHashGet(pPrev->leftCache, &uid, sizeof(uid));
return (NULL == num) ? false : true;
}
static void updatePostJoinCurrTableInfo(SStbJoinDynCtrlInfo* pStbJoin) {
SStbJoinPrevJoinCtx* pPrev = &pStbJoin->ctx.prev;
SStbJoinPostJoinCtx* pPost = &pStbJoin->ctx.post;
SStbJoinTableList* pNode = pPrev->pListHead;
int32_t* leftVgId = pNode->pLeftVg + pNode->readIdx;
int32_t* rightVgId = pNode->pRightVg + pNode->readIdx;
int64_t* leftUid = pNode->pLeftUid + pNode->readIdx;
int64_t* rightUid = pNode->pRightUid + pNode->readIdx;
int64_t readIdx = pNode->readIdx + 1;
int64_t rightPrevUid = pPost->rightCurrUid;
pPost->leftCurrUid = *leftUid;
pPost->rightCurrUid = *rightUid;
pPost->leftVgId = *leftVgId;
pPost->rightVgId = *rightVgId;
while (true) {
if (readIdx < pNode->uidNum) {
pPost->rightNextUid = *(pNode->pRightUid + readIdx);
break;
}
pNode = pNode->pNext;
if (NULL == pNode) {
pPost->rightNextUid = 0;
break;
}
rightUid = pNode->pRightUid;
readIdx = 0;
}
pPost->leftNeedCache = tableNeedCache(*leftUid, pPrev, pPost, false, pStbJoin->basic.batchFetch);
pPost->rightNeedCache = tableNeedCache(*rightUid, pPrev, pPost, true, pStbJoin->basic.batchFetch);
if (pPost->rightNeedCache && rightPrevUid != pPost->rightCurrUid) {
tSimpleHashPut(pPrev->rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid), NULL, 0);
pStbJoin->execInfo.rightCacheNum++;
}
}
static int32_t buildGroupCacheOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t vgId, int64_t tbUid, bool needCache, SOperatorParam* pChild) {
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (pChild) {
(*ppRes)->pChildren = taosArrayInit(1, POINTER_BYTES);
if (NULL == *ppRes) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (NULL == taosArrayPush((*ppRes)->pChildren, &pChild)) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} else {
(*ppRes)->pChildren = NULL;
}
SGcOperatorParam* pGc = taosMemoryMalloc(sizeof(SGcOperatorParam));
if (NULL == pGc) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pGc->sessionId = atomic_add_fetch_64(&gSessionId, 1);
pGc->downstreamIdx = downstreamIdx;
pGc->vgId = vgId;
pGc->tbUid = tbUid;
pGc->needCache = needCache;
(*ppRes)->opType = QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE;
(*ppRes)->downstreamIdx = downstreamIdx;
(*ppRes)->value = pGc;
return TSDB_CODE_SUCCESS;
}
static int32_t buildGroupCacheNotifyOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t vgId, int64_t tbUid) {
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(*ppRes)->pChildren = NULL;
SGcNotifyOperatorParam* pGc = taosMemoryMalloc(sizeof(SGcNotifyOperatorParam));
if (NULL == pGc) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pGc->downstreamIdx = downstreamIdx;
pGc->vgId = vgId;
pGc->tbUid = tbUid;
(*ppRes)->opType = QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE;
(*ppRes)->downstreamIdx = downstreamIdx;
(*ppRes)->value = pGc;
return TSDB_CODE_SUCCESS;
}
static int32_t buildExchangeOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t* pVgId, int64_t* pUid) {
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(*ppRes)->pChildren = NULL;
SExchangeOperatorParam* pExc = taosMemoryMalloc(sizeof(SExchangeOperatorParam));
if (NULL == pExc) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pExc->multiParams = false;
pExc->basic.vgId = *pVgId;
pExc->basic.tableSeq = true;
pExc->basic.srcOpType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN;
pExc->basic.uidList = taosArrayInit(1, sizeof(int64_t));
if (NULL == pExc->basic.uidList) {
taosMemoryFree(pExc);
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArrayPush(pExc->basic.uidList, pUid);
(*ppRes)->opType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE;
(*ppRes)->downstreamIdx = downstreamIdx;
(*ppRes)->value = pExc;
return TSDB_CODE_SUCCESS;
}
static int32_t buildBatchExchangeOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, SSHashObj* pVg) {
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(*ppRes)->pChildren = NULL;
SExchangeOperatorBatchParam* pExc = taosMemoryMalloc(sizeof(SExchangeOperatorBatchParam));
if (NULL == pExc) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pExc->multiParams = true;
pExc->pBatchs = tSimpleHashInit(tSimpleHashGetSize(pVg), taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT));
if (NULL == pExc->pBatchs) {
taosMemoryFree(pExc);
return TSDB_CODE_OUT_OF_MEMORY;
}
tSimpleHashSetFreeFp(pExc->pBatchs, freeExchangeGetBasicOperatorParam);
SExchangeOperatorBasicParam basic;
basic.srcOpType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN;
int32_t iter = 0;
void* p = NULL;
while (NULL != (p = tSimpleHashIterate(pVg, p, &iter))) {
int32_t* pVgId = tSimpleHashGetKey(p, NULL);
SArray* pUidList = *(SArray**)p;
basic.vgId = *pVgId;
basic.uidList = pUidList;
basic.tableSeq = false;
tSimpleHashPut(pExc->pBatchs, pVgId, sizeof(*pVgId), &basic, sizeof(basic));
qTrace("build downstreamIdx %d batch scan, vgId:%d, uidNum:%" PRId64, downstreamIdx, *pVgId, (int64_t)taosArrayGetSize(pUidList));
*(SArray**)p = NULL;
}
(*ppRes)->opType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE;
(*ppRes)->downstreamIdx = downstreamIdx;
(*ppRes)->value = pExc;
return TSDB_CODE_SUCCESS;
}
static int32_t buildMergeJoinOperatorParam(SOperatorParam** ppRes, bool initParam, SOperatorParam* pChild0, SOperatorParam* pChild1) {
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(*ppRes)->pChildren = taosArrayInit(2, POINTER_BYTES);
if (NULL == *ppRes) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (NULL == taosArrayPush((*ppRes)->pChildren, &pChild0)) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (NULL == taosArrayPush((*ppRes)->pChildren, &pChild1)) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SSortMergeJoinOperatorParam* pJoin = taosMemoryMalloc(sizeof(SSortMergeJoinOperatorParam));
if (NULL == pJoin) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pJoin->initDownstream = initParam;
(*ppRes)->opType = QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN;
(*ppRes)->value = pJoin;
return TSDB_CODE_SUCCESS;
}
static int32_t buildMergeJoinNotifyOperatorParam(SOperatorParam** ppRes, SOperatorParam* pChild0, SOperatorParam* pChild1) {
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(*ppRes)->pChildren = taosArrayInit(2, POINTER_BYTES);
if (NULL == *ppRes) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (pChild0 && NULL == taosArrayPush((*ppRes)->pChildren, &pChild0)) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (pChild1 && NULL == taosArrayPush((*ppRes)->pChildren, &pChild1)) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(*ppRes)->opType = QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN;
(*ppRes)->value = NULL;
return TSDB_CODE_SUCCESS;
}
static int32_t buildBatchTableScanOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, SSHashObj* pVg) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t vgNum = tSimpleHashGetSize(pVg);
if (vgNum <= 0 || vgNum > 1) {
qError("Invalid vgroup num %d to build table scan operator param", vgNum);
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
int32_t iter = 0;
void* p = NULL;
while (NULL != (p = tSimpleHashIterate(pVg, p, &iter))) {
int32_t* pVgId = tSimpleHashGetKey(p, NULL);
SArray* pUidList = *(SArray**)p;
code = buildTableScanOperatorParam(ppRes, pUidList, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false);
if (code) {
return code;
}
taosArrayDestroy(pUidList);
*(SArray**)p = NULL;
}
return TSDB_CODE_SUCCESS;
}
static int32_t buildSingleTableScanOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t* pVgId, int64_t* pUid) {
SArray* pUidList = taosArrayInit(1, sizeof(int64_t));
if (NULL == pUidList) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArrayPush(pUidList, pUid);
int32_t code = buildTableScanOperatorParam(ppRes, pUidList, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, true);
taosArrayDestroy(pUidList);
if (code) {
return code;
}
return TSDB_CODE_SUCCESS;
}
static int32_t buildSeqStbJoinOperatorParam(SDynQueryCtrlOperatorInfo* pInfo, SStbJoinPrevJoinCtx* pPrev, SStbJoinPostJoinCtx* pPost, SOperatorParam** ppParam) {
int64_t rowIdx = pPrev->pListHead->readIdx;
SOperatorParam* pSrcParam0 = NULL;
SOperatorParam* pSrcParam1 = NULL;
SOperatorParam* pGcParam0 = NULL;
SOperatorParam* pGcParam1 = NULL;
int32_t* leftVg = pPrev->pListHead->pLeftVg + rowIdx;
int64_t* leftUid = pPrev->pListHead->pLeftUid + rowIdx;
int32_t* rightVg = pPrev->pListHead->pRightVg + rowIdx;
int64_t* rightUid = pPrev->pListHead->pRightUid + rowIdx;
int32_t code = TSDB_CODE_SUCCESS;
qError("start %" PRId64 ":%" PRId64 "th stbJoin, left:%d,%" PRIu64 " - right:%d,%" PRIu64,
rowIdx, pPrev->tableNum, *leftVg, *leftUid, *rightVg, *rightUid);
updatePostJoinCurrTableInfo(&pInfo->stbJoin);
if (pInfo->stbJoin.basic.batchFetch) {
if (pPrev->leftHash) {
code = pInfo->stbJoin.basic.srcScan[0] ? buildBatchTableScanOperatorParam(&pSrcParam0, 0, pPrev->leftHash) : buildBatchExchangeOperatorParam(&pSrcParam0, 0, pPrev->leftHash);
if (TSDB_CODE_SUCCESS == code) {
code = pInfo->stbJoin.basic.srcScan[1] ? buildBatchTableScanOperatorParam(&pSrcParam1, 1, pPrev->rightHash) : buildBatchExchangeOperatorParam(&pSrcParam1, 1, pPrev->rightHash);
}
if (TSDB_CODE_SUCCESS == code) {
tSimpleHashCleanup(pPrev->leftHash);
tSimpleHashCleanup(pPrev->rightHash);
pPrev->leftHash = NULL;
pPrev->rightHash = NULL;
}
}
} else {
code = pInfo->stbJoin.basic.srcScan[0] ? buildSingleTableScanOperatorParam(&pSrcParam0, 0, leftVg, leftUid) : buildExchangeOperatorParam(&pSrcParam0, 0, leftVg, leftUid);
if (TSDB_CODE_SUCCESS == code) {
code = pInfo->stbJoin.basic.srcScan[1] ? buildSingleTableScanOperatorParam(&pSrcParam1, 1, rightVg, rightUid) : buildExchangeOperatorParam(&pSrcParam1, 1, rightVg, rightUid);
}
}
if (TSDB_CODE_SUCCESS == code) {
code = buildGroupCacheOperatorParam(&pGcParam0, 0, *leftVg, *leftUid, pPost->leftNeedCache, pSrcParam0);
}
if (TSDB_CODE_SUCCESS == code) {
code = buildGroupCacheOperatorParam(&pGcParam1, 1, *rightVg, *rightUid, pPost->rightNeedCache, pSrcParam1);
}
if (TSDB_CODE_SUCCESS == code) {
code = buildMergeJoinOperatorParam(ppParam, pSrcParam0 ? true : false, pGcParam0, pGcParam1);
}
return code;
}
static void seqJoinLaunchNewRetrieveImpl(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
SDynQueryCtrlOperatorInfo* pInfo = pOperator->info;
SStbJoinDynCtrlInfo* pStbJoin = (SStbJoinDynCtrlInfo*)&pInfo->stbJoin;
SStbJoinPrevJoinCtx* pPrev = &pStbJoin->ctx.prev;
SStbJoinPostJoinCtx* pPost = &pStbJoin->ctx.post;
SOperatorParam* pParam = NULL;
int32_t code = buildSeqStbJoinOperatorParam(pInfo, pPrev, pPost, &pParam);
if (TSDB_CODE_SUCCESS != code) {
pOperator->pTaskInfo->code = code;
T_LONG_JMP(pOperator->pTaskInfo->env, pOperator->pTaskInfo->code);
}
qError("dynamic post task begin");
*ppRes = pOperator->pDownstream[1]->fpSet.getNextExtFn(pOperator->pDownstream[1], pParam);
if (*ppRes) {
pPost->isStarted = true;
pStbJoin->execInfo.postBlkNum++;
pStbJoin->execInfo.postBlkRows += (*ppRes)->info.rows;
qError("join res block retrieved");
} else {
qError("Empty join res block retrieved");
}
}
static int32_t notifySeqJoinTableCacheEnd(SOperatorInfo* pOperator, SStbJoinPostJoinCtx* pPost, bool leftTable) {
SOperatorParam* pGcParam = NULL;
SOperatorParam* pMergeJoinParam = NULL;
int32_t downstreamId = leftTable ? 0 : 1;
int32_t vgId = leftTable ? pPost->leftVgId : pPost->rightVgId;
int64_t uid = leftTable ? pPost->leftCurrUid : pPost->rightCurrUid;
qError("notify table %" PRIu64 " in vgId %d downstreamId %d cache end", uid, vgId, downstreamId);
int32_t code = buildGroupCacheNotifyOperatorParam(&pGcParam, downstreamId, vgId, uid);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
code = buildMergeJoinNotifyOperatorParam(&pMergeJoinParam, pGcParam, NULL);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
return optrDefaultNotifyFn(pOperator->pDownstream[1], pMergeJoinParam);
}
static void handleSeqJoinCurrRetrieveEnd(SOperatorInfo* pOperator, SStbJoinDynCtrlInfo* pStbJoin) {
SStbJoinPostJoinCtx* pPost = &pStbJoin->ctx.post;
pPost->isStarted = false;
if (pStbJoin->basic.batchFetch) {
return;
}
if (pPost->leftNeedCache) {
uint32_t* num = tSimpleHashGet(pStbJoin->ctx.prev.leftCache, &pPost->leftCurrUid, sizeof(pPost->leftCurrUid));
if (--(*num) <= 0) {
tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, &pPost->leftCurrUid, sizeof(pPost->leftCurrUid));
notifySeqJoinTableCacheEnd(pOperator, pPost, true);
}
}
if (!pPost->rightNeedCache) {
void* v = tSimpleHashGet(pStbJoin->ctx.prev.rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid));
if (NULL != v) {
tSimpleHashRemove(pStbJoin->ctx.prev.rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid));
notifySeqJoinTableCacheEnd(pOperator, pPost, false);
}
}
}
static FORCE_INLINE void seqJoinContinueCurrRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
SDynQueryCtrlOperatorInfo* pInfo = pOperator->info;
SStbJoinPostJoinCtx* pPost = &pInfo->stbJoin.ctx.post;
SStbJoinPrevJoinCtx* pPrev = &pInfo->stbJoin.ctx.prev;
if (!pPost->isStarted) {
return;
}
qDebug("%s dynQueryCtrl continue to retrieve block from post op", GET_TASKID(pOperator->pTaskInfo));
*ppRes = getNextBlockFromDownstream(pOperator, 1);
if (NULL == *ppRes) {
handleSeqJoinCurrRetrieveEnd(pOperator, &pInfo->stbJoin);
pPrev->pListHead->readIdx++;
} else {
pInfo->stbJoin.execInfo.postBlkNum++;
pInfo->stbJoin.execInfo.postBlkRows += (*ppRes)->info.rows;
return;
}
}
static FORCE_INLINE int32_t addToJoinVgroupHash(SSHashObj* pHash, void* pKey, int32_t keySize, void* pVal, int32_t valSize) {
SArray** ppArray = tSimpleHashGet(pHash, pKey, keySize);
if (NULL == ppArray) {
SArray* pArray = taosArrayInit(10, valSize);
if (NULL == pArray) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (NULL == taosArrayPush(pArray, pVal)) {
taosArrayDestroy(pArray);
return TSDB_CODE_OUT_OF_MEMORY;
}
if (tSimpleHashPut(pHash, pKey, keySize, &pArray, POINTER_BYTES)) {
taosArrayDestroy(pArray);
return TSDB_CODE_OUT_OF_MEMORY;
}
return TSDB_CODE_SUCCESS;
}
if (NULL == taosArrayPush(*ppArray, pVal)) {
return TSDB_CODE_OUT_OF_MEMORY;
}
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE int32_t addToJoinTableHash(SSHashObj* pHash, SSHashObj* pOnceHash, void* pKey, int32_t keySize) {
uint32_t* pNum = tSimpleHashGet(pHash, pKey, keySize);
if (NULL == pNum) {
uint32_t n = 1;
if (tSimpleHashPut(pHash, pKey, keySize, &n, sizeof(n))) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (tSimpleHashPut(pOnceHash, pKey, keySize, NULL, 0)) {
return TSDB_CODE_OUT_OF_MEMORY;
}
return TSDB_CODE_SUCCESS;
}
switch (*pNum) {
case 0:
break;
case UINT32_MAX:
*pNum = 0;
break;
default:
if (1 == (*pNum)) {
tSimpleHashRemove(pOnceHash, pKey, keySize);
}
(*pNum)++;
break;
}
return TSDB_CODE_SUCCESS;
}
static void freeStbJoinTableList(SStbJoinTableList* pList) {
if (NULL == pList) {
return;
}
taosMemoryFree(pList->pLeftVg);
taosMemoryFree(pList->pLeftUid);
taosMemoryFree(pList->pRightVg);
taosMemoryFree(pList->pRightUid);
taosMemoryFree(pList);
}
static int32_t appendStbJoinTableList(SStbJoinPrevJoinCtx* pCtx, int64_t rows, int32_t* pLeftVg, int64_t* pLeftUid, int32_t* pRightVg, int64_t* pRightUid) {
SStbJoinTableList* pNew = taosMemoryMalloc(sizeof(SStbJoinTableList));
if (NULL == pNew) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pNew->pLeftVg = taosMemoryMalloc(rows * sizeof(*pLeftVg));
pNew->pLeftUid = taosMemoryMalloc(rows * sizeof(*pLeftUid));
pNew->pRightVg = taosMemoryMalloc(rows * sizeof(*pRightVg));
pNew->pRightUid = taosMemoryMalloc(rows * sizeof(*pRightUid));
if (NULL == pNew->pLeftVg || NULL == pNew->pLeftUid || NULL == pNew->pRightVg || NULL == pNew->pRightUid) {
freeStbJoinTableList(pNew);
return TSDB_CODE_OUT_OF_MEMORY;
}
memcpy(pNew->pLeftVg, pLeftVg, rows * sizeof(*pLeftVg));
memcpy(pNew->pLeftUid, pLeftUid, rows * sizeof(*pLeftUid));
memcpy(pNew->pRightVg, pRightVg, rows * sizeof(*pRightVg));
memcpy(pNew->pRightUid, pRightUid, rows * sizeof(*pRightUid));
pNew->readIdx = 0;
pNew->uidNum = rows;
pNew->pNext = NULL;
if (pCtx->pListTail) {
pCtx->pListTail->pNext = pNew;
pCtx->pListTail = pNew;
} else {
pCtx->pListHead = pNew;
pCtx->pListTail= pNew;
}
return TSDB_CODE_SUCCESS;
}
static void doBuildStbJoinTableHash(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
SDynQueryCtrlOperatorInfo* pInfo = pOperator->info;
SStbJoinDynCtrlInfo* pStbJoin = (SStbJoinDynCtrlInfo*)&pInfo->stbJoin;
SColumnInfoData* pVg0 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.vgSlot[0]);
SColumnInfoData* pVg1 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.vgSlot[1]);
SColumnInfoData* pUid0 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.uidSlot[0]);
SColumnInfoData* pUid1 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.uidSlot[1]);
if (pStbJoin->basic.batchFetch) {
for (int32_t i = 0; i < pBlock->info.rows; ++i) {
int32_t* leftVg = (int32_t*)(pVg0->pData + pVg0->info.bytes * i);
int64_t* leftUid = (int64_t*)(pUid0->pData + pUid0->info.bytes * i);
int32_t* rightVg = (int32_t*)(pVg1->pData + pVg1->info.bytes * i);
int64_t* rightUid = (int64_t*)(pUid1->pData + pUid1->info.bytes * i);
code = addToJoinVgroupHash(pStbJoin->ctx.prev.leftHash, leftVg, sizeof(*leftVg), leftUid, sizeof(*leftUid));
if (TSDB_CODE_SUCCESS != code) {
break;
}
code = addToJoinVgroupHash(pStbJoin->ctx.prev.rightHash, rightVg, sizeof(*rightVg), rightUid, sizeof(*rightUid));
if (TSDB_CODE_SUCCESS != code) {
break;
}
}
} else {
for (int32_t i = 0; i < pBlock->info.rows; ++i) {
int64_t* leftUid = (int64_t*)(pUid0->pData + pUid0->info.bytes * i);
code = addToJoinTableHash(pStbJoin->ctx.prev.leftCache, pStbJoin->ctx.prev.onceTable, leftUid, sizeof(*leftUid));
if (TSDB_CODE_SUCCESS != code) {
break;
}
}
}
if (TSDB_CODE_SUCCESS == code) {
code = appendStbJoinTableList(&pStbJoin->ctx.prev, pBlock->info.rows, (int32_t*)pVg0->pData, (int64_t*)pUid0->pData, (int32_t*)pVg1->pData, (int64_t*)pUid1->pData);
if (TSDB_CODE_SUCCESS == code) {
pStbJoin->ctx.prev.tableNum += pBlock->info.rows;
}
}
if (TSDB_CODE_SUCCESS != code) {
pOperator->pTaskInfo->code = code;
T_LONG_JMP(pOperator->pTaskInfo->env, pOperator->pTaskInfo->code);
}
}
static void postProcessStbJoinTableHash(SOperatorInfo* pOperator) {
SDynQueryCtrlOperatorInfo* pInfo = pOperator->info;
SStbJoinDynCtrlInfo* pStbJoin = (SStbJoinDynCtrlInfo*)&pInfo->stbJoin;
if (pStbJoin->basic.batchFetch) {
return;
}
if (tSimpleHashGetSize(pStbJoin->ctx.prev.leftCache) == tSimpleHashGetSize(pStbJoin->ctx.prev.onceTable)) {
tSimpleHashClear(pStbJoin->ctx.prev.leftCache);
return;
}
uint64_t* pUid = NULL;
int32_t iter = 0;
while (NULL != (pUid = tSimpleHashIterate(pStbJoin->ctx.prev.onceTable, pUid, &iter))) {
tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, pUid, sizeof(*pUid));
}
pStbJoin->execInfo.leftCacheNum = tSimpleHashGetSize(pStbJoin->ctx.prev.leftCache);
qError("more than 1 ref build table num %" PRId64, (int64_t)tSimpleHashGetSize(pStbJoin->ctx.prev.leftCache));
// debug only
iter = 0;
uint32_t* num = NULL;
while (NULL != (num = tSimpleHashIterate(pStbJoin->ctx.prev.leftCache, num, &iter))) {
ASSERT(*num > 1);
}
}
static void buildStbJoinTableList(SOperatorInfo* pOperator) {
SDynQueryCtrlOperatorInfo* pInfo = pOperator->info;
SStbJoinDynCtrlInfo* pStbJoin = (SStbJoinDynCtrlInfo*)&pInfo->stbJoin;
while (true) {
SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
if (NULL == pBlock) {
break;
}
pStbJoin->execInfo.prevBlkNum++;
pStbJoin->execInfo.prevBlkRows += pBlock->info.rows;
doBuildStbJoinTableHash(pOperator, pBlock);
}
postProcessStbJoinTableHash(pOperator);
pStbJoin->ctx.prev.joinBuild = true;
}
static void seqJoinLaunchNewRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
SDynQueryCtrlOperatorInfo* pInfo = pOperator->info;
SStbJoinDynCtrlInfo* pStbJoin = (SStbJoinDynCtrlInfo*)&pInfo->stbJoin;
SStbJoinPrevJoinCtx* pPrev = &pStbJoin->ctx.prev;
SStbJoinTableList* pNode = pPrev->pListHead;
while (pNode) {
if (pNode->readIdx >= pNode->uidNum) {
pPrev->pListHead = pNode->pNext;
freeStbJoinTableList(pNode);
pNode = pPrev->pListHead;
continue;
}
seqJoinLaunchNewRetrieveImpl(pOperator, ppRes);
if (*ppRes) {
return;
}
handleSeqJoinCurrRetrieveEnd(pOperator, pStbJoin);
pPrev->pListHead->readIdx++;
}
*ppRes = NULL;
setOperatorCompleted(pOperator);
return;
}
static FORCE_INLINE SSDataBlock* seqStableJoinComposeRes(SStbJoinDynCtrlInfo* pStbJoin, SSDataBlock* pBlock) {
pBlock->info.id.blockId = pStbJoin->outputBlkId;
return pBlock;
}
SSDataBlock* seqStableJoin(SOperatorInfo* pOperator) {
SDynQueryCtrlOperatorInfo* pInfo = pOperator->info;
SStbJoinDynCtrlInfo* pStbJoin = (SStbJoinDynCtrlInfo*)&pInfo->stbJoin;
SSDataBlock* pRes = NULL;
if (pOperator->status == OP_EXEC_DONE) {
return pRes;
}
int64_t st = 0;
if (pOperator->cost.openCost == 0) {
st = taosGetTimestampUs();
}
if (!pStbJoin->ctx.prev.joinBuild) {
buildStbJoinTableList(pOperator);
if (pStbJoin->execInfo.prevBlkRows <= 0) {
setOperatorCompleted(pOperator);
goto _return;
}
}
seqJoinContinueCurrRetrieve(pOperator, &pRes);
if (pRes) {
goto _return;
}
seqJoinLaunchNewRetrieve(pOperator, &pRes);
_return:
if (pOperator->cost.openCost == 0) {
pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
}
return pRes ? seqStableJoinComposeRes(pStbJoin, pRes) : NULL;
}
int32_t initSeqStbJoinTableHash(SStbJoinPrevJoinCtx* pPrev, bool batchFetch) {
if (batchFetch) {
pPrev->leftHash = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT));
if (NULL == pPrev->leftHash) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pPrev->rightHash = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT));
if (NULL == pPrev->rightHash) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} else {
pPrev->leftCache = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
if (NULL == pPrev->leftCache) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pPrev->rightCache = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
if (NULL == pPrev->rightCache) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pPrev->onceTable = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
if (NULL == pPrev->onceTable) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
return TSDB_CODE_SUCCESS;
}
SOperatorInfo* createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream,
SDynQueryCtrlPhysiNode* pPhyciNode, SExecTaskInfo* pTaskInfo) {
SDynQueryCtrlOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SDynQueryCtrlOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
__optr_fn_t nextFp = NULL;
int32_t code = TSDB_CODE_SUCCESS;
if (pOperator == NULL || pInfo == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
}
pTaskInfo->dynamicTask = pPhyciNode->node.dynamicOp;
code = appendDownstream(pOperator, pDownstream, numOfDownstream);
if (TSDB_CODE_SUCCESS != code) {
goto _error;
}
pInfo->qType = pPhyciNode->qType;
switch (pInfo->qType) {
case DYN_QTYPE_STB_HASH:
memcpy(&pInfo->stbJoin.basic, &pPhyciNode->stbJoin, sizeof(pPhyciNode->stbJoin));
pInfo->stbJoin.outputBlkId = pPhyciNode->node.pOutputDataBlockDesc->dataBlockId;
code = initSeqStbJoinTableHash(&pInfo->stbJoin.ctx.prev, pInfo->stbJoin.basic.batchFetch);
if (TSDB_CODE_SUCCESS != code) {
goto _error;
}
nextFp = seqStableJoin;
break;
default:
qError("unsupported dynamic query ctrl type: %d", pInfo->qType);
code = TSDB_CODE_INVALID_PARA;
goto _error;
}
setOperatorInfo(pOperator, "DynQueryCtrlOperator", QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL, false, OP_NOT_OPENED, pInfo, pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, nextFp, NULL, destroyDynQueryCtrlOperator, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
return pOperator;
_error:
if (pInfo != NULL) {
destroyDynQueryCtrlOperator(pInfo);
}
taosMemoryFree(pOperator);
pTaskInfo->code = code;
return NULL;
}

4
source/libs/executor/src/eventwindowoperator.c
View File

@ -123,7 +123,7 @@ SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo
setOperatorInfo(pOperator, "EventWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, true, OP_NOT_OPENED, pInfo,
pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, eventWindowAggregate, NULL, destroyEWindowOperatorInfo,
optrDefaultBufFn, NULL);
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -183,7 +183,7 @@ static SSDataBlock* eventWindowAggregate(SOperatorInfo* pOperator) {
SOperatorInfo* downstream = pOperator->pDownstream[0];
while (1) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
if (pBlock == NULL) {
break;
}

178
source/libs/executor/src/exchangeoperator.c
View File

@ -40,6 +40,9 @@ typedef struct SSourceDataInfo {
int32_t code;
EX_SOURCE_STATUS status;
const char* taskId;
SArray* pSrcUidList;
int32_t srcOpType;
bool tableSeq;
} SSourceDataInfo;
static void destroyExchangeOperatorInfo(void* param);
@ -67,6 +70,8 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
return;
}
SSourceDataInfo* pDataInfo = NULL;
while (1) {
qDebug("prepare wait for ready, %p, %s", pExchangeInfo, GET_TASKID(pTaskInfo));
tsem_wait(&pExchangeInfo->ready);
@ -76,7 +81,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
}
for (int32_t i = 0; i < totalSources; ++i) {
SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, i);
pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, i);
if (pDataInfo->status == EX_SOURCE_DATA_EXHAUSTED) {
continue;
}
@ -91,17 +96,26 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
}
SRetrieveTableRsp* pRsp = pDataInfo->pRsp;
SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, i);
SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pDataInfo->index);
// todo
SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo;
if (pRsp->numOfRows == 0) {
if (NULL != pDataInfo->pSrcUidList) {
pDataInfo->status = EX_SOURCE_DATA_NOT_READY;
code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i);
if (code != TSDB_CODE_SUCCESS) {
taosMemoryFreeClear(pDataInfo->pRsp);
goto _error;
}
} else {
pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64
", totalRows:%" PRIu64 ", try next %d/%" PRIzu,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows,
pExchangeInfo->loadInfo.totalRows, i + 1, totalSources);
taosMemoryFreeClear(pDataInfo->pRsp);
}
break;
}
@ -131,7 +145,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
taosMemoryFreeClear(pDataInfo->pRsp);
if (pDataInfo->status != EX_SOURCE_DATA_EXHAUSTED) {
if (pDataInfo->status != EX_SOURCE_DATA_EXHAUSTED || NULL != pDataInfo->pSrcUidList) {
pDataInfo->status = EX_SOURCE_DATA_NOT_READY;
code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i);
if (code != TSDB_CODE_SUCCESS) {
@ -157,11 +171,6 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) {
SExchangeInfo* pExchangeInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
return NULL;
}
size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources);
SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo;
@ -204,6 +213,11 @@ static SSDataBlock* loadRemoteData(SOperatorInfo* pOperator) {
SExchangeInfo* pExchangeInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
}
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
@ -244,6 +258,10 @@ static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo, const
return TSDB_CODE_OUT_OF_MEMORY;
}
if (pInfo->dynamicOp) {
return TSDB_CODE_SUCCESS;
}
for (int32_t i = 0; i < numOfSources; ++i) {
SSourceDataInfo dataInfo = {0};
dataInfo.status = EX_SOURCE_DATA_NOT_READY;
@ -272,9 +290,18 @@ static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo*
return TSDB_CODE_OUT_OF_MEMORY;
}
if (pExNode->node.dynamicOp) {
pInfo->pHashSources = tSimpleHashInit(numOfSources * 2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT));
if (NULL == pInfo->pHashSources) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
for (int32_t i = 0; i < numOfSources; ++i) {
SDownstreamSourceNode* pNode = (SDownstreamSourceNode*)nodesListGetNode((SNodeList*)pExNode->pSrcEndPoints, i);
taosArrayPush(pInfo->pSources, pNode);
SExchangeSrcIndex idx = {.srcIdx = i, .inUseIdx = -1};
tSimpleHashPut(pInfo->pHashSources, &pNode->addr.nodeId, sizeof(pNode->addr.nodeId), &idx, sizeof(idx));
}
initLimitInfo(pExNode->node.pLimit, pExNode->node.pSlimit, &pInfo->limitInfo);
@ -290,6 +317,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode
goto _error;
}
pInfo->dynamicOp = pExNode->node.dynamicOp;
int32_t code = initExchangeOperator(pExNode, pInfo, GET_TASKID(pTaskInfo));
if (code != TSDB_CODE_SUCCESS) {
goto _error;
@ -316,7 +344,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode
}
pOperator->fpSet =
createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, optrDefaultBufFn, NULL);
createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
return pOperator;
_error:
@ -354,6 +382,7 @@ void doDestroyExchangeOperatorInfo(void* param) {
taosArrayDestroyEx(pExInfo->pRecycledBlocks, freeBlock);
blockDataDestroy(pExInfo->pDummyBlock);
tSimpleHashCleanup(pExInfo->pHashSources);
tsem_destroy(&pExInfo->ready);
taosMemoryFreeClear(param);
@ -402,14 +431,45 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) {
return code;
}
int32_t buildTableScanOperatorParam(SOperatorParam** ppRes, SArray* pUidList, int32_t srcOpType, bool tableSeq) {
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) {
return TSDB_CODE_OUT_OF_MEMORY;
}
STableScanOperatorParam* pScan = taosMemoryMalloc(sizeof(STableScanOperatorParam));
if (NULL == pScan) {
taosMemoryFreeClear(*ppRes);
return TSDB_CODE_OUT_OF_MEMORY;
}
pScan->pUidList = taosArrayDup(pUidList, NULL);
if (NULL == pScan->pUidList) {
taosMemoryFree(pScan);
taosMemoryFreeClear(*ppRes);
return TSDB_CODE_OUT_OF_MEMORY;
}
pScan->tableSeq = tableSeq;
(*ppRes)->opType = srcOpType;
(*ppRes)->downstreamIdx = 0;
(*ppRes)->value = pScan;
(*ppRes)->pChildren = NULL;
return TSDB_CODE_SUCCESS;
}
int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex) {
size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources);
SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, sourceIndex);
SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex);
pDataInfo->startTime = taosGetTimestampUs();
if (EX_SOURCE_DATA_NOT_READY != pDataInfo->status) {
return TSDB_CODE_SUCCESS;
}
ASSERT(pDataInfo->status == EX_SOURCE_DATA_NOT_READY);
pDataInfo->status = EX_SOURCE_DATA_STARTED;
SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pDataInfo->index);
pDataInfo->startTime = taosGetTimestampUs();
size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources);
SFetchRspHandleWrapper* pWrapper = taosMemoryCalloc(1, sizeof(SFetchRspHandleWrapper));
pWrapper->exchangeId = pExchangeInfo->self;
@ -429,11 +489,22 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas
req.taskId = pSource->taskId;
req.queryId = pTaskInfo->id.queryId;
req.execId = pSource->execId;
if (pDataInfo->pSrcUidList) {
int32_t code = buildTableScanOperatorParam(&req.pOpParam, pDataInfo->pSrcUidList, pDataInfo->srcOpType, pDataInfo->tableSeq);
taosArrayDestroy(pDataInfo->pSrcUidList);
pDataInfo->pSrcUidList = NULL;
if (TSDB_CODE_SUCCESS != code) {
pTaskInfo->code = code;
taosMemoryFree(pWrapper);
return pTaskInfo->code;
}
}
int32_t msgSize = tSerializeSResFetchReq(NULL, 0, &req);
if (msgSize < 0) {
pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pWrapper);
freeOperatorParam(req.pOpParam, OP_GET_PARAM);
return pTaskInfo->code;
}
@ -441,6 +512,7 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas
if (NULL == msg) {
pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pWrapper);
freeOperatorParam(req.pOpParam, OP_GET_PARAM);
return pTaskInfo->code;
}
@ -448,9 +520,12 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas
pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pWrapper);
taosMemoryFree(msg);
freeOperatorParam(req.pOpParam, OP_GET_PARAM);
return pTaskInfo->code;
}
freeOperatorParam(req.pOpParam, OP_GET_PARAM);
qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %p, %d/%" PRIzu,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId,
pSource->execId, pExchangeInfo, sourceIndex, totalSources);
@ -560,7 +635,7 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) {
SExchangeInfo* pExchangeInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources);
size_t totalSources = taosArrayGetSize(pExchangeInfo->pSourceDataInfo);
int64_t startTs = taosGetTimestampUs();
// Asynchronously send all fetch requests to all sources.
@ -693,14 +768,81 @@ _error:
return code;
}
int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) {
if (OPTR_IS_OPENED(pOperator)) {
int32_t addSingleExchangeSource(SOperatorInfo* pOperator, SExchangeOperatorBasicParam* pBasicParam) {
SExchangeInfo* pExchangeInfo = pOperator->info;
SExchangeSrcIndex* pIdx = tSimpleHashGet(pExchangeInfo->pHashSources, &pBasicParam->vgId, sizeof(pBasicParam->vgId));
if (NULL == pIdx) {
qError("No exchange source for vgId: %d", pBasicParam->vgId);
return TSDB_CODE_INVALID_PARA;
}
if (pIdx->inUseIdx < 0) {
SSourceDataInfo dataInfo = {0};
dataInfo.status = EX_SOURCE_DATA_NOT_READY;
dataInfo.taskId = GET_TASKID(pOperator->pTaskInfo);
dataInfo.index = pIdx->srcIdx;
dataInfo.pSrcUidList = taosArrayDup(pBasicParam->uidList, NULL);
dataInfo.srcOpType = pBasicParam->srcOpType;
dataInfo.tableSeq = pBasicParam->tableSeq;
taosArrayPush(pExchangeInfo->pSourceDataInfo, &dataInfo);
pIdx->inUseIdx = taosArrayGetSize(pExchangeInfo->pSourceDataInfo) - 1;
} else {
SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pIdx->inUseIdx);
if (pDataInfo->status == EX_SOURCE_DATA_EXHAUSTED) {
pDataInfo->status = EX_SOURCE_DATA_NOT_READY;
}
pDataInfo->pSrcUidList = taosArrayDup(pBasicParam->uidList, NULL);
pDataInfo->srcOpType = pBasicParam->srcOpType;
pDataInfo->tableSeq = pBasicParam->tableSeq;
}
return TSDB_CODE_SUCCESS;
}
int32_t addDynamicExchangeSource(SOperatorInfo* pOperator) {
SExchangeInfo* pExchangeInfo = pOperator->info;
int32_t code = TSDB_CODE_SUCCESS;
SExchangeOperatorBasicParam* pBasicParam = NULL;
SExchangeOperatorParam* pParam = (SExchangeOperatorParam*)pOperator->pOperatorGetParam->value;
if (pParam->multiParams) {
SExchangeOperatorBatchParam* pBatch = (SExchangeOperatorBatchParam*)pOperator->pOperatorGetParam->value;
int32_t iter = 0;
while (NULL != (pBasicParam = tSimpleHashIterate(pBatch->pBatchs, pBasicParam, &iter))) {
code = addSingleExchangeSource(pOperator, pBasicParam);
if (code) {
return code;
}
}
} else {
pBasicParam = &pParam->basic;
code = addSingleExchangeSource(pOperator, pBasicParam);
}
freeOperatorParam(pOperator->pOperatorGetParam, OP_GET_PARAM);
pOperator->pOperatorGetParam = NULL;
return TSDB_CODE_SUCCESS;
}
int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) {
SExchangeInfo* pExchangeInfo = pOperator->info;
int32_t code = TSDB_CODE_SUCCESS;
if ((OPTR_IS_OPENED(pOperator) && !pExchangeInfo->dynamicOp) || (pExchangeInfo->dynamicOp && NULL == pOperator->pOperatorGetParam)) {
return TSDB_CODE_SUCCESS;
}
if (pExchangeInfo->dynamicOp) {
code = addDynamicExchangeSource(pOperator);
if (code) {
return code;
}
}
int64_t st = taosGetTimestampUs();
SExchangeInfo* pExchangeInfo = pOperator->info;
if (!pExchangeInfo->seqLoadData) {
int32_t code = prepareConcurrentlyLoad(pOperator);
if (code != TSDB_CODE_SUCCESS) {

34
source/libs/executor/src/executil.c
View File

@ -926,20 +926,31 @@ SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, S
return pResBlock;
}
static void doSetQualifiedUid(SArray* pUidList, const SArray* pUidTagList, bool* pResultList) {
static int32_t doSetQualifiedUid(STableListInfo* pListInfo, SArray* pUidList, const SArray* pUidTagList, bool* pResultList, bool addUid) {
taosArrayClear(pUidList);
STableKeyInfo info = {.uid = 0, .groupId = 0};
int32_t numOfTables = taosArrayGetSize(pUidTagList);
for (int32_t i = 0; i < numOfTables; ++i) {
if (pResultList[i]) {
uint64_t uid = ((STUidTagInfo*)taosArrayGet(pUidTagList, i))->uid;
qDebug("tagfilter get uid:%" PRId64 ", res:%d", uid, pResultList[i]);
if (pResultList[i]) {
info.uid = uid;
void* p = taosArrayPush(pListInfo->pTableList, &info);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (addUid) {
taosArrayPush(pUidList, &uid);
}
}
}
return TSDB_CODE_SUCCESS;
}
static void copyExistedUids(SArray* pUidTagList, const SArray* pUidList) {
int32_t numOfExisted = taosArrayGetSize(pUidList);
if (numOfExisted == 0) {
@ -954,7 +965,8 @@ static void copyExistedUids(SArray* pUidTagList, const SArray* pUidList) {
}
static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SNode* pTagCond, void* pVnode,
SIdxFltStatus status, SStorageAPI* pAPI) {
SIdxFltStatus status, SStorageAPI* pAPI, bool addUid, bool* listAdded) {
*listAdded = false;
if (pTagCond == NULL) {
return TSDB_CODE_SUCCESS;
}
@ -1043,7 +1055,12 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN
goto end;
}
doSetQualifiedUid(pUidList, pUidTagList, (bool*)output.columnData->pData);
code = doSetQualifiedUid(pListInfo, pUidList, pUidTagList, (bool*)output.columnData->pData, addUid);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
goto end;
}
*listAdded = true;
end:
taosHashCleanup(ctx.colHash);
@ -1061,6 +1078,7 @@ int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, S
STableListInfo* pListInfo, uint8_t* digest, const char* idstr, SStorageAPI* pStorageAPI) {
int32_t code = TSDB_CODE_SUCCESS;
size_t numOfTables = 0;
bool listAdded = false;
pListInfo->idInfo.suid = pScanNode->suid;
pListInfo->idInfo.tableType = pScanNode->tableType;
@ -1073,7 +1091,7 @@ int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, S
if (pStorageAPI->metaFn.isTableExisted(pVnode, pScanNode->uid)) {
taosArrayPush(pUidList, &pScanNode->uid);
}
code = doFilterByTagCond(pListInfo, pUidList, pTagCond, pVnode, status, pStorageAPI);
code = doFilterByTagCond(pListInfo, pUidList, pTagCond, pVnode, status, pStorageAPI, false, &listAdded);
if (code != TSDB_CODE_SUCCESS) {
goto _end;
}
@ -1117,7 +1135,7 @@ int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, S
}
}
code = doFilterByTagCond(pListInfo, pUidList, pTagCond, pVnode, status, pStorageAPI);
code = doFilterByTagCond(pListInfo, pUidList, pTagCond, pVnode, status, pStorageAPI, tsTagFilterCache, &listAdded);
if (code != TSDB_CODE_SUCCESS) {
goto _end;
}
@ -1141,6 +1159,7 @@ int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, S
}
_end:
if (!listAdded) {
numOfTables = taosArrayGetSize(pUidList);
for (int i = 0; i < numOfTables; i++) {
STableKeyInfo info = {.uid = *(uint64_t*)taosArrayGet(pUidList, i), .groupId = 0};
@ -1153,6 +1172,7 @@ _end:
qTrace("tagfilter get uid:%" PRIu64 ", %s", info.uid, idstr);
}
}
taosArrayDestroy(pUidList);
return code;
@ -1254,7 +1274,7 @@ int32_t getGroupIdFromTagsVal(void* pVnode, uint64_t uid, SNodeList* pGroupNode,
return TSDB_CODE_SUCCESS;
}
SArray* extractPartitionColInfo(SNodeList* pNodeList) {
SArray* makeColumnArrayFromList(SNodeList* pNodeList) {
if (!pNodeList) {
return NULL;
}

55
source/libs/executor/src/executor.c
View File

@ -474,28 +474,48 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI
}
int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion,
int32_t* tversion) {
int32_t* tversion, int32_t idx) {
ASSERT(tinfo != NULL && dbName != NULL && tableName != NULL);
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
if (pTaskInfo->schemaInfo.sw == NULL) {
return TSDB_CODE_SUCCESS;
if (taosArrayGetSize(pTaskInfo->schemaInfos) <= idx) {
return -1;
}
*sversion = pTaskInfo->schemaInfo.sw->version;
*tversion = pTaskInfo->schemaInfo.tversion;
if (pTaskInfo->schemaInfo.dbname) {
strcpy(dbName, pTaskInfo->schemaInfo.dbname);
SSchemaInfo* pSchemaInfo = taosArrayGet(pTaskInfo->schemaInfos, idx);
*sversion = pSchemaInfo->sw->version;
*tversion = pSchemaInfo->tversion;
if (pSchemaInfo->dbname) {
strcpy(dbName, pSchemaInfo->dbname);
} else {
dbName[0] = 0;
}
if (pTaskInfo->schemaInfo.tablename) {
strcpy(tableName, pTaskInfo->schemaInfo.tablename);
if (pSchemaInfo->tablename) {
strcpy(tableName, pSchemaInfo->tablename);
} else {
tableName[0] = 0;
}
return 0;
return TSDB_CODE_SUCCESS;
}
bool qIsDynamicExecTask(qTaskInfo_t tinfo) {
return ((SExecTaskInfo*)tinfo)->dynamicTask;
}
void destroyOperatorParam(SOperatorParam* pParam) {
if (NULL == pParam) {
return;
}
//TODO
}
void qUpdateOperatorParam(qTaskInfo_t tinfo, void* pParam) {
destroyOperatorParam(((SExecTaskInfo*)tinfo)->pOpParam);
((SExecTaskInfo*)tinfo)->pOpParam = pParam;
((SExecTaskInfo*)tinfo)->paramSet = false;
}
int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan,
@ -506,7 +526,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
qDebug("start to create task, TID:0x%" PRIx64 " QID:0x%" PRIx64 ", vgId:%d", taskId, pSubplan->id.queryId, vgId);
int32_t code = createExecTaskInfo(pSubplan, pTask, readHandle, taskId, vgId, sql, model);
if (code != TSDB_CODE_SUCCESS) {
if (code != TSDB_CODE_SUCCESS || NULL == *pTask) {
qError("failed to createExecTaskInfo, code: %s", tstrerror(code));
goto _error;
}
@ -588,12 +608,19 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
int64_t st = taosGetTimestampUs();
int32_t blockIndex = 0;
if (pTaskInfo->pOpParam && !pTaskInfo->paramSet) {
pTaskInfo->paramSet = true;
pRes = pTaskInfo->pRoot->fpSet.getNextExtFn(pTaskInfo->pRoot, pTaskInfo->pOpParam);
} else {
pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot);
}
int32_t rowsThreshold = pTaskInfo->pSubplan->rowsThreshold;
if (!pTaskInfo->pSubplan->dynamicRowThreshold || 4096 <= pTaskInfo->pSubplan->rowsThreshold) {
rowsThreshold = 4096;
}
while ((pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot)) != NULL) {
int32_t blockIndex = 0;
while (pRes != NULL) {
SSDataBlock* p = NULL;
if (blockIndex >= taosArrayGetSize(pTaskInfo->pResultBlockList)) {
SSDataBlock* p1 = createOneDataBlock(pRes, true);
@ -613,6 +640,8 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
if (current >= rowsThreshold) {
break;
}
pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot);
}
if (pTaskInfo->pSubplan->dynamicRowThreshold) {
pTaskInfo->pSubplan->rowsThreshold -= current;

136
source/libs/executor/src/executorInt.c
View File

@ -1071,6 +1071,142 @@ void streamOpReloadState(SOperatorInfo* pOperator) {
}
}
void freeOperatorParamImpl(SOperatorParam* pParam, SOperatorParamType type) {
int32_t childrenNum = taosArrayGetSize(pParam->pChildren);
for (int32_t i = 0; i < childrenNum; ++i) {
SOperatorParam* pChild = taosArrayGetP(pParam->pChildren, i);
freeOperatorParam(pChild, type);
}
taosArrayDestroy(pParam->pChildren);
taosMemoryFree(pParam->value);
taosMemoryFree(pParam);
}
void freeExchangeGetBasicOperatorParam(void* pParam) {
SExchangeOperatorBasicParam* pBasic = (SExchangeOperatorBasicParam*)pParam;
taosArrayDestroy(pBasic->uidList);
}
void freeExchangeGetOperatorParam(SOperatorParam* pParam) {
SExchangeOperatorParam* pExcParam = (SExchangeOperatorParam*)pParam->value;
if (pExcParam->multiParams) {
SExchangeOperatorBatchParam* pExcBatch = (SExchangeOperatorBatchParam*)pParam->value;
tSimpleHashCleanup(pExcBatch->pBatchs);
} else {
freeExchangeGetBasicOperatorParam(&pExcParam->basic);
}
freeOperatorParamImpl(pParam, OP_GET_PARAM);
}
void freeExchangeNotifyOperatorParam(SOperatorParam* pParam) {
freeOperatorParamImpl(pParam, OP_NOTIFY_PARAM);
}
void freeGroupCacheGetOperatorParam(SOperatorParam* pParam) {
freeOperatorParamImpl(pParam, OP_GET_PARAM);
}
void freeGroupCacheNotifyOperatorParam(SOperatorParam* pParam) {
freeOperatorParamImpl(pParam, OP_NOTIFY_PARAM);
}
void freeMergeJoinGetOperatorParam(SOperatorParam* pParam) {
freeOperatorParamImpl(pParam, OP_GET_PARAM);
}
void freeMergeJoinNotifyOperatorParam(SOperatorParam* pParam) {
freeOperatorParamImpl(pParam, OP_NOTIFY_PARAM);
}
void freeTableScanGetOperatorParam(SOperatorParam* pParam) {
STableScanOperatorParam* pTableScanParam = (STableScanOperatorParam*)pParam->value;
taosArrayDestroy(pTableScanParam->pUidList);
freeOperatorParamImpl(pParam, OP_GET_PARAM);
}
void freeTableScanNotifyOperatorParam(SOperatorParam* pParam) {
freeOperatorParamImpl(pParam, OP_NOTIFY_PARAM);
}
void freeOperatorParam(SOperatorParam* pParam, SOperatorParamType type) {
if (NULL == pParam) {
return;
}
switch (pParam->opType) {
case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:
type == OP_GET_PARAM ? freeExchangeGetOperatorParam(pParam) : freeExchangeNotifyOperatorParam(pParam);
break;
case QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE:
type == OP_GET_PARAM ? freeGroupCacheGetOperatorParam(pParam) : freeGroupCacheNotifyOperatorParam(pParam);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN:
type == OP_GET_PARAM ? freeMergeJoinGetOperatorParam(pParam) : freeMergeJoinNotifyOperatorParam(pParam);
break;
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:
type == OP_GET_PARAM ? freeTableScanGetOperatorParam(pParam) : freeTableScanNotifyOperatorParam(pParam);
break;
default:
qError("unsupported op %d param, type %d", pParam->opType, type);
break;
}
}
void freeResetOperatorParams(struct SOperatorInfo* pOperator, SOperatorParamType type, bool allFree) {
SOperatorParam** ppParam = NULL;
SOperatorParam*** pppDownstramParam = NULL;
switch (type) {
case OP_GET_PARAM:
ppParam = &pOperator->pOperatorGetParam;
pppDownstramParam = &pOperator->pDownstreamGetParams;
break;
case OP_NOTIFY_PARAM:
ppParam = &pOperator->pOperatorNotifyParam;
pppDownstramParam = &pOperator->pDownstreamNotifyParams;
break;
default:
return;
}
if (*ppParam) {
freeOperatorParam(*ppParam, type);
*ppParam = NULL;
}
if (*pppDownstramParam) {
for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) {
if ((*pppDownstramParam)[i]) {
freeOperatorParam((*pppDownstramParam)[i], type);
(*pppDownstramParam)[i] = NULL;
}
}
if (allFree) {
taosMemoryFreeClear(*pppDownstramParam);
}
}
}
FORCE_INLINE SSDataBlock* getNextBlockFromDownstreamImpl(struct SOperatorInfo* pOperator, int32_t idx, bool clearParam) {
if (pOperator->pDownstreamGetParams && pOperator->pDownstreamGetParams[idx]) {
qDebug("DynOp: op %s start to get block from downstream %s", pOperator->name, pOperator->pDownstream[idx]->name);
SSDataBlock* pBlock = pOperator->pDownstream[idx]->fpSet.getNextExtFn(pOperator->pDownstream[idx], pOperator->pDownstreamGetParams[idx]);
if (clearParam) {
freeOperatorParam(pOperator->pDownstreamGetParams[idx], OP_GET_PARAM);
pOperator->pDownstreamGetParams[idx] = NULL;
}
return pBlock;
}
return pOperator->pDownstream[idx]->fpSet.getNextFn(pOperator->pDownstream[idx]);
}
bool compareVal(const char* v, const SStateKeys* pKey) {
if (IS_VAR_DATA_TYPE(pKey->type)) {
if (varDataLen(v) != varDataLen(pKey->pData)) {

8
source/libs/executor/src/filloperator.c
View File

@ -197,7 +197,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
}
while (1) {
SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream);
SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
if (pBlock == NULL) {
if (pInfo->totalInputRows == 0 &&
(pInfo->pFillInfo->type != TSDB_FILL_NULL_F && pInfo->pFillInfo->type != TSDB_FILL_SET_VALUE_F)) {
@ -444,7 +444,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
setOperatorInfo(pOperator, "FillOperator", QUERY_NODE_PHYSICAL_PLAN_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo);
pOperator->exprSupp.numOfExprs = pInfo->numOfExpr;
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, optrDefaultBufFn, NULL);
createOperatorFpSet(optrDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -1312,7 +1312,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) {
while (1) {
if (pInfo->srcRowIndex >= pInfo->pSrcBlock->info.rows || pInfo->pSrcBlock->info.rows == 0) {
// If there are delete datablocks, we receive them first.
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
if (pBlock == NULL) {
pOperator->status = OP_RES_TO_RETURN;
pInfo->pFillInfo->preRowKey = INT64_MIN;
@ -1569,7 +1569,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo,
pTaskInfo);
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, optrDefaultBufFn, NULL);
createOperatorFpSet(optrDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState);
code = appendDownstream(pOperator, &downstream, 1);

1395
source/libs/executor/src/groupcacheoperator.c Executable file
View File

File diff suppressed because it is too large Load Diff

16
source/libs/executor/src/groupoperator.c
View File

@ -383,7 +383,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
SOperatorInfo* downstream = pOperator->pDownstream[0];
while (1) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
if (pBlock == NULL) {
break;
}
@ -480,7 +480,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode*
pInfo->binfo.outputTsOrder = pAggNode->node.outputTsOrder;
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, hashGroupbyAggregate, NULL, destroyGroupOperatorInfo,
optrDefaultBufFn, NULL);
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
@ -754,7 +754,7 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) {
SOperatorInfo* downstream = pOperator->pDownstream[0];
while (1) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
if (pBlock == NULL) {
break;
}
@ -844,7 +844,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition
int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pPartNode->pTargets, NULL, &numOfCols);
pInfo->pGroupCols = extractPartitionColInfo(pPartNode->pPartitionKeys);
pInfo->pGroupCols = makeColumnArrayFromList(pPartNode->pPartitionKeys);
if (pPartNode->pExprs != NULL) {
int32_t num = 0;
@ -906,7 +906,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition
pOperator->exprSupp.pExprInfo = pExprInfo;
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, optrDefaultBufFn, NULL);
createOperatorFpSet(optrDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -1111,7 +1111,7 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) {
SOperatorInfo* downstream = pOperator->pDownstream[0];
{
pInfo->pInputDataBlock = NULL;
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
if (pBlock == NULL) {
setOperatorCompleted(pOperator);
return NULL;
@ -1244,7 +1244,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
goto _error;
}
pInfo->partitionSup.pGroupCols = extractPartitionColInfo(pPartNode->part.pPartitionKeys);
pInfo->partitionSup.pGroupCols = makeColumnArrayFromList(pPartNode->part.pPartitionKeys);
if (pPartNode->part.pExprs != NULL) {
int32_t num = 0;
@ -1324,7 +1324,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
pOperator->exprSupp.numOfExprs = numOfCols;
pOperator->exprSupp.pExprInfo = pExprInfo;
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamHashPartition, NULL,
destroyStreamPartitionOperatorInfo, optrDefaultBufFn, NULL);
destroyStreamPartitionOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState);
initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup);

917
source/libs/executor/src/hashjoinoperator.c Executable file
View File

@ -0,0 +1,917 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "executorInt.h"
#include "filter.h"
#include "function.h"
#include "operator.h"
#include "os.h"
#include "querynodes.h"
#include "querytask.h"
#include "tcompare.h"
#include "tdatablock.h"
#include "thash.h"
#include "tmsg.h"
#include "ttypes.h"
#include "hashjoin.h"
static int64_t getSingleKeyRowsNum(SBufRowInfo* pRow) {
int64_t rows = 0;
while (pRow) {
rows++;
pRow = pRow->next;
}
return rows;
}
static int64_t getRowsNumOfKeyHash(SSHashObj* pHash) {
SGroupData* pGroup = NULL;
int32_t iter = 0;
int64_t rowsNum = 0;
while (NULL != (pGroup = tSimpleHashIterate(pHash, pGroup, &iter))) {
int32_t* pKey = tSimpleHashGetKey(pGroup, NULL);
int64_t rows = getSingleKeyRowsNum(pGroup->rows);
qTrace("build_key:%d, rows:%" PRId64, *pKey, rows);
rowsNum += rows;
}
return rowsNum;
}
static int32_t initHJoinKeyColsInfo(SHJoinTableInfo* pTable, SNodeList* pList) {
pTable->keyNum = LIST_LENGTH(pList);
pTable->keyCols = taosMemoryMalloc(pTable->keyNum * sizeof(SHJoinColInfo));
if (NULL == pTable->keyCols) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int64_t bufSize = 0;
int32_t i = 0;
SNode* pNode = NULL;
FOREACH(pNode, pList) {
SColumnNode* pColNode = (SColumnNode*)pNode;
pTable->keyCols[i].srcSlot = pColNode->slotId;
pTable->keyCols[i].vardata = IS_VAR_DATA_TYPE(pColNode->node.resType.type);
pTable->keyCols[i].bytes = pColNode->node.resType.bytes;
bufSize += pColNode->node.resType.bytes;
++i;
}
if (pTable->keyNum > 1) {
pTable->keyBuf = taosMemoryMalloc(bufSize);
if (NULL == pTable->keyBuf) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
return TSDB_CODE_SUCCESS;
}
static void getHJoinValColNum(SNodeList* pList, int32_t blkId, int32_t* colNum) {
*colNum = 0;
SNode* pNode = NULL;
FOREACH(pNode, pList) {
STargetNode* pTarget = (STargetNode*)pNode;
SColumnNode* pCol = (SColumnNode*)pTarget->pExpr;
if (pCol->dataBlockId == blkId) {
(*colNum)++;
}
}
}
static bool valColInKeyCols(int16_t slotId, int32_t keyNum, SHJoinColInfo* pKeys, int32_t* pKeyIdx) {
for (int32_t i = 0; i < keyNum; ++i) {
if (pKeys[i].srcSlot == slotId) {
*pKeyIdx = i;
return true;
}
}
return false;
}
static int32_t initJoinValColsInfo(SHJoinTableInfo* pTable, SNodeList* pList) {
getHJoinValColNum(pList, pTable->blkId, &pTable->valNum);
if (pTable->valNum == 0) {
return TSDB_CODE_SUCCESS;
}
pTable->valCols = taosMemoryMalloc(pTable->valNum * sizeof(SHJoinColInfo));
if (NULL == pTable->valCols) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t i = 0;
int32_t colNum = 0;
SNode* pNode = NULL;
FOREACH(pNode, pList) {
STargetNode* pTarget = (STargetNode*)pNode;
SColumnNode* pColNode = (SColumnNode*)pTarget->pExpr;
if (pColNode->dataBlockId == pTable->blkId) {
if (valColInKeyCols(pColNode->slotId, pTable->keyNum, pTable->keyCols, &pTable->valCols[i].srcSlot)) {
pTable->valCols[i].keyCol = true;
} else {
pTable->valCols[i].keyCol = false;
pTable->valCols[i].srcSlot = pColNode->slotId;
pTable->valColExist = true;
colNum++;
}
pTable->valCols[i].dstSlot = pTarget->slotId;
pTable->valCols[i].vardata = IS_VAR_DATA_TYPE(pColNode->node.resType.type);
if (pTable->valCols[i].vardata) {
if (NULL == pTable->valVarCols) {
pTable->valVarCols = taosArrayInit(pTable->valNum, sizeof(int32_t));
if (NULL == pTable->valVarCols) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
taosArrayPush(pTable->valVarCols, &i);
}
pTable->valCols[i].bytes = pColNode->node.resType.bytes;
if (!pTable->valCols[i].keyCol && !pTable->valCols[i].vardata) {
pTable->valBufSize += pColNode->node.resType.bytes;
}
i++;
}
}
pTable->valBitMapSize = BitmapLen(colNum);
pTable->valBufSize += pTable->valBitMapSize;
return TSDB_CODE_SUCCESS;
}
static int32_t initHJoinTableInfo(SHJoinOperatorInfo* pJoin, SHashJoinPhysiNode* pJoinNode, SOperatorInfo** pDownstream, int32_t idx, SQueryStat* pStat) {
SNodeList* pKeyList = NULL;
SHJoinTableInfo* pTable = &pJoin->tbs[idx];
pTable->downStream = pDownstream[idx];
pTable->blkId = pDownstream[idx]->resultDataBlockId;
if (0 == idx) {
pKeyList = pJoinNode->pOnLeft;
} else {
pKeyList = pJoinNode->pOnRight;
}
int32_t code = initHJoinKeyColsInfo(pTable, pKeyList);
if (code) {
return code;
}
code = initJoinValColsInfo(pTable, pJoinNode->pTargets);
if (code) {
return code;
}
memcpy(&pTable->inputStat, pStat, sizeof(*pStat));
return TSDB_CODE_SUCCESS;
}
static void setHJoinBuildAndProbeTable(SHJoinOperatorInfo* pInfo, SHashJoinPhysiNode* pJoinNode) {
int32_t buildIdx = 0;
int32_t probeIdx = 1;
pInfo->joinType = pJoinNode->joinType;
switch (pInfo->joinType) {
case JOIN_TYPE_INNER:
if (pInfo->tbs[0].inputStat.inputRowNum <= pInfo->tbs[1].inputStat.inputRowNum) {
buildIdx = 0;
probeIdx = 1;
} else {
buildIdx = 1;
probeIdx = 0;
}
break;
case JOIN_TYPE_LEFT:
buildIdx = 1;
probeIdx = 0;
break;
case JOIN_TYPE_RIGHT:
buildIdx = 0;
probeIdx = 1;
break;
default:
break;
}
pInfo->pBuild = &pInfo->tbs[buildIdx];
pInfo->pProbe = &pInfo->tbs[probeIdx];
pInfo->pBuild->downStreamIdx = buildIdx;
pInfo->pProbe->downStreamIdx = probeIdx;
}
static int32_t buildHJoinResColMap(SHJoinOperatorInfo* pInfo, SHashJoinPhysiNode* pJoinNode) {
pInfo->pResColNum = pJoinNode->pTargets->length;
pInfo->pResColMap = taosMemoryCalloc(pJoinNode->pTargets->length, sizeof(int8_t));
if (NULL == pInfo->pResColMap) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SNode* pNode = NULL;
int32_t i = 0;
FOREACH(pNode, pJoinNode->pTargets) {
STargetNode* pTarget = (STargetNode*)pNode;
SColumnNode* pCol = (SColumnNode*)pTarget->pExpr;
if (pCol->dataBlockId == pInfo->pBuild->blkId) {
pInfo->pResColMap[i] = 1;
}
i++;
}
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE int32_t addPageToHJoinBuf(SArray* pRowBufs) {
SBufPageInfo page;
page.pageSize = HASH_JOIN_DEFAULT_PAGE_SIZE;
page.offset = 0;
page.data = taosMemoryMalloc(page.pageSize);
if (NULL == page.data) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArrayPush(pRowBufs, &page);
return TSDB_CODE_SUCCESS;
}
static int32_t initHJoinBufPages(SHJoinOperatorInfo* pInfo) {
pInfo->pRowBufs = taosArrayInit(32, sizeof(SBufPageInfo));
if (NULL == pInfo->pRowBufs) {
return TSDB_CODE_OUT_OF_MEMORY;
}
return addPageToHJoinBuf(pInfo->pRowBufs);
}
static void freeHJoinTableInfo(SHJoinTableInfo* pTable) {
taosMemoryFreeClear(pTable->keyCols);
taosMemoryFreeClear(pTable->keyBuf);
taosMemoryFreeClear(pTable->valCols);
taosArrayDestroy(pTable->valVarCols);
}
static void freeHJoinBufPage(void* param) {
SBufPageInfo* pInfo = (SBufPageInfo*)param;
taosMemoryFree(pInfo->data);
}
static void destroyHJoinKeyHash(SSHashObj** ppHash) {
if (NULL == ppHash || NULL == (*ppHash)) {
return;
}
void* pIte = NULL;
int32_t iter = 0;
while ((pIte = tSimpleHashIterate(*ppHash, pIte, &iter)) != NULL) {
SGroupData* pGroup = pIte;
SBufRowInfo* pRow = pGroup->rows;
SBufRowInfo* pNext = NULL;
while (pRow) {
pNext = pRow->next;
taosMemoryFree(pRow);
pRow = pNext;
}
}
tSimpleHashCleanup(*ppHash);
*ppHash = NULL;
}
static void destroyHashJoinOperator(void* param) {
SHJoinOperatorInfo* pJoinOperator = (SHJoinOperatorInfo*)param;
qError("hashJoin exec info, buildBlk:%" PRId64 ", buildRows:%" PRId64 ", probeBlk:%" PRId64 ", probeRows:%" PRId64 ", resRows:%" PRId64,
pJoinOperator->execInfo.buildBlkNum, pJoinOperator->execInfo.buildBlkRows, pJoinOperator->execInfo.probeBlkNum,
pJoinOperator->execInfo.probeBlkRows, pJoinOperator->execInfo.resRows);
destroyHJoinKeyHash(&pJoinOperator->pKeyHash);
freeHJoinTableInfo(&pJoinOperator->tbs[0]);
freeHJoinTableInfo(&pJoinOperator->tbs[1]);
pJoinOperator->pRes = blockDataDestroy(pJoinOperator->pRes);
taosMemoryFreeClear(pJoinOperator->pResColMap);
taosArrayDestroyEx(pJoinOperator->pRowBufs, freeHJoinBufPage);
nodesDestroyNode(pJoinOperator->pCond);
taosMemoryFreeClear(param);
}
static FORCE_INLINE char* retrieveColDataFromRowBufs(SArray* pRowBufs, SBufRowInfo* pRow) {
SBufPageInfo *pPage = taosArrayGet(pRowBufs, pRow->pageId);
return pPage->data + pRow->offset;
}
static FORCE_INLINE int32_t copyHJoinResRowsToBlock(SHJoinOperatorInfo* pJoin, int32_t rowNum, SBufRowInfo* pStart, SSDataBlock* pRes) {
SHJoinTableInfo* pBuild = pJoin->pBuild;
SHJoinTableInfo* pProbe = pJoin->pProbe;
int32_t buildIdx = 0, buildValIdx = 0;
int32_t probeIdx = 0;
SBufRowInfo* pRow = pStart;
int32_t code = 0;
for (int32_t r = 0; r < rowNum; ++r) {
char* pData = retrieveColDataFromRowBufs(pJoin->pRowBufs, pRow);
char* pValData = pData + pBuild->valBitMapSize;
char* pKeyData = pProbe->keyData;
buildIdx = buildValIdx = probeIdx = 0;
for (int32_t i = 0; i < pJoin->pResColNum; ++i) {
if (pJoin->pResColMap[i]) {
SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pBuild->valCols[buildIdx].dstSlot);
if (pBuild->valCols[buildIdx].keyCol) {
code = colDataSetVal(pDst, pRes->info.rows + r, pKeyData, false);
if (code) {
return code;
}
pKeyData += pBuild->valCols[buildIdx].vardata ? varDataTLen(pKeyData) : pBuild->valCols[buildIdx].bytes;
} else {
if (colDataIsNull_f(pData, buildValIdx)) {
code = colDataSetVal(pDst, pRes->info.rows + r, NULL, true);
if (code) {
return code;
}
} else {
code = colDataSetVal(pDst, pRes->info.rows + r, pValData, false);
if (code) {
return code;
}
pValData += pBuild->valCols[buildIdx].vardata ? varDataTLen(pValData) : pBuild->valCols[buildIdx].bytes;
}
buildValIdx++;
}
buildIdx++;
} else if (0 == r) {
SColumnInfoData* pSrc = taosArrayGet(pJoin->ctx.pProbeData->pDataBlock, pProbe->valCols[probeIdx].srcSlot);
SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pProbe->valCols[probeIdx].dstSlot);
code = colDataCopyNItems(pDst, pRes->info.rows, colDataGetData(pSrc, pJoin->ctx.probeIdx), rowNum, colDataIsNull_s(pSrc, pJoin->ctx.probeIdx));
if (code) {
return code;
}
probeIdx++;
}
}
pRow = pRow->next;
}
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE void appendHJoinResToBlock(struct SOperatorInfo* pOperator, SSDataBlock* pRes, bool* allFetched) {
SHJoinOperatorInfo* pJoin = pOperator->info;
SHJoinCtx* pCtx = &pJoin->ctx;
SBufRowInfo* pStart = pCtx->pBuildRow;
int32_t rowNum = 0;
int32_t resNum = pRes->info.rows;
while (pCtx->pBuildRow && (resNum < pRes->info.capacity)) {
rowNum++;
resNum++;
pCtx->pBuildRow = pCtx->pBuildRow->next;
}
pJoin->execInfo.resRows += rowNum;
int32_t code = copyHJoinResRowsToBlock(pJoin, rowNum, pStart, pRes);
if (code) {
pOperator->pTaskInfo->code = code;
T_LONG_JMP(pOperator->pTaskInfo->env, code);
}
pRes->info.rows = resNum;
*allFetched = pCtx->pBuildRow ? false : true;
}
static FORCE_INLINE void copyKeyColsDataToBuf(SHJoinTableInfo* pTable, int32_t rowIdx, size_t *pBufLen) {
char *pData = NULL;
size_t bufLen = 0;
if (1 == pTable->keyNum) {
if (pTable->keyCols[0].vardata) {
pData = pTable->keyCols[0].data + pTable->keyCols[0].offset[rowIdx];
bufLen = varDataTLen(pData);
} else {
pData = pTable->keyCols[0].data + pTable->keyCols[0].bytes * rowIdx;
bufLen = pTable->keyCols[0].bytes;
}
pTable->keyData = pData;
} else {
for (int32_t i = 0; i < pTable->keyNum; ++i) {
if (pTable->keyCols[i].vardata) {
pData = pTable->keyCols[i].data + pTable->keyCols[i].offset[rowIdx];
memcpy(pTable->keyBuf + bufLen, pData, varDataTLen(pData));
bufLen += varDataTLen(pData);
} else {
pData = pTable->keyCols[i].data + pTable->keyCols[i].bytes * rowIdx;
memcpy(pTable->keyBuf + bufLen, pData, pTable->keyCols[i].bytes);
bufLen += pTable->keyCols[i].bytes;
}
}
pTable->keyData = pTable->keyBuf;
}
if (pBufLen) {
*pBufLen = bufLen;
}
}
static void doHashJoinImpl(struct SOperatorInfo* pOperator) {
SHJoinOperatorInfo* pJoin = pOperator->info;
SHJoinTableInfo* pProbe = pJoin->pProbe;
SHJoinCtx* pCtx = &pJoin->ctx;
SSDataBlock* pRes = pJoin->pRes;
size_t bufLen = 0;
bool allFetched = false;
if (pJoin->ctx.pBuildRow) {
appendHJoinResToBlock(pOperator, pRes, &allFetched);
if (pRes->info.rows >= pRes->info.capacity) {
if (allFetched) {
++pCtx->probeIdx;
}
return;
} else {
++pCtx->probeIdx;
}
}
for (; pCtx->probeIdx < pCtx->pProbeData->info.rows; ++pCtx->probeIdx) {
copyKeyColsDataToBuf(pProbe, pCtx->probeIdx, &bufLen);
SGroupData* pGroup = tSimpleHashGet(pJoin->pKeyHash, pProbe->keyData, bufLen);
/*
size_t keySize = 0;
int32_t* pKey = tSimpleHashGetKey(pGroup, &keySize);
ASSERT(keySize == bufLen && 0 == memcmp(pKey, pProbe->keyData, bufLen));
int64_t rows = getSingleKeyRowsNum(pGroup->rows);
pJoin->execInfo.expectRows += rows;
qTrace("hash_key:%d, rows:%" PRId64, *pKey, rows);
*/
if (pGroup) {
pCtx->pBuildRow = pGroup->rows;
appendHJoinResToBlock(pOperator, pRes, &allFetched);
if (pRes->info.rows >= pRes->info.capacity) {
if (allFetched) {
++pCtx->probeIdx;
}
return;
}
} else {
qTrace("no key matched");
}
}
pCtx->rowRemains = false;
}
static int32_t setKeyColsData(SSDataBlock* pBlock, SHJoinTableInfo* pTable) {
for (int32_t i = 0; i < pTable->keyNum; ++i) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, pTable->keyCols[i].srcSlot);
if (pTable->keyCols[i].vardata != IS_VAR_DATA_TYPE(pCol->info.type)) {
qError("column type mismatch, idx:%d, slotId:%d, type:%d, vardata:%d", i, pTable->keyCols[i].srcSlot, pCol->info.type, pTable->keyCols[i].vardata);
return TSDB_CODE_INVALID_PARA;
}
if (pTable->keyCols[i].bytes != pCol->info.bytes) {
qError("column bytes mismatch, idx:%d, slotId:%d, bytes:%d, %d", i, pTable->keyCols[i].srcSlot, pCol->info.bytes, pTable->keyCols[i].bytes);
return TSDB_CODE_INVALID_PARA;
}
pTable->keyCols[i].data = pCol->pData;
if (pTable->keyCols[i].vardata) {
pTable->keyCols[i].offset = pCol->varmeta.offset;
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t setValColsData(SSDataBlock* pBlock, SHJoinTableInfo* pTable) {
if (!pTable->valColExist) {
return TSDB_CODE_SUCCESS;
}
for (int32_t i = 0; i < pTable->valNum; ++i) {
if (pTable->valCols[i].keyCol) {
continue;
}
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, pTable->valCols[i].srcSlot);
if (pTable->valCols[i].vardata != IS_VAR_DATA_TYPE(pCol->info.type)) {
qError("column type mismatch, idx:%d, slotId:%d, type:%d, vardata:%d", i, pTable->valCols[i].srcSlot, pCol->info.type, pTable->valCols[i].vardata);
return TSDB_CODE_INVALID_PARA;
}
if (pTable->valCols[i].bytes != pCol->info.bytes) {
qError("column bytes mismatch, idx:%d, slotId:%d, bytes:%d, %d", i, pTable->valCols[i].srcSlot, pCol->info.bytes, pTable->valCols[i].bytes);
return TSDB_CODE_INVALID_PARA;
}
if (!pTable->valCols[i].vardata) {
pTable->valCols[i].bitMap = pCol->nullbitmap;
}
pTable->valCols[i].data = pCol->pData;
if (pTable->valCols[i].vardata) {
pTable->valCols[i].offset = pCol->varmeta.offset;
}
}
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE void copyValColsDataToBuf(SHJoinTableInfo* pTable, int32_t rowIdx) {
if (!pTable->valColExist) {
return;
}
char *pData = NULL;
size_t bufLen = pTable->valBitMapSize;
memset(pTable->valData, 0, pTable->valBitMapSize);
for (int32_t i = 0, m = 0; i < pTable->valNum; ++i) {
if (pTable->valCols[i].keyCol) {
continue;
}
if (pTable->valCols[i].vardata) {
if (-1 == pTable->valCols[i].offset[rowIdx]) {
colDataSetNull_f(pTable->valData, m);
} else {
pData = pTable->valCols[i].data + pTable->valCols[i].offset[rowIdx];
memcpy(pTable->valData + bufLen, pData, varDataTLen(pData));
bufLen += varDataTLen(pData);
}
} else {
if (colDataIsNull_f(pTable->valCols[i].bitMap, rowIdx)) {
colDataSetNull_f(pTable->valData, m);
} else {
pData = pTable->valCols[i].data + pTable->valCols[i].bytes * rowIdx;
memcpy(pTable->valData + bufLen, pData, pTable->valCols[i].bytes);
bufLen += pTable->valCols[i].bytes;
}
}
m++;
}
}
static FORCE_INLINE int32_t getValBufFromPages(SArray* pPages, int32_t bufSize, char** pBuf, SBufRowInfo* pRow) {
if (0 == bufSize) {
pRow->pageId = -1;
return TSDB_CODE_SUCCESS;
}
if (bufSize > HASH_JOIN_DEFAULT_PAGE_SIZE) {
qError("invalid join value buf size:%d", bufSize);
return TSDB_CODE_INVALID_PARA;
}
do {
SBufPageInfo* page = taosArrayGetLast(pPages);
if ((page->pageSize - page->offset) >= bufSize) {
*pBuf = page->data + page->offset;
pRow->pageId = taosArrayGetSize(pPages) - 1;
pRow->offset = page->offset;
page->offset += bufSize;
return TSDB_CODE_SUCCESS;
}
int32_t code = addPageToHJoinBuf(pPages);
if (code) {
return code;
}
} while (true);
}
static FORCE_INLINE int32_t getHJoinValBufSize(SHJoinTableInfo* pTable, int32_t rowIdx) {
if (NULL == pTable->valVarCols) {
return pTable->valBufSize;
}
int32_t* varColIdx = NULL;
int32_t bufLen = pTable->valBufSize;
int32_t varColNum = taosArrayGetSize(pTable->valVarCols);
for (int32_t i = 0; i < varColNum; ++i) {
varColIdx = taosArrayGet(pTable->valVarCols, i);
char* pData = pTable->valCols[*varColIdx].data + pTable->valCols[*varColIdx].offset[rowIdx];
bufLen += varDataTLen(pData);
}
return bufLen;
}
static int32_t addRowToHashImpl(SHJoinOperatorInfo* pJoin, SGroupData* pGroup, SHJoinTableInfo* pTable, size_t keyLen, int32_t rowIdx) {
SGroupData group = {0};
SBufRowInfo* pRow = NULL;
if (NULL == pGroup) {
group.rows = taosMemoryMalloc(sizeof(SBufRowInfo));
if (NULL == group.rows) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pRow = group.rows;
} else {
pRow = taosMemoryMalloc(sizeof(SBufRowInfo));
if (NULL == pRow) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
int32_t code = getValBufFromPages(pJoin->pRowBufs, getHJoinValBufSize(pTable, rowIdx), &pTable->valData, pRow);
if (code) {
return code;
}
if (NULL == pGroup) {
pRow->next = NULL;
if (tSimpleHashPut(pJoin->pKeyHash, pTable->keyData, keyLen, &group, sizeof(group))) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} else {
pRow->next = pGroup->rows;
pGroup->rows = pRow;
}
return TSDB_CODE_SUCCESS;
}
static int32_t addRowToHash(SHJoinOperatorInfo* pJoin, SSDataBlock* pBlock, size_t keyLen, int32_t rowIdx) {
SHJoinTableInfo* pBuild = pJoin->pBuild;
int32_t code = setValColsData(pBlock, pBuild);
if (code) {
return code;
}
SGroupData* pGroup = tSimpleHashGet(pJoin->pKeyHash, pBuild->keyData, keyLen);
code = addRowToHashImpl(pJoin, pGroup, pBuild, keyLen, rowIdx);
if (code) {
return code;
}
copyValColsDataToBuf(pBuild, rowIdx);
return TSDB_CODE_SUCCESS;
}
static int32_t addBlockRowsToHash(SSDataBlock* pBlock, SHJoinOperatorInfo* pJoin) {
SHJoinTableInfo* pBuild = pJoin->pBuild;
int32_t code = setKeyColsData(pBlock, pBuild);
if (code) {
return code;
}
size_t bufLen = 0;
for (int32_t i = 0; i < pBlock->info.rows; ++i) {
copyKeyColsDataToBuf(pBuild, i, &bufLen);
code = addRowToHash(pJoin, pBlock, bufLen, i);
if (code) {
return code;
}
}
return code;
}
static int32_t buildHJoinKeyHash(struct SOperatorInfo* pOperator) {
SHJoinOperatorInfo* pJoin = pOperator->info;
SSDataBlock* pBlock = NULL;
int32_t code = TSDB_CODE_SUCCESS;
while (true) {
pBlock = getNextBlockFromDownstream(pOperator, pJoin->pBuild->downStreamIdx);
if (NULL == pBlock) {
break;
}
pJoin->execInfo.buildBlkNum++;
pJoin->execInfo.buildBlkRows += pBlock->info.rows;
code = addBlockRowsToHash(pBlock, pJoin);
if (code) {
return code;
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t launchBlockHashJoin(struct SOperatorInfo* pOperator, SSDataBlock* pBlock) {
SHJoinOperatorInfo* pJoin = pOperator->info;
SHJoinTableInfo* pProbe = pJoin->pProbe;
int32_t code = setKeyColsData(pBlock, pProbe);
if (code) {
return code;
}
code = setValColsData(pBlock, pProbe);
if (code) {
return code;
}
pJoin->ctx.probeIdx = 0;
pJoin->ctx.pBuildRow = NULL;
pJoin->ctx.pProbeData = pBlock;
pJoin->ctx.rowRemains = true;
doHashJoinImpl(pOperator);
return TSDB_CODE_SUCCESS;
}
static void setHJoinDone(struct SOperatorInfo* pOperator) {
setOperatorCompleted(pOperator);
SHJoinOperatorInfo* pInfo = pOperator->info;
destroyHJoinKeyHash(&pInfo->pKeyHash);
qError("hash Join done");
}
static SSDataBlock* doHashJoin(struct SOperatorInfo* pOperator) {
SHJoinOperatorInfo* pJoin = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
int32_t code = TSDB_CODE_SUCCESS;
SSDataBlock* pRes = pJoin->pRes;
pRes->info.rows = 0;
int64_t st = 0;
if (pOperator->cost.openCost == 0) {
st = taosGetTimestampUs();
}
if (pOperator->status == OP_EXEC_DONE) {
goto _return;
}
if (!pJoin->keyHashBuilt) {
pJoin->keyHashBuilt = true;
code = buildHJoinKeyHash(pOperator);
if (code) {
pTaskInfo->code = code;
T_LONG_JMP(pTaskInfo->env, code);
}
if (tSimpleHashGetSize(pJoin->pKeyHash) <= 0) {
setHJoinDone(pOperator);
goto _return;
}
//qTrace("build table rows:%" PRId64, getRowsNumOfKeyHash(pJoin->pKeyHash));
}
if (pJoin->ctx.rowRemains) {
doHashJoinImpl(pOperator);
if (pRes->info.rows >= pRes->info.capacity && pOperator->exprSupp.pFilterInfo != NULL) {
doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL);
}
if (pRes->info.rows > 0) {
return pRes;
}
}
while (true) {
SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, pJoin->pProbe->downStreamIdx);
if (NULL == pBlock) {
setHJoinDone(pOperator);
break;
}
pJoin->execInfo.probeBlkNum++;
pJoin->execInfo.probeBlkRows += pBlock->info.rows;
code = launchBlockHashJoin(pOperator, pBlock);
if (code) {
pTaskInfo->code = code;
T_LONG_JMP(pTaskInfo->env, code);
}
if (pRes->info.rows < pOperator->resultInfo.threshold) {
continue;
}
if (pOperator->exprSupp.pFilterInfo != NULL) {
doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL);
}
if (pRes->info.rows > 0) {
break;
}
}
_return:
if (pOperator->cost.openCost == 0) {
pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
}
return (pRes->info.rows > 0) ? pRes : NULL;
}
SOperatorInfo* createHashJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream,
SHashJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) {
SHJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SHJoinOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
int32_t code = TSDB_CODE_SUCCESS;
if (pOperator == NULL || pInfo == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
}
int32_t numOfCols = 0;
pInfo->pRes = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc);
initResultSizeInfo(&pOperator->resultInfo, 4096);
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
setOperatorInfo(pOperator, "HashJoinOperator", QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN, false, OP_NOT_OPENED, pInfo, pTaskInfo);
initHJoinTableInfo(pInfo, pJoinNode, pDownstream, 0, &pJoinNode->inputStat[0]);
initHJoinTableInfo(pInfo, pJoinNode, pDownstream, 1, &pJoinNode->inputStat[1]);
setHJoinBuildAndProbeTable(pInfo, pJoinNode);
code = buildHJoinResColMap(pInfo, pJoinNode);
if (code) {
goto _error;
}
code = initHJoinBufPages(pInfo);
if (code) {
goto _error;
}
size_t hashCap = pInfo->pBuild->inputStat.inputRowNum > 0 ? (pInfo->pBuild->inputStat.inputRowNum * 1.5) : 1024;
pInfo->pKeyHash = tSimpleHashInit(hashCap, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY));
if (pInfo->pKeyHash == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
}
if (pJoinNode->pFilterConditions != NULL && pJoinNode->node.pConditions != NULL) {
pInfo->pCond = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION);
if (pInfo->pCond == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
}
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pInfo->pCond);
pLogicCond->pParameterList = nodesMakeList();
if (pLogicCond->pParameterList == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
}
nodesListMakeAppend(&pLogicCond->pParameterList, nodesCloneNode(pJoinNode->pFilterConditions));
nodesListMakeAppend(&pLogicCond->pParameterList, nodesCloneNode(pJoinNode->node.pConditions));
pLogicCond->condType = LOGIC_COND_TYPE_AND;
} else if (pJoinNode->pFilterConditions != NULL) {
pInfo->pCond = nodesCloneNode(pJoinNode->pFilterConditions);
} else if (pJoinNode->node.pConditions != NULL) {
pInfo->pCond = nodesCloneNode(pJoinNode->node.pConditions);
} else {
pInfo->pCond = NULL;
}
code = filterInitFromNode(pInfo->pCond, &pOperator->exprSupp.pFilterInfo, 0);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
code = appendDownstream(pOperator, pDownstream, numOfDownstream);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doHashJoin, NULL, destroyHashJoinOperator, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
qError("create hash Join operator done");
return pOperator;
_error:
if (pInfo != NULL) {
destroyHashJoinOperator(pInfo);
}
taosMemoryFree(pOperator);
pTaskInfo->code = code;
return NULL;
}

223
source/libs/executor/src/joinoperator.c → source/libs/executor/src/mergejoinoperator.c
View File

@ -26,7 +26,7 @@
#include "tmsg.h"
#include "ttypes.h"
typedef struct SJoinRowCtx {
typedef struct SMJoinRowCtx {
bool rowRemains;
int64_t ts;
SArray* leftRowLocations;
@ -37,12 +37,15 @@ typedef struct SJoinRowCtx {
bool rightUseBuildTable;
SArray* rightRowLocations;
} SJoinRowCtx;
} SMJoinRowCtx;
typedef struct SJoinOperatorInfo {
typedef struct SMJoinOperatorInfo {
SSDataBlock* pRes;
int32_t joinType;
int32_t inputOrder;
bool downstreamInitDone[2];
bool downstreamFetchDone[2];
int16_t downstreamResBlkId[2];
SSDataBlock* pLeft;
int32_t leftPos;
@ -63,24 +66,24 @@ typedef struct SJoinOperatorInfo {
int32_t rightEqOnCondKeyLen;
SSHashObj* rightBuildTable;
SJoinRowCtx rowCtx;
} SJoinOperatorInfo;
SMJoinRowCtx rowCtx;
int64_t resRows;
} SMJoinOperatorInfo;
static void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode);
static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator);
static void destroyMergeJoinOperator(void* param);
static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstream, int32_t num,
SSortMergeJoinPhysiNode* pJoinNode, const char* idStr);
static void extractTimeCondition(SMJoinOperatorInfo* pInfo, SSortMergeJoinPhysiNode* pJoinNode, const char* idStr);
static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstream, int32_t num,
SSortMergeJoinPhysiNode* pJoinNode, const char* idStr) {
SNode* pMergeCondition = pJoinNode->pMergeCondition;
if (nodeType(pMergeCondition) != QUERY_NODE_OPERATOR) {
static void extractTimeCondition(SMJoinOperatorInfo* pInfo, SSortMergeJoinPhysiNode* pJoinNode, const char* idStr) {
SNode* pPrimKeyCond = pJoinNode->pPrimKeyCond;
if (nodeType(pPrimKeyCond) != QUERY_NODE_OPERATOR) {
qError("not support this in join operator, %s", idStr);
return; // do not handle this
}
SOperatorNode* pNode = (SOperatorNode*)pMergeCondition;
SOperatorNode* pNode = (SOperatorNode*)pPrimKeyCond;
SColumnNode* col1 = (SColumnNode*)pNode->pLeft;
SColumnNode* col2 = (SColumnNode*)pNode->pRight;
SColumnNode* leftTsCol = NULL;
@ -89,13 +92,13 @@ static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDown
leftTsCol = col1;
rightTsCol = col2;
} else {
if (col1->dataBlockId == pDownstream[0]->resultDataBlockId) {
ASSERT(col2->dataBlockId == pDownstream[1]->resultDataBlockId);
if (col1->dataBlockId == pInfo->downstreamResBlkId[0]) {
ASSERT(col2->dataBlockId == pInfo->downstreamResBlkId[1]);
leftTsCol = col1;
rightTsCol = col2;
} else {
ASSERT(col1->dataBlockId == pDownstream[1]->resultDataBlockId);
ASSERT(col2->dataBlockId == pDownstream[0]->resultDataBlockId);
ASSERT(col1->dataBlockId == pInfo->downstreamResBlkId[1]);
ASSERT(col2->dataBlockId == pInfo->downstreamResBlkId[0]);
leftTsCol = col2;
rightTsCol = col1;
}
@ -104,11 +107,11 @@ static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDown
setJoinColumnInfo(&pInfo->rightCol, rightTsCol);
}
static void extractEqualOnCondColsFromOper(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstreams, SOperatorNode* pOperNode,
static void extractEqualOnCondColsFromOper(SMJoinOperatorInfo* pInfo, SOperatorNode* pOperNode,
SColumn* pLeft, SColumn* pRight) {
SColumnNode* pLeftNode = (SColumnNode*)pOperNode->pLeft;
SColumnNode* pRightNode = (SColumnNode*)pOperNode->pRight;
if (pLeftNode->dataBlockId == pRightNode->dataBlockId || pLeftNode->dataBlockId == pDownstreams[0]->resultDataBlockId) {
if (pLeftNode->dataBlockId == pRightNode->dataBlockId || pLeftNode->dataBlockId == pInfo->downstreamResBlkId[0]) {
*pLeft = extractColumnFromColumnNode((SColumnNode*)pOperNode->pLeft);
*pRight = extractColumnFromColumnNode((SColumnNode*)pOperNode->pRight);
} else {
@ -117,7 +120,7 @@ static void extractEqualOnCondColsFromOper(SJoinOperatorInfo* pInfo, SOperatorIn
}
}
static void extractEqualOnCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownStream, SNode* pEqualOnCondNode,
static void extractEqualOnCondCols(SMJoinOperatorInfo* pInfo, SNode* pEqualOnCondNode,
SArray* leftTagEqCols, SArray* rightTagEqCols) {
SColumn left = {0};
SColumn right = {0};
@ -125,7 +128,7 @@ static void extractEqualOnCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pDo
SNode* pNode = NULL;
FOREACH(pNode, ((SLogicConditionNode*)pEqualOnCondNode)->pParameterList) {
SOperatorNode* pOperNode = (SOperatorNode*)pNode;
extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right);
extractEqualOnCondColsFromOper(pInfo, pOperNode, &left, &right);
taosArrayPush(leftTagEqCols, &left);
taosArrayPush(rightTagEqCols, &right);
}
@ -134,7 +137,7 @@ static void extractEqualOnCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pDo
if (nodeType(pEqualOnCondNode) == QUERY_NODE_OPERATOR) {
SOperatorNode* pOperNode = (SOperatorNode*)pEqualOnCondNode;
extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right);
extractEqualOnCondColsFromOper(pInfo, pOperNode, &left, &right);
taosArrayPush(leftTagEqCols, &left);
taosArrayPush(rightTagEqCols, &right);
}
@ -201,10 +204,24 @@ static int32_t fillKeyBufFromTagCols(SArray* pCols, SSDataBlock* pBlock, int32_t
return (int32_t)(pStart - (char*)pKey);
}
SOperatorInfo** buildMergeJoinDownstreams(SMJoinOperatorInfo* pInfo, SOperatorInfo** pDownstream) {
SOperatorInfo** p = taosMemoryMalloc(2 * POINTER_BYTES);
if (p) {
p[0] = pDownstream[0];
p[1] = pDownstream[0];
pInfo->downstreamResBlkId[0] = getOperatorResultBlockId(p[0], 0);
pInfo->downstreamResBlkId[1] = getOperatorResultBlockId(p[1], 1);
}
return p;
}
SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream,
SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) {
SJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SJoinOperatorInfo));
SMJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMJoinOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
bool newDownstreams = false;
int32_t code = TSDB_CODE_SUCCESS;
if (pOperator == NULL || pInfo == NULL) {
@ -212,6 +229,19 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t
goto _error;
}
if (1 == numOfDownstream) {
newDownstreams = true;
pDownstream = buildMergeJoinDownstreams(pInfo, pDownstream);
if (NULL == pDownstream) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
}
numOfDownstream = 2;
} else {
pInfo->downstreamResBlkId[0] = getOperatorResultBlockId(pDownstream[0], 0);
pInfo->downstreamResBlkId[1] = getOperatorResultBlockId(pDownstream[1], 1);
}
int32_t numOfCols = 0;
pInfo->pRes = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc);
@ -223,9 +253,9 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t
pOperator->exprSupp.pExprInfo = pExprInfo;
pOperator->exprSupp.numOfExprs = numOfCols;
extractTimeCondition(pInfo, pDownstream, numOfDownstream, pJoinNode, GET_TASKID(pTaskInfo));
extractTimeCondition(pInfo, pJoinNode, GET_TASKID(pTaskInfo));
if (pJoinNode->pOnConditions != NULL && pJoinNode->node.pConditions != NULL) {
if (pJoinNode->pOtherOnCond != NULL && pJoinNode->node.pConditions != NULL) {
pInfo->pCondAfterMerge = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION);
if (pInfo->pCondAfterMerge == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
@ -239,11 +269,13 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t
goto _error;
}
nodesListMakeAppend(&pLogicCond->pParameterList, nodesCloneNode(pJoinNode->pOnConditions));
nodesListMakeAppend(&pLogicCond->pParameterList, nodesCloneNode(pJoinNode->pOtherOnCond));
nodesListMakeAppend(&pLogicCond->pParameterList, nodesCloneNode(pJoinNode->node.pConditions));
pLogicCond->condType = LOGIC_COND_TYPE_AND;
} else if (pJoinNode->pOnConditions != NULL) {
pInfo->pCondAfterMerge = nodesCloneNode(pJoinNode->pOnConditions);
} else if (pJoinNode->pOtherOnCond != NULL) {
pInfo->pCondAfterMerge = nodesCloneNode(pJoinNode->pOtherOnCond);
} else if (pJoinNode->pColEqCond != NULL) {
pInfo->pCondAfterMerge = nodesCloneNode(pJoinNode->pColEqCond);
} else if (pJoinNode->node.pConditions != NULL) {
pInfo->pCondAfterMerge = nodesCloneNode(pJoinNode->node.pConditions);
} else {
@ -262,21 +294,27 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t
pInfo->inputOrder = TSDB_ORDER_DESC;
}
pInfo->pColEqualOnConditions = pJoinNode->pColEqualOnConditions;
pInfo->pColEqualOnConditions = pJoinNode->pColEqCond;
if (pInfo->pColEqualOnConditions != NULL) {
pInfo->leftEqOnCondCols = taosArrayInit(4, sizeof(SColumn));
pInfo->rightEqOnCondCols = taosArrayInit(4, sizeof(SColumn));
extractEqualOnCondCols(pInfo, pDownstream, pInfo->pColEqualOnConditions, pInfo->leftEqOnCondCols, pInfo->rightEqOnCondCols);
extractEqualOnCondCols(pInfo, pInfo->pColEqualOnConditions, pInfo->leftEqOnCondCols, pInfo->rightEqOnCondCols);
initTagColskeyBuf(&pInfo->leftEqOnCondKeyLen, &pInfo->leftEqOnCondKeyBuf, pInfo->leftEqOnCondCols);
initTagColskeyBuf(&pInfo->rightEqOnCondKeyLen, &pInfo->rightEqOnCondKeyBuf, pInfo->rightEqOnCondCols);
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pInfo->rightBuildTable = tSimpleHashInit(256, hashFn);
}
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
code = appendDownstream(pOperator, pDownstream, numOfDownstream);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
if (newDownstreams) {
taosMemoryFree(pDownstream);
}
pOperator->numOfRealDownstream = newDownstreams ? 1 : 2;
return pOperator;
@ -284,6 +322,9 @@ _error:
if (pInfo != NULL) {
destroyMergeJoinOperator(pInfo);
}
if (newDownstreams) {
taosMemoryFree(pDownstream);
}
taosMemoryFree(pOperator);
pTaskInfo->code = code;
@ -311,7 +352,7 @@ static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) {
}
void destroyMergeJoinOperator(void* param) {
SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param;
SMJoinOperatorInfo* pJoinOperator = (SMJoinOperatorInfo*)param;
if (pJoinOperator->pColEqualOnConditions != NULL) {
mergeJoinDestoryBuildTable(pJoinOperator->rightBuildTable);
taosMemoryFreeClear(pJoinOperator->rightEqOnCondKeyBuf);
@ -334,7 +375,7 @@ void destroyMergeJoinOperator(void* param) {
static void mergeJoinJoinLeftRight(struct SOperatorInfo* pOperator, SSDataBlock* pRes, int32_t currRow,
SSDataBlock* pLeftBlock, int32_t leftPos, SSDataBlock* pRightBlock,
int32_t rightPos) {
SJoinOperatorInfo* pJoinInfo = pOperator->info;
SMJoinOperatorInfo* pJoinInfo = pOperator->info;
for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) {
SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, i);
@ -411,13 +452,14 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator
SArray* createdBlocks) {
ASSERT(whichChild == 0 || whichChild == 1);
SJoinOperatorInfo* pJoinInfo = pOperator->info;
SMJoinOperatorInfo* pJoinInfo = pOperator->info;
int32_t endPos = -1;
SSDataBlock* dataBlock = startDataBlock;
mergeJoinGetBlockRowsEqualTs(dataBlock, tsSlotId, startPos, timestamp, &endPos, rowLocations, createdBlocks);
while (endPos == dataBlock->info.rows) {
SOperatorInfo* ds = pOperator->pDownstream[whichChild];
dataBlock = ds->fpSet.getNextFn(ds);
dataBlock = getNextBlockFromDownstreamRemain(pOperator, whichChild);
qError("merge join %s got block for same ts, rows:%" PRId64, whichChild == 0 ? "left" : "right", dataBlock ? dataBlock->info.rows : 0);
if (whichChild == 0) {
pJoinInfo->leftPos = 0;
pJoinInfo->pLeft = dataBlock;
@ -427,7 +469,7 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator
}
if (dataBlock == NULL) {
setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
pJoinInfo->downstreamFetchDone[whichChild] = true;
endPos = -1;
break;
}
@ -444,7 +486,7 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator
return 0;
}
static int32_t mergeJoinFillBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations) {
static int32_t mergeJoinFillBuildTable(SMJoinOperatorInfo* pInfo, SArray* rightRowLocations) {
for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) {
SRowLocation* rightRow = taosArrayGet(rightRowLocations, i);
int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightEqOnCondCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightEqOnCondKeyBuf);
@ -465,7 +507,7 @@ static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock*
int32_t rightRowIdx, bool useBuildTableTSRange, SArray* rightRowLocations, bool* pReachThreshold) {
*pReachThreshold = false;
uint32_t limitRowNum = pOperator->resultInfo.threshold;
SJoinOperatorInfo* pJoinInfo = pOperator->info;
SMJoinOperatorInfo* pJoinInfo = pOperator->info;
size_t leftNumJoin = taosArrayGetSize(leftRowLocations);
int32_t i,j;
@ -508,7 +550,7 @@ static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock*
return TSDB_CODE_SUCCESS;
}
static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* leftRowLocations, SArray* leftCreatedBlocks,
static void mergeJoinDestroyTSRangeCtx(SMJoinOperatorInfo* pJoinInfo, SArray* leftRowLocations, SArray* leftCreatedBlocks,
SArray* rightCreatedBlocks, bool rightUseBuildTable, SArray* rightRowLocations) {
for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) {
SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i);
@ -546,7 +588,7 @@ static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* lef
static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t timestamp, SSDataBlock* pRes,
int32_t* nRows) {
int32_t code = TSDB_CODE_SUCCESS;
SJoinOperatorInfo* pJoinInfo = pOperator->info;
SMJoinOperatorInfo* pJoinInfo = pOperator->info;
SArray* leftRowLocations = NULL;
SArray* rightRowLocations = NULL;
SArray* leftCreatedBlocks = NULL;
@ -613,30 +655,63 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t
return TSDB_CODE_SUCCESS;
}
static void setMergeJoinDone(SOperatorInfo* pOperator) {
setOperatorCompleted(pOperator);
if (pOperator->pDownstreamGetParams) {
freeOperatorParam(pOperator->pDownstreamGetParams[0], OP_GET_PARAM);
freeOperatorParam(pOperator->pDownstreamGetParams[1], OP_GET_PARAM);
pOperator->pDownstreamGetParams[0] = NULL;
pOperator->pDownstreamGetParams[1] = NULL;
}
}
static bool mergeJoinGetNextTimestamp(SOperatorInfo* pOperator, int64_t* pLeftTs, int64_t* pRightTs) {
SJoinOperatorInfo* pJoinInfo = pOperator->info;
SMJoinOperatorInfo* pJoinInfo = pOperator->info;
bool leftEmpty = false;
if (pJoinInfo->pLeft == NULL || pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) {
SOperatorInfo* ds1 = pOperator->pDownstream[0];
pJoinInfo->pLeft = ds1->fpSet.getNextFn(ds1);
if (!pJoinInfo->downstreamFetchDone[0]) {
pJoinInfo->pLeft = getNextBlockFromDownstreamRemain(pOperator, 0);
pJoinInfo->downstreamInitDone[0] = true;
pJoinInfo->leftPos = 0;
qError("merge join left got block, rows:%" PRId64, pJoinInfo->pLeft ? pJoinInfo->pLeft->info.rows : 0);
} else {
pJoinInfo->pLeft = NULL;
}
if (pJoinInfo->pLeft == NULL) {
setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
if (pOperator->pOperatorGetParam && ((SSortMergeJoinOperatorParam*)pOperator->pOperatorGetParam->value)->initDownstream && !pJoinInfo->downstreamInitDone[1]) {
leftEmpty = true;
} else {
setMergeJoinDone(pOperator);
return false;
}
}
}
if (pJoinInfo->pRight == NULL || pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) {
SOperatorInfo* ds2 = pOperator->pDownstream[1];
pJoinInfo->pRight = ds2->fpSet.getNextFn(ds2);
if (!pJoinInfo->downstreamFetchDone[1]) {
pJoinInfo->pRight = getNextBlockFromDownstreamRemain(pOperator, 1);
pJoinInfo->downstreamInitDone[1] = true;
pJoinInfo->rightPos = 0;
qError("merge join right got block, rows:%" PRId64, pJoinInfo->pRight ? pJoinInfo->pRight->info.rows : 0);
} else {
pJoinInfo->pRight = NULL;
}
if (pJoinInfo->pRight == NULL) {
setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
setMergeJoinDone(pOperator);
return false;
} else {
if (leftEmpty) {
setMergeJoinDone(pOperator);
return false;
}
}
}
// only the timestamp match support for ordinary table
SColumnInfoData* pLeftCol = taosArrayGet(pJoinInfo->pLeft->pDataBlock, pJoinInfo->leftCol.slotId);
char* pLeftVal = colDataGetData(pLeftCol, pJoinInfo->leftPos);
@ -650,7 +725,7 @@ static bool mergeJoinGetNextTimestamp(SOperatorInfo* pOperator, int64_t* pLeftTs
}
static void doMergeJoinImpl(struct SOperatorInfo* pOperator, SSDataBlock* pRes) {
SJoinOperatorInfo* pJoinInfo = pOperator->info;
SMJoinOperatorInfo* pJoinInfo = pOperator->info;
int32_t nrows = pRes->info.rows;
@ -694,8 +769,40 @@ static void doMergeJoinImpl(struct SOperatorInfo* pOperator, SSDataBlock* pRes)
}
}
void resetMergeJoinOperator(struct SOperatorInfo* pOperator) {
SMJoinOperatorInfo* pJoinInfo = pOperator->info;
if (pJoinInfo->rowCtx.rowRemains) {
mergeJoinDestroyTSRangeCtx(pJoinInfo, pJoinInfo->rowCtx.leftRowLocations, pJoinInfo->rowCtx.leftCreatedBlocks, pJoinInfo->rowCtx.rightCreatedBlocks,
pJoinInfo->rowCtx.rightUseBuildTable, pJoinInfo->rowCtx.rightRowLocations);
}
pJoinInfo->pLeft = NULL;
pJoinInfo->leftPos = 0;
pJoinInfo->pRight = NULL;
pJoinInfo->rightPos = 0;
pJoinInfo->downstreamFetchDone[0] = false;
pJoinInfo->downstreamFetchDone[1] = false;
pJoinInfo->downstreamInitDone[0] = false;
pJoinInfo->downstreamInitDone[1] = false;
pJoinInfo->resRows = 0;
pOperator->status = OP_OPENED;
}
SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) {
SJoinOperatorInfo* pJoinInfo = pOperator->info;
SMJoinOperatorInfo* pJoinInfo = pOperator->info;
if (pOperator->status == OP_EXEC_DONE) {
if (NULL == pOperator->pDownstreamGetParams || NULL == pOperator->pDownstreamGetParams[0] || NULL == pOperator->pDownstreamGetParams[1]) {
qError("total merge join res rows:%" PRId64, pJoinInfo->resRows);
return NULL;
} else {
resetMergeJoinOperator(pOperator);
qError("start new merge join");
}
}
int64_t st = 0;
if (pOperator->cost.openCost == 0) {
st = taosGetTimestampUs();
}
SSDataBlock* pRes = pJoinInfo->pRes;
blockDataCleanup(pRes);
@ -713,6 +820,24 @@ SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) {
if (pRes->info.rows >= pOperator->resultInfo.threshold) {
break;
}
if (pOperator->status == OP_EXEC_DONE) {
break;
}
return (pRes->info.rows > 0) ? pRes : NULL;
}
if (pOperator->cost.openCost == 0) {
pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
}
if (pRes->info.rows > 0) {
pJoinInfo->resRows += pRes->info.rows;
qError("merge join returns res rows:%" PRId64, pRes->info.rows);
return pRes;
} else {
qError("total merge join res rows:%" PRId64, pJoinInfo->resRows);
return NULL;
}
}

213
source/libs/executor/src/operator.c
View File

@ -30,7 +30,7 @@
SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup,
__optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn,
__optr_explain_fn_t explain) {
__optr_explain_fn_t explain, __optr_get_ext_fn_t nextExtFn, __optr_notify_fn_t notifyFn) {
SOperatorFpSet fpSet = {
._openFn = openFn,
.getNextFn = nextFn,
@ -38,6 +38,8 @@ SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn,
.closeFn = closeFn,
.reqBufFn = reqBufFn,
.getExplainFn = explain,
.getNextExtFn = nextExtFn,
.notifyFn = notifyFn,
.releaseStreamStateFn = NULL,
.reloadStreamStateFn = NULL,
};
@ -64,6 +66,7 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t
memcpy(p->pDownstream, pDownstream, num * POINTER_BYTES);
p->numOfDownstream = num;
p->numOfRealDownstream = num;
return TSDB_CODE_SUCCESS;
}
@ -279,7 +282,6 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR
SOperatorInfo* pOperator = NULL;
if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) {
STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
// NOTE: this is an patch to fix the physical plan
// TODO remove it later
if (pTableScanNode->scan.node.pLimit != NULL) {
@ -287,8 +289,20 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR
}
STableListInfo* pTableListInfo = tableListCreate();
int32_t code =
createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle,
int32_t code = initQueriedTableSchemaInfo(pHandle, &pTableScanNode->scan, dbname, pTaskInfo);
if (code) {
pTaskInfo->code = code;
tableListDestroy(pTableListInfo);
return NULL;
}
if (pTableScanNode->scan.node.dynamicOp) {
pTaskInfo->dynamicTask = true;
pTableListInfo->idInfo.suid = pTableScanNode->scan.suid;
pTableListInfo->idInfo.tableType = pTableScanNode->scan.tableType;
} else {
code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle,
pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo);
if (code) {
pTaskInfo->code = code;
@ -296,12 +310,6 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR
qError("failed to createScanTableListInfo, code:%s, %s", tstrerror(code), idstr);
return NULL;
}
code = initQueriedTableSchemaInfo(pHandle, &pTableScanNode->scan, dbname, pTaskInfo);
if (code) {
pTaskInfo->code = code;
tableListDestroy(pTableListInfo);
return NULL;
}
pOperator = createTableScanOperatorInfo(pTableScanNode, pHandle, pTableListInfo, pTaskInfo);
@ -361,7 +369,7 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR
}
}
pTaskInfo->schemaInfo.qsw = extractQueriedColumnSchema(&pTableScanNode->scan);
//pTaskInfo->schemaInfo.qsw = extractQueriedColumnSchema(&pTableScanNode->scan);
pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTableListInfo, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) {
SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode;
@ -429,6 +437,7 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR
pOperator = createProjectOperatorInfo(NULL, (SProjectPhysiNode*)pPhyNode, pTaskInfo);
} else {
terrno = TSDB_CODE_INVALID_PARA;
pTaskInfo->code = terrno;
return NULL;
}
@ -442,6 +451,8 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR
size_t size = LIST_LENGTH(pPhyNode->pChildren);
SOperatorInfo** ops = taosMemoryCalloc(size, POINTER_BYTES);
if (ops == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
pTaskInfo->code = terrno;
return NULL;
}
@ -510,6 +521,8 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR
pOptr = createStreamStateAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN == type) {
pOptr = createMergeJoinOperatorInfo(ops, size, (SSortMergeJoinPhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN == type) {
pOptr = createHashJoinOperatorInfo(ops, size, (SHashJoinPhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_FILL == type) {
pOptr = createFillOperatorInfo(ops[0], (SFillPhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL == type) {
@ -520,8 +533,13 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR
pOptr = createTimeSliceOperatorInfo(ops[0], pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT == type) {
pOptr = createEventwindowOperatorInfo(ops[0], pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE == type) {
pOptr = createGroupCacheOperatorInfo(ops, size, (SGroupCachePhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL == type) {
pOptr = createDynQueryCtrlOperatorInfo(ops, size, (SDynQueryCtrlPhysiNode*)pPhyNode, pTaskInfo);
} else {
terrno = TSDB_CODE_INVALID_PARA;
pTaskInfo->code = terrno;
taosMemoryFree(ops);
return NULL;
}
@ -534,17 +552,21 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR
return pOptr;
}
void destroyOperator(SOperatorInfo* pOperator) {
if (pOperator == NULL) {
return;
}
freeResetOperatorParams(pOperator, OP_GET_PARAM, true);
freeResetOperatorParams(pOperator, OP_NOTIFY_PARAM, true);
if (pOperator->fpSet.closeFn != NULL) {
pOperator->fpSet.closeFn(pOperator->info);
}
if (pOperator->pDownstream != NULL) {
for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) {
for (int32_t i = 0; i < pOperator->numOfRealDownstream; ++i) {
destroyOperator(pOperator->pDownstream[i]);
}
@ -586,3 +608,170 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInf
return TSDB_CODE_SUCCESS;
}
int32_t mergeOperatorParams(SOperatorParam* pDst, SOperatorParam* pSrc) {
if (pDst->opType != pSrc->opType) {
qError("different optype %d:%d for merge operator params", pDst->opType, pSrc->opType);
return TSDB_CODE_INVALID_PARA;
}
switch (pDst->opType) {
case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: {
SExchangeOperatorParam* pDExc = pDst->value;
SExchangeOperatorParam* pSExc = pSrc->value;
if (!pDExc->multiParams) {
if (pSExc->basic.vgId != pDExc->basic.vgId) {
SExchangeOperatorBatchParam* pBatch = taosMemoryMalloc(sizeof(SExchangeOperatorBatchParam));
if (NULL == pBatch) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pBatch->multiParams = true;
pBatch->pBatchs = tSimpleHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT));
if (NULL == pBatch->pBatchs) {
taosMemoryFree(pBatch);
return TSDB_CODE_OUT_OF_MEMORY;
}
tSimpleHashSetFreeFp(pBatch->pBatchs, freeExchangeGetBasicOperatorParam);
tSimpleHashPut(pBatch->pBatchs, &pDExc->basic.vgId, sizeof(pDExc->basic.vgId), &pDExc->basic, sizeof(pDExc->basic));
tSimpleHashPut(pBatch->pBatchs, &pSExc->basic.vgId, sizeof(pSExc->basic.vgId), &pSExc->basic, sizeof(pSExc->basic));
taosMemoryFree(pDst->value);
pDst->value = pBatch;
} else {
taosArrayAddAll(pDExc->basic.uidList, pSExc->basic.uidList);
}
} else {
SExchangeOperatorBatchParam* pBatch = pDst->value;
SExchangeOperatorBasicParam* pBasic = tSimpleHashGet(pBatch->pBatchs, &pSExc->basic.vgId, sizeof(pSExc->basic.vgId));
if (pBasic) {
taosArrayAddAll(pBasic->uidList, pSExc->basic.uidList);
} else {
tSimpleHashPut(pBatch->pBatchs, &pSExc->basic.vgId, sizeof(pSExc->basic.vgId), &pSExc->basic, sizeof(pSExc->basic));
}
}
break;
}
default:
qError("invalid optype %d for merge operator params", pDst->opType);
return TSDB_CODE_INVALID_PARA;
}
return TSDB_CODE_SUCCESS;
}
int32_t setOperatorParams(struct SOperatorInfo* pOperator, SOperatorParam* pInput, SOperatorParamType type) {
SOperatorParam** ppParam = NULL;
SOperatorParam*** pppDownstramParam = NULL;
switch (type) {
case OP_GET_PARAM:
ppParam = &pOperator->pOperatorGetParam;
pppDownstramParam = &pOperator->pDownstreamGetParams;
break;
case OP_NOTIFY_PARAM:
ppParam = &pOperator->pOperatorNotifyParam;
pppDownstramParam = &pOperator->pDownstreamNotifyParams;
break;
default:
return TSDB_CODE_INVALID_PARA;
}
freeResetOperatorParams(pOperator, type, false);
if (NULL == pInput) {
return TSDB_CODE_SUCCESS;
}
*ppParam = (pInput->opType == pOperator->operatorType) ? pInput : NULL;
if (NULL == *pppDownstramParam) {
*pppDownstramParam = taosMemoryCalloc(pOperator->numOfDownstream, POINTER_BYTES);
if (NULL == *pppDownstramParam) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
if (NULL == *ppParam) {
for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) {
(*pppDownstramParam)[i] = pInput;
}
return TSDB_CODE_SUCCESS;
}
memset(*pppDownstramParam, 0, pOperator->numOfDownstream * POINTER_BYTES);
int32_t childrenNum = taosArrayGetSize((*ppParam)->pChildren);
if (childrenNum <= 0) {
return TSDB_CODE_SUCCESS;
}
for (int32_t i = 0; i < childrenNum; ++i) {
SOperatorParam* pChild = *(SOperatorParam**)taosArrayGet((*ppParam)->pChildren, i);
if ((*pppDownstramParam)[pChild->downstreamIdx]) {
int32_t code = mergeOperatorParams((*pppDownstramParam)[pChild->downstreamIdx], pChild);
if (code) {
return code;
}
} else {
(*pppDownstramParam)[pChild->downstreamIdx] = pChild;
}
}
taosArrayDestroy((*ppParam)->pChildren);
(*ppParam)->pChildren = NULL;
return TSDB_CODE_SUCCESS;
}
SSDataBlock* getNextBlockFromDownstream(struct SOperatorInfo* pOperator, int32_t idx) {
return getNextBlockFromDownstreamImpl(pOperator, idx, true);
}
SSDataBlock* getNextBlockFromDownstreamRemain(struct SOperatorInfo* pOperator, int32_t idx) {
return getNextBlockFromDownstreamImpl(pOperator, idx, false);
}
SSDataBlock* optrDefaultGetNextExtFn(struct SOperatorInfo* pOperator, SOperatorParam* pParam) {
int32_t code = setOperatorParams(pOperator, pParam, OP_GET_PARAM);
if (TSDB_CODE_SUCCESS != code) {
pOperator->pTaskInfo->code = code;
T_LONG_JMP(pOperator->pTaskInfo->env, pOperator->pTaskInfo->code);
}
return pOperator->fpSet.getNextFn(pOperator);
}
int32_t optrDefaultNotifyFn(struct SOperatorInfo* pOperator, SOperatorParam* pParam) {
int32_t code = setOperatorParams(pOperator, pParam, OP_NOTIFY_PARAM);
if (TSDB_CODE_SUCCESS == code && pOperator->fpSet.notifyFn && pOperator->pOperatorNotifyParam) {
code = pOperator->fpSet.notifyFn(pOperator, pOperator->pOperatorNotifyParam);
}
if (TSDB_CODE_SUCCESS == code) {
for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) {
if (pOperator->pDownstreamNotifyParams[i]) {
code = optrDefaultNotifyFn(pOperator->pDownstream[i], pOperator->pDownstreamNotifyParams[i]);
if (TSDB_CODE_SUCCESS != code) {
break;
}
pOperator->pDownstreamNotifyParams[i] = NULL;
}
}
}
if (TSDB_CODE_SUCCESS != code) {
pOperator->pTaskInfo->code = code;
T_LONG_JMP(pOperator->pTaskInfo->env, pOperator->pTaskInfo->code);
}
return code;
}
int16_t getOperatorResultBlockId(struct SOperatorInfo* pOperator, int32_t idx) {
if (pOperator->transparent) {
return getOperatorResultBlockId(pOperator->pDownstream[idx], 0);
}
return pOperator->resultDataBlockId;
}

8
source/libs/executor/src/projectoperator.c
View File

@ -149,7 +149,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys
setOperatorInfo(pOperator, "ProjectOperator", QUERY_NODE_PHYSICAL_PLAN_PROJECT, false, OP_NOT_OPENED, pInfo,
pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doProjectOperation, NULL, destroyProjectOperatorInfo,
optrDefaultBufFn, NULL);
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamOperatorReleaseState, streamOperatorReloadState);
code = appendDownstream(pOperator, &downstream, 1);
@ -280,7 +280,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
blockDataCleanup(pRes);
// The downstream exec may change the value of the newgroup, so use a local variable instead.
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
if (pBlock == NULL) {
qDebug("set op close, exec %d, status %d rows %" PRId64 , pTaskInfo->execModel, pOperator->status, pFinalRes->info.rows);
setOperatorCompleted(pOperator);
@ -440,7 +440,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
setOperatorInfo(pOperator, "IndefinitOperator", QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, false, OP_NOT_OPENED, pInfo,
pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doApplyIndefinitFunction, NULL, destroyIndefinitOperatorInfo,
optrDefaultBufFn, NULL);
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -525,7 +525,7 @@ SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) {
if (pInfo->pRes->info.rows < pOperator->resultInfo.threshold) {
while (1) {
// The downstream exec may change the value of the newgroup, so use a local variable instead.
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
if (pBlock == NULL) {
setOperatorCompleted(pOperator);
break;

32
source/libs/executor/src/querytask.c
View File

@ -54,8 +54,11 @@ SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOP
pTaskInfo->id.vgId = vgId;
pTaskInfo->id.queryId = queryId;
pTaskInfo->id.taskId = taskId;
pTaskInfo->id.str = taosMemoryMalloc(64);
buildTaskId(taskId, queryId, pTaskInfo->id.str);
pTaskInfo->schemaInfos = taosArrayInit(1, sizeof(SSchemaInfo));
return pTaskInfo;
}
@ -106,7 +109,9 @@ int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHand
}
}
void cleanupQueriedTableScanInfo(SSchemaInfo* pSchemaInfo) {
void cleanupQueriedTableScanInfo(void* p) {
SSchemaInfo* pSchemaInfo = p;
taosMemoryFreeClear(pSchemaInfo->dbname);
taosMemoryFreeClear(pSchemaInfo->tablename);
tDeleteSchemaWrapper(pSchemaInfo->sw);
@ -132,14 +137,14 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo
return terrno;
}
SSchemaInfo* pSchemaInfo = &pTaskInfo->schemaInfo;
SSchemaInfo schemaInfo = {0};
pSchemaInfo->tablename = taosStrdup(mr.me.name);
pSchemaInfo->dbname = taosStrdup(dbName);
schemaInfo.tablename = taosStrdup(mr.me.name);
schemaInfo.dbname = taosStrdup(dbName);
if (mr.me.type == TSDB_SUPER_TABLE) {
pSchemaInfo->sw = tCloneSSchemaWrapper(&mr.me.stbEntry.schemaRow);
pSchemaInfo->tversion = mr.me.stbEntry.schemaTag.version;
schemaInfo.sw = tCloneSSchemaWrapper(&mr.me.stbEntry.schemaRow);
schemaInfo.tversion = mr.me.stbEntry.schemaTag.version;
} else if (mr.me.type == TSDB_CHILD_TABLE) {
tDecoderClear(&mr.coder);
@ -147,18 +152,23 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo
code = pAPI->metaReaderFn.getEntryGetUidCache(&mr, suid);
if (code != TSDB_CODE_SUCCESS) {
pAPI->metaReaderFn.clearReader(&mr);
taosMemoryFree(schemaInfo.tablename);
taosMemoryFree(schemaInfo.dbname);
return terrno;
}
pSchemaInfo->sw = tCloneSSchemaWrapper(&mr.me.stbEntry.schemaRow);
pSchemaInfo->tversion = mr.me.stbEntry.schemaTag.version;
schemaInfo.sw = tCloneSSchemaWrapper(&mr.me.stbEntry.schemaRow);
schemaInfo.tversion = mr.me.stbEntry.schemaTag.version;
} else {
pSchemaInfo->sw = tCloneSSchemaWrapper(&mr.me.ntbEntry.schemaRow);
schemaInfo.sw = tCloneSSchemaWrapper(&mr.me.ntbEntry.schemaRow);
}
pAPI->metaReaderFn.clearReader(&mr);
pSchemaInfo->qsw = extractQueriedColumnSchema(pScanNode);
schemaInfo.qsw = extractQueriedColumnSchema(pScanNode);
taosArrayPush(pTaskInfo->schemaInfos, &schemaInfo);
return TSDB_CODE_SUCCESS;
}
@ -211,7 +221,7 @@ void doDestroyTask(SExecTaskInfo* pTaskInfo) {
destroyOperator(pTaskInfo->pRoot);
pTaskInfo->pRoot = NULL;
cleanupQueriedTableScanInfo(&pTaskInfo->schemaInfo);
taosArrayDestroyEx(pTaskInfo->schemaInfos, cleanupQueriedTableScanInfo);
cleanupStreamInfo(&pTaskInfo->streamInfo);
if (!pTaskInfo->localFetch.localExec) {

241
source/libs/executor/src/scanoperator.c
View File

@ -782,11 +782,167 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) {
return NULL;
}
static int32_t createTableListInfoFromParam(SOperatorInfo* pOperator) {
STableScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
int32_t code = 0;
STableListInfo* pListInfo = pInfo->base.pTableListInfo;
STableScanOperatorParam* pParam = (STableScanOperatorParam*)pOperator->pOperatorGetParam->value;
int32_t num = taosArrayGetSize(pParam->pUidList);
if (num <= 0) {
qError("empty table scan uid list");
return TSDB_CODE_INVALID_PARA;
}
qError("vgId:%d add total %d dynamic tables to scan, tableSeq:%d, exist num:%" PRId64 ", operator status:%d",
pTaskInfo->id.vgId, num, pParam->tableSeq, (int64_t)taosArrayGetSize(pListInfo->pTableList), pOperator->status);
if (pParam->tableSeq) {
pListInfo->oneTableForEachGroup = true;
if (taosArrayGetSize(pListInfo->pTableList) > 0) {
taosHashClear(pListInfo->map);
taosArrayClear(pListInfo->pTableList);
pOperator->status = OP_EXEC_DONE;
}
} else {
pListInfo->oneTableForEachGroup = false;
pListInfo->numOfOuputGroups = 1;
}
STableKeyInfo info = {.groupId = 0};
int32_t tableIdx = 0;
for (int32_t i = 0; i < num; ++i) {
uint64_t* pUid = taosArrayGet(pParam->pUidList, i);
if (taosHashPut(pListInfo->map, pUid, sizeof(uint64_t), &tableIdx, sizeof(int32_t))) {
if (TSDB_CODE_DUP_KEY == terrno) {
continue;
}
return TSDB_CODE_OUT_OF_MEMORY;
}
info.uid = *pUid;
void* p = taosArrayPush(pListInfo->pTableList, &info);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tableIdx++;
qError("add dynamic table scan uid:%" PRIu64 ", %s", info.uid, GET_TASKID(pTaskInfo));
}
return code;
}
static SSDataBlock* startNextGroupScan(SOperatorInfo* pOperator) {
STableScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) {
setOperatorCompleted(pOperator);
if (pOperator->dynamicTask) {
taosArrayClear(pInfo->base.pTableListInfo->pTableList);
taosHashClear(pInfo->base.pTableListInfo->map);
}
return NULL;
}
// reset value for the next group data output
pOperator->status = OP_OPENED;
resetLimitInfoForNextGroup(&pInfo->base.limitInfo);
int32_t num = 0;
STableKeyInfo* pList = NULL;
tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num);
pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, pList, num);
pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond);
pInfo->scanTimes = 0;
SSDataBlock* result = doGroupedTableScan(pOperator);
if (result != NULL) {
if (pOperator->dynamicTask) {
result->info.id.groupId = result->info.id.uid;
}
return result;
}
return NULL;
}
static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) {
STableScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
if (pInfo->currentGroupId == -1) {
if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) {
setOperatorCompleted(pOperator);
return NULL;
}
int32_t num = 0;
STableKeyInfo* pList = NULL;
tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num);
ASSERT(pInfo->base.dataReader == NULL);
int32_t code = pAPI->tsdReader.tsdReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock,
(void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly, &pInfo->pIgnoreTables);
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
}
if (pInfo->pResBlock->info.capacity > pOperator->resultInfo.capacity) {
pOperator->resultInfo.capacity = pInfo->pResBlock->info.capacity;
}
}
SSDataBlock* result = doGroupedTableScan(pOperator);
if (result != NULL) {
if (pOperator->dynamicTask) {
result->info.id.groupId = result->info.id.uid;
}
return result;
}
while (true) {
result = startNextGroupScan(pOperator);
if (result || pOperator->status == OP_EXEC_DONE) {
return result;
}
}
return result;
}
static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
STableScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
if (pOperator->pOperatorGetParam) {
pOperator->dynamicTask = true;
int32_t code = createTableListInfoFromParam(pOperator);
freeOperatorParam(pOperator->pOperatorGetParam, OP_GET_PARAM);
pOperator->pOperatorGetParam = NULL;
if (code != TSDB_CODE_SUCCESS) {
pTaskInfo->code = code;
T_LONG_JMP(pTaskInfo->env, code);
}
if (pOperator->status == OP_EXEC_DONE) {
pInfo->currentGroupId = -1;
pOperator->status = OP_OPENED;
SSDataBlock* result = NULL;
while (true) {
result = startNextGroupScan(pOperator);
if (result || pOperator->status == OP_EXEC_DONE) {
return result;
}
}
}
}
// scan table one by one sequentially
if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) {
int32_t numOfTables = 0; // tableListGetSize(pTaskInfo->pTableListInfo);
@ -821,56 +977,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
pInfo->scanTimes = 0;
}
} else { // scan table group by group sequentially
if (pInfo->currentGroupId == -1) {
if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) {
setOperatorCompleted(pOperator);
return NULL;
}
int32_t num = 0;
STableKeyInfo* pList = NULL;
tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num);
ASSERT(pInfo->base.dataReader == NULL);
int32_t code = pAPI->tsdReader.tsdReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock,
(void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly, &pInfo->pIgnoreTables);
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
}
if (pInfo->pResBlock->info.capacity > pOperator->resultInfo.capacity) {
pOperator->resultInfo.capacity = pInfo->pResBlock->info.capacity;
}
}
SSDataBlock* result = doGroupedTableScan(pOperator);
if (result != NULL) {
return result;
}
while (1) {
if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) {
setOperatorCompleted(pOperator);
return NULL;
}
// reset value for the next group data output
pOperator->status = OP_OPENED;
resetLimitInfoForNextGroup(&pInfo->base.limitInfo);
int32_t num = 0;
STableKeyInfo* pList = NULL;
tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num);
pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, pList, num);
pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond);
pInfo->scanTimes = 0;
result = doGroupedTableScan(pOperator);
if (result != NULL) {
return result;
}
}
return groupSeqTableScan(pOperator);
}
}
@ -979,7 +1086,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
taosLRUCacheSetStrictCapacity(pInfo->base.metaCache.pTableMetaEntryCache, false);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo,
optrDefaultBufFn, getTableScannerExecInfo);
optrDefaultBufFn, getTableScannerExecInfo, optrDefaultGetNextExtFn, NULL);
// for non-blocking operator, the open cost is always 0
pOperator->cost.openCost = 0;
@ -1004,7 +1111,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo*
setOperatorInfo(pOperator, "TableSeqScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, false, OP_NOT_OPENED,
pInfo, pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableScanImpl, NULL, NULL, optrDefaultBufFn, NULL);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableScanImpl, NULL, NULL, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
return pOperator;
}
@ -2316,7 +2423,7 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT
setOperatorInfo(pOperator, "RawScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo,
pTaskInfo);
pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, destroyRawScanOperatorInfo, optrDefaultBufFn, NULL);
pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, destroyRawScanOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
return pOperator;
_end:
@ -2591,7 +2698,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
__optr_fn_t nextFn = (pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM) ? doStreamScan : doQueueScan;
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, optrDefaultBufFn, NULL);
createOperatorFpSet(optrDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamScanReleaseState, streamScanReloadState);
return pOperator;
@ -2629,9 +2736,13 @@ static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes,
SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId);
// refactor later
if (fmIsScanPseudoColumnFunc(pExprInfo[j].pExpr->_function.functionId)) {
if (FUNCTION_TYPE_TBNAME == pExprInfo[j].pExpr->_function.functionType) {
STR_TO_VARSTR(str, (*mr).me.name);
colDataSetVal(pDst, (count), str, false);
} else if (FUNCTION_TYPE_TBUID == pExprInfo[j].pExpr->_function.functionType) {
colDataSetVal(pDst, (count), (char*)&(*mr).me.uid, false);
} else if (FUNCTION_TYPE_VGID == pExprInfo[j].pExpr->_function.functionType) {
colDataSetVal(pDst, (count), (char*)&pTaskInfo->id.vgId, false);
} else { // it is a tag value
STagVal val = {0};
val.cid = pExprInfo[j].base.pParam[0].pCol->colId;
@ -2758,12 +2869,18 @@ static void tagScanFilterByTagCond(SArray* aUidTags, SNode* pTagCond, SArray* aF
}
static void tagScanFillOneCellWithTag(const STUidTagInfo* pUidTagInfo, SExprInfo* pExprInfo, SColumnInfoData* pColInfo, int rowIndex, const SStorageAPI* pAPI, void* pVnode) {
if (fmIsScanPseudoColumnFunc(pExprInfo->pExpr->_function.functionId)) { // tbname
static void tagScanFillOneCellWithTag(SOperatorInfo* pOperator, const STUidTagInfo* pUidTagInfo, SExprInfo* pExprInfo, SColumnInfoData* pColInfo, int rowIndex, const SStorageAPI* pAPI, void* pVnode) {
if (QUERY_NODE_FUNCTION == pExprInfo->pExpr->nodeType) {
if (FUNCTION_TYPE_TBNAME == pExprInfo->pExpr->_function.functionType) { // tbname
char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(str, "ctbidx");
colDataSetVal(pColInfo, rowIndex, str, false);
} else if (FUNCTION_TYPE_TBUID == pExprInfo->pExpr->_function.functionType) {
colDataSetVal(pColInfo, rowIndex, (char*)&pUidTagInfo->uid, false);
} else if (FUNCTION_TYPE_VGID == pExprInfo->pExpr->_function.functionType) {
colDataSetVal(pColInfo, rowIndex, (char*)&pOperator->pTaskInfo->id.vgId, false);
}
} else {
STagVal tagVal = {0};
tagVal.cid = pExprInfo->base.pParam[0].pCol->colId;
@ -2800,7 +2917,7 @@ static int32_t tagScanFillResultBlock(SOperatorInfo* pOperator, SSDataBlock* pRe
STUidTagInfo* pUidTagInfo = taosArrayGet(aUidTags, idx);
for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId);
tagScanFillOneCellWithTag(pUidTagInfo, &pExprInfo[j], pDst, i, pAPI, pInfo->readHandle.vnode);
tagScanFillOneCellWithTag(pOperator, pUidTagInfo, &pExprInfo[j], pDst, i, pAPI, pInfo->readHandle.vnode);
}
}
} else {
@ -2809,7 +2926,7 @@ static int32_t tagScanFillResultBlock(SOperatorInfo* pOperator, SSDataBlock* pRe
STUidTagInfo* pUidTagInfo = taosArrayGet(aUidTags, i);
for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId);
tagScanFillOneCellWithTag(pUidTagInfo, &pExprInfo[j], pDst, i, pAPI, pInfo->readHandle.vnode);
tagScanFillOneCellWithTag(pOperator, pUidTagInfo, &pExprInfo[j], pDst, i, pAPI, pInfo->readHandle.vnode);
}
}
}
@ -3012,7 +3129,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
}
__optr_fn_t tagScanNextFn = (pTagScanNode->onlyMetaCtbIdx) ? doTagScanFromCtbIdx : doTagScanFromMetaEntry;
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, tagScanNextFn, NULL, destroyTagScanOperatorInfo, optrDefaultBufFn, NULL);
createOperatorFpSet(optrDummyOpenFn, tagScanNextFn, NULL, destroyTagScanOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
return pOperator;
@ -3418,7 +3535,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
pOperator->exprSupp.numOfExprs = numOfCols;
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableMergeScan, NULL, destroyTableMergeScanOperatorInfo,
optrDefaultBufFn, getTableMergeScanExplainExecInfo);
optrDefaultBufFn, getTableMergeScanExplainExecInfo, optrDefaultGetNextExtFn, NULL);
pOperator->cost.openCost = 0;
return pOperator;
@ -3568,7 +3685,7 @@ SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* readHandle, STableC
setOperatorInfo(pOperator, "TableCountScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, false, OP_NOT_OPENED,
pInfo, pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableCountScan, NULL, destoryTableCountScanOperator,
optrDefaultBufFn, NULL);
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
return pOperator;
_error:

8
source/libs/executor/src/sortoperator.c
View File

@ -91,7 +91,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode*
// TODO dynamic set the available sort buffer
pOperator->fpSet =
createOperatorFpSet(doOpenSortOperator, doSort, NULL, destroySortOperatorInfo, optrDefaultBufFn, getExplainExecInfo);
createOperatorFpSet(doOpenSortOperator, doSort, NULL, destroySortOperatorInfo, optrDefaultBufFn, getExplainExecInfo, optrDefaultGetNextExtFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -461,7 +461,7 @@ SSDataBlock* doGroupSort(SOperatorInfo* pOperator) {
if (!pInfo->hasGroupId) {
pInfo->hasGroupId = true;
pInfo->prefetchedSortInput = pOperator->pDownstream[0]->fpSet.getNextFn(pOperator->pDownstream[0]);
pInfo->prefetchedSortInput = getNextBlockFromDownstream(pOperator, 0);
if (pInfo->prefetchedSortInput == NULL) {
setOperatorCompleted(pOperator);
return NULL;
@ -556,7 +556,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort
pInfo->pSortInfo = createSortInfo(pSortPhyNode->pSortKeys);
setOperatorInfo(pOperator, "GroupSortOperator", QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, false, OP_NOT_OPENED, pInfo, pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doGroupSort, NULL, destroyGroupSortOperatorInfo,
optrDefaultBufFn, getGroupSortExplainExecInfo);
optrDefaultBufFn, getGroupSortExplainExecInfo, optrDefaultGetNextExtFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -845,7 +845,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size
setOperatorInfo(pOperator, "MultiwayMergeOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE, false, OP_NOT_OPENED, pInfo, pTaskInfo);
pOperator->fpSet = createOperatorFpSet(openMultiwayMergeOperator, doMultiwayMerge, NULL,
destroyMultiwayMergeOperatorInfo, optrDefaultBufFn, getMultiwayMergeExplainExecInfo);
destroyMultiwayMergeOperatorInfo, optrDefaultBufFn, getMultiwayMergeExplainExecInfo, optrDefaultGetNextExtFn, NULL);
code = appendDownstream(pOperator, downStreams, numStreams);
if (code != TSDB_CODE_SUCCESS) {

10
source/libs/executor/src/streamtimewindowoperator.c
View File

@ -1283,7 +1283,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
pOperator->info = pInfo;
pOperator->fpSet = createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo,
optrDefaultBufFn, NULL);
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState);
if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) {
initIntervalDownStream(downstream, pPhyNode->type, pInfo);
@ -2253,7 +2253,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true,
OP_NOT_OPENED, pInfo, pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo,
optrDefaultBufFn, NULL);
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState);
if (downstream) {
@ -2390,7 +2390,7 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream
pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR);
blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionSemiAgg, NULL,
destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL);
destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
}
setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), pPhyNode->type, false, OP_NOT_OPENED, pInfo, pTaskInfo);
@ -2929,7 +2929,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED,
pInfo, pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo,
optrDefaultBufFn, NULL);
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState);
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup);
code = appendDownstream(pOperator, &downstream, 1);
@ -3173,7 +3173,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED,
pInfo, pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamIntervalAgg, NULL,
destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL);
destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState);
pInfo->statestore = pTaskInfo->storageAPI.stateStore;

6
source/libs/executor/src/sysscanoperator.c
View File

@ -933,7 +933,7 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo,
} else {
int32_t bufSize = IS_VAR_DATA_TYPE(tagType) ? (tagLen + VARSTR_HEADER_SIZE)
: (3 + DBL_MANT_DIG - DBL_MIN_EXP + VARSTR_HEADER_SIZE);
tagVarChar = taosMemoryMalloc(bufSize);
tagVarChar = taosMemoryCalloc(1, bufSize + 1);
int32_t len = -1;
convertTagDataToStr(varDataVal(tagVarChar), tagType, tagData, tagLen, &len);
varDataSetLen(tagVarChar, len);
@ -1788,7 +1788,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
pInfo, pTaskInfo);
pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock);
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, optrDefaultBufFn, NULL);
createOperatorFpSet(optrDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
return pOperator;
_error:
@ -2325,7 +2325,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi
setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false,
OP_NOT_OPENED, pInfo, pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo,
optrDefaultBufFn, NULL);
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
return pOperator;
_error:

4
source/libs/executor/src/timesliceoperator.c
View File

@ -900,7 +900,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
}
while (1) {
SSDataBlock* pBlock = pSliceInfo->pRemainRes ? pSliceInfo->pRemainRes : downstream->fpSet.getNextFn(downstream);
SSDataBlock* pBlock = pSliceInfo->pRemainRes ? pSliceInfo->pRemainRes : getNextBlockFromDownstream(pOperator, 0);
if (pBlock == NULL) {
setOperatorCompleted(pOperator);
break;
@ -1011,7 +1011,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode
setOperatorInfo(pOperator, "TimeSliceOperator", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, false, OP_NOT_OPENED, pInfo,
pTaskInfo);
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, optrDefaultBufFn, NULL);
createOperatorFpSet(optrDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);

20
source/libs/executor/src/timewindowoperator.c
View File

@ -858,7 +858,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
int64_t st = taosGetTimestampUs();
while (1) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
if (pBlock == NULL) {
break;
}
@ -982,7 +982,7 @@ static int32_t openStateWindowAggOptr(SOperatorInfo* pOperator) {
SOperatorInfo* downstream = pOperator->pDownstream[0];
while (1) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
if (pBlock == NULL) {
break;
}
@ -1300,7 +1300,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
pInfo, pTaskInfo);
pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo,
optrDefaultBufFn, NULL);
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -1423,7 +1423,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
SOperatorInfo* downstream = pOperator->pDownstream[0];
while (1) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
if (pBlock == NULL) {
break;
}
@ -1523,7 +1523,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi
setOperatorInfo(pOperator, "StateWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, true, OP_NOT_OPENED, pInfo,
pTaskInfo);
pOperator->fpSet = createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo,
optrDefaultBufFn, NULL);
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -1599,7 +1599,7 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW
setOperatorInfo(pOperator, "SessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, true, OP_NOT_OPENED,
pInfo, pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo,
optrDefaultBufFn, NULL);
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
pOperator->pTaskInfo = pTaskInfo;
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -1733,7 +1733,7 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
while (1) {
SSDataBlock* pBlock = NULL;
if (pMiaInfo->prefetchedBlock == NULL) {
pBlock = downstream->fpSet.getNextFn(downstream);
pBlock = getNextBlockFromDownstream(pOperator, 0);
} else {
pBlock = pMiaInfo->prefetchedBlock;
pMiaInfo->prefetchedBlock = NULL;
@ -1884,7 +1884,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream,
false, OP_NOT_OPENED, miaInfo, pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo,
optrDefaultBufFn, NULL);
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -2065,7 +2065,7 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) {
while (1) {
SSDataBlock* pBlock = NULL;
if (miaInfo->prefetchedBlock == NULL) {
pBlock = downstream->fpSet.getNextFn(downstream);
pBlock = getNextBlockFromDownstream(pOperator, 0);
} else {
pBlock = miaInfo->prefetchedBlock;
miaInfo->groupId = pBlock->info.id.groupId;
@ -2171,7 +2171,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge
setOperatorInfo(pOperator, "TimeMergeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, false,
OP_NOT_OPENED, pMergeIntervalInfo, pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo,
optrDefaultBufFn, NULL);
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {

36
source/libs/function/src/builtins.c
View File

@ -681,6 +681,21 @@ static int32_t translateTbnameColumn(SFunctionNode* pFunc, char* pErrBuf, int32_
return TSDB_CODE_SUCCESS;
}
static int32_t translateTbUidColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
// pseudo column do not need to check parameters
pFunc->node.resType =
(SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
return TSDB_CODE_SUCCESS;
}
static int32_t translateVgIdColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
// pseudo column do not need to check parameters
pFunc->node.resType =
(SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes, .type = TSDB_DATA_TYPE_INT};
return TSDB_CODE_SUCCESS;
}
static int32_t translateTopBot(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
if (2 != numOfParams) {
@ -3597,6 +3612,27 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.sprocessFunc = containsProperlyFunction,
.finalizeFunc = NULL
},
{
.name = "_tbuid",
.type = FUNCTION_TYPE_TBUID,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_SCAN_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC,
.translateFunc = translateTbUidColumn,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = qTbUidFunction,
.finalizeFunc = NULL
},
{
.name = "_vgid",
.type = FUNCTION_TYPE_VGID,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_SCAN_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC,
.translateFunc = translateVgIdColumn,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = qVgIdFunction,
.finalizeFunc = NULL
},
};
// clang-format on

2
source/libs/nodes/CMakeLists.txt
View File

@ -7,7 +7,7 @@ target_include_directories(
)
target_link_libraries(
nodes
PRIVATE os util common qcom
PRIVATE os util common qcom function
)
if(${BUILD_TEST})

63
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -259,6 +259,8 @@ static int32_t tempTableNodeCopy(const STempTableNode* pSrc, STempTableNode* pDs
static int32_t joinTableNodeCopy(const SJoinTableNode* pSrc, SJoinTableNode* pDst) {
COPY_BASE_OBJECT_FIELD(table, tableNodeCopy);
COPY_SCALAR_FIELD(joinType);
COPY_SCALAR_FIELD(hasSubQuery);
COPY_SCALAR_FIELD(isLowLevelJoin);
CLONE_NODE_FIELD(pLeft);
CLONE_NODE_FIELD(pRight);
CLONE_NODE_FIELD(pOnCond);
@ -348,6 +350,24 @@ static int32_t caseWhenNodeCopy(const SCaseWhenNode* pSrc, SCaseWhenNode* pDst)
return TSDB_CODE_SUCCESS;
}
static int32_t copyHintValue(const SHintNode* pSrc, SHintNode* pDst) {
if (NULL == pSrc->value) {
pDst->value = NULL;
return TSDB_CODE_SUCCESS;
}
switch (pSrc->option) {
default:
break;
}
return TSDB_CODE_SUCCESS;
}
static int32_t hintNodeCopy(const SHintNode* pSrc, SHintNode* pDst) {
COPY_SCALAR_FIELD(type);
COPY_SCALAR_FIELD(option);
return copyHintValue(pSrc, pDst);
}
static int32_t logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) {
CLONE_NODE_LIST_FIELD(pTargets);
CLONE_NODE_FIELD(pConditions);
@ -361,7 +381,9 @@ static int32_t logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) {
COPY_SCALAR_FIELD(groupAction);
COPY_SCALAR_FIELD(inputTsOrder);
COPY_SCALAR_FIELD(outputTsOrder);
COPY_SCALAR_FIELD(dynamicOp);
COPY_SCALAR_FIELD(forceCreateNonBlockingOptr);
CLONE_NODE_LIST_FIELD(pHint);
return TSDB_CODE_SUCCESS;
}
@ -406,10 +428,14 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) {
static int32_t logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
COPY_SCALAR_FIELD(joinType);
CLONE_NODE_FIELD(pMergeCondition);
CLONE_NODE_FIELD(pOnConditions);
CLONE_NODE_FIELD(pColEqualOnConditions);
COPY_SCALAR_FIELD(joinAlgo);
CLONE_NODE_FIELD(pPrimKeyEqCond);
CLONE_NODE_FIELD(pColEqCond);
CLONE_NODE_FIELD(pTagEqCond);
CLONE_NODE_FIELD(pTagOnCond);
CLONE_NODE_FIELD(pOtherOnCond);
COPY_SCALAR_FIELD(isSingleTableJoin);
COPY_SCALAR_FIELD(hasSubQuery);
return TSDB_CODE_SUCCESS;
}
@ -533,6 +559,26 @@ static int32_t logicInterpFuncCopy(const SInterpFuncLogicNode* pSrc, SInterpFunc
return TSDB_CODE_SUCCESS;
}
static int32_t logicGroupCacheCopy(const SGroupCacheLogicNode* pSrc, SGroupCacheLogicNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
COPY_SCALAR_FIELD(grpColsMayBeNull);
COPY_SCALAR_FIELD(grpByUid);
COPY_SCALAR_FIELD(globalGrp);
COPY_SCALAR_FIELD(batchFetch);
CLONE_NODE_LIST_FIELD(pGroupCols);
return TSDB_CODE_SUCCESS;
}
static int32_t logicDynQueryCtrlCopy(const SDynQueryCtrlLogicNode* pSrc, SDynQueryCtrlLogicNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
COPY_SCALAR_FIELD(qType);
COPY_SCALAR_FIELD(stbJoin.batchFetch);
CLONE_NODE_LIST_FIELD(stbJoin.pVgList);
CLONE_NODE_LIST_FIELD(stbJoin.pUidList);
COPY_OBJECT_FIELD(stbJoin.srcScan, sizeof(pDst->stbJoin.srcScan));
return TSDB_CODE_SUCCESS;
}
static int32_t logicSubplanCopy(const SLogicSubplan* pSrc, SLogicSubplan* pDst) {
COPY_OBJECT_FIELD(id, sizeof(SSubplanId));
CLONE_NODE_FIELD_EX(pNode, SLogicNode*);
@ -548,6 +594,7 @@ static int32_t physiNodeCopy(const SPhysiNode* pSrc, SPhysiNode* pDst) {
CLONE_NODE_LIST_FIELD(pChildren);
COPY_SCALAR_FIELD(inputTsOrder);
COPY_SCALAR_FIELD(outputTsOrder);
COPY_SCALAR_FIELD(dynamicOp);
COPY_SCALAR_FIELD(forceCreateNonBlockingOptr);
return TSDB_CODE_SUCCESS;
}
@ -680,6 +727,7 @@ static int32_t selectStmtCopy(const SSelectStmt* pSrc, SSelectStmt* pDst) {
COPY_SCALAR_FIELD(timeLineResMode);
COPY_SCALAR_FIELD(hasAggFuncs);
COPY_SCALAR_FIELD(hasRepeatScanFuncs);
CLONE_NODE_LIST_FIELD(pHint);
return TSDB_CODE_SUCCESS;
}
@ -768,6 +816,9 @@ SNode* nodesCloneNode(const SNode* pNode) {
case QUERY_NODE_CASE_WHEN:
code = caseWhenNodeCopy((const SCaseWhenNode*)pNode, (SCaseWhenNode*)pDst);
break;
case QUERY_NODE_HINT:
code = hintNodeCopy((const SHintNode*)pNode, (SHintNode*)pDst);
break;
case QUERY_NODE_SELECT_STMT:
code = selectStmtCopy((const SSelectStmt*)pNode, (SSelectStmt*)pDst);
break;
@ -810,6 +861,12 @@ SNode* nodesCloneNode(const SNode* pNode) {
case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC:
code = logicInterpFuncCopy((const SInterpFuncLogicNode*)pNode, (SInterpFuncLogicNode*)pDst);
break;
case QUERY_NODE_LOGIC_PLAN_GROUP_CACHE:
code = logicGroupCacheCopy((const SGroupCacheLogicNode*)pNode, (SGroupCacheLogicNode*)pDst);
break;
case QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL:
code = logicDynQueryCtrlCopy((const SDynQueryCtrlLogicNode*)pNode, (SDynQueryCtrlLogicNode*)pDst);
break;
case QUERY_NODE_LOGIC_SUBPLAN:
code = logicSubplanCopy((const SLogicSubplan*)pNode, (SLogicSubplan*)pDst);
break;

396
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -197,10 +197,8 @@ const char* nodesNodeName(ENodeType type) {
return "ShowDnodesStmt";
case QUERY_NODE_SHOW_MNODES_STMT:
return "ShowMnodesStmt";
/*
case QUERY_NODE_SHOW_MODULES_STMT:
return "ShowModulesStmt";
*/
case QUERY_NODE_SHOW_QNODES_STMT:
return "ShowQnodesStmt";
case QUERY_NODE_SHOW_SNODES_STMT:
@ -297,6 +295,10 @@ const char* nodesNodeName(ENodeType type) {
return "LogicIndefRowsFunc";
case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC:
return "LogicInterpFunc";
case QUERY_NODE_LOGIC_PLAN_GROUP_CACHE:
return "LogicGroupCache";
case QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL:
return "LogicDynamicQueryCtrl";
case QUERY_NODE_LOGIC_SUBPLAN:
return "LogicSubplan";
case QUERY_NODE_LOGIC_PLAN:
@ -326,7 +328,9 @@ const char* nodesNodeName(ENodeType type) {
case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return "PhysiProject";
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN:
return "PhysiJoin";
return "PhysiMergeJoin";
case QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN:
return "PhysiHashJoin";
case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG:
return "PhysiAgg";
case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:
@ -378,6 +382,10 @@ const char* nodesNodeName(ENodeType type) {
return "PhysiQueryInsert";
case QUERY_NODE_PHYSICAL_PLAN_DELETE:
return "PhysiDelete";
case QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE:
return "PhysiGroupCache";
case QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL:
return "PhysiDynamicQueryCtrl";
case QUERY_NODE_PHYSICAL_SUBPLAN:
return "PhysiSubplan";
case QUERY_NODE_PHYSICAL_PLAN:
@ -1180,6 +1188,97 @@ static int32_t jsonToLogicInterpFuncNode(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkGroupCacheLogicPlanGrpColsMayBeNull = "GroupColsMayBeNull";
static const char* jkGroupCacheLogicPlanGroupByUid = "GroupByUid";
static const char* jkGroupCacheLogicPlanGlobalGroup = "GlobalGroup";
static const char* jkGroupCacheLogicPlanGroupCols = "GroupCols";
static int32_t logicGroupCacheNodeToJson(const void* pObj, SJson* pJson) {
const SGroupCacheLogicNode* pNode = (const SGroupCacheLogicNode*)pObj;
int32_t code = logicPlanNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkGroupCacheLogicPlanGrpColsMayBeNull, pNode->grpColsMayBeNull);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkGroupCacheLogicPlanGroupByUid, pNode->grpByUid);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkGroupCacheLogicPlanGlobalGroup, pNode->globalGrp);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkGroupCacheLogicPlanGroupCols, pNode->pGroupCols);
}
return code;
}
static int32_t jsonToLogicGroupCacheNode(const SJson* pJson, void* pObj) {
SGroupCacheLogicNode* pNode = (SGroupCacheLogicNode*)pObj;
int32_t code = jsonToLogicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkGroupCacheLogicPlanGrpColsMayBeNull, &pNode->grpColsMayBeNull);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkGroupCacheLogicPlanGroupByUid, &pNode->grpByUid);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkGroupCacheLogicPlanGlobalGroup, &pNode->globalGrp);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkGroupCacheLogicPlanGroupCols, &pNode->pGroupCols);
}
return code;
}
static const char* jkDynQueryCtrlLogicPlanQueryType = "QueryType";
static const char* jkDynQueryCtrlLogicPlanStbJoinBatchFetch = "BatchFetch";
static const char* jkDynQueryCtrlLogicPlanStbJoinVgList = "VgroupList";
static const char* jkDynQueryCtrlLogicPlanStbJoinUidList = "UidList";
static int32_t logicDynQueryCtrlNodeToJson(const void* pObj, SJson* pJson) {
const SDynQueryCtrlLogicNode* pNode = (const SDynQueryCtrlLogicNode*)pObj;
int32_t code = logicPlanNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDynQueryCtrlLogicPlanQueryType, pNode->qType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkDynQueryCtrlLogicPlanStbJoinBatchFetch, pNode->stbJoin.batchFetch);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkDynQueryCtrlLogicPlanStbJoinVgList, pNode->stbJoin.pVgList);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkDynQueryCtrlLogicPlanStbJoinUidList, pNode->stbJoin.pUidList);
}
return code;
}
static int32_t jsonToLogicDynQueryCtrlNode(const SJson* pJson, void* pObj) {
SDynQueryCtrlLogicNode* pNode = (SDynQueryCtrlLogicNode*)pObj;
int32_t code = jsonToLogicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkDynQueryCtrlLogicPlanQueryType, pNode->qType, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetBoolValue(pJson, jkDynQueryCtrlLogicPlanStbJoinBatchFetch, &pNode->stbJoin.batchFetch);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkDynQueryCtrlLogicPlanStbJoinVgList, &pNode->stbJoin.pVgList);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkDynQueryCtrlLogicPlanStbJoinUidList, &pNode->stbJoin.pUidList);
}
return code;
}
static const char* jkSubplanIdQueryId = "QueryId";
static const char* jkSubplanIdGroupId = "GroupId";
static const char* jkSubplanIdSubplanId = "SubplanId";
@ -1434,9 +1533,11 @@ static int32_t jsonToLogicPlan(const SJson* pJson, void* pObj) {
}
static const char* jkJoinLogicPlanJoinType = "JoinType";
static const char* jkJoinLogicPlanOnConditions = "OnConditions";
static const char* jkJoinLogicPlanMergeCondition = "MergeConditions";
static const char* jkJoinLogicPlanColEqualOnConditions = "ColumnEqualOnConditions";
static const char* jkJoinLogicPlanJoinAlgo = "JoinAlgo";
static const char* jkJoinLogicPlanOnConditions = "OtherOnCond";
static const char* jkJoinLogicPlanPrimKeyEqCondition = "PrimKeyEqCond";
static const char* jkJoinLogicPlanColEqCondition = "ColumnEqCond";
static const char* jkJoinLogicPlanTagEqCondition = "TagEqCond";
static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) {
const SJoinLogicNode* pNode = (const SJoinLogicNode*)pObj;
@ -1446,13 +1547,19 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) {
code = tjsonAddIntegerToObject(pJson, jkJoinLogicPlanJoinType, pNode->joinType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinLogicPlanMergeCondition, nodeToJson, pNode->pMergeCondition);
code = tjsonAddIntegerToObject(pJson, jkJoinLogicPlanJoinAlgo, pNode->joinAlgo);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pOnConditions);
code = tjsonAddObject(pJson, jkJoinLogicPlanPrimKeyEqCondition, nodeToJson, pNode->pPrimKeyEqCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinLogicPlanColEqualOnConditions, nodeToJson, pNode->pColEqualOnConditions);
code = tjsonAddObject(pJson, jkJoinLogicPlanColEqCondition, nodeToJson, pNode->pColEqCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinLogicPlanTagEqCondition, nodeToJson, pNode->pTagEqCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pOtherOnCond);
}
return code;
}
@ -1465,14 +1572,21 @@ static int32_t jsonToLogicJoinNode(const SJson* pJson, void* pObj) {
tjsonGetNumberValue(pJson, jkJoinLogicPlanJoinType, pNode->joinType, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinLogicPlanMergeCondition, &pNode->pMergeCondition);
tjsonGetNumberValue(pJson, jkJoinLogicPlanJoinAlgo, pNode->joinAlgo, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinLogicPlanOnConditions, &pNode->pOnConditions);
code = jsonToNodeObject(pJson, jkJoinLogicPlanPrimKeyEqCondition, &pNode->pPrimKeyEqCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinLogicPlanColEqualOnConditions, &pNode->pColEqualOnConditions);
code = jsonToNodeObject(pJson, jkJoinLogicPlanColEqCondition, &pNode->pColEqCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinLogicPlanTagEqCondition, &pNode->pTagEqCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinLogicPlanOnConditions, &pNode->pOtherOnCond);
}
return code;
}
@ -1931,12 +2045,16 @@ static int32_t jsonToPhysiProjectNode(const SJson* pJson, void* pObj) {
static const char* jkJoinPhysiPlanJoinType = "JoinType";
static const char* jkJoinPhysiPlanInputTsOrder = "InputTsOrder";
static const char* jkJoinPhysiPlanMergeCondition = "MergeCondition";
static const char* jkJoinPhysiPlanOnLeftCols = "OnLeftColumns";
static const char* jkJoinPhysiPlanOnRightCols = "OnRightColumns";
static const char* jkJoinPhysiPlanPrimKeyCondition = "PrimKeyCondition";
static const char* jkJoinPhysiPlanOnConditions = "OnConditions";
static const char* jkJoinPhysiPlanTargets = "Targets";
static const char* jkJoinPhysiPlanColEqualOnConditions = "ColumnEqualOnConditions";
static const char* jkJoinPhysiPlanInputRowNum = "InputRowNum";
static const char* jkJoinPhysiPlanInputRowSize = "InputRowSize";
static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) {
static int32_t physiMergeJoinNodeToJson(const void* pObj, SJson* pJson) {
const SSortMergeJoinPhysiNode* pNode = (const SSortMergeJoinPhysiNode*)pObj;
int32_t code = physicPlanNodeToJson(pObj, pJson);
@ -1944,21 +2062,21 @@ static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanJoinType, pNode->joinType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanMergeCondition, nodeToJson, pNode->pMergeCondition);
code = tjsonAddObject(pJson, jkJoinPhysiPlanPrimKeyCondition, nodeToJson, pNode->pPrimKeyCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanOnConditions, nodeToJson, pNode->pOnConditions);
code = tjsonAddObject(pJson, jkJoinPhysiPlanOnConditions, nodeToJson, pNode->pOtherOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkJoinPhysiPlanTargets, pNode->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanColEqualOnConditions, nodeToJson, pNode->pColEqualOnConditions);
code = tjsonAddObject(pJson, jkJoinPhysiPlanColEqualOnConditions, nodeToJson, pNode->pColEqCond);
}
return code;
}
static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) {
static int32_t jsonToPhysiMergeJoinNode(const SJson* pJson, void* pObj) {
SSortMergeJoinPhysiNode* pNode = (SSortMergeJoinPhysiNode*)pObj;
int32_t code = jsonToPhysicPlanNode(pJson, pObj);
@ -1966,20 +2084,90 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanJoinType, pNode->joinType, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanOnConditions, &pNode->pOnConditions);
code = jsonToNodeObject(pJson, jkJoinPhysiPlanOnConditions, &pNode->pOtherOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanMergeCondition, &pNode->pMergeCondition);
code = jsonToNodeObject(pJson, jkJoinPhysiPlanPrimKeyCondition, &pNode->pPrimKeyCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanColEqualOnConditions, &pNode->pColEqualOnConditions);
code = jsonToNodeObject(pJson, jkJoinPhysiPlanColEqualOnConditions, &pNode->pColEqCond);
}
return code;
}
static int32_t physiHashJoinNodeToJson(const void* pObj, SJson* pJson) {
const SHashJoinPhysiNode* pNode = (const SHashJoinPhysiNode*)pObj;
int32_t code = physicPlanNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanJoinType, pNode->joinType);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkJoinPhysiPlanOnLeftCols, pNode->pOnLeft);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkJoinPhysiPlanOnRightCols, pNode->pOnRight);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanOnConditions, nodeToJson, pNode->pFilterConditions);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkJoinPhysiPlanTargets, pNode->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanInputRowNum, pNode->inputStat[0].inputRowNum);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanInputRowSize, pNode->inputStat[0].inputRowSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanInputRowNum, pNode->inputStat[1].inputRowNum);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanInputRowSize, pNode->inputStat[1].inputRowSize);
}
return code;
}
static int32_t jsonToPhysiHashJoinNode(const SJson* pJson, void* pObj) {
SHashJoinPhysiNode* pNode = (SHashJoinPhysiNode*)pObj;
int32_t code = jsonToPhysicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanJoinType, pNode->joinType, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkJoinPhysiPlanOnLeftCols, &pNode->pOnLeft);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkJoinPhysiPlanOnRightCols, &pNode->pOnRight);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanOnConditions, &pNode->pFilterConditions);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanInputRowNum, pNode->inputStat[0].inputRowNum, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanInputRowSize, pNode->inputStat[0].inputRowSize, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanInputRowNum, pNode->inputStat[1].inputRowNum, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanInputRowSize, pNode->inputStat[1].inputRowSize, code);
}
return code;
}
static const char* jkAggPhysiPlanExprs = "Exprs";
static const char* jkAggPhysiPlanGroupKeys = "GroupKeys";
static const char* jkAggPhysiPlanAggFuncs = "AggFuncs";
@ -2795,6 +2983,148 @@ static int32_t jsonToPhysiDeleteNode(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkGroupCachePhysiPlanGroupCols = "GroupColumns";
static const char* jkGroupCachePhysiPlanGrpColsMayBeNull = "GroupColumnsMayBeNull";
static const char* jkGroupCachePhysiPlanGroupByUid = "GroupByUid";
static const char* jkGroupCachePhysiPlanGlobalGroup = "GlobalGroup";
static const char* jkGroupCachePhysiPlanBatchFetch = "BatchFetch";
static int32_t physiGroupCacheNodeToJson(const void* pObj, SJson* pJson) {
const SGroupCachePhysiNode* pNode = (const SGroupCachePhysiNode*)pObj;
int32_t code = physicPlanNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkGroupCachePhysiPlanGrpColsMayBeNull, pNode->grpColsMayBeNull);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkGroupCachePhysiPlanGroupByUid, pNode->grpByUid);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkGroupCachePhysiPlanGlobalGroup, pNode->globalGrp);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkGroupCachePhysiPlanBatchFetch, pNode->batchFetch);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkGroupCachePhysiPlanGroupCols, pNode->pGroupCols);
}
return code;
}
static int32_t jsonToPhysiGroupCacheNode(const SJson* pJson, void* pObj) {
SGroupCachePhysiNode* pNode = (SGroupCachePhysiNode*)pObj;
int32_t code = jsonToPhysicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkGroupCachePhysiPlanGrpColsMayBeNull, &pNode->grpColsMayBeNull);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkGroupCachePhysiPlanGroupByUid, &pNode->grpByUid);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkGroupCachePhysiPlanGlobalGroup, &pNode->globalGrp);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkGroupCachePhysiPlanBatchFetch, &pNode->batchFetch);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkGroupCachePhysiPlanGroupCols, &pNode->pGroupCols);
}
return code;
}
static const char* jkDynQueryCtrlPhysiPlanQueryType = "QueryType";
static const char* jkDynQueryCtrlPhysiPlanBatchFetch = "BatchFetch";
static const char* jkDynQueryCtrlPhysiPlanVgSlot0 = "VgSlot[0]";
static const char* jkDynQueryCtrlPhysiPlanVgSlot1 = "VgSlot[1]";
static const char* jkDynQueryCtrlPhysiPlanUidSlot0 = "UidSlot[0]";
static const char* jkDynQueryCtrlPhysiPlanUidSlot1 = "UidSlot[1]";
static const char* jkDynQueryCtrlPhysiPlanSrcScan0 = "SrcScan[0]";
static const char* jkDynQueryCtrlPhysiPlanSrcScan1 = "SrcScan[1]";
static int32_t physiDynQueryCtrlNodeToJson(const void* pObj, SJson* pJson) {
const SDynQueryCtrlPhysiNode* pNode = (const SDynQueryCtrlPhysiNode*)pObj;
int32_t code = physicPlanNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDynQueryCtrlPhysiPlanQueryType, pNode->qType);
}
if (TSDB_CODE_SUCCESS == code) {
switch (pNode->qType) {
case DYN_QTYPE_STB_HASH: {
code = tjsonAddBoolToObject(pJson, jkDynQueryCtrlPhysiPlanBatchFetch, pNode->stbJoin.batchFetch);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDynQueryCtrlPhysiPlanVgSlot0, pNode->stbJoin.vgSlot[0]);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDynQueryCtrlPhysiPlanVgSlot1, pNode->stbJoin.vgSlot[1]);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDynQueryCtrlPhysiPlanUidSlot0, pNode->stbJoin.uidSlot[0]);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDynQueryCtrlPhysiPlanUidSlot1, pNode->stbJoin.uidSlot[1]);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkDynQueryCtrlPhysiPlanSrcScan0, pNode->stbJoin.srcScan[0]);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkDynQueryCtrlPhysiPlanSrcScan1, pNode->stbJoin.srcScan[1]);
}
break;
}
default:
return TSDB_CODE_INVALID_PARA;
}
}
return code;
}
static int32_t jsonToPhysiDynQueryCtrlNode(const SJson* pJson, void* pObj) {
SDynQueryCtrlPhysiNode* pNode = (SDynQueryCtrlPhysiNode*)pObj;
int32_t code = jsonToPhysicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkDynQueryCtrlPhysiPlanQueryType, pNode->qType, code);
}
if (TSDB_CODE_SUCCESS == code) {
switch (pNode->qType) {
case DYN_QTYPE_STB_HASH: {
tjsonGetNumberValue(pJson, jkDynQueryCtrlPhysiPlanQueryType, pNode->qType, code);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkDynQueryCtrlPhysiPlanBatchFetch, &pNode->stbJoin.batchFetch);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkDynQueryCtrlPhysiPlanVgSlot0, pNode->stbJoin.vgSlot[0], code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkDynQueryCtrlPhysiPlanVgSlot1, pNode->stbJoin.vgSlot[1], code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkDynQueryCtrlPhysiPlanUidSlot0, pNode->stbJoin.uidSlot[0], code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkDynQueryCtrlPhysiPlanUidSlot1, pNode->stbJoin.uidSlot[1], code);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkDynQueryCtrlPhysiPlanSrcScan0, &pNode->stbJoin.srcScan[0]);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkDynQueryCtrlPhysiPlanSrcScan1, &pNode->stbJoin.srcScan[1]);
}
break;
}
default:
return TSDB_CODE_INVALID_PARA;
}
}
return code;
}
static const char* jkQueryNodeAddrId = "Id";
static const char* jkQueryNodeAddrInUse = "InUse";
static const char* jkQueryNodeAddrNumOfEps = "NumOfEps";
@ -6631,6 +6961,10 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return logicIndefRowsFuncNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC:
return logicInterpFuncNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_GROUP_CACHE:
return logicGroupCacheNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL:
return logicDynQueryCtrlNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_SUBPLAN:
return logicSubplanToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN:
@ -6651,7 +6985,9 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return physiProjectNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN:
return physiJoinNodeToJson(pObj, pJson);
return physiMergeJoinNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN:
return physiHashJoinNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG:
return physiAggNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:
@ -6697,6 +7033,10 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return physiQueryInsertNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_DELETE:
return physiDeleteNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE:
return physiGroupCacheNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL:
return physiDynQueryCtrlNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_SUBPLAN:
return subplanToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN:
@ -6950,6 +7290,10 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToLogicIndefRowsFuncNode(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC:
return jsonToLogicInterpFuncNode(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN_GROUP_CACHE:
return jsonToLogicGroupCacheNode(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL:
return jsonToLogicDynQueryCtrlNode(pJson, pObj);
case QUERY_NODE_LOGIC_SUBPLAN:
return jsonToLogicSubplan(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN:
@ -6970,7 +7314,9 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return jsonToPhysiProjectNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN:
return jsonToPhysiJoinNode(pJson, pObj);
return jsonToPhysiMergeJoinNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN:
return jsonToPhysiHashJoinNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG:
return jsonToPhysiAggNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:
@ -7014,6 +7360,10 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToPhysiQueryInsertNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_DELETE:
return jsonToPhysiDeleteNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE:
return jsonToPhysiGroupCacheNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL:
return jsonToPhysiDynQueryCtrlNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_SUBPLAN:
return jsonToSubplan(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN:

304
source/libs/nodes/src/nodesMsgFuncs.c
View File

@ -1854,6 +1854,7 @@ enum {
PHY_NODE_CODE_SLIMIT,
PHY_NODE_CODE_INPUT_TS_ORDER,
PHY_NODE_CODE_OUTPUT_TS_ORDER,
PHY_NODE_CODE_DYNAMIC_OP,
PHY_NODE_CODE_FORCE_NONBLOCKING_OPTR
};
@ -1879,6 +1880,9 @@ static int32_t physiNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeEnum(pEncoder, PHY_NODE_CODE_OUTPUT_TS_ORDER, pNode->outputTsOrder);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_NODE_CODE_DYNAMIC_OP, pNode->dynamicOp);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_NODE_CODE_FORCE_NONBLOCKING_OPTR, pNode->forceCreateNonBlockingOptr);
}
@ -1914,8 +1918,12 @@ static int32_t msgToPhysiNode(STlvDecoder* pDecoder, void* pObj) {
case PHY_NODE_CODE_OUTPUT_TS_ORDER:
code = tlvDecodeEnum(pTlv, &pNode->outputTsOrder, sizeof(pNode->outputTsOrder));
break;
case PHY_NODE_CODE_DYNAMIC_OP:
code = tlvDecodeBool(pTlv, &pNode->dynamicOp);
break;
case PHY_NODE_CODE_FORCE_NONBLOCKING_OPTR:
code = tlvDecodeBool(pTlv, &pNode->forceCreateNonBlockingOptr);
break;
default:
break;
}
@ -2380,14 +2388,14 @@ static int32_t msgToPhysiProjectNode(STlvDecoder* pDecoder, void* pObj) {
enum {
PHY_SORT_MERGE_JOIN_CODE_BASE_NODE = 1,
PHY_SORT_MERGE_JOIN_CODE_JOIN_TYPE,
PHY_SORT_MERGE_JOIN_CODE_MERGE_CONDITION,
PHY_SORT_MERGE_JOIN_CODE_PRIM_KEY_CONDITION,
PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS,
PHY_SORT_MERGE_JOIN_CODE_TARGETS,
PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER,
PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS
};
static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
static int32_t physiMergeJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
const SSortMergeJoinPhysiNode* pNode = (const SSortMergeJoinPhysiNode*)pObj;
int32_t code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_BASE_NODE, physiNodeToMsg, &pNode->node);
@ -2395,21 +2403,21 @@ static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_JOIN_TYPE, pNode->joinType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_MERGE_CONDITION, nodeToMsg, pNode->pMergeCondition);
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_PRIM_KEY_CONDITION, nodeToMsg, pNode->pPrimKeyCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS, nodeToMsg, pNode->pOnConditions);
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS, nodeToMsg, pNode->pOtherOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TARGETS, nodeListToMsg, pNode->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pColEqualOnConditions);
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pColEqCond);
}
return code;
}
static int32_t msgToPhysiJoinNode(STlvDecoder* pDecoder, void* pObj) {
static int32_t msgToPhysiMergeJoinNode(STlvDecoder* pDecoder, void* pObj) {
SSortMergeJoinPhysiNode* pNode = (SSortMergeJoinPhysiNode*)pObj;
int32_t code = TSDB_CODE_SUCCESS;
@ -2422,17 +2430,17 @@ static int32_t msgToPhysiJoinNode(STlvDecoder* pDecoder, void* pObj) {
case PHY_SORT_MERGE_JOIN_CODE_JOIN_TYPE:
code = tlvDecodeEnum(pTlv, &pNode->joinType, sizeof(pNode->joinType));
break;
case PHY_SORT_MERGE_JOIN_CODE_MERGE_CONDITION:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pMergeCondition);
case PHY_SORT_MERGE_JOIN_CODE_PRIM_KEY_CONDITION:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pPrimKeyCond);
break;
case PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pOnConditions);
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pOtherOnCond);
break;
case PHY_SORT_MERGE_JOIN_CODE_TARGETS:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets);
break;
case PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pColEqualOnConditions);
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pColEqCond);
break;
default:
break;
@ -2442,6 +2450,100 @@ static int32_t msgToPhysiJoinNode(STlvDecoder* pDecoder, void* pObj) {
return code;
}
enum {
PHY_HASH_JOIN_CODE_BASE_NODE = 1,
PHY_HASH_JOIN_CODE_JOIN_TYPE,
PHY_HASH_JOIN_CODE_ON_LEFT_COLUMN,
PHY_HASH_JOIN_CODE_ON_RIGHT_COLUMN,
PHY_HASH_JOIN_CODE_ON_CONDITIONS,
PHY_HASH_JOIN_CODE_TARGETS,
PHY_HASH_JOIN_CODE_INPUT_ROW_NUM0,
PHY_HASH_JOIN_CODE_INPUT_ROW_SIZE0,
PHY_HASH_JOIN_CODE_INPUT_ROW_NUM1,
PHY_HASH_JOIN_CODE_INPUT_ROW_SIZE1
};
static int32_t physiHashJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
const SHashJoinPhysiNode* pNode = (const SHashJoinPhysiNode*)pObj;
int32_t code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_BASE_NODE, physiNodeToMsg, &pNode->node);
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeEnum(pEncoder, PHY_HASH_JOIN_CODE_JOIN_TYPE, pNode->joinType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_ON_LEFT_COLUMN, nodeListToMsg, pNode->pOnLeft);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_ON_RIGHT_COLUMN, nodeListToMsg, pNode->pOnRight);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_ON_CONDITIONS, nodeToMsg, pNode->pFilterConditions);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_TARGETS, nodeListToMsg, pNode->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI64(pEncoder, PHY_HASH_JOIN_CODE_INPUT_ROW_NUM0, pNode->inputStat[0].inputRowNum);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI64(pEncoder, PHY_HASH_JOIN_CODE_INPUT_ROW_NUM1, pNode->inputStat[1].inputRowNum);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, PHY_HASH_JOIN_CODE_INPUT_ROW_SIZE0, pNode->inputStat[0].inputRowSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, PHY_HASH_JOIN_CODE_INPUT_ROW_SIZE1, pNode->inputStat[1].inputRowSize);
}
return code;
}
static int32_t msgToPhysiHashJoinNode(STlvDecoder* pDecoder, void* pObj) {
SHashJoinPhysiNode* pNode = (SHashJoinPhysiNode*)pObj;
int32_t code = TSDB_CODE_SUCCESS;
STlv* pTlv = NULL;
tlvForEach(pDecoder, pTlv, code) {
switch (pTlv->type) {
case PHY_HASH_JOIN_CODE_BASE_NODE:
code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node);
break;
case PHY_HASH_JOIN_CODE_JOIN_TYPE:
code = tlvDecodeEnum(pTlv, &pNode->joinType, sizeof(pNode->joinType));
break;
case PHY_HASH_JOIN_CODE_ON_LEFT_COLUMN:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pOnLeft);
break;
case PHY_HASH_JOIN_CODE_ON_RIGHT_COLUMN:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pOnRight);
break;
case PHY_HASH_JOIN_CODE_ON_CONDITIONS:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pFilterConditions);
break;
case PHY_HASH_JOIN_CODE_TARGETS:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets);
break;
case PHY_HASH_JOIN_CODE_INPUT_ROW_NUM0:
code = tlvDecodeI64(pTlv, &pNode->inputStat[0].inputRowNum);
break;
case PHY_HASH_JOIN_CODE_INPUT_ROW_NUM1:
code = tlvDecodeI64(pTlv, &pNode->inputStat[1].inputRowNum);
break;
case PHY_HASH_JOIN_CODE_INPUT_ROW_SIZE0:
code = tlvDecodeI32(pTlv, &pNode->inputStat[0].inputRowSize);
break;
case PHY_HASH_JOIN_CODE_INPUT_ROW_SIZE1:
code = tlvDecodeI32(pTlv, &pNode->inputStat[1].inputRowSize);
break;
default:
break;
}
}
return code;
}
enum {
PHY_AGG_CODE_BASE_NODE = 1,
PHY_AGG_CODE_EXPR,
@ -3474,6 +3576,166 @@ static int32_t msgToPhysiDeleteNode(STlvDecoder* pDecoder, void* pObj) {
return code;
}
enum {
PHY_GROUP_CACHE_CODE_BASE_NODE = 1,
PHY_GROUP_CACHE_CODE_GROUP_COLS_MAY_BE_NULL,
PHY_GROUP_CACHE_CODE_GROUP_BY_UID,
PHY_GROUP_CACHE_CODE_GLOBAL_GROUP,
PHY_GROUP_CACHE_CODE_BATCH_FETCH,
PHY_GROUP_CACHE_CODE_GROUP_COLUMNS
};
static int32_t physiGroupCacheNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
const SGroupCachePhysiNode* pNode = (const SGroupCachePhysiNode*)pObj;
int32_t code = tlvEncodeObj(pEncoder, PHY_GROUP_CACHE_CODE_BASE_NODE, physiNodeToMsg, &pNode->node);
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_GROUP_CACHE_CODE_GROUP_COLUMNS, nodeListToMsg, pNode->pGroupCols);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_GROUP_CACHE_CODE_GROUP_COLS_MAY_BE_NULL, pNode->grpColsMayBeNull);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_GROUP_CACHE_CODE_GROUP_BY_UID, pNode->grpByUid);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_GROUP_CACHE_CODE_GLOBAL_GROUP, pNode->globalGrp);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_GROUP_CACHE_CODE_BATCH_FETCH, pNode->batchFetch);
}
return code;
}
static int32_t msgToPhysiGroupCacheNode(STlvDecoder* pDecoder, void* pObj) {
SGroupCachePhysiNode* pNode = (SGroupCachePhysiNode*)pObj;
int32_t code = TSDB_CODE_SUCCESS;
STlv* pTlv = NULL;
tlvForEach(pDecoder, pTlv, code) {
switch (pTlv->type) {
case PHY_GROUP_CACHE_CODE_BASE_NODE:
code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node);
break;
case PHY_GROUP_CACHE_CODE_GROUP_COLUMNS:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pGroupCols);
break;
case PHY_GROUP_CACHE_CODE_GROUP_COLS_MAY_BE_NULL:
code = tlvDecodeBool(pTlv, &pNode->grpColsMayBeNull);
break;
case PHY_GROUP_CACHE_CODE_GROUP_BY_UID:
code = tlvDecodeBool(pTlv, &pNode->grpByUid);
break;
case PHY_GROUP_CACHE_CODE_GLOBAL_GROUP:
code = tlvDecodeBool(pTlv, &pNode->globalGrp);
break;
case PHY_GROUP_CACHE_CODE_BATCH_FETCH:
code = tlvDecodeBool(pTlv, &pNode->batchFetch);
break;
default:
break;
}
}
return code;
}
enum {
PHY_DYN_QUERY_CTRL_CODE_BASE_NODE = 1,
PHY_DYN_QUERY_CTRL_CODE_QUERY_TYPE,
PHY_DYN_QUERY_CTRL_CODE_STB_JOIN_BATCH_FETCH,
PHY_DYN_QUERY_CTRL_CODE_STB_JOIN_VG_SLOT0,
PHY_DYN_QUERY_CTRL_CODE_STB_JOIN_VG_SLOT1,
PHY_DYN_QUERY_CTRL_CODE_STB_JOIN_UID_SLOT0,
PHY_DYN_QUERY_CTRL_CODE_STB_JOIN_UID_SLOT1,
PHY_DYN_QUERY_CTRL_CODE_STB_JOIN_SRC_SCAN0,
PHY_DYN_QUERY_CTRL_CODE_STB_JOIN_SRC_SCAN1,
};
static int32_t physiDynQueryCtrlNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
const SDynQueryCtrlPhysiNode* pNode = (const SDynQueryCtrlPhysiNode*)pObj;
int32_t code = tlvEncodeObj(pEncoder, PHY_DYN_QUERY_CTRL_CODE_BASE_NODE, physiNodeToMsg, &pNode->node);
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeEnum(pEncoder, PHY_DYN_QUERY_CTRL_CODE_QUERY_TYPE, pNode->qType);
}
if (TSDB_CODE_SUCCESS == code) {
switch (pNode->qType) {
case DYN_QTYPE_STB_HASH: {
code = tlvEncodeBool(pEncoder, PHY_DYN_QUERY_CTRL_CODE_STB_JOIN_BATCH_FETCH, pNode->stbJoin.batchFetch);
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeEnum(pEncoder, PHY_DYN_QUERY_CTRL_CODE_STB_JOIN_VG_SLOT0, pNode->stbJoin.vgSlot[0]);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeEnum(pEncoder, PHY_DYN_QUERY_CTRL_CODE_STB_JOIN_VG_SLOT1, pNode->stbJoin.vgSlot[1]);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeEnum(pEncoder, PHY_DYN_QUERY_CTRL_CODE_STB_JOIN_UID_SLOT0, pNode->stbJoin.uidSlot[0]);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeEnum(pEncoder, PHY_DYN_QUERY_CTRL_CODE_STB_JOIN_UID_SLOT1, pNode->stbJoin.uidSlot[1]);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_DYN_QUERY_CTRL_CODE_STB_JOIN_SRC_SCAN0, pNode->stbJoin.srcScan[0]);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_DYN_QUERY_CTRL_CODE_STB_JOIN_SRC_SCAN1, pNode->stbJoin.srcScan[1]);
}
break;
}
default:
return TSDB_CODE_INVALID_PARA;
}
}
return code;
}
static int32_t msgToPhysiDynQueryCtrlNode(STlvDecoder* pDecoder, void* pObj) {
SDynQueryCtrlPhysiNode* pNode = (SDynQueryCtrlPhysiNode*)pObj;
int32_t code = TSDB_CODE_SUCCESS;
STlv* pTlv = NULL;
tlvForEach(pDecoder, pTlv, code) {
switch (pTlv->type) {
case PHY_DYN_QUERY_CTRL_CODE_BASE_NODE:
code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node);
break;
case PHY_DYN_QUERY_CTRL_CODE_QUERY_TYPE:
code = tlvDecodeEnum(pTlv, &pNode->qType, sizeof(pNode->qType));
break;
case PHY_DYN_QUERY_CTRL_CODE_STB_JOIN_BATCH_FETCH:
code = tlvDecodeBool(pTlv, &pNode->stbJoin.batchFetch);
break;
case PHY_DYN_QUERY_CTRL_CODE_STB_JOIN_VG_SLOT0:
code = tlvDecodeEnum(pTlv, &pNode->stbJoin.vgSlot[0], sizeof(pNode->stbJoin.vgSlot[0]));
break;
case PHY_DYN_QUERY_CTRL_CODE_STB_JOIN_VG_SLOT1:
code = tlvDecodeEnum(pTlv, &pNode->stbJoin.vgSlot[1], sizeof(pNode->stbJoin.vgSlot[1]));
break;
case PHY_DYN_QUERY_CTRL_CODE_STB_JOIN_UID_SLOT0:
code = tlvDecodeEnum(pTlv, &pNode->stbJoin.uidSlot[0], sizeof(pNode->stbJoin.uidSlot[0]));
break;
case PHY_DYN_QUERY_CTRL_CODE_STB_JOIN_UID_SLOT1:
code = tlvDecodeEnum(pTlv, &pNode->stbJoin.uidSlot[1], sizeof(pNode->stbJoin.uidSlot[1]));
break;
case PHY_DYN_QUERY_CTRL_CODE_STB_JOIN_SRC_SCAN0:
code = tlvDecodeBool(pTlv, &pNode->stbJoin.srcScan[0]);
break;
case PHY_DYN_QUERY_CTRL_CODE_STB_JOIN_SRC_SCAN1:
code = tlvDecodeBool(pTlv, &pNode->stbJoin.srcScan[1]);
break;
default:
break;
}
}
return code;
}
enum { SUBPLAN_ID_CODE_QUERY_ID = 1, SUBPLAN_ID_CODE_GROUP_ID, SUBPLAN_ID_CODE_SUBPLAN_ID };
static int32_t subplanIdInlineToMsg(const void* pObj, STlvEncoder* pEncoder) {
@ -3796,7 +4058,10 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
code = physiProjectNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN:
code = physiJoinNodeToMsg(pObj, pEncoder);
code = physiMergeJoinNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN:
code = physiHashJoinNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG:
code = physiAggNodeToMsg(pObj, pEncoder);
@ -3857,6 +4122,12 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
case QUERY_NODE_PHYSICAL_PLAN_DELETE:
code = physiDeleteNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE:
code = physiGroupCacheNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL:
code = physiDynQueryCtrlNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_SUBPLAN:
code = subplanToMsg(pObj, pEncoder);
break;
@ -3941,7 +4212,10 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) {
code = msgToPhysiProjectNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN:
code = msgToPhysiJoinNode(pDecoder, pObj);
code = msgToPhysiMergeJoinNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN:
code = msgToPhysiHashJoinNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG:
code = msgToPhysiAggNode(pDecoder, pObj);
@ -4002,6 +4276,12 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) {
case QUERY_NODE_PHYSICAL_PLAN_DELETE:
code = msgToPhysiDeleteNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE:
code = msgToPhysiGroupCacheNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL:
code = msgToPhysiDynQueryCtrlNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_SUBPLAN:
code = msgToSubplan(pDecoder, pObj);
break;

93
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -22,6 +22,7 @@
#include "tdatablock.h"
#include "thash.h"
#include "tref.h"
#include "functionMgt.h"
typedef struct SNodeMemChunk {
int32_t availableSize;
@ -301,6 +302,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SCaseWhenNode));
case QUERY_NODE_EVENT_WINDOW:
return makeNode(type, sizeof(SEventWindowNode));
case QUERY_NODE_HINT:
return makeNode(type, sizeof(SHintNode));
case QUERY_NODE_SET_OPERATOR:
return makeNode(type, sizeof(SSetOperator));
case QUERY_NODE_SELECT_STMT:
@ -406,7 +409,7 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SRevokeStmt));
case QUERY_NODE_SHOW_DNODES_STMT:
case QUERY_NODE_SHOW_MNODES_STMT:
// case QUERY_NODE_SHOW_MODULES_STMT:
case QUERY_NODE_SHOW_MODULES_STMT:
case QUERY_NODE_SHOW_QNODES_STMT:
case QUERY_NODE_SHOW_SNODES_STMT:
case QUERY_NODE_SHOW_BNODES_STMT:
@ -490,6 +493,10 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SIndefRowsFuncLogicNode));
case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC:
return makeNode(type, sizeof(SInterpFuncLogicNode));
case QUERY_NODE_LOGIC_PLAN_GROUP_CACHE:
return makeNode(type, sizeof(SGroupCacheLogicNode));
case QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL:
return makeNode(type, sizeof(SDynQueryCtrlLogicNode));
case QUERY_NODE_LOGIC_SUBPLAN:
return makeNode(type, sizeof(SLogicSubplan));
case QUERY_NODE_LOGIC_PLAN:
@ -516,6 +523,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SProjectPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN:
return makeNode(type, sizeof(SSortMergeJoinPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN:
return makeNode(type, sizeof(SHashJoinPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG:
return makeNode(type, sizeof(SAggPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:
@ -571,6 +580,10 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SQueryInserterNode));
case QUERY_NODE_PHYSICAL_PLAN_DELETE:
return makeNode(type, sizeof(SDataDeleterNode));
case QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE:
return makeNode(type, sizeof(SGroupCachePhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL:
return makeNode(type, sizeof(SDynQueryCtrlPhysiNode));
case QUERY_NODE_PHYSICAL_SUBPLAN:
return makeNode(type, sizeof(SSubplan));
case QUERY_NODE_PHYSICAL_PLAN:
@ -598,6 +611,7 @@ static void destroyLogicNode(SLogicNode* pNode) {
nodesDestroyList(pNode->pChildren);
nodesDestroyNode(pNode->pLimit);
nodesDestroyNode(pNode->pSlimit);
nodesDestroyList(pNode->pHint);
}
static void destroyPhysiNode(SPhysiNode* pNode) {
@ -646,6 +660,15 @@ static void destroyTableCfg(STableCfg* pCfg) {
static void destroySmaIndex(void* pIndex) { taosMemoryFree(((STableIndexInfo*)pIndex)->expr); }
static void destroyHintValue(EHintOption option, void* value) {
switch (option) {
default:
break;
}
taosMemoryFree(value);
}
void nodesDestroyNode(SNode* pNode) {
if (NULL == pNode) {
return;
@ -804,6 +827,11 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(pEvent->pEndCond);
break;
}
case QUERY_NODE_HINT: {
SHintNode* pHint = (SHintNode*)pNode;
destroyHintValue(pHint->option, pHint->value);
break;
}
case QUERY_NODE_SET_OPERATOR: {
SSetOperator* pStmt = (SSetOperator*)pNode;
nodesDestroyList(pStmt->pProjectionList);
@ -830,6 +858,7 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyList(pStmt->pOrderByList);
nodesDestroyNode((SNode*)pStmt->pLimit);
nodesDestroyNode((SNode*)pStmt->pSlimit);
nodesDestroyList(pStmt->pHint);
break;
}
case QUERY_NODE_VNODE_MODIFY_STMT: {
@ -982,7 +1011,7 @@ void nodesDestroyNode(SNode* pNode) {
break;
case QUERY_NODE_SHOW_DNODES_STMT:
case QUERY_NODE_SHOW_MNODES_STMT:
// case QUERY_NODE_SHOW_MODULES_STMT:
case QUERY_NODE_SHOW_MODULES_STMT:
case QUERY_NODE_SHOW_QNODES_STMT:
case QUERY_NODE_SHOW_SNODES_STMT:
case QUERY_NODE_SHOW_BNODES_STMT:
@ -1095,9 +1124,9 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_LOGIC_PLAN_JOIN: {
SJoinLogicNode* pLogicNode = (SJoinLogicNode*)pNode;
destroyLogicNode((SLogicNode*)pLogicNode);
nodesDestroyNode(pLogicNode->pMergeCondition);
nodesDestroyNode(pLogicNode->pOnConditions);
nodesDestroyNode(pLogicNode->pColEqualOnConditions);
nodesDestroyNode(pLogicNode->pPrimKeyEqCond);
nodesDestroyNode(pLogicNode->pOtherOnCond);
nodesDestroyNode(pLogicNode->pColEqCond);
break;
}
case QUERY_NODE_LOGIC_PLAN_AGG: {
@ -1183,6 +1212,17 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(pLogicNode->pTimeSeries);
break;
}
case QUERY_NODE_LOGIC_PLAN_GROUP_CACHE: {
SGroupCacheLogicNode* pLogicNode = (SGroupCacheLogicNode*)pNode;
destroyLogicNode((SLogicNode*)pLogicNode);
nodesDestroyList(pLogicNode->pGroupCols);
break;
}
case QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL: {
SDynQueryCtrlLogicNode* pLogicNode = (SDynQueryCtrlLogicNode*)pNode;
destroyLogicNode((SLogicNode*)pLogicNode);
break;
}
case QUERY_NODE_LOGIC_SUBPLAN: {
SLogicSubplan* pSubplan = (SLogicSubplan*)pNode;
nodesDestroyList(pSubplan->pChildren);
@ -1227,10 +1267,23 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: {
SSortMergeJoinPhysiNode* pPhyNode = (SSortMergeJoinPhysiNode*)pNode;
destroyPhysiNode((SPhysiNode*)pPhyNode);
nodesDestroyNode(pPhyNode->pMergeCondition);
nodesDestroyNode(pPhyNode->pOnConditions);
nodesDestroyNode(pPhyNode->pPrimKeyCond);
nodesDestroyNode(pPhyNode->pOtherOnCond);
nodesDestroyList(pPhyNode->pTargets);
nodesDestroyNode(pPhyNode->pColEqualOnConditions);
nodesDestroyNode(pPhyNode->pColEqCond);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN: {
SHashJoinPhysiNode* pPhyNode = (SHashJoinPhysiNode*)pNode;
destroyPhysiNode((SPhysiNode*)pPhyNode);
nodesDestroyList(pPhyNode->pOnLeft);
nodesDestroyList(pPhyNode->pOnRight);
nodesDestroyNode(pPhyNode->pFilterConditions);
nodesDestroyList(pPhyNode->pTargets);
nodesDestroyNode(pPhyNode->pPrimKeyCond);
nodesDestroyNode(pPhyNode->pColEqCond);
nodesDestroyNode(pPhyNode->pTagEqCond);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: {
@ -1351,6 +1404,17 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(pSink->pEndTs);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE: {
SGroupCachePhysiNode* pPhyNode = (SGroupCachePhysiNode*)pNode;
destroyPhysiNode((SPhysiNode*)pPhyNode);
nodesDestroyList(pPhyNode->pGroupCols);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL: {
SDynQueryCtrlPhysiNode* pPhyNode = (SDynQueryCtrlPhysiNode*)pNode;
destroyPhysiNode((SPhysiNode*)pPhyNode);
break;
}
case QUERY_NODE_PHYSICAL_SUBPLAN: {
SSubplan* pSubplan = (SSubplan*)pNode;
nodesClearList(pSubplan->pChildren);
@ -1857,7 +1921,7 @@ static EDealRes doCollect(SCollectColumnsCxt* pCxt, SColumnNode* pCol, SNode* pN
static bool isCollectType(ECollectColType collectType, EColumnType colType) {
return COLLECT_COL_TYPE_ALL == collectType
? true
: (COLLECT_COL_TYPE_TAG == collectType ? COLUMN_TYPE_TAG == colType : COLUMN_TYPE_TAG != colType);
: (COLLECT_COL_TYPE_TAG == collectType ? COLUMN_TYPE_TAG == colType : (COLUMN_TYPE_TAG != colType && COLUMN_TYPE_TBNAME != colType));
}
static EDealRes collectColumns(SNode* pNode, void* pContext) {
@ -1936,6 +2000,7 @@ int32_t nodesCollectColumnsFromNode(SNode* node, const char* pTableAlias, EColle
typedef struct SCollectFuncsCxt {
int32_t errCode;
char* tableAlias;
FFuncClassifier classifier;
SNodeList* pFuncs;
SHashObj* pFuncsSet;
@ -1945,6 +2010,13 @@ static EDealRes collectFuncs(SNode* pNode, void* pContext) {
SCollectFuncsCxt* pCxt = (SCollectFuncsCxt*)pContext;
if (QUERY_NODE_FUNCTION == nodeType(pNode) && pCxt->classifier(((SFunctionNode*)pNode)->funcId) &&
!(((SExprNode*)pNode)->orderAlias)) {
SFunctionNode* pFunc = (SFunctionNode*)pNode;
if (FUNCTION_TYPE_TBNAME == pFunc->funcType && pCxt->tableAlias) {
SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0);
if (pVal && strcmp(pVal->literal, pCxt->tableAlias)) {
return DEAL_RES_CONTINUE;
}
}
SExprNode* pExpr = (SExprNode*)pNode;
if (NULL == taosHashGet(pCxt->pFuncsSet, &pExpr, sizeof(SExprNode*))) {
pCxt->errCode = nodesListStrictAppend(pCxt->pFuncs, nodesCloneNode(pNode));
@ -1967,13 +2039,14 @@ static int32_t funcNodeEqual(const void* pLeft, const void* pRight, size_t len)
return nodesEqualNode(*(const SNode**)pLeft, *(const SNode**)pRight) ? 0 : 1;
}
int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, FFuncClassifier classifier, SNodeList** pFuncs) {
int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, char* tableAlias, FFuncClassifier classifier, SNodeList** pFuncs) {
if (NULL == pSelect || NULL == pFuncs) {
return TSDB_CODE_FAILED;
}
SCollectFuncsCxt cxt = {.errCode = TSDB_CODE_SUCCESS,
.classifier = classifier,
.tableAlias = tableAlias,
.pFuncs = (NULL == *pFuncs ? nodesMakeList() : *pFuncs),
.pFuncsSet = taosHashInit(4, funcNodeHash, false, false)};
if (NULL == cxt.pFuncs || NULL == cxt.pFuncsSet) {

3
source/libs/parser/inc/parAst.h
View File

@ -101,6 +101,7 @@ SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode
SNode* createColumnNode(SAstCreateContext* pCxt, SToken* pTableAlias, SToken* pColumnName);
SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral);
SNodeList* createHintNodeList(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* createIdentifierValueNode(SAstCreateContext* pCxt, SToken* pLiteral);
SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt);
@ -142,7 +143,7 @@ SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit);
SNode* addRangeClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pRange);
SNode* addEveryClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pEvery);
SNode* addFillClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pFill);
SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable);
SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable, SNodeList* pHint);
SNode* setSelectStmtTagMode(SAstCreateContext* pCxt, SNode* pStmt, bool bSelectTags);
SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight);

2
source/libs/parser/inc/parUtil.h
View File

@ -116,7 +116,7 @@ int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName,
int32_t getTableCfgFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableCfg** pOutput);
int32_t getDnodeListFromCache(SParseMetaCache* pMetaCache, SArray** pDnodes);
void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request);
SNode* createSelectStmtImpl(bool isDistinct, SNodeList* pProjectionList, SNode* pTable);
SNode* createSelectStmtImpl(bool isDistinct, SNodeList* pProjectionList, SNode* pTable, SNodeList* pHint);
/**
* @brief return a - b with overflow check

11
source/libs/parser/inc/sql.y
View File

@ -1009,11 +1009,11 @@ join_type(A) ::= INNER.
/************************************************ query_specification *************************************************/
query_specification(A) ::=
SELECT tag_mode_opt(M) set_quantifier_opt(B) select_list(C) from_clause_opt(D)
SELECT hint_list(M) tag_mode_opt(N) set_quantifier_opt(B) select_list(C) from_clause_opt(D)
where_clause_opt(E) partition_by_clause_opt(F) range_opt(J) every_opt(K)
fill_opt(L) twindow_clause_opt(G) group_by_clause_opt(H) having_clause_opt(I). {
A = createSelectStmt(pCxt, B, C, D);
A = setSelectStmtTagMode(pCxt, A, M);
A = createSelectStmt(pCxt, B, C, D, M);
A = setSelectStmtTagMode(pCxt, A, N);
A = addWhereClause(pCxt, A, E);
A = addPartitionByClause(pCxt, A, F);
A = addWindowClauseClause(pCxt, A, G);
@ -1024,6 +1024,11 @@ query_specification(A) ::=
A = addFillClause(pCxt, A, L);
}
%type hint_list { SNodeList* }
%destructor hint_list { nodesDestroyList($$); }
hint_list(A) ::= . { A = createHintNodeList(pCxt, NULL); }
hint_list(A) ::= NK_HINT(B). { A = createHintNodeList(pCxt, &B); }
%type tag_mode_opt { bool }
%destructor tag_mode_opt { }
tag_mode_opt(A) ::= . { A = false; }

121
source/libs/parser/src/parAstCreater.c
View File

@ -348,6 +348,123 @@ SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken*
return (SNode*)val;
}
bool addHintNodeToList(SAstCreateContext* pCxt, SNodeList** ppHintList, EHintOption opt, SToken* paramList, int32_t paramNum) {
void* value = NULL;
switch (opt) {
case HINT_BATCH_SCAN:
case HINT_NO_BATCH_SCAN: {
if (paramNum > 0) {
return true;
}
break;
}
default:
return true;
}
SHintNode* hint = (SHintNode*)nodesMakeNode(QUERY_NODE_HINT);
CHECK_OUT_OF_MEM(hint);
hint->option = opt;
hint->value = value;
if (NULL == *ppHintList) {
*ppHintList = nodesMakeList();
CHECK_OUT_OF_MEM(*ppHintList);
}
pCxt->errCode = nodesListStrictAppend(*ppHintList, (SNode*)hint);
if (pCxt->errCode) {
return true;
}
return false;
}
SNodeList* createHintNodeList(SAstCreateContext* pCxt, const SToken* pLiteral) {
CHECK_PARSER_STATUS(pCxt);
if (NULL == pLiteral || pLiteral->n <= 5) {
return NULL;
}
SNodeList* pHintList = NULL;
char* hint = strndup(pLiteral->z + 3, pLiteral->n - 5);
int32_t i = 0;
bool quit = false;
bool inParamList = false;
bool lastComma = false;
EHintOption opt = 0;
int32_t paramNum = 0;
SToken paramList[10];
while (!quit) {
SToken t0 = {0};
if (hint[i] == 0) {
break;
}
t0.n = tGetToken(&hint[i], &t0.type);
t0.z = hint + i;
i += t0.n;
switch (t0.type) {
case TK_BATCH_SCAN:
lastComma = false;
if (0 != opt || inParamList) {
quit = true;
break;
}
opt = HINT_BATCH_SCAN;
break;
case TK_NO_BATCH_SCAN:
lastComma = false;
if (0 != opt || inParamList) {
quit = true;
break;
}
opt = HINT_NO_BATCH_SCAN;
break;
case TK_NK_LP:
lastComma = false;
if (0 == opt || inParamList) {
quit = true;
}
inParamList = true;
break;
case TK_NK_RP:
lastComma = false;
if (0 == opt || !inParamList) {
quit = true;
} else {
quit = addHintNodeToList(pCxt, &pHintList, opt, paramList, paramNum);
inParamList = false;
paramNum = 0;
opt = 0;
}
break;
case TK_NK_ID:
lastComma = false;
if (0 == opt || !inParamList) {
quit = true;
} else {
paramList[paramNum++] = t0;
}
break;
case TK_NK_COMMA:
if (lastComma) {
quit = true;
}
lastComma = true;
break;
case TK_NK_SPACE:
break;
default:
lastComma = false;
quit = true;
break;
}
}
taosMemoryFree(hint);
return pHintList;
}
SNode* createIdentifierValueNode(SAstCreateContext* pCxt, SToken* pLiteral) {
trimEscape(pLiteral);
return createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, pLiteral);
@ -845,9 +962,9 @@ SNode* addFillClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pFill) {
return pStmt;
}
SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable) {
SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable, SNodeList* pHint) {
CHECK_PARSER_STATUS(pCxt);
SNode* select = createSelectStmtImpl(isDistinct, pProjectionList, pTable);
SNode* select = createSelectStmtImpl(isDistinct, pProjectionList, pTable, pHint);
CHECK_OUT_OF_MEM(select);
return select;
}

2
source/libs/parser/src/parAstParser.c
View File

@ -690,10 +690,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
return collectMetaKeyFromShowDnodes(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_MNODES_STMT:
return collectMetaKeyFromShowMnodes(pCxt, (SShowStmt*)pStmt);
/*
case QUERY_NODE_SHOW_MODULES_STMT:
return collectMetaKeyFromShowModules(pCxt, (SShowStmt*)pStmt);
*/
case QUERY_NODE_SHOW_QNODES_STMT:
return collectMetaKeyFromShowQnodes(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_SNODES_STMT:

2
source/libs/parser/src/parAuthenticator.c
View File

@ -263,7 +263,7 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
return authAlterTable(pCxt, (SAlterTableStmt*)pStmt);
case QUERY_NODE_SHOW_DNODES_STMT:
case QUERY_NODE_SHOW_MNODES_STMT:
// case QUERY_NODE_SHOW_MODULES_STMT:
case QUERY_NODE_SHOW_MODULES_STMT:
case QUERY_NODE_SHOW_QNODES_STMT:
case QUERY_NODE_SHOW_SNODES_STMT:
case QUERY_NODE_SHOW_BNODES_STMT:

8
source/libs/parser/src/parTokenizer.c
View File

@ -42,6 +42,7 @@ static SKeyword keywordTable[] = {
{"ASC", TK_ASC},
{"AT_ONCE", TK_AT_ONCE},
{"BALANCE", TK_BALANCE},
{"BATCH_SCAN", TK_BATCH_SCAN},
{"BETWEEN", TK_BETWEEN},
{"BIGINT", TK_BIGINT},
{"BINARY", TK_BINARY},
@ -153,6 +154,7 @@ static SKeyword keywordTable[] = {
{"NONE", TK_NONE},
{"NOT", TK_NOT},
{"NOW", TK_NOW},
{"NO_BATCH_SCAN", TK_NO_BATCH_SCAN},
{"NULL", TK_NULL},
{"NULL_F", TK_NULL_F},
{"NULLS", TK_NULLS},
@ -398,10 +400,14 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) {
*tokenId = TK_NK_SLASH;
return 1;
}
bool isHint = false;
if (z[2] == '+') {
isHint = true;
}
for (i = 3; z[i] && (z[i] != '/' || z[i - 1] != '*'); i++) {
}
if (z[i]) i++;
*tokenId = TK_NK_COMMENT;
*tokenId = isHint ? TK_NK_HINT : TK_NK_COMMENT;
return i;
}
case '%': {

13
source/libs/parser/src/parTranslater.c
View File

@ -92,7 +92,6 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = {
.numOfShowCols = 1,
.pShowCols = {"*"}
},
/*
{
.showType = QUERY_NODE_SHOW_MODULES_STMT,
.pDbName = TSDB_INFORMATION_SCHEMA_DB,
@ -100,7 +99,6 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = {
.numOfShowCols = 1,
.pShowCols = {"*"}
},
*/
{
.showType = QUERY_NODE_SHOW_QNODES_STMT,
.pDbName = TSDB_INFORMATION_SCHEMA_DB,
@ -2810,6 +2808,13 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) {
pJoinTable->table.precision = calcJoinTablePrecision(pJoinTable);
pJoinTable->table.singleTable = joinTableIsSingleTable(pJoinTable);
code = translateExpr(pCxt, &pJoinTable->pOnCond);
pJoinTable->hasSubQuery = (nodeType(pJoinTable->pLeft) != QUERY_NODE_REAL_TABLE) || (nodeType(pJoinTable->pRight) != QUERY_NODE_REAL_TABLE);
if (nodeType(pJoinTable->pLeft) == QUERY_NODE_JOIN_TABLE) {
((SJoinTableNode*)pJoinTable->pLeft)->isLowLevelJoin = true;
}
if (nodeType(pJoinTable->pRight) == QUERY_NODE_JOIN_TABLE) {
((SJoinTableNode*)pJoinTable->pRight)->isLowLevelJoin = true;
}
}
break;
}
@ -6178,7 +6183,7 @@ static int32_t buildQueryForTableTopic(STranslateContext* pCxt, SCreateTopicStmt
strcpy(realTable->table.dbName, pStmt->subDbName);
strcpy(realTable->table.tableName, pStmt->subSTbName);
strcpy(realTable->table.tableAlias, pStmt->subSTbName);
*pSelect = createSelectStmtImpl(true, pProjection, (SNode*)realTable);
*pSelect = createSelectStmtImpl(true, pProjection, (SNode*)realTable, NULL);
((SSelectStmt*)*pSelect)->pWhere = nodesCloneNode(pStmt->pWhere);
pCxt->pParseCxt->topicQuery = true;
code = translateQuery(pCxt, *pSelect);
@ -9199,7 +9204,7 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
case QUERY_NODE_SHOW_USERS_STMT:
case QUERY_NODE_SHOW_DNODES_STMT:
case QUERY_NODE_SHOW_MNODES_STMT:
// case QUERY_NODE_SHOW_MODULES_STMT:
case QUERY_NODE_SHOW_MODULES_STMT:
case QUERY_NODE_SHOW_QNODES_STMT:
case QUERY_NODE_SHOW_FUNCTIONS_STMT:
case QUERY_NODE_SHOW_INDEXES_STMT:

5
source/libs/parser/src/parUtil.c
View File

@ -692,7 +692,7 @@ int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalog
}
SNode* createSelectStmtImpl(bool isDistinct, SNodeList* pProjectionList, SNode* pTable) {
SNode* createSelectStmtImpl(bool isDistinct, SNodeList* pProjectionList, SNode* pTable, SNodeList* pHint) {
SSelectStmt* select = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT);
if (NULL == select) {
return NULL;
@ -704,6 +704,7 @@ SNode* createSelectStmtImpl(bool isDistinct, SNodeList* pProjectionList, SNode*
select->timeLineResMode = select->isDistinct ? TIME_LINE_NONE : TIME_LINE_GLOBAL;
select->onlyHasKeepOrderFunc = true;
select->timeRange = TSWINDOW_INITIALIZER;
select->pHint = pHint;
return (SNode*)select;
}
@ -1157,3 +1158,5 @@ int64_t int64SafeSub(int64_t a, int64_t b) {
}
return res;
}

6701
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

4
source/libs/planner/inc/planInt.h
View File

@ -21,6 +21,7 @@ extern "C" {
#endif
#include "planner.h"
#include "tsimplehash.h"
#include "taoserror.h"
#define planFatal(param, ...) qFatal("PLAN: " param, ##__VA_ARGS__)
@ -43,6 +44,9 @@ int32_t splitLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan);
int32_t scaleOutLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SQueryLogicPlan** pLogicPlan);
int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList);
bool getBatchScanOptionFromHint(SNodeList* pList);
SLogicNode* getLogicNodeRootNode(SLogicNode* pCurr);
int32_t collectTableAliasFromNodes(SNode* pNode, SSHashObj** ppRes);
bool isPartTableAgg(SAggLogicNode* pAgg);
bool isPartTagAgg(SAggLogicNode* pAgg);
bool isPartTableWinodw(SWindowLogicNode* pWindow);

87
source/libs/planner/src/planLogicCreater.c
View File

@ -44,6 +44,11 @@ static void setColumnInfo(SFunctionNode* pFunc, SColumnNode* pCol, bool isPartit
switch (pFunc->funcType) {
case FUNCTION_TYPE_TBNAME:
pCol->colType = COLUMN_TYPE_TBNAME;
SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0);
if (pVal) {
strcpy(pCol->tableName, pVal->literal);
strcpy(pCol->tableAlias, pVal->literal);
}
break;
case FUNCTION_TYPE_WSTART:
if (!isPartitionBy) {
@ -420,16 +425,26 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
}
if (TSDB_CODE_SUCCESS == code) {
code = nodesCollectFuncs(pSelect, SQL_CLAUSE_FROM, fmIsScanPseudoColumnFunc, &pScan->pScanPseudoCols);
}
// rewrite the expression in subsequent clauses
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExprsForSelect(pScan->pScanPseudoCols, pSelect, SQL_CLAUSE_FROM, NULL);
code = nodesCollectFuncs(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, fmIsScanPseudoColumnFunc, &pScan->pScanPseudoCols);
}
pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->tableType, pSelect->tagScan);
// rewrite the expression in subsequent clauses
if (TSDB_CODE_SUCCESS == code) {
SNodeList* pNewScanPseudoCols = NULL;
code = rewriteExprsForSelect(pScan->pScanPseudoCols, pSelect, SQL_CLAUSE_FROM, NULL);
/*
if (TSDB_CODE_SUCCESS == code && NULL != pScan->pScanPseudoCols) {
code = createColumnByRewriteExprs(pScan->pScanPseudoCols, &pNewScanPseudoCols);
if (TSDB_CODE_SUCCESS == code) {
nodesDestroyList(pScan->pScanPseudoCols);
pScan->pScanPseudoCols = pNewScanPseudoCols;
}
}
*/
}
if (NULL != pScan->pScanCols) {
pScan->hasNormalCols = true;
}
@ -488,11 +503,14 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
}
pJoin->joinType = pJoinTable->joinType;
pJoin->joinAlgo = JOIN_ALGO_UNKNOWN;
pJoin->isSingleTableJoin = pJoinTable->table.singleTable;
pJoin->hasSubQuery = pJoinTable->hasSubQuery;
pJoin->node.inputTsOrder = ORDER_ASC;
pJoin->node.groupAction = GROUP_ACTION_CLEAR;
pJoin->node.requireDataOrder = DATA_ORDER_LEVEL_GLOBAL;
pJoin->node.resultDataOrder = DATA_ORDER_LEVEL_GLOBAL;
pJoin->node.resultDataOrder = DATA_ORDER_LEVEL_NONE;
pJoin->isLowLevelJoin = pJoinTable->isLowLevelJoin;
int32_t code = TSDB_CODE_SUCCESS;
@ -520,20 +538,51 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
// set on conditions
if (TSDB_CODE_SUCCESS == code && NULL != pJoinTable->pOnCond) {
pJoin->pOnConditions = nodesCloneNode(pJoinTable->pOnCond);
if (NULL == pJoin->pOnConditions) {
pJoin->pOtherOnCond = nodesCloneNode(pJoinTable->pOnCond);
if (NULL == pJoin->pOtherOnCond) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
// set the output
if (TSDB_CODE_SUCCESS == code) {
SNodeList* pColList = NULL;
if (QUERY_NODE_REAL_TABLE == nodeType(pJoinTable->pLeft) && !pJoin->isLowLevelJoin) {
code = nodesCollectColumns(pSelect, SQL_CLAUSE_WHERE, ((SRealTableNode*)pJoinTable->pLeft)->table.tableAlias, COLLECT_COL_TYPE_ALL, &pColList);
} else {
pJoin->node.pTargets = nodesCloneList(pLeft->pTargets);
if (NULL == pJoin->node.pTargets) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
if (TSDB_CODE_SUCCESS == code && NULL != pColList) {
code = createColumnByRewriteExprs(pColList, &pJoin->node.pTargets);
}
}
if (TSDB_CODE_SUCCESS == code) {
code = nodesListStrictAppendList(pJoin->node.pTargets, nodesCloneList(pRight->pTargets));
SNodeList* pColList = NULL;
if (QUERY_NODE_REAL_TABLE == nodeType(pJoinTable->pRight) && !pJoin->isLowLevelJoin) {
code = nodesCollectColumns(pSelect, SQL_CLAUSE_WHERE, ((SRealTableNode*)pJoinTable->pRight)->table.tableAlias, COLLECT_COL_TYPE_ALL, &pColList);
} else {
if (pJoin->node.pTargets) {
nodesListStrictAppendList(pJoin->node.pTargets, nodesCloneList(pRight->pTargets));
} else {
pJoin->node.pTargets = nodesCloneList(pRight->pTargets);
if (NULL == pJoin->node.pTargets) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
}
if (TSDB_CODE_SUCCESS == code && NULL != pColList) {
code = createColumnByRewriteExprs(pColList, &pJoin->node.pTargets);
}
}
if (NULL == pJoin->node.pTargets) {
pJoin->node.pTargets = nodesCloneList(pLeft->pTargets);
if (NULL == pJoin->node.pTargets) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
@ -642,7 +691,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect,
// set grouyp keys, agg funcs and having conditions
if (TSDB_CODE_SUCCESS == code) {
code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, fmIsAggFunc, &pAgg->pAggFuncs);
code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, NULL, fmIsAggFunc, &pAgg->pAggFuncs);
}
// rewrite the expression in subsequent clauses
@ -713,7 +762,7 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt
pIdfRowsFunc->node.resultDataOrder = pIdfRowsFunc->node.requireDataOrder;
// indefinite rows functions and _select_values functions
int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsVectorFunc, &pIdfRowsFunc->pFuncs);
int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, NULL, fmIsVectorFunc, &pIdfRowsFunc->pFuncs);
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExprsForSelect(pIdfRowsFunc->pFuncs, pSelect, SQL_CLAUSE_SELECT, NULL);
}
@ -751,7 +800,7 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p
pInterpFunc->node.resultDataOrder = pInterpFunc->node.requireDataOrder;
// interp functions and _group_key functions
int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, isInterpFunc, &pInterpFunc->pFuncs);
int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, NULL, isInterpFunc, &pInterpFunc->pFuncs);
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExprsForSelect(pInterpFunc->pFuncs, pSelect, SQL_CLAUSE_SELECT, NULL);
}
@ -797,7 +846,7 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm
pWindow->node.inputTsOrder = ORDER_ASC;
pWindow->node.outputTsOrder = ORDER_ASC;
int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_WINDOW, fmIsWindowClauseFunc, &pWindow->pFuncs);
int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_WINDOW, NULL, fmIsWindowClauseFunc, &pWindow->pFuncs);
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExprsForSelect(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW, NULL);
}
@ -1328,11 +1377,17 @@ static int32_t createSelectFromLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p
}
static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
int32_t code = TSDB_CODE_SUCCESS;
if (NULL == pSelect->pFromTable) {
return createSelectWithoutFromLogicNode(pCxt, pSelect, pLogicNode);
code = createSelectWithoutFromLogicNode(pCxt, pSelect, pLogicNode);
} else {
return createSelectFromLogicNode(pCxt, pSelect, pLogicNode);
code = createSelectFromLogicNode(pCxt, pSelect, pLogicNode);
}
if (TSDB_CODE_SUCCESS == code && NULL != *pLogicNode) {
(*pLogicNode)->stmtRoot = true;
TSWAP((*pLogicNode)->pHint, pSelect->pHint);
}
return code;
}
static int32_t createSetOpRootLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetOperator, FCreateSetOpLogicNode func,

684
source/libs/planner/src/planOptimizer.c
View File

@ -50,8 +50,8 @@ typedef struct SOsdInfo {
} SOsdInfo;
typedef struct SCpdIsMultiTableCondCxt {
SNodeList* pLeftCols;
SNodeList* pRightCols;
SSHashObj* pLeftTbls;
SSHashObj* pRightTbls;
bool havaLeftCol;
bool haveRightCol;
} SCpdIsMultiTableCondCxt;
@ -505,12 +505,21 @@ static bool pushDownCondOptBelongThisTable(SNode* pCondCol, SNodeList* pTableCol
return false;
}
static bool pushDownCondOptColInTableList(SNode* pCondCol, SSHashObj* pTables) {
SColumnNode* pTableCol = (SColumnNode*)pCondCol;
if (NULL == tSimpleHashGet(pTables, pTableCol->tableAlias, strlen(pTableCol->tableAlias))) {
return false;
}
return true;
}
static EDealRes pushDownCondOptIsCrossTableCond(SNode* pNode, void* pContext) {
SCpdIsMultiTableCondCxt* pCxt = pContext;
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
if (pushDownCondOptBelongThisTable(pNode, pCxt->pLeftCols)) {
if (pushDownCondOptColInTableList(pNode, pCxt->pLeftTbls)) {
pCxt->havaLeftCol = true;
} else if (pushDownCondOptBelongThisTable(pNode, pCxt->pRightCols)) {
} else if (pushDownCondOptColInTableList(pNode, pCxt->pRightTbls)) {
pCxt->haveRightCol = true;
}
return pCxt->havaLeftCol && pCxt->haveRightCol ? DEAL_RES_END : DEAL_RES_CONTINUE;
@ -518,10 +527,10 @@ static EDealRes pushDownCondOptIsCrossTableCond(SNode* pNode, void* pContext) {
return DEAL_RES_CONTINUE;
}
static ECondAction pushDownCondOptGetCondAction(EJoinType joinType, SNodeList* pLeftCols, SNodeList* pRightCols,
static ECondAction pushDownCondOptGetCondAction(EJoinType joinType, SSHashObj* pLeftTbls, SSHashObj* pRightTbls,
SNode* pNode) {
SCpdIsMultiTableCondCxt cxt = {
.pLeftCols = pLeftCols, .pRightCols = pRightCols, .havaLeftCol = false, .haveRightCol = false};
.pLeftTbls = pLeftTbls, .pRightTbls = pRightTbls, .havaLeftCol = false, .haveRightCol = false};
nodesWalkExpr(pNode, pushDownCondOptIsCrossTableCond, &cxt);
return (JOIN_TYPE_INNER != joinType
? COND_ACTION_STAY
@ -534,12 +543,14 @@ static int32_t pushDownCondOptPartLogicCond(SJoinLogicNode* pJoin, SNode** pOnCo
SNode** pRightChildCond) {
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pJoin->node.pConditions;
if (LOGIC_COND_TYPE_AND != pLogicCond->condType) {
return TSDB_CODE_SUCCESS;
return TSDB_CODE_PLAN_NOT_SUPPORT_JOIN_COND;
}
SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets;
SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets;
int32_t code = TSDB_CODE_SUCCESS;
SSHashObj* pLeftTables = NULL;
SSHashObj* pRightTables = NULL;
collectTableAliasFromNodes(nodesListGetNode(pJoin->node.pChildren, 0), &pLeftTables);
collectTableAliasFromNodes(nodesListGetNode(pJoin->node.pChildren, 1), &pRightTables);
SNodeList* pOnConds = NULL;
SNodeList* pLeftChildConds = NULL;
@ -547,7 +558,7 @@ static int32_t pushDownCondOptPartLogicCond(SJoinLogicNode* pJoin, SNode** pOnCo
SNodeList* pRemainConds = NULL;
SNode* pCond = NULL;
FOREACH(pCond, pLogicCond->pParameterList) {
ECondAction condAction = pushDownCondOptGetCondAction(pJoin->joinType, pLeftCols, pRightCols, pCond);
ECondAction condAction = pushDownCondOptGetCondAction(pJoin->joinType, pLeftTables, pRightTables, pCond);
if (COND_ACTION_PUSH_JOIN == condAction) {
code = nodesListMakeAppend(&pOnConds, nodesCloneNode(pCond));
} else if (COND_ACTION_PUSH_LEFT_CHILD == condAction) {
@ -562,6 +573,9 @@ static int32_t pushDownCondOptPartLogicCond(SJoinLogicNode* pJoin, SNode** pOnCo
}
}
tSimpleHashCleanup(pLeftTables);
tSimpleHashCleanup(pRightTables);
SNode* pTempOnCond = NULL;
SNode* pTempLeftChildCond = NULL;
SNode* pTempRightChildCond = NULL;
@ -601,10 +615,17 @@ static int32_t pushDownCondOptPartLogicCond(SJoinLogicNode* pJoin, SNode** pOnCo
static int32_t pushDownCondOptPartOpCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond,
SNode** pRightChildCond) {
SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets;
SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets;
SSHashObj* pLeftTables = NULL;
SSHashObj* pRightTables = NULL;
collectTableAliasFromNodes(nodesListGetNode(pJoin->node.pChildren, 0), &pLeftTables);
collectTableAliasFromNodes(nodesListGetNode(pJoin->node.pChildren, 1), &pRightTables);
ECondAction condAction =
pushDownCondOptGetCondAction(pJoin->joinType, pLeftCols, pRightCols, pJoin->node.pConditions);
pushDownCondOptGetCondAction(pJoin->joinType, pLeftTables, pRightTables, pJoin->node.pConditions);
tSimpleHashCleanup(pLeftTables);
tSimpleHashCleanup(pRightTables);
if (COND_ACTION_STAY == condAction) {
return TSDB_CODE_SUCCESS;
} else if (COND_ACTION_PUSH_JOIN == condAction) {
@ -628,14 +649,14 @@ static int32_t pushDownCondOptPartCond(SJoinLogicNode* pJoin, SNode** pOnCond, S
}
static int32_t pushDownCondOptPushCondToOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode** pCond) {
return pushDownCondOptAppendCond(&pJoin->pOnConditions, pCond);
return pushDownCondOptAppendCond(&pJoin->pOtherOnCond, pCond);
}
static int32_t pushDownCondOptPushCondToChild(SOptimizeContext* pCxt, SLogicNode* pChild, SNode** pCond) {
return pushDownCondOptAppendCond(&pChild->pConditions, pCond);
}
static bool pushDownCondOptIsPriKey(SNode* pNode, SNodeList* pTableCols) {
static bool pushDownCondOptIsPriKey(SNode* pNode, SSHashObj* pTables) {
if (QUERY_NODE_COLUMN != nodeType(pNode)) {
return false;
}
@ -643,7 +664,7 @@ static bool pushDownCondOptIsPriKey(SNode* pNode, SNodeList* pTableCols) {
if (PRIMARYKEY_TIMESTAMP_COL_ID != pCol->colId || TSDB_SYSTEM_TABLE == pCol->tableType) {
return false;
}
return pushDownCondOptBelongThisTable(pNode, pTableCols);
return pushDownCondOptColInTableList(pNode, pTables);
}
static bool pushDownCondOptIsPriKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) {
@ -656,26 +677,37 @@ static bool pushDownCondOptIsPriKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond
return false;
}
SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets;
SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets;
if (pushDownCondOptIsPriKey(pOper->pLeft, pLeftCols)) {
return pushDownCondOptIsPriKey(pOper->pRight, pRightCols);
} else if (pushDownCondOptIsPriKey(pOper->pLeft, pRightCols)) {
return pushDownCondOptIsPriKey(pOper->pRight, pLeftCols);
}
return false;
SSHashObj* pLeftTables = NULL;
SSHashObj* pRightTables = NULL;
collectTableAliasFromNodes(nodesListGetNode(pJoin->node.pChildren, 0), &pLeftTables);
collectTableAliasFromNodes(nodesListGetNode(pJoin->node.pChildren, 1), &pRightTables);
bool res = false;
if (pushDownCondOptIsPriKey(pOper->pLeft, pLeftTables)) {
res = pushDownCondOptIsPriKey(pOper->pRight, pRightTables);
} else if (pushDownCondOptIsPriKey(pOper->pLeft, pRightTables)) {
res = pushDownCondOptIsPriKey(pOper->pRight, pLeftTables);
}
static bool pushDownCondOptContainPriKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) {
tSimpleHashCleanup(pLeftTables);
tSimpleHashCleanup(pRightTables);
return res;
}
static bool pushDownCondOptContainPriKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond, bool* errCond) {
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pCond)) {
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pCond;
if (LOGIC_COND_TYPE_AND != pLogicCond->condType) {
if (errCond) {
*errCond = true;
}
return false;
}
bool hasPrimaryKeyEqualCond = false;
SNode* pCond = NULL;
FOREACH(pCond, pLogicCond->pParameterList) {
if (pushDownCondOptContainPriKeyEqualCond(pJoin, pCond)) {
if (pushDownCondOptContainPriKeyEqualCond(pJoin, pCond, NULL)) {
hasPrimaryKeyEqualCond = true;
break;
}
@ -687,26 +719,33 @@ static bool pushDownCondOptContainPriKeyEqualCond(SJoinLogicNode* pJoin, SNode*
}
static int32_t pushDownCondOptCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
if (NULL == pJoin->pOnConditions) {
if (NULL == pJoin->pOtherOnCond) {
return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN);
}
if (!pushDownCondOptContainPriKeyEqualCond(pJoin, pJoin->pOnConditions)) {
bool errCond = false;
if (!pushDownCondOptContainPriKeyEqualCond(pJoin, pJoin->pOtherOnCond, &errCond)) {
if (errCond) {
return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_NOT_SUPPORT_JOIN_COND);
}
return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL);
}
return TSDB_CODE_SUCCESS;
}
static int32_t pushDownCondOptPartJoinOnCondLogicCond(SJoinLogicNode* pJoin, SNode** ppMergeCond, SNode** ppOnCond) {
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOnConditions);
static int32_t pushDownCondOptPartJoinOnCondLogicCond(SJoinLogicNode* pJoin, SNode** ppPrimKeyEqCond, SNode** ppOnCond) {
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOtherOnCond);
int32_t code = TSDB_CODE_SUCCESS;
SNodeList* pOnConds = NULL;
SNode* pCond = NULL;
FOREACH(pCond, pLogicCond->pParameterList) {
WHERE_EACH(pCond, pLogicCond->pParameterList) {
if (pushDownCondOptIsPriKeyEqualCond(pJoin, pCond)) {
*ppMergeCond = nodesCloneNode(pCond);
nodesDestroyNode(*ppPrimKeyEqCond);
*ppPrimKeyEqCond = nodesCloneNode(pCond);
ERASE_NODE(pLogicCond->pParameterList);
} else {
code = nodesListMakeAppend(&pOnConds, nodesCloneNode(pCond));
WHERE_NEXT;
}
}
@ -715,10 +754,10 @@ static int32_t pushDownCondOptPartJoinOnCondLogicCond(SJoinLogicNode* pJoin, SNo
code = nodesMergeConds(&pTempOnCond, &pOnConds);
}
if (TSDB_CODE_SUCCESS == code && NULL != *ppMergeCond) {
if (TSDB_CODE_SUCCESS == code && NULL != *ppPrimKeyEqCond) {
*ppOnCond = pTempOnCond;
nodesDestroyNode(pJoin->pOnConditions);
pJoin->pOnConditions = NULL;
nodesDestroyNode(pJoin->pOtherOnCond);
pJoin->pOtherOnCond = NULL;
return TSDB_CODE_SUCCESS;
} else {
nodesDestroyList(pOnConds);
@ -727,35 +766,34 @@ static int32_t pushDownCondOptPartJoinOnCondLogicCond(SJoinLogicNode* pJoin, SNo
}
}
static int32_t pushDownCondOptPartJoinOnCond(SJoinLogicNode* pJoin, SNode** ppMergeCond, SNode** ppOnCond) {
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) &&
LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOnConditions))->condType) {
return pushDownCondOptPartJoinOnCondLogicCond(pJoin, ppMergeCond, ppOnCond);
static int32_t pushDownCondOptPartJoinOnCond(SJoinLogicNode* pJoin, SNode** ppPrimKeyEqCond, SNode** ppOnCond) {
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOtherOnCond) &&
LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOtherOnCond))->condType) {
return pushDownCondOptPartJoinOnCondLogicCond(pJoin, ppPrimKeyEqCond, ppOnCond);
}
if (pushDownCondOptIsPriKeyEqualCond(pJoin, pJoin->pOnConditions)) {
*ppMergeCond = nodesCloneNode(pJoin->pOnConditions);
if (pushDownCondOptIsPriKeyEqualCond(pJoin, pJoin->pOtherOnCond)) {
*ppPrimKeyEqCond = pJoin->pOtherOnCond;
*ppOnCond = NULL;
nodesDestroyNode(pJoin->pOnConditions);
pJoin->pOnConditions = NULL;
pJoin->pOtherOnCond = NULL;
return TSDB_CODE_SUCCESS;
} else {
return TSDB_CODE_PLAN_INTERNAL_ERROR;
}
}
static int32_t pushDownCondOptJoinExtractMergeCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
static int32_t pushDownCondOptJoinExtractCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
int32_t code = pushDownCondOptCheckJoinOnCond(pCxt, pJoin);
SNode* pJoinMergeCond = NULL;
SNode* pPrimKeyEqCond = NULL;
SNode* pJoinOnCond = NULL;
if (TSDB_CODE_SUCCESS == code) {
code = pushDownCondOptPartJoinOnCond(pJoin, &pJoinMergeCond, &pJoinOnCond);
code = pushDownCondOptPartJoinOnCond(pJoin, &pPrimKeyEqCond, &pJoinOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
pJoin->pMergeCondition = pJoinMergeCond;
pJoin->pOnConditions = pJoinOnCond;
pJoin->pPrimKeyEqCond = pPrimKeyEqCond;
pJoin->pOtherOnCond = pJoinOnCond;
} else {
nodesDestroyNode(pJoinMergeCond);
nodesDestroyNode(pPrimKeyEqCond);
nodesDestroyNode(pJoinOnCond);
}
return code;
@ -769,85 +807,168 @@ static bool pushDownCondOptIsTableColumn(SNode* pNode, SNodeList* pTableCols) {
return pushDownCondOptBelongThisTable(pNode, pTableCols);
}
static bool pushDownCondOptIsColEqualOnCond(SJoinLogicNode* pJoin, SNode* pCond) {
static bool pushDownCondOptIsColEqualOnCond(SJoinLogicNode* pJoin, SNode* pCond, bool* allTags) {
if (QUERY_NODE_OPERATOR != nodeType(pCond)) {
return false;
}
SOperatorNode* pOper = (SOperatorNode*)pCond;
if (OP_TYPE_EQUAL != pOper->opType) {
return false;
}
if (QUERY_NODE_COLUMN != nodeType(pOper->pLeft) || QUERY_NODE_COLUMN != nodeType(pOper->pRight)) {
return false;
}
SColumnNode* pLeft = (SColumnNode*)(pOper->pLeft);
SColumnNode* pRight = (SColumnNode*)(pOper->pRight);
// TODO: add cast to operator and remove this restriction of optimization
*allTags = (COLUMN_TYPE_TAG == pLeft->colType) && (COLUMN_TYPE_TAG == pRight->colType);
if (OP_TYPE_EQUAL != pOper->opType) {
return false;
}
if (pLeft->node.resType.type != pRight->node.resType.type ||
pLeft->node.resType.bytes != pRight->node.resType.bytes) {
return false;
}
SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets;
SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets;
bool isEqual = false;
if (pushDownCondOptIsTableColumn(pOper->pLeft, pLeftCols)) {
return pushDownCondOptIsTableColumn(pOper->pRight, pRightCols);
isEqual = pushDownCondOptIsTableColumn(pOper->pRight, pRightCols);
} else if (pushDownCondOptIsTableColumn(pOper->pLeft, pRightCols)) {
return pushDownCondOptIsTableColumn(pOper->pRight, pLeftCols);
isEqual = pushDownCondOptIsTableColumn(pOper->pRight, pLeftCols);
}
return false;
if (isEqual) {
}
return isEqual;
}
static int32_t pushDownCondOptJoinExtractColEqualOnLogicCond(SJoinLogicNode* pJoin) {
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOnConditions);
static int32_t pushDownCondOptJoinExtractEqualOnLogicCond(SJoinLogicNode* pJoin) {
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOtherOnCond);
int32_t code = TSDB_CODE_SUCCESS;
SNodeList* pEqualOnConds = NULL;
SNodeList* pColEqOnConds = NULL;
SNodeList* pTagEqOnConds = NULL;
SNodeList* pTagOnConds = NULL;
SNode* pCond = NULL;
bool allTags = false;
FOREACH(pCond, pLogicCond->pParameterList) {
if (pushDownCondOptIsColEqualOnCond(pJoin, pCond)) {
code = nodesListMakeAppend(&pEqualOnConds, nodesCloneNode(pCond));
allTags = false;
if (pushDownCondOptIsColEqualOnCond(pJoin, pCond, &allTags)) {
if (allTags) {
code = nodesListMakeAppend(&pTagEqOnConds, nodesCloneNode(pCond));
} else {
code = nodesListMakeAppend(&pColEqOnConds, nodesCloneNode(pCond));
}
} else if (allTags) {
code = nodesListMakeAppend(&pTagOnConds, nodesCloneNode(pCond));
}
if (code) {
break;
}
}
SNode* pTempTagEqCond = NULL;
SNode* pTempColEqCond = NULL;
SNode* pTempTagOnCond = NULL;
if (TSDB_CODE_SUCCESS == code) {
code = nodesMergeConds(&pTempTagEqCond, &pEqualOnConds);
code = nodesMergeConds(&pTempColEqCond, &pColEqOnConds);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodesMergeConds(&pTempTagEqCond, &pTagEqOnConds);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodesMergeConds(&pTempTagOnCond, &pTagOnConds);
}
if (TSDB_CODE_SUCCESS == code) {
pJoin->pColEqualOnConditions = pTempTagEqCond;
pJoin->pColEqCond = pTempColEqCond;
pJoin->pTagEqCond = pTempTagEqCond;
pJoin->pTagOnCond = pTempTagOnCond;
return TSDB_CODE_SUCCESS;
} else {
nodesDestroyList(pEqualOnConds);
nodesDestroyList(pColEqOnConds);
nodesDestroyList(pTagEqOnConds);
return TSDB_CODE_PLAN_INTERNAL_ERROR;
}
return TSDB_CODE_SUCCESS;
}
static int32_t pushDownCondOptJoinExtractColEqualOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
if (NULL == pJoin->pOnConditions) {
pJoin->pColEqualOnConditions = NULL;
static int32_t pushDownCondOptJoinExtractEqualOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
if (NULL == pJoin->pOtherOnCond) {
pJoin->pColEqCond = NULL;
pJoin->pTagEqCond = NULL;
return TSDB_CODE_SUCCESS;
}
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) &&
LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOnConditions))->condType) {
return pushDownCondOptJoinExtractColEqualOnLogicCond(pJoin);
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOtherOnCond) &&
LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOtherOnCond))->condType) {
return pushDownCondOptJoinExtractEqualOnLogicCond(pJoin);
}
if (pushDownCondOptIsColEqualOnCond(pJoin, pJoin->pOnConditions)) {
pJoin->pColEqualOnConditions = nodesCloneNode(pJoin->pOnConditions);
bool allTags = false;
if (pushDownCondOptIsColEqualOnCond(pJoin, pJoin->pOtherOnCond, &allTags)) {
if (allTags) {
pJoin->pTagEqCond = nodesCloneNode(pJoin->pOtherOnCond);
} else {
pJoin->pColEqCond = nodesCloneNode(pJoin->pOtherOnCond);
}
} else if (allTags) {
pJoin->pTagOnCond = nodesCloneNode(pJoin->pOtherOnCond);
}
return TSDB_CODE_SUCCESS;
}
static int32_t pushDownCondOptAppendFilterCol(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
if (NULL == pJoin->pOtherOnCond) {
return TSDB_CODE_SUCCESS;
}
int32_t code = TSDB_CODE_SUCCESS;
SNodeList* pCondCols = nodesMakeList();
SNodeList* pTargets = NULL;
if (NULL == pCondCols) {
code = TSDB_CODE_OUT_OF_MEMORY;
} else {
code = nodesCollectColumnsFromNode(pJoin->pOtherOnCond, NULL, COLLECT_COL_TYPE_ALL, &pCondCols);
}
if (TSDB_CODE_SUCCESS == code) {
code = createColumnByRewriteExprs(pCondCols, &pTargets);
}
nodesDestroyList(pCondCols);
if (TSDB_CODE_SUCCESS == code) {
SNode* pNode = NULL;
FOREACH(pNode, pTargets) {
SNode* pTmp = NULL;
bool found = false;
FOREACH(pTmp, pJoin->node.pTargets) {
if (nodesEqualNode(pTmp, pNode)) {
found = true;
break;
}
}
if (!found) {
nodesListStrictAppend(pJoin->node.pTargets, nodesCloneNode(pNode));
}
}
}
nodesDestroyList(pTargets);
return code;
}
static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
if (OPTIMIZE_FLAG_TEST_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) {
return TSDB_CODE_SUCCESS;
}
if (pJoin->joinAlgo != JOIN_ALGO_UNKNOWN) {
return TSDB_CODE_SUCCESS;
}
if (NULL == pJoin->node.pConditions) {
int32_t code = pushDownCondOptJoinExtractMergeCond(pCxt, pJoin);
int32_t code = pushDownCondOptJoinExtractCond(pCxt, pJoin);
if (TSDB_CODE_SUCCESS == code) {
OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE);
pCxt->optimized = true;
@ -872,11 +993,15 @@ static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* p
}
if (TSDB_CODE_SUCCESS == code) {
code = pushDownCondOptJoinExtractMergeCond(pCxt, pJoin);
code = pushDownCondOptJoinExtractCond(pCxt, pJoin);
}
if (TSDB_CODE_SUCCESS == code) {
code = pushDownCondOptJoinExtractColEqualOnCond(pCxt, pJoin);
code = pushDownCondOptJoinExtractEqualOnCond(pCxt, pJoin);
}
if (TSDB_CODE_SUCCESS == code) {
code = pushDownCondOptAppendFilterCol(pCxt, pJoin);
}
if (TSDB_CODE_SUCCESS == code) {
@ -1230,6 +1355,7 @@ static int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool group
}
case QUERY_NODE_LOGIC_PLAN_AGG:
case QUERY_NODE_LOGIC_PLAN_PARTITION:
case QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL:
*pNotOptimize = true;
return TSDB_CODE_SUCCESS;
default:
@ -1848,10 +1974,13 @@ static bool eliminateProjOptCanChildConditionUseChildTargets(SLogicNode* pChild,
nodesWalkExpr(pChild->pConditions, eliminateProjOptCanUseNewChildTargetsImpl, &cxt);
if (!cxt.canUse) return false;
}
if (QUERY_NODE_LOGIC_PLAN_JOIN == nodeType(pChild) && NULL != ((SJoinLogicNode*)pChild)->pOnConditions) {
if (QUERY_NODE_LOGIC_PLAN_JOIN == nodeType(pChild) && ((SJoinLogicNode*)pChild)->joinAlgo != JOIN_ALGO_UNKNOWN) {
return false;
}
if (QUERY_NODE_LOGIC_PLAN_JOIN == nodeType(pChild) && ((SJoinLogicNode*)pChild)->pOtherOnCond) {
SJoinLogicNode* pJoinLogicNode = (SJoinLogicNode*)pChild;
CheckNewChildTargetsCxt cxt = {.pNewChildTargets = pNewChildTargets, .canUse = false};
nodesWalkExpr(pJoinLogicNode->pOnConditions, eliminateProjOptCanUseNewChildTargetsImpl, &cxt);
nodesWalkExpr(pJoinLogicNode->pOtherOnCond, eliminateProjOptCanUseNewChildTargetsImpl, &cxt);
if (!cxt.canUse) return false;
}
return true;
@ -3057,23 +3186,413 @@ static int32_t sortNonPriKeyOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLog
return TSDB_CODE_SUCCESS;
}
static bool stbJoinOptShouldBeOptimized(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_JOIN != nodeType(pNode)) {
return false;
}
SJoinLogicNode* pJoin = (SJoinLogicNode*)pNode;
if (pJoin->isSingleTableJoin || NULL == pJoin->pTagEqCond || pNode->pChildren->length != 2
|| pJoin->hasSubQuery || pJoin->joinAlgo != JOIN_ALGO_UNKNOWN || pJoin->isLowLevelJoin) {
if (pJoin->joinAlgo == JOIN_ALGO_UNKNOWN) {
pJoin->joinAlgo = JOIN_ALGO_MERGE;
}
return false;
}
return true;
}
int32_t stbJoinOptAddFuncToScanNode(char* funcName, SScanLogicNode* pScan) {
SFunctionNode* pUidFunc = createFunction(funcName, NULL);
snprintf(pUidFunc->node.aliasName, sizeof(pUidFunc->node.aliasName), "%s.%p",
pUidFunc->functionName, pUidFunc);
nodesListStrictAppend(pScan->pScanPseudoCols, (SNode *)pUidFunc);
return createColumnByRewriteExpr((SNode*)pUidFunc, &pScan->node.pTargets);
}
int32_t stbJoinOptRewriteToTagScan(SLogicNode* pJoin, SNode* pNode) {
SScanLogicNode* pScan = (SScanLogicNode*)pNode;
SJoinLogicNode* pJoinNode = (SJoinLogicNode*)pJoin;
pScan->scanType = SCAN_TYPE_TAG;
NODES_DESTORY_LIST(pScan->pScanCols);
NODES_DESTORY_NODE(pScan->node.pConditions);
pScan->node.requireDataOrder = DATA_ORDER_LEVEL_NONE;
pScan->node.resultDataOrder = DATA_ORDER_LEVEL_NONE;
pScan->onlyMetaCtbIdx = true;
SNodeList* pTags = nodesMakeList();
int32_t code = nodesCollectColumnsFromNode(pJoinNode->pTagEqCond, NULL, COLLECT_COL_TYPE_TAG, &pTags);
if (TSDB_CODE_SUCCESS == code) {
code = nodesCollectColumnsFromNode(pJoinNode->pTagOnCond, NULL, COLLECT_COL_TYPE_TAG, &pTags);
}
if (TSDB_CODE_SUCCESS == code) {
SNode* pTarget = NULL;
SNode* pTag = NULL;
bool found = false;
WHERE_EACH(pTarget, pScan->node.pTargets) {
found = false;
FOREACH(pTag, pTags) {
if (nodesEqualNode(pTarget, pTag)) {
found = true;
break;
}
}
if (!found) {
ERASE_NODE(pScan->node.pTargets);
} else {
WHERE_NEXT;
}
}
}
if (TSDB_CODE_SUCCESS == code) {
code = stbJoinOptAddFuncToScanNode("_tbuid", pScan);
}
if (TSDB_CODE_SUCCESS == code) {
code = stbJoinOptAddFuncToScanNode("_vgid", pScan);
}
if (code) {
nodesDestroyList(pTags);
}
return code;
}
static int32_t stbJoinOptCreateTagScanNode(SLogicNode* pJoin, SNodeList** ppList) {
SNodeList* pList = nodesCloneList(pJoin->pChildren);
if (NULL == pList) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t code = TSDB_CODE_SUCCESS;
SNode* pNode = NULL;
FOREACH(pNode, pList) {
code = stbJoinOptRewriteToTagScan(pJoin, pNode);
if (code) {
break;
}
}
if (TSDB_CODE_SUCCESS == code) {
*ppList = pList;
} else {
nodesDestroyList(pList);
}
return code;
}
static int32_t stbJoinOptCreateTagHashJoinNode(SLogicNode* pOrig, SNodeList* pChildren, SLogicNode** ppLogic) {
SJoinLogicNode* pOrigJoin = (SJoinLogicNode*)pOrig;
SJoinLogicNode* pJoin = (SJoinLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_JOIN);
if (NULL == pJoin) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pJoin->joinType = pOrigJoin->joinType;
pJoin->joinAlgo = JOIN_ALGO_HASH;
pJoin->isSingleTableJoin = pOrigJoin->isSingleTableJoin;
pJoin->hasSubQuery = pOrigJoin->hasSubQuery;
pJoin->node.inputTsOrder = pOrigJoin->node.inputTsOrder;
pJoin->node.groupAction = pOrigJoin->node.groupAction;
pJoin->node.requireDataOrder = DATA_ORDER_LEVEL_NONE;
pJoin->node.resultDataOrder = DATA_ORDER_LEVEL_NONE;
pJoin->pTagEqCond = nodesCloneNode(pOrigJoin->pTagEqCond);
pJoin->pOtherOnCond = nodesCloneNode(pOrigJoin->pTagOnCond);
int32_t code = TSDB_CODE_SUCCESS;
pJoin->node.pChildren = pChildren;
SNode* pNode = NULL;
FOREACH(pNode, pChildren) {
SScanLogicNode* pScan = (SScanLogicNode*)pNode;
SNode* pCol = NULL;
FOREACH(pCol, pScan->pScanPseudoCols) {
if (QUERY_NODE_FUNCTION == nodeType(pCol) && (((SFunctionNode*)pCol)->funcType == FUNCTION_TYPE_TBUID || ((SFunctionNode*)pCol)->funcType == FUNCTION_TYPE_VGID)) {
code = createColumnByRewriteExpr(pCol, &pJoin->node.pTargets);
if (code) {
break;
}
}
}
if (code) {
break;
}
pScan->node.pParent = (SLogicNode*)pJoin;
}
if (TSDB_CODE_SUCCESS == code) {
*ppLogic = (SLogicNode*)pJoin;
} else {
nodesDestroyNode((SNode*)pJoin);
}
return code;
}
static int32_t stbJoinOptCreateTableScanNodes(SLogicNode* pJoin, SNodeList** ppList, bool* srcScan) {
SNodeList* pList = nodesCloneList(pJoin->pChildren);
if (NULL == pList) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t code = TSDB_CODE_SUCCESS;
int32_t i = 0;
SNode* pNode = NULL;
FOREACH(pNode, pList) {
SScanLogicNode* pScan = (SScanLogicNode*)pNode;
//code = stbJoinOptAddFuncToScanNode("_tbuid", pScan);
//if (code) {
// break;
//}
pScan->node.dynamicOp = true;
*(srcScan + i++) = pScan->pVgroupList->numOfVgroups <= 1;
pScan->scanType = SCAN_TYPE_TABLE;
}
if (TSDB_CODE_SUCCESS == code) {
*ppList = pList;
} else {
nodesDestroyList(pList);
*ppList = NULL;
}
return code;
}
static int32_t stbJoinOptCreateGroupCacheNode(SLogicNode* pRoot, SNodeList* pChildren, SLogicNode** ppLogic) {
int32_t code = TSDB_CODE_SUCCESS;
SGroupCacheLogicNode* pGrpCache = (SGroupCacheLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_GROUP_CACHE);
if (NULL == pGrpCache) {
return TSDB_CODE_OUT_OF_MEMORY;
}
//pGrpCache->node.dynamicOp = true;
pGrpCache->grpColsMayBeNull = false;
pGrpCache->grpByUid = true;
pGrpCache->batchFetch = getBatchScanOptionFromHint(pRoot->pHint);
pGrpCache->node.pChildren = pChildren;
pGrpCache->node.pTargets = nodesMakeList();
if (NULL == pGrpCache->node.pTargets) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
if (TSDB_CODE_SUCCESS == code) {
SScanLogicNode* pScan = (SScanLogicNode*)nodesListGetNode(pChildren, 0);
code = nodesListStrictAppendList(pGrpCache->node.pTargets, nodesCloneList(pScan->node.pTargets));
}
SScanLogicNode* pScan = (SScanLogicNode*)nodesListGetNode(pChildren, 0);
SNode* pCol = NULL;
FOREACH(pCol, pScan->pScanPseudoCols) {
if (QUERY_NODE_FUNCTION == nodeType(pCol) && (((SFunctionNode*)pCol)->funcType == FUNCTION_TYPE_TBUID || ((SFunctionNode*)pCol)->funcType == FUNCTION_TYPE_VGID)) {
code = createColumnByRewriteExpr(pCol, &pGrpCache->pGroupCols);
if (code) {
break;
}
}
}
bool hasCond = false;
SNode* pNode = NULL;
FOREACH(pNode, pChildren) {
SScanLogicNode* pScan = (SScanLogicNode*)pNode;
if (pScan->node.pConditions) {
hasCond = true;
}
pScan->node.pParent = (SLogicNode*)pGrpCache;
}
pGrpCache->globalGrp = false;
if (TSDB_CODE_SUCCESS == code) {
*ppLogic = (SLogicNode*)pGrpCache;
} else {
nodesDestroyNode((SNode*)pGrpCache);
}
return code;
}
static void stbJoinOptRemoveTagEqCond(SJoinLogicNode* pJoin) {
if (QUERY_NODE_OPERATOR == nodeType(pJoin->pOtherOnCond) && nodesEqualNode(pJoin->pOtherOnCond, pJoin->pTagEqCond)) {
NODES_DESTORY_NODE(pJoin->pOtherOnCond);
return;
}
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOtherOnCond)) {
SLogicConditionNode* pLogic = (SLogicConditionNode*)pJoin->pOtherOnCond;
SNode* pNode = NULL;
FOREACH(pNode, pLogic->pParameterList) {
if (nodesEqualNode(pNode, pJoin->pTagEqCond)) {
ERASE_NODE(pLogic->pParameterList);
break;
} else if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pTagEqCond)) {
SLogicConditionNode* pTags = (SLogicConditionNode*)pJoin->pTagEqCond;
SNode* pTag = NULL;
FOREACH(pTag, pTags->pParameterList) {
if (nodesEqualNode(pTag, pNode)) {
ERASE_NODE(pLogic->pParameterList);
break;
}
}
}
}
if (pLogic->pParameterList->length <= 0) {
NODES_DESTORY_NODE(pJoin->pOtherOnCond);
}
}
}
static int32_t stbJoinOptCreateMergeJoinNode(SLogicNode* pOrig, SLogicNode* pChild, SLogicNode** ppLogic) {
SJoinLogicNode* pOrigJoin = (SJoinLogicNode*)pOrig;
SJoinLogicNode* pJoin = (SJoinLogicNode*)nodesCloneNode((SNode*)pOrig);
if (NULL == pJoin) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pJoin->joinAlgo = JOIN_ALGO_MERGE;
//pJoin->node.dynamicOp = true;
stbJoinOptRemoveTagEqCond(pJoin);
NODES_DESTORY_NODE(pJoin->pTagEqCond);
SNode* pNode = NULL;
FOREACH(pNode, pJoin->node.pChildren) {
ERASE_NODE(pJoin->node.pChildren);
}
nodesListStrictAppend(pJoin->node.pChildren, (SNode *)pChild);
pChild->pParent = (SLogicNode*)pJoin;
*ppLogic = (SLogicNode*)pJoin;
return TSDB_CODE_SUCCESS;
}
static int32_t stbJoinOptCreateDynQueryCtrlNode(SLogicNode* pRoot, SLogicNode* pPrev, SLogicNode* pPost, bool* srcScan, SLogicNode** ppDynNode) {
int32_t code = TSDB_CODE_SUCCESS;
SDynQueryCtrlLogicNode* pDynCtrl = (SDynQueryCtrlLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL);
if (NULL == pDynCtrl) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pDynCtrl->qType = DYN_QTYPE_STB_HASH;
pDynCtrl->stbJoin.batchFetch = getBatchScanOptionFromHint(pRoot->pHint);
memcpy(pDynCtrl->stbJoin.srcScan, srcScan, sizeof(pDynCtrl->stbJoin.srcScan));
if (TSDB_CODE_SUCCESS == code) {
pDynCtrl->node.pChildren = nodesMakeList();
if (NULL == pDynCtrl->node.pChildren) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
if (TSDB_CODE_SUCCESS == code) {
pDynCtrl->stbJoin.pVgList = nodesMakeList();
if (NULL == pDynCtrl->stbJoin.pVgList) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
if (TSDB_CODE_SUCCESS == code) {
pDynCtrl->stbJoin.pUidList = nodesMakeList();
if (NULL == pDynCtrl->stbJoin.pUidList) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
SJoinLogicNode* pHJoin = (SJoinLogicNode*)pPrev;
nodesListStrictAppend(pDynCtrl->stbJoin.pUidList, nodesListGetNode(pHJoin->node.pTargets, 0));
nodesListStrictAppend(pDynCtrl->stbJoin.pUidList, nodesListGetNode(pHJoin->node.pTargets, 2));
nodesListStrictAppend(pDynCtrl->stbJoin.pVgList, nodesListGetNode(pHJoin->node.pTargets, 1));
nodesListStrictAppend(pDynCtrl->stbJoin.pVgList, nodesListGetNode(pHJoin->node.pTargets, 3));
if (TSDB_CODE_SUCCESS == code) {
nodesListStrictAppend(pDynCtrl->node.pChildren, (SNode*)pPrev);
nodesListStrictAppend(pDynCtrl->node.pChildren, (SNode*)pPost);
pDynCtrl->node.pTargets = nodesCloneList(pPost->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
pPrev->pParent = (SLogicNode*)pDynCtrl;
pPost->pParent = (SLogicNode*)pDynCtrl;
*ppDynNode = (SLogicNode*)pDynCtrl;
} else {
nodesDestroyNode((SNode*)pDynCtrl);
*ppDynNode = NULL;
}
return code;
}
static int32_t stbJoinOptRewriteStableJoin(SOptimizeContext* pCxt, SLogicNode* pJoin, SLogicSubplan* pLogicSubplan) {
SNodeList* pTagScanNodes = NULL;
SNodeList* pTbScanNodes = NULL;
SLogicNode* pGrpCacheNode = NULL;
SLogicNode* pHJoinNode = NULL;
SLogicNode* pMJoinNode = NULL;
SLogicNode* pDynNode = NULL;
bool srcScan[2] = {0};
int32_t code = stbJoinOptCreateTagScanNode(pJoin, &pTagScanNodes);
if (TSDB_CODE_SUCCESS == code) {
code = stbJoinOptCreateTagHashJoinNode(pJoin, pTagScanNodes, &pHJoinNode);
}
if (TSDB_CODE_SUCCESS == code) {
code = stbJoinOptCreateTableScanNodes(pJoin, &pTbScanNodes, srcScan);
}
if (TSDB_CODE_SUCCESS == code) {
code = stbJoinOptCreateGroupCacheNode(getLogicNodeRootNode(pJoin), pTbScanNodes, &pGrpCacheNode);
}
if (TSDB_CODE_SUCCESS == code) {
code = stbJoinOptCreateMergeJoinNode(pJoin, pGrpCacheNode, &pMJoinNode);
}
if (TSDB_CODE_SUCCESS == code) {
code = stbJoinOptCreateDynQueryCtrlNode(getLogicNodeRootNode(pJoin), pHJoinNode, pMJoinNode, srcScan, &pDynNode);
}
if (TSDB_CODE_SUCCESS == code) {
code = replaceLogicNode(pLogicSubplan, pJoin, (SLogicNode*)pDynNode);
}
if (TSDB_CODE_SUCCESS == code) {
nodesDestroyNode((SNode*)pJoin);
pCxt->optimized = true;
}
return code;
}
static int32_t stableJoinOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
SLogicNode* pNode = optFindPossibleNode(pLogicSubplan->pNode, stbJoinOptShouldBeOptimized);
if (NULL == pNode) {
return TSDB_CODE_SUCCESS;
}
return stbJoinOptRewriteStableJoin(pCxt, pNode, pLogicSubplan);
}
// clang-format off
static const SOptimizeRule optimizeRuleSet[] = {
{.pName = "ScanPath", .optimizeFunc = scanPathOptimize},
{.pName = "PushDownCondition", .optimizeFunc = pushDownCondOptimize},
{.pName = "StableJoin", .optimizeFunc = stableJoinOptimize},
{.pName = "sortNonPriKeyOptimize", .optimizeFunc = sortNonPriKeyOptimize},
{.pName = "SortPrimaryKey", .optimizeFunc = sortPrimaryKeyOptimize},
{.pName = "SmaIndex", .optimizeFunc = smaIndexOptimize},
{.pName = "PushDownLimit", .optimizeFunc = pushDownLimitOptimize},
{.pName = "PartitionTags", .optimizeFunc = partTagsOptimize},
{.pName = "MergeProjects", .optimizeFunc = mergeProjectsOptimize},
{.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize},
{.pName = "EliminateSetOperator", .optimizeFunc = eliminateSetOpOptimize},
{.pName = "RewriteTail", .optimizeFunc = rewriteTailOptimize},
{.pName = "RewriteUnique", .optimizeFunc = rewriteUniqueOptimize},
{.pName = "LastRowScan", .optimizeFunc = lastRowScanOptimize},
{.pName = "TagScan", .optimizeFunc = tagScanOptimize},
{.pName = "TableCountScan", .optimizeFunc = tableCountScanOptimize},
{.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize},
{.pName = "EliminateSetOperator", .optimizeFunc = eliminateSetOpOptimize},
};
// clang-format on
@ -3086,9 +3605,9 @@ static void dumpLogicSubplan(const char* pRuleName, SLogicSubplan* pSubplan) {
char* pStr = NULL;
nodesNodeToString((SNode*)pSubplan, false, &pStr, NULL);
if (NULL == pRuleName) {
qDebugL("before optimize: %s", pStr);
qDebugL("before optimize, JsonPlan: %s", pStr);
} else {
qDebugL("apply optimize %s rule: %s", pRuleName, pStr);
qDebugL("apply optimize %s rule, JsonPlan: %s", pRuleName, pStr);
}
taosMemoryFree(pStr);
}
@ -3108,6 +3627,7 @@ static int32_t applyOptimizeRule(SPlanContext* pCxt, SLogicSubplan* pLogicSubpla
if (cxt.optimized) {
optimized = true;
dumpLogicSubplan(optimizeRuleSet[i].pName, pLogicSubplan);
break;
}
}
} while (optimized);

396
source/libs/planner/src/planPhysiCreater.c
View File

@ -53,6 +53,17 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) {
return sprintf(pKey, "%s", pCol->colName);
}
return sprintf(pKey, "%s.%s", pCol->tableAlias, pCol->colName);
} else if (QUERY_NODE_FUNCTION == nodeType(pNode)) {
SFunctionNode* pFunc = (SFunctionNode*)pNode;
if (FUNCTION_TYPE_TBNAME == pFunc->funcType) {
SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0);
if (pVal) {
if (NULL != pStmtName && '\0' != pStmtName[0]) {
return sprintf(pKey, "%s.%s", pStmtName, ((SExprNode*)pNode)->aliasName);
}
return sprintf(pKey, "%s.%s", pVal->literal, ((SExprNode*)pNode)->aliasName);
}
}
}
if (NULL != pStmtName && '\0' != pStmtName[0]) {
@ -366,6 +377,7 @@ static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode
TSWAP(pPhysiNode->pLimit, pLogicNode->pLimit);
TSWAP(pPhysiNode->pSlimit, pLogicNode->pSlimit);
pPhysiNode->dynamicOp = pLogicNode->dynamicOp;
pPhysiNode->inputTsOrder = pLogicNode->inputTsOrder;
pPhysiNode->outputTsOrder = pLogicNode->outputTsOrder;
@ -681,7 +693,64 @@ static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan,
return TSDB_CODE_FAILED;
}
static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode,
static int32_t mergeEqCond(SNode** ppDst, SNode** ppSrc) {
if (NULL == *ppSrc) {
return TSDB_CODE_SUCCESS;
}
if (NULL == *ppDst) {
*ppDst = *ppSrc;
*ppSrc = NULL;
return TSDB_CODE_SUCCESS;
}
if (QUERY_NODE_LOGIC_CONDITION == nodeType(*ppSrc)) {
TSWAP(*ppDst, *ppSrc);
}
if (QUERY_NODE_LOGIC_CONDITION == nodeType(*ppDst)) {
SLogicConditionNode* pLogic = (SLogicConditionNode*)*ppDst;
if (QUERY_NODE_LOGIC_CONDITION == nodeType(*ppSrc)) {
nodesListStrictAppendList(pLogic->pParameterList, ((SLogicConditionNode*)(*ppSrc))->pParameterList);
((SLogicConditionNode*)(*ppSrc))->pParameterList = NULL;
} else {
nodesListStrictAppend(pLogic->pParameterList, *ppSrc);
*ppSrc = NULL;
}
nodesDestroyNode(*ppSrc);
*ppSrc = NULL;
return TSDB_CODE_SUCCESS;
}
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION);
if (NULL == pLogicCond) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pLogicCond->node.resType.type = TSDB_DATA_TYPE_BOOL;
pLogicCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
pLogicCond->condType = LOGIC_COND_TYPE_AND;
pLogicCond->pParameterList = nodesMakeList();
nodesListStrictAppend(pLogicCond->pParameterList, *ppSrc);
nodesListStrictAppend(pLogicCond->pParameterList, *ppDst);
*ppDst = (SNode*)pLogicCond;
*ppSrc = NULL;
return TSDB_CODE_SUCCESS;
}
static int32_t getJoinDataBlockDescNode(SNodeList* pChildren, int32_t idx, SDataBlockDescNode** ppDesc) {
if (2 == pChildren->length) {
*ppDesc = ((SPhysiNode*)nodesListGetNode(pChildren, idx))->pOutputDataBlockDesc;
} else if (1 == pChildren->length && nodeType(nodesListGetNode(pChildren, 0)) == QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE) {
SGroupCachePhysiNode* pGrpCache = (SGroupCachePhysiNode*)nodesListGetNode(pChildren, 0);
*ppDesc = ((SPhysiNode*)nodesListGetNode(pGrpCache->node.pChildren, idx))->pOutputDataBlockDesc;
} else {
planError("Invalid join children num:%d or child type:%d", pChildren->length, nodeType(nodesListGetNode(pChildren, 0)));
return TSDB_CODE_PLAN_INTERNAL_ERROR;
}
return TSDB_CODE_SUCCESS;
}
static int32_t createMergeJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode,
SPhysiNode** pPhyNode) {
SSortMergeJoinPhysiNode* pJoin =
(SSortMergeJoinPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pJoinLogicNode, QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN);
@ -689,46 +758,220 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren
return TSDB_CODE_OUT_OF_MEMORY;
}
pJoin->joinType = pJoinLogicNode->joinType;
pJoin->node.inputTsOrder = pJoinLogicNode->node.inputTsOrder;
SDataBlockDescNode* pLeftDesc = NULL;
SDataBlockDescNode* pRightDesc = NULL;
int32_t code = getJoinDataBlockDescNode(pChildren, 0, &pLeftDesc);
if (TSDB_CODE_SUCCESS == code) {
code = getJoinDataBlockDescNode(pChildren, 1, &pRightDesc);
}
if (TSDB_CODE_SUCCESS == code) {
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pPrimKeyEqCond,
&pJoin->pPrimKeyCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = setListSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->node.pTargets,
&pJoin->pTargets);
}
if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pOtherOnCond) {
code = setNodeSlotId(pCxt, ((SPhysiNode*)pJoin)->pOutputDataBlockDesc->dataBlockId, -1,
pJoinLogicNode->pOtherOnCond, &pJoin->pOtherOnCond);
}
if (TSDB_CODE_SUCCESS == code && ((NULL != pJoinLogicNode->pColEqCond) || (NULL != pJoinLogicNode->pTagEqCond))) {
code = mergeEqCond(&pJoinLogicNode->pColEqCond, &pJoinLogicNode->pTagEqCond);
}
if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pColEqCond) {
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pColEqCond, &pJoin->pColEqCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin);
}
if (TSDB_CODE_SUCCESS == code) {
code = addDataBlockSlots(pCxt, pJoin->pTargets, pJoin->node.pOutputDataBlockDesc);
}
if (TSDB_CODE_SUCCESS == code) {
*pPhyNode = (SPhysiNode*)pJoin;
} else {
nodesDestroyNode((SNode*)pJoin);
}
return code;
}
static int32_t extractHashJoinOpCols(int16_t lBlkId, int16_t rBlkId, SNode* pEq, SHashJoinPhysiNode* pJoin) {
if (QUERY_NODE_OPERATOR == nodeType(pEq)) {
SOperatorNode* pOp = (SOperatorNode*)pEq;
SColumnNode* pLeft = (SColumnNode*)pOp->pLeft;
SColumnNode* pRight = (SColumnNode*)pOp->pRight;
if (lBlkId == pLeft->dataBlockId && rBlkId == pRight->dataBlockId) {
nodesListStrictAppend(pJoin->pOnLeft, nodesCloneNode(pOp->pLeft));
nodesListStrictAppend(pJoin->pOnRight, nodesCloneNode(pOp->pRight));
} else if (rBlkId == pLeft->dataBlockId && lBlkId == pRight->dataBlockId) {
nodesListStrictAppend(pJoin->pOnLeft, nodesCloneNode(pOp->pRight));
nodesListStrictAppend(pJoin->pOnRight, nodesCloneNode(pOp->pLeft));
} else {
planError("Invalid join equal cond, lbid:%d, rbid:%d, oplid:%d, oprid:%d", lBlkId, rBlkId, pLeft->dataBlockId, pRight->dataBlockId);
return TSDB_CODE_PLAN_INTERNAL_ERROR;
}
return TSDB_CODE_SUCCESS;
}
planError("Invalid join equal node type:%d", nodeType(pEq));
return TSDB_CODE_PLAN_INTERNAL_ERROR;
}
static int32_t extractHashJoinOnCols(int16_t lBlkId, int16_t rBlkId, SNode* pEq, SHashJoinPhysiNode* pJoin) {
if (NULL == pEq) {
return TSDB_CODE_SUCCESS;
}
int32_t code = TSDB_CODE_SUCCESS;
if (QUERY_NODE_OPERATOR == nodeType(pEq)) {
code = extractHashJoinOpCols(lBlkId, rBlkId, pEq, pJoin);
} else if (QUERY_NODE_LOGIC_CONDITION == nodeType(pEq)) {
SLogicConditionNode* pLogic = (SLogicConditionNode*)pEq;
SNode* pNode = NULL;
FOREACH(pNode, pLogic->pParameterList) {
code = extractHashJoinOpCols(lBlkId, rBlkId, pNode, pJoin);
if (code) {
break;
}
}
} else {
planError("Invalid join equal node type:%d", nodeType(pEq));
return TSDB_CODE_PLAN_INTERNAL_ERROR;
}
return code;
}
static int32_t createHashJoinColList(int16_t lBlkId, int16_t rBlkId, SNode* pEq1, SNode* pEq2, SNode* pEq3, SHashJoinPhysiNode* pJoin) {
int32_t code = TSDB_CODE_SUCCESS;
pJoin->pOnLeft = nodesMakeList();
pJoin->pOnRight = nodesMakeList();
if (NULL == pJoin->pOnLeft || NULL == pJoin->pOnRight) {
return TSDB_CODE_OUT_OF_MEMORY;
}
code = extractHashJoinOnCols(lBlkId, rBlkId, pEq1, pJoin);
if (TSDB_CODE_SUCCESS == code) {
code = extractHashJoinOnCols(lBlkId, rBlkId, pEq2, pJoin);
}
if (TSDB_CODE_SUCCESS == code) {
code = extractHashJoinOnCols(lBlkId, rBlkId, pEq3, pJoin);
}
if (TSDB_CODE_SUCCESS == code && pJoin->pOnLeft->length <= 0) {
planError("Invalid join equal column num: %d", pJoin->pOnLeft->length);
code = TSDB_CODE_PLAN_INTERNAL_ERROR;
}
return code;
}
static int32_t sortHashJoinTargets(int16_t lBlkId, int16_t rBlkId, SHashJoinPhysiNode* pJoin) {
SNode* pNode = NULL;
char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN];
SSHashObj* pHash = tSimpleHashInit(pJoin->pTargets->length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY));
if (NULL == pHash) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SNodeList* pNew = nodesMakeList();
FOREACH(pNode, pJoin->pTargets) {
SColumnNode* pCol = (SColumnNode*)pNode;
int32_t len = getSlotKey(pNode, NULL, name);
tSimpleHashPut(pHash, name, len, &pCol, POINTER_BYTES);
}
nodesClearList(pJoin->pTargets);
pJoin->pTargets = pNew;
FOREACH(pNode, pJoin->pOnLeft) {
SColumnNode* pCol = (SColumnNode*)pNode;
int32_t len = getSlotKey(pNode, NULL, name);
SNode** p = tSimpleHashGet(pHash, name, len);
if (p) {
nodesListStrictAppend(pJoin->pTargets, *p);
tSimpleHashRemove(pHash, name, len);
}
}
FOREACH(pNode, pJoin->pOnRight) {
SColumnNode* pCol = (SColumnNode*)pNode;
int32_t len = getSlotKey(pNode, NULL, name);
SNode** p = tSimpleHashGet(pHash, name, len);
if (p) {
nodesListStrictAppend(pJoin->pTargets, *p);
tSimpleHashRemove(pHash, name, len);
}
}
if (tSimpleHashGetSize(pHash) > 0) {
SNode** p = NULL;
int32_t iter = 0;
while (1) {
p = tSimpleHashIterate(pHash, p, &iter);
if (p == NULL) {
break;
}
nodesListStrictAppend(pJoin->pTargets, *p);
}
}
tSimpleHashCleanup(pHash);
return TSDB_CODE_SUCCESS;
}
static int32_t createHashJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode,
SPhysiNode** pPhyNode) {
SHashJoinPhysiNode* pJoin =
(SHashJoinPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pJoinLogicNode, QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN);
if (NULL == pJoin) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SDataBlockDescNode* pLeftDesc = ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc;
SDataBlockDescNode* pRightDesc = ((SPhysiNode*)nodesListGetNode(pChildren, 1))->pOutputDataBlockDesc;
int32_t code = TSDB_CODE_SUCCESS;
pJoin->joinType = pJoinLogicNode->joinType;
pJoin->node.inputTsOrder = pJoinLogicNode->node.inputTsOrder;
setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pMergeCondition,
&pJoin->pMergeCondition);
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pPrimKeyEqCond, &pJoin->pPrimKeyCond);
if (TSDB_CODE_SUCCESS == code) {
code = setListSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->node.pTargets,
&pJoin->pTargets);
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pColEqCond, &pJoin->pColEqCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = addDataBlockSlots(pCxt, pJoin->pTargets, pJoin->node.pOutputDataBlockDesc);
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pTagEqCond, &pJoin->pTagEqCond);
}
if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pOnConditions) {
SNodeList* pCondCols = nodesMakeList();
if (NULL == pCondCols) {
code = TSDB_CODE_OUT_OF_MEMORY;
} else {
code = nodesCollectColumnsFromNode(pJoinLogicNode->pOnConditions, NULL, COLLECT_COL_TYPE_ALL, &pCondCols);
if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pOtherOnCond) {
code = setNodeSlotId(pCxt, ((SPhysiNode*)pJoin)->pOutputDataBlockDesc->dataBlockId, -1, pJoinLogicNode->pOtherOnCond, &pJoin->pFilterConditions);
}
if (TSDB_CODE_SUCCESS == code) {
code = addDataBlockSlots(pCxt, pCondCols, pJoin->node.pOutputDataBlockDesc);
}
nodesDestroyList(pCondCols);
}
if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pOnConditions) {
code = setNodeSlotId(pCxt, ((SPhysiNode*)pJoin)->pOutputDataBlockDesc->dataBlockId, -1,
pJoinLogicNode->pOnConditions, &pJoin->pOnConditions);
}
if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pColEqualOnConditions) {
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pColEqualOnConditions, &pJoin->pColEqualOnConditions);
code = setListSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->node.pTargets, &pJoin->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin);
}
if (TSDB_CODE_SUCCESS == code) {
code = createHashJoinColList(pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoin->pPrimKeyCond, pJoin->pColEqCond, pJoin->pTagEqCond, pJoin);
}
if (TSDB_CODE_SUCCESS == code) {
code = sortHashJoinTargets(pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoin);
}
if (TSDB_CODE_SUCCESS == code) {
code = addDataBlockSlots(pCxt, pJoin->pTargets, pJoin->node.pOutputDataBlockDesc);
}
if (TSDB_CODE_SUCCESS == code) {
*pPhyNode = (SPhysiNode*)pJoin;
@ -739,6 +982,105 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren
return code;
}
static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode,
SPhysiNode** pPhyNode) {
switch (pJoinLogicNode->joinAlgo) {
case JOIN_ALGO_MERGE:
return createMergeJoinPhysiNode(pCxt, pChildren, pJoinLogicNode, pPhyNode);
case JOIN_ALGO_HASH:
return createHashJoinPhysiNode(pCxt, pChildren, pJoinLogicNode, pPhyNode);
default:
planError("Invalid join algorithm:%d", pJoinLogicNode->joinAlgo);
break;
}
return TSDB_CODE_FAILED;
}
static int32_t createGroupCachePhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SGroupCacheLogicNode* pLogicNode,
SPhysiNode** pPhyNode) {
SGroupCachePhysiNode* pGrpCache =
(SGroupCachePhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pLogicNode, QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE);
if (NULL == pGrpCache) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pGrpCache->grpColsMayBeNull = pLogicNode->grpColsMayBeNull;
pGrpCache->grpByUid = pLogicNode->grpByUid;
pGrpCache->globalGrp = pLogicNode->globalGrp;
pGrpCache->batchFetch = pLogicNode->batchFetch;
SDataBlockDescNode* pChildDesc = ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc;
int32_t code = TSDB_CODE_SUCCESS;
/*
if (TSDB_CODE_SUCCESS == code) {
code = setListSlotId(pCxt, pChildDesc->dataBlockId, -1, pLogicNode->pGroupCols, &pGrpCache->pGroupCols);
}
*/
if (TSDB_CODE_SUCCESS == code) {
*pPhyNode = (SPhysiNode*)pGrpCache;
} else {
nodesDestroyNode((SNode*)pGrpCache);
}
return code;
}
static int32_t updateDynQueryCtrlStbJoinInfo(SPhysiPlanContext* pCxt, SNodeList* pChildren, SDynQueryCtrlLogicNode* pLogicNode,
SDynQueryCtrlPhysiNode* pDynCtrl) {
SDataBlockDescNode* pPrevDesc = ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc;
SNodeList* pVgList = NULL;
SNodeList* pUidList = NULL;
int32_t code = setListSlotId(pCxt, pPrevDesc->dataBlockId, -1, pLogicNode->stbJoin.pVgList, &pVgList);
if (TSDB_CODE_SUCCESS == code) {
code = setListSlotId(pCxt, pPrevDesc->dataBlockId, -1, pLogicNode->stbJoin.pUidList, &pUidList);
}
if (TSDB_CODE_SUCCESS == code) {
memcpy(pDynCtrl->stbJoin.srcScan, pLogicNode->stbJoin.srcScan, sizeof(pDynCtrl->stbJoin.srcScan));
SNode* pNode = NULL;
int32_t i = 0;
FOREACH(pNode, pVgList) {
pDynCtrl->stbJoin.vgSlot[i] = ((SColumnNode*)pNode)->slotId;
++i;
}
i = 0;
FOREACH(pNode, pUidList) {
pDynCtrl->stbJoin.uidSlot[i] = ((SColumnNode*)pNode)->slotId;
++i;
}
pDynCtrl->stbJoin.batchFetch = pLogicNode->stbJoin.batchFetch;
}
return code;
}
static int32_t createDynQueryCtrlPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SDynQueryCtrlLogicNode* pLogicNode,
SPhysiNode** pPhyNode) {
int32_t code = TSDB_CODE_SUCCESS;
SDynQueryCtrlPhysiNode* pDynCtrl =
(SDynQueryCtrlPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pLogicNode, QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL);
if (NULL == pDynCtrl) {
return TSDB_CODE_OUT_OF_MEMORY;
}
switch (pLogicNode->qType) {
case DYN_QTYPE_STB_HASH:
code = updateDynQueryCtrlStbJoinInfo(pCxt, pChildren, pLogicNode, pDynCtrl);
break;
default:
planError("Invalid dyn query ctrl type:%d", pLogicNode->qType);
return TSDB_CODE_PLAN_INTERNAL_ERROR;
}
if (TSDB_CODE_SUCCESS == code) {
pDynCtrl->qType = pLogicNode->qType;
*pPhyNode = (SPhysiNode*)pDynCtrl;
}
return code;
}
typedef struct SRewritePrecalcExprsCxt {
int32_t errCode;
int32_t planNodeId;
@ -1655,6 +1997,10 @@ static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode
return createInterpFuncPhysiNode(pCxt, pChildren, (SInterpFuncLogicNode*)pLogicNode, pPhyNode);
case QUERY_NODE_LOGIC_PLAN_MERGE:
return createMergePhysiNode(pCxt, (SMergeLogicNode*)pLogicNode, pPhyNode);
case QUERY_NODE_LOGIC_PLAN_GROUP_CACHE:
return createGroupCachePhysiNode(pCxt, pChildren, (SGroupCacheLogicNode*)pLogicNode, pPhyNode);
case QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL:
return createDynQueryCtrlPhysiNode(pCxt, pChildren, (SDynQueryCtrlLogicNode*)pLogicNode, pPhyNode);
default:
break;
}

13
source/libs/planner/src/planSpliter.c
View File

@ -108,6 +108,7 @@ static int32_t splCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pChild, SE
pExchange->srcStartGroupId = pCxt->groupId;
pExchange->srcEndGroupId = pCxt->groupId;
pExchange->node.precision = pChild->precision;
pExchange->node.dynamicOp = pChild->dynamicOp;
pExchange->node.pTargets = nodesCloneList(pChild->pTargets);
if (NULL == pExchange->node.pTargets) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -280,7 +281,7 @@ static bool stbSplNeedSplitWindow(bool streamQuery, SLogicNode* pNode) {
}
static bool stbSplNeedSplitJoin(bool streamQuery, SJoinLogicNode* pJoin) {
if (pJoin->isSingleTableJoin) {
if (pJoin->isSingleTableJoin || JOIN_ALGO_HASH == pJoin->joinAlgo) {
return false;
}
SNode* pChild = NULL;
@ -1171,7 +1172,11 @@ static int32_t stbSplSplitJoinNodeImpl(SSplitContext* pCxt, SLogicSubplan* pSubp
SNode* pChild = NULL;
FOREACH(pChild, pJoin->node.pChildren) {
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild)) {
//if (pJoin->node.dynamicOp) {
// code = TSDB_CODE_SUCCESS;
//} else {
code = stbSplSplitMergeScanNode(pCxt, pSubplan, (SScanLogicNode*)pChild, false);
//}
} else if (QUERY_NODE_LOGIC_PLAN_JOIN == nodeType(pChild)) {
code = stbSplSplitJoinNodeImpl(pCxt, pSubplan, (SJoinLogicNode*)pChild);
} else {
@ -1187,7 +1192,9 @@ static int32_t stbSplSplitJoinNodeImpl(SSplitContext* pCxt, SLogicSubplan* pSubp
static int32_t stbSplSplitJoinNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
int32_t code = stbSplSplitJoinNodeImpl(pCxt, pInfo->pSubplan, (SJoinLogicNode*)pInfo->pSplitNode);
if (TSDB_CODE_SUCCESS == code) {
//if (!pInfo->pSplitNode->dynamicOp) {
pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE;
//}
SPLIT_FLAG_SET_MASK(pInfo->pSubplan->splitFlag, SPLIT_FLAG_STABLE_SPLIT);
}
return code;
@ -1589,9 +1596,9 @@ static void dumpLogicSubplan(const char* pRuleName, SLogicSubplan* pSubplan) {
char* pStr = NULL;
nodesNodeToString((SNode*)pSubplan, false, &pStr, NULL);
if (NULL == pRuleName) {
qDebugL("before split: %s", pStr);
qDebugL("before split, JsonPlan: %s", pStr);
} else {
qDebugL("apply split %s rule: %s", pRuleName, pStr);
qDebugL("apply split %s rule, JsonPlan: %s", pRuleName, pStr);
}
taosMemoryFree(pStr);
}

Some files were not shown because too many files have changed in this diff Show More